--
-- failure_mx_metadata_sync.sql
--
CREATE SCHEMA IF NOT EXISTS mx_metadata_sync;
SET SEARCH_PATH = mx_metadata_sync;
SET citus.shard_count TO 2;
SET citus.next_shard_id TO 16000000;
SET citus.shard_replication_factor TO 1;
SELECT pg_backend_pid() as pid \gset
SELECT citus.mitmproxy('conn.allow()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

CREATE TABLE t1 (id int PRIMARY KEY);
SELECT create_distributed_table('t1', 'id');
 create_distributed_table
---------------------------------------------------------------------

(1 row)

INSERT INTO t1 SELECT x FROM generate_series(1,100) AS f(x);
-- Initially turn metadata sync off because we'll ingest errors to start/stop metadata sync operations
SELECT stop_metadata_sync_to_node('localhost', :worker_2_proxy_port);
NOTICE:  dropping metadata on the node (localhost,9060)
 stop_metadata_sync_to_node
---------------------------------------------------------------------

(1 row)

SELECT hasmetadata FROM pg_dist_node WHERE nodeport=:worker_2_proxy_port;
 hasmetadata
---------------------------------------------------------------------
 f
(1 row)

-- Failure to set groupid in the worker
SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE pg_dist_local_group SET groupid").cancel(' || :pid || ')');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT start_metadata_sync_to_node('localhost', :worker_2_proxy_port);
ERROR:  canceling statement due to user request
SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE pg_dist_local_group SET groupid").kill()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT start_metadata_sync_to_node('localhost', :worker_2_proxy_port);
ERROR:  server closed the connection unexpectedly
	This probably means the server terminated abnormally
	before or while processing the request.
CONTEXT:  while executing command on localhost:xxxxx
-- Failure to drop all tables in pg_dist_partition
SELECT citus.mitmproxy('conn.onQuery(query="^SELECT worker_drop_distributed_table").cancel(' || :pid || ')');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT start_metadata_sync_to_node('localhost', :worker_2_proxy_port);
ERROR:  canceling statement due to user request
SELECT citus.mitmproxy('conn.onQuery(query="^SELECT worker_drop_distributed_table").kill()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT start_metadata_sync_to_node('localhost', :worker_2_proxy_port);
ERROR:  server closed the connection unexpectedly
	This probably means the server terminated abnormally
	before or while processing the request.
CONTEXT:  while executing command on localhost:xxxxx
-- Failure to truncate pg_dist_node in the worker
SELECT citus.mitmproxy('conn.onQuery(query="^TRUNCATE pg_dist_node CASCADE").cancel(' || :pid || ')');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT start_metadata_sync_to_node('localhost', :worker_2_proxy_port);
ERROR:  canceling statement due to user request
SELECT citus.mitmproxy('conn.onQuery(query="^TRUNCATE pg_dist_node CASCADE").kill()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT start_metadata_sync_to_node('localhost', :worker_2_proxy_port);
ERROR:  server closed the connection unexpectedly
	This probably means the server terminated abnormally
	before or while processing the request.
CONTEXT:  while executing command on localhost:xxxxx
-- Failure to populate pg_dist_node in the worker
SELECT citus.mitmproxy('conn.onQuery(query="^INSERT INTO pg_dist_node").cancel(' || :pid || ')');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT start_metadata_sync_to_node('localhost', :worker_2_proxy_port);
ERROR:  canceling statement due to user request
SELECT citus.mitmproxy('conn.onQuery(query="^INSERT INTO pg_dist_node").kill()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT start_metadata_sync_to_node('localhost', :worker_2_proxy_port);
ERROR:  server closed the connection unexpectedly
	This probably means the server terminated abnormally
	before or while processing the request.
CONTEXT:  while executing command on localhost:xxxxx
-- Verify that coordinator knows worker does not have valid metadata
SELECT hasmetadata FROM pg_dist_node WHERE nodeport=:worker_2_proxy_port;
 hasmetadata
---------------------------------------------------------------------
 f
(1 row)

-- Verify we can sync metadata after unsuccessful attempts
SELECT citus.mitmproxy('conn.allow()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT start_metadata_sync_to_node('localhost', :worker_2_proxy_port);
 start_metadata_sync_to_node
---------------------------------------------------------------------

(1 row)

SELECT hasmetadata FROM pg_dist_node WHERE nodeport=:worker_2_proxy_port;
 hasmetadata
---------------------------------------------------------------------
 t
(1 row)

-- Check failures on DDL command propagation
CREATE TABLE t2 (id int PRIMARY KEY);
SELECT citus.mitmproxy('conn.onParse(query="citus_internal_add_placement_metadata").kill()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT create_distributed_table('t2', 'id');
ERROR:  server closed the connection unexpectedly
	This probably means the server terminated abnormally
	before or while processing the request.
CONTEXT:  while executing command on localhost:xxxxx
SELECT citus.mitmproxy('conn.onParse(query="citus_internal_add_shard_metadata").cancel(' || :pid || ')');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT create_distributed_table('t2', 'id');
ERROR:  canceling statement due to user request
-- Verify that the table was not distributed
SELECT count(*) > 0 AS is_table_distributed
FROM pg_dist_partition
WHERE logicalrelid='t2'::regclass;
 is_table_distributed
---------------------------------------------------------------------
 f
(1 row)

-- Failure to set groupid in the worker
SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE pg_dist_local_group SET groupid").cancel(' || :pid || ')');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT stop_metadata_sync_to_node('localhost', :worker_2_proxy_port);
NOTICE:  dropping metadata on the node (localhost,9060)
ERROR:  canceling statement due to user request
SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE pg_dist_local_group SET groupid").kill()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT stop_metadata_sync_to_node('localhost', :worker_2_proxy_port);
NOTICE:  dropping metadata on the node (localhost,9060)
WARNING:  server closed the connection unexpectedly
	This probably means the server terminated abnormally
	before or while processing the request.
CONTEXT:  while executing command on localhost:xxxxx
WARNING:  server closed the connection unexpectedly
	This probably means the server terminated abnormally
	before or while processing the request.
connection not open
CONTEXT:  while executing command on localhost:xxxxx
WARNING:  server closed the connection unexpectedly
	This probably means the server terminated abnormally
	before or while processing the request.
connection not open
connection not open
CONTEXT:  while executing command on localhost:xxxxx
WARNING:  server closed the connection unexpectedly
	This probably means the server terminated abnormally
	before or while processing the request.
connection not open
connection not open
connection not open
CONTEXT:  while executing command on localhost:xxxxx
WARNING:  server closed the connection unexpectedly
	This probably means the server terminated abnormally
	before or while processing the request.
connection not open
connection not open
connection not open
connection not open
CONTEXT:  while executing command on localhost:xxxxx
 stop_metadata_sync_to_node
---------------------------------------------------------------------

(1 row)

-- Failure to drop all tables in pg_dist_partition
SELECT citus.mitmproxy('conn.onQuery(query="^SELECT worker_drop_distributed_table").cancel(' || :pid || ')');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT stop_metadata_sync_to_node('localhost', :worker_2_proxy_port);
NOTICE:  dropping metadata on the node (localhost,9060)
ERROR:  canceling statement due to user request
SELECT citus.mitmproxy('conn.onQuery(query="^SELECT worker_drop_distributed_table").kill()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT stop_metadata_sync_to_node('localhost', :worker_2_proxy_port);
NOTICE:  dropping metadata on the node (localhost,9060)
WARNING:  server closed the connection unexpectedly
	This probably means the server terminated abnormally
	before or while processing the request.
CONTEXT:  while executing command on localhost:xxxxx
WARNING:  server closed the connection unexpectedly
	This probably means the server terminated abnormally
	before or while processing the request.
connection not open
CONTEXT:  while executing command on localhost:xxxxx
WARNING:  server closed the connection unexpectedly
	This probably means the server terminated abnormally
	before or while processing the request.
connection not open
connection not open
CONTEXT:  while executing command on localhost:xxxxx
WARNING:  server closed the connection unexpectedly
	This probably means the server terminated abnormally
	before or while processing the request.
connection not open
connection not open
connection not open
CONTEXT:  while executing command on localhost:xxxxx
WARNING:  server closed the connection unexpectedly
	This probably means the server terminated abnormally
	before or while processing the request.
connection not open
connection not open
connection not open
connection not open
CONTEXT:  while executing command on localhost:xxxxx
 stop_metadata_sync_to_node
---------------------------------------------------------------------

(1 row)

-- Failure to truncate pg_dist_node in the worker
SELECT citus.mitmproxy('conn.onQuery(query="^TRUNCATE pg_dist_node CASCADE").cancel(' || :pid || ')');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT stop_metadata_sync_to_node('localhost', :worker_2_proxy_port);
NOTICE:  dropping metadata on the node (localhost,9060)
ERROR:  canceling statement due to user request
SELECT citus.mitmproxy('conn.onQuery(query="^TRUNCATE pg_dist_node CASCADE").kill()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT stop_metadata_sync_to_node('localhost', :worker_2_proxy_port);
NOTICE:  dropping metadata on the node (localhost,9060)
WARNING:  server closed the connection unexpectedly
	This probably means the server terminated abnormally
	before or while processing the request.
CONTEXT:  while executing command on localhost:xxxxx
WARNING:  server closed the connection unexpectedly
	This probably means the server terminated abnormally
	before or while processing the request.
connection not open
CONTEXT:  while executing command on localhost:xxxxx
WARNING:  server closed the connection unexpectedly
	This probably means the server terminated abnormally
	before or while processing the request.
connection not open
connection not open
CONTEXT:  while executing command on localhost:xxxxx
WARNING:  server closed the connection unexpectedly
	This probably means the server terminated abnormally
	before or while processing the request.
connection not open
connection not open
connection not open
CONTEXT:  while executing command on localhost:xxxxx
WARNING:  server closed the connection unexpectedly
	This probably means the server terminated abnormally
	before or while processing the request.
connection not open
connection not open
connection not open
connection not open
CONTEXT:  while executing command on localhost:xxxxx
 stop_metadata_sync_to_node
---------------------------------------------------------------------

(1 row)

\c - - - :worker_2_port
SELECT count(*) FROM pg_dist_node;
 count
---------------------------------------------------------------------
     2
(1 row)

\c - - - :master_port
SELECT hasmetadata FROM pg_dist_node WHERE nodeport=:worker_2_proxy_port;
 hasmetadata
---------------------------------------------------------------------
 f
(1 row)

-- Verify we can drop metadata after unsuccessful attempts
SELECT citus.mitmproxy('conn.allow()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT stop_metadata_sync_to_node('localhost', :worker_2_proxy_port);
NOTICE:  dropping metadata on the node (localhost,9060)
 stop_metadata_sync_to_node
---------------------------------------------------------------------

(1 row)

\c - - - :worker_2_port
SELECT count(*) FROM pg_dist_node;
 count
---------------------------------------------------------------------
     0
(1 row)

\c - - - :master_port
SELECT hasmetadata FROM pg_dist_node WHERE nodeport=:worker_2_proxy_port;
 hasmetadata
---------------------------------------------------------------------
 f
(1 row)

-- turn metadata sync back on
SELECT start_metadata_sync_to_node('localhost', :worker_2_proxy_port);
 start_metadata_sync_to_node
---------------------------------------------------------------------

(1 row)

SET SEARCH_PATH = mx_metadata_sync;
DROP TABLE t1;
DROP TABLE t2;
DROP SCHEMA mx_metadata_sync CASCADE;