--
-- failure_mx_metadata_sync.sql
--
CREATE SCHEMA IF NOT EXISTS mx_metadata_sync;
SET SEARCH_PATH = mx_metadata_sync;
SET citus.shard_count TO 2;
SET citus.next_shard_id TO 16000000;
SET citus.shard_replication_factor TO 1;
SELECT pg_backend_pid() as pid \gset
SELECT citus.mitmproxy('conn.allow()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

\set VERBOSITY terse
SET client_min_messages TO ERROR;
CREATE TABLE t1 (id int PRIMARY KEY);
SELECT create_distributed_table('t1', 'id');
 create_distributed_table
---------------------------------------------------------------------

(1 row)

INSERT INTO t1 SELECT x FROM generate_series(1,100) AS f(x);
-- Initially turn metadata sync off because we'll ingest errors to start/stop metadata sync operations
SELECT stop_metadata_sync_to_node('localhost', :worker_2_proxy_port);
 stop_metadata_sync_to_node
---------------------------------------------------------------------

(1 row)

SELECT hasmetadata FROM pg_dist_node WHERE nodeport=:worker_2_proxy_port;
 hasmetadata
---------------------------------------------------------------------
 f
(1 row)

-- Failure to set groupid in the worker
SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE pg_dist_local_group SET groupid").cancel(' || :pid || ')');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT citus_activate_node('localhost', :worker_2_proxy_port);
ERROR:  canceling statement due to user request
SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE pg_dist_local_group SET groupid").kill()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT citus_activate_node('localhost', :worker_2_proxy_port);
ERROR:  failure on connection marked as essential: localhost:xxxxx
-- Failure to drop all tables in pg_dist_partition
SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM pg_dist_partition").cancel(' || :pid || ')');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT citus_activate_node('localhost', :worker_2_proxy_port);
ERROR:  canceling statement due to user request
SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM pg_dist_partition").kill()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT citus_activate_node('localhost', :worker_2_proxy_port);
ERROR:  failure on connection marked as essential: localhost:xxxxx
-- Failure to delete pg_dist_node entries from the worker
SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM pg_dist_node").cancel(' || :pid || ')');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT start_metadata_sync_to_node('localhost', :worker_2_proxy_port);
ERROR:  canceling statement due to user request
SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM pg_dist_node").kill()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT start_metadata_sync_to_node('localhost', :worker_2_proxy_port);
ERROR:  failure on connection marked as essential: localhost:xxxxx
-- Failure to populate pg_dist_node in the worker
SELECT citus.mitmproxy('conn.onQuery(query="INSERT INTO pg_dist_node").cancel(' || :pid || ')');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT start_metadata_sync_to_node('localhost', :worker_2_proxy_port);
ERROR:  canceling statement due to user request
SELECT citus.mitmproxy('conn.onQuery(query="INSERT INTO pg_dist_node").kill()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT start_metadata_sync_to_node('localhost', :worker_2_proxy_port);
ERROR:  failure on connection marked as essential: localhost:xxxxx
-- Verify that coordinator knows worker does not have valid metadata
SELECT hasmetadata FROM pg_dist_node WHERE nodeport=:worker_2_proxy_port;
 hasmetadata
---------------------------------------------------------------------
 f
(1 row)

-- Verify we can activate node after unsuccessful attempts
SELECT citus.mitmproxy('conn.allow()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT 1 FROM citus_activate_node('localhost', :worker_2_proxy_port);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

SELECT hasmetadata FROM pg_dist_node WHERE nodeport=:worker_2_proxy_port;
 hasmetadata
---------------------------------------------------------------------
 t
(1 row)

-- Check failures on DDL command propagation
CREATE TABLE t2 (id int PRIMARY KEY);
SELECT citus.mitmproxy('conn.onParse(query="citus_internal_add_placement_metadata").kill()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT create_distributed_table('t2', 'id');
ERROR:  server closed the connection unexpectedly
	This probably means the server terminated abnormally
	before or while processing the request.
SELECT citus.mitmproxy('conn.onParse(query="citus_internal_add_shard_metadata").cancel(' || :pid || ')');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT create_distributed_table('t2', 'id');
ERROR:  canceling statement due to user request
-- Verify that the table was not distributed
SELECT count(*) > 0 AS is_table_distributed
FROM pg_dist_partition
WHERE logicalrelid='t2'::regclass;
 is_table_distributed
---------------------------------------------------------------------
 f
(1 row)

-- Failure to set groupid in the worker
SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE pg_dist_local_group SET groupid").cancel(' || :pid || ')');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT stop_metadata_sync_to_node('localhost', :worker_2_proxy_port);
ERROR:  canceling statement due to user request
SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE pg_dist_local_group SET groupid").kill()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT stop_metadata_sync_to_node('localhost', :worker_2_proxy_port);
 stop_metadata_sync_to_node
---------------------------------------------------------------------

(1 row)

-- Failure to delete pg_dist_node entries from the worker
SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM pg_dist_node").cancel(' || :pid || ')');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT stop_metadata_sync_to_node('localhost', :worker_2_proxy_port);
ERROR:  canceling statement due to user request
SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM pg_dist_node").kill()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT stop_metadata_sync_to_node('localhost', :worker_2_proxy_port);
 stop_metadata_sync_to_node
---------------------------------------------------------------------

(1 row)

\c - - - :worker_2_port
SELECT count(*) FROM pg_dist_node;
 count
---------------------------------------------------------------------
     2
(1 row)

\c - - - :master_port
SELECT hasmetadata FROM pg_dist_node WHERE nodeport=:worker_2_proxy_port;
 hasmetadata
---------------------------------------------------------------------
 f
(1 row)

-- Verify we can stop metadata sync after unsuccessful attempts
SELECT citus.mitmproxy('conn.allow()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT stop_metadata_sync_to_node('localhost', :worker_2_proxy_port);
NOTICE:  dropping metadata on the node (localhost,9060)
 stop_metadata_sync_to_node
---------------------------------------------------------------------

(1 row)

\c - - - :worker_2_port
SELECT count(*) FROM pg_dist_node;
 count
---------------------------------------------------------------------
     0
(1 row)

\c - - - :master_port
SELECT hasmetadata FROM pg_dist_node WHERE nodeport=:worker_2_proxy_port;
 hasmetadata
---------------------------------------------------------------------
 f
(1 row)

-- turn metadata sync back on
SELECT start_metadata_sync_to_node('localhost', :worker_2_proxy_port);
 start_metadata_sync_to_node
---------------------------------------------------------------------

(1 row)

SET SEARCH_PATH = mx_metadata_sync;
DROP TABLE t1;
DROP TABLE t2;
DROP SCHEMA mx_metadata_sync CASCADE;