Turn metadata sync on in failure schedule

src/test/regress/expected/failure_copy_on_hash.out

@@ -282,7 +282,6 @@ SELECT citus.mitmproxy('conn.kill()');
 ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  COPY test_table_2, line 1: "1,2"
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy
 ---------------------------------------------------------------------

src/test/regress/expected/failure_copy_to_reference.out

@@ -36,7 +36,6 @@ SELECT citus.mitmproxy('conn.kill()');
 
 \copy test_table FROM STDIN DELIMITER ','
 ERROR:  failure on connection marked as essential: localhost:xxxxx
-CONTEXT:  COPY test_table, line 1: "1,2"
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy
 ---------------------------------------------------------------------
@@ -64,7 +63,6 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 
 \copy test_table FROM STDIN DELIMITER ','
 ERROR:  failure on connection marked as essential: localhost:xxxxx
-CONTEXT:  COPY test_table, line 1: "1,2"
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy
 ---------------------------------------------------------------------
@@ -92,7 +90,6 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 
 \copy test_table FROM STDIN DELIMITER ','
 ERROR:  canceling statement due to user request
-CONTEXT:  COPY test_table, line 1: "1,2"
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy
 ---------------------------------------------------------------------

src/test/regress/expected/failure_create_index_concurrently.out

@@ -38,7 +38,7 @@ SELECT citus.mitmproxy('conn.allow()');
 (1 row)
 
 -- verify index is not created
-SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test%' $$)
+SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test_%' $$)
 WHERE nodeport = :worker_2_proxy_port;
  nodename  | nodeport | success | result
 ---------------------------------------------------------------------
@@ -153,7 +153,7 @@ SELECT citus.mitmproxy('conn.allow()');
 (1 row)
 
 -- verify index is not dropped at worker 2
-SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test%' $$)
+SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test_%' $$)
 WHERE nodeport = :worker_2_proxy_port;
  nodename  | nodeport | success | result
 ---------------------------------------------------------------------
@@ -186,7 +186,7 @@ NOTICE:  drop cascades to 2 other objects
 DETAIL:  drop cascades to table index_schema.index_test
 drop cascades to table index_schema.index_test_2
 -- verify index is not at worker 2 upon cleanup
-SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test%' $$)
+SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test_%' $$)
 WHERE nodeport = :worker_2_proxy_port;
  nodename  | nodeport | success | result
 ---------------------------------------------------------------------

src/test/regress/expected/failure_ddl.out

@@ -9,7 +9,7 @@ SET search_path TO 'ddl_failure';
 -- do not cache any connections
 SET citus.max_cached_conns_per_worker TO 0;
 -- we don't want to see the prepared transaction numbers in the warnings
-SET client_min_messages TO ERROR;
+SET client_min_messages TO WARNING;
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy
 ---------------------------------------------------------------------
@@ -69,9 +69,12 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 ALTER TABLE test_table ADD COLUMN new_column INT;
-ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
+WARNING:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
+connection not open
+CONTEXT:  while executing command on localhost:xxxxx
+ERROR:  failure on connection marked as essential: localhost:xxxxx
 SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass;
   array_agg
 ---------------------------------------------------------------------
@@ -153,6 +156,9 @@ SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT").cancel(' ||  pg_backend_pi
 (1 row)
 
 ALTER TABLE test_table ADD COLUMN new_column INT;
+WARNING:
+CONTEXT:  while executing command on localhost:xxxxx
+WARNING:  failed to commit transaction on localhost:xxxxx
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy
 ---------------------------------------------------------------------
@@ -365,9 +371,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 ALTER TABLE test_table DROP COLUMN new_column;
-ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
+ERROR:  failure on connection marked as essential: localhost:xxxxx
 SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass;
        array_agg
 ---------------------------------------------------------------------
@@ -735,9 +739,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 ALTER TABLE test_table ADD COLUMN new_column INT;
-ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
+ERROR:  failure on connection marked as essential: localhost:xxxxx
 SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass;
   array_agg
 ---------------------------------------------------------------------
@@ -1030,9 +1032,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 ALTER TABLE test_table ADD COLUMN new_column INT;
-ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
+ERROR:  failure on connection marked as essential: localhost:xxxxx
 SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass;
   array_agg
 ---------------------------------------------------------------------

src/test/regress/expected/failure_mx_metadata_sync.out

@@ -21,7 +21,14 @@ SELECT create_distributed_table('t1', 'id');
 (1 row)
 
 INSERT INTO t1 SELECT x FROM generate_series(1,100) AS f(x);
--- Initial metadata status
+-- Initially turn metadata sync off because we'll ingest errors to start/stop metadata sync operations
+SELECT stop_metadata_sync_to_node('localhost', :worker_2_proxy_port);
+NOTICE:  dropping metadata on the node (localhost,9060)
+ stop_metadata_sync_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
 SELECT hasmetadata FROM pg_dist_node WHERE nodeport=:worker_2_proxy_port;
  hasmetadata
 ---------------------------------------------------------------------
@@ -365,6 +372,13 @@ SELECT hasmetadata FROM pg_dist_node WHERE nodeport=:worker_2_proxy_port;
  f
 (1 row)
 
+-- turn metadata sync back on
+SELECT start_metadata_sync_to_node('localhost', :worker_2_proxy_port);
+ start_metadata_sync_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
 SET SEARCH_PATH = mx_metadata_sync;
 DROP TABLE t1;
 DROP TABLE t2;

src/test/regress/expected/failure_truncate.out

@@ -100,9 +100,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 TRUNCATE test_table;
-ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
+ERROR:  failure on connection marked as essential: localhost:xxxxx
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy
 ---------------------------------------------------------------------
@@ -649,9 +647,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 TRUNCATE test_table;
-ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
+ERROR:  failure on connection marked as essential: localhost:xxxxx
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy
 ---------------------------------------------------------------------
@@ -1027,9 +1023,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 TRUNCATE test_table;
-ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
+ERROR:  failure on connection marked as essential: localhost:xxxxx
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy
 ---------------------------------------------------------------------

src/test/regress/failure_schedule

@@ -1,5 +1,4 @@
 # import this file (from psql you can use \i) to use mitmproxy manually
-test: turn_mx_off
 test: failure_test_helpers
 
 # this should only be run by pg_regress_multi, you don't need it
@@ -17,22 +16,25 @@ test: failure_add_disable_node
 test: failure_copy_to_reference
 test: failure_copy_on_hash
 test: failure_create_reference_table
+test: check_mx
+test: turn_mx_off
 test: failure_create_distributed_table_non_empty
 test: failure_create_table
+test: failure_single_select
+test: failure_connection_establishment
+test: turn_mx_on
 
 test: failure_multi_shard_update_delete
 test: failure_cte_subquery
 test: failure_insert_select_via_coordinator
 test: failure_multi_dml
 test: failure_vacuum
-test: failure_single_select
 test: failure_ref_tables
 test: failure_insert_select_pushdown
 test: failure_single_mod
 test: failure_savepoints
 test: failure_multi_row_insert
 test: failure_mx_metadata_sync
-test: failure_connection_establishment
 
 # this test syncs metadata to the workers
 test: failure_failover_to_local_execution
@@ -45,3 +47,4 @@ test: ensure_no_intermediate_data_leak
 # in the shared memory
 # --------
 test: ensure_no_shared_connection_leak
+test: check_mx

src/test/regress/sql/failure_copy_on_hash.sql

@@ -142,14 +142,6 @@ SELECT create_distributed_table('test_table_2','id');
 SELECT citus.mitmproxy('conn.kill()');
 
 \COPY test_table_2 FROM stdin delimiter ',';
-1,2
-3,4
-6,7
-8,9
-9,10
-11,12
-13,14
-\.
 
 SELECT citus.mitmproxy('conn.allow()');
 SELECT pds.logicalrelid, pdsd.shardid, pdsd.shardstate

src/test/regress/sql/failure_copy_to_reference.sql

@@ -25,10 +25,6 @@ CREATE VIEW unhealthy_shard_count AS
 -- response we get from the worker
 SELECT citus.mitmproxy('conn.kill()');
 \copy test_table FROM STDIN DELIMITER ','
-1,2
-2,3
-3,4
-\.
 SELECT citus.mitmproxy('conn.allow()');
 SELECT * FROM unhealthy_shard_count;
 SELECT count(*) FROM test_table;
@@ -36,10 +32,6 @@ SELECT count(*) FROM test_table;
 -- kill as soon as the coordinator sends begin
 SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED").kill()');
 \copy test_table FROM STDIN DELIMITER ','
-1,2
-2,3
-3,4
-\.
 SELECT citus.mitmproxy('conn.allow()');
 SELECT * FROM unhealthy_shard_count;
 SELECT count(*) FROM test_table;
@@ -47,10 +39,6 @@ SELECT count(*) FROM test_table;
 -- cancel as soon as the coordinator sends begin
 SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED").cancel(' ||  pg_backend_pid() || ')');
 \copy test_table FROM STDIN DELIMITER ','
-1,2
-2,3
-3,4
-\.
 SELECT citus.mitmproxy('conn.allow()');
 SELECT * FROM unhealthy_shard_count;
 SELECT count(*) FROM test_table;

src/test/regress/sql/failure_create_index_concurrently.sql

@@ -20,7 +20,7 @@ CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1);
 
 SELECT citus.mitmproxy('conn.allow()');
 -- verify index is not created
-SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test%' $$)
+SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test_%' $$)
 WHERE nodeport = :worker_2_proxy_port;
 
 
@@ -78,7 +78,7 @@ DROP INDEX CONCURRENTLY IF EXISTS idx_index_test;
 SELECT citus.mitmproxy('conn.allow()');
 
 -- verify index is not dropped at worker 2
-SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test%' $$)
+SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test_%' $$)
 WHERE nodeport = :worker_2_proxy_port;
 
 -- test unique concurrent index creation failure when there are duplicates
@@ -97,5 +97,5 @@ RESET SEARCH_PATH;
 DROP SCHEMA index_schema CASCADE;
 
 -- verify index is not at worker 2 upon cleanup
-SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test%' $$)
+SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test_%' $$)
 WHERE nodeport = :worker_2_proxy_port;

src/test/regress/sql/failure_ddl.sql

@@ -14,7 +14,7 @@ SET search_path TO 'ddl_failure';
 SET citus.max_cached_conns_per_worker TO 0;
 
 -- we don't want to see the prepared transaction numbers in the warnings
-SET client_min_messages TO ERROR;
+SET client_min_messages TO WARNING;
 
 SELECT citus.mitmproxy('conn.allow()');
 

src/test/regress/sql/failure_mx_metadata_sync.sql

@@ -14,7 +14,8 @@ CREATE TABLE t1 (id int PRIMARY KEY);
 SELECT create_distributed_table('t1', 'id');
 INSERT INTO t1 SELECT x FROM generate_series(1,100) AS f(x);
 
--- Initial metadata status
+-- Initially turn metadata sync off because we'll ingest errors to start/stop metadata sync operations
+SELECT stop_metadata_sync_to_node('localhost', :worker_2_proxy_port);
 SELECT hasmetadata FROM pg_dist_node WHERE nodeport=:worker_2_proxy_port;
 
 -- Failure to set groupid in the worker
@@ -95,6 +96,10 @@ SELECT count(*) FROM pg_dist_node;
 
 \c - - - :master_port
 SELECT hasmetadata FROM pg_dist_node WHERE nodeport=:worker_2_proxy_port;
+
+-- turn metadata sync back on
+SELECT start_metadata_sync_to_node('localhost', :worker_2_proxy_port);
+
 SET SEARCH_PATH = mx_metadata_sync;
 DROP TABLE t1;
 DROP TABLE t2;