diff --git a/src/test/regress/expected/failure_copy_on_hash.out b/src/test/regress/expected/failure_copy_on_hash.out index 749345392..9707d2d2e 100644 --- a/src/test/regress/expected/failure_copy_on_hash.out +++ b/src/test/regress/expected/failure_copy_on_hash.out @@ -282,7 +282,6 @@ SELECT citus.mitmproxy('conn.kill()'); ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. -CONTEXT: COPY test_table_2, line 1: "1,2" SELECT citus.mitmproxy('conn.allow()'); mitmproxy --------------------------------------------------------------------- diff --git a/src/test/regress/expected/failure_copy_to_reference.out b/src/test/regress/expected/failure_copy_to_reference.out index a3cb3a2d8..a26e7290f 100644 --- a/src/test/regress/expected/failure_copy_to_reference.out +++ b/src/test/regress/expected/failure_copy_to_reference.out @@ -36,7 +36,6 @@ SELECT citus.mitmproxy('conn.kill()'); \copy test_table FROM STDIN DELIMITER ',' ERROR: failure on connection marked as essential: localhost:xxxxx -CONTEXT: COPY test_table, line 1: "1,2" SELECT citus.mitmproxy('conn.allow()'); mitmproxy --------------------------------------------------------------------- @@ -64,7 +63,6 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R \copy test_table FROM STDIN DELIMITER ',' ERROR: failure on connection marked as essential: localhost:xxxxx -CONTEXT: COPY test_table, line 1: "1,2" SELECT citus.mitmproxy('conn.allow()'); mitmproxy --------------------------------------------------------------------- @@ -92,7 +90,6 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R \copy test_table FROM STDIN DELIMITER ',' ERROR: canceling statement due to user request -CONTEXT: COPY test_table, line 1: "1,2" SELECT citus.mitmproxy('conn.allow()'); mitmproxy --------------------------------------------------------------------- diff --git a/src/test/regress/expected/failure_create_index_concurrently.out b/src/test/regress/expected/failure_create_index_concurrently.out index 0a040c17c..1a0dc4dec 100644 --- a/src/test/regress/expected/failure_create_index_concurrently.out +++ b/src/test/regress/expected/failure_create_index_concurrently.out @@ -38,7 +38,7 @@ SELECT citus.mitmproxy('conn.allow()'); (1 row) -- verify index is not created -SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test%' $$) +SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test_%' $$) WHERE nodeport = :worker_2_proxy_port; nodename | nodeport | success | result --------------------------------------------------------------------- @@ -153,7 +153,7 @@ SELECT citus.mitmproxy('conn.allow()'); (1 row) -- verify index is not dropped at worker 2 -SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test%' $$) +SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test_%' $$) WHERE nodeport = :worker_2_proxy_port; nodename | nodeport | success | result --------------------------------------------------------------------- @@ -186,7 +186,7 @@ NOTICE: drop cascades to 2 other objects DETAIL: drop cascades to table index_schema.index_test drop cascades to table index_schema.index_test_2 -- verify index is not at worker 2 upon cleanup -SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test%' $$) +SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test_%' $$) WHERE nodeport = :worker_2_proxy_port; nodename | nodeport | success | result --------------------------------------------------------------------- diff --git a/src/test/regress/expected/failure_ddl.out b/src/test/regress/expected/failure_ddl.out index 20082792a..9a30156c4 100644 --- a/src/test/regress/expected/failure_ddl.out +++ b/src/test/regress/expected/failure_ddl.out @@ -9,7 +9,7 @@ SET search_path TO 'ddl_failure'; -- do not cache any connections SET citus.max_cached_conns_per_worker TO 0; -- we don't want to see the prepared transaction numbers in the warnings -SET client_min_messages TO ERROR; +SET client_min_messages TO WARNING; SELECT citus.mitmproxy('conn.allow()'); mitmproxy --------------------------------------------------------------------- @@ -69,9 +69,12 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R (1 row) ALTER TABLE test_table ADD COLUMN new_column INT; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly +WARNING: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. +connection not open +CONTEXT: while executing command on localhost:xxxxx +ERROR: failure on connection marked as essential: localhost:xxxxx SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass; array_agg --------------------------------------------------------------------- @@ -153,6 +156,9 @@ SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT").cancel(' || pg_backend_pi (1 row) ALTER TABLE test_table ADD COLUMN new_column INT; +WARNING: +CONTEXT: while executing command on localhost:xxxxx +WARNING: failed to commit transaction on localhost:xxxxx SELECT citus.mitmproxy('conn.allow()'); mitmproxy --------------------------------------------------------------------- @@ -365,9 +371,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R (1 row) ALTER TABLE test_table DROP COLUMN new_column; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly - This probably means the server terminated abnormally - before or while processing the request. +ERROR: failure on connection marked as essential: localhost:xxxxx SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass; array_agg --------------------------------------------------------------------- @@ -735,9 +739,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R (1 row) ALTER TABLE test_table ADD COLUMN new_column INT; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly - This probably means the server terminated abnormally - before or while processing the request. +ERROR: failure on connection marked as essential: localhost:xxxxx SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass; array_agg --------------------------------------------------------------------- @@ -1030,9 +1032,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R (1 row) ALTER TABLE test_table ADD COLUMN new_column INT; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly - This probably means the server terminated abnormally - before or while processing the request. +ERROR: failure on connection marked as essential: localhost:xxxxx SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass; array_agg --------------------------------------------------------------------- diff --git a/src/test/regress/expected/failure_mx_metadata_sync.out b/src/test/regress/expected/failure_mx_metadata_sync.out index 85180dcc1..f3856410a 100644 --- a/src/test/regress/expected/failure_mx_metadata_sync.out +++ b/src/test/regress/expected/failure_mx_metadata_sync.out @@ -21,7 +21,14 @@ SELECT create_distributed_table('t1', 'id'); (1 row) INSERT INTO t1 SELECT x FROM generate_series(1,100) AS f(x); --- Initial metadata status +-- Initially turn metadata sync off because we'll ingest errors to start/stop metadata sync operations +SELECT stop_metadata_sync_to_node('localhost', :worker_2_proxy_port); +NOTICE: dropping metadata on the node (localhost,9060) + stop_metadata_sync_to_node +--------------------------------------------------------------------- + +(1 row) + SELECT hasmetadata FROM pg_dist_node WHERE nodeport=:worker_2_proxy_port; hasmetadata --------------------------------------------------------------------- @@ -365,6 +372,13 @@ SELECT hasmetadata FROM pg_dist_node WHERE nodeport=:worker_2_proxy_port; f (1 row) +-- turn metadata sync back on +SELECT start_metadata_sync_to_node('localhost', :worker_2_proxy_port); + start_metadata_sync_to_node +--------------------------------------------------------------------- + +(1 row) + SET SEARCH_PATH = mx_metadata_sync; DROP TABLE t1; DROP TABLE t2; diff --git a/src/test/regress/expected/failure_truncate.out b/src/test/regress/expected/failure_truncate.out index 645ec0392..4d9c0d6d6 100644 --- a/src/test/regress/expected/failure_truncate.out +++ b/src/test/regress/expected/failure_truncate.out @@ -100,9 +100,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R (1 row) TRUNCATE test_table; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly - This probably means the server terminated abnormally - before or while processing the request. +ERROR: failure on connection marked as essential: localhost:xxxxx SELECT citus.mitmproxy('conn.allow()'); mitmproxy --------------------------------------------------------------------- @@ -649,9 +647,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R (1 row) TRUNCATE test_table; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly - This probably means the server terminated abnormally - before or while processing the request. +ERROR: failure on connection marked as essential: localhost:xxxxx SELECT citus.mitmproxy('conn.allow()'); mitmproxy --------------------------------------------------------------------- @@ -1027,9 +1023,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R (1 row) TRUNCATE test_table; -ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly - This probably means the server terminated abnormally - before or while processing the request. +ERROR: failure on connection marked as essential: localhost:xxxxx SELECT citus.mitmproxy('conn.allow()'); mitmproxy --------------------------------------------------------------------- diff --git a/src/test/regress/failure_schedule b/src/test/regress/failure_schedule index 17b4d0c14..4353913b7 100644 --- a/src/test/regress/failure_schedule +++ b/src/test/regress/failure_schedule @@ -1,5 +1,4 @@ # import this file (from psql you can use \i) to use mitmproxy manually -test: turn_mx_off test: failure_test_helpers # this should only be run by pg_regress_multi, you don't need it @@ -17,22 +16,25 @@ test: failure_add_disable_node test: failure_copy_to_reference test: failure_copy_on_hash test: failure_create_reference_table +test: check_mx +test: turn_mx_off test: failure_create_distributed_table_non_empty test: failure_create_table +test: failure_single_select +test: failure_connection_establishment +test: turn_mx_on test: failure_multi_shard_update_delete test: failure_cte_subquery test: failure_insert_select_via_coordinator test: failure_multi_dml test: failure_vacuum -test: failure_single_select test: failure_ref_tables test: failure_insert_select_pushdown test: failure_single_mod test: failure_savepoints test: failure_multi_row_insert test: failure_mx_metadata_sync -test: failure_connection_establishment # this test syncs metadata to the workers test: failure_failover_to_local_execution @@ -45,3 +47,4 @@ test: ensure_no_intermediate_data_leak # in the shared memory # -------- test: ensure_no_shared_connection_leak +test: check_mx diff --git a/src/test/regress/sql/failure_copy_on_hash.sql b/src/test/regress/sql/failure_copy_on_hash.sql index c9f98bc5c..d60242803 100644 --- a/src/test/regress/sql/failure_copy_on_hash.sql +++ b/src/test/regress/sql/failure_copy_on_hash.sql @@ -142,14 +142,6 @@ SELECT create_distributed_table('test_table_2','id'); SELECT citus.mitmproxy('conn.kill()'); \COPY test_table_2 FROM stdin delimiter ','; -1,2 -3,4 -6,7 -8,9 -9,10 -11,12 -13,14 -\. SELECT citus.mitmproxy('conn.allow()'); SELECT pds.logicalrelid, pdsd.shardid, pdsd.shardstate diff --git a/src/test/regress/sql/failure_copy_to_reference.sql b/src/test/regress/sql/failure_copy_to_reference.sql index 15b5037b9..b6c8d6c81 100644 --- a/src/test/regress/sql/failure_copy_to_reference.sql +++ b/src/test/regress/sql/failure_copy_to_reference.sql @@ -25,10 +25,6 @@ CREATE VIEW unhealthy_shard_count AS -- response we get from the worker SELECT citus.mitmproxy('conn.kill()'); \copy test_table FROM STDIN DELIMITER ',' -1,2 -2,3 -3,4 -\. SELECT citus.mitmproxy('conn.allow()'); SELECT * FROM unhealthy_shard_count; SELECT count(*) FROM test_table; @@ -36,10 +32,6 @@ SELECT count(*) FROM test_table; -- kill as soon as the coordinator sends begin SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED").kill()'); \copy test_table FROM STDIN DELIMITER ',' -1,2 -2,3 -3,4 -\. SELECT citus.mitmproxy('conn.allow()'); SELECT * FROM unhealthy_shard_count; SELECT count(*) FROM test_table; @@ -47,10 +39,6 @@ SELECT count(*) FROM test_table; -- cancel as soon as the coordinator sends begin SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED").cancel(' || pg_backend_pid() || ')'); \copy test_table FROM STDIN DELIMITER ',' -1,2 -2,3 -3,4 -\. SELECT citus.mitmproxy('conn.allow()'); SELECT * FROM unhealthy_shard_count; SELECT count(*) FROM test_table; diff --git a/src/test/regress/sql/failure_create_index_concurrently.sql b/src/test/regress/sql/failure_create_index_concurrently.sql index 4f624bd22..23b07b4b4 100644 --- a/src/test/regress/sql/failure_create_index_concurrently.sql +++ b/src/test/regress/sql/failure_create_index_concurrently.sql @@ -20,7 +20,7 @@ CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1); SELECT citus.mitmproxy('conn.allow()'); -- verify index is not created -SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test%' $$) +SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test_%' $$) WHERE nodeport = :worker_2_proxy_port; @@ -78,7 +78,7 @@ DROP INDEX CONCURRENTLY IF EXISTS idx_index_test; SELECT citus.mitmproxy('conn.allow()'); -- verify index is not dropped at worker 2 -SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test%' $$) +SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test_%' $$) WHERE nodeport = :worker_2_proxy_port; -- test unique concurrent index creation failure when there are duplicates @@ -97,5 +97,5 @@ RESET SEARCH_PATH; DROP SCHEMA index_schema CASCADE; -- verify index is not at worker 2 upon cleanup -SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test%' $$) +SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test_%' $$) WHERE nodeport = :worker_2_proxy_port; diff --git a/src/test/regress/sql/failure_ddl.sql b/src/test/regress/sql/failure_ddl.sql index 2618771ab..a13d563e0 100644 --- a/src/test/regress/sql/failure_ddl.sql +++ b/src/test/regress/sql/failure_ddl.sql @@ -14,7 +14,7 @@ SET search_path TO 'ddl_failure'; SET citus.max_cached_conns_per_worker TO 0; -- we don't want to see the prepared transaction numbers in the warnings -SET client_min_messages TO ERROR; +SET client_min_messages TO WARNING; SELECT citus.mitmproxy('conn.allow()'); diff --git a/src/test/regress/sql/failure_mx_metadata_sync.sql b/src/test/regress/sql/failure_mx_metadata_sync.sql index 9c7db9a68..171d49a1a 100644 --- a/src/test/regress/sql/failure_mx_metadata_sync.sql +++ b/src/test/regress/sql/failure_mx_metadata_sync.sql @@ -14,7 +14,8 @@ CREATE TABLE t1 (id int PRIMARY KEY); SELECT create_distributed_table('t1', 'id'); INSERT INTO t1 SELECT x FROM generate_series(1,100) AS f(x); --- Initial metadata status +-- Initially turn metadata sync off because we'll ingest errors to start/stop metadata sync operations +SELECT stop_metadata_sync_to_node('localhost', :worker_2_proxy_port); SELECT hasmetadata FROM pg_dist_node WHERE nodeport=:worker_2_proxy_port; -- Failure to set groupid in the worker @@ -95,6 +96,10 @@ SELECT count(*) FROM pg_dist_node; \c - - - :master_port SELECT hasmetadata FROM pg_dist_node WHERE nodeport=:worker_2_proxy_port; + +-- turn metadata sync back on +SELECT start_metadata_sync_to_node('localhost', :worker_2_proxy_port); + SET SEARCH_PATH = mx_metadata_sync; DROP TABLE t1; DROP TABLE t2;