From e182215d96e351ac21e6f30ad8e611ae8eca8b47 Mon Sep 17 00:00:00 2001 From: Onder Kalaci Date: Tue, 10 Mar 2020 18:44:54 +0100 Subject: [PATCH] Improve connection error message from the worker nodes We currently put the actual error message to the detail part. However, many drivers don't show detail part. As connection errors are somehow common, and hard to trace back, can't we added the detail to the message itself. In addition to that, we changed "connection error" message, as it was confusing to the users who think that the error was happening while connecting to the coordinator. In fact, this error is showing up when the coordinator fails to connect remote nodes. --- .../distributed/connection/remote_commands.c | 21 +- .../expected/failure_1pc_copy_append.out | 3 +- .../expected/failure_1pc_copy_hash.out | 3 +- .../failure_connection_establishment.out | 2 +- .../regress/expected/failure_copy_on_hash.out | 15 +- ...ure_create_distributed_table_non_empty.out | 27 +- .../failure_create_reference_table.out | 9 +- .../regress/expected/failure_create_table.out | 33 +- .../regress/expected/failure_cte_subquery.out | 12 +- src/test/regress/expected/failure_ddl.out | 36 +- .../expected/failure_distributed_results.out | 3 +- .../failure_insert_select_pushdown.out | 6 +- .../failure_insert_select_repartition.out | 24 +- .../regress/expected/failure_multi_dml.out | 9 +- .../expected/failure_multi_row_insert.out | 15 +- .../failure_multi_shard_update_delete.out | 48 +- .../regress/expected/failure_ref_tables.out | 9 +- .../failure_replicated_partitions.out | 3 +- .../regress/expected/failure_savepoints.out | 19 +- .../regress/expected/failure_single_mod.out | 9 +- .../expected/failure_single_select.out | 18 +- .../regress/expected/failure_truncate.out | 30 +- src/test/regress/expected/failure_vacuum.out | 9 +- .../expected/multi_modifying_xacts.out | 24 +- .../expected/multi_mx_node_metadata.out | 14 +- .../multi_replicate_reference_table.out | 20 +- .../regress/expected/multi_router_planner.out | 6 +- .../regress/expected/node_conninfo_reload.out | 21 +- src/test/regress/output/multi_copy.source | 478 +++++++++--------- .../regress/sql/multi_mx_node_metadata.sql | 2 + .../sql/multi_replicate_reference_table.sql | 19 +- 31 files changed, 425 insertions(+), 522 deletions(-) diff --git a/src/backend/distributed/connection/remote_commands.c b/src/backend/distributed/connection/remote_commands.c index a2160a891..e9e0570b3 100644 --- a/src/backend/distributed/connection/remote_commands.c +++ b/src/backend/distributed/connection/remote_commands.c @@ -277,10 +277,23 @@ ReportConnectionError(MultiConnection *connection, int elevel) messageDetail = pchomp(PQerrorMessage(pgConn)); } - ereport(elevel, (errcode(ERRCODE_CONNECTION_FAILURE), - errmsg("connection error: %s:%d", nodeName, nodePort), - messageDetail != NULL ? - errdetail("%s", ApplyLogRedaction(messageDetail)) : 0)); + if (messageDetail) + { + /* + * We don't use ApplyLogRedaction(messageDetail) as we expect any error + * detail that requires log reduction should have done it locally. + */ + ereport(elevel, (errcode(ERRCODE_CONNECTION_FAILURE), + errmsg("connection to the remote node %s:%d failed with the " + "following error: %s", nodeName, nodePort, + messageDetail))); + } + else + { + ereport(elevel, (errcode(ERRCODE_CONNECTION_FAILURE), + errmsg("connection to the remote node %s:%d failed", + nodeName, nodePort))); + } } diff --git a/src/test/regress/expected/failure_1pc_copy_append.out b/src/test/regress/expected/failure_1pc_copy_append.out index 0bbf05175..6f05eab3c 100644 --- a/src/test/regress/expected/failure_1pc_copy_append.out +++ b/src/test/regress/expected/failure_1pc_copy_append.out @@ -168,8 +168,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="SELECT|COPY").kill()'); (1 row) SELECT count(1) FROM copy_test; -WARNING: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. count diff --git a/src/test/regress/expected/failure_1pc_copy_hash.out b/src/test/regress/expected/failure_1pc_copy_hash.out index cc9584d02..272872dd9 100644 --- a/src/test/regress/expected/failure_1pc_copy_hash.out +++ b/src/test/regress/expected/failure_1pc_copy_hash.out @@ -219,8 +219,7 @@ SELECT citus.mitmproxy('conn.killall()'); (1 row) COPY copy_test FROM PROGRAM 'echo 0, 0 && echo 1, 1 && echo 2, 4 && echo 3, 9' WITH CSV; -WARNING: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. CONTEXT: COPY copy_test, line 1: "0, 0" diff --git a/src/test/regress/expected/failure_connection_establishment.out b/src/test/regress/expected/failure_connection_establishment.out index 30beab4ef..93dbf1a10 100644 --- a/src/test/regress/expected/failure_connection_establishment.out +++ b/src/test/regress/expected/failure_connection_establishment.out @@ -227,7 +227,7 @@ RESET client_min_messages; -- verify get_global_active_transactions works when a timeout happens on a connection SELECT get_global_active_transactions(); WARNING: could not establish connection after 400 ms -WARNING: connection error: localhost:xxxxx +WARNING: connection to the remote node localhost:xxxxx failed get_global_active_transactions --------------------------------------------------------------------- (0 rows) diff --git a/src/test/regress/expected/failure_copy_on_hash.out b/src/test/regress/expected/failure_copy_on_hash.out index b867005af..439bbe94d 100644 --- a/src/test/regress/expected/failure_copy_on_hash.out +++ b/src/test/regress/expected/failure_copy_on_hash.out @@ -36,8 +36,7 @@ SELECT citus.mitmproxy('conn.kill()'); (1 row) \COPY test_table FROM stdin delimiter ','; -WARNING: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. CONTEXT: COPY test_table, line 1: "1,2" @@ -276,23 +275,19 @@ SELECT citus.mitmproxy('conn.kill()'); (1 row) \COPY test_table_2 FROM stdin delimiter ','; -WARNING: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. CONTEXT: COPY test_table_2, line 1: "1,2" -WARNING: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. CONTEXT: COPY test_table_2, line 2: "3,4" -WARNING: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. CONTEXT: COPY test_table_2, line 3: "6,7" -WARNING: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. CONTEXT: COPY test_table_2, line 5: "9,10" diff --git a/src/test/regress/expected/failure_create_distributed_table_non_empty.out b/src/test/regress/expected/failure_create_distributed_table_non_empty.out index 2fea6e43a..6b7ec6ae4 100644 --- a/src/test/regress/expected/failure_create_distributed_table_non_empty.out +++ b/src/test/regress/expected/failure_create_distributed_table_non_empty.out @@ -26,8 +26,7 @@ SELECT citus.mitmproxy('conn.kill()'); (1 row) SELECT create_distributed_table('test_table', 'id'); -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT count(*) FROM pg_dist_shard WHERE logicalrelid='create_distributed_table_non_empty_failure.test_table'::regclass; @@ -125,8 +124,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R (1 row) SELECT create_distributed_table('test_table', 'id'); -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT citus.mitmproxy('conn.allow()'); @@ -190,8 +188,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").kill()'); (1 row) SELECT create_distributed_table('test_table', 'id'); -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT count(*) FROM pg_dist_shard WHERE logicalrelid='create_distributed_table_non_empty_failure.test_table'::regclass; @@ -473,8 +470,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R (1 row) SELECT create_distributed_table('test_table', 'id', colocate_with => 'colocated_table'); -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT count(*) FROM pg_dist_shard WHERE logicalrelid='create_distributed_table_non_empty_failure.test_table'::regclass; @@ -561,8 +557,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^SELECT worker_apply_shard_ddl_comma (1 row) SELECT create_distributed_table('test_table', 'id', colocate_with => 'colocated_table'); -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT count(*) FROM pg_dist_shard WHERE logicalrelid='create_distributed_table_non_empty_failure.test_table'::regclass; @@ -599,8 +594,7 @@ SELECT citus.mitmproxy('conn.kill()'); (1 row) SELECT create_distributed_table('test_table', 'id'); -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT citus.mitmproxy('conn.allow()'); @@ -667,8 +661,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R (1 row) SELECT create_distributed_table('test_table', 'id'); -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT citus.mitmproxy('conn.allow()'); @@ -732,8 +725,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").kill()'); (1 row) SELECT create_distributed_table('test_table', 'id'); -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT count(*) FROM pg_dist_shard WHERE logicalrelid='create_distributed_table_non_empty_failure.test_table'::regclass; @@ -946,8 +938,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R (1 row) SELECT create_distributed_table('test_table', 'id', colocate_with => 'colocated_table'); -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT count(*) FROM pg_dist_shard WHERE logicalrelid='create_distributed_table_non_empty_failure.test_table'::regclass; diff --git a/src/test/regress/expected/failure_create_reference_table.out b/src/test/regress/expected/failure_create_reference_table.out index 13b383b8b..bfb29cebf 100644 --- a/src/test/regress/expected/failure_create_reference_table.out +++ b/src/test/regress/expected/failure_create_reference_table.out @@ -25,8 +25,7 @@ SELECT citus.mitmproxy('conn.onQuery().kill()'); (1 row) SELECT create_reference_table('ref_table'); -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT count(*) FROM pg_dist_shard_placement; @@ -43,8 +42,7 @@ SELECT citus.mitmproxy('conn.onCommandComplete(command="BEGIN").kill()'); (1 row) SELECT create_reference_table('ref_table'); -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT count(*) FROM pg_dist_shard_placement; @@ -76,8 +74,7 @@ SELECT citus.mitmproxy('conn.onCommandComplete(command="SELECT 1").kill()'); (1 row) SELECT create_reference_table('ref_table'); -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT count(*) FROM pg_dist_shard_placement; diff --git a/src/test/regress/expected/failure_create_table.out b/src/test/regress/expected/failure_create_table.out index fe839b791..48946b5e5 100644 --- a/src/test/regress/expected/failure_create_table.out +++ b/src/test/regress/expected/failure_create_table.out @@ -20,8 +20,7 @@ SELECT citus.mitmproxy('conn.kill()'); (1 row) SELECT create_distributed_table('test_table','id'); -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT citus.mitmproxy('conn.allow()'); @@ -89,8 +88,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R (1 row) SELECT create_distributed_table('test_table','id'); -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT citus.mitmproxy('conn.allow()'); @@ -120,8 +118,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="SELECT worker_apply_shard_ddl_comman (1 row) SELECT create_distributed_table('test_table','id'); -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT citus.mitmproxy('conn.allow()'); @@ -154,8 +151,7 @@ BEGIN; (1 row) SELECT create_distributed_table('test_table', 'id'); -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. COMMIT; @@ -225,8 +221,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").kill()'); (1 row) SELECT create_distributed_table('test_table','id',colocate_with=>'temp_table'); -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT citus.mitmproxy('conn.allow()'); @@ -347,8 +342,7 @@ SELECT citus.mitmproxy('conn.kill()'); BEGIN; SELECT create_distributed_table('test_table','id'); -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. ROLLBACK; @@ -384,8 +378,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R BEGIN; SELECT create_distributed_table('test_table','id'); -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. ROLLBACK; @@ -463,8 +456,7 @@ SELECT citus.mitmproxy('conn.kill()'); BEGIN; SELECT create_distributed_table('test_table','id'); -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. ROLLBACK; @@ -496,8 +488,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").kill()'); BEGIN; SELECT create_distributed_table('test_table','id'); -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. ROLLBACK; @@ -533,8 +524,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R BEGIN; SELECT create_distributed_table('test_table','id'); -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. ROLLBACK; @@ -615,8 +605,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R (1 row) SELECT master_create_worker_shards('test_table_2', 4, 2); -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT count(*) FROM pg_dist_shard; diff --git a/src/test/regress/expected/failure_cte_subquery.out b/src/test/regress/expected/failure_cte_subquery.out index bad22eeed..866c591ac 100644 --- a/src/test/regress/expected/failure_cte_subquery.out +++ b/src/test/regress/expected/failure_cte_subquery.out @@ -88,8 +88,7 @@ FROM ORDER BY 1 DESC LIMIT 5 ) as foo WHERE foo.user_id = cte.user_id; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. -- kill at the third copy (pull) @@ -122,8 +121,7 @@ FROM ORDER BY 1 DESC LIMIT 5 ) as foo WHERE foo.user_id = cte.user_id; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. -- cancel at the first copy (push) @@ -262,8 +260,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^DELETE FROM").kill()'); WITH cte_delete as (DELETE FROM users_table WHERE user_name in ('A', 'D') RETURNING *) INSERT INTO users_table SELECT * FROM cte_delete; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. -- verify contents are the same @@ -378,8 +375,7 @@ BEGIN; SET LOCAL citus.multi_shard_modify_mode = 'sequential'; WITH cte_delete as (DELETE FROM users_table WHERE user_name in ('A', 'D') RETURNING *) INSERT INTO users_table SELECT * FROM cte_delete; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. END; diff --git a/src/test/regress/expected/failure_ddl.out b/src/test/regress/expected/failure_ddl.out index 8b31e291c..80bd88a96 100644 --- a/src/test/regress/expected/failure_ddl.out +++ b/src/test/regress/expected/failure_ddl.out @@ -37,8 +37,7 @@ SELECT citus.mitmproxy('conn.onAuthenticationOk().kill()'); (1 row) ALTER TABLE test_table ADD COLUMN new_column INT; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass; @@ -71,8 +70,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R (1 row) ALTER TABLE test_table ADD COLUMN new_column INT; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass; @@ -104,8 +102,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_apply_shard_ddl_command").kil (1 row) ALTER TABLE test_table ADD COLUMN new_column INT; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. -- show that we've never commited the changes @@ -349,8 +346,7 @@ SELECT citus.mitmproxy('conn.onAuthenticationOk().kill()'); (1 row) ALTER TABLE test_table DROP COLUMN new_column; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass; @@ -383,8 +379,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R (1 row) ALTER TABLE test_table DROP COLUMN new_column; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass; @@ -416,8 +411,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_apply_shard_ddl_command").kil (1 row) ALTER TABLE test_table DROP COLUMN new_column; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass; @@ -720,8 +714,7 @@ SELECT citus.mitmproxy('conn.onAuthenticationOk().kill()'); (1 row) ALTER TABLE test_table ADD COLUMN new_column INT; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass; @@ -754,8 +747,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R (1 row) ALTER TABLE test_table ADD COLUMN new_column INT; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass; @@ -787,8 +779,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_apply_shard_ddl_command").kil (1 row) ALTER TABLE test_table ADD COLUMN new_column INT; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass; @@ -1048,8 +1039,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R (1 row) ALTER TABLE test_table ADD COLUMN new_column INT; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass; @@ -1081,8 +1071,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_apply_shard_ddl_command").kil (1 row) ALTER TABLE test_table ADD COLUMN new_column INT; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. -- kill as soon as the coordinator after it sends worker_apply_shard_ddl_command 2nd time @@ -1093,8 +1082,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_apply_shard_ddl_command").aft (1 row) ALTER TABLE test_table ADD COLUMN new_column INT; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. -- cancel as soon as the coordinator after it sends worker_apply_shard_ddl_command 2nd time diff --git a/src/test/regress/expected/failure_distributed_results.out b/src/test/regress/expected/failure_distributed_results.out index 5681a5173..4065b648a 100644 --- a/src/test/regress/expected/failure_distributed_results.out +++ b/src/test/regress/expected/failure_distributed_results.out @@ -85,8 +85,7 @@ CREATE TABLE distributed_result_info AS SELECT resultId, nodeport, rowcount, targetShardId, targetShardIndex FROM partition_task_list_results('test', $$ SELECT * FROM source_table $$, 'target_table') NATURAL JOIN pg_dist_node; -WARNING: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT * FROM distributed_result_info ORDER BY resultId; diff --git a/src/test/regress/expected/failure_insert_select_pushdown.out b/src/test/regress/expected/failure_insert_select_pushdown.out index b92b49b36..1661b6ff7 100644 --- a/src/test/regress/expected/failure_insert_select_pushdown.out +++ b/src/test/regress/expected/failure_insert_select_pushdown.out @@ -44,8 +44,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT INTO insert_select_pushdown" (1 row) INSERT INTO events_summary SELECT user_id, event_id, count(*) FROM events_table GROUP BY 1,2; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. --verify nothing is modified @@ -98,8 +97,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT INTO insert_select_pushdown" (1 row) INSERT INTO events_table SELECT * FROM events_table; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. --verify nothing is modified diff --git a/src/test/regress/expected/failure_insert_select_repartition.out b/src/test/regress/expected/failure_insert_select_repartition.out index a0d91e0ef..0cc9a5e86 100644 --- a/src/test/regress/expected/failure_insert_select_repartition.out +++ b/src/test/regress/expected/failure_insert_select_repartition.out @@ -54,8 +54,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_partition_query_result").kill (1 row) INSERT INTO target_table SELECT * FROM source_table; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT * FROM target_table ORDER BY a; @@ -70,12 +69,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_partition_query_result").kill (1 row) INSERT INTO target_table SELECT * FROM replicated_source_table; -WARNING: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. -WARNING: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT * FROM target_table ORDER BY a; @@ -106,8 +103,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="fetch_intermediate_results").kill()' (1 row) INSERT INTO target_table SELECT * FROM source_table; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT * FROM target_table ORDER BY a; @@ -122,8 +118,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="fetch_intermediate_results").kill()' (1 row) INSERT INTO target_table SELECT * FROM replicated_source_table; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT * FROM target_table ORDER BY a; @@ -143,8 +138,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="read_intermediate_results").kill()') (1 row) INSERT INTO target_table SELECT * FROM source_table; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT * FROM target_table ORDER BY a; @@ -159,8 +153,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="read_intermediate_results").kill()') (1 row) INSERT INTO target_table SELECT * FROM replicated_source_table; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT * FROM target_table ORDER BY a; @@ -179,8 +172,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="read_intermediate_results").kill()') (1 row) INSERT INTO replicated_target_table SELECT * FROM source_table; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT * FROM replicated_target_table; diff --git a/src/test/regress/expected/failure_multi_dml.out b/src/test/regress/expected/failure_multi_dml.out index 89f48536a..d4061eb1d 100644 --- a/src/test/regress/expected/failure_multi_dml.out +++ b/src/test/regress/expected/failure_multi_dml.out @@ -33,8 +33,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^DELETE").kill()'); BEGIN; DELETE FROM dml_test WHERE id = 1; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. DELETE FROM dml_test WHERE id = 2; @@ -96,8 +95,7 @@ BEGIN; DELETE FROM dml_test WHERE id = 1; DELETE FROM dml_test WHERE id = 2; INSERT INTO dml_test VALUES (5, 'Epsilon'); -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. UPDATE dml_test SET name = 'alpha' WHERE id = 1; @@ -154,8 +152,7 @@ DELETE FROM dml_test WHERE id = 1; DELETE FROM dml_test WHERE id = 2; INSERT INTO dml_test VALUES (5, 'Epsilon'); UPDATE dml_test SET name = 'alpha' WHERE id = 1; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. UPDATE dml_test SET name = 'gamma' WHERE id = 3; diff --git a/src/test/regress/expected/failure_multi_row_insert.out b/src/test/regress/expected/failure_multi_row_insert.out index a5ab9b357..cdc0d82af 100644 --- a/src/test/regress/expected/failure_multi_row_insert.out +++ b/src/test/regress/expected/failure_multi_row_insert.out @@ -43,8 +43,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()'); (1 row) INSERT INTO distributed_table VALUES (1,1), (1,2), (1,3); -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. -- this test is broken, see https://github.com/citusdata/citus/issues/2460 @@ -58,8 +57,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()'); (1 row) INSERT INTO distributed_table VALUES (1,7), (5,8); -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. -- this test is broken, see https://github.com/citusdata/citus/issues/2460 @@ -73,8 +71,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()'); (1 row) INSERT INTO distributed_table VALUES (1,11), (6,12); -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").cancel(' || :pid || ')'); @@ -93,8 +90,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").after(1).kill()'); (1 row) INSERT INTO distributed_table VALUES (1,15), (6,16); -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").after(1).cancel(' || :pid || ')'); @@ -113,8 +109,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()'); (1 row) INSERT INTO distributed_table VALUES (2,19),(1,20); -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").cancel(' || :pid || ')'); diff --git a/src/test/regress/expected/failure_multi_shard_update_delete.out b/src/test/regress/expected/failure_multi_shard_update_delete.out index 68860e5b0..631e04807 100644 --- a/src/test/regress/expected/failure_multi_shard_update_delete.out +++ b/src/test/regress/expected/failure_multi_shard_update_delete.out @@ -64,8 +64,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM").kill()'); -- issue a multi shard delete DELETE FROM t2 WHERE b = 2; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. -- verify nothing is deleted @@ -83,8 +82,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM multi_shard.t2_201005"). (1 row) DELETE FROM t2 WHERE b = 2; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. -- verify nothing is deleted @@ -146,8 +144,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()'); -- issue a multi shard update UPDATE t2 SET c = 4 WHERE b = 2; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. -- verify nothing is updated @@ -165,8 +162,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="UPDATE multi_shard.t2_201005").kill( (1 row) UPDATE t2 SET c = 4 WHERE b = 2; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. -- verify nothing is updated @@ -222,8 +218,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM").kill()'); -- issue a multi shard delete DELETE FROM t2 WHERE b = 2; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. -- verify nothing is deleted @@ -241,8 +236,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM multi_shard.t2_201005"). (1 row) DELETE FROM t2 WHERE b = 2; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. -- verify nothing is deleted @@ -304,8 +298,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()'); -- issue a multi shard update UPDATE t2 SET c = 4 WHERE b = 2; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. -- verify nothing is updated @@ -323,8 +316,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="UPDATE multi_shard.t2_201005").kill( (1 row) UPDATE t2 SET c = 4 WHERE b = 2; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. -- verify nothing is updated @@ -397,8 +389,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM").kill()'); (1 row) DELETE FROM r1 WHERE a = 2; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. -- verify nothing is deleted @@ -415,8 +406,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM").kill()'); (1 row) DELETE FROM t2 WHERE b = 2; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. -- verify nothing is deleted @@ -500,8 +490,7 @@ UPDATE t3 SET c = q.c FROM ( SELECT b, max(c) as c FROM t2 GROUP BY b) q WHERE t3.b = q.b RETURNING *; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. --- verify nothing is updated @@ -559,8 +548,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="UPDATE multi_shard.t3_201013").kill( (1 row) UPDATE t3 SET b = 2 WHERE b = 1; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. -- verify nothing is updated @@ -594,8 +582,7 @@ SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FRO -- following will fail UPDATE t3 SET b = 2 WHERE b = 1; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. END; @@ -613,8 +600,7 @@ SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FRO (1 row) UPDATE t3 SET b = 1 WHERE b = 2 RETURNING *; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. -- verify nothing is updated @@ -633,8 +619,7 @@ SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FRO (1 row) UPDATE t3 SET b = 2 WHERE b = 1; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. -- verify nothing is updated @@ -668,8 +653,7 @@ SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FRO -- following will fail UPDATE t3 SET b = 2 WHERE b = 1; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. END; diff --git a/src/test/regress/expected/failure_ref_tables.out b/src/test/regress/expected/failure_ref_tables.out index a397f8dfd..79f79b09d 100644 --- a/src/test/regress/expected/failure_ref_tables.out +++ b/src/test/regress/expected/failure_ref_tables.out @@ -33,8 +33,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()'); (1 row) INSERT INTO ref_table VALUES (5, 6); -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT COUNT(*) FROM ref_table WHERE key=5; @@ -51,8 +50,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()'); (1 row) UPDATE ref_table SET key=7 RETURNING value; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT COUNT(*) FROM ref_table WHERE key=7; @@ -71,8 +69,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()'); BEGIN; DELETE FROM ref_table WHERE key=5; UPDATE ref_table SET key=value; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. COMMIT; diff --git a/src/test/regress/expected/failure_replicated_partitions.out b/src/test/regress/expected/failure_replicated_partitions.out index 3d6d748a8..bfa8189af 100644 --- a/src/test/regress/expected/failure_replicated_partitions.out +++ b/src/test/regress/expected/failure_replicated_partitions.out @@ -29,8 +29,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()'); (1 row) INSERT INTO partitioned_table VALUES (0, 0); -WARNING: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. -- use both placements diff --git a/src/test/regress/expected/failure_savepoints.out b/src/test/regress/expected/failure_savepoints.out index 873eaca5c..48d2e154d 100644 --- a/src/test/regress/expected/failure_savepoints.out +++ b/src/test/regress/expected/failure_savepoints.out @@ -40,8 +40,7 @@ INSERT INTO artists VALUES (5, 'Asher Lev'); SAVEPOINT s1; WARNING: connection not open CONTEXT: while executing command on localhost:xxxxx -WARNING: connection error: localhost:xxxxx -DETAIL: connection not open +WARNING: connection to the remote node localhost:xxxxx failed with the following error: connection not open WARNING: connection not open CONTEXT: while executing command on localhost:xxxxx WARNING: connection not open @@ -74,8 +73,7 @@ RELEASE SAVEPOINT s1; WARNING: AbortSubTransaction while in COMMIT state WARNING: connection not open CONTEXT: while executing command on localhost:xxxxx -WARNING: connection error: localhost:xxxxx -DETAIL: connection not open +WARNING: connection to the remote node localhost:xxxxx failed with the following error: connection not open WARNING: connection not open CONTEXT: while executing command on localhost:xxxxx WARNING: connection not open @@ -132,8 +130,7 @@ RELEASE SAVEPOINT s2; WARNING: AbortSubTransaction while in COMMIT state WARNING: connection not open CONTEXT: while executing command on localhost:xxxxx -WARNING: connection error: localhost:xxxxx -DETAIL: connection not open +WARNING: connection to the remote node localhost:xxxxx failed with the following error: connection not open WARNING: connection not open CONTEXT: while executing command on localhost:xxxxx WARNING: connection not open @@ -213,7 +210,7 @@ ROLLBACK TO SAVEPOINT s1; WARNING: connection not open WARNING: connection not open WARNING: connection not open -WARNING: connection error: localhost:xxxxx +WARNING: connection to the remote node localhost:xxxxx failed with the following error: connection not open WARNING: connection not open WARNING: connection not open COMMIT; @@ -248,7 +245,7 @@ BEGIN; INSERT INTO researchers VALUES (7, 4, 'Jan Plaza'); SAVEPOINT s1; WARNING: connection not open -WARNING: connection error: localhost:xxxxx +WARNING: connection to the remote node localhost:xxxxx failed with the following error: connection not open WARNING: connection not open WARNING: connection not open ERROR: connection not open @@ -321,7 +318,7 @@ ROLLBACK TO s1; RELEASE SAVEPOINT s1; WARNING: AbortSubTransaction while in COMMIT state WARNING: connection not open -WARNING: connection error: localhost:xxxxx +WARNING: connection to the remote node localhost:xxxxx failed with the following error: connection not open WARNING: connection not open WARNING: connection not open WARNING: savepoint "savepoint_3" does not exist @@ -370,10 +367,10 @@ WARNING: connection not open WARNING: connection not open SELECT * FROM ref; WARNING: connection not open -WARNING: connection error: localhost:xxxxx +WARNING: connection to the remote node localhost:xxxxx failed with the following error: connection not open WARNING: connection not open WARNING: connection not open -ERROR: connection error: localhost:xxxxx +ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open END; -- clean up SELECT citus.mitmproxy('conn.allow()'); diff --git a/src/test/regress/expected/failure_single_mod.out b/src/test/regress/expected/failure_single_mod.out index 65c5ffbef..90d9c40e2 100644 --- a/src/test/regress/expected/failure_single_mod.out +++ b/src/test/regress/expected/failure_single_mod.out @@ -27,8 +27,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()'); (1 row) INSERT INTO mod_test VALUES (2, 6); -WARNING: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT COUNT(*) FROM mod_test WHERE key=2; @@ -63,8 +62,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()'); (1 row) UPDATE mod_test SET value='ok' WHERE key=2 RETURNING key; -WARNING: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. key @@ -102,8 +100,7 @@ INSERT INTO mod_test VALUES (2, 6); INSERT INTO mod_test VALUES (2, 7); DELETE FROM mod_test WHERE key=2 AND value = '7'; UPDATE mod_test SET value='ok' WHERE key=2; -WARNING: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. COMMIT; diff --git a/src/test/regress/expected/failure_single_select.out b/src/test/regress/expected/failure_single_select.out index d5087779f..d46b801ec 100644 --- a/src/test/regress/expected/failure_single_select.out +++ b/src/test/regress/expected/failure_single_select.out @@ -28,8 +28,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").kill()'); (1 row) SELECT * FROM select_test WHERE key = 3; -WARNING: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. key | value @@ -38,8 +37,7 @@ DETAIL: server closed the connection unexpectedly (1 row) SELECT * FROM select_test WHERE key = 3; -WARNING: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. key | value @@ -57,8 +55,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").kill()'); BEGIN; INSERT INTO select_test VALUES (3, 'more data'); SELECT * FROM select_test WHERE key = 3; -WARNING: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. key | value @@ -69,8 +66,7 @@ DETAIL: server closed the connection unexpectedly INSERT INTO select_test VALUES (3, 'even more data'); SELECT * FROM select_test WHERE key = 3; -WARNING: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. key | value @@ -160,8 +156,7 @@ SELECT * FROM select_test WHERE key = 3; INSERT INTO select_test VALUES (3, 'even more data'); SELECT * FROM select_test WHERE key = 3; -WARNING: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. key | value @@ -215,8 +210,7 @@ SELECT * FROM select_test WHERE key = 1; (1 row) SELECT * FROM select_test WHERE key = 1; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. -- now the same test with query cancellation diff --git a/src/test/regress/expected/failure_truncate.out b/src/test/regress/expected/failure_truncate.out index 28cbe1b46..fb40401a5 100644 --- a/src/test/regress/expected/failure_truncate.out +++ b/src/test/regress/expected/failure_truncate.out @@ -44,8 +44,7 @@ SELECT citus.mitmproxy('conn.onAuthenticationOk().kill()'); (1 row) TRUNCATE test_table; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT citus.mitmproxy('conn.allow()'); @@ -102,8 +101,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R (1 row) TRUNCATE test_table; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT citus.mitmproxy('conn.allow()'); @@ -159,8 +157,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="TRUNCATE TABLE truncate_failure.test (1 row) TRUNCATE test_table; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT citus.mitmproxy('conn.allow()'); @@ -433,8 +430,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^TRUNCATE TABLE").after(2).kill()'); (1 row) TRUNCATE reference_table CASCADE; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT citus.mitmproxy('conn.allow()'); @@ -577,8 +573,7 @@ SELECT citus.mitmproxy('conn.onAuthenticationOk().kill()'); (1 row) TRUNCATE test_table; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT citus.mitmproxy('conn.allow()'); @@ -635,8 +630,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R (1 row) TRUNCATE test_table; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT citus.mitmproxy('conn.allow()'); @@ -692,8 +686,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^TRUNCATE TABLE truncate_failure.tes (1 row) TRUNCATE test_table; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT citus.mitmproxy('conn.allow()'); @@ -956,8 +949,7 @@ SELECT citus.mitmproxy('conn.onAuthenticationOk().kill()'); (1 row) TRUNCATE test_table; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT citus.mitmproxy('conn.allow()'); @@ -1014,8 +1006,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R (1 row) TRUNCATE test_table; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT citus.mitmproxy('conn.allow()'); @@ -1071,8 +1062,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="TRUNCATE TABLE truncate_failure.test (1 row) TRUNCATE test_table; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT citus.mitmproxy('conn.allow()'); diff --git a/src/test/regress/expected/failure_vacuum.out b/src/test/regress/expected/failure_vacuum.out index 66801ec22..fddaaff91 100644 --- a/src/test/regress/expected/failure_vacuum.out +++ b/src/test/regress/expected/failure_vacuum.out @@ -31,8 +31,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^VACUUM").kill()'); (1 row) VACUUM vacuum_test; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT citus.mitmproxy('conn.onQuery(query="^ANALYZE").kill()'); @@ -42,8 +41,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^ANALYZE").kill()'); (1 row) ANALYZE vacuum_test; -WARNING: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT").kill()'); @@ -111,8 +109,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^VACUUM.*other").kill()'); (1 row) VACUUM vacuum_test, other_vacuum_test; -ERROR: connection error: localhost:xxxxx -DETAIL: server closed the connection unexpectedly +ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT citus.mitmproxy('conn.onQuery(query="^VACUUM.*other").cancel(' || pg_backend_pid() || ')'); diff --git a/src/test/regress/expected/multi_modifying_xacts.out b/src/test/regress/expected/multi_modifying_xacts.out index 7b48d85b0..24b6e7133 100644 --- a/src/test/regress/expected/multi_modifying_xacts.out +++ b/src/test/regress/expected/multi_modifying_xacts.out @@ -1178,15 +1178,15 @@ NOTICE: not propagating ALTER ROLE ... RENAME TO commands to worker nodes \c - test_user - :master_port -- should fail since the worker doesn't have test_user anymore INSERT INTO reference_failure_test VALUES (1, '1'); -ERROR: connection error: localhost:xxxxx +ERROR: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist -- the same as the above, but wrapped within a transaction BEGIN; INSERT INTO reference_failure_test VALUES (1, '1'); -ERROR: connection error: localhost:xxxxx +ERROR: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist COMMIT; BEGIN; COPY reference_failure_test FROM STDIN WITH (FORMAT 'csv'); -ERROR: connection error: localhost:xxxxx +ERROR: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist COMMIT; -- show that no data go through the table and shard states are good SET client_min_messages to 'ERROR'; @@ -1211,8 +1211,8 @@ ORDER BY s.logicalrelid, sp.shardstate; BEGIN; COPY numbers_hash_failure_test FROM STDIN WITH (FORMAT 'csv'); -WARNING: connection error: localhost:xxxxx -WARNING: connection error: localhost:xxxxx +WARNING: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist +WARNING: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist -- some placements are invalid before abort SELECT shardid, shardstate, nodename, nodeport FROM pg_dist_shard_placement JOIN pg_dist_shard USING (shardid) @@ -1233,8 +1233,8 @@ ORDER BY shardid, nodeport; ABORT; -- verify nothing is inserted SELECT count(*) FROM numbers_hash_failure_test; -WARNING: connection error: localhost:xxxxx -WARNING: connection error: localhost:xxxxx +WARNING: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist +WARNING: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist count --------------------------------------------------------------------- 0 @@ -1259,8 +1259,8 @@ ORDER BY shardid, nodeport; BEGIN; COPY numbers_hash_failure_test FROM STDIN WITH (FORMAT 'csv'); -WARNING: connection error: localhost:xxxxx -WARNING: connection error: localhost:xxxxx +WARNING: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist +WARNING: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist -- check shard states before commit SELECT shardid, shardstate, nodename, nodeport FROM pg_dist_shard_placement JOIN pg_dist_shard USING (shardid) @@ -1298,8 +1298,8 @@ ORDER BY shardid, nodeport; -- verify data is inserted SELECT count(*) FROM numbers_hash_failure_test; -WARNING: connection error: localhost:xxxxx -WARNING: connection error: localhost:xxxxx +WARNING: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist +WARNING: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist count --------------------------------------------------------------------- 2 @@ -1312,7 +1312,7 @@ NOTICE: not propagating ALTER ROLE ... RENAME TO commands to worker nodes \c - test_user - :master_port -- fails on all shard placements INSERT INTO numbers_hash_failure_test VALUES (2,2); -ERROR: connection error: localhost:xxxxx +ERROR: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" does not exist -- connect back to the master with the proper user to continue the tests \c - :default_user - :master_port SET citus.next_shard_id TO 1200020; diff --git a/src/test/regress/expected/multi_mx_node_metadata.out b/src/test/regress/expected/multi_mx_node_metadata.out index c602678e6..d5e5f8a9c 100644 --- a/src/test/regress/expected/multi_mx_node_metadata.out +++ b/src/test/regress/expected/multi_mx_node_metadata.out @@ -9,6 +9,7 @@ SET citus.replication_model TO streaming; SET citus.shard_count TO 8; SET citus.shard_replication_factor TO 1; SET citus.replicate_reference_tables_on_activate TO off; +\set VERBOSITY terse -- Simulates a readonly node by setting default_transaction_read_only. CREATE FUNCTION mark_node_readonly(hostname TEXT, port INTEGER, isreadonly BOOLEAN) RETURNS TEXT @@ -319,7 +320,6 @@ SET citus.log_remote_commands = ON; -- trigger caching for prepared statements EXECUTE foo; NOTICE: issuing SELECT count(*) AS count FROM public.dist_table_1_102010 dist_table_1 WHERE (a OPERATOR(pg_catalog.=) 1) -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx count --------------------------------------------------------------------- 0 @@ -327,7 +327,6 @@ DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx EXECUTE foo; NOTICE: issuing SELECT count(*) AS count FROM public.dist_table_1_102010 dist_table_1 WHERE (a OPERATOR(pg_catalog.=) 1) -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx count --------------------------------------------------------------------- 0 @@ -335,7 +334,6 @@ DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx EXECUTE foo; NOTICE: issuing SELECT count(*) AS count FROM public.dist_table_1_102010 dist_table_1 WHERE (a OPERATOR(pg_catalog.=) 1) -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx count --------------------------------------------------------------------- 0 @@ -343,7 +341,6 @@ DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx EXECUTE foo; NOTICE: issuing SELECT count(*) AS count FROM public.dist_table_1_102010 dist_table_1 WHERE (a OPERATOR(pg_catalog.=) 1) -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx count --------------------------------------------------------------------- 0 @@ -351,7 +348,6 @@ DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx EXECUTE foo; NOTICE: issuing SELECT count(*) AS count FROM public.dist_table_1_102010 dist_table_1 WHERE (a OPERATOR(pg_catalog.=) 1) -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx count --------------------------------------------------------------------- 0 @@ -359,7 +355,6 @@ DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx EXECUTE foo; NOTICE: issuing SELECT count(*) AS count FROM public.dist_table_1_102010 dist_table_1 WHERE (a OPERATOR(pg_catalog.=) 1) -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx count --------------------------------------------------------------------- 0 @@ -367,7 +362,6 @@ DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx EXECUTE foo; NOTICE: issuing SELECT count(*) AS count FROM public.dist_table_1_102010 dist_table_1 WHERE (a OPERATOR(pg_catalog.=) 1) -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx count --------------------------------------------------------------------- 0 @@ -381,7 +375,6 @@ SELECT master_update_node(:nodeid_1, '127.0.0.1', :worker_1_port); SELECT wait_until_metadata_sync(30000); NOTICE: issuing LISTEN metadata_sync -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx wait_until_metadata_sync --------------------------------------------------------------------- @@ -390,7 +383,6 @@ DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -- make sure the nodename changed. EXECUTE foo; NOTICE: issuing SELECT count(*) AS count FROM public.dist_table_1_102010 dist_table_1 WHERE (a OPERATOR(pg_catalog.=) 1) -DETAIL: on server postgres@127.0.0.1:57637 connectionId: xxxxxxx count --------------------------------------------------------------------- 0 @@ -500,8 +492,6 @@ UPDATE pg_dist_node SET metadatasynced = TRUE WHERE nodeid IN (:nodeid_1, :nodei -- should error out SELECT 1 FROM master_disable_node('localhost', 1); ERROR: Disabling localhost:xxxxx failed -DETAIL: connection error: localhost:xxxxx -HINT: If you are using MX, try stop_metadata_sync_to_node(hostname, port) for nodes that are down before disabling them. -- try again after stopping metadata sync SELECT stop_metadata_sync_to_node('localhost', 1); stop_metadata_sync_to_node @@ -566,8 +556,6 @@ UPDATE pg_dist_node SET metadatasynced = TRUE WHERE nodeid IN (:nodeid_1, :nodei -- should error out SELECT 1 FROM master_disable_node('localhost', :worker_2_port); ERROR: Disabling localhost:xxxxx failed -DETAIL: connection error: localhost:xxxxx -HINT: If you are using MX, try stop_metadata_sync_to_node(hostname, port) for nodes that are down before disabling them. -- try again after stopping metadata sync SELECT stop_metadata_sync_to_node('localhost', 1); stop_metadata_sync_to_node diff --git a/src/test/regress/expected/multi_replicate_reference_table.out b/src/test/regress/expected/multi_replicate_reference_table.out index db1229bf1..d09b2e378 100644 --- a/src/test/regress/expected/multi_replicate_reference_table.out +++ b/src/test/regress/expected/multi_replicate_reference_table.out @@ -1088,10 +1088,24 @@ END; -- test adding an invalid node while we have reference tables to replicate -- set client message level to ERROR and verbosity to terse to supporess -- OS-dependent host name resolution warnings -SET client_min_messages to ERROR; \set VERBOSITY terse -SELECT master_add_node('invalid-node-name', 9999); -ERROR: connection error: invalid-node-name:9999 +SET client_min_messages to ERROR; +DO $$ +DECLARE + errors_received INTEGER; +BEGIN +errors_received := 0; + BEGIN + SELECT master_add_node('invalid-node-name', 9999); + EXCEPTION WHEN OTHERS THEN + IF SQLERRM LIKE 'connection to the remote node%%' THEN + errors_received := errors_received + 1; + END IF; + END; +RAISE '(%/1) failed to add node', errors_received; +END; +$$; +ERROR: (1/1) failed to add node -- drop unnecassary tables DROP TABLE initially_not_replicated_reference_table; -- reload pg_dist_shard_placement table diff --git a/src/test/regress/expected/multi_router_planner.out b/src/test/regress/expected/multi_router_planner.out index 37cac63a9..ab52b50ea 100644 --- a/src/test/regress/expected/multi_router_planner.out +++ b/src/test/regress/expected/multi_router_planner.out @@ -2442,8 +2442,7 @@ GRANT INSERT ON ALL TABLES IN SCHEMA public TO router_user; -- we will fail to connect to worker 2, since the user does not exist BEGIN; INSERT INTO failure_test VALUES (1, 1); -WARNING: connection error: localhost:xxxxx -DETAIL: FATAL: role "router_user" does not exist +WARNING: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "router_user" does not exist SELECT shardid, shardstate, nodename, nodeport FROM pg_dist_shard_placement WHERE shardid IN ( SELECT shardid FROM pg_dist_shard @@ -2460,8 +2459,7 @@ SELECT shardid, shardstate, nodename, nodeport FROM pg_dist_shard_placement ROLLBACK; INSERT INTO failure_test VALUES (2, 1); -WARNING: connection error: localhost:xxxxx -DETAIL: FATAL: role "router_user" does not exist +WARNING: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "router_user" does not exist SELECT shardid, shardstate, nodename, nodeport FROM pg_dist_shard_placement WHERE shardid IN ( SELECT shardid FROM pg_dist_shard diff --git a/src/test/regress/expected/node_conninfo_reload.out b/src/test/regress/expected/node_conninfo_reload.out index c0049b5e5..8a82eebee 100644 --- a/src/test/regress/expected/node_conninfo_reload.out +++ b/src/test/regress/expected/node_conninfo_reload.out @@ -46,8 +46,7 @@ show citus.node_conninfo; -- Should give a connection error because of bad sslmode select count(*) from test where a = 0; -ERROR: connection error: localhost:xxxxx -DETAIL: invalid sslmode value: "doesnotexist" +ERROR: connection to the remote node localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist" -- Reset it again ALTER SYSTEM RESET citus.node_conninfo; select pg_reload_conf(); @@ -114,8 +113,7 @@ select count(*) from test where a = 0; COMMIT; -- Should fail now with connection error, when transaction is finished select count(*) from test where a = 0; -ERROR: connection error: localhost:xxxxx -DETAIL: invalid sslmode value: "doesnotexist" +ERROR: connection to the remote node localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist" -- Reset it again ALTER SYSTEM RESET citus.node_conninfo; select pg_reload_conf(); @@ -177,8 +175,7 @@ select count(*) from test where a = 0; COMMIT; -- Should fail now, when transaction is finished select count(*) from test where a = 0; -ERROR: connection error: localhost:xxxxx -DETAIL: invalid sslmode value: "doesnotexist" +ERROR: connection to the remote node localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist" -- Reset it again ALTER SYSTEM RESET citus.node_conninfo; select pg_reload_conf(); @@ -231,13 +228,11 @@ show citus.node_conninfo; -- Should fail since a different shard is accessed and thus a new connection -- will to be created. select count(*) from test where a = 0; -ERROR: connection error: localhost:xxxxx -DETAIL: invalid sslmode value: "doesnotexist" +ERROR: connection to the remote node localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist" COMMIT; -- Should still fail now, when transaction is finished select count(*) from test where a = 0; -ERROR: connection error: localhost:xxxxx -DETAIL: invalid sslmode value: "doesnotexist" +ERROR: connection to the remote node localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist" -- Reset it again ALTER SYSTEM RESET citus.node_conninfo; select pg_reload_conf(); @@ -298,8 +293,7 @@ select count(*) from test; COMMIT; -- Should fail now, when transaction is finished select count(*) from test; -ERROR: connection error: localhost:xxxxx -DETAIL: invalid sslmode value: "doesnotexist" +ERROR: connection to the remote node localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist" -- Reset it again ALTER SYSTEM RESET citus.node_conninfo; select pg_reload_conf(); @@ -355,8 +349,7 @@ ERROR: division by zero ROLLBACK; -- Should fail now, when transaction is finished select count(*) from test; -ERROR: connection error: localhost:xxxxx -DETAIL: invalid sslmode value: "doesnotexist" +ERROR: connection to the remote node localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist" -- Reset it again ALTER SYSTEM RESET citus.node_conninfo; select pg_reload_conf(); diff --git a/src/test/regress/output/multi_copy.source b/src/test/regress/output/multi_copy.source index 99c5a3d9f..e8f68f43d 100644 --- a/src/test/regress/output/multi_copy.source +++ b/src/test/regress/output/multi_copy.source @@ -14,9 +14,9 @@ CREATE TABLE customer_copy_hash ( c_comment varchar(117), primary key (c_custkey)); SELECT master_create_distributed_table('customer_copy_hash', 'c_custkey', 'hash'); - master_create_distributed_table ---------------------------------- - + master_create_distributed_table +--------------------------------------------------------------------- + (1 row) -- Test COPY into empty hash-partitioned table @@ -25,9 +25,9 @@ ERROR: could not find any shards into which to copy DETAIL: No shards exist for distributed table "customer_copy_hash". HINT: Run master_create_worker_shards to create shards and try again. SELECT master_create_worker_shards('customer_copy_hash', 64, 1); - master_create_worker_shards ------------------------------ - + master_create_worker_shards +--------------------------------------------------------------------- + (1 row) -- Test empty copy @@ -38,8 +38,8 @@ ERROR: invalid input syntax for integer: "1,customer1" CONTEXT: COPY customer_copy_hash, line 1, column c_custkey: "1,customer1" -- Confirm that no data was copied SELECT count(*) FROM customer_copy_hash; - count -------- + count +--------------------------------------------------------------------- 0 (1 row) @@ -50,8 +50,8 @@ ERROR: duplicate key value violates unique constraint "customer_copy_hash_pkey_ DETAIL: Key (c_custkey)=(2) already exists. -- Confirm that no data was copied SELECT count(*) FROM customer_copy_hash; - count -------- + count +--------------------------------------------------------------------- 0 (1 row) @@ -60,8 +60,8 @@ COPY customer_copy_hash (c_custkey, c_name) FROM STDIN WITH (FORMAT 'csv', HEADER true, FORCE_NULL (c_custkey)); -- Confirm that only first row was skipped SELECT count(*) FROM customer_copy_hash; - count -------- + count +--------------------------------------------------------------------- 3 (1 row) @@ -70,8 +70,8 @@ COPY customer_copy_hash (c_custkey, c_name, c_address) FROM STDIN WITH (FORMAT 'csv', QUOTE '"', FORCE_NOT_NULL (c_address)); -- Confirm that value is not null SELECT count(c_address) FROM customer_copy_hash WHERE c_custkey = 4; - count -------- + count +--------------------------------------------------------------------- 1 (1 row) @@ -80,8 +80,8 @@ COPY customer_copy_hash (c_custkey, c_name, c_address) FROM STDIN WITH (FORMAT 'csv', QUOTE '"', FORCE_NULL (c_address)); -- Confirm that value is null SELECT count(c_address) FROM customer_copy_hash WHERE c_custkey = 5; - count -------- + count +--------------------------------------------------------------------- 0 (1 row) @@ -92,8 +92,8 @@ ERROR: null value in column "c_name" violates not-null constraint DETAIL: Failing row contains (8, null, null, null, null, null, null, null). -- Confirm that no data was copied SELECT count(*) FROM customer_copy_hash; - count -------- + count +--------------------------------------------------------------------- 5 (1 row) @@ -102,8 +102,8 @@ COPY customer_copy_hash (c_custkey, c_name) FROM PROGRAM 'echo 9 customer9' WITH (DELIMITER ' '); -- Confirm that data was copied SELECT count(*) FROM customer_copy_hash WHERE c_custkey = 9; - count -------- + count +--------------------------------------------------------------------- 1 (1 row) @@ -111,8 +111,8 @@ SELECT count(*) FROM customer_copy_hash WHERE c_custkey = 9; COPY customer_copy_hash FROM '@abs_srcdir@/data/customer.2.data' WITH (DELIMITER '|'); -- Confirm that data was copied SELECT count(*) FROM customer_copy_hash; - count -------- + count +--------------------------------------------------------------------- 1006 (1 row) @@ -120,28 +120,28 @@ SELECT count(*) FROM customer_copy_hash; \copy customer_copy_hash FROM '@abs_srcdir@/data/customer.3.data' WITH (DELIMITER '|'); -- Confirm that data was copied SELECT count(*) FROM customer_copy_hash; - count -------- + count +--------------------------------------------------------------------- 2006 (1 row) -- Make sure that master_update_shard_statistics() only updates shard length for -- hash-partitioned tables SELECT master_update_shard_statistics(560000); - master_update_shard_statistics --------------------------------- + master_update_shard_statistics +--------------------------------------------------------------------- 8192 (1 row) SELECT shardid, shardminvalue, shardmaxvalue FROM pg_dist_shard WHERE shardid = 560000; - shardid | shardminvalue | shardmaxvalue ----------+---------------+--------------- + shardid | shardminvalue | shardmaxvalue +--------------------------------------------------------------------- 560000 | -2147483648 | -2080374785 (1 row) SELECT shardid, shardlength FROM pg_dist_shard_placement WHERE shardid = 560000; - shardid | shardlength ----------+------------- + shardid | shardlength +--------------------------------------------------------------------- 560000 | 8192 (1 row) @@ -151,15 +151,15 @@ CREATE TABLE customer_with_default( c_name varchar(25) not null, c_time timestamp default now()); SELECT master_create_distributed_table('customer_with_default', 'c_custkey', 'hash'); - master_create_distributed_table ---------------------------------- - + master_create_distributed_table +--------------------------------------------------------------------- + (1 row) SELECT master_create_worker_shards('customer_with_default', 64, 1); - master_create_worker_shards ------------------------------ - + master_create_worker_shards +--------------------------------------------------------------------- + (1 row) -- Test with default values for now() function @@ -167,8 +167,8 @@ COPY customer_with_default (c_custkey, c_name) FROM STDIN WITH (FORMAT 'csv'); -- Confirm that data was copied with now() function SELECT count(*) FROM customer_with_default where c_time IS NOT NULL; - count -------- + count +--------------------------------------------------------------------- 2 (1 row) @@ -177,8 +177,8 @@ ALTER TABLE customer_copy_hash ADD COLUMN extra1 INT DEFAULT 0; ALTER TABLE customer_copy_hash ADD COLUMN extra2 INT DEFAULT 0; COPY customer_copy_hash (c_custkey, c_name, extra1, extra2) FROM STDIN CSV; SELECT * FROM customer_copy_hash WHERE extra1 = 1; - c_custkey | c_name | c_address | c_nationkey | c_phone | c_acctbal | c_mktsegment | c_comment | extra1 | extra2 ------------+------------+-----------+-------------+---------+-----------+--------------+-----------+--------+-------- + c_custkey | c_name | c_address | c_nationkey | c_phone | c_acctbal | c_mktsegment | c_comment | extra1 | extra2 +--------------------------------------------------------------------- 10 | customer10 | | | | | | | 1 | 5 (1 row) @@ -186,8 +186,8 @@ SELECT * FROM customer_copy_hash WHERE extra1 = 1; ALTER TABLE customer_copy_hash DROP COLUMN extra1; COPY customer_copy_hash (c_custkey, c_name, extra2) FROM STDIN CSV; SELECT * FROM customer_copy_hash WHERE c_custkey = 11; - c_custkey | c_name | c_address | c_nationkey | c_phone | c_acctbal | c_mktsegment | c_comment | extra2 ------------+------------+-----------+-------------+---------+-----------+--------------+-----------+-------- + c_custkey | c_name | c_address | c_nationkey | c_phone | c_acctbal | c_mktsegment | c_comment | extra2 +--------------------------------------------------------------------- 11 | customer11 | | | | | | | 5 (1 row) @@ -195,9 +195,9 @@ SELECT * FROM customer_copy_hash WHERE c_custkey = 11; ALTER TABLE customer_copy_hash DROP COLUMN extra2; COPY customer_copy_hash (c_custkey, c_name) FROM STDIN CSV; SELECT * FROM customer_copy_hash WHERE c_custkey = 12; - c_custkey | c_name | c_address | c_nationkey | c_phone | c_acctbal | c_mktsegment | c_comment ------------+------------+-----------+-------------+---------+-----------+--------------+----------- - 12 | customer12 | | | | | | + c_custkey | c_name | c_address | c_nationkey | c_phone | c_acctbal | c_mktsegment | c_comment +--------------------------------------------------------------------- + 12 | customer12 | | | | | | (1 row) -- Create a new range-partitioned table into which to COPY @@ -212,9 +212,9 @@ CREATE TABLE customer_copy_range ( c_comment varchar(117), primary key (c_custkey)); SELECT master_create_distributed_table('customer_copy_range', 'c_custkey', 'range'); - master_create_distributed_table ---------------------------------- - + master_create_distributed_table +--------------------------------------------------------------------- + (1 row) -- Test COPY into empty range-partitioned table @@ -234,15 +234,15 @@ COPY customer_copy_range FROM '@abs_srcdir@/data/customer.1.data' WITH (DELIMITE -- Check whether data went into the right shard (maybe) SELECT min(c_custkey), max(c_custkey), avg(c_custkey), count(*) FROM customer_copy_range WHERE c_custkey <= 500; - min | max | avg | count ------+-----+----------------------+------- + min | max | avg | count +--------------------------------------------------------------------- 1 | 500 | 250.5000000000000000 | 500 (1 row) -- Check whether data was copied SELECT count(*) FROM customer_copy_range; - count -------- + count +--------------------------------------------------------------------- 1000 (1 row) @@ -250,14 +250,14 @@ SELECT count(*) FROM customer_copy_range; UPDATE pg_dist_shard SET shardminvalue = 1501, shardmaxvalue = 2000 WHERE shardid = :new_shard_id; SELECT shardid, shardminvalue, shardmaxvalue FROM pg_dist_shard WHERE shardid = :new_shard_id; - shardid | shardminvalue | shardmaxvalue ----------+---------------+--------------- + shardid | shardminvalue | shardmaxvalue +--------------------------------------------------------------------- 560129 | 1501 | 2000 (1 row) SELECT shardid, shardlength FROM pg_dist_shard_placement WHERE shardid = :new_shard_id; - shardid | shardlength ----------+------------- + shardid | shardlength +--------------------------------------------------------------------- 560129 | 0 560129 | 0 (2 rows) @@ -265,20 +265,20 @@ SELECT shardid, shardlength FROM pg_dist_shard_placement WHERE shardid = :new_sh -- Update shard statistics for range-partitioned shard and check that only the -- shard length is updated. SELECT master_update_shard_statistics(:new_shard_id); - master_update_shard_statistics --------------------------------- + master_update_shard_statistics +--------------------------------------------------------------------- 131072 (1 row) SELECT shardid, shardminvalue, shardmaxvalue FROM pg_dist_shard WHERE shardid = :new_shard_id; - shardid | shardminvalue | shardmaxvalue ----------+---------------+--------------- + shardid | shardminvalue | shardmaxvalue +--------------------------------------------------------------------- 560129 | 1501 | 2000 (1 row) SELECT shardid, shardlength FROM pg_dist_shard_placement WHERE shardid = :new_shard_id; - shardid | shardlength ----------+------------- + shardid | shardlength +--------------------------------------------------------------------- 560129 | 131072 560129 | 131072 (2 rows) @@ -297,9 +297,9 @@ CREATE TABLE customer_copy_append ( c_mktsegment char(10), c_comment varchar(117)); SELECT master_create_distributed_table('customer_copy_append', 'c_custkey', 'append'); - master_create_distributed_table ---------------------------------- - + master_create_distributed_table +--------------------------------------------------------------------- + (1 row) -- Test syntax error @@ -308,8 +308,8 @@ ERROR: invalid input syntax for integer: "notinteger" CONTEXT: COPY customer_copy_append, line 3, column c_custkey: "notinteger" -- Test that no shard is created for failing copy SELECT count(*) FROM pg_dist_shard WHERE logicalrelid = 'customer_copy_append'::regclass; - count -------- + count +--------------------------------------------------------------------- 0 (1 row) @@ -317,8 +317,8 @@ SELECT count(*) FROM pg_dist_shard WHERE logicalrelid = 'customer_copy_append':: COPY customer_copy_append FROM STDIN; -- Test that no shard is created for copying zero rows SELECT count(*) FROM pg_dist_shard WHERE logicalrelid = 'customer_copy_append'::regclass; - count -------- + count +--------------------------------------------------------------------- 0 (1 row) @@ -326,44 +326,44 @@ SELECT count(*) FROM pg_dist_shard WHERE logicalrelid = 'customer_copy_append':: COPY customer_copy_append(c_custkey, c_name) FROM STDIN WITH (FORMAT 'csv'); -- Check whether data was copied properly SELECT * FROM customer_copy_append; - c_custkey | c_name | c_address | c_nationkey | c_phone | c_acctbal | c_mktsegment | c_comment ------------+-----------+-----------+-------------+---------+-----------+--------------+----------- - 1 | customer1 | | | | | | - 2 | customer2 | | | | | | + c_custkey | c_name | c_address | c_nationkey | c_phone | c_acctbal | c_mktsegment | c_comment +--------------------------------------------------------------------- + 1 | customer1 | | | | | | + 2 | customer2 | | | | | | (2 rows) -- Manipulate manipulate and check shard statistics for append-partitioned table shard UPDATE pg_dist_shard SET shardminvalue = 1501, shardmaxvalue = 2000 WHERE shardid = 560131; UPDATE pg_dist_shard_placement SET shardlength = 0 WHERE shardid = 560131; SELECT shardid, shardminvalue, shardmaxvalue FROM pg_dist_shard WHERE shardid = 560131; - shardid | shardminvalue | shardmaxvalue ----------+---------------+--------------- + shardid | shardminvalue | shardmaxvalue +--------------------------------------------------------------------- 560131 | 1501 | 2000 (1 row) SELECT shardid, shardlength FROM pg_dist_shard_placement WHERE shardid = 560131; - shardid | shardlength ----------+------------- + shardid | shardlength +--------------------------------------------------------------------- 560131 | 0 560131 | 0 (2 rows) -- Update shard statistics for append-partitioned shard SELECT master_update_shard_statistics(560131); - master_update_shard_statistics --------------------------------- + master_update_shard_statistics +--------------------------------------------------------------------- 8192 (1 row) SELECT shardid, shardminvalue, shardmaxvalue FROM pg_dist_shard WHERE shardid = 560131; - shardid | shardminvalue | shardmaxvalue ----------+---------------+--------------- + shardid | shardminvalue | shardmaxvalue +--------------------------------------------------------------------- 560131 | 1 | 2 (1 row) SELECT shardid, shardlength FROM pg_dist_shard_placement WHERE shardid = 560131; - shardid | shardlength ----------+------------- + shardid | shardlength +--------------------------------------------------------------------- 560131 | 8192 560131 | 8192 (2 rows) @@ -387,17 +387,17 @@ CREATE TABLE lineitem_copy_append ( l_shipmode char(10) not null, l_comment varchar(44) not null); SELECT master_create_distributed_table('lineitem_copy_append', 'l_orderkey', 'append'); - master_create_distributed_table ---------------------------------- - + master_create_distributed_table +--------------------------------------------------------------------- + (1 row) -- Test multiple shard creation SET citus.shard_max_size TO '256kB'; COPY lineitem_copy_append FROM '@abs_srcdir@/data/lineitem.1.data' with delimiter '|'; SELECT count(*) FROM pg_dist_shard WHERE logicalrelid = 'lineitem_copy_append'::regclass; - count -------- + count +--------------------------------------------------------------------- 5 (1 row) @@ -417,8 +417,8 @@ ORDER BY pg_dist_shard.shardid DESC LIMIT 5; - shardid | nodeport ----------+---------- + shardid | nodeport +--------------------------------------------------------------------- 560141 | 57637 560140 | 57638 560139 | 57637 @@ -438,9 +438,9 @@ CREATE TABLE append.customer_copy ( c_mktsegment char(10), c_comment varchar(117)); SELECT master_create_distributed_table('append.customer_copy', 'c_custkey', 'append'); - master_create_distributed_table ---------------------------------- - + master_create_distributed_table +--------------------------------------------------------------------- + (1 row) -- Test copy from the master node @@ -448,8 +448,8 @@ COPY append.customer_copy FROM '@abs_srcdir@/data/customer.1.data' with (delimit COPY append.customer_copy FROM '@abs_srcdir@/data/customer.2.data' with (delimiter '|'); -- Test the content of the table SELECT min(c_custkey), max(c_custkey), avg(c_acctbal), count(*) FROM append.customer_copy; - min | max | avg | count ------+------+-----------------------+------- + min | max | avg | count +--------------------------------------------------------------------- 1 | 7000 | 4443.8028800000000000 | 2000 (1 row) @@ -458,23 +458,23 @@ CREATE TABLE "customer_with_special_\\_character"( c_custkey integer, c_name varchar(25) not null); SELECT master_create_distributed_table('"customer_with_special_\\_character"', 'c_custkey', 'hash'); - master_create_distributed_table ---------------------------------- - + master_create_distributed_table +--------------------------------------------------------------------- + (1 row) SELECT master_create_worker_shards('"customer_with_special_\\_character"', 4, 1); - master_create_worker_shards ------------------------------ - + master_create_worker_shards +--------------------------------------------------------------------- + (1 row) COPY "customer_with_special_\\_character" (c_custkey, c_name) FROM STDIN WITH (FORMAT 'csv'); -- Confirm that data was copied SELECT count(*) FROM "customer_with_special_\\_character"; - count -------- + count +--------------------------------------------------------------------- 2 (1 row) @@ -483,23 +483,23 @@ CREATE TABLE "1_customer"( c_custkey integer, c_name varchar(25) not null); SELECT master_create_distributed_table('"1_customer"', 'c_custkey', 'hash'); - master_create_distributed_table ---------------------------------- - + master_create_distributed_table +--------------------------------------------------------------------- + (1 row) SELECT master_create_worker_shards('"1_customer"', 4, 1); - master_create_worker_shards ------------------------------ - + master_create_worker_shards +--------------------------------------------------------------------- + (1 row) COPY "1_customer" (c_custkey, c_name) FROM STDIN WITH (FORMAT 'csv'); -- Confirm that data was copied SELECT count(*) FROM "1_customer"; - count -------- + count +--------------------------------------------------------------------- 2 (1 row) @@ -518,23 +518,23 @@ CREATE TABLE packed_numbers_hash ( packed_numbers number_pack[] ); SELECT master_create_distributed_table('packed_numbers_hash', 'id', 'hash'); - master_create_distributed_table ---------------------------------- - + master_create_distributed_table +--------------------------------------------------------------------- + (1 row) SELECT master_create_worker_shards('packed_numbers_hash', 4, 1); - master_create_worker_shards ------------------------------ - + master_create_worker_shards +--------------------------------------------------------------------- + (1 row) COPY (SELECT 1, ARRAY[ROW(42, 42), ROW(42, 42)]) TO :'temp_dir''copy_test_array_of_composite'; COPY packed_numbers_hash FROM :'temp_dir''copy_test_array_of_composite'; -- Verify data is actually copied SELECT * FROM packed_numbers_hash; - id | packed_numbers -----+----------------------- + id | packed_numbers +--------------------------------------------------------------------- 1 | {"(42,42)","(42,42)"} (1 row) @@ -544,23 +544,23 @@ CREATE TABLE super_packed_numbers_hash ( super_packed_number super_number_pack ); SELECT master_create_distributed_table('super_packed_numbers_hash', 'id', 'hash'); - master_create_distributed_table ---------------------------------- - + master_create_distributed_table +--------------------------------------------------------------------- + (1 row) SELECT master_create_worker_shards('super_packed_numbers_hash', 4, 1); - master_create_worker_shards ------------------------------ - + master_create_worker_shards +--------------------------------------------------------------------- + (1 row) COPY (SELECT 1, ROW(ROW(42, 42), ROW(42, 42))) TO :'temp_dir''copy_test_composite_of_composite'; COPY super_packed_numbers_hash FROM :'temp_dir''copy_test_composite_of_composite'; -- Verify data is actually copied SELECT * FROM super_packed_numbers_hash; - id | super_packed_number -----+----------------------- + id | super_packed_number +--------------------------------------------------------------------- 1 | ("(42,42)","(42,42)") (1 row) @@ -570,16 +570,16 @@ CREATE TABLE packed_numbers_append ( packed_numbers number_pack[] ); SELECT master_create_distributed_table('packed_numbers_append', 'id', 'append'); - master_create_distributed_table ---------------------------------- - + master_create_distributed_table +--------------------------------------------------------------------- + (1 row) COPY packed_numbers_append FROM :'temp_dir''copy_test_array_of_composite'; -- Verify data is actually copied SELECT * FROM packed_numbers_append; - id | packed_numbers -----+----------------------- + id | packed_numbers +--------------------------------------------------------------------- 1 | {"(42,42)","(42,42)"} (1 row) @@ -589,16 +589,16 @@ CREATE TABLE super_packed_numbers_append ( super_packed_number super_number_pack ); SELECT master_create_distributed_table('super_packed_numbers_append', 'id', 'append'); - master_create_distributed_table ---------------------------------- - + master_create_distributed_table +--------------------------------------------------------------------- + (1 row) COPY super_packed_numbers_append FROM :'temp_dir''copy_test_composite_of_composite'; -- Verify data is actually copied SELECT * FROM super_packed_numbers_append; - id | super_packed_number -----+----------------------- + id | super_packed_number +--------------------------------------------------------------------- 1 | ("(42,42)","(42,42)") (1 row) @@ -608,32 +608,32 @@ CREATE TABLE composite_partition_column_table( composite_column number_pack ); SELECT master_create_distributed_table('composite_partition_column_table', 'composite_column', 'append'); - master_create_distributed_table ---------------------------------- - + master_create_distributed_table +--------------------------------------------------------------------- + (1 row) \COPY composite_partition_column_table FROM STDIN WITH (FORMAT 'csv'); WARNING: function min(number_pack) does not exist HINT: No function matches the given name and argument types. You might need to add explicit type casts. -CONTEXT: while executing command on localhost:57638 +CONTEXT: while executing command on localhost:xxxxx WARNING: could not get statistics for shard public.composite_partition_column_table_560162 DETAIL: Setting shard statistics to NULL -ERROR: failure on connection marked as essential: localhost:57638 +ERROR: failure on connection marked as essential: localhost:xxxxx -- Test copy on append distributed tables do not create shards on removed workers CREATE TABLE numbers_append (a int, b int); SELECT master_create_distributed_table('numbers_append', 'a', 'append'); - master_create_distributed_table ---------------------------------- - + master_create_distributed_table +--------------------------------------------------------------------- + (1 row) -- no shards is created yet SELECT shardid, nodename, nodeport FROM pg_dist_shard_placement join pg_dist_shard using(shardid) WHERE logicalrelid = 'numbers_append'::regclass order by placementid; - shardid | nodename | nodeport ----------+----------+---------- + shardid | nodename | nodeport +--------------------------------------------------------------------- (0 rows) COPY numbers_append FROM STDIN WITH (FORMAT 'csv'); @@ -642,18 +642,18 @@ COPY numbers_append FROM STDIN WITH (FORMAT 'csv'); SELECT shardid, nodename, nodeport FROM pg_dist_shard_placement join pg_dist_shard using(shardid) WHERE logicalrelid = 'numbers_append'::regclass order by placementid; - shardid | nodename | nodeport ----------+-----------+---------- + shardid | nodename | nodeport +--------------------------------------------------------------------- 560163 | localhost | 57637 560164 | localhost | 57638 (2 rows) -- disable the first node SELECT master_disable_node('localhost', :worker_1_port); -NOTICE: Node localhost:57637 has active shard placements. Some queries may fail after this operation. Use SELECT master_activate_node('localhost', 57637) to activate this node back. - master_disable_node ---------------------- - +NOTICE: Node localhost:xxxxx has active shard placements. Some queries may fail after this operation. Use SELECT master_activate_node('localhost', 57637) to activate this node back. + master_disable_node +--------------------------------------------------------------------- + (1 row) -- set replication factor to 1 so that copy will @@ -665,8 +665,8 @@ COPY numbers_append FROM STDIN WITH (FORMAT 'csv'); SELECT shardid, nodename, nodeport FROM pg_dist_shard_placement join pg_dist_shard using(shardid) WHERE logicalrelid = 'numbers_append'::regclass order by placementid; - shardid | nodename | nodeport ----------+-----------+---------- + shardid | nodename | nodeport +--------------------------------------------------------------------- 560163 | localhost | 57637 560164 | localhost | 57638 560165 | localhost | 57638 @@ -676,8 +676,8 @@ SELECT shardid, nodename, nodeport -- add the node back SET client_min_messages TO ERROR; SELECT 1 FROM master_activate_node('localhost', :worker_1_port); - ?column? ----------- + ?column? +--------------------------------------------------------------------- 1 (1 row) @@ -689,8 +689,8 @@ COPY numbers_append FROM STDIN WITH (FORMAT 'csv'); SELECT shardid, nodename, nodeport FROM pg_dist_shard_placement join pg_dist_shard using(shardid) WHERE logicalrelid = 'numbers_append'::regclass order by placementid; - shardid | nodename | nodeport ----------+-----------+---------- + shardid | nodename | nodeport +--------------------------------------------------------------------- 560163 | localhost | 57637 560164 | localhost | 57638 560165 | localhost | 57638 @@ -708,8 +708,8 @@ CREATE USER test_user; NOTICE: not propagating CREATE ROLE/USER commands to worker nodes HINT: Connect to worker nodes directly to manually create all necessary users and roles. SELECT * FROM run_command_on_workers('CREATE USER test_user'); - nodename | nodeport | success | result ------------+----------+---------+------------- + nodename | nodeport | success | result +--------------------------------------------------------------------- localhost | 57637 | t | CREATE ROLE localhost | 57638 | t | CREATE ROLE (2 rows) @@ -718,9 +718,9 @@ SELECT * FROM run_command_on_workers('CREATE USER test_user'); SET citus.shard_count to 4; CREATE TABLE numbers_hash (a int, b int); SELECT create_distributed_table('numbers_hash', 'a'); - create_distributed_table --------------------------- - + create_distributed_table +--------------------------------------------------------------------- + (1 row) COPY numbers_hash FROM STDIN WITH (FORMAT 'csv'); @@ -728,8 +728,8 @@ COPY numbers_hash FROM STDIN WITH (FORMAT 'csv'); SELECT shardid, shardstate, nodename, nodeport FROM pg_dist_shard_placement join pg_dist_shard using(shardid) WHERE logicalrelid = 'numbers_hash'::regclass order by shardid, nodeport; - shardid | shardstate | nodename | nodeport ----------+------------+-----------+---------- + shardid | shardstate | nodename | nodeport +--------------------------------------------------------------------- 560169 | 1 | localhost | 57637 560169 | 1 | localhost | 57638 560170 | 1 | localhost | 57637 @@ -743,25 +743,25 @@ SELECT shardid, shardstate, nodename, nodeport -- create a reference table CREATE TABLE numbers_reference(a int, b int); SELECT create_reference_table('numbers_reference'); - create_reference_table ------------------------- - + create_reference_table +--------------------------------------------------------------------- + (1 row) COPY numbers_reference FROM STDIN WITH (FORMAT 'csv'); -- create another hash distributed table CREATE TABLE numbers_hash_other(a int, b int); SELECT create_distributed_table('numbers_hash_other', 'a'); - create_distributed_table --------------------------- - + create_distributed_table +--------------------------------------------------------------------- + (1 row) SELECT shardid, shardstate, nodename, nodeport FROM pg_dist_shard_placement join pg_dist_shard using(shardid) WHERE logicalrelid = 'numbers_hash_other'::regclass order by shardid, nodeport; - shardid | shardstate | nodename | nodeport ----------+------------+-----------+---------- + shardid | shardstate | nodename | nodeport +--------------------------------------------------------------------- 560174 | 1 | localhost | 57637 560174 | 1 | localhost | 57638 560175 | 1 | localhost | 57637 @@ -783,24 +783,20 @@ ALTER USER test_user WITH nologin; \c - test_user - :master_port -- reissue copy COPY numbers_hash FROM STDIN WITH (FORMAT 'csv'); -WARNING: connection error: localhost:57637 -DETAIL: FATAL: role "test_user" is not permitted to log in +WARNING: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" is not permitted to log in CONTEXT: COPY numbers_hash, line 1: "1,1" -WARNING: connection error: localhost:57637 -DETAIL: FATAL: role "test_user" is not permitted to log in +WARNING: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" is not permitted to log in CONTEXT: COPY numbers_hash, line 2: "2,2" -WARNING: connection error: localhost:57637 -DETAIL: FATAL: role "test_user" is not permitted to log in +WARNING: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" is not permitted to log in CONTEXT: COPY numbers_hash, line 3: "3,3" -WARNING: connection error: localhost:57637 -DETAIL: FATAL: role "test_user" is not permitted to log in +WARNING: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" is not permitted to log in CONTEXT: COPY numbers_hash, line 6: "6,6" -- verify shards in the first worker as marked invalid SELECT shardid, shardstate, nodename, nodeport FROM pg_dist_shard_placement join pg_dist_shard using(shardid) WHERE logicalrelid = 'numbers_hash'::regclass order by shardid, nodeport; - shardid | shardstate | nodename | nodeport ----------+------------+-----------+---------- + shardid | shardstate | nodename | nodeport +--------------------------------------------------------------------- 560169 | 3 | localhost | 57637 560169 | 1 | localhost | 57638 560170 | 3 | localhost | 57637 @@ -813,15 +809,14 @@ SELECT shardid, shardstate, nodename, nodeport -- try to insert into a reference table copy should fail COPY numbers_reference FROM STDIN WITH (FORMAT 'csv'); -ERROR: connection error: localhost:57637 -DETAIL: FATAL: role "test_user" is not permitted to log in +ERROR: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" is not permitted to log in CONTEXT: COPY numbers_reference, line 1: "3,1" -- verify shards for reference table are still valid SELECT shardid, shardstate, nodename, nodeport FROM pg_dist_shard_placement join pg_dist_shard using(shardid) WHERE logicalrelid = 'numbers_reference'::regclass order by placementid; - shardid | shardstate | nodename | nodeport ----------+------------+-----------+---------- + shardid | shardstate | nodename | nodeport +--------------------------------------------------------------------- 560173 | 1 | localhost | 57637 560173 | 1 | localhost | 57638 (2 rows) @@ -830,22 +825,19 @@ SELECT shardid, shardstate, nodename, nodeport -- since it can not insert into either copies of a shard. shards are expected to -- stay valid since the operation is rolled back. COPY numbers_hash_other FROM STDIN WITH (FORMAT 'csv'); -WARNING: connection error: localhost:57637 -DETAIL: FATAL: role "test_user" is not permitted to log in +WARNING: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" is not permitted to log in CONTEXT: COPY numbers_hash_other, line 1: "1,1" -WARNING: connection error: localhost:57637 -DETAIL: FATAL: role "test_user" is not permitted to log in +WARNING: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" is not permitted to log in CONTEXT: COPY numbers_hash_other, line 2: "2,2" -WARNING: connection error: localhost:57637 -DETAIL: FATAL: role "test_user" is not permitted to log in +WARNING: connection to the remote node localhost:xxxxx failed with the following error: FATAL: role "test_user" is not permitted to log in CONTEXT: COPY numbers_hash_other, line 3: "3,3" -- verify shards for numbers_hash_other are still valid -- since copy has failed altogether SELECT shardid, shardstate, nodename, nodeport FROM pg_dist_shard_placement join pg_dist_shard using(shardid) WHERE logicalrelid = 'numbers_hash_other'::regclass order by shardid, nodeport; - shardid | shardstate | nodename | nodeport ----------+------------+-----------+---------- + shardid | shardstate | nodename | nodeport +--------------------------------------------------------------------- 560174 | 3 | localhost | 57637 560174 | 1 | localhost | 57638 560175 | 3 | localhost | 57637 @@ -872,9 +864,9 @@ DROP TABLE numbers_reference; SET citus.shard_count to 4; CREATE TABLE numbers_hash(a int, b int); SELECT create_distributed_table('numbers_hash', 'a'); - create_distributed_table --------------------------- - + create_distributed_table +--------------------------------------------------------------------- + (1 row) \c - - - :worker_1_port @@ -883,12 +875,12 @@ ALTER TABLE numbers_hash_560180 DROP COLUMN b; -- operation will fail to modify a shard and roll back COPY numbers_hash FROM STDIN WITH (FORMAT 'csv'); ERROR: column "b" of relation "numbers_hash_560180" does not exist -CONTEXT: while executing command on localhost:57637 +CONTEXT: while executing command on localhost:xxxxx COPY numbers_hash, line 6: "6,6" -- verify no row is inserted SELECT count(a) FROM numbers_hash; - count -------- + count +--------------------------------------------------------------------- 0 (1 row) @@ -896,8 +888,8 @@ SELECT count(a) FROM numbers_hash; SELECT shardid, shardstate, nodename, nodeport FROM pg_dist_shard_placement join pg_dist_shard using(shardid) WHERE logicalrelid = 'numbers_hash'::regclass order by shardid, nodeport; - shardid | shardstate | nodename | nodeport ----------+------------+-----------+---------- + shardid | shardstate | nodename | nodeport +--------------------------------------------------------------------- 560178 | 1 | localhost | 57637 560178 | 1 | localhost | 57638 560179 | 1 | localhost | 57637 @@ -910,8 +902,8 @@ SELECT shardid, shardstate, nodename, nodeport DROP TABLE numbers_hash; SELECT * FROM run_command_on_workers('DROP USER test_user'); - nodename | nodeport | success | result ------------+----------+---------+----------- + nodename | nodeport | success | result +--------------------------------------------------------------------- localhost | 57637 | t | DROP ROLE localhost | 57638 | t | DROP ROLE (2 rows) @@ -923,15 +915,15 @@ col1 aclitem NOT NULL, col2 character varying(255) NOT NULL ); SELECT create_reference_table('test_binaryless_builtin'); - create_reference_table ------------------------- - + create_reference_table +--------------------------------------------------------------------- + (1 row) \COPY test_binaryless_builtin FROM STDIN WITH (format CSV) SELECT * FROM test_binaryless_builtin; - col1 | col2 ----------------------+------- + col1 | col2 +--------------------------------------------------------------------- postgres=r/postgres | test (1 row) @@ -940,9 +932,9 @@ DROP TABLE test_binaryless_builtin; BEGIN; CREATE TABLE tt1(id int); SELECT create_distributed_table('tt1','id'); - create_distributed_table --------------------------- - + create_distributed_table +--------------------------------------------------------------------- + (1 row) \copy tt1 from STDIN; @@ -951,24 +943,24 @@ END; -- Test dropping a column in front of the partition column CREATE TABLE drop_copy_test_table (col1 int, col2 int, col3 int, col4 int); SELECT create_distributed_table('drop_copy_test_table','col3'); - create_distributed_table --------------------------- - + create_distributed_table +--------------------------------------------------------------------- + (1 row) ALTER TABLE drop_copy_test_table drop column col1; COPY drop_copy_test_table (col2,col3,col4) from STDIN with CSV; SELECT * FROM drop_copy_test_table WHERE col3 = 1; - col2 | col3 | col4 -------+------+------ - | 1 | + col2 | col3 | col4 +--------------------------------------------------------------------- + | 1 | (1 row) ALTER TABLE drop_copy_test_table drop column col4; COPY drop_copy_test_table (col2,col3) from STDIN with CSV; SELECT * FROM drop_copy_test_table WHERE col3 = 1; - col2 | col3 -------+------ + col2 | col3 +--------------------------------------------------------------------- | 1 | 1 (2 rows) @@ -977,8 +969,8 @@ DROP TABLE drop_copy_test_table; -- There should be no "tt1" shard on the worker nodes \c - - - :worker_1_port SELECT relname FROM pg_class WHERE relname LIKE 'tt1%'; - relname ---------- + relname +--------------------------------------------------------------------- (0 rows) \c - - - :master_port @@ -991,9 +983,9 @@ NOTICE: Copying data from local table... NOTICE: copying the data has completed DETAIL: The local data in the table is longer visible, but is still on disk. HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.trigger_flush$$) - create_distributed_table --------------------------- - + create_distributed_table +--------------------------------------------------------------------- + (1 row) ABORT; @@ -1003,9 +995,9 @@ SET citus.shard_count TO 3; SET citus.multi_shard_modify_mode TO 'sequential'; CREATE UNLOGGED TABLE trigger_switchover(a int, b int, c int, d int, e int, f int, g int, h int); SELECT create_distributed_table('trigger_switchover','a'); - create_distributed_table --------------------------- - + create_distributed_table +--------------------------------------------------------------------- + (1 row) INSERT INTO trigger_switchover @@ -1014,16 +1006,16 @@ ABORT; -- copy into a table with a JSONB column CREATE TABLE copy_jsonb (key text, value jsonb, extra jsonb default '["default"]'::jsonb); SELECT create_distributed_table('copy_jsonb', 'key', colocate_with => 'none'); - create_distributed_table --------------------------- - + create_distributed_table +--------------------------------------------------------------------- + (1 row) -- JSONB from text should work \COPY copy_jsonb (key, value) FROM STDIN SELECT * FROM copy_jsonb ORDER BY key; - key | value | extra --------+----------------------------+------------- + key | value | extra +--------------------------------------------------------------------- blue | {"b": 255, "g": 0, "r": 0} | ["default"] green | {"b": 0, "g": 255, "r": 0} | ["default"] (2 rows) @@ -1032,8 +1024,8 @@ SELECT * FROM copy_jsonb ORDER BY key; COPY copy_jsonb TO :'temp_dir''copy_jsonb.pgcopy' WITH (format binary); COPY copy_jsonb FROM :'temp_dir''copy_jsonb.pgcopy' WITH (format binary); SELECT * FROM copy_jsonb ORDER BY key; - key | value | extra --------+----------------------------+------------- + key | value | extra +--------------------------------------------------------------------- blue | {"b": 255, "g": 0, "r": 0} | ["default"] blue | {"b": 255, "g": 0, "r": 0} | ["default"] green | {"b": 0, "g": 255, "r": 0} | ["default"] @@ -1042,7 +1034,7 @@ SELECT * FROM copy_jsonb ORDER BY key; -- JSONB parsing error without validation: no line number \COPY copy_jsonb (key, value) FROM STDIN -ERROR: invalid input syntax for type json +ERROR: invalid input syntax for json DETAIL: The input string ended unexpectedly. CONTEXT: JSON data, line 1: {"r":255,"g":0,"b":0 COPY copy_jsonb, line 1, column value: "{"r":255,"g":0,"b":0" @@ -1051,8 +1043,8 @@ SET citus.skip_jsonb_validation_in_copy TO off; -- JSONB from text should work \COPY copy_jsonb (key, value) FROM STDIN SELECT * FROM copy_jsonb ORDER BY key; - key | value | extra --------+----------------------------+------------- + key | value | extra +--------------------------------------------------------------------- blue | {"b": 255, "g": 0, "r": 0} | ["default"] green | {"b": 0, "g": 255, "r": 0} | ["default"] (2 rows) @@ -1061,8 +1053,8 @@ SELECT * FROM copy_jsonb ORDER BY key; COPY copy_jsonb TO :'temp_dir''copy_jsonb.pgcopy' WITH (format binary); COPY copy_jsonb FROM :'temp_dir''copy_jsonb.pgcopy' WITH (format binary); SELECT * FROM copy_jsonb ORDER BY key; - key | value | extra --------+----------------------------+------------- + key | value | extra +--------------------------------------------------------------------- blue | {"b": 255, "g": 0, "r": 0} | ["default"] blue | {"b": 255, "g": 0, "r": 0} | ["default"] green | {"b": 0, "g": 255, "r": 0} | ["default"] @@ -1071,7 +1063,7 @@ SELECT * FROM copy_jsonb ORDER BY key; -- JSONB parsing error with validation: should see line number \COPY copy_jsonb (key, value) FROM STDIN -ERROR: invalid input syntax for type json +ERROR: invalid input syntax for json DETAIL: The input string ended unexpectedly. CONTEXT: JSON data, line 1: {"r":255,"g":0,"b":0 COPY copy_jsonb, line 1, column value: "{"r":255,"g":0,"b":0" diff --git a/src/test/regress/sql/multi_mx_node_metadata.sql b/src/test/regress/sql/multi_mx_node_metadata.sql index 4dee21780..44ada118e 100644 --- a/src/test/regress/sql/multi_mx_node_metadata.sql +++ b/src/test/regress/sql/multi_mx_node_metadata.sql @@ -13,6 +13,8 @@ SET citus.shard_count TO 8; SET citus.shard_replication_factor TO 1; SET citus.replicate_reference_tables_on_activate TO off; +\set VERBOSITY terse + -- Simulates a readonly node by setting default_transaction_read_only. CREATE FUNCTION mark_node_readonly(hostname TEXT, port INTEGER, isreadonly BOOLEAN) RETURNS TEXT diff --git a/src/test/regress/sql/multi_replicate_reference_table.sql b/src/test/regress/sql/multi_replicate_reference_table.sql index ed71e5b76..8acd08f32 100644 --- a/src/test/regress/sql/multi_replicate_reference_table.sql +++ b/src/test/regress/sql/multi_replicate_reference_table.sql @@ -669,10 +669,23 @@ END; -- test adding an invalid node while we have reference tables to replicate -- set client message level to ERROR and verbosity to terse to supporess -- OS-dependent host name resolution warnings -SET client_min_messages to ERROR; \set VERBOSITY terse - -SELECT master_add_node('invalid-node-name', 9999); +SET client_min_messages to ERROR; +DO $$ +DECLARE + errors_received INTEGER; +BEGIN +errors_received := 0; + BEGIN + SELECT master_add_node('invalid-node-name', 9999); + EXCEPTION WHEN OTHERS THEN + IF SQLERRM LIKE 'connection to the remote node%%' THEN + errors_received := errors_received + 1; + END IF; + END; +RAISE '(%/1) failed to add node', errors_received; +END; +$$; -- drop unnecassary tables DROP TABLE initially_not_replicated_reference_table;