Improve connection error message from the worker nodes

We currently put the actual error message to the detail part. However,
many drivers don't show detail part.

As connection errors are somehow common, and hard to trace back, can't
we added the detail to the message itself.

In addition to that, we changed "connection error" message, as it
was confusing to the users who think that the error was happening
while connecting to the coordinator. In fact, this error is showing
up when the coordinator fails to connect remote nodes.

src/backend/distributed/connection/remote_commands.c

@@ -277,10 +277,23 @@ ReportConnectionError(MultiConnection *connection, int elevel)
 		messageDetail = pchomp(PQerrorMessage(pgConn));
 	}
 
+	if (messageDetail)
+	{
+		/*
+		 * We don't use ApplyLogRedaction(messageDetail) as we expect any error
+		 * detail that requires log reduction should have done it locally.
+		 */
 		ereport(elevel, (errcode(ERRCODE_CONNECTION_FAILURE),
-					 errmsg("connection error: %s:%d", nodeName, nodePort),
+						 errmsg("connection to the remote node %s:%d failed with the "
-					 messageDetail != NULL ?
+								"following error: %s", nodeName, nodePort,
-					 errdetail("%s", ApplyLogRedaction(messageDetail)) : 0));
+								messageDetail)));
+	}
+	else
+	{
+		ereport(elevel, (errcode(ERRCODE_CONNECTION_FAILURE),
+						 errmsg("connection to the remote node %s:%d failed",
+								nodeName, nodePort)));
+	}
 }
 
 

src/test/regress/expected/failure_1pc_copy_append.out

@@ -168,8 +168,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="SELECT|COPY").kill()');
 (1 row)
 
 SELECT count(1) FROM copy_test;
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
  count

src/test/regress/expected/failure_1pc_copy_hash.out

@@ -219,8 +219,7 @@ SELECT citus.mitmproxy('conn.killall()');
 (1 row)
 
 COPY copy_test FROM PROGRAM 'echo 0, 0 && echo 1, 1 && echo 2, 4 && echo 3, 9' WITH CSV;
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 CONTEXT:  COPY copy_test, line 1: "0, 0"

src/test/regress/expected/failure_connection_establishment.out

@@ -227,7 +227,7 @@ RESET client_min_messages;
 -- verify get_global_active_transactions works when a timeout happens on a connection
 SELECT get_global_active_transactions();
 WARNING:  could not establish connection after 400 ms
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed
  get_global_active_transactions
 ---------------------------------------------------------------------
 (0 rows)

src/test/regress/expected/failure_copy_on_hash.out

@@ -36,8 +36,7 @@ SELECT citus.mitmproxy('conn.kill()');
 (1 row)
 
 \COPY test_table FROM stdin delimiter ',';
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 CONTEXT:  COPY test_table, line 1: "1,2"
@@ -276,23 +275,19 @@ SELECT citus.mitmproxy('conn.kill()');
 (1 row)
 
 \COPY test_table_2 FROM stdin delimiter ',';
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 CONTEXT:  COPY test_table_2, line 1: "1,2"
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 CONTEXT:  COPY test_table_2, line 2: "3,4"
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 CONTEXT:  COPY test_table_2, line 3: "6,7"
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 CONTEXT:  COPY test_table_2, line 5: "9,10"

src/test/regress/expected/failure_create_distributed_table_non_empty.out

@@ -26,8 +26,7 @@ SELECT citus.mitmproxy('conn.kill()');
 (1 row)
 
 SELECT create_distributed_table('test_table', 'id');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT count(*) FROM pg_dist_shard WHERE logicalrelid='create_distributed_table_non_empty_failure.test_table'::regclass;
@@ -125,8 +124,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 SELECT create_distributed_table('test_table', 'id');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
@@ -190,8 +188,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").kill()');
 (1 row)
 
 SELECT create_distributed_table('test_table', 'id');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT count(*) FROM pg_dist_shard WHERE logicalrelid='create_distributed_table_non_empty_failure.test_table'::regclass;
@@ -473,8 +470,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 SELECT create_distributed_table('test_table', 'id', colocate_with => 'colocated_table');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT count(*) FROM pg_dist_shard WHERE logicalrelid='create_distributed_table_non_empty_failure.test_table'::regclass;
@@ -561,8 +557,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^SELECT worker_apply_shard_ddl_comma
 (1 row)
 
 SELECT create_distributed_table('test_table', 'id', colocate_with => 'colocated_table');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT count(*) FROM pg_dist_shard WHERE logicalrelid='create_distributed_table_non_empty_failure.test_table'::regclass;
@@ -599,8 +594,7 @@ SELECT citus.mitmproxy('conn.kill()');
 (1 row)
 
 SELECT create_distributed_table('test_table', 'id');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
@@ -667,8 +661,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 SELECT create_distributed_table('test_table', 'id');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
@@ -732,8 +725,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").kill()');
 (1 row)
 
 SELECT create_distributed_table('test_table', 'id');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT count(*) FROM pg_dist_shard WHERE logicalrelid='create_distributed_table_non_empty_failure.test_table'::regclass;
@@ -946,8 +938,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 SELECT create_distributed_table('test_table', 'id', colocate_with => 'colocated_table');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT count(*) FROM pg_dist_shard WHERE logicalrelid='create_distributed_table_non_empty_failure.test_table'::regclass;

src/test/regress/expected/failure_create_reference_table.out

@@ -25,8 +25,7 @@ SELECT citus.mitmproxy('conn.onQuery().kill()');
 (1 row)
 
 SELECT create_reference_table('ref_table');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT count(*) FROM pg_dist_shard_placement;
@@ -43,8 +42,7 @@ SELECT citus.mitmproxy('conn.onCommandComplete(command="BEGIN").kill()');
 (1 row)
 
 SELECT create_reference_table('ref_table');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT count(*) FROM pg_dist_shard_placement;
@@ -76,8 +74,7 @@ SELECT citus.mitmproxy('conn.onCommandComplete(command="SELECT 1").kill()');
 (1 row)
 
 SELECT create_reference_table('ref_table');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT count(*) FROM pg_dist_shard_placement;

src/test/regress/expected/failure_create_table.out

@@ -20,8 +20,7 @@ SELECT citus.mitmproxy('conn.kill()');
 (1 row)
 
 SELECT create_distributed_table('test_table','id');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
@@ -89,8 +88,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 SELECT create_distributed_table('test_table','id');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
@@ -120,8 +118,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="SELECT worker_apply_shard_ddl_comman
 (1 row)
 
 SELECT create_distributed_table('test_table','id');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
@@ -154,8 +151,7 @@ BEGIN;
 (1 row)
 
     SELECT create_distributed_table('test_table', 'id');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 COMMIT;
@@ -225,8 +221,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").kill()');
 (1 row)
 
 SELECT create_distributed_table('test_table','id',colocate_with=>'temp_table');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
@@ -347,8 +342,7 @@ SELECT citus.mitmproxy('conn.kill()');
 
 BEGIN;
 SELECT create_distributed_table('test_table','id');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 ROLLBACK;
@@ -384,8 +378,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 
 BEGIN;
 SELECT create_distributed_table('test_table','id');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 ROLLBACK;
@@ -463,8 +456,7 @@ SELECT citus.mitmproxy('conn.kill()');
 
 BEGIN;
 SELECT create_distributed_table('test_table','id');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 ROLLBACK;
@@ -496,8 +488,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").kill()');
 
 BEGIN;
 SELECT create_distributed_table('test_table','id');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 ROLLBACK;
@@ -533,8 +524,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 
 BEGIN;
 SELECT create_distributed_table('test_table','id');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 ROLLBACK;
@@ -615,8 +605,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 SELECT master_create_worker_shards('test_table_2', 4, 2);
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT count(*) FROM pg_dist_shard;

src/test/regress/expected/failure_cte_subquery.out

@@ -88,8 +88,7 @@ FROM
      ORDER BY 1 DESC LIMIT 5
      ) as foo
 	  WHERE foo.user_id = cte.user_id;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- kill at the third copy (pull)
@@ -122,8 +121,7 @@ FROM
      ORDER BY 1 DESC LIMIT 5
      ) as foo
 	  WHERE foo.user_id = cte.user_id;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- cancel at the first copy (push)
@@ -262,8 +260,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^DELETE FROM").kill()');
 
 WITH cte_delete as (DELETE FROM users_table WHERE user_name in ('A', 'D') RETURNING *)
 INSERT INTO users_table SELECT * FROM cte_delete;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- verify contents are the same
@@ -378,8 +375,7 @@ BEGIN;
 SET LOCAL citus.multi_shard_modify_mode = 'sequential';
 WITH cte_delete as (DELETE FROM users_table WHERE user_name in ('A', 'D') RETURNING *)
 INSERT INTO users_table SELECT * FROM cte_delete;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 END;

src/test/regress/expected/failure_ddl.out

@@ -37,8 +37,7 @@ SELECT citus.mitmproxy('conn.onAuthenticationOk().kill()');
 (1 row)
 
 ALTER TABLE test_table ADD COLUMN new_column INT;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass;
@@ -71,8 +70,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 ALTER TABLE test_table ADD COLUMN new_column INT;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass;
@@ -104,8 +102,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_apply_shard_ddl_command").kil
 (1 row)
 
 ALTER TABLE test_table ADD COLUMN new_column INT;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- show that we've never commited the changes
@@ -349,8 +346,7 @@ SELECT citus.mitmproxy('conn.onAuthenticationOk().kill()');
 (1 row)
 
 ALTER TABLE test_table DROP COLUMN new_column;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass;
@@ -383,8 +379,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 ALTER TABLE test_table DROP COLUMN new_column;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass;
@@ -416,8 +411,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_apply_shard_ddl_command").kil
 (1 row)
 
 ALTER TABLE test_table DROP COLUMN new_column;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass;
@@ -720,8 +714,7 @@ SELECT citus.mitmproxy('conn.onAuthenticationOk().kill()');
 (1 row)
 
 ALTER TABLE test_table ADD COLUMN new_column INT;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass;
@@ -754,8 +747,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 ALTER TABLE test_table ADD COLUMN new_column INT;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass;
@@ -787,8 +779,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_apply_shard_ddl_command").kil
 (1 row)
 
 ALTER TABLE test_table ADD COLUMN new_column INT;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass;
@@ -1048,8 +1039,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 ALTER TABLE test_table ADD COLUMN new_column INT;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass;
@@ -1081,8 +1071,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_apply_shard_ddl_command").kil
 (1 row)
 
 ALTER TABLE test_table ADD COLUMN new_column INT;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- kill as soon as the coordinator after it sends worker_apply_shard_ddl_command 2nd time
@@ -1093,8 +1082,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_apply_shard_ddl_command").aft
 (1 row)
 
 ALTER TABLE test_table ADD COLUMN new_column INT;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- cancel as soon as the coordinator after it sends worker_apply_shard_ddl_command 2nd time

src/test/regress/expected/failure_distributed_results.out

@@ -85,8 +85,7 @@ CREATE TABLE distributed_result_info AS
   SELECT resultId, nodeport, rowcount, targetShardId, targetShardIndex
   FROM partition_task_list_results('test', $$ SELECT * FROM source_table $$, 'target_table')
           NATURAL JOIN pg_dist_node;
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT * FROM distributed_result_info ORDER BY resultId;

src/test/regress/expected/failure_insert_select_pushdown.out

@@ -44,8 +44,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT INTO insert_select_pushdown"
 (1 row)
 
 INSERT INTO events_summary SELECT user_id, event_id, count(*) FROM events_table GROUP BY 1,2;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 --verify nothing is modified
@@ -98,8 +97,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT INTO insert_select_pushdown"
 (1 row)
 
 INSERT INTO events_table SELECT * FROM events_table;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 --verify nothing is modified

src/test/regress/expected/failure_insert_select_repartition.out

@@ -54,8 +54,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_partition_query_result").kill
 (1 row)
 
 INSERT INTO target_table SELECT * FROM source_table;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT * FROM target_table ORDER BY a;
@@ -70,12 +69,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_partition_query_result").kill
 (1 row)
 
 INSERT INTO target_table SELECT * FROM replicated_source_table;
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT * FROM target_table ORDER BY a;
@@ -106,8 +103,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="fetch_intermediate_results").kill()'
 (1 row)
 
 INSERT INTO target_table SELECT * FROM source_table;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT * FROM target_table ORDER BY a;
@@ -122,8 +118,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="fetch_intermediate_results").kill()'
 (1 row)
 
 INSERT INTO target_table SELECT * FROM replicated_source_table;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT * FROM target_table ORDER BY a;
@@ -143,8 +138,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="read_intermediate_results").kill()')
 (1 row)
 
 INSERT INTO target_table SELECT * FROM source_table;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT * FROM target_table ORDER BY a;
@@ -159,8 +153,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="read_intermediate_results").kill()')
 (1 row)
 
 INSERT INTO target_table SELECT * FROM replicated_source_table;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT * FROM target_table ORDER BY a;
@@ -179,8 +172,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="read_intermediate_results").kill()')
 (1 row)
 
 INSERT INTO replicated_target_table SELECT * FROM source_table;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT * FROM replicated_target_table;

src/test/regress/expected/failure_multi_dml.out

@@ -33,8 +33,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^DELETE").kill()');
 
 BEGIN;
 DELETE FROM dml_test WHERE id = 1;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 DELETE FROM dml_test WHERE id = 2;
@@ -96,8 +95,7 @@ BEGIN;
 DELETE FROM dml_test WHERE id = 1;
 DELETE FROM dml_test WHERE id = 2;
 INSERT INTO dml_test VALUES (5, 'Epsilon');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 UPDATE dml_test SET name = 'alpha' WHERE id = 1;
@@ -154,8 +152,7 @@ DELETE FROM dml_test WHERE id = 1;
 DELETE FROM dml_test WHERE id = 2;
 INSERT INTO dml_test VALUES (5, 'Epsilon');
 UPDATE dml_test SET name = 'alpha' WHERE id = 1;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 UPDATE dml_test SET name = 'gamma' WHERE id = 3;

src/test/regress/expected/failure_multi_row_insert.out

@@ -43,8 +43,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()');
 (1 row)
 
 INSERT INTO distributed_table VALUES (1,1), (1,2), (1,3);
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- this test is broken, see https://github.com/citusdata/citus/issues/2460
@@ -58,8 +57,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()');
 (1 row)
 
 INSERT INTO distributed_table VALUES (1,7), (5,8);
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- this test is broken, see https://github.com/citusdata/citus/issues/2460
@@ -73,8 +71,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()');
 (1 row)
 
 INSERT INTO distributed_table VALUES (1,11), (6,12);
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").cancel(' || :pid || ')');
@@ -93,8 +90,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").after(1).kill()');
 (1 row)
 
 INSERT INTO distributed_table VALUES (1,15), (6,16);
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").after(1).cancel(' || :pid || ')');
@@ -113,8 +109,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()');
 (1 row)
 
 INSERT INTO distributed_table VALUES (2,19),(1,20);
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").cancel(' || :pid || ')');

src/test/regress/expected/failure_multi_shard_update_delete.out

@@ -64,8 +64,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM").kill()');
 
 -- issue a multi shard delete
 DELETE FROM t2 WHERE b = 2;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- verify nothing is deleted
@@ -83,8 +82,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM multi_shard.t2_201005").
 (1 row)
 
 DELETE FROM t2 WHERE b = 2;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- verify nothing is deleted
@@ -146,8 +144,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()');
 
 -- issue a multi shard update
 UPDATE t2 SET c = 4 WHERE b = 2;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- verify nothing is updated
@@ -165,8 +162,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="UPDATE multi_shard.t2_201005").kill(
 (1 row)
 
 UPDATE t2 SET c = 4 WHERE b = 2;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- verify nothing is updated
@@ -222,8 +218,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM").kill()');
 
 -- issue a multi shard delete
 DELETE FROM t2 WHERE b = 2;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- verify nothing is deleted
@@ -241,8 +236,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM multi_shard.t2_201005").
 (1 row)
 
 DELETE FROM t2 WHERE b = 2;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- verify nothing is deleted
@@ -304,8 +298,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()');
 
 -- issue a multi shard update
 UPDATE t2 SET c = 4 WHERE b = 2;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- verify nothing is updated
@@ -323,8 +316,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="UPDATE multi_shard.t2_201005").kill(
 (1 row)
 
 UPDATE t2 SET c = 4 WHERE b = 2;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- verify nothing is updated
@@ -397,8 +389,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM").kill()');
 (1 row)
 
 DELETE FROM r1 WHERE a = 2;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- verify nothing is deleted
@@ -415,8 +406,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM").kill()');
 (1 row)
 
 DELETE FROM t2 WHERE b = 2;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- verify nothing is deleted
@@ -500,8 +490,7 @@ UPDATE t3 SET c = q.c FROM (
 	SELECT b, max(c) as c FROM t2  GROUP BY b) q
 WHERE t3.b = q.b
 RETURNING *;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 --- verify nothing is updated
@@ -559,8 +548,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="UPDATE multi_shard.t3_201013").kill(
 (1 row)
 
 UPDATE t3 SET b = 2 WHERE b = 1;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- verify nothing is updated
@@ -594,8 +582,7 @@ SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FRO
 
 -- following will fail
 UPDATE t3 SET b = 2 WHERE b = 1;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 END;
@@ -613,8 +600,7 @@ SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FRO
 (1 row)
 
 UPDATE t3 SET b = 1 WHERE b = 2 RETURNING *;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- verify nothing is updated
@@ -633,8 +619,7 @@ SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FRO
 (1 row)
 
 UPDATE t3 SET b = 2 WHERE b = 1;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- verify nothing is updated
@@ -668,8 +653,7 @@ SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FRO
 
 -- following will fail
 UPDATE t3 SET b = 2 WHERE b = 1;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 END;

src/test/regress/expected/failure_ref_tables.out

@@ -33,8 +33,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()');
 (1 row)
 
 INSERT INTO ref_table VALUES (5, 6);
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT COUNT(*) FROM ref_table WHERE key=5;
@@ -51,8 +50,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()');
 (1 row)
 
 UPDATE ref_table SET key=7 RETURNING value;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT COUNT(*) FROM ref_table WHERE key=7;
@@ -71,8 +69,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()');
 BEGIN;
 DELETE FROM ref_table WHERE key=5;
 UPDATE ref_table SET key=value;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 COMMIT;

src/test/regress/expected/failure_replicated_partitions.out

@@ -29,8 +29,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()');
 (1 row)
 
 INSERT INTO partitioned_table VALUES (0, 0);
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- use both placements

src/test/regress/expected/failure_savepoints.out

@@ -40,8 +40,7 @@ INSERT INTO artists VALUES (5, 'Asher Lev');
 SAVEPOINT s1;
 WARNING:  connection not open
 CONTEXT:  while executing command on localhost:xxxxx
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: connection not open
-DETAIL:  connection not open
 WARNING:  connection not open
 CONTEXT:  while executing command on localhost:xxxxx
 WARNING:  connection not open
@@ -74,8 +73,7 @@ RELEASE SAVEPOINT s1;
 WARNING:  AbortSubTransaction while in COMMIT state
 WARNING:  connection not open
 CONTEXT:  while executing command on localhost:xxxxx
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: connection not open
-DETAIL:  connection not open
 WARNING:  connection not open
 CONTEXT:  while executing command on localhost:xxxxx
 WARNING:  connection not open
@@ -132,8 +130,7 @@ RELEASE SAVEPOINT s2;
 WARNING:  AbortSubTransaction while in COMMIT state
 WARNING:  connection not open
 CONTEXT:  while executing command on localhost:xxxxx
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: connection not open
-DETAIL:  connection not open
 WARNING:  connection not open
 CONTEXT:  while executing command on localhost:xxxxx
 WARNING:  connection not open
@@ -213,7 +210,7 @@ ROLLBACK TO SAVEPOINT s1;
 WARNING:  connection not open
 WARNING:  connection not open
 WARNING:  connection not open
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: connection not open
 WARNING:  connection not open
 WARNING:  connection not open
 COMMIT;
@@ -248,7 +245,7 @@ BEGIN;
 INSERT INTO researchers VALUES (7, 4, 'Jan Plaza');
 SAVEPOINT s1;
 WARNING:  connection not open
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: connection not open
 WARNING:  connection not open
 WARNING:  connection not open
 ERROR:  connection not open
@@ -321,7 +318,7 @@ ROLLBACK TO s1;
 RELEASE SAVEPOINT s1;
 WARNING:  AbortSubTransaction while in COMMIT state
 WARNING:  connection not open
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: connection not open
 WARNING:  connection not open
 WARNING:  connection not open
 WARNING:  savepoint "savepoint_3" does not exist
@@ -370,10 +367,10 @@ WARNING:  connection not open
 WARNING:  connection not open
 SELECT * FROM ref;
 WARNING:  connection not open
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: connection not open
 WARNING:  connection not open
 WARNING:  connection not open
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: connection not open
 END;
 -- clean up
 SELECT citus.mitmproxy('conn.allow()');

src/test/regress/expected/failure_single_mod.out

@@ -27,8 +27,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()');
 (1 row)
 
 INSERT INTO mod_test VALUES (2, 6);
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT COUNT(*) FROM mod_test WHERE key=2;
@@ -63,8 +62,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()');
 (1 row)
 
 UPDATE mod_test SET value='ok' WHERE key=2 RETURNING key;
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
  key
@@ -102,8 +100,7 @@ INSERT INTO mod_test VALUES (2, 6);
 INSERT INTO mod_test VALUES (2, 7);
 DELETE FROM mod_test WHERE key=2 AND value = '7';
 UPDATE mod_test SET value='ok' WHERE key=2;
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 COMMIT;

src/test/regress/expected/failure_single_select.out

@@ -28,8 +28,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").kill()');
 (1 row)
 
 SELECT * FROM select_test WHERE key = 3;
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
  key |   value
@@ -38,8 +37,7 @@ DETAIL:  server closed the connection unexpectedly
 (1 row)
 
 SELECT * FROM select_test WHERE key = 3;
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
  key |   value
@@ -57,8 +55,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").kill()');
 BEGIN;
 INSERT INTO select_test VALUES (3, 'more data');
 SELECT * FROM select_test WHERE key = 3;
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
  key |   value
@@ -69,8 +66,7 @@ DETAIL:  server closed the connection unexpectedly
 
 INSERT INTO select_test VALUES (3, 'even more data');
 SELECT * FROM select_test WHERE key = 3;
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
  key |     value
@@ -160,8 +156,7 @@ SELECT * FROM select_test WHERE key = 3;
 
 INSERT INTO select_test VALUES (3, 'even more data');
 SELECT * FROM select_test WHERE key = 3;
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
  key |     value
@@ -215,8 +210,7 @@ SELECT * FROM select_test WHERE key = 1;
 (1 row)
 
 SELECT * FROM select_test WHERE key = 1;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- now the same test with query cancellation

src/test/regress/expected/failure_truncate.out

@@ -44,8 +44,7 @@ SELECT citus.mitmproxy('conn.onAuthenticationOk().kill()');
 (1 row)
 
 TRUNCATE test_table;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
@@ -102,8 +101,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 TRUNCATE test_table;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
@@ -159,8 +157,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="TRUNCATE TABLE truncate_failure.test
 (1 row)
 
 TRUNCATE test_table;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
@@ -433,8 +430,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^TRUNCATE TABLE").after(2).kill()');
 (1 row)
 
 TRUNCATE reference_table CASCADE;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
@@ -577,8 +573,7 @@ SELECT citus.mitmproxy('conn.onAuthenticationOk().kill()');
 (1 row)
 
 TRUNCATE test_table;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
@@ -635,8 +630,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 TRUNCATE test_table;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
@@ -692,8 +686,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^TRUNCATE TABLE truncate_failure.tes
 (1 row)
 
 TRUNCATE test_table;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
@@ -956,8 +949,7 @@ SELECT citus.mitmproxy('conn.onAuthenticationOk().kill()');
 (1 row)
 
 TRUNCATE test_table;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
@@ -1014,8 +1006,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 TRUNCATE test_table;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
@@ -1071,8 +1062,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="TRUNCATE TABLE truncate_failure.test
 (1 row)
 
 TRUNCATE test_table;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');

src/test/regress/expected/failure_vacuum.out

@@ -31,8 +31,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^VACUUM").kill()');
 (1 row)
 
 VACUUM vacuum_test;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.onQuery(query="^ANALYZE").kill()');
@@ -42,8 +41,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^ANALYZE").kill()');
 (1 row)
 
 ANALYZE vacuum_test;
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT").kill()');
@@ -111,8 +109,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^VACUUM.*other").kill()');
 (1 row)
 
 VACUUM vacuum_test, other_vacuum_test;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.onQuery(query="^VACUUM.*other").cancel(' ||  pg_backend_pid() || ')');

src/test/regress/expected/multi_modifying_xacts.out

@@ -1178,15 +1178,15 @@ NOTICE:  not propagating ALTER ROLE ... RENAME TO commands to worker nodes
  \c - test_user - :master_port
 -- should fail since the worker doesn't have test_user anymore
 INSERT INTO reference_failure_test VALUES (1, '1');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "test_user" does not exist
 -- the same as the above, but wrapped within a transaction
 BEGIN;
 INSERT INTO reference_failure_test VALUES (1, '1');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "test_user" does not exist
 COMMIT;
 BEGIN;
 COPY reference_failure_test FROM STDIN WITH (FORMAT 'csv');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "test_user" does not exist
 COMMIT;
 -- show that no data go through the table and shard states are good
 SET client_min_messages to 'ERROR';
@@ -1211,8 +1211,8 @@ ORDER BY s.logicalrelid, sp.shardstate;
 
 BEGIN;
 COPY numbers_hash_failure_test FROM STDIN WITH (FORMAT 'csv');
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "test_user" does not exist
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "test_user" does not exist
 -- some placements are invalid before abort
 SELECT shardid, shardstate, nodename, nodeport
 FROM pg_dist_shard_placement JOIN pg_dist_shard USING (shardid)
@@ -1233,8 +1233,8 @@ ORDER BY shardid, nodeport;
 ABORT;
 -- verify nothing is inserted
 SELECT count(*) FROM numbers_hash_failure_test;
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "test_user" does not exist
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "test_user" does not exist
  count
 ---------------------------------------------------------------------
      0
@@ -1259,8 +1259,8 @@ ORDER BY shardid, nodeport;
 
 BEGIN;
 COPY numbers_hash_failure_test FROM STDIN WITH (FORMAT 'csv');
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "test_user" does not exist
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "test_user" does not exist
 -- check shard states before commit
 SELECT shardid, shardstate, nodename, nodeport
 FROM pg_dist_shard_placement JOIN pg_dist_shard USING (shardid)
@@ -1298,8 +1298,8 @@ ORDER BY shardid, nodeport;
 
 -- verify data is inserted
 SELECT count(*) FROM numbers_hash_failure_test;
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "test_user" does not exist
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "test_user" does not exist
  count
 ---------------------------------------------------------------------
      2
@@ -1312,7 +1312,7 @@ NOTICE:  not propagating ALTER ROLE ... RENAME TO commands to worker nodes
 \c - test_user - :master_port
 -- fails on all shard placements
 INSERT INTO numbers_hash_failure_test VALUES (2,2);
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "test_user" does not exist
 -- connect back to the master with the proper user to continue the tests
 \c - :default_user - :master_port
 SET citus.next_shard_id TO 1200020;

src/test/regress/expected/multi_mx_node_metadata.out

@@ -9,6 +9,7 @@ SET citus.replication_model TO streaming;
 SET citus.shard_count TO 8;
 SET citus.shard_replication_factor TO 1;
 SET citus.replicate_reference_tables_on_activate TO off;
+\set VERBOSITY terse
 -- Simulates a readonly node by setting default_transaction_read_only.
 CREATE FUNCTION mark_node_readonly(hostname TEXT, port INTEGER, isreadonly BOOLEAN)
     RETURNS TEXT
@@ -319,7 +320,6 @@ SET citus.log_remote_commands = ON;
 -- trigger caching for prepared statements
 EXECUTE foo;
 NOTICE:  issuing SELECT count(*) AS count FROM public.dist_table_1_102010 dist_table_1 WHERE (a OPERATOR(pg_catalog.=) 1)
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
  count
 ---------------------------------------------------------------------
      0
@@ -327,7 +327,6 @@ DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 
 EXECUTE foo;
 NOTICE:  issuing SELECT count(*) AS count FROM public.dist_table_1_102010 dist_table_1 WHERE (a OPERATOR(pg_catalog.=) 1)
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
  count
 ---------------------------------------------------------------------
      0
@@ -335,7 +334,6 @@ DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 
 EXECUTE foo;
 NOTICE:  issuing SELECT count(*) AS count FROM public.dist_table_1_102010 dist_table_1 WHERE (a OPERATOR(pg_catalog.=) 1)
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
  count
 ---------------------------------------------------------------------
      0
@@ -343,7 +341,6 @@ DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 
 EXECUTE foo;
 NOTICE:  issuing SELECT count(*) AS count FROM public.dist_table_1_102010 dist_table_1 WHERE (a OPERATOR(pg_catalog.=) 1)
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
  count
 ---------------------------------------------------------------------
      0
@@ -351,7 +348,6 @@ DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 
 EXECUTE foo;
 NOTICE:  issuing SELECT count(*) AS count FROM public.dist_table_1_102010 dist_table_1 WHERE (a OPERATOR(pg_catalog.=) 1)
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
  count
 ---------------------------------------------------------------------
      0
@@ -359,7 +355,6 @@ DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 
 EXECUTE foo;
 NOTICE:  issuing SELECT count(*) AS count FROM public.dist_table_1_102010 dist_table_1 WHERE (a OPERATOR(pg_catalog.=) 1)
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
  count
 ---------------------------------------------------------------------
      0
@@ -367,7 +362,6 @@ DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 
 EXECUTE foo;
 NOTICE:  issuing SELECT count(*) AS count FROM public.dist_table_1_102010 dist_table_1 WHERE (a OPERATOR(pg_catalog.=) 1)
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
  count
 ---------------------------------------------------------------------
      0
@@ -381,7 +375,6 @@ SELECT master_update_node(:nodeid_1, '127.0.0.1', :worker_1_port);
 
 SELECT wait_until_metadata_sync(30000);
 NOTICE:  issuing LISTEN metadata_sync
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
  wait_until_metadata_sync
 ---------------------------------------------------------------------
 
@@ -390,7 +383,6 @@ DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 -- make sure the nodename changed.
 EXECUTE foo;
 NOTICE:  issuing SELECT count(*) AS count FROM public.dist_table_1_102010 dist_table_1 WHERE (a OPERATOR(pg_catalog.=) 1)
-DETAIL:  on server postgres@127.0.0.1:57637 connectionId: xxxxxxx
  count
 ---------------------------------------------------------------------
      0
@@ -500,8 +492,6 @@ UPDATE pg_dist_node SET metadatasynced = TRUE WHERE nodeid IN (:nodeid_1, :nodei
 -- should error out
 SELECT 1 FROM master_disable_node('localhost', 1);
 ERROR:  Disabling localhost:xxxxx failed
-DETAIL:  connection error: localhost:xxxxx
-HINT:  If you are using MX, try stop_metadata_sync_to_node(hostname, port) for nodes that are down before disabling them.
 -- try again after stopping metadata sync
 SELECT stop_metadata_sync_to_node('localhost', 1);
  stop_metadata_sync_to_node
@@ -566,8 +556,6 @@ UPDATE pg_dist_node SET metadatasynced = TRUE WHERE nodeid IN (:nodeid_1, :nodei
 -- should error out
 SELECT 1 FROM master_disable_node('localhost', :worker_2_port);
 ERROR:  Disabling localhost:xxxxx failed
-DETAIL:  connection error: localhost:xxxxx
-HINT:  If you are using MX, try stop_metadata_sync_to_node(hostname, port) for nodes that are down before disabling them.
 -- try again after stopping metadata sync
 SELECT stop_metadata_sync_to_node('localhost', 1);
  stop_metadata_sync_to_node

src/test/regress/expected/multi_replicate_reference_table.out

@@ -1088,10 +1088,24 @@ END;
 -- test adding an invalid node while we have reference tables to replicate
 -- set client message level to ERROR and verbosity to terse to supporess
 -- OS-dependent host name resolution warnings
-SET client_min_messages to ERROR;
 \set VERBOSITY terse
+SET client_min_messages to ERROR;
+DO $$
+DECLARE
+        errors_received INTEGER;
+BEGIN
+errors_received := 0;
+        BEGIN
 		SELECT master_add_node('invalid-node-name', 9999);
-ERROR:  connection error: invalid-node-name:9999
+        EXCEPTION WHEN OTHERS THEN
+                IF SQLERRM LIKE 'connection to the remote node%%' THEN
+                        errors_received := errors_received + 1;
+                END IF;
+        END;
+RAISE '(%/1) failed to add node', errors_received;
+END;
+$$;
+ERROR:  (1/1) failed to add node
 -- drop unnecassary tables
 DROP TABLE initially_not_replicated_reference_table;
 -- reload pg_dist_shard_placement table

src/test/regress/expected/multi_router_planner.out

@@ -2442,8 +2442,7 @@ GRANT INSERT ON ALL TABLES IN SCHEMA public TO router_user;
 -- we will fail to connect to worker 2, since the user does not exist
 BEGIN;
 INSERT INTO failure_test VALUES (1, 1);
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "router_user" does not exist
-DETAIL:  FATAL:  role "router_user" does not exist
 SELECT shardid, shardstate, nodename, nodeport FROM pg_dist_shard_placement
 	WHERE shardid IN (
 		SELECT shardid FROM pg_dist_shard
@@ -2460,8 +2459,7 @@ SELECT shardid, shardstate, nodename, nodeport FROM pg_dist_shard_placement
 
 ROLLBACK;
 INSERT INTO failure_test VALUES (2, 1);
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "router_user" does not exist
-DETAIL:  FATAL:  role "router_user" does not exist
 SELECT shardid, shardstate, nodename, nodeport FROM pg_dist_shard_placement
 	WHERE shardid IN (
 		SELECT shardid FROM pg_dist_shard

src/test/regress/expected/node_conninfo_reload.out

@@ -46,8 +46,7 @@ show citus.node_conninfo;
 
 -- Should give a connection error because of bad sslmode
 select count(*) from test where a = 0;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist"
-DETAIL:  invalid sslmode value: "doesnotexist"
 -- Reset it again
 ALTER SYSTEM RESET citus.node_conninfo;
 select pg_reload_conf();
@@ -114,8 +113,7 @@ select count(*) from test where a = 0;
 COMMIT;
 -- Should fail now with connection error, when transaction is finished
 select count(*) from test where a = 0;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist"
-DETAIL:  invalid sslmode value: "doesnotexist"
 -- Reset it again
 ALTER SYSTEM RESET citus.node_conninfo;
 select pg_reload_conf();
@@ -177,8 +175,7 @@ select count(*) from test where a = 0;
 COMMIT;
 -- Should fail now, when transaction is finished
 select count(*) from test where a = 0;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist"
-DETAIL:  invalid sslmode value: "doesnotexist"
 -- Reset it again
 ALTER SYSTEM RESET citus.node_conninfo;
 select pg_reload_conf();
@@ -231,13 +228,11 @@ show citus.node_conninfo;
 -- Should fail since a different shard is accessed and thus a new connection
 -- will to be created.
 select count(*) from test where a = 0;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist"
-DETAIL:  invalid sslmode value: "doesnotexist"
 COMMIT;
 -- Should still fail now, when transaction is finished
 select count(*) from test where a = 0;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist"
-DETAIL:  invalid sslmode value: "doesnotexist"
 -- Reset it again
 ALTER SYSTEM RESET citus.node_conninfo;
 select pg_reload_conf();
@@ -298,8 +293,7 @@ select count(*) from test;
 COMMIT;
 -- Should fail now, when transaction is finished
 select count(*) from test;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist"
-DETAIL:  invalid sslmode value: "doesnotexist"
 -- Reset it again
 ALTER SYSTEM RESET citus.node_conninfo;
 select pg_reload_conf();
@@ -355,8 +349,7 @@ ERROR:  division by zero
 ROLLBACK;
 -- Should fail now, when transaction is finished
 select count(*) from test;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist"
-DETAIL:  invalid sslmode value: "doesnotexist"
 -- Reset it again
 ALTER SYSTEM RESET citus.node_conninfo;
 select pg_reload_conf();

src/test/regress/output/multi_copy.source

@@ -15,7 +15,7 @@ CREATE TABLE customer_copy_hash (
 		primary key (c_custkey));
 SELECT master_create_distributed_table('customer_copy_hash', 'c_custkey', 'hash');
  master_create_distributed_table
----------------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -26,7 +26,7 @@ DETAIL:  No shards exist for distributed table "customer_copy_hash".
 HINT:  Run master_create_worker_shards to create shards and try again.
 SELECT master_create_worker_shards('customer_copy_hash', 64, 1);
  master_create_worker_shards
------------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -39,7 +39,7 @@ CONTEXT:  COPY customer_copy_hash, line 1, column c_custkey: "1,customer1"
 -- Confirm that no data was copied
 SELECT count(*) FROM customer_copy_hash;
  count
--------
+---------------------------------------------------------------------
      0
 (1 row)
 
@@ -51,7 +51,7 @@ DETAIL:  Key (c_custkey)=(2) already exists.
 -- Confirm that no data was copied
 SELECT count(*) FROM customer_copy_hash;
  count
--------
+---------------------------------------------------------------------
      0
 (1 row)
 
@@ -61,7 +61,7 @@ WITH (FORMAT 'csv', HEADER true, FORCE_NULL (c_custkey));
 -- Confirm that only first row was skipped
 SELECT count(*) FROM customer_copy_hash;
  count
--------
+---------------------------------------------------------------------
      3
 (1 row)
 
@@ -71,7 +71,7 @@ WITH (FORMAT 'csv', QUOTE '"', FORCE_NOT_NULL (c_address));
 -- Confirm that value is not null
 SELECT count(c_address) FROM customer_copy_hash WHERE c_custkey = 4;
  count
--------
+---------------------------------------------------------------------
      1
 (1 row)
 
@@ -81,7 +81,7 @@ WITH (FORMAT 'csv', QUOTE '"', FORCE_NULL (c_address));
 -- Confirm that value is null
 SELECT count(c_address) FROM customer_copy_hash WHERE c_custkey = 5;
  count
--------
+---------------------------------------------------------------------
      0
 (1 row)
 
@@ -93,7 +93,7 @@ DETAIL:  Failing row contains (8, null, null, null, null, null, null, null).
 -- Confirm that no data was copied
 SELECT count(*) FROM customer_copy_hash;
  count
--------
+---------------------------------------------------------------------
      5
 (1 row)
 
@@ -103,7 +103,7 @@ WITH (DELIMITER ' ');
 -- Confirm that data was copied
 SELECT count(*) FROM customer_copy_hash WHERE c_custkey = 9;
  count
--------
+---------------------------------------------------------------------
      1
 (1 row)
 
@@ -112,7 +112,7 @@ COPY customer_copy_hash FROM '@abs_srcdir@/data/customer.2.data' WITH (DELIMITER
 -- Confirm that data was copied
 SELECT count(*) FROM customer_copy_hash;
  count
--------
+---------------------------------------------------------------------
   1006
 (1 row)
 
@@ -121,7 +121,7 @@ SELECT count(*) FROM customer_copy_hash;
 -- Confirm that data was copied
 SELECT count(*) FROM customer_copy_hash;
  count
--------
+---------------------------------------------------------------------
   2006
 (1 row)
 
@@ -129,19 +129,19 @@ SELECT count(*) FROM customer_copy_hash;
 -- hash-partitioned tables
 SELECT master_update_shard_statistics(560000);
  master_update_shard_statistics
---------------------------------
+---------------------------------------------------------------------
                            8192
 (1 row)
 
 SELECT shardid, shardminvalue, shardmaxvalue FROM pg_dist_shard WHERE shardid = 560000;
  shardid | shardminvalue | shardmaxvalue
----------+---------------+---------------
+---------------------------------------------------------------------
   560000 | -2147483648   | -2080374785
 (1 row)
 
 SELECT shardid, shardlength FROM pg_dist_shard_placement WHERE shardid = 560000;
  shardid | shardlength
----------+-------------
+---------------------------------------------------------------------
   560000 |        8192
 (1 row)
 
@@ -152,13 +152,13 @@ CREATE TABLE customer_with_default(
         c_time timestamp default now());
 SELECT master_create_distributed_table('customer_with_default', 'c_custkey', 'hash');
  master_create_distributed_table
----------------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
 SELECT master_create_worker_shards('customer_with_default', 64, 1);
  master_create_worker_shards
------------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -168,7 +168,7 @@ WITH (FORMAT 'csv');
 -- Confirm that data was copied with now() function
 SELECT count(*) FROM customer_with_default where c_time IS NOT NULL;
  count
--------
+---------------------------------------------------------------------
      2
 (1 row)
 
@@ -178,7 +178,7 @@ ALTER TABLE customer_copy_hash ADD COLUMN extra2 INT DEFAULT 0;
 COPY customer_copy_hash (c_custkey, c_name, extra1, extra2) FROM STDIN CSV;
 SELECT * FROM customer_copy_hash WHERE extra1 = 1;
  c_custkey |   c_name   | c_address | c_nationkey | c_phone | c_acctbal | c_mktsegment | c_comment | extra1 | extra2
------------+------------+-----------+-------------+---------+-----------+--------------+-----------+--------+--------
+---------------------------------------------------------------------
         10 | customer10 |           |             |         |           |              |           |      1 |      5
 (1 row)
 
@@ -187,7 +187,7 @@ ALTER TABLE customer_copy_hash DROP COLUMN extra1;
 COPY customer_copy_hash (c_custkey, c_name, extra2) FROM STDIN CSV;
 SELECT * FROM customer_copy_hash WHERE c_custkey = 11;
  c_custkey |   c_name   | c_address | c_nationkey | c_phone | c_acctbal | c_mktsegment | c_comment | extra2
------------+------------+-----------+-------------+---------+-----------+--------------+-----------+--------
+---------------------------------------------------------------------
         11 | customer11 |           |             |         |           |              |           |      5
 (1 row)
 
@@ -196,7 +196,7 @@ ALTER TABLE customer_copy_hash DROP COLUMN extra2;
 COPY customer_copy_hash (c_custkey, c_name) FROM STDIN CSV;
 SELECT * FROM customer_copy_hash WHERE c_custkey = 12;
  c_custkey |   c_name   | c_address | c_nationkey | c_phone | c_acctbal | c_mktsegment | c_comment
------------+------------+-----------+-------------+---------+-----------+--------------+-----------
+---------------------------------------------------------------------
         12 | customer12 |           |             |         |           |              |
 (1 row)
 
@@ -213,7 +213,7 @@ CREATE TABLE customer_copy_range (
 		primary key (c_custkey));
 SELECT master_create_distributed_table('customer_copy_range', 'c_custkey', 'range');
  master_create_distributed_table
----------------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -235,14 +235,14 @@ COPY customer_copy_range FROM '@abs_srcdir@/data/customer.1.data' WITH (DELIMITE
 SELECT min(c_custkey), max(c_custkey), avg(c_custkey), count(*)
 FROM customer_copy_range WHERE c_custkey <= 500;
  min | max |         avg          | count
------+-----+----------------------+-------
+---------------------------------------------------------------------
    1 | 500 | 250.5000000000000000 |   500
 (1 row)
 
 -- Check whether data was copied
 SELECT count(*) FROM customer_copy_range;
  count
--------
+---------------------------------------------------------------------
   1000
 (1 row)
 
@@ -251,13 +251,13 @@ UPDATE pg_dist_shard SET shardminvalue = 1501, shardmaxvalue = 2000
 WHERE shardid = :new_shard_id;
 SELECT shardid, shardminvalue, shardmaxvalue FROM pg_dist_shard WHERE shardid = :new_shard_id;
  shardid | shardminvalue | shardmaxvalue
----------+---------------+---------------
+---------------------------------------------------------------------
   560129 | 1501          | 2000
 (1 row)
 
 SELECT shardid, shardlength FROM pg_dist_shard_placement WHERE shardid = :new_shard_id;
  shardid | shardlength
----------+-------------
+---------------------------------------------------------------------
   560129 |           0
   560129 |           0
 (2 rows)
@@ -266,19 +266,19 @@ SELECT shardid, shardlength FROM pg_dist_shard_placement WHERE shardid = :new_sh
 -- shard length is updated.
 SELECT master_update_shard_statistics(:new_shard_id);
  master_update_shard_statistics
---------------------------------
+---------------------------------------------------------------------
                          131072
 (1 row)
 
 SELECT shardid, shardminvalue, shardmaxvalue FROM pg_dist_shard WHERE shardid = :new_shard_id;
  shardid | shardminvalue | shardmaxvalue
----------+---------------+---------------
+---------------------------------------------------------------------
   560129 | 1501          | 2000
 (1 row)
 
 SELECT shardid, shardlength FROM pg_dist_shard_placement WHERE shardid = :new_shard_id;
  shardid | shardlength
----------+-------------
+---------------------------------------------------------------------
   560129 |      131072
   560129 |      131072
 (2 rows)
@@ -298,7 +298,7 @@ CREATE TABLE customer_copy_append (
         c_comment varchar(117));
 SELECT master_create_distributed_table('customer_copy_append', 'c_custkey', 'append');
  master_create_distributed_table
----------------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -309,7 +309,7 @@ CONTEXT:  COPY customer_copy_append, line 3, column c_custkey: "notinteger"
 -- Test that no shard is created for failing copy
 SELECT count(*) FROM pg_dist_shard WHERE logicalrelid = 'customer_copy_append'::regclass;
  count
--------
+---------------------------------------------------------------------
      0
 (1 row)
 
@@ -318,7 +318,7 @@ COPY customer_copy_append FROM STDIN;
 -- Test that no shard is created for copying zero rows
 SELECT count(*) FROM pg_dist_shard WHERE logicalrelid = 'customer_copy_append'::regclass;
  count
--------
+---------------------------------------------------------------------
      0
 (1 row)
 
@@ -327,7 +327,7 @@ COPY customer_copy_append(c_custkey, c_name) FROM STDIN WITH (FORMAT 'csv');
 -- Check whether data was copied properly
 SELECT * FROM customer_copy_append;
  c_custkey |  c_name   | c_address | c_nationkey | c_phone | c_acctbal | c_mktsegment | c_comment
------------+-----------+-----------+-------------+---------+-----------+--------------+-----------
+---------------------------------------------------------------------
          1 | customer1 |           |             |         |           |              |
          2 | customer2 |           |             |         |           |              |
 (2 rows)
@@ -337,13 +337,13 @@ UPDATE pg_dist_shard SET shardminvalue = 1501, shardmaxvalue = 2000 WHERE shardi
 UPDATE pg_dist_shard_placement SET shardlength = 0 WHERE shardid = 560131;
 SELECT shardid, shardminvalue, shardmaxvalue FROM pg_dist_shard WHERE shardid = 560131;
  shardid | shardminvalue | shardmaxvalue
----------+---------------+---------------
+---------------------------------------------------------------------
   560131 | 1501          | 2000
 (1 row)
 
 SELECT shardid, shardlength FROM pg_dist_shard_placement WHERE shardid = 560131;
  shardid | shardlength
----------+-------------
+---------------------------------------------------------------------
   560131 |           0
   560131 |           0
 (2 rows)
@@ -351,19 +351,19 @@ SELECT shardid, shardlength FROM pg_dist_shard_placement WHERE shardid = 560131;
 -- Update shard statistics for append-partitioned shard
 SELECT master_update_shard_statistics(560131);
  master_update_shard_statistics
---------------------------------
+---------------------------------------------------------------------
                            8192
 (1 row)
 
 SELECT shardid, shardminvalue, shardmaxvalue FROM pg_dist_shard WHERE shardid = 560131;
  shardid | shardminvalue | shardmaxvalue
----------+---------------+---------------
+---------------------------------------------------------------------
   560131 | 1             | 2
 (1 row)
 
 SELECT shardid, shardlength FROM pg_dist_shard_placement WHERE shardid = 560131;
  shardid | shardlength
----------+-------------
+---------------------------------------------------------------------
   560131 |        8192
   560131 |        8192
 (2 rows)
@@ -388,7 +388,7 @@ CREATE TABLE lineitem_copy_append (
         l_comment varchar(44) not null);
 SELECT master_create_distributed_table('lineitem_copy_append', 'l_orderkey', 'append');
  master_create_distributed_table
----------------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -397,7 +397,7 @@ SET citus.shard_max_size TO '256kB';
 COPY lineitem_copy_append FROM '@abs_srcdir@/data/lineitem.1.data' with delimiter '|';
 SELECT count(*) FROM pg_dist_shard WHERE logicalrelid = 'lineitem_copy_append'::regclass;
  count
--------
+---------------------------------------------------------------------
      5
 (1 row)
 
@@ -418,7 +418,7 @@ ORDER BY
 LIMIT
         5;
  shardid | nodeport
----------+----------
+---------------------------------------------------------------------
   560141 |    57637
   560140 |    57638
   560139 |    57637
@@ -439,7 +439,7 @@ CREATE TABLE append.customer_copy (
         c_comment varchar(117));
 SELECT master_create_distributed_table('append.customer_copy', 'c_custkey', 'append');
  master_create_distributed_table
----------------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -449,7 +449,7 @@ COPY append.customer_copy FROM '@abs_srcdir@/data/customer.2.data' with (delimit
 -- Test the content of the table
 SELECT min(c_custkey), max(c_custkey), avg(c_acctbal), count(*) FROM append.customer_copy;
  min | max  |          avg          | count
------+------+-----------------------+-------
+---------------------------------------------------------------------
    1 | 7000 | 4443.8028800000000000 |  2000
 (1 row)
 
@@ -459,13 +459,13 @@ CREATE TABLE "customer_with_special_\\_character"(
         c_name varchar(25) not null);
 SELECT master_create_distributed_table('"customer_with_special_\\_character"', 'c_custkey', 'hash');
  master_create_distributed_table
----------------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
 SELECT master_create_worker_shards('"customer_with_special_\\_character"', 4, 1);
  master_create_worker_shards
------------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -474,7 +474,7 @@ WITH (FORMAT 'csv');
 -- Confirm that data was copied
 SELECT count(*) FROM "customer_with_special_\\_character";
  count
--------
+---------------------------------------------------------------------
      2
 (1 row)
 
@@ -484,13 +484,13 @@ CREATE TABLE "1_customer"(
         c_name varchar(25) not null);
 SELECT master_create_distributed_table('"1_customer"', 'c_custkey', 'hash');
  master_create_distributed_table
----------------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
 SELECT master_create_worker_shards('"1_customer"', 4, 1);
  master_create_worker_shards
------------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -499,7 +499,7 @@ WITH (FORMAT 'csv');
 -- Confirm that data was copied
 SELECT count(*) FROM "1_customer";
  count
--------
+---------------------------------------------------------------------
      2
 (1 row)
 
@@ -519,13 +519,13 @@ CREATE TABLE packed_numbers_hash (
 );
 SELECT master_create_distributed_table('packed_numbers_hash', 'id', 'hash');
  master_create_distributed_table
----------------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
 SELECT master_create_worker_shards('packed_numbers_hash', 4, 1);
  master_create_worker_shards
------------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -534,7 +534,7 @@ COPY packed_numbers_hash FROM :'temp_dir''copy_test_array_of_composite';
 -- Verify data is actually copied
 SELECT * FROM packed_numbers_hash;
  id |    packed_numbers
-----+-----------------------
+---------------------------------------------------------------------
   1 | {"(42,42)","(42,42)"}
 (1 row)
 
@@ -545,13 +545,13 @@ CREATE TABLE super_packed_numbers_hash (
 );
 SELECT master_create_distributed_table('super_packed_numbers_hash', 'id', 'hash');
  master_create_distributed_table
----------------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
 SELECT master_create_worker_shards('super_packed_numbers_hash', 4, 1);
  master_create_worker_shards
------------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -560,7 +560,7 @@ COPY super_packed_numbers_hash FROM :'temp_dir''copy_test_composite_of_composite
 -- Verify data is actually copied
 SELECT * FROM super_packed_numbers_hash;
  id |  super_packed_number
-----+-----------------------
+---------------------------------------------------------------------
   1 | ("(42,42)","(42,42)")
 (1 row)
 
@@ -571,7 +571,7 @@ CREATE TABLE packed_numbers_append (
 );
 SELECT master_create_distributed_table('packed_numbers_append', 'id', 'append');
  master_create_distributed_table
----------------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -579,7 +579,7 @@ COPY packed_numbers_append FROM :'temp_dir''copy_test_array_of_composite';
 -- Verify data is actually copied
 SELECT * FROM packed_numbers_append;
  id |    packed_numbers
-----+-----------------------
+---------------------------------------------------------------------
   1 | {"(42,42)","(42,42)"}
 (1 row)
 
@@ -590,7 +590,7 @@ CREATE TABLE super_packed_numbers_append (
 );
 SELECT master_create_distributed_table('super_packed_numbers_append', 'id', 'append');
  master_create_distributed_table
----------------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -598,7 +598,7 @@ COPY super_packed_numbers_append FROM :'temp_dir''copy_test_composite_of_composi
 -- Verify data is actually copied
 SELECT * FROM super_packed_numbers_append;
  id |  super_packed_number
-----+-----------------------
+---------------------------------------------------------------------
   1 | ("(42,42)","(42,42)")
 (1 row)
 
@@ -609,22 +609,22 @@ CREATE TABLE composite_partition_column_table(
 );
 SELECT master_create_distributed_table('composite_partition_column_table', 'composite_column', 'append');
  master_create_distributed_table
----------------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
 \COPY composite_partition_column_table FROM STDIN WITH (FORMAT 'csv');
 WARNING:  function min(number_pack) does not exist
 HINT:  No function matches the given name and argument types. You might need to add explicit type casts.
-CONTEXT:  while executing command on localhost:57638
+CONTEXT:  while executing command on localhost:xxxxx
 WARNING:  could not get statistics for shard public.composite_partition_column_table_560162
 DETAIL:  Setting shard statistics to NULL
-ERROR:  failure on connection marked as essential: localhost:57638
+ERROR:  failure on connection marked as essential: localhost:xxxxx
 -- Test copy on append distributed tables do not create shards on removed workers
 CREATE TABLE numbers_append (a int, b int);
 SELECT master_create_distributed_table('numbers_append', 'a', 'append');
  master_create_distributed_table
----------------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -633,7 +633,7 @@ SELECT shardid, nodename, nodeport
 	FROM pg_dist_shard_placement join pg_dist_shard using(shardid)
 	WHERE logicalrelid = 'numbers_append'::regclass order by placementid;
  shardid | nodename | nodeport
----------+----------+----------
+---------------------------------------------------------------------
 (0 rows)
 
 COPY numbers_append FROM STDIN WITH (FORMAT 'csv');
@@ -643,16 +643,16 @@ SELECT shardid, nodename, nodeport
 	FROM pg_dist_shard_placement join pg_dist_shard using(shardid)
 	WHERE logicalrelid = 'numbers_append'::regclass order by placementid;
  shardid | nodename  | nodeport
----------+-----------+----------
+---------------------------------------------------------------------
   560163 | localhost |    57637
   560164 | localhost |    57638
 (2 rows)
 
 -- disable the first node
 SELECT master_disable_node('localhost', :worker_1_port);
-NOTICE:  Node localhost:57637 has active shard placements. Some queries may fail after this operation. Use SELECT master_activate_node('localhost', 57637) to activate this node back.
+NOTICE:  Node localhost:xxxxx has active shard placements. Some queries may fail after this operation. Use SELECT master_activate_node('localhost', 57637) to activate this node back.
  master_disable_node
----------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -666,7 +666,7 @@ SELECT shardid, nodename, nodeport
 	FROM pg_dist_shard_placement join pg_dist_shard using(shardid)
 	WHERE logicalrelid = 'numbers_append'::regclass order by placementid;
  shardid | nodename  | nodeport
----------+-----------+----------
+---------------------------------------------------------------------
   560163 | localhost |    57637
   560164 | localhost |    57638
   560165 | localhost |    57638
@@ -677,7 +677,7 @@ SELECT shardid, nodename, nodeport
 SET client_min_messages TO ERROR;
 SELECT 1 FROM master_activate_node('localhost', :worker_1_port);
  ?column?
-----------
+---------------------------------------------------------------------
         1
 (1 row)
 
@@ -690,7 +690,7 @@ SELECT shardid, nodename, nodeport
 	FROM pg_dist_shard_placement join pg_dist_shard using(shardid)
 	WHERE logicalrelid = 'numbers_append'::regclass order by placementid;
  shardid | nodename  | nodeport
----------+-----------+----------
+---------------------------------------------------------------------
   560163 | localhost |    57637
   560164 | localhost |    57638
   560165 | localhost |    57638
@@ -709,7 +709,7 @@ NOTICE:  not propagating CREATE ROLE/USER commands to worker nodes
 HINT:  Connect to worker nodes directly to manually create all necessary users and roles.
 SELECT * FROM run_command_on_workers('CREATE USER test_user');
  nodename  | nodeport | success |   result
------------+----------+---------+-------------
+---------------------------------------------------------------------
  localhost |    57637 | t       | CREATE ROLE
  localhost |    57638 | t       | CREATE ROLE
 (2 rows)
@@ -719,7 +719,7 @@ SET citus.shard_count to 4;
 CREATE TABLE numbers_hash (a int, b int);
 SELECT create_distributed_table('numbers_hash', 'a');
  create_distributed_table
---------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -729,7 +729,7 @@ SELECT shardid, shardstate, nodename, nodeport
 	FROM pg_dist_shard_placement join pg_dist_shard using(shardid)
 	WHERE logicalrelid = 'numbers_hash'::regclass order by shardid, nodeport;
  shardid | shardstate | nodename  | nodeport
----------+------------+-----------+----------
+---------------------------------------------------------------------
   560169 |          1 | localhost |    57637
   560169 |          1 | localhost |    57638
   560170 |          1 | localhost |    57637
@@ -744,7 +744,7 @@ SELECT shardid, shardstate, nodename, nodeport
 CREATE TABLE numbers_reference(a int, b int);
 SELECT create_reference_table('numbers_reference');
  create_reference_table
-------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -753,7 +753,7 @@ COPY numbers_reference FROM STDIN WITH (FORMAT 'csv');
 CREATE TABLE numbers_hash_other(a int, b int);
 SELECT create_distributed_table('numbers_hash_other', 'a');
  create_distributed_table
---------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -761,7 +761,7 @@ SELECT shardid, shardstate, nodename, nodeport
 	FROM pg_dist_shard_placement join pg_dist_shard using(shardid)
 	WHERE logicalrelid = 'numbers_hash_other'::regclass order by shardid, nodeport;
  shardid | shardstate | nodename  | nodeport
----------+------------+-----------+----------
+---------------------------------------------------------------------
   560174 |          1 | localhost |    57637
   560174 |          1 | localhost |    57638
   560175 |          1 | localhost |    57637
@@ -783,24 +783,20 @@ ALTER USER test_user WITH nologin;
 \c - test_user - :master_port
 -- reissue copy
 COPY numbers_hash FROM STDIN WITH (FORMAT 'csv');
-WARNING:  connection error: localhost:57637
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "test_user" is not permitted to log in
-DETAIL:  FATAL:  role "test_user" is not permitted to log in
 CONTEXT:  COPY numbers_hash, line 1: "1,1"
-WARNING:  connection error: localhost:57637
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "test_user" is not permitted to log in
-DETAIL:  FATAL:  role "test_user" is not permitted to log in
 CONTEXT:  COPY numbers_hash, line 2: "2,2"
-WARNING:  connection error: localhost:57637
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "test_user" is not permitted to log in
-DETAIL:  FATAL:  role "test_user" is not permitted to log in
 CONTEXT:  COPY numbers_hash, line 3: "3,3"
-WARNING:  connection error: localhost:57637
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "test_user" is not permitted to log in
-DETAIL:  FATAL:  role "test_user" is not permitted to log in
 CONTEXT:  COPY numbers_hash, line 6: "6,6"
 -- verify shards in the first worker as marked invalid
 SELECT shardid, shardstate, nodename, nodeport
 	FROM pg_dist_shard_placement join pg_dist_shard using(shardid)
 	WHERE logicalrelid = 'numbers_hash'::regclass order by shardid, nodeport;
  shardid | shardstate | nodename  | nodeport
----------+------------+-----------+----------
+---------------------------------------------------------------------
   560169 |          3 | localhost |    57637
   560169 |          1 | localhost |    57638
   560170 |          3 | localhost |    57637
@@ -813,15 +809,14 @@ SELECT shardid, shardstate, nodename, nodeport
 
 -- try to insert into a reference table copy should fail
 COPY numbers_reference FROM STDIN WITH (FORMAT 'csv');
-ERROR:  connection error: localhost:57637
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "test_user" is not permitted to log in
-DETAIL:  FATAL:  role "test_user" is not permitted to log in
 CONTEXT:  COPY numbers_reference, line 1: "3,1"
 -- verify shards for reference table are still valid
 SELECT shardid, shardstate, nodename, nodeport
 	FROM pg_dist_shard_placement join pg_dist_shard using(shardid)
 	WHERE logicalrelid = 'numbers_reference'::regclass order by placementid;
  shardid | shardstate | nodename  | nodeport
----------+------------+-----------+----------
+---------------------------------------------------------------------
   560173 |          1 | localhost |    57637
   560173 |          1 | localhost |    57638
 (2 rows)
@@ -830,14 +825,11 @@ SELECT shardid, shardstate, nodename, nodeport
 -- since it can not insert into either copies of a shard. shards are expected to
 -- stay valid since the operation is rolled back.
 COPY numbers_hash_other FROM STDIN WITH (FORMAT 'csv');
-WARNING:  connection error: localhost:57637
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "test_user" is not permitted to log in
-DETAIL:  FATAL:  role "test_user" is not permitted to log in
 CONTEXT:  COPY numbers_hash_other, line 1: "1,1"
-WARNING:  connection error: localhost:57637
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "test_user" is not permitted to log in
-DETAIL:  FATAL:  role "test_user" is not permitted to log in
 CONTEXT:  COPY numbers_hash_other, line 2: "2,2"
-WARNING:  connection error: localhost:57637
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "test_user" is not permitted to log in
-DETAIL:  FATAL:  role "test_user" is not permitted to log in
 CONTEXT:  COPY numbers_hash_other, line 3: "3,3"
 -- verify shards for numbers_hash_other are still valid
 -- since copy has failed altogether
@@ -845,7 +837,7 @@ SELECT shardid, shardstate, nodename, nodeport
 	FROM pg_dist_shard_placement join pg_dist_shard using(shardid)
 	WHERE logicalrelid = 'numbers_hash_other'::regclass order by shardid, nodeport;
  shardid | shardstate | nodename  | nodeport
----------+------------+-----------+----------
+---------------------------------------------------------------------
   560174 |          3 | localhost |    57637
   560174 |          1 | localhost |    57638
   560175 |          3 | localhost |    57637
@@ -873,7 +865,7 @@ SET citus.shard_count to 4;
 CREATE TABLE numbers_hash(a int, b int);
 SELECT create_distributed_table('numbers_hash', 'a');
  create_distributed_table
---------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -883,12 +875,12 @@ ALTER TABLE numbers_hash_560180 DROP COLUMN b;
 -- operation will fail to modify a shard and roll back
 COPY numbers_hash FROM STDIN WITH (FORMAT 'csv');
 ERROR:  column "b" of relation "numbers_hash_560180" does not exist
-CONTEXT:  while executing command on localhost:57637
+CONTEXT:  while executing command on localhost:xxxxx
 COPY numbers_hash, line 6: "6,6"
 -- verify no row is inserted
 SELECT count(a) FROM numbers_hash;
  count
--------
+---------------------------------------------------------------------
      0
 (1 row)
 
@@ -897,7 +889,7 @@ SELECT shardid, shardstate, nodename, nodeport
 	FROM pg_dist_shard_placement join pg_dist_shard using(shardid)
 	WHERE logicalrelid = 'numbers_hash'::regclass order by shardid, nodeport;
  shardid | shardstate | nodename  | nodeport
----------+------------+-----------+----------
+---------------------------------------------------------------------
   560178 |          1 | localhost |    57637
   560178 |          1 | localhost |    57638
   560179 |          1 | localhost |    57637
@@ -911,7 +903,7 @@ SELECT shardid, shardstate, nodename, nodeport
 DROP TABLE numbers_hash;
 SELECT * FROM run_command_on_workers('DROP USER test_user');
  nodename  | nodeport | success |  result
------------+----------+---------+-----------
+---------------------------------------------------------------------
  localhost |    57637 | t       | DROP ROLE
  localhost |    57638 | t       | DROP ROLE
 (2 rows)
@@ -924,14 +916,14 @@ col2 character varying(255) NOT NULL
 );
 SELECT create_reference_table('test_binaryless_builtin');
  create_reference_table
-------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
 \COPY test_binaryless_builtin FROM STDIN WITH (format CSV)
 SELECT * FROM test_binaryless_builtin;
         col1         | col2
----------------------+-------
+---------------------------------------------------------------------
  postgres=r/postgres |  test
 (1 row)
 
@@ -941,7 +933,7 @@ BEGIN;
 CREATE TABLE tt1(id int);
 SELECT create_distributed_table('tt1','id');
  create_distributed_table
---------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -952,7 +944,7 @@ END;
 CREATE TABLE drop_copy_test_table (col1 int, col2 int, col3 int, col4 int);
 SELECT create_distributed_table('drop_copy_test_table','col3');
  create_distributed_table
---------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -960,7 +952,7 @@ ALTER TABLE drop_copy_test_table drop column col1;
 COPY drop_copy_test_table (col2,col3,col4) from STDIN with CSV;
 SELECT * FROM drop_copy_test_table WHERE col3 = 1;
  col2 | col3 | col4
-------+------+------
+---------------------------------------------------------------------
       |    1 |
 (1 row)
 
@@ -968,7 +960,7 @@ ALTER TABLE drop_copy_test_table drop column col4;
 COPY drop_copy_test_table (col2,col3) from STDIN with CSV;
 SELECT * FROM drop_copy_test_table WHERE col3 = 1;
  col2 | col3
-------+------
+---------------------------------------------------------------------
       |    1
       |    1
 (2 rows)
@@ -978,7 +970,7 @@ DROP TABLE drop_copy_test_table;
 \c - - - :worker_1_port
 SELECT relname FROM pg_class WHERE relname LIKE 'tt1%';
  relname
----------
+---------------------------------------------------------------------
 (0 rows)
 
 \c - - - :master_port
@@ -992,7 +984,7 @@ NOTICE:  copying the data has completed
 DETAIL:  The local data in the table is longer visible, but is still on disk.
 HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.trigger_flush$$)
  create_distributed_table
---------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -1004,7 +996,7 @@ SET citus.multi_shard_modify_mode TO 'sequential';
 CREATE UNLOGGED TABLE trigger_switchover(a int, b int, c int, d int, e int, f int, g int, h int);
 SELECT create_distributed_table('trigger_switchover','a');
  create_distributed_table
---------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -1015,7 +1007,7 @@ ABORT;
 CREATE TABLE copy_jsonb (key text, value jsonb, extra jsonb default '["default"]'::jsonb);
 SELECT create_distributed_table('copy_jsonb', 'key', colocate_with => 'none');
  create_distributed_table
---------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -1023,7 +1015,7 @@ SELECT create_distributed_table('copy_jsonb', 'key', colocate_with => 'none');
 \COPY copy_jsonb (key, value) FROM STDIN
 SELECT * FROM copy_jsonb ORDER BY key;
   key  |           value            |    extra
--------+----------------------------+-------------
+---------------------------------------------------------------------
  blue  | {"b": 255, "g": 0, "r": 0} | ["default"]
  green | {"b": 0, "g": 255, "r": 0} | ["default"]
 (2 rows)
@@ -1033,7 +1025,7 @@ COPY copy_jsonb TO :'temp_dir''copy_jsonb.pgcopy' WITH (format binary);
 COPY copy_jsonb FROM :'temp_dir''copy_jsonb.pgcopy' WITH (format binary);
 SELECT * FROM copy_jsonb ORDER BY key;
   key  |           value            |    extra
--------+----------------------------+-------------
+---------------------------------------------------------------------
  blue  | {"b": 255, "g": 0, "r": 0} | ["default"]
  blue  | {"b": 255, "g": 0, "r": 0} | ["default"]
  green | {"b": 0, "g": 255, "r": 0} | ["default"]
@@ -1042,7 +1034,7 @@ SELECT * FROM copy_jsonb ORDER BY key;
 
 -- JSONB parsing error without validation: no line number
 \COPY copy_jsonb (key, value) FROM STDIN
-ERROR:  invalid input syntax for type json
+ERROR:  invalid input syntax for json
 DETAIL:  The input string ended unexpectedly.
 CONTEXT:  JSON data, line 1: {"r":255,"g":0,"b":0
 COPY copy_jsonb, line 1, column value: "{"r":255,"g":0,"b":0"
@@ -1052,7 +1044,7 @@ SET citus.skip_jsonb_validation_in_copy TO off;
 \COPY copy_jsonb (key, value) FROM STDIN
 SELECT * FROM copy_jsonb ORDER BY key;
   key  |           value            |    extra
--------+----------------------------+-------------
+---------------------------------------------------------------------
  blue  | {"b": 255, "g": 0, "r": 0} | ["default"]
  green | {"b": 0, "g": 255, "r": 0} | ["default"]
 (2 rows)
@@ -1062,7 +1054,7 @@ COPY copy_jsonb TO :'temp_dir''copy_jsonb.pgcopy' WITH (format binary);
 COPY copy_jsonb FROM :'temp_dir''copy_jsonb.pgcopy' WITH (format binary);
 SELECT * FROM copy_jsonb ORDER BY key;
   key  |           value            |    extra
--------+----------------------------+-------------
+---------------------------------------------------------------------
  blue  | {"b": 255, "g": 0, "r": 0} | ["default"]
  blue  | {"b": 255, "g": 0, "r": 0} | ["default"]
  green | {"b": 0, "g": 255, "r": 0} | ["default"]
@@ -1071,7 +1063,7 @@ SELECT * FROM copy_jsonb ORDER BY key;
 
 -- JSONB parsing error with validation: should see line number
 \COPY copy_jsonb (key, value) FROM STDIN
-ERROR:  invalid input syntax for type json
+ERROR:  invalid input syntax for json
 DETAIL:  The input string ended unexpectedly.
 CONTEXT:  JSON data, line 1: {"r":255,"g":0,"b":0
 COPY copy_jsonb, line 1, column value: "{"r":255,"g":0,"b":0"

src/test/regress/sql/multi_mx_node_metadata.sql

@@ -13,6 +13,8 @@ SET citus.shard_count TO 8;
 SET citus.shard_replication_factor TO 1;
 SET citus.replicate_reference_tables_on_activate TO off;
 
+\set VERBOSITY terse
+
 -- Simulates a readonly node by setting default_transaction_read_only.
 CREATE FUNCTION mark_node_readonly(hostname TEXT, port INTEGER, isreadonly BOOLEAN)
     RETURNS TEXT

src/test/regress/sql/multi_replicate_reference_table.sql

@@ -669,10 +669,23 @@ END;
 -- test adding an invalid node while we have reference tables to replicate
 -- set client message level to ERROR and verbosity to terse to supporess
 -- OS-dependent host name resolution warnings
-SET client_min_messages to ERROR;
 \set VERBOSITY terse
-
+SET client_min_messages to ERROR;
+DO $$
+DECLARE
+        errors_received INTEGER;
+BEGIN
+errors_received := 0;
+        BEGIN
 		SELECT master_add_node('invalid-node-name', 9999);
+        EXCEPTION WHEN OTHERS THEN
+                IF SQLERRM LIKE 'connection to the remote node%%' THEN
+                        errors_received := errors_received + 1;
+                END IF;
+        END;
+RAISE '(%/1) failed to add node', errors_received;
+END;
+$$;
 
 -- drop unnecassary tables
 DROP TABLE initially_not_replicated_reference_table;