Merge pull request #3606 from citusdata/improve_error_messages

Improve connection error message from the worker nodes

src/backend/distributed/connection/remote_commands.c

@@ -277,10 +277,23 @@ ReportConnectionError(MultiConnection *connection, int elevel)
 		messageDetail = pchomp(PQerrorMessage(pgConn));
 	}
 
+	if (messageDetail)
+	{
+		/*
+		 * We don't use ApplyLogRedaction(messageDetail) as we expect any error
+		 * detail that requires log reduction should have done it locally.
+		 */
 		ereport(elevel, (errcode(ERRCODE_CONNECTION_FAILURE),
-					 errmsg("connection error: %s:%d", nodeName, nodePort),
+						 errmsg("connection to the remote node %s:%d failed with the "
-					 messageDetail != NULL ?
+								"following error: %s", nodeName, nodePort,
-					 errdetail("%s", ApplyLogRedaction(messageDetail)) : 0));
+								messageDetail)));
+	}
+	else
+	{
+		ereport(elevel, (errcode(ERRCODE_CONNECTION_FAILURE),
+						 errmsg("connection to the remote node %s:%d failed",
+								nodeName, nodePort)));
+	}
 }
 
 

src/test/regress/expected/failure_1pc_copy_append.out

@@ -168,8 +168,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="SELECT|COPY").kill()');
 (1 row)
 
 SELECT count(1) FROM copy_test;
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
  count

src/test/regress/expected/failure_1pc_copy_hash.out

@@ -219,8 +219,7 @@ SELECT citus.mitmproxy('conn.killall()');
 (1 row)
 
 COPY copy_test FROM PROGRAM 'echo 0, 0 && echo 1, 1 && echo 2, 4 && echo 3, 9' WITH CSV;
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 CONTEXT:  COPY copy_test, line 1: "0, 0"

src/test/regress/expected/failure_connection_establishment.out

@@ -227,7 +227,7 @@ RESET client_min_messages;
 -- verify get_global_active_transactions works when a timeout happens on a connection
 SELECT get_global_active_transactions();
 WARNING:  could not establish connection after 400 ms
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed
  get_global_active_transactions
 ---------------------------------------------------------------------
 (0 rows)

src/test/regress/expected/failure_copy_on_hash.out

@@ -36,8 +36,7 @@ SELECT citus.mitmproxy('conn.kill()');
 (1 row)
 
 \COPY test_table FROM stdin delimiter ',';
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 CONTEXT:  COPY test_table, line 1: "1,2"
@@ -276,23 +275,19 @@ SELECT citus.mitmproxy('conn.kill()');
 (1 row)
 
 \COPY test_table_2 FROM stdin delimiter ',';
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 CONTEXT:  COPY test_table_2, line 1: "1,2"
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 CONTEXT:  COPY test_table_2, line 2: "3,4"
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 CONTEXT:  COPY test_table_2, line 3: "6,7"
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 CONTEXT:  COPY test_table_2, line 5: "9,10"

src/test/regress/expected/failure_create_distributed_table_non_empty.out

@@ -26,8 +26,7 @@ SELECT citus.mitmproxy('conn.kill()');
 (1 row)
 
 SELECT create_distributed_table('test_table', 'id');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT count(*) FROM pg_dist_shard WHERE logicalrelid='create_distributed_table_non_empty_failure.test_table'::regclass;
@@ -125,8 +124,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 SELECT create_distributed_table('test_table', 'id');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
@@ -190,8 +188,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").kill()');
 (1 row)
 
 SELECT create_distributed_table('test_table', 'id');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT count(*) FROM pg_dist_shard WHERE logicalrelid='create_distributed_table_non_empty_failure.test_table'::regclass;
@@ -473,8 +470,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 SELECT create_distributed_table('test_table', 'id', colocate_with => 'colocated_table');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT count(*) FROM pg_dist_shard WHERE logicalrelid='create_distributed_table_non_empty_failure.test_table'::regclass;
@@ -561,8 +557,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^SELECT worker_apply_shard_ddl_comma
 (1 row)
 
 SELECT create_distributed_table('test_table', 'id', colocate_with => 'colocated_table');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT count(*) FROM pg_dist_shard WHERE logicalrelid='create_distributed_table_non_empty_failure.test_table'::regclass;
@@ -599,8 +594,7 @@ SELECT citus.mitmproxy('conn.kill()');
 (1 row)
 
 SELECT create_distributed_table('test_table', 'id');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
@@ -667,8 +661,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 SELECT create_distributed_table('test_table', 'id');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
@@ -732,8 +725,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").kill()');
 (1 row)
 
 SELECT create_distributed_table('test_table', 'id');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT count(*) FROM pg_dist_shard WHERE logicalrelid='create_distributed_table_non_empty_failure.test_table'::regclass;
@@ -946,8 +938,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 SELECT create_distributed_table('test_table', 'id', colocate_with => 'colocated_table');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT count(*) FROM pg_dist_shard WHERE logicalrelid='create_distributed_table_non_empty_failure.test_table'::regclass;

src/test/regress/expected/failure_create_reference_table.out

@@ -25,8 +25,7 @@ SELECT citus.mitmproxy('conn.onQuery().kill()');
 (1 row)
 
 SELECT create_reference_table('ref_table');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT count(*) FROM pg_dist_shard_placement;
@@ -43,8 +42,7 @@ SELECT citus.mitmproxy('conn.onCommandComplete(command="BEGIN").kill()');
 (1 row)
 
 SELECT create_reference_table('ref_table');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT count(*) FROM pg_dist_shard_placement;
@@ -76,8 +74,7 @@ SELECT citus.mitmproxy('conn.onCommandComplete(command="SELECT 1").kill()');
 (1 row)
 
 SELECT create_reference_table('ref_table');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT count(*) FROM pg_dist_shard_placement;

src/test/regress/expected/failure_create_table.out

@@ -20,8 +20,7 @@ SELECT citus.mitmproxy('conn.kill()');
 (1 row)
 
 SELECT create_distributed_table('test_table','id');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
@@ -89,8 +88,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 SELECT create_distributed_table('test_table','id');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
@@ -120,8 +118,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="SELECT worker_apply_shard_ddl_comman
 (1 row)
 
 SELECT create_distributed_table('test_table','id');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
@@ -154,8 +151,7 @@ BEGIN;
 (1 row)
 
     SELECT create_distributed_table('test_table', 'id');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 COMMIT;
@@ -225,8 +221,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").kill()');
 (1 row)
 
 SELECT create_distributed_table('test_table','id',colocate_with=>'temp_table');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
@@ -347,8 +342,7 @@ SELECT citus.mitmproxy('conn.kill()');
 
 BEGIN;
 SELECT create_distributed_table('test_table','id');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 ROLLBACK;
@@ -384,8 +378,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 
 BEGIN;
 SELECT create_distributed_table('test_table','id');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 ROLLBACK;
@@ -463,8 +456,7 @@ SELECT citus.mitmproxy('conn.kill()');
 
 BEGIN;
 SELECT create_distributed_table('test_table','id');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 ROLLBACK;
@@ -496,8 +488,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").kill()');
 
 BEGIN;
 SELECT create_distributed_table('test_table','id');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 ROLLBACK;
@@ -533,8 +524,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 
 BEGIN;
 SELECT create_distributed_table('test_table','id');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 ROLLBACK;
@@ -615,8 +605,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 SELECT master_create_worker_shards('test_table_2', 4, 2);
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT count(*) FROM pg_dist_shard;

src/test/regress/expected/failure_cte_subquery.out

@@ -88,8 +88,7 @@ FROM
      ORDER BY 1 DESC LIMIT 5
      ) as foo
 	  WHERE foo.user_id = cte.user_id;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- kill at the third copy (pull)
@@ -122,8 +121,7 @@ FROM
      ORDER BY 1 DESC LIMIT 5
      ) as foo
 	  WHERE foo.user_id = cte.user_id;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- cancel at the first copy (push)
@@ -262,8 +260,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^DELETE FROM").kill()');
 
 WITH cte_delete as (DELETE FROM users_table WHERE user_name in ('A', 'D') RETURNING *)
 INSERT INTO users_table SELECT * FROM cte_delete;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- verify contents are the same
@@ -378,8 +375,7 @@ BEGIN;
 SET LOCAL citus.multi_shard_modify_mode = 'sequential';
 WITH cte_delete as (DELETE FROM users_table WHERE user_name in ('A', 'D') RETURNING *)
 INSERT INTO users_table SELECT * FROM cte_delete;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 END;

src/test/regress/expected/failure_ddl.out

@@ -37,8 +37,7 @@ SELECT citus.mitmproxy('conn.onAuthenticationOk().kill()');
 (1 row)
 
 ALTER TABLE test_table ADD COLUMN new_column INT;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass;
@@ -71,8 +70,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 ALTER TABLE test_table ADD COLUMN new_column INT;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass;
@@ -104,8 +102,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_apply_shard_ddl_command").kil
 (1 row)
 
 ALTER TABLE test_table ADD COLUMN new_column INT;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- show that we've never commited the changes
@@ -349,8 +346,7 @@ SELECT citus.mitmproxy('conn.onAuthenticationOk().kill()');
 (1 row)
 
 ALTER TABLE test_table DROP COLUMN new_column;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass;
@@ -383,8 +379,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 ALTER TABLE test_table DROP COLUMN new_column;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass;
@@ -416,8 +411,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_apply_shard_ddl_command").kil
 (1 row)
 
 ALTER TABLE test_table DROP COLUMN new_column;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass;
@@ -720,8 +714,7 @@ SELECT citus.mitmproxy('conn.onAuthenticationOk().kill()');
 (1 row)
 
 ALTER TABLE test_table ADD COLUMN new_column INT;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass;
@@ -754,8 +747,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 ALTER TABLE test_table ADD COLUMN new_column INT;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass;
@@ -787,8 +779,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_apply_shard_ddl_command").kil
 (1 row)
 
 ALTER TABLE test_table ADD COLUMN new_column INT;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass;
@@ -1048,8 +1039,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 ALTER TABLE test_table ADD COLUMN new_column INT;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass;
@@ -1081,8 +1071,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_apply_shard_ddl_command").kil
 (1 row)
 
 ALTER TABLE test_table ADD COLUMN new_column INT;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- kill as soon as the coordinator after it sends worker_apply_shard_ddl_command 2nd time
@@ -1093,8 +1082,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_apply_shard_ddl_command").aft
 (1 row)
 
 ALTER TABLE test_table ADD COLUMN new_column INT;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- cancel as soon as the coordinator after it sends worker_apply_shard_ddl_command 2nd time

src/test/regress/expected/failure_distributed_results.out

@@ -85,8 +85,7 @@ CREATE TABLE distributed_result_info AS
   SELECT resultId, nodeport, rowcount, targetShardId, targetShardIndex
   FROM partition_task_list_results('test', $$ SELECT * FROM source_table $$, 'target_table')
           NATURAL JOIN pg_dist_node;
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT * FROM distributed_result_info ORDER BY resultId;

src/test/regress/expected/failure_insert_select_pushdown.out

@@ -44,8 +44,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT INTO insert_select_pushdown"
 (1 row)
 
 INSERT INTO events_summary SELECT user_id, event_id, count(*) FROM events_table GROUP BY 1,2;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 --verify nothing is modified
@@ -98,8 +97,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT INTO insert_select_pushdown"
 (1 row)
 
 INSERT INTO events_table SELECT * FROM events_table;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 --verify nothing is modified

src/test/regress/expected/failure_insert_select_repartition.out

@@ -54,8 +54,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_partition_query_result").kill
 (1 row)
 
 INSERT INTO target_table SELECT * FROM source_table;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT * FROM target_table ORDER BY a;
@@ -70,12 +69,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_partition_query_result").kill
 (1 row)
 
 INSERT INTO target_table SELECT * FROM replicated_source_table;
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT * FROM target_table ORDER BY a;
@@ -106,8 +103,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="fetch_intermediate_results").kill()'
 (1 row)
 
 INSERT INTO target_table SELECT * FROM source_table;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT * FROM target_table ORDER BY a;
@@ -122,8 +118,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="fetch_intermediate_results").kill()'
 (1 row)
 
 INSERT INTO target_table SELECT * FROM replicated_source_table;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT * FROM target_table ORDER BY a;
@@ -143,8 +138,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="read_intermediate_results").kill()')
 (1 row)
 
 INSERT INTO target_table SELECT * FROM source_table;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT * FROM target_table ORDER BY a;
@@ -159,8 +153,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="read_intermediate_results").kill()')
 (1 row)
 
 INSERT INTO target_table SELECT * FROM replicated_source_table;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT * FROM target_table ORDER BY a;
@@ -179,8 +172,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="read_intermediate_results").kill()')
 (1 row)
 
 INSERT INTO replicated_target_table SELECT * FROM source_table;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT * FROM replicated_target_table;

src/test/regress/expected/failure_multi_dml.out

@@ -33,8 +33,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^DELETE").kill()');
 
 BEGIN;
 DELETE FROM dml_test WHERE id = 1;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 DELETE FROM dml_test WHERE id = 2;
@@ -96,8 +95,7 @@ BEGIN;
 DELETE FROM dml_test WHERE id = 1;
 DELETE FROM dml_test WHERE id = 2;
 INSERT INTO dml_test VALUES (5, 'Epsilon');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 UPDATE dml_test SET name = 'alpha' WHERE id = 1;
@@ -154,8 +152,7 @@ DELETE FROM dml_test WHERE id = 1;
 DELETE FROM dml_test WHERE id = 2;
 INSERT INTO dml_test VALUES (5, 'Epsilon');
 UPDATE dml_test SET name = 'alpha' WHERE id = 1;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 UPDATE dml_test SET name = 'gamma' WHERE id = 3;

src/test/regress/expected/failure_multi_row_insert.out

@@ -43,8 +43,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()');
 (1 row)
 
 INSERT INTO distributed_table VALUES (1,1), (1,2), (1,3);
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- this test is broken, see https://github.com/citusdata/citus/issues/2460
@@ -58,8 +57,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()');
 (1 row)
 
 INSERT INTO distributed_table VALUES (1,7), (5,8);
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- this test is broken, see https://github.com/citusdata/citus/issues/2460
@@ -73,8 +71,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()');
 (1 row)
 
 INSERT INTO distributed_table VALUES (1,11), (6,12);
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").cancel(' || :pid || ')');
@@ -93,8 +90,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").after(1).kill()');
 (1 row)
 
 INSERT INTO distributed_table VALUES (1,15), (6,16);
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").after(1).cancel(' || :pid || ')');
@@ -113,8 +109,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()');
 (1 row)
 
 INSERT INTO distributed_table VALUES (2,19),(1,20);
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").cancel(' || :pid || ')');

src/test/regress/expected/failure_multi_shard_update_delete.out

@@ -64,8 +64,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM").kill()');
 
 -- issue a multi shard delete
 DELETE FROM t2 WHERE b = 2;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- verify nothing is deleted
@@ -83,8 +82,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM multi_shard.t2_201005").
 (1 row)
 
 DELETE FROM t2 WHERE b = 2;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- verify nothing is deleted
@@ -146,8 +144,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()');
 
 -- issue a multi shard update
 UPDATE t2 SET c = 4 WHERE b = 2;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- verify nothing is updated
@@ -165,8 +162,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="UPDATE multi_shard.t2_201005").kill(
 (1 row)
 
 UPDATE t2 SET c = 4 WHERE b = 2;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- verify nothing is updated
@@ -222,8 +218,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM").kill()');
 
 -- issue a multi shard delete
 DELETE FROM t2 WHERE b = 2;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- verify nothing is deleted
@@ -241,8 +236,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM multi_shard.t2_201005").
 (1 row)
 
 DELETE FROM t2 WHERE b = 2;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- verify nothing is deleted
@@ -304,8 +298,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()');
 
 -- issue a multi shard update
 UPDATE t2 SET c = 4 WHERE b = 2;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- verify nothing is updated
@@ -323,8 +316,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="UPDATE multi_shard.t2_201005").kill(
 (1 row)
 
 UPDATE t2 SET c = 4 WHERE b = 2;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- verify nothing is updated
@@ -397,8 +389,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM").kill()');
 (1 row)
 
 DELETE FROM r1 WHERE a = 2;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- verify nothing is deleted
@@ -415,8 +406,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM").kill()');
 (1 row)
 
 DELETE FROM t2 WHERE b = 2;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- verify nothing is deleted
@@ -500,8 +490,7 @@ UPDATE t3 SET c = q.c FROM (
 	SELECT b, max(c) as c FROM t2  GROUP BY b) q
 WHERE t3.b = q.b
 RETURNING *;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 --- verify nothing is updated
@@ -559,8 +548,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="UPDATE multi_shard.t3_201013").kill(
 (1 row)
 
 UPDATE t3 SET b = 2 WHERE b = 1;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- verify nothing is updated
@@ -594,8 +582,7 @@ SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FRO
 
 -- following will fail
 UPDATE t3 SET b = 2 WHERE b = 1;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 END;
@@ -613,8 +600,7 @@ SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FRO
 (1 row)
 
 UPDATE t3 SET b = 1 WHERE b = 2 RETURNING *;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- verify nothing is updated
@@ -633,8 +619,7 @@ SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FRO
 (1 row)
 
 UPDATE t3 SET b = 2 WHERE b = 1;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- verify nothing is updated
@@ -668,8 +653,7 @@ SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FRO
 
 -- following will fail
 UPDATE t3 SET b = 2 WHERE b = 1;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 END;

src/test/regress/expected/failure_ref_tables.out

@@ -33,8 +33,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()');
 (1 row)
 
 INSERT INTO ref_table VALUES (5, 6);
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT COUNT(*) FROM ref_table WHERE key=5;
@@ -51,8 +50,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()');
 (1 row)
 
 UPDATE ref_table SET key=7 RETURNING value;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT COUNT(*) FROM ref_table WHERE key=7;
@@ -71,8 +69,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()');
 BEGIN;
 DELETE FROM ref_table WHERE key=5;
 UPDATE ref_table SET key=value;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 COMMIT;

src/test/regress/expected/failure_replicated_partitions.out

@@ -29,8 +29,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()');
 (1 row)
 
 INSERT INTO partitioned_table VALUES (0, 0);
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- use both placements

src/test/regress/expected/failure_savepoints.out

@@ -40,8 +40,7 @@ INSERT INTO artists VALUES (5, 'Asher Lev');
 SAVEPOINT s1;
 WARNING:  connection not open
 CONTEXT:  while executing command on localhost:xxxxx
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: connection not open
-DETAIL:  connection not open
 WARNING:  connection not open
 CONTEXT:  while executing command on localhost:xxxxx
 WARNING:  connection not open
@@ -74,8 +73,7 @@ RELEASE SAVEPOINT s1;
 WARNING:  AbortSubTransaction while in COMMIT state
 WARNING:  connection not open
 CONTEXT:  while executing command on localhost:xxxxx
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: connection not open
-DETAIL:  connection not open
 WARNING:  connection not open
 CONTEXT:  while executing command on localhost:xxxxx
 WARNING:  connection not open
@@ -132,8 +130,7 @@ RELEASE SAVEPOINT s2;
 WARNING:  AbortSubTransaction while in COMMIT state
 WARNING:  connection not open
 CONTEXT:  while executing command on localhost:xxxxx
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: connection not open
-DETAIL:  connection not open
 WARNING:  connection not open
 CONTEXT:  while executing command on localhost:xxxxx
 WARNING:  connection not open
@@ -213,7 +210,7 @@ ROLLBACK TO SAVEPOINT s1;
 WARNING:  connection not open
 WARNING:  connection not open
 WARNING:  connection not open
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: connection not open
 WARNING:  connection not open
 WARNING:  connection not open
 COMMIT;
@@ -248,7 +245,7 @@ BEGIN;
 INSERT INTO researchers VALUES (7, 4, 'Jan Plaza');
 SAVEPOINT s1;
 WARNING:  connection not open
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: connection not open
 WARNING:  connection not open
 WARNING:  connection not open
 ERROR:  connection not open
@@ -321,7 +318,7 @@ ROLLBACK TO s1;
 RELEASE SAVEPOINT s1;
 WARNING:  AbortSubTransaction while in COMMIT state
 WARNING:  connection not open
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: connection not open
 WARNING:  connection not open
 WARNING:  connection not open
 WARNING:  savepoint "savepoint_3" does not exist
@@ -370,10 +367,10 @@ WARNING:  connection not open
 WARNING:  connection not open
 SELECT * FROM ref;
 WARNING:  connection not open
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: connection not open
 WARNING:  connection not open
 WARNING:  connection not open
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: connection not open
 END;
 -- clean up
 SELECT citus.mitmproxy('conn.allow()');

src/test/regress/expected/failure_single_mod.out

@@ -27,8 +27,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()');
 (1 row)
 
 INSERT INTO mod_test VALUES (2, 6);
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT COUNT(*) FROM mod_test WHERE key=2;
@@ -63,8 +62,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()');
 (1 row)
 
 UPDATE mod_test SET value='ok' WHERE key=2 RETURNING key;
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
  key
@@ -102,8 +100,7 @@ INSERT INTO mod_test VALUES (2, 6);
 INSERT INTO mod_test VALUES (2, 7);
 DELETE FROM mod_test WHERE key=2 AND value = '7';
 UPDATE mod_test SET value='ok' WHERE key=2;
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 COMMIT;

src/test/regress/expected/failure_single_select.out

@@ -28,8 +28,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").kill()');
 (1 row)
 
 SELECT * FROM select_test WHERE key = 3;
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
  key |   value
@@ -38,8 +37,7 @@ DETAIL:  server closed the connection unexpectedly
 (1 row)
 
 SELECT * FROM select_test WHERE key = 3;
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
  key |   value
@@ -57,8 +55,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").kill()');
 BEGIN;
 INSERT INTO select_test VALUES (3, 'more data');
 SELECT * FROM select_test WHERE key = 3;
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
  key |   value
@@ -69,8 +66,7 @@ DETAIL:  server closed the connection unexpectedly
 
 INSERT INTO select_test VALUES (3, 'even more data');
 SELECT * FROM select_test WHERE key = 3;
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
  key |     value
@@ -160,8 +156,7 @@ SELECT * FROM select_test WHERE key = 3;
 
 INSERT INTO select_test VALUES (3, 'even more data');
 SELECT * FROM select_test WHERE key = 3;
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
  key |     value
@@ -215,8 +210,7 @@ SELECT * FROM select_test WHERE key = 1;
 (1 row)
 
 SELECT * FROM select_test WHERE key = 1;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 -- now the same test with query cancellation

src/test/regress/expected/failure_truncate.out

@@ -44,8 +44,7 @@ SELECT citus.mitmproxy('conn.onAuthenticationOk().kill()');
 (1 row)
 
 TRUNCATE test_table;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
@@ -102,8 +101,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 TRUNCATE test_table;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
@@ -159,8 +157,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="TRUNCATE TABLE truncate_failure.test
 (1 row)
 
 TRUNCATE test_table;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
@@ -433,8 +430,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^TRUNCATE TABLE").after(2).kill()');
 (1 row)
 
 TRUNCATE reference_table CASCADE;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
@@ -577,8 +573,7 @@ SELECT citus.mitmproxy('conn.onAuthenticationOk().kill()');
 (1 row)
 
 TRUNCATE test_table;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
@@ -635,8 +630,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 TRUNCATE test_table;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
@@ -692,8 +686,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^TRUNCATE TABLE truncate_failure.tes
 (1 row)
 
 TRUNCATE test_table;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
@@ -956,8 +949,7 @@ SELECT citus.mitmproxy('conn.onAuthenticationOk().kill()');
 (1 row)
 
 TRUNCATE test_table;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
@@ -1014,8 +1006,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 TRUNCATE test_table;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
@@ -1071,8 +1062,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="TRUNCATE TABLE truncate_failure.test
 (1 row)
 
 TRUNCATE test_table;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');

src/test/regress/expected/failure_vacuum.out

@@ -31,8 +31,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^VACUUM").kill()');
 (1 row)
 
 VACUUM vacuum_test;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.onQuery(query="^ANALYZE").kill()');
@@ -42,8 +41,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^ANALYZE").kill()');
 (1 row)
 
 ANALYZE vacuum_test;
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT").kill()');
@@ -111,8 +109,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^VACUUM.*other").kill()');
 (1 row)
 
 VACUUM vacuum_test, other_vacuum_test;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
-DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
 SELECT citus.mitmproxy('conn.onQuery(query="^VACUUM.*other").cancel(' ||  pg_backend_pid() || ')');

src/test/regress/expected/multi_modifying_xacts.out

@@ -1178,15 +1178,15 @@ NOTICE:  not propagating ALTER ROLE ... RENAME TO commands to worker nodes
  \c - test_user - :master_port
 -- should fail since the worker doesn't have test_user anymore
 INSERT INTO reference_failure_test VALUES (1, '1');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "test_user" does not exist
 -- the same as the above, but wrapped within a transaction
 BEGIN;
 INSERT INTO reference_failure_test VALUES (1, '1');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "test_user" does not exist
 COMMIT;
 BEGIN;
 COPY reference_failure_test FROM STDIN WITH (FORMAT 'csv');
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "test_user" does not exist
 COMMIT;
 -- show that no data go through the table and shard states are good
 SET client_min_messages to 'ERROR';
@@ -1211,8 +1211,8 @@ ORDER BY s.logicalrelid, sp.shardstate;
 
 BEGIN;
 COPY numbers_hash_failure_test FROM STDIN WITH (FORMAT 'csv');
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "test_user" does not exist
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "test_user" does not exist
 -- some placements are invalid before abort
 SELECT shardid, shardstate, nodename, nodeport
 FROM pg_dist_shard_placement JOIN pg_dist_shard USING (shardid)
@@ -1233,8 +1233,8 @@ ORDER BY shardid, nodeport;
 ABORT;
 -- verify nothing is inserted
 SELECT count(*) FROM numbers_hash_failure_test;
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "test_user" does not exist
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "test_user" does not exist
  count
 ---------------------------------------------------------------------
      0
@@ -1259,8 +1259,8 @@ ORDER BY shardid, nodeport;
 
 BEGIN;
 COPY numbers_hash_failure_test FROM STDIN WITH (FORMAT 'csv');
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "test_user" does not exist
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "test_user" does not exist
 -- check shard states before commit
 SELECT shardid, shardstate, nodename, nodeport
 FROM pg_dist_shard_placement JOIN pg_dist_shard USING (shardid)
@@ -1298,8 +1298,8 @@ ORDER BY shardid, nodeport;
 
 -- verify data is inserted
 SELECT count(*) FROM numbers_hash_failure_test;
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "test_user" does not exist
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "test_user" does not exist
  count
 ---------------------------------------------------------------------
      2
@@ -1312,7 +1312,7 @@ NOTICE:  not propagating ALTER ROLE ... RENAME TO commands to worker nodes
 \c - test_user - :master_port
 -- fails on all shard placements
 INSERT INTO numbers_hash_failure_test VALUES (2,2);
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "test_user" does not exist
 -- connect back to the master with the proper user to continue the tests
 \c - :default_user - :master_port
 SET citus.next_shard_id TO 1200020;

src/test/regress/expected/multi_mx_node_metadata.out

@@ -9,6 +9,7 @@ SET citus.replication_model TO streaming;
 SET citus.shard_count TO 8;
 SET citus.shard_replication_factor TO 1;
 SET citus.replicate_reference_tables_on_activate TO off;
+\set VERBOSITY terse
 -- Simulates a readonly node by setting default_transaction_read_only.
 CREATE FUNCTION mark_node_readonly(hostname TEXT, port INTEGER, isreadonly BOOLEAN)
     RETURNS TEXT
@@ -319,7 +320,6 @@ SET citus.log_remote_commands = ON;
 -- trigger caching for prepared statements
 EXECUTE foo;
 NOTICE:  issuing SELECT count(*) AS count FROM public.dist_table_1_102010 dist_table_1 WHERE (a OPERATOR(pg_catalog.=) 1)
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
  count
 ---------------------------------------------------------------------
      0
@@ -327,7 +327,6 @@ DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 
 EXECUTE foo;
 NOTICE:  issuing SELECT count(*) AS count FROM public.dist_table_1_102010 dist_table_1 WHERE (a OPERATOR(pg_catalog.=) 1)
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
  count
 ---------------------------------------------------------------------
      0
@@ -335,7 +334,6 @@ DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 
 EXECUTE foo;
 NOTICE:  issuing SELECT count(*) AS count FROM public.dist_table_1_102010 dist_table_1 WHERE (a OPERATOR(pg_catalog.=) 1)
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
  count
 ---------------------------------------------------------------------
      0
@@ -343,7 +341,6 @@ DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 
 EXECUTE foo;
 NOTICE:  issuing SELECT count(*) AS count FROM public.dist_table_1_102010 dist_table_1 WHERE (a OPERATOR(pg_catalog.=) 1)
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
  count
 ---------------------------------------------------------------------
      0
@@ -351,7 +348,6 @@ DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 
 EXECUTE foo;
 NOTICE:  issuing SELECT count(*) AS count FROM public.dist_table_1_102010 dist_table_1 WHERE (a OPERATOR(pg_catalog.=) 1)
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
  count
 ---------------------------------------------------------------------
      0
@@ -359,7 +355,6 @@ DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 
 EXECUTE foo;
 NOTICE:  issuing SELECT count(*) AS count FROM public.dist_table_1_102010 dist_table_1 WHERE (a OPERATOR(pg_catalog.=) 1)
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
  count
 ---------------------------------------------------------------------
      0
@@ -367,7 +362,6 @@ DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 
 EXECUTE foo;
 NOTICE:  issuing SELECT count(*) AS count FROM public.dist_table_1_102010 dist_table_1 WHERE (a OPERATOR(pg_catalog.=) 1)
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
  count
 ---------------------------------------------------------------------
      0
@@ -381,7 +375,6 @@ SELECT master_update_node(:nodeid_1, '127.0.0.1', :worker_1_port);
 
 SELECT wait_until_metadata_sync(30000);
 NOTICE:  issuing LISTEN metadata_sync
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
  wait_until_metadata_sync
 ---------------------------------------------------------------------
 
@@ -390,7 +383,6 @@ DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 -- make sure the nodename changed.
 EXECUTE foo;
 NOTICE:  issuing SELECT count(*) AS count FROM public.dist_table_1_102010 dist_table_1 WHERE (a OPERATOR(pg_catalog.=) 1)
-DETAIL:  on server postgres@127.0.0.1:57637 connectionId: xxxxxxx
  count
 ---------------------------------------------------------------------
      0
@@ -500,8 +492,6 @@ UPDATE pg_dist_node SET metadatasynced = TRUE WHERE nodeid IN (:nodeid_1, :nodei
 -- should error out
 SELECT 1 FROM master_disable_node('localhost', 1);
 ERROR:  Disabling localhost:xxxxx failed
-DETAIL:  connection error: localhost:xxxxx
-HINT:  If you are using MX, try stop_metadata_sync_to_node(hostname, port) for nodes that are down before disabling them.
 -- try again after stopping metadata sync
 SELECT stop_metadata_sync_to_node('localhost', 1);
  stop_metadata_sync_to_node
@@ -566,8 +556,6 @@ UPDATE pg_dist_node SET metadatasynced = TRUE WHERE nodeid IN (:nodeid_1, :nodei
 -- should error out
 SELECT 1 FROM master_disable_node('localhost', :worker_2_port);
 ERROR:  Disabling localhost:xxxxx failed
-DETAIL:  connection error: localhost:xxxxx
-HINT:  If you are using MX, try stop_metadata_sync_to_node(hostname, port) for nodes that are down before disabling them.
 -- try again after stopping metadata sync
 SELECT stop_metadata_sync_to_node('localhost', 1);
  stop_metadata_sync_to_node

src/test/regress/expected/multi_replicate_reference_table.out

@@ -1088,10 +1088,24 @@ END;
 -- test adding an invalid node while we have reference tables to replicate
 -- set client message level to ERROR and verbosity to terse to supporess
 -- OS-dependent host name resolution warnings
-SET client_min_messages to ERROR;
 \set VERBOSITY terse
-SELECT master_add_node('invalid-node-name', 9999);
+SET client_min_messages to ERROR;
-ERROR:  connection error: invalid-node-name:9999
+DO $$
+DECLARE
+        errors_received INTEGER;
+BEGIN
+errors_received := 0;
+        BEGIN
+		SELECT master_add_node('invalid-node-name', 9999);
+        EXCEPTION WHEN OTHERS THEN
+                IF SQLERRM LIKE 'connection to the remote node%%' THEN
+                        errors_received := errors_received + 1;
+                END IF;
+        END;
+RAISE '(%/1) failed to add node', errors_received;
+END;
+$$;
+ERROR:  (1/1) failed to add node
 -- drop unnecassary tables
 DROP TABLE initially_not_replicated_reference_table;
 -- reload pg_dist_shard_placement table

src/test/regress/expected/multi_router_planner.out

@@ -2442,8 +2442,7 @@ GRANT INSERT ON ALL TABLES IN SCHEMA public TO router_user;
 -- we will fail to connect to worker 2, since the user does not exist
 BEGIN;
 INSERT INTO failure_test VALUES (1, 1);
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "router_user" does not exist
-DETAIL:  FATAL:  role "router_user" does not exist
 SELECT shardid, shardstate, nodename, nodeport FROM pg_dist_shard_placement
 	WHERE shardid IN (
 		SELECT shardid FROM pg_dist_shard
@@ -2460,8 +2459,7 @@ SELECT shardid, shardstate, nodename, nodeport FROM pg_dist_shard_placement
 
 ROLLBACK;
 INSERT INTO failure_test VALUES (2, 1);
-WARNING:  connection error: localhost:xxxxx
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "router_user" does not exist
-DETAIL:  FATAL:  role "router_user" does not exist
 SELECT shardid, shardstate, nodename, nodeport FROM pg_dist_shard_placement
 	WHERE shardid IN (
 		SELECT shardid FROM pg_dist_shard

src/test/regress/expected/node_conninfo_reload.out

@@ -46,8 +46,7 @@ show citus.node_conninfo;
 
 -- Should give a connection error because of bad sslmode
 select count(*) from test where a = 0;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist"
-DETAIL:  invalid sslmode value: "doesnotexist"
 -- Reset it again
 ALTER SYSTEM RESET citus.node_conninfo;
 select pg_reload_conf();
@@ -114,8 +113,7 @@ select count(*) from test where a = 0;
 COMMIT;
 -- Should fail now with connection error, when transaction is finished
 select count(*) from test where a = 0;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist"
-DETAIL:  invalid sslmode value: "doesnotexist"
 -- Reset it again
 ALTER SYSTEM RESET citus.node_conninfo;
 select pg_reload_conf();
@@ -177,8 +175,7 @@ select count(*) from test where a = 0;
 COMMIT;
 -- Should fail now, when transaction is finished
 select count(*) from test where a = 0;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist"
-DETAIL:  invalid sslmode value: "doesnotexist"
 -- Reset it again
 ALTER SYSTEM RESET citus.node_conninfo;
 select pg_reload_conf();
@@ -231,13 +228,11 @@ show citus.node_conninfo;
 -- Should fail since a different shard is accessed and thus a new connection
 -- will to be created.
 select count(*) from test where a = 0;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist"
-DETAIL:  invalid sslmode value: "doesnotexist"
 COMMIT;
 -- Should still fail now, when transaction is finished
 select count(*) from test where a = 0;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist"
-DETAIL:  invalid sslmode value: "doesnotexist"
 -- Reset it again
 ALTER SYSTEM RESET citus.node_conninfo;
 select pg_reload_conf();
@@ -298,8 +293,7 @@ select count(*) from test;
 COMMIT;
 -- Should fail now, when transaction is finished
 select count(*) from test;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist"
-DETAIL:  invalid sslmode value: "doesnotexist"
 -- Reset it again
 ALTER SYSTEM RESET citus.node_conninfo;
 select pg_reload_conf();
@@ -355,8 +349,7 @@ ERROR:  division by zero
 ROLLBACK;
 -- Should fail now, when transaction is finished
 select count(*) from test;
-ERROR:  connection error: localhost:xxxxx
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: invalid sslmode value: "doesnotexist"
-DETAIL:  invalid sslmode value: "doesnotexist"
 -- Reset it again
 ALTER SYSTEM RESET citus.node_conninfo;
 select pg_reload_conf();

src/test/regress/output/multi_copy.source

@@ -15,7 +15,7 @@ CREATE TABLE customer_copy_hash (
 		primary key (c_custkey));
 SELECT master_create_distributed_table('customer_copy_hash', 'c_custkey', 'hash');
  master_create_distributed_table
----------------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -26,7 +26,7 @@ DETAIL:  No shards exist for distributed table "customer_copy_hash".
 HINT:  Run master_create_worker_shards to create shards and try again.
 SELECT master_create_worker_shards('customer_copy_hash', 64, 1);
  master_create_worker_shards
------------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -39,7 +39,7 @@ CONTEXT:  COPY customer_copy_hash, line 1, column c_custkey: "1,customer1"
 -- Confirm that no data was copied
 SELECT count(*) FROM customer_copy_hash;
  count
--------
+---------------------------------------------------------------------
      0
 (1 row)
 
@@ -51,7 +51,7 @@ DETAIL:  Key (c_custkey)=(2) already exists.
 -- Confirm that no data was copied
 SELECT count(*) FROM customer_copy_hash;
  count
--------
+---------------------------------------------------------------------
      0
 (1 row)
 
@@ -61,7 +61,7 @@ WITH (FORMAT 'csv', HEADER true, FORCE_NULL (c_custkey));
 -- Confirm that only first row was skipped
 SELECT count(*) FROM customer_copy_hash;
  count
--------
+---------------------------------------------------------------------
      3
 (1 row)
 
@@ -71,7 +71,7 @@ WITH (FORMAT 'csv', QUOTE '"', FORCE_NOT_NULL (c_address));
 -- Confirm that value is not null
 SELECT count(c_address) FROM customer_copy_hash WHERE c_custkey = 4;
  count
--------
+---------------------------------------------------------------------
      1
 (1 row)
 
@@ -81,7 +81,7 @@ WITH (FORMAT 'csv', QUOTE '"', FORCE_NULL (c_address));
 -- Confirm that value is null
 SELECT count(c_address) FROM customer_copy_hash WHERE c_custkey = 5;
  count
--------
+---------------------------------------------------------------------
      0
 (1 row)
 
@@ -93,7 +93,7 @@ DETAIL:  Failing row contains (8, null, null, null, null, null, null, null).
 -- Confirm that no data was copied
 SELECT count(*) FROM customer_copy_hash;
  count
--------
+---------------------------------------------------------------------
      5
 (1 row)
 
@@ -103,7 +103,7 @@ WITH (DELIMITER ' ');
 -- Confirm that data was copied
 SELECT count(*) FROM customer_copy_hash WHERE c_custkey = 9;
  count
--------
+---------------------------------------------------------------------
      1
 (1 row)
 
@@ -112,7 +112,7 @@ COPY customer_copy_hash FROM '@abs_srcdir@/data/customer.2.data' WITH (DELIMITER
 -- Confirm that data was copied
 SELECT count(*) FROM customer_copy_hash;
  count
--------
+---------------------------------------------------------------------
   1006
 (1 row)
 
@@ -121,7 +121,7 @@ SELECT count(*) FROM customer_copy_hash;
 -- Confirm that data was copied
 SELECT count(*) FROM customer_copy_hash;
  count
--------
+---------------------------------------------------------------------
   2006
 (1 row)
 
@@ -129,19 +129,19 @@ SELECT count(*) FROM customer_copy_hash;
 -- hash-partitioned tables
 SELECT master_update_shard_statistics(560000);
  master_update_shard_statistics
---------------------------------
+---------------------------------------------------------------------
                            8192
 (1 row)
 
 SELECT shardid, shardminvalue, shardmaxvalue FROM pg_dist_shard WHERE shardid = 560000;
  shardid | shardminvalue | shardmaxvalue
----------+---------------+---------------
+---------------------------------------------------------------------
   560000 | -2147483648   | -2080374785
 (1 row)
 
 SELECT shardid, shardlength FROM pg_dist_shard_placement WHERE shardid = 560000;
  shardid | shardlength
----------+-------------
+---------------------------------------------------------------------
   560000 |        8192
 (1 row)
 
@@ -152,13 +152,13 @@ CREATE TABLE customer_with_default(
         c_time timestamp default now());
 SELECT master_create_distributed_table('customer_with_default', 'c_custkey', 'hash');
  master_create_distributed_table
----------------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
 SELECT master_create_worker_shards('customer_with_default', 64, 1);
  master_create_worker_shards
------------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -168,7 +168,7 @@ WITH (FORMAT 'csv');
 -- Confirm that data was copied with now() function
 SELECT count(*) FROM customer_with_default where c_time IS NOT NULL;
  count
--------
+---------------------------------------------------------------------
      2
 (1 row)
 
@@ -178,7 +178,7 @@ ALTER TABLE customer_copy_hash ADD COLUMN extra2 INT DEFAULT 0;
 COPY customer_copy_hash (c_custkey, c_name, extra1, extra2) FROM STDIN CSV;
 SELECT * FROM customer_copy_hash WHERE extra1 = 1;
  c_custkey |   c_name   | c_address | c_nationkey | c_phone | c_acctbal | c_mktsegment | c_comment | extra1 | extra2
------------+------------+-----------+-------------+---------+-----------+--------------+-----------+--------+--------
+---------------------------------------------------------------------
         10 | customer10 |           |             |         |           |              |           |      1 |      5
 (1 row)
 
@@ -187,7 +187,7 @@ ALTER TABLE customer_copy_hash DROP COLUMN extra1;
 COPY customer_copy_hash (c_custkey, c_name, extra2) FROM STDIN CSV;
 SELECT * FROM customer_copy_hash WHERE c_custkey = 11;
  c_custkey |   c_name   | c_address | c_nationkey | c_phone | c_acctbal | c_mktsegment | c_comment | extra2
------------+------------+-----------+-------------+---------+-----------+--------------+-----------+--------
+---------------------------------------------------------------------
         11 | customer11 |           |             |         |           |              |           |      5
 (1 row)
 
@@ -196,7 +196,7 @@ ALTER TABLE customer_copy_hash DROP COLUMN extra2;
 COPY customer_copy_hash (c_custkey, c_name) FROM STDIN CSV;
 SELECT * FROM customer_copy_hash WHERE c_custkey = 12;
  c_custkey |   c_name   | c_address | c_nationkey | c_phone | c_acctbal | c_mktsegment | c_comment
------------+------------+-----------+-------------+---------+-----------+--------------+-----------
+---------------------------------------------------------------------
         12 | customer12 |           |             |         |           |              |
 (1 row)
 
@@ -213,7 +213,7 @@ CREATE TABLE customer_copy_range (
 		primary key (c_custkey));
 SELECT master_create_distributed_table('customer_copy_range', 'c_custkey', 'range');
  master_create_distributed_table
----------------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -235,14 +235,14 @@ COPY customer_copy_range FROM '@abs_srcdir@/data/customer.1.data' WITH (DELIMITE
 SELECT min(c_custkey), max(c_custkey), avg(c_custkey), count(*)
 FROM customer_copy_range WHERE c_custkey <= 500;
  min | max |         avg          | count
------+-----+----------------------+-------
+---------------------------------------------------------------------
    1 | 500 | 250.5000000000000000 |   500
 (1 row)
 
 -- Check whether data was copied
 SELECT count(*) FROM customer_copy_range;
  count
--------
+---------------------------------------------------------------------
   1000
 (1 row)
 
@@ -251,13 +251,13 @@ UPDATE pg_dist_shard SET shardminvalue = 1501, shardmaxvalue = 2000
 WHERE shardid = :new_shard_id;
 SELECT shardid, shardminvalue, shardmaxvalue FROM pg_dist_shard WHERE shardid = :new_shard_id;
  shardid | shardminvalue | shardmaxvalue
----------+---------------+---------------
+---------------------------------------------------------------------
   560129 | 1501          | 2000
 (1 row)
 
 SELECT shardid, shardlength FROM pg_dist_shard_placement WHERE shardid = :new_shard_id;
  shardid | shardlength
----------+-------------
+---------------------------------------------------------------------
   560129 |           0
   560129 |           0
 (2 rows)
@@ -266,19 +266,19 @@ SELECT shardid, shardlength FROM pg_dist_shard_placement WHERE shardid = :new_sh
 -- shard length is updated.
 SELECT master_update_shard_statistics(:new_shard_id);
  master_update_shard_statistics
---------------------------------
+---------------------------------------------------------------------
                          131072
 (1 row)
 
 SELECT shardid, shardminvalue, shardmaxvalue FROM pg_dist_shard WHERE shardid = :new_shard_id;
  shardid | shardminvalue | shardmaxvalue
----------+---------------+---------------
+---------------------------------------------------------------------
   560129 | 1501          | 2000
 (1 row)
 
 SELECT shardid, shardlength FROM pg_dist_shard_placement WHERE shardid = :new_shard_id;
  shardid | shardlength
----------+-------------
+---------------------------------------------------------------------
   560129 |      131072
   560129 |      131072
 (2 rows)
@@ -298,7 +298,7 @@ CREATE TABLE customer_copy_append (
         c_comment varchar(117));
 SELECT master_create_distributed_table('customer_copy_append', 'c_custkey', 'append');
  master_create_distributed_table
----------------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -309,7 +309,7 @@ CONTEXT:  COPY customer_copy_append, line 3, column c_custkey: "notinteger"
 -- Test that no shard is created for failing copy
 SELECT count(*) FROM pg_dist_shard WHERE logicalrelid = 'customer_copy_append'::regclass;
  count
--------
+---------------------------------------------------------------------
      0
 (1 row)
 
@@ -318,7 +318,7 @@ COPY customer_copy_append FROM STDIN;
 -- Test that no shard is created for copying zero rows
 SELECT count(*) FROM pg_dist_shard WHERE logicalrelid = 'customer_copy_append'::regclass;
  count
--------
+---------------------------------------------------------------------
      0
 (1 row)
 
@@ -327,7 +327,7 @@ COPY customer_copy_append(c_custkey, c_name) FROM STDIN WITH (FORMAT 'csv');
 -- Check whether data was copied properly
 SELECT * FROM customer_copy_append;
  c_custkey |  c_name   | c_address | c_nationkey | c_phone | c_acctbal | c_mktsegment | c_comment
------------+-----------+-----------+-------------+---------+-----------+--------------+-----------
+---------------------------------------------------------------------
          1 | customer1 |           |             |         |           |              |
          2 | customer2 |           |             |         |           |              |
 (2 rows)
@@ -337,13 +337,13 @@ UPDATE pg_dist_shard SET shardminvalue = 1501, shardmaxvalue = 2000 WHERE shardi
 UPDATE pg_dist_shard_placement SET shardlength = 0 WHERE shardid = 560131;
 SELECT shardid, shardminvalue, shardmaxvalue FROM pg_dist_shard WHERE shardid = 560131;
  shardid | shardminvalue | shardmaxvalue
----------+---------------+---------------
+---------------------------------------------------------------------
   560131 | 1501          | 2000
 (1 row)
 
 SELECT shardid, shardlength FROM pg_dist_shard_placement WHERE shardid = 560131;
  shardid | shardlength
----------+-------------
+---------------------------------------------------------------------
   560131 |           0
   560131 |           0
 (2 rows)
@@ -351,19 +351,19 @@ SELECT shardid, shardlength FROM pg_dist_shard_placement WHERE shardid = 560131;
 -- Update shard statistics for append-partitioned shard
 SELECT master_update_shard_statistics(560131);
  master_update_shard_statistics
---------------------------------
+---------------------------------------------------------------------
                            8192
 (1 row)
 
 SELECT shardid, shardminvalue, shardmaxvalue FROM pg_dist_shard WHERE shardid = 560131;
  shardid | shardminvalue | shardmaxvalue
----------+---------------+---------------
+---------------------------------------------------------------------
   560131 | 1             | 2
 (1 row)
 
 SELECT shardid, shardlength FROM pg_dist_shard_placement WHERE shardid = 560131;
  shardid | shardlength
----------+-------------
+---------------------------------------------------------------------
   560131 |        8192
   560131 |        8192
 (2 rows)
@@ -388,7 +388,7 @@ CREATE TABLE lineitem_copy_append (
         l_comment varchar(44) not null);
 SELECT master_create_distributed_table('lineitem_copy_append', 'l_orderkey', 'append');
  master_create_distributed_table
----------------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -397,7 +397,7 @@ SET citus.shard_max_size TO '256kB';
 COPY lineitem_copy_append FROM '@abs_srcdir@/data/lineitem.1.data' with delimiter '|';
 SELECT count(*) FROM pg_dist_shard WHERE logicalrelid = 'lineitem_copy_append'::regclass;
  count
--------
+---------------------------------------------------------------------
      5
 (1 row)
 
@@ -418,7 +418,7 @@ ORDER BY
 LIMIT
         5;
  shardid | nodeport
----------+----------
+---------------------------------------------------------------------
   560141 |    57637
   560140 |    57638
   560139 |    57637
@@ -439,7 +439,7 @@ CREATE TABLE append.customer_copy (
         c_comment varchar(117));
 SELECT master_create_distributed_table('append.customer_copy', 'c_custkey', 'append');
  master_create_distributed_table
----------------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -449,7 +449,7 @@ COPY append.customer_copy FROM '@abs_srcdir@/data/customer.2.data' with (delimit
 -- Test the content of the table
 SELECT min(c_custkey), max(c_custkey), avg(c_acctbal), count(*) FROM append.customer_copy;
  min | max  |          avg          | count
------+------+-----------------------+-------
+---------------------------------------------------------------------
    1 | 7000 | 4443.8028800000000000 |  2000
 (1 row)
 
@@ -459,13 +459,13 @@ CREATE TABLE "customer_with_special_\\_character"(
         c_name varchar(25) not null);
 SELECT master_create_distributed_table('"customer_with_special_\\_character"', 'c_custkey', 'hash');
  master_create_distributed_table
----------------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
 SELECT master_create_worker_shards('"customer_with_special_\\_character"', 4, 1);
  master_create_worker_shards
------------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -474,7 +474,7 @@ WITH (FORMAT 'csv');
 -- Confirm that data was copied
 SELECT count(*) FROM "customer_with_special_\\_character";
  count
--------
+---------------------------------------------------------------------
      2
 (1 row)
 
@@ -484,13 +484,13 @@ CREATE TABLE "1_customer"(
         c_name varchar(25) not null);
 SELECT master_create_distributed_table('"1_customer"', 'c_custkey', 'hash');
  master_create_distributed_table
----------------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
 SELECT master_create_worker_shards('"1_customer"', 4, 1);
  master_create_worker_shards
------------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -499,7 +499,7 @@ WITH (FORMAT 'csv');
 -- Confirm that data was copied
 SELECT count(*) FROM "1_customer";
  count
--------
+---------------------------------------------------------------------
      2
 (1 row)
 
@@ -519,13 +519,13 @@ CREATE TABLE packed_numbers_hash (
 );
 SELECT master_create_distributed_table('packed_numbers_hash', 'id', 'hash');
  master_create_distributed_table
----------------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
 SELECT master_create_worker_shards('packed_numbers_hash', 4, 1);
  master_create_worker_shards
------------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -534,7 +534,7 @@ COPY packed_numbers_hash FROM :'temp_dir''copy_test_array_of_composite';
 -- Verify data is actually copied
 SELECT * FROM packed_numbers_hash;
  id |    packed_numbers
-----+-----------------------
+---------------------------------------------------------------------
   1 | {"(42,42)","(42,42)"}
 (1 row)
 
@@ -545,13 +545,13 @@ CREATE TABLE super_packed_numbers_hash (
 );
 SELECT master_create_distributed_table('super_packed_numbers_hash', 'id', 'hash');
  master_create_distributed_table
----------------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
 SELECT master_create_worker_shards('super_packed_numbers_hash', 4, 1);
  master_create_worker_shards
------------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -560,7 +560,7 @@ COPY super_packed_numbers_hash FROM :'temp_dir''copy_test_composite_of_composite
 -- Verify data is actually copied
 SELECT * FROM super_packed_numbers_hash;
  id |  super_packed_number
-----+-----------------------
+---------------------------------------------------------------------
   1 | ("(42,42)","(42,42)")
 (1 row)
 
@@ -571,7 +571,7 @@ CREATE TABLE packed_numbers_append (
 );
 SELECT master_create_distributed_table('packed_numbers_append', 'id', 'append');
  master_create_distributed_table
----------------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -579,7 +579,7 @@ COPY packed_numbers_append FROM :'temp_dir''copy_test_array_of_composite';
 -- Verify data is actually copied
 SELECT * FROM packed_numbers_append;
  id |    packed_numbers
-----+-----------------------
+---------------------------------------------------------------------
   1 | {"(42,42)","(42,42)"}
 (1 row)
 
@@ -590,7 +590,7 @@ CREATE TABLE super_packed_numbers_append (
 );
 SELECT master_create_distributed_table('super_packed_numbers_append', 'id', 'append');
  master_create_distributed_table
----------------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -598,7 +598,7 @@ COPY super_packed_numbers_append FROM :'temp_dir''copy_test_composite_of_composi
 -- Verify data is actually copied
 SELECT * FROM super_packed_numbers_append;
  id |  super_packed_number
-----+-----------------------
+---------------------------------------------------------------------
   1 | ("(42,42)","(42,42)")
 (1 row)
 
@@ -609,22 +609,22 @@ CREATE TABLE composite_partition_column_table(
 );
 SELECT master_create_distributed_table('composite_partition_column_table', 'composite_column', 'append');
  master_create_distributed_table
----------------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
 \COPY composite_partition_column_table FROM STDIN WITH (FORMAT 'csv');
 WARNING:  function min(number_pack) does not exist
 HINT:  No function matches the given name and argument types. You might need to add explicit type casts.
-CONTEXT:  while executing command on localhost:57638
+CONTEXT:  while executing command on localhost:xxxxx
 WARNING:  could not get statistics for shard public.composite_partition_column_table_560162
 DETAIL:  Setting shard statistics to NULL
-ERROR:  failure on connection marked as essential: localhost:57638
+ERROR:  failure on connection marked as essential: localhost:xxxxx
 -- Test copy on append distributed tables do not create shards on removed workers
 CREATE TABLE numbers_append (a int, b int);
 SELECT master_create_distributed_table('numbers_append', 'a', 'append');
  master_create_distributed_table
----------------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -633,7 +633,7 @@ SELECT shardid, nodename, nodeport
 	FROM pg_dist_shard_placement join pg_dist_shard using(shardid)
 	WHERE logicalrelid = 'numbers_append'::regclass order by placementid;
  shardid | nodename | nodeport
----------+----------+----------
+---------------------------------------------------------------------
 (0 rows)
 
 COPY numbers_append FROM STDIN WITH (FORMAT 'csv');
@@ -643,16 +643,16 @@ SELECT shardid, nodename, nodeport
 	FROM pg_dist_shard_placement join pg_dist_shard using(shardid)
 	WHERE logicalrelid = 'numbers_append'::regclass order by placementid;
  shardid | nodename  | nodeport
----------+-----------+----------
+---------------------------------------------------------------------
   560163 | localhost |    57637
   560164 | localhost |    57638
 (2 rows)
 
 -- disable the first node
 SELECT master_disable_node('localhost', :worker_1_port);
-NOTICE:  Node localhost:57637 has active shard placements. Some queries may fail after this operation. Use SELECT master_activate_node('localhost', 57637) to activate this node back.
+NOTICE:  Node localhost:xxxxx has active shard placements. Some queries may fail after this operation. Use SELECT master_activate_node('localhost', 57637) to activate this node back.
  master_disable_node
----------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -666,7 +666,7 @@ SELECT shardid, nodename, nodeport
 	FROM pg_dist_shard_placement join pg_dist_shard using(shardid)
 	WHERE logicalrelid = 'numbers_append'::regclass order by placementid;
  shardid | nodename  | nodeport
----------+-----------+----------
+---------------------------------------------------------------------
   560163 | localhost |    57637
   560164 | localhost |    57638
   560165 | localhost |    57638
@@ -677,7 +677,7 @@ SELECT shardid, nodename, nodeport
 SET client_min_messages TO ERROR;
 SELECT 1 FROM master_activate_node('localhost', :worker_1_port);
  ?column?
-----------
+---------------------------------------------------------------------
         1
 (1 row)
 
@@ -690,7 +690,7 @@ SELECT shardid, nodename, nodeport
 	FROM pg_dist_shard_placement join pg_dist_shard using(shardid)
 	WHERE logicalrelid = 'numbers_append'::regclass order by placementid;
  shardid | nodename  | nodeport
----------+-----------+----------
+---------------------------------------------------------------------
   560163 | localhost |    57637
   560164 | localhost |    57638
   560165 | localhost |    57638
@@ -709,7 +709,7 @@ NOTICE:  not propagating CREATE ROLE/USER commands to worker nodes
 HINT:  Connect to worker nodes directly to manually create all necessary users and roles.
 SELECT * FROM run_command_on_workers('CREATE USER test_user');
  nodename  | nodeport | success |   result
------------+----------+---------+-------------
+---------------------------------------------------------------------
  localhost |    57637 | t       | CREATE ROLE
  localhost |    57638 | t       | CREATE ROLE
 (2 rows)
@@ -719,7 +719,7 @@ SET citus.shard_count to 4;
 CREATE TABLE numbers_hash (a int, b int);
 SELECT create_distributed_table('numbers_hash', 'a');
  create_distributed_table
---------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -729,7 +729,7 @@ SELECT shardid, shardstate, nodename, nodeport
 	FROM pg_dist_shard_placement join pg_dist_shard using(shardid)
 	WHERE logicalrelid = 'numbers_hash'::regclass order by shardid, nodeport;
  shardid | shardstate | nodename  | nodeport
----------+------------+-----------+----------
+---------------------------------------------------------------------
   560169 |          1 | localhost |    57637
   560169 |          1 | localhost |    57638
   560170 |          1 | localhost |    57637
@@ -744,7 +744,7 @@ SELECT shardid, shardstate, nodename, nodeport
 CREATE TABLE numbers_reference(a int, b int);
 SELECT create_reference_table('numbers_reference');
  create_reference_table
-------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -753,7 +753,7 @@ COPY numbers_reference FROM STDIN WITH (FORMAT 'csv');
 CREATE TABLE numbers_hash_other(a int, b int);
 SELECT create_distributed_table('numbers_hash_other', 'a');
  create_distributed_table
---------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -761,7 +761,7 @@ SELECT shardid, shardstate, nodename, nodeport
 	FROM pg_dist_shard_placement join pg_dist_shard using(shardid)
 	WHERE logicalrelid = 'numbers_hash_other'::regclass order by shardid, nodeport;
  shardid | shardstate | nodename  | nodeport
----------+------------+-----------+----------
+---------------------------------------------------------------------
   560174 |          1 | localhost |    57637
   560174 |          1 | localhost |    57638
   560175 |          1 | localhost |    57637
@@ -783,24 +783,20 @@ ALTER USER test_user WITH nologin;
 \c - test_user - :master_port
 -- reissue copy
 COPY numbers_hash FROM STDIN WITH (FORMAT 'csv');
-WARNING:  connection error: localhost:57637
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "test_user" is not permitted to log in
-DETAIL:  FATAL:  role "test_user" is not permitted to log in
 CONTEXT:  COPY numbers_hash, line 1: "1,1"
-WARNING:  connection error: localhost:57637
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "test_user" is not permitted to log in
-DETAIL:  FATAL:  role "test_user" is not permitted to log in
 CONTEXT:  COPY numbers_hash, line 2: "2,2"
-WARNING:  connection error: localhost:57637
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "test_user" is not permitted to log in
-DETAIL:  FATAL:  role "test_user" is not permitted to log in
 CONTEXT:  COPY numbers_hash, line 3: "3,3"
-WARNING:  connection error: localhost:57637
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "test_user" is not permitted to log in
-DETAIL:  FATAL:  role "test_user" is not permitted to log in
 CONTEXT:  COPY numbers_hash, line 6: "6,6"
 -- verify shards in the first worker as marked invalid
 SELECT shardid, shardstate, nodename, nodeport
 	FROM pg_dist_shard_placement join pg_dist_shard using(shardid)
 	WHERE logicalrelid = 'numbers_hash'::regclass order by shardid, nodeport;
  shardid | shardstate | nodename  | nodeport
----------+------------+-----------+----------
+---------------------------------------------------------------------
   560169 |          3 | localhost |    57637
   560169 |          1 | localhost |    57638
   560170 |          3 | localhost |    57637
@@ -813,15 +809,14 @@ SELECT shardid, shardstate, nodename, nodeport
 
 -- try to insert into a reference table copy should fail
 COPY numbers_reference FROM STDIN WITH (FORMAT 'csv');
-ERROR:  connection error: localhost:57637
+ERROR:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "test_user" is not permitted to log in
-DETAIL:  FATAL:  role "test_user" is not permitted to log in
 CONTEXT:  COPY numbers_reference, line 1: "3,1"
 -- verify shards for reference table are still valid
 SELECT shardid, shardstate, nodename, nodeport
 	FROM pg_dist_shard_placement join pg_dist_shard using(shardid)
 	WHERE logicalrelid = 'numbers_reference'::regclass order by placementid;
  shardid | shardstate | nodename  | nodeport
----------+------------+-----------+----------
+---------------------------------------------------------------------
   560173 |          1 | localhost |    57637
   560173 |          1 | localhost |    57638
 (2 rows)
@@ -830,14 +825,11 @@ SELECT shardid, shardstate, nodename, nodeport
 -- since it can not insert into either copies of a shard. shards are expected to
 -- stay valid since the operation is rolled back.
 COPY numbers_hash_other FROM STDIN WITH (FORMAT 'csv');
-WARNING:  connection error: localhost:57637
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "test_user" is not permitted to log in
-DETAIL:  FATAL:  role "test_user" is not permitted to log in
 CONTEXT:  COPY numbers_hash_other, line 1: "1,1"
-WARNING:  connection error: localhost:57637
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "test_user" is not permitted to log in
-DETAIL:  FATAL:  role "test_user" is not permitted to log in
 CONTEXT:  COPY numbers_hash_other, line 2: "2,2"
-WARNING:  connection error: localhost:57637
+WARNING:  connection to the remote node localhost:xxxxx failed with the following error: FATAL:  role "test_user" is not permitted to log in
-DETAIL:  FATAL:  role "test_user" is not permitted to log in
 CONTEXT:  COPY numbers_hash_other, line 3: "3,3"
 -- verify shards for numbers_hash_other are still valid
 -- since copy has failed altogether
@@ -845,7 +837,7 @@ SELECT shardid, shardstate, nodename, nodeport
 	FROM pg_dist_shard_placement join pg_dist_shard using(shardid)
 	WHERE logicalrelid = 'numbers_hash_other'::regclass order by shardid, nodeport;
  shardid | shardstate | nodename  | nodeport
----------+------------+-----------+----------
+---------------------------------------------------------------------
   560174 |          3 | localhost |    57637
   560174 |          1 | localhost |    57638
   560175 |          3 | localhost |    57637
@@ -873,7 +865,7 @@ SET citus.shard_count to 4;
 CREATE TABLE numbers_hash(a int, b int);
 SELECT create_distributed_table('numbers_hash', 'a');
  create_distributed_table
---------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -883,12 +875,12 @@ ALTER TABLE numbers_hash_560180 DROP COLUMN b;
 -- operation will fail to modify a shard and roll back
 COPY numbers_hash FROM STDIN WITH (FORMAT 'csv');
 ERROR:  column "b" of relation "numbers_hash_560180" does not exist
-CONTEXT:  while executing command on localhost:57637
+CONTEXT:  while executing command on localhost:xxxxx
 COPY numbers_hash, line 6: "6,6"
 -- verify no row is inserted
 SELECT count(a) FROM numbers_hash;
  count
--------
+---------------------------------------------------------------------
      0
 (1 row)
 
@@ -897,7 +889,7 @@ SELECT shardid, shardstate, nodename, nodeport
 	FROM pg_dist_shard_placement join pg_dist_shard using(shardid)
 	WHERE logicalrelid = 'numbers_hash'::regclass order by shardid, nodeport;
  shardid | shardstate | nodename  | nodeport
----------+------------+-----------+----------
+---------------------------------------------------------------------
   560178 |          1 | localhost |    57637
   560178 |          1 | localhost |    57638
   560179 |          1 | localhost |    57637
@@ -911,7 +903,7 @@ SELECT shardid, shardstate, nodename, nodeport
 DROP TABLE numbers_hash;
 SELECT * FROM run_command_on_workers('DROP USER test_user');
  nodename  | nodeport | success |  result
------------+----------+---------+-----------
+---------------------------------------------------------------------
  localhost |    57637 | t       | DROP ROLE
  localhost |    57638 | t       | DROP ROLE
 (2 rows)
@@ -924,14 +916,14 @@ col2 character varying(255) NOT NULL
 );
 SELECT create_reference_table('test_binaryless_builtin');
  create_reference_table
-------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
 \COPY test_binaryless_builtin FROM STDIN WITH (format CSV)
 SELECT * FROM test_binaryless_builtin;
         col1         | col2
----------------------+-------
+---------------------------------------------------------------------
  postgres=r/postgres |  test
 (1 row)
 
@@ -941,7 +933,7 @@ BEGIN;
 CREATE TABLE tt1(id int);
 SELECT create_distributed_table('tt1','id');
  create_distributed_table
---------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -952,7 +944,7 @@ END;
 CREATE TABLE drop_copy_test_table (col1 int, col2 int, col3 int, col4 int);
 SELECT create_distributed_table('drop_copy_test_table','col3');
  create_distributed_table
---------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -960,7 +952,7 @@ ALTER TABLE drop_copy_test_table drop column col1;
 COPY drop_copy_test_table (col2,col3,col4) from STDIN with CSV;
 SELECT * FROM drop_copy_test_table WHERE col3 = 1;
  col2 | col3 | col4
-------+------+------
+---------------------------------------------------------------------
       |    1 |
 (1 row)
 
@@ -968,7 +960,7 @@ ALTER TABLE drop_copy_test_table drop column col4;
 COPY drop_copy_test_table (col2,col3) from STDIN with CSV;
 SELECT * FROM drop_copy_test_table WHERE col3 = 1;
  col2 | col3
-------+------
+---------------------------------------------------------------------
       |    1
       |    1
 (2 rows)
@@ -978,7 +970,7 @@ DROP TABLE drop_copy_test_table;
 \c - - - :worker_1_port
 SELECT relname FROM pg_class WHERE relname LIKE 'tt1%';
  relname
----------
+---------------------------------------------------------------------
 (0 rows)
 
 \c - - - :master_port
@@ -992,7 +984,7 @@ NOTICE:  copying the data has completed
 DETAIL:  The local data in the table is longer visible, but is still on disk.
 HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.trigger_flush$$)
  create_distributed_table
---------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -1004,7 +996,7 @@ SET citus.multi_shard_modify_mode TO 'sequential';
 CREATE UNLOGGED TABLE trigger_switchover(a int, b int, c int, d int, e int, f int, g int, h int);
 SELECT create_distributed_table('trigger_switchover','a');
  create_distributed_table
---------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -1015,7 +1007,7 @@ ABORT;
 CREATE TABLE copy_jsonb (key text, value jsonb, extra jsonb default '["default"]'::jsonb);
 SELECT create_distributed_table('copy_jsonb', 'key', colocate_with => 'none');
  create_distributed_table
---------------------------
+---------------------------------------------------------------------
 
 (1 row)
 
@@ -1023,7 +1015,7 @@ SELECT create_distributed_table('copy_jsonb', 'key', colocate_with => 'none');
 \COPY copy_jsonb (key, value) FROM STDIN
 SELECT * FROM copy_jsonb ORDER BY key;
   key  |           value            |    extra
--------+----------------------------+-------------
+---------------------------------------------------------------------
  blue  | {"b": 255, "g": 0, "r": 0} | ["default"]
  green | {"b": 0, "g": 255, "r": 0} | ["default"]
 (2 rows)
@@ -1033,7 +1025,7 @@ COPY copy_jsonb TO :'temp_dir''copy_jsonb.pgcopy' WITH (format binary);
 COPY copy_jsonb FROM :'temp_dir''copy_jsonb.pgcopy' WITH (format binary);
 SELECT * FROM copy_jsonb ORDER BY key;
   key  |           value            |    extra
--------+----------------------------+-------------
+---------------------------------------------------------------------
  blue  | {"b": 255, "g": 0, "r": 0} | ["default"]
  blue  | {"b": 255, "g": 0, "r": 0} | ["default"]
  green | {"b": 0, "g": 255, "r": 0} | ["default"]
@@ -1042,7 +1034,7 @@ SELECT * FROM copy_jsonb ORDER BY key;
 
 -- JSONB parsing error without validation: no line number
 \COPY copy_jsonb (key, value) FROM STDIN
-ERROR:  invalid input syntax for type json
+ERROR:  invalid input syntax for json
 DETAIL:  The input string ended unexpectedly.
 CONTEXT:  JSON data, line 1: {"r":255,"g":0,"b":0
 COPY copy_jsonb, line 1, column value: "{"r":255,"g":0,"b":0"
@@ -1052,7 +1044,7 @@ SET citus.skip_jsonb_validation_in_copy TO off;
 \COPY copy_jsonb (key, value) FROM STDIN
 SELECT * FROM copy_jsonb ORDER BY key;
   key  |           value            |    extra
--------+----------------------------+-------------
+---------------------------------------------------------------------
  blue  | {"b": 255, "g": 0, "r": 0} | ["default"]
  green | {"b": 0, "g": 255, "r": 0} | ["default"]
 (2 rows)
@@ -1062,7 +1054,7 @@ COPY copy_jsonb TO :'temp_dir''copy_jsonb.pgcopy' WITH (format binary);
 COPY copy_jsonb FROM :'temp_dir''copy_jsonb.pgcopy' WITH (format binary);
 SELECT * FROM copy_jsonb ORDER BY key;
   key  |           value            |    extra
--------+----------------------------+-------------
+---------------------------------------------------------------------
  blue  | {"b": 255, "g": 0, "r": 0} | ["default"]
  blue  | {"b": 255, "g": 0, "r": 0} | ["default"]
  green | {"b": 0, "g": 255, "r": 0} | ["default"]
@@ -1071,7 +1063,7 @@ SELECT * FROM copy_jsonb ORDER BY key;
 
 -- JSONB parsing error with validation: should see line number
 \COPY copy_jsonb (key, value) FROM STDIN
-ERROR:  invalid input syntax for type json
+ERROR:  invalid input syntax for json
 DETAIL:  The input string ended unexpectedly.
 CONTEXT:  JSON data, line 1: {"r":255,"g":0,"b":0
 COPY copy_jsonb, line 1, column value: "{"r":255,"g":0,"b":0"

src/test/regress/sql/multi_mx_node_metadata.sql

@@ -13,6 +13,8 @@ SET citus.shard_count TO 8;
 SET citus.shard_replication_factor TO 1;
 SET citus.replicate_reference_tables_on_activate TO off;
 
+\set VERBOSITY terse
+
 -- Simulates a readonly node by setting default_transaction_read_only.
 CREATE FUNCTION mark_node_readonly(hostname TEXT, port INTEGER, isreadonly BOOLEAN)
     RETURNS TEXT

src/test/regress/sql/multi_replicate_reference_table.sql

@@ -669,10 +669,23 @@ END;
 -- test adding an invalid node while we have reference tables to replicate
 -- set client message level to ERROR and verbosity to terse to supporess
 -- OS-dependent host name resolution warnings
-SET client_min_messages to ERROR;
 \set VERBOSITY terse
-
+SET client_min_messages to ERROR;
-SELECT master_add_node('invalid-node-name', 9999);
+DO $$
+DECLARE
+        errors_received INTEGER;
+BEGIN
+errors_received := 0;
+        BEGIN
+		SELECT master_add_node('invalid-node-name', 9999);
+        EXCEPTION WHEN OTHERS THEN
+                IF SQLERRM LIKE 'connection to the remote node%%' THEN
+                        errors_received := errors_received + 1;
+                END IF;
+        END;
+RAISE '(%/1) failed to add node', errors_received;
+END;
+$$;
 
 -- drop unnecassary tables
 DROP TABLE initially_not_replicated_reference_table;