Remove real-time/router executors (step 1) (#3125)

See #3125 for details on each item.

* Remove real-time/router executor tests-1

These are the ones which doesn't have '_%d' in the test
output files.

* Remove real-time/router executor tests-2

These are the ones which has in the test
output files.

* Move the tests outputs to correct place

* Make sure that single shard commits use 2PC on adaptive executor

It looks like we've messed the tests in #2891. Fixing back.

* Use adaptive executor for all router queries

This becomes important because when task-tracker is picked, we
used to pick router executor, which doesn't make sense.

* Remove explicit references to real-time/router executors in the tests

* JobExecutorType never picks real-time/router executors

* Make sure to go incremental in test output numbers

* Even users cannot pick real-time anymore

* Do not use real-time/router custom scans

* Get rid of unnecessary normalizations

* Reflect unneeded normalizations

* Get rid of unnecessary test output file

src/backend/distributed/executor/multi_server_executor.c

@@ -45,14 +45,10 @@ MultiExecutorType
 JobExecutorType(DistributedPlan *distributedPlan)
 {
 	Job *job = distributedPlan->workerJob;
-	List *workerNodeList = NIL;
-	int workerNodeCount = 0;
-	int taskCount = 0;
-	double tasksPerNode = 0.;
 	MultiExecutorType executorType = TaskExecutorType;
 	bool routerExecutablePlan = distributedPlan->routerExecutable;
 
-	/* check if can switch to router executor */
+	/* debug distribution column value */
 	if (routerExecutablePlan)
 	{
 		if (log_min_messages <= DEBUG2 || client_min_messages <= DEBUG2)
@@ -76,12 +72,7 @@ JobExecutorType(DistributedPlan *distributedPlan)
 			}
 		}
 
-		if (TaskExecutorType == MULTI_EXECUTOR_ADAPTIVE)
+		return MULTI_EXECUTOR_ADAPTIVE;
-		{
-			return TaskExecutorType;
-		}
-
-		return MULTI_EXECUTOR_ROUTER;
 	}
 
 	if (distributedPlan->insertSelectSubquery != NULL)
@@ -97,43 +88,7 @@ JobExecutorType(DistributedPlan *distributedPlan)
 
 	Assert(distributedPlan->modLevel == ROW_MODIFY_READONLY);
 
-	workerNodeList = ActiveReadableNodeList();
+	if (executorType == MULTI_EXECUTOR_ADAPTIVE)
-	workerNodeCount = list_length(workerNodeList);
-	taskCount = list_length(job->taskList);
-	tasksPerNode = taskCount / ((double) workerNodeCount);
-
-	if (executorType == MULTI_EXECUTOR_REAL_TIME)
-	{
-		double reasonableConnectionCount = 0;
-
-		/* if we need to open too many connections per worker, warn the user */
-		if (tasksPerNode >= MaxConnections)
-		{
-			ereport(WARNING, (errmsg("this query uses more connections than the "
-									 "configured max_connections limit"),
-							  errhint("Consider increasing max_connections or setting "
-									  "citus.task_executor_type to "
-									  "\"task-tracker\".")));
-		}
-
-		/*
-		 * If we need to open too many outgoing connections, warn the user.
-		 * The real-time executor caps the number of tasks it starts by the same limit,
-		 * but we still issue this warning because it degrades performance.
-		 */
-		reasonableConnectionCount = MaxMasterConnectionCount();
-		if (taskCount >= reasonableConnectionCount)
-		{
-			ereport(WARNING, (errmsg("this query uses more file descriptors than the "
-									 "configured max_files_per_process limit"),
-							  errhint("Consider increasing max_files_per_process or "
-									  "setting citus.task_executor_type to "
-									  "\"task-tracker\".")));
-		}
-	}
-
-	if (executorType == MULTI_EXECUTOR_REAL_TIME ||
-		executorType == MULTI_EXECUTOR_ADAPTIVE)
 	{
 		/* if we have repartition jobs with real time executor and repartition
 		 * joins are not enabled, error out. Otherwise, switch to task-tracker
@@ -158,6 +113,11 @@ JobExecutorType(DistributedPlan *distributedPlan)
 	}
 	else
 	{
+		List *workerNodeList = ActiveReadableNodeList();
+		int workerNodeCount = list_length(workerNodeList);
+		int taskCount = list_length(job->taskList);
+		double tasksPerNode = taskCount / ((double) workerNodeCount);
+
 		/* if we have more tasks per node than what can be tracked, warn the user */
 		if (tasksPerNode >= MaxTrackedTasksPerNode)
 		{

src/backend/distributed/planner/distributed_planner.c

@@ -984,24 +984,12 @@ FinalizePlan(PlannedStmt *localPlan, DistributedPlan *distributedPlan)
 			break;
 		}
 
-		case MULTI_EXECUTOR_REAL_TIME:
-		{
-			customScan->methods = &RealTimeCustomScanMethods;
-			break;
-		}
-
 		case MULTI_EXECUTOR_TASK_TRACKER:
 		{
 			customScan->methods = &TaskTrackerCustomScanMethods;
 			break;
 		}
 
-		case MULTI_EXECUTOR_ROUTER:
-		{
-			customScan->methods = &RouterCustomScanMethods;
-			break;
-		}
-
 		case MULTI_EXECUTOR_COORDINATOR_INSERT_SELECT:
 		{
 			customScan->methods = &CoordinatorInsertSelectCustomScanMethods;

src/backend/distributed/shared_library_init.c

@@ -112,7 +112,7 @@ static const struct config_enum_entry replication_model_options[] = {
 
 static const struct config_enum_entry task_executor_type_options[] = {
 	{ "adaptive", MULTI_EXECUTOR_ADAPTIVE, false },
-	{ "real-time", MULTI_EXECUTOR_REAL_TIME, false },
+	{ "real-time", MULTI_EXECUTOR_ADAPTIVE, false }, /* ignore real-time executor, always use adaptive */
 	{ "task-tracker", MULTI_EXECUTOR_TASK_TRACKER, false },
 	{ NULL, 0, false }
 };

src/test/regress/bin/normalize.sed

@@ -16,6 +16,7 @@ s/node group [12] (but|does)/node group \1/
 
 # Differing names can have differing table column widths
 s/(-+\|)+-+/---/g
+s/.*-------------.*/---------------------------------------------------------------------/g
 
 # In foreign_key_to_reference_table, normalize shard table names, etc in
 # the generated plan
@@ -47,14 +48,6 @@ s/name_len_12345678901234567890123456789012345678_fcd8ab6f_[0-9]+/name_len_12345
 # normalize pkey constraints in multi_insert_select.sql
 s/"(raw_events_second_user_id_value_1_key_|agg_events_user_id_value_1_agg_key_)[0-9]+"/"\1xxxxxxx"/g
 
-# normalize explain outputs, basically wipeout the executor name from the output
-s/.*Custom Scan \(Citus.*/Custom Scan \(Citus\)/g
-s/.*-------------.*/---------------------------------------------------------------------/g
-s/.* QUERY PLAN .*/                              QUERY PLAN                              /g
-s/.*Custom Plan Provider.*Citus.*/              \"Custom Plan Provider\": \"Citus\",     /g
-s/.*Custom-Plan-Provide.*/\<Custom-Plan-Provider\>Citus Unified\<\/Custom-Plan-Provider\>     /g
-s/ +$//g
-
 # normalize failed task ids
 s/ERROR:  failed to execute task [0-9]+/ERROR:  failed to execute task X/g
 

src/test/regress/expected/failure_1pc_copy_append_9.out

@@ -1,294 +0,0 @@
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
--- do not cache any connections
-SET citus.max_cached_conns_per_worker TO 1;
-SET citus.shard_count = 1;
-SET citus.shard_replication_factor = 2; -- one shard per worker
-SET citus.multi_shard_commit_protocol TO '1pc';
-SET citus.next_shard_id TO 100400;
-ALTER SEQUENCE pg_catalog.pg_dist_placement_placementid_seq RESTART 100;
-CREATE TABLE copy_test (key int, value int);
-SELECT create_distributed_table('copy_test', 'key', 'append');
- create_distributed_table 
---------------------------
- 
-(1 row)
-
-SELECT citus.clear_network_traffic();
- clear_network_traffic 
------------------------
- 
-(1 row)
-
-COPY copy_test FROM PROGRAM 'echo 0, 0 && echo 1, 1 && echo 2, 4 && echo 3, 9' WITH CSV;
-SELECT count(1) FROM copy_test;
- count 
--------
-     4
-(1 row)
-
-SELECT citus.dump_network_traffic();
-                                                                                                                                                                                                                                                                                                                                                                                              dump_network_traffic                                                                                                                                                                                                                                                                                                                                                                                               
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
- (0,coordinator,"[initial message]")
- (0,worker,"['AuthenticationOk()', 'ParameterStatus(application_name=citus)', 'ParameterStatus(client_encoding=UTF8)', 'ParameterStatus(DateStyle=ISO, MDY)', 'ParameterStatus(integer_datetimes=on)', 'ParameterStatus(IntervalStyle=postgres)', 'ParameterStatus(is_superuser=on)', 'ParameterStatus(server_encoding=UTF8)', 'ParameterStatus(server_version=XXX)', 'ParameterStatus(session_authorization=postgres)', 'ParameterStatus(standard_conforming_strings=on)', 'ParameterStatus(TimeZone=XXX)', 'BackendKeyData(XXX)', 'ReadyForQuery(state=idle)']")
- (0,coordinator,"[""Query(query=SELECT worker_apply_shard_ddl_command (100400, 'CREATE TABLE public.copy_test (key integer, value integer)'))""]")
- (0,worker,"[""RowDescription(fieldcount=1,fields=['F(name=worker_apply_shard_ddl_command,tableoid=0,colattrnum=0,typoid=2278,typlen=4,typmod=-1,format_code=0)'])"", 'DataRow(columncount=1,columns=[""C(length=0,value=b\\'\\')""])', 'CommandComplete(command=SELECT 1)', 'ReadyForQuery(state=idle)']")
- (0,coordinator,"[""Query(query=SELECT worker_apply_shard_ddl_command (100400, 'ALTER TABLE public.copy_test OWNER TO postgres'))""]")
- (0,worker,"[""RowDescription(fieldcount=1,fields=['F(name=worker_apply_shard_ddl_command,tableoid=0,colattrnum=0,typoid=2278,typlen=4,typmod=-1,format_code=0)'])"", 'DataRow(columncount=1,columns=[""C(length=0,value=b\\'\\')""])', 'CommandComplete(command=SELECT 1)', 'ReadyForQuery(state=idle)']")
- (0,coordinator,"[""Query(query=BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(0, XX, 'XXXX-XX-XX XX:XX:XX.XXXXXX-XX');)""]")
- (0,worker,"['CommandComplete(command=BEGIN)', ""RowDescription(fieldcount=1,fields=['F(name=assign_distributed_transaction_id,tableoid=0,colattrnum=0,typoid=2278,typlen=4,typmod=-1,format_code=0)'])"", 'DataRow(columncount=1,columns=[""C(length=0,value=b\\'\\')""])', 'CommandComplete(command=SELECT 1)', 'ReadyForQuery(state=in_transaction_block)']")
- (0,coordinator,"['Query(query=COPY public.copy_test_XXXXXX FROM STDIN WITH (FORMAT BINARY))']")
- (0,worker,"[""Backend(type=G,body=b'\\\\x01\\\\x00\\\\x02\\\\x00\\\\x01\\\\x00\\\\x01')""]")
- (0,coordinator,"[""CopyData(data=b'PGCOPY\\\\n\\\\xff\\\\r\\\\n\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00')"", ""CopyData(data=b'\\\\x00\\\\x02\\\\x00\\\\x00\\\\x00\\\\x04\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x04\\\\x00\\\\x00\\\\x00\\\\x00')"", ""CopyData(data=b'\\\\x00\\\\x02\\\\x00\\\\x00\\\\x00\\\\x04\\\\x00\\\\x00\\\\x00\\\\x01\\\\x00\\\\x00\\\\x00\\\\x04\\\\x00\\\\x00\\\\x00\\\\x01')"", ""CopyData(data=b'\\\\x00\\\\x02\\\\x00\\\\x00\\\\x00\\\\x04\\\\x00\\\\x00\\\\x00\\\\x02\\\\x00\\\\x00\\\\x00\\\\x04\\\\x00\\\\x00\\\\x00\\\\x04')"", ""CopyData(data=b'\\\\x00\\\\x02\\\\x00\\\\x00\\\\x00\\\\x04\\\\x00\\\\x00\\\\x00\\\\x03\\\\x00\\\\x00\\\\x00\\\\x04\\\\x00\\\\x00\\\\x00\\\\t')"", ""CopyData(data=b'\\\\xff\\\\xff')"", 'CopyDone()']")
- (0,worker,"['CommandComplete(command=COPY 4)', 'ReadyForQuery(state=in_transaction_block)']")
- (0,coordinator,"[""Query(query=SELECT pg_table_size('public.copy_test_XXXXXX'))""]")
- (0,worker,"[""RowDescription(fieldcount=1,fields=['F(name=pg_table_size,tableoid=0,colattrnum=0,typoid=20,typlen=8,typmod=-1,format_code=0)'])"", 'DataRow(columncount=1,columns=[""C(length=0,value=b\\'\\')""])', 'CommandComplete(command=SELECT 1)', 'ReadyForQuery(state=in_transaction_block)']")
- (0,coordinator,"['Query(query=SELECT min(key), max(key) FROM public.copy_test_XXXXXX)']")
- (0,worker,"[""RowDescription(fieldcount=2,fields=['F(name=min,tableoid=0,colattrnum=0,typoid=23,typlen=4,typmod=-1,format_code=0)', 'F(name=max,tableoid=0,colattrnum=0,typoid=23,typlen=4,typmod=-1,format_code=0)'])"", 'DataRow(columncount=2,columns=[""C(length=0,value=b\\'\\')"", ""C(length=1,value=b\\'0\\')""])', 'CommandComplete(command=SELECT 1)', 'ReadyForQuery(state=in_transaction_block)']")
- (0,coordinator,"['Query(query=COMMIT)']")
- (0,worker,"['CommandComplete(command=COMMIT)', 'ReadyForQuery(state=idle)']")
- (0,coordinator,"['Query(query=COPY (SELECT count(1) AS count FROM copy_test_100400 copy_test WHERE true) TO STDOUT)']")
- (0,worker,"[""CopyOutResponse(format=0,columncount=1,columns=['Anonymous(format=0)'])"", ""CopyData(data=b'4\\\\n')"", 'CopyDone()', 'CommandComplete(command=COPY 1)', 'ReadyForQuery(state=idle)']")
-(20 rows)
-
----- all of the following tests test behavior with 2 shard placements ----
-SHOW citus.shard_replication_factor;
- citus.shard_replication_factor 
---------------------------------
- 2
-(1 row)
-
----- kill the connection when we try to create the shard ----
-SELECT citus.mitmproxy('conn.onQuery(query="worker_apply_shard_ddl_command").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-COPY copy_test FROM PROGRAM 'echo 0, 0 && echo 1, 1 && echo 2, 4 && echo 3, 9' WITH CSV;
-ERROR:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
-SELECT * FROM pg_dist_shard s, pg_dist_shard_placement p
-  WHERE (s.shardid = p.shardid) AND s.logicalrelid = 'copy_test'::regclass
-  ORDER BY placementid;
- logicalrelid | shardid | shardstorage | shardminvalue | shardmaxvalue | shardid | shardstate | shardlength | nodename  | nodeport | placementid 
---------------+---------+--------------+---------------+---------------+---------+------------+-------------+-----------+----------+-------------
- copy_test    |  100400 | t            | 0             | 3             |  100400 |          1 |        8192 | localhost |    57637 |         100
- copy_test    |  100400 | t            | 0             | 3             |  100400 |          1 |        8192 | localhost |     9060 |         101
-(2 rows)
-
-SELECT count(1) FROM copy_test;
- count 
--------
-     4
-(1 row)
-
----- kill the connection when we try to start a transaction ----
-SELECT citus.mitmproxy('conn.onQuery(query="assign_distributed_transaction_id").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-COPY copy_test FROM PROGRAM 'echo 0, 0 && echo 1, 1 && echo 2, 4 && echo 3, 9' WITH CSV;
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
-ERROR:  failure on connection marked as essential: localhost:9060
-SELECT * FROM pg_dist_shard s, pg_dist_shard_placement p
-  WHERE (s.shardid = p.shardid) AND s.logicalrelid = 'copy_test'::regclass
-  ORDER BY placementid;
- logicalrelid | shardid | shardstorage | shardminvalue | shardmaxvalue | shardid | shardstate | shardlength | nodename  | nodeport | placementid 
---------------+---------+--------------+---------------+---------------+---------+------------+-------------+-----------+----------+-------------
- copy_test    |  100400 | t            | 0             | 3             |  100400 |          1 |        8192 | localhost |    57637 |         100
- copy_test    |  100400 | t            | 0             | 3             |  100400 |          1 |        8192 | localhost |     9060 |         101
-(2 rows)
-
-SELECT count(1) FROM copy_test;
- count 
--------
-     4
-(1 row)
-
----- kill the connection when we start the COPY ----
-SELECT citus.mitmproxy('conn.onQuery(query="FROM STDIN WITH").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-COPY copy_test FROM PROGRAM 'echo 0, 0 && echo 1, 1 && echo 2, 4 && echo 3, 9' WITH CSV;
-ERROR:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
-SELECT * FROM pg_dist_shard s, pg_dist_shard_placement p
-  WHERE (s.shardid = p.shardid) AND s.logicalrelid = 'copy_test'::regclass
-  ORDER BY placementid;
- logicalrelid | shardid | shardstorage | shardminvalue | shardmaxvalue | shardid | shardstate | shardlength | nodename  | nodeport | placementid 
---------------+---------+--------------+---------------+---------------+---------+------------+-------------+-----------+----------+-------------
- copy_test    |  100400 | t            | 0             | 3             |  100400 |          1 |        8192 | localhost |    57637 |         100
- copy_test    |  100400 | t            | 0             | 3             |  100400 |          1 |        8192 | localhost |     9060 |         101
-(2 rows)
-
-SELECT count(1) FROM copy_test;
- count 
--------
-     4
-(1 row)
-
----- kill the connection when we send the data ----
-SELECT citus.mitmproxy('conn.onCopyData().kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-COPY copy_test FROM PROGRAM 'echo 0, 0 && echo 1, 1 && echo 2, 4 && echo 3, 9' WITH CSV;
-ERROR:  failed to COPY to shard 100404 on localhost:9060
-SELECT * FROM pg_dist_shard s, pg_dist_shard_placement p
-  WHERE (s.shardid = p.shardid) AND s.logicalrelid = 'copy_test'::regclass
-  ORDER BY placementid;
- logicalrelid | shardid | shardstorage | shardminvalue | shardmaxvalue | shardid | shardstate | shardlength | nodename  | nodeport | placementid 
---------------+---------+--------------+---------------+---------------+---------+------------+-------------+-----------+----------+-------------
- copy_test    |  100400 | t            | 0             | 3             |  100400 |          1 |        8192 | localhost |    57637 |         100
- copy_test    |  100400 | t            | 0             | 3             |  100400 |          1 |        8192 | localhost |     9060 |         101
-(2 rows)
-
-SELECT citus.mitmproxy('conn.onQuery(query="SELECT|COPY").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT count(1) FROM copy_test;
-WARNING:  could not consume data from worker node
- count 
--------
-     4
-(1 row)
-
----- cancel the connection when we send the data ----
-SELECT citus.mitmproxy('conn.onQuery(query="SELECT|COPY").cancel(' ||  pg_backend_pid() || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-COPY copy_test FROM PROGRAM 'echo 0, 0 && echo 1, 1 && echo 2, 4 && echo 3, 9' WITH CSV;
-ERROR:  canceling statement due to user request
-SELECT * FROM pg_dist_shard s, pg_dist_shard_placement p
-  WHERE (s.shardid = p.shardid) AND s.logicalrelid = 'copy_test'::regclass
-  ORDER BY placementid;
- logicalrelid | shardid | shardstorage | shardminvalue | shardmaxvalue | shardid | shardstate | shardlength | nodename  | nodeport | placementid 
---------------+---------+--------------+---------------+---------------+---------+------------+-------------+-----------+----------+-------------
- copy_test    |  100400 | t            | 0             | 3             |  100400 |          1 |        8192 | localhost |    57637 |         100
- copy_test    |  100400 | t            | 0             | 3             |  100400 |          1 |        8192 | localhost |     9060 |         101
-(2 rows)
-
-SELECT count(1) FROM copy_test;
-ERROR:  canceling statement due to user request
----- kill the connection when we try to get the size of the table ----
-SELECT citus.mitmproxy('conn.onQuery(query="pg_table_size").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-COPY copy_test FROM PROGRAM 'echo 0, 0 && echo 1, 1 && echo 2, 4 && echo 3, 9' WITH CSV;
-WARNING:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
-ERROR:  failure on connection marked as essential: localhost:9060
-SELECT * FROM pg_dist_shard s, pg_dist_shard_placement p
-  WHERE (s.shardid = p.shardid) AND s.logicalrelid = 'copy_test'::regclass
-  ORDER BY placementid;
- logicalrelid | shardid | shardstorage | shardminvalue | shardmaxvalue | shardid | shardstate | shardlength | nodename  | nodeport | placementid 
---------------+---------+--------------+---------------+---------------+---------+------------+-------------+-----------+----------+-------------
- copy_test    |  100400 | t            | 0             | 3             |  100400 |          1 |        8192 | localhost |    57637 |         100
- copy_test    |  100400 | t            | 0             | 3             |  100400 |          1 |        8192 | localhost |     9060 |         101
-(2 rows)
-
-SELECT count(1) FROM copy_test;
- count 
--------
-     4
-(1 row)
-
----- kill the connection when we try to get the min, max of the table ----
-SELECT citus.mitmproxy('conn.onQuery(query="SELECT min\(key\), max\(key\)").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-COPY copy_test FROM PROGRAM 'echo 0, 0 && echo 1, 1 && echo 2, 4 && echo 3, 9' WITH CSV;
-WARNING:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
-ERROR:  failure on connection marked as essential: localhost:9060
-SELECT * FROM pg_dist_shard s, pg_dist_shard_placement p
-  WHERE (s.shardid = p.shardid) AND s.logicalrelid = 'copy_test'::regclass
-  ORDER BY placementid;
- logicalrelid | shardid | shardstorage | shardminvalue | shardmaxvalue | shardid | shardstate | shardlength | nodename  | nodeport | placementid 
---------------+---------+--------------+---------------+---------------+---------+------------+-------------+-----------+----------+-------------
- copy_test    |  100400 | t            | 0             | 3             |  100400 |          1 |        8192 | localhost |    57637 |         100
- copy_test    |  100400 | t            | 0             | 3             |  100400 |          1 |        8192 | localhost |     9060 |         101
-(2 rows)
-
-SELECT count(1) FROM copy_test;
- count 
--------
-     4
-(1 row)
-
----- kill the connection when we try to COMMIT ----
-SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-COPY copy_test FROM PROGRAM 'echo 0, 0 && echo 1, 1 && echo 2, 4 && echo 3, 9' WITH CSV;
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
-WARNING:  failed to commit transaction on localhost:9060
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
-SELECT * FROM pg_dist_shard s, pg_dist_shard_placement p
-  WHERE (s.shardid = p.shardid) AND s.logicalrelid = 'copy_test'::regclass
-  ORDER BY placementid;
- logicalrelid | shardid | shardstorage | shardminvalue | shardmaxvalue | shardid | shardstate | shardlength | nodename  | nodeport | placementid 
---------------+---------+--------------+---------------+---------------+---------+------------+-------------+-----------+----------+-------------
- copy_test    |  100400 | t            | 0             | 3             |  100400 |          1 |        8192 | localhost |    57637 |         100
- copy_test    |  100400 | t            | 0             | 3             |  100400 |          1 |        8192 | localhost |     9060 |         101
- copy_test    |  100408 | t            | 0             | 3             |  100408 |          1 |        8192 | localhost |    57637 |         112
- copy_test    |  100408 | t            | 0             | 3             |  100408 |          3 |        8192 | localhost |     9060 |         113
-(4 rows)
-
-SELECT count(1) FROM copy_test;
- count 
--------
-     8
-(1 row)
-
--- ==== Clean up, we're done here ====
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-DROP TABLE copy_test;

src/test/regress/expected/failure_connection_establishment.out

@@ -44,8 +44,7 @@ SELECT citus.mitmproxy('conn.delay(500)');
 (1 row)
 
 ALTER TABLE products ADD CONSTRAINT p_key PRIMARY KEY(product_no);
-WARNING:  could not establish connection after 400 ms
+ERROR:  could not establish any connections to the node localhost:9060 after 400 ms
-ERROR:  connection error: localhost:9060
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy 
 -----------
@@ -170,7 +169,7 @@ SELECT citus.mitmproxy('conn.delay(500)');
 (1 row)
 
 SELECT count(*) FROM single_replicatated;
-ERROR:  failed to execute task 1
+ERROR:  could not establish any connections to the node localhost:9060 after 400 ms
 SET citus.force_max_query_parallelization TO OFF;
 -- one similar test, but this time on modification queries
 -- to see that connection establishement failures could

src/test/regress/expected/failure_connection_establishment_9.out

@@ -1,244 +0,0 @@
---
--- failure_connection_establishment.sql tests some behaviour of connection management when
--- it fails to connect.
---
--- Failure cases covered:
---  - timeout
---
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-CREATE SCHEMA fail_connect;
-SET search_path TO 'fail_connect';
-SET citus.shard_count TO 4;
-SET citus.max_cached_conns_per_worker TO 0;
-ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 1450000;
-ALTER SEQUENCE pg_catalog.pg_dist_placement_placementid_seq RESTART 1450000;
-CREATE TABLE products (
-    product_no integer,
-    name text,
-    price numeric
-);
-SELECT create_distributed_table('products', 'product_no');
- create_distributed_table 
---------------------------
- 
-(1 row)
-
--- Can only add primary key constraint on distribution column (or group of columns
--- including distribution column)
--- Command below should error out since 'name' is not a distribution column
-ALTER TABLE products ADD CONSTRAINT p_key PRIMARY KEY(name);
-ERROR:  cannot create constraint on "products"
-DETAIL:  Distributed relations cannot have UNIQUE, EXCLUDE, or PRIMARY KEY constraints that do not include the partition column (with an equality operator if EXCLUDE).
--- we will insert a connection delay here as this query was the cause for an investigation
--- into connection establishment problems
-SET citus.node_connection_timeout TO 400;
-SELECT citus.mitmproxy('conn.delay(500)');
- mitmproxy 
------------
- 
-(1 row)
-
-ALTER TABLE products ADD CONSTRAINT p_key PRIMARY KEY(product_no);
-ERROR:  could not establish any connections to the node localhost:9060 after 400 ms
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-CREATE TABLE r1 (
-    id int PRIMARY KEY,
-    name text
-);
-INSERT INTO r1 (id, name) VALUES
-(1,'foo'),
-(2,'bar'),
-(3,'baz');
-SELECT create_reference_table('r1');
-NOTICE:  Copying data from local table...
- create_reference_table 
-------------------------
- 
-(1 row)
-
-SELECT citus.clear_network_traffic();
- clear_network_traffic 
------------------------
- 
-(1 row)
-
-SELECT citus.mitmproxy('conn.delay(500)');
- mitmproxy 
------------
- 
-(1 row)
-
--- we cannot control which replica of the reference table will be queried and there is
--- only one specific client we can control the connection for.
--- by using round-robin task_assignment_policy we can force to hit both machines. 
--- and in the end, dumping the network traffic shows that the connection establishment
--- is initiated to the node behind the proxy
-SET client_min_messages TO ERROR;
-SET citus.task_assignment_policy TO 'round-robin';
--- suppress the warning since we can't control which shard is chose first. Failure of this
--- test would be if one of the queries does not return the result but an error.
-SELECT name FROM r1 WHERE id = 2;
- name 
-------
- bar
-(1 row)
-
-SELECT name FROM r1 WHERE id = 2;
- name 
-------
- bar
-(1 row)
-
--- verify a connection attempt was made to the intercepted node, this would have cause the
--- connection to have been delayed and thus caused a timeout
-SELECT citus.dump_network_traffic();
-        dump_network_traffic         
--------------------------------------
- (0,coordinator,"[initial message]")
-(1 row)
-
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
--- similar test with the above but this time on a 
--- distributed table instead of a reference table
--- and with citus.force_max_query_parallelization is set
-SET citus.force_max_query_parallelization TO ON;
-SELECT citus.mitmproxy('conn.delay(500)');
- mitmproxy 
------------
- 
-(1 row)
-
--- suppress the warning since we can't control which shard is chose first. Failure of this
--- test would be if one of the queries does not return the result but an error.
-SELECT count(*) FROM products;
- count 
--------
-     0
-(1 row)
-
-SELECT count(*) FROM products;
- count 
--------
-     0
-(1 row)
-
--- use OFFSET 1 to prevent printing the line where source 
--- is the worker
-SELECT citus.dump_network_traffic() ORDER BY 1 OFFSET 1;
-        dump_network_traffic         
--------------------------------------
- (1,coordinator,"[initial message]")
-(1 row)
-
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SET citus.shard_replication_factor TO 1;
-CREATE TABLE single_replicatated(key int);
-SELECT create_distributed_table('single_replicatated', 'key');
- create_distributed_table 
---------------------------
- 
-(1 row)
-
--- this time the table is single replicated and we're still using the
--- the max parallelization flag, so the query should fail
-SET citus.force_max_query_parallelization TO ON;
-SELECT citus.mitmproxy('conn.delay(500)');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT count(*) FROM single_replicatated;
-ERROR:  could not establish any connections to the node localhost:9060 after 400 ms
-SET citus.force_max_query_parallelization TO OFF;
--- one similar test, but this time on modification queries
--- to see that connection establishement failures could
--- mark placement INVALID
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-SELECT 
-	count(*) as invalid_placement_count
-FROM 
-	pg_dist_shard_placement 
-WHERE 
-	shardstate = 3 AND 
-	shardid IN (SELECT shardid from pg_dist_shard where logicalrelid = 'products'::regclass);
- invalid_placement_count 
--------------------------
-                       0
-(1 row)
-
-SELECT citus.mitmproxy('conn.delay(500)');
- mitmproxy 
------------
- 
-(1 row)
-
-INSERT INTO products VALUES (100, '100', 100);
-COMMIT;
-SELECT 
-	count(*) as invalid_placement_count
-FROM 
-	pg_dist_shard_placement 
-WHERE 
-	shardstate = 3 AND 
-	shardid IN (SELECT shardid from pg_dist_shard where logicalrelid = 'products'::regclass);
- invalid_placement_count 
--------------------------
-                       1
-(1 row)
-
--- show that INSERT went through
-SELECT count(*) FROM products WHERE product_no = 100;
- count 
--------
-     1
-(1 row)
-
-RESET client_min_messages;
--- verify get_global_active_transactions works when a timeout happens on a connection
-SELECT get_global_active_transactions();
-WARNING:  could not establish connection after 400 ms
-WARNING:  connection error: localhost:9060
- get_global_active_transactions 
---------------------------------
-(0 rows)
-
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SET citus.node_connection_timeout TO DEFAULT;
-DROP SCHEMA fail_connect CASCADE;
-NOTICE:  drop cascades to 3 other objects
-DETAIL:  drop cascades to table products
-drop cascades to table r1
-drop cascades to table single_replicatated
-SET search_path TO default;

src/test/regress/expected/failure_create_distributed_table_non_empty.out

@@ -125,10 +125,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 SELECT create_distributed_table('test_table', 'id');
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
 ERROR:  connection error: localhost:9060
-DETAIL:  connection not open
+DETAIL:  server closed the connection unexpectedly
+	This probably means the server terminated abnormally
+	before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy 
 -----------
@@ -159,13 +159,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 SELECT create_distributed_table('test_table', 'id');
-WARNING:  cancel requests are ignored during shard creation
+ERROR:  canceling statement due to user request
-NOTICE:  Copying data from local table...
- create_distributed_table 
---------------------------
- 
-(1 row)
-
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy 
 -----------
@@ -175,7 +169,7 @@ SELECT citus.mitmproxy('conn.allow()');
 SELECT count(*) FROM pg_dist_shard WHERE logicalrelid='create_distributed_table_non_empty_failure.test_table'::regclass;
  count 
 -------
-     4
+     0
 (1 row)
 
 SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.schemata WHERE schema_name = 'create_distributed_table_non_empty_failure'$$);
@@ -196,10 +190,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").kill()');
 (1 row)
 
 SELECT create_distributed_table('test_table', 'id');
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 SELECT count(*) FROM pg_dist_shard WHERE logicalrelid='create_distributed_table_non_empty_failure.test_table'::regclass;
  count 
 -------
@@ -474,7 +468,9 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 
 SELECT create_distributed_table('test_table', 'id', colocate_with => 'colocated_table');
 ERROR:  connection error: localhost:9060
-DETAIL:  connection not open
+DETAIL:  server closed the connection unexpectedly
+	This probably means the server terminated abnormally
+	before or while processing the request.
 SELECT count(*) FROM pg_dist_shard WHERE logicalrelid='create_distributed_table_non_empty_failure.test_table'::regclass;
  count 
 -------
@@ -559,10 +555,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="^SELECT worker_apply_shard_ddl_comma
 (1 row)
 
 SELECT create_distributed_table('test_table', 'id', colocate_with => 'colocated_table');
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 SELECT count(*) FROM pg_dist_shard WHERE logicalrelid='create_distributed_table_non_empty_failure.test_table'::regclass;
  count 
 -------
@@ -666,7 +662,9 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 
 SELECT create_distributed_table('test_table', 'id');
 ERROR:  connection error: localhost:9060
-DETAIL:  connection not open
+DETAIL:  server closed the connection unexpectedly
+	This probably means the server terminated abnormally
+	before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy 
 -----------
@@ -697,11 +695,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 SELECT create_distributed_table('test_table', 'id');
- create_distributed_table 
+ERROR:  canceling statement due to user request
---------------------------
- 
-(1 row)
-
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy 
 -----------
@@ -711,7 +705,7 @@ SELECT citus.mitmproxy('conn.allow()');
 SELECT count(*) FROM pg_dist_shard WHERE logicalrelid='create_distributed_table_non_empty_failure.test_table'::regclass;
  count 
 -------
-     4
+     0
 (1 row)
 
 SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.schemata WHERE schema_name = 'create_distributed_table_non_empty_failure'$$);
@@ -732,10 +726,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").kill()');
 (1 row)
 
 SELECT create_distributed_table('test_table', 'id');
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 SELECT count(*) FROM pg_dist_shard WHERE logicalrelid='create_distributed_table_non_empty_failure.test_table'::regclass;
  count 
 -------
@@ -947,7 +941,9 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 
 SELECT create_distributed_table('test_table', 'id', colocate_with => 'colocated_table');
 ERROR:  connection error: localhost:9060
-DETAIL:  connection not open
+DETAIL:  server closed the connection unexpectedly
+	This probably means the server terminated abnormally
+	before or while processing the request.
 SELECT count(*) FROM pg_dist_shard WHERE logicalrelid='create_distributed_table_non_empty_failure.test_table'::regclass;
  count 
 -------

src/test/regress/expected/failure_create_reference_table.out

@@ -25,10 +25,10 @@ SELECT citus.mitmproxy('conn.onQuery().kill()');
 (1 row)
 
 SELECT create_reference_table('ref_table');
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
 ERROR:  connection error: localhost:9060
-DETAIL:  connection not open
+DETAIL:  server closed the connection unexpectedly
+	This probably means the server terminated abnormally
+	before or while processing the request.
 SELECT count(*) FROM pg_dist_shard_placement;
  count 
 -------
@@ -43,10 +43,10 @@ SELECT citus.mitmproxy('conn.onCommandComplete(command="BEGIN").kill()');
 (1 row)
 
 SELECT create_reference_table('ref_table');
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
 ERROR:  connection error: localhost:9060
-DETAIL:  connection not open
+DETAIL:  server closed the connection unexpectedly
+	This probably means the server terminated abnormally
+	before or while processing the request.
 SELECT count(*) FROM pg_dist_shard_placement;
  count 
 -------
@@ -76,10 +76,10 @@ SELECT citus.mitmproxy('conn.onCommandComplete(command="SELECT 1").kill()');
 (1 row)
 
 SELECT create_reference_table('ref_table');
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
 ERROR:  connection error: localhost:9060
-DETAIL:  connection not open
+DETAIL:  server closed the connection unexpectedly
+	This probably means the server terminated abnormally
+	before or while processing the request.
 SELECT count(*) FROM pg_dist_shard_placement;
  count 
 -------

src/test/regress/expected/failure_create_reference_table_9.out

@@ -1,255 +0,0 @@
---
--- Failure tests for creating reference table
---
-CREATE SCHEMA failure_reference_table;
-SET search_path TO 'failure_reference_table';
-SET citus.next_shard_id TO 10000000;
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
--- this is merely used to get the schema creation propagated. Without there are failures
--- not related to reference tables but schema creation due to dependency creation on workers
-CREATE TYPE schema_proc AS (a int);
-DROP TYPE schema_proc;
-CREATE TABLE ref_table(id int);
-INSERT INTO ref_table VALUES(1),(2),(3);
--- Kill on sending first query to worker node, should error
--- out and not create any placement
-SELECT citus.mitmproxy('conn.onQuery().kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT create_reference_table('ref_table');
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-SELECT count(*) FROM pg_dist_shard_placement;
- count 
--------
-     0
-(1 row)
-
--- Kill after creating transaction on worker node
-SELECT citus.mitmproxy('conn.onCommandComplete(command="BEGIN").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT create_reference_table('ref_table');
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-SELECT count(*) FROM pg_dist_shard_placement;
- count 
--------
-     0
-(1 row)
-
--- Cancel after creating transaction on worker node
-SELECT citus.mitmproxy('conn.onCommandComplete(command="BEGIN").cancel(' || pg_backend_pid() || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT create_reference_table('ref_table');
-ERROR:  canceling statement due to user request
-SELECT count(*) FROM pg_dist_shard_placement;
- count 
--------
-     0
-(1 row)
-
--- Kill after copying data to worker node
-SELECT citus.mitmproxy('conn.onCommandComplete(command="SELECT 1").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT create_reference_table('ref_table');
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-SELECT count(*) FROM pg_dist_shard_placement;
- count 
--------
-     0
-(1 row)
-
--- Cancel after copying data to worker node
-SELECT citus.mitmproxy('conn.onCommandComplete(command="SELECT 1").cancel(' || pg_backend_pid() || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT create_reference_table('ref_table');
-ERROR:  canceling statement due to user request
-SELECT count(*) FROM pg_dist_shard_placement;
- count 
--------
-     0
-(1 row)
-
--- Kill after copying data to worker node
-SELECT citus.mitmproxy('conn.onCommandComplete(command="COPY 3").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT create_reference_table('ref_table');
-NOTICE:  Copying data from local table...
-ERROR:  failed to COPY to shard 10000005 on localhost:9060
-SELECT count(*) FROM pg_dist_shard_placement;
- count 
--------
-     0
-(1 row)
-
--- Cancel after copying data to worker node
-SELECT citus.mitmproxy('conn.onCommandComplete(command="COPY 3").cancel(' || pg_backend_pid() || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT create_reference_table('ref_table');
-NOTICE:  Copying data from local table...
-ERROR:  canceling statement due to user request
-SELECT count(*) FROM pg_dist_shard_placement;
- count 
--------
-     0
-(1 row)
-
--- we don't want to see the prepared transaction numbers in the warnings
-SET client_min_messages TO ERROR;
--- Kill after preparing transaction. Since we don't commit after preparing, we recover
--- prepared transaction afterwards.
-SELECT citus.mitmproxy('conn.onCommandComplete(command="PREPARE TRANSACTION").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT create_reference_table('ref_table');
-ERROR:  connection not open
-CONTEXT:  while executing command on localhost:9060
-SELECT count(*) FROM pg_dist_shard_placement;
- count 
--------
-     0
-(1 row)
-
-SELECT recover_prepared_transactions();
- recover_prepared_transactions 
--------------------------------
-                             1
-(1 row)
-
--- Kill after commiting prepared, this should succeed
-SELECT citus.mitmproxy('conn.onCommandComplete(command="COMMIT PREPARED").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT create_reference_table('ref_table');
- create_reference_table 
-------------------------
- 
-(1 row)
-
-SELECT shardid, nodeport, shardstate FROM pg_dist_shard_placement ORDER BY shardid, nodeport;
- shardid  | nodeport | shardstate 
-----------+----------+------------
- 10000008 |     9060 |          1
- 10000008 |    57637 |          1
-(2 rows)
-
-SET client_min_messages TO NOTICE;
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-DROP TABLE ref_table;
-DROP SCHEMA failure_reference_table;
-CREATE SCHEMA failure_reference_table;
-CREATE TABLE ref_table(id int);
-INSERT INTO ref_table VALUES(1),(2),(3);
--- Test in transaction
-SELECT citus.mitmproxy('conn.onQuery().kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-SELECT create_reference_table('ref_table');
-WARNING:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
-ERROR:  failure on connection marked as essential: localhost:9060
-COMMIT;
--- kill on ROLLBACK, should be rollbacked
-SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-SELECT create_reference_table('ref_table');
-NOTICE:  Copying data from local table...
- create_reference_table 
-------------------------
- 
-(1 row)
-
-ROLLBACK;
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
-SELECT * FROM pg_dist_shard_placement ORDER BY shardid, nodeport;
- shardid | shardstate | shardlength | nodename | nodeport | placementid 
----------+------------+-------------+----------+----------+-------------
-(0 rows)
-
--- cancel when the coordinator send ROLLBACK, should be rollbacked. We ignore cancellations
--- during the ROLLBACK.
-SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").cancel(' || pg_backend_pid() || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-SELECT create_reference_table('ref_table');
-NOTICE:  Copying data from local table...
- create_reference_table 
-------------------------
- 
-(1 row)
-
-ROLLBACK;
-SELECT * FROM pg_dist_shard_placement ORDER BY shardid, nodeport;
- shardid | shardstate | shardlength | nodename | nodeport | placementid 
----------+------------+-------------+----------+----------+-------------
-(0 rows)
-
-DROP SCHEMA failure_reference_table CASCADE;
-NOTICE:  drop cascades to table ref_table
-SET search_path TO default;

src/test/regress/expected/failure_create_table.out

@@ -89,10 +89,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 SELECT create_distributed_table('test_table','id');
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
 ERROR:  connection error: localhost:9060
-DETAIL:  connection not open
+DETAIL:  server closed the connection unexpectedly
+	This probably means the server terminated abnormally
+	before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy 
 -----------
@@ -120,10 +120,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="SELECT worker_apply_shard_ddl_comman
 (1 row)
 
 SELECT create_distributed_table('test_table','id');
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy 
 -----------
@@ -154,10 +154,10 @@ BEGIN;
 (1 row)
 
     SELECT create_distributed_table('test_table', 'id');
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 COMMIT;
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy 
@@ -188,12 +188,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 SELECT create_distributed_table('test_table','id');
-WARNING:  cancel requests are ignored during shard creation
+ERROR:  canceling statement due to user request
- create_distributed_table 
---------------------------
- 
-(1 row)
-
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy 
 -----------
@@ -203,14 +198,14 @@ SELECT citus.mitmproxy('conn.allow()');
 SELECT count(*) FROM pg_dist_shard;
  count 
 -------
-     4
+     0
 (1 row)
 
 SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
  run_command_on_workers 
 ------------------------
- (localhost,9060,t,2)
+ (localhost,9060,t,0)
- (localhost,57637,t,2)
+ (localhost,57637,t,0)
 (2 rows)
 
 DROP TABLE test_table;
@@ -230,10 +225,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").kill()');
 (1 row)
 
 SELECT create_distributed_table('test_table','id',colocate_with=>'temp_table');
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy 
 -----------
@@ -389,10 +384,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 
 BEGIN;
 SELECT create_distributed_table('test_table','id');
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
 ERROR:  connection error: localhost:9060
-DETAIL:  connection not open
+DETAIL:  server closed the connection unexpectedly
+	This probably means the server terminated abnormally
+	before or while processing the request.
 ROLLBACK;
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy 
@@ -425,12 +420,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 
 BEGIN;
 SELECT create_distributed_table('test_table','id');
-WARNING:  cancel requests are ignored during shard creation
+ERROR:  canceling statement due to user request
- create_distributed_table 
---------------------------
- 
-(1 row)
-
 COMMIT;
 SELECT recover_prepared_transactions();
  recover_prepared_transactions 
@@ -447,14 +437,14 @@ SELECT citus.mitmproxy('conn.allow()');
 SELECT count(*) FROM pg_dist_shard;
  count 
 -------
-     4
+     0
 (1 row)
 
 SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
  run_command_on_workers 
 ------------------------
- (localhost,9060,t,2)
+ (localhost,9060,t,0)
- (localhost,57637,t,2)
+ (localhost,57637,t,0)
 (2 rows)
 
 -- drop tables and schema and recreate to start from a non-distributed schema again
@@ -506,10 +496,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").kill()');
 
 BEGIN;
 SELECT create_distributed_table('test_table','id');
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 ROLLBACK;
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy 
@@ -543,10 +533,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 
 BEGIN;
 SELECT create_distributed_table('test_table','id');
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
 ERROR:  connection error: localhost:9060
-DETAIL:  connection not open
+DETAIL:  server closed the connection unexpectedly
+	This probably means the server terminated abnormally
+	before or while processing the request.
 ROLLBACK;
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy 
@@ -578,12 +568,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 
 BEGIN;
 SELECT create_distributed_table('test_table','id');
-WARNING:  cancel requests are ignored during shard creation
+ERROR:  canceling statement due to user request
- create_distributed_table 
---------------------------
- 
-(1 row)
-
 COMMIT;
 SELECT recover_prepared_transactions();
  recover_prepared_transactions 
@@ -600,14 +585,14 @@ SELECT citus.mitmproxy('conn.allow()');
 SELECT count(*) FROM pg_dist_shard;
  count 
 -------
-     4
+     0
 (1 row)
 
 SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
  run_command_on_workers 
 ------------------------
- (localhost,9060,t,2)
+ (localhost,9060,t,0)
- (localhost,57637,t,2)
+ (localhost,57637,t,0)
 (2 rows)
 
 DROP TABLE test_table;
@@ -630,10 +615,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 SELECT master_create_worker_shards('test_table_2', 4, 2);
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
 ERROR:  connection error: localhost:9060
-DETAIL:  connection not open
+DETAIL:  server closed the connection unexpectedly
+	This probably means the server terminated abnormally
+	before or while processing the request.
 SELECT count(*) FROM pg_dist_shard;
  count 
 -------

src/test/regress/expected/failure_create_table_9.out

@@ -1,713 +0,0 @@
---
--- failure_create_table adds failure tests for creating table without data.
---
-CREATE SCHEMA failure_create_table;
-SET search_path TO 'failure_create_table';
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SET citus.shard_replication_factor TO 1;
-SET citus.shard_count to 4;
-CREATE TABLE test_table(id int, value_1 int);
--- Kill connection before sending query to the worker 
-SELECT citus.mitmproxy('conn.kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT create_distributed_table('test_table','id');
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT count(*) FROM pg_dist_shard;
- count 
--------
-     0
-(1 row)
-
-SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
- run_command_on_workers 
-------------------------
- (localhost,9060,t,0)
- (localhost,57637,t,0)
-(2 rows)
-
--- kill as soon as the coordinator sends CREATE SCHEMA
--- Since schemas are created in separate transaction, schema will
--- be created only on the node which is not behind the proxy.
--- https://github.com/citusdata/citus/pull/1652
-SELECT citus.mitmproxy('conn.onQuery(query="^CREATE SCHEMA").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT create_distributed_table('test_table', 'id');
-ERROR:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT count(*) FROM pg_dist_shard;
- count 
--------
-     0
-(1 row)
-
-SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.schemata WHERE schema_name = 'failure_create_table'$$);
- run_command_on_workers 
-------------------------
- (localhost,9060,t,0)
- (localhost,57637,t,1)
-(2 rows)
-
--- this is merely used to get the schema creation propagated. Without there are failures
--- not related to reference tables but schema creation due to dependency creation on workers
-CREATE TYPE schema_proc AS (a int);
-DROP TYPE schema_proc;
--- Now, kill the connection while opening transaction on workers.
-SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT create_distributed_table('test_table','id');
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT count(*) FROM pg_dist_shard;
- count 
--------
-     0
-(1 row)
-
-SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
- run_command_on_workers 
-------------------------
- (localhost,9060,t,0)
- (localhost,57637,t,0)
-(2 rows)
-
--- Now, kill the connection after sending create table command with worker_apply_shard_ddl_command UDF
-SELECT citus.mitmproxy('conn.onQuery(query="SELECT worker_apply_shard_ddl_command").after(1).kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT create_distributed_table('test_table','id');
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT count(*) FROM pg_dist_shard;
- count 
--------
-     0
-(1 row)
-
-SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
- run_command_on_workers 
-------------------------
- (localhost,9060,t,0)
- (localhost,57637,t,0)
-(2 rows)
-
--- Kill the connection while creating a distributed table in sequential mode on sending create command
--- with worker_apply_shard_ddl_command UDF.
-BEGIN;
-    SET LOCAL citus.multi_shard_modify_mode TO 'sequential';
-    SELECT citus.mitmproxy('conn.onQuery(query="SELECT worker_apply_shard_ddl_command").after(1).kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-    SELECT create_distributed_table('test_table', 'id');
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-COMMIT;
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT count(*) FROM pg_dist_shard;
- count 
--------
-     0
-(1 row)
-
-SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
- run_command_on_workers 
-------------------------
- (localhost,9060,t,0)
- (localhost,57637,t,0)
-(2 rows)
-
--- Now, cancel the connection while creating transaction
--- workers. Note that, cancel requests will be ignored during
--- shard creation.
-SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED").cancel(' || pg_backend_pid() || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT create_distributed_table('test_table','id');
-ERROR:  canceling statement due to user request
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT count(*) FROM pg_dist_shard;
- count 
--------
-     0
-(1 row)
-
-SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
- run_command_on_workers 
-------------------------
- (localhost,9060,t,0)
- (localhost,57637,t,0)
-(2 rows)
-
-DROP TABLE test_table;
-CREATE TABLE test_table(id int, value_1 int);
--- Kill and cancel the connection with colocate_with option while sending the create table command
-CREATE TABLE temp_table(id int, value_1 int);
-SELECT create_distributed_table('temp_table','id');
- create_distributed_table 
---------------------------
- 
-(1 row)
-
-SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT create_distributed_table('test_table','id',colocate_with=>'temp_table');
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT count(*) FROM pg_dist_shard;
- count 
--------
-     4
-(1 row)
-
-SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
- run_command_on_workers 
-------------------------
- (localhost,9060,t,0)
- (localhost,57637,t,0)
-(2 rows)
-
-SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").cancel(' || pg_backend_pid() || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT create_distributed_table('test_table','id',colocate_with=>'temp_table');
-ERROR:  canceling statement due to user request
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT count(*) FROM pg_dist_shard;
- count 
--------
-     4
-(1 row)
-
-SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
- run_command_on_workers 
-------------------------
- (localhost,9060,t,0)
- (localhost,57637,t,0)
-(2 rows)
-
--- Kill and cancel the connection after worker sends "PREPARE TRANSACTION" ack with colocate_with option
-SELECT citus.mitmproxy('conn.onCommandComplete(command="PREPARE TRANSACTION").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT create_distributed_table('test_table','id',colocate_with=>'temp_table');
-ERROR:  connection not open
-CONTEXT:  while executing command on localhost:9060
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT count(*) FROM pg_dist_shard;
- count 
--------
-     4
-(1 row)
-
-SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
- run_command_on_workers 
-------------------------
- (localhost,9060,t,0)
- (localhost,57637,t,0)
-(2 rows)
-
-SELECT citus.mitmproxy('conn.onCommandComplete(command="PREPARE TRANSACTION").cancel(' ||  pg_backend_pid() || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT create_distributed_table('test_table','id',colocate_with=>'temp_table');
-ERROR:  canceling statement due to user request
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT count(*) FROM pg_dist_shard;
- count 
--------
-     4
-(1 row)
-
-SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
- run_command_on_workers 
-------------------------
- (localhost,9060,t,0)
- (localhost,57637,t,0)
-(2 rows)
-
--- drop tables and schema and recreate to start from a non-distributed schema again
-DROP TABLE temp_table;
-DROP TABLE test_table;
-DROP SCHEMA failure_create_table;
-CREATE SCHEMA failure_create_table;
-CREATE TABLE test_table(id int, value_1 int);
--- Test inside transaction
--- Kill connection before sending query to the worker 
-SELECT citus.mitmproxy('conn.kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-SELECT create_distributed_table('test_table','id');
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-ROLLBACK;
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT count(*) FROM pg_dist_shard;
- count 
--------
-     0
-(1 row)
-
-SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
- run_command_on_workers 
-------------------------
- (localhost,9060,t,0)
- (localhost,57637,t,0)
-(2 rows)
-
--- this is merely used to get the schema creation propagated. Without there are failures
--- not related to reference tables but schema creation due to dependency creation on workers
-CREATE TYPE schema_proc AS (a int);
-DROP TYPE schema_proc;
--- Now, kill the connection while creating transaction on workers in transaction.
-SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-SELECT create_distributed_table('test_table','id');
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-ROLLBACK;
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT count(*) FROM pg_dist_shard;
- count 
--------
-     0
-(1 row)
-
-SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
- run_command_on_workers 
-------------------------
- (localhost,9060,t,0)
- (localhost,57637,t,0)
-(2 rows)
-
--- Now, cancel the connection while creating the transaction on
--- workers. Note that, cancel requests will be ignored during
--- shard creation again in transaction if we're not relying on the
--- executor. So, we'll have two output files
-SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED").cancel(' || pg_backend_pid() || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-SELECT create_distributed_table('test_table','id');
-ERROR:  canceling statement due to user request
-COMMIT;
-SELECT recover_prepared_transactions();
- recover_prepared_transactions 
--------------------------------
-                             1
-(1 row)
-
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT count(*) FROM pg_dist_shard;
- count 
--------
-     0
-(1 row)
-
-SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
- run_command_on_workers 
-------------------------
- (localhost,9060,t,0)
- (localhost,57637,t,0)
-(2 rows)
-
--- drop tables and schema and recreate to start from a non-distributed schema again
-DROP TABLE test_table;
-DROP SCHEMA failure_create_table;
-CREATE SCHEMA failure_create_table;
-CREATE TABLE test_table(id int, value_1 int);
--- Test inside transaction and with 1PC
-SET citus.multi_shard_commit_protocol TO "1pc";
--- Kill connection before sending query to the worker with 1pc.
-SELECT citus.mitmproxy('conn.kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-SELECT create_distributed_table('test_table','id');
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-ROLLBACK;
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT count(*) FROM pg_dist_shard;
- count 
--------
-     0
-(1 row)
-
-SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
- run_command_on_workers 
-------------------------
- (localhost,9060,t,0)
- (localhost,57637,t,0)
-(2 rows)
-
--- Kill connection while sending create table command with 1pc.
-SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-SELECT create_distributed_table('test_table','id');
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-ROLLBACK;
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT count(*) FROM pg_dist_shard;
- count 
--------
-     0
-(1 row)
-
-SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
- run_command_on_workers 
-------------------------
- (localhost,9060,t,0)
- (localhost,57637,t,0)
-(2 rows)
-
--- this is merely used to get the schema creation propagated. Without there are failures
--- not related to reference tables but schema creation due to dependency creation on workers
-CREATE TYPE schema_proc AS (a int);
-DROP TYPE schema_proc;
--- Now, kill the connection while opening transactions on workers with 1pc. Transaction will be opened due to BEGIN.
-SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-SELECT create_distributed_table('test_table','id');
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-ROLLBACK;
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT count(*) FROM pg_dist_shard;
- count 
--------
-     0
-(1 row)
-
-SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
- run_command_on_workers 
-------------------------
- (localhost,9060,t,0)
- (localhost,57637,t,0)
-(2 rows)
-
--- Now, cancel the connection while creating transactions on
--- workers with 1pc. Note that, cancel requests will be ignored during
--- shard creation unless the executor is used. So, we'll have two output files
-SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED").cancel(' || pg_backend_pid() || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-SELECT create_distributed_table('test_table','id');
-ERROR:  canceling statement due to user request
-COMMIT;
-SELECT recover_prepared_transactions();
- recover_prepared_transactions 
--------------------------------
-                             0
-(1 row)
-
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT count(*) FROM pg_dist_shard;
- count 
--------
-     0
-(1 row)
-
-SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
- run_command_on_workers 
-------------------------
- (localhost,9060,t,0)
- (localhost,57637,t,0)
-(2 rows)
-
-DROP TABLE test_table;
-DROP SCHEMA failure_create_table;
-CREATE SCHEMA failure_create_table;
--- Test master_create_worker_shards with 2pc
-SET citus.multi_shard_commit_protocol TO "2pc";
-CREATE TABLE test_table_2(id int, value_1 int);
-SELECT master_create_distributed_table('test_table_2', 'id', 'hash');
- master_create_distributed_table 
----------------------------------
- 
-(1 row)
-
--- Kill connection before sending query to the worker
-SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT master_create_worker_shards('test_table_2', 4, 2);
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-SELECT count(*) FROM pg_dist_shard;
- count 
--------
-     0
-(1 row)
-
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT count(*) FROM pg_dist_shard;
- count 
--------
-     0
-(1 row)
-
-SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
- run_command_on_workers 
-------------------------
- (localhost,9060,t,0)
- (localhost,57637,t,0)
-(2 rows)
-
--- Kill the connection after worker sends "PREPARE TRANSACTION" ack
-SELECT citus.mitmproxy('conn.onCommandComplete(command="^PREPARE TRANSACTION").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT master_create_worker_shards('test_table_2', 4, 2);
-ERROR:  connection not open
-CONTEXT:  while executing command on localhost:9060
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT count(*) FROM pg_dist_shard;
- count 
--------
-     0
-(1 row)
-
-SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
- run_command_on_workers 
-------------------------
- (localhost,9060,t,0)
- (localhost,57637,t,0)
-(2 rows)
-
--- Cancel the connection after sending prepare transaction in master_create_worker_shards
-SELECT citus.mitmproxy('conn.onCommandComplete(command="PREPARE TRANSACTION").cancel(' ||  pg_backend_pid() || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT master_create_worker_shards('test_table_2', 4, 2);
-ERROR:  canceling statement due to user request
--- Show that there is no pending transaction
-SELECT recover_prepared_transactions();
- recover_prepared_transactions 
--------------------------------
-                             1
-(1 row)
-
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT count(*) FROM pg_dist_shard;
- count 
--------
-     0
-(1 row)
-
-SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
- run_command_on_workers 
-------------------------
- (localhost,9060,t,0)
- (localhost,57637,t,0)
-(2 rows)
-
-DROP SCHEMA failure_create_table CASCADE;
-NOTICE:  drop cascades to table test_table_2
-SET search_path TO default;

src/test/regress/expected/failure_cte_subquery.out

@@ -83,10 +83,10 @@ FROM
      ORDER BY 1 DESC LIMIT 5
      ) as foo 
 	  WHERE foo.user_id = cte.user_id;
-WARNING:  could not consume data from worker node
+ERROR:  connection error: localhost:9060
-WARNING:  could not consume data from worker node
+DETAIL:  server closed the connection unexpectedly
-WARNING:  could not consume data from worker node
+	This probably means the server terminated abnormally
-ERROR:  failed to execute task 1
+	before or while processing the request.
 	  
 -- kill at the third copy (pull)
 SELECT citus.mitmproxy('conn.onQuery(query="SELECT DISTINCT users_table.user").kill()');
@@ -118,9 +118,10 @@ FROM
      ORDER BY 1 DESC LIMIT 5
      ) as foo 
 	  WHERE foo.user_id = cte.user_id;
-WARNING:  could not consume data from worker node
+ERROR:  connection error: localhost:9060
-WARNING:  could not consume data from worker node
+DETAIL:  server closed the connection unexpectedly
-ERROR:  failed to execute task 1
+	This probably means the server terminated abnormally
+	before or while processing the request.
 -- cancel at the first copy (push)
 SELECT citus.mitmproxy('conn.onQuery(query="^COPY").cancel(' || :pid || ')');
  mitmproxy 
@@ -257,10 +258,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="^DELETE FROM").kill()');
 
 WITH cte_delete as (DELETE FROM users_table WHERE user_name in ('A', 'D') RETURNING *)
 INSERT INTO users_table SELECT * FROM cte_delete;
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 -- verify contents are the same
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy 
@@ -373,10 +374,10 @@ BEGIN;
 SET LOCAL citus.multi_shard_modify_mode = 'sequential';
 WITH cte_delete as (DELETE FROM users_table WHERE user_name in ('A', 'D') RETURNING *)
 INSERT INTO users_table SELECT * FROM cte_delete;
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 END;
 RESET SEARCH_PATH;
 SELECT citus.mitmproxy('conn.allow()');

src/test/regress/expected/failure_cte_subquery_9.out

@@ -1,393 +0,0 @@
-CREATE SCHEMA cte_failure;
-SET SEARCH_PATH=cte_failure;
-SET citus.shard_count to 2;
-SET citus.shard_replication_factor to 1;
-SET citus.next_shard_id TO 16000000;
-SELECT pg_backend_pid() as pid \gset
-CREATE TABLE users_table (user_id int, user_name text);
-CREATE TABLE events_table(user_id int, event_id int, event_type int);
-SELECT create_distributed_table('users_table', 'user_id');
- create_distributed_table 
---------------------------
- 
-(1 row)
-
-SELECT create_distributed_table('events_table', 'user_id');
- create_distributed_table 
---------------------------
- 
-(1 row)
-
-CREATE TABLE users_table_local AS SELECT * FROM users_table;
--- kill at the first copy (push)
-SELECT citus.mitmproxy('conn.onQuery(query="^COPY").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-WITH cte AS (
-	WITH local_cte AS (
-		SELECT * FROM users_table_local
-	),
-	dist_cte AS (
-		SELECT user_id FROM events_table
-	)
-	SELECT dist_cte.user_id FROM local_cte join dist_cte on dist_cte.user_id=local_cte.user_id
-)
-SELECT 
-	count(*) 
-FROM 
-	cte,
-	  (SELECT 
-    	DISTINCT users_table.user_id 
-     FROM 
-     	users_table, events_table 
-     WHERE 
-     	users_table.user_id = events_table.user_id AND 
-     event_type IN (1,2,3,4)
-     ORDER BY 1 DESC LIMIT 5
-     ) as foo 
-	  WHERE foo.user_id = cte.user_id;
-ERROR:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
--- kill at the second copy (pull)
-SELECT citus.mitmproxy('conn.onQuery(query="SELECT user_id FROM cte_failure.events_table_16000002").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-WITH cte AS (
-	WITH local_cte AS (
-		SELECT * FROM users_table_local
-	),
-	dist_cte AS (
-		SELECT user_id FROM events_table
-	)
-	SELECT dist_cte.user_id FROM local_cte join dist_cte on dist_cte.user_id=local_cte.user_id
-)
-SELECT 
-	count(*) 
-FROM 
-	cte,
-	  (SELECT 
-    	DISTINCT users_table.user_id 
-     FROM 
-     	users_table, events_table 
-     WHERE 
-     	users_table.user_id = events_table.user_id AND 
-     event_type IN (1,2,3,4)
-     ORDER BY 1 DESC LIMIT 5
-     ) as foo 
-	  WHERE foo.user_id = cte.user_id;
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-	  
--- kill at the third copy (pull)
-SELECT citus.mitmproxy('conn.onQuery(query="SELECT DISTINCT users_table.user").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-WITH cte AS (
-	WITH local_cte AS (
-		SELECT * FROM users_table_local
-	),
-	dist_cte AS (
-		SELECT user_id FROM events_table
-	)
-	SELECT dist_cte.user_id FROM local_cte join dist_cte on dist_cte.user_id=local_cte.user_id
-)
-SELECT 
-	count(*) 
-FROM 
-	cte,
-	  (SELECT 
-    	DISTINCT users_table.user_id 
-     FROM 
-     	users_table, events_table 
-     WHERE 
-     	users_table.user_id = events_table.user_id AND 
-     event_type IN (1,2,3,4)
-     ORDER BY 1 DESC LIMIT 5
-     ) as foo 
-	  WHERE foo.user_id = cte.user_id;
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
--- cancel at the first copy (push)
-SELECT citus.mitmproxy('conn.onQuery(query="^COPY").cancel(' || :pid || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-WITH cte AS (
-	WITH local_cte AS (
-		SELECT * FROM users_table_local
-	),
-	dist_cte AS (
-		SELECT user_id FROM events_table
-	)
-	SELECT dist_cte.user_id FROM local_cte join dist_cte on dist_cte.user_id=local_cte.user_id
-)
-SELECT 
-	count(*) 
-FROM 
-	cte,
-	  (SELECT 
-    	DISTINCT users_table.user_id 
-     FROM 
-     	users_table, events_table 
-     WHERE 
-     	users_table.user_id = events_table.user_id AND 
-     event_type IN (1,2,3,4)
-     ORDER BY 1 DESC LIMIT 5
-     ) as foo 
-	  WHERE foo.user_id = cte.user_id;
-ERROR:  canceling statement due to user request
--- cancel at the second copy (pull)
-SELECT citus.mitmproxy('conn.onQuery(query="SELECT user_id FROM").cancel(' || :pid || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-WITH cte AS (
-	WITH local_cte AS (
-		SELECT * FROM users_table_local
-	),
-	dist_cte AS (
-		SELECT user_id FROM events_table
-	)
-	SELECT dist_cte.user_id FROM local_cte join dist_cte on dist_cte.user_id=local_cte.user_id
-)
-SELECT 
-	count(*) 
-FROM 
-	cte,
-	  (SELECT 
-    	DISTINCT users_table.user_id 
-     FROM 
-     	users_table, events_table 
-     WHERE 
-     	users_table.user_id = events_table.user_id AND 
-     event_type IN (1,2,3,4)
-     ORDER BY 1 DESC LIMIT 5
-     ) as foo 
-	  WHERE foo.user_id = cte.user_id;
-ERROR:  canceling statement due to user request
--- cancel at the third copy (pull)
-SELECT citus.mitmproxy('conn.onQuery(query="SELECT DISTINCT users_table.user").cancel(' || :pid || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-WITH cte AS (
-	WITH local_cte AS (
-		SELECT * FROM users_table_local
-	),
-	dist_cte AS (
-		SELECT user_id FROM events_table
-	)
-	SELECT dist_cte.user_id FROM local_cte join dist_cte on dist_cte.user_id=local_cte.user_id
-)
-SELECT 
-	count(*) 
-FROM 
-	cte,
-	  (SELECT 
-    	DISTINCT users_table.user_id 
-     FROM 
-     	users_table, events_table 
-     WHERE 
-     	users_table.user_id = events_table.user_id AND 
-     event_type IN (1,2,3,4)
-     ORDER BY 1 DESC LIMIT 5
-     ) as foo 
-	  WHERE foo.user_id = cte.user_id;
-ERROR:  canceling statement due to user request
--- distributed update tests
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
--- insert some rows
-INSERT INTO users_table VALUES (1, 'A'), (2, 'B'), (3, 'C'), (4, 'D'), (5, 'E');
-INSERT INTO events_table VALUES (1,1,1), (1,2,1), (1,3,1), (2,1, 4), (3, 4,1), (5, 1, 2), (5, 2, 1), (5, 2,2);
-SELECT * FROM users_table ORDER BY 1, 2;
- user_id | user_name 
----------+-----------
-       1 | A
-       2 | B
-       3 | C
-       4 | D
-       5 | E
-(5 rows)
-
--- following will delete and insert the same rows
-WITH cte_delete as (DELETE FROM users_table WHERE user_name in ('A', 'D') RETURNING *)
-INSERT INTO users_table SELECT * FROM cte_delete;
--- verify contents are the same
-SELECT * FROM users_table ORDER BY 1, 2;
- user_id | user_name 
----------+-----------
-       1 | A
-       2 | B
-       3 | C
-       4 | D
-       5 | E
-(5 rows)
-
--- kill connection during deletion
-SELECT citus.mitmproxy('conn.onQuery(query="^DELETE FROM").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-WITH cte_delete as (DELETE FROM users_table WHERE user_name in ('A', 'D') RETURNING *)
-INSERT INTO users_table SELECT * FROM cte_delete;
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
--- verify contents are the same
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT * FROM users_table ORDER BY 1, 2;
- user_id | user_name 
----------+-----------
-       1 | A
-       2 | B
-       3 | C
-       4 | D
-       5 | E
-(5 rows)
-
--- kill connection during insert
-SELECT citus.mitmproxy('conn.onQuery(query="^COPY").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-WITH cte_delete as (DELETE FROM users_table WHERE user_name in ('A', 'D') RETURNING *)
-INSERT INTO users_table SELECT * FROM cte_delete;
-ERROR:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
--- verify contents are the same
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT * FROM users_table ORDER BY 1, 2;
- user_id | user_name 
----------+-----------
-       1 | A
-       2 | B
-       3 | C
-       4 | D
-       5 | E
-(5 rows)
-
--- cancel during deletion
-SELECT citus.mitmproxy('conn.onQuery(query="^DELETE FROM").cancel(' || :pid || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-WITH cte_delete as (DELETE FROM users_table WHERE user_name in ('A', 'D') RETURNING *)
-INSERT INTO users_table SELECT * FROM cte_delete;
-ERROR:  canceling statement due to user request
--- verify contents are the same
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT * FROM users_table ORDER BY 1, 2;
- user_id | user_name 
----------+-----------
-       1 | A
-       2 | B
-       3 | C
-       4 | D
-       5 | E
-(5 rows)
-
--- cancel during insert
-SELECT citus.mitmproxy('conn.onQuery(query="^COPY").cancel(' || :pid || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-WITH cte_delete as (DELETE FROM users_table WHERE user_name in ('A', 'D') RETURNING *)
-INSERT INTO users_table SELECT * FROM cte_delete;
-ERROR:  canceling statement due to user request
--- verify contents are the same
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT * FROM users_table ORDER BY 1, 2;
- user_id | user_name 
----------+-----------
-       1 | A
-       2 | B
-       3 | C
-       4 | D
-       5 | E
-(5 rows)
-
--- test sequential delete/insert
-SELECT citus.mitmproxy('conn.onQuery(query="^DELETE FROM").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-SET LOCAL citus.multi_shard_modify_mode = 'sequential';
-WITH cte_delete as (DELETE FROM users_table WHERE user_name in ('A', 'D') RETURNING *)
-INSERT INTO users_table SELECT * FROM cte_delete;
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-END;
-RESET SEARCH_PATH;
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-DROP SCHEMA cte_failure CASCADE;
-NOTICE:  drop cascades to 3 other objects
-DETAIL:  drop cascades to table cte_failure.users_table
-drop cascades to table cte_failure.events_table
-drop cascades to table cte_failure.users_table_local

src/test/regress/expected/failure_ddl.out

@@ -70,7 +70,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 ALTER TABLE test_table ADD COLUMN new_column INT;
-ERROR:  failure on connection marked as essential: localhost:9060
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
+	This probably means the server terminated abnormally
+	before or while processing the request.
 SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass;
   array_agg  
 -------------
@@ -100,10 +103,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_apply_shard_ddl_command").kil
 (1 row)
 
 ALTER TABLE test_table ADD COLUMN new_column INT;
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 -- show that we've never commited the changes
 SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass;
   array_agg  
@@ -379,7 +382,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 ALTER TABLE test_table DROP COLUMN new_column;
-ERROR:  failure on connection marked as essential: localhost:9060
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
+	This probably means the server terminated abnormally
+	before or while processing the request.
 SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass;
        array_agg        
 ------------------------
@@ -409,10 +415,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_apply_shard_ddl_command").kil
 (1 row)
 
 ALTER TABLE test_table DROP COLUMN new_column;
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass;
        array_agg        
 ------------------------
@@ -747,7 +753,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 ALTER TABLE test_table ADD COLUMN new_column INT;
-ERROR:  failure on connection marked as essential: localhost:9060
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
+	This probably means the server terminated abnormally
+	before or while processing the request.
 SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass;
   array_agg  
 -------------
@@ -777,10 +786,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_apply_shard_ddl_command").kil
 (1 row)
 
 ALTER TABLE test_table ADD COLUMN new_column INT;
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass;
   array_agg  
 -------------
@@ -1038,7 +1047,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 ALTER TABLE test_table ADD COLUMN new_column INT;
-ERROR:  failure on connection marked as essential: localhost:9060
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
+	This probably means the server terminated abnormally
+	before or while processing the request.
 SELECT array_agg(name::text ORDER BY name::text) FROM public.table_attrs where relid = 'test_table'::regclass;
   array_agg  
 -------------
@@ -1068,10 +1080,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_apply_shard_ddl_command").kil
 (1 row)
 
 ALTER TABLE test_table ADD COLUMN new_column INT;
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 -- kill as soon as the coordinator after it sends worker_apply_shard_ddl_command 2nd time
 SELECT citus.mitmproxy('conn.onQuery(query="worker_apply_shard_ddl_command").after(2).kill()');
  mitmproxy 
@@ -1080,10 +1092,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="worker_apply_shard_ddl_command").aft
 (1 row)
 
 ALTER TABLE test_table ADD COLUMN new_column INT;
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 -- cancel as soon as the coordinator after it sends worker_apply_shard_ddl_command 2nd time
 SELECT citus.mitmproxy('conn.onQuery(query="worker_apply_shard_ddl_command").after(2).cancel(' ||  pg_backend_pid() || ')');
  mitmproxy 

src/test/regress/expected/failure_insert_select_pushdown.out

@@ -44,10 +44,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT INTO insert_select_pushdown"
 (1 row)
 
 INSERT INTO events_summary SELECT user_id, event_id, count(*) FROM events_table GROUP BY 1,2;
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 --verify nothing is modified
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy 
@@ -98,10 +98,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT INTO insert_select_pushdown"
 (1 row)
 
 INSERT INTO events_table SELECT * FROM events_table;
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 --verify nothing is modified
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy 

src/test/regress/expected/failure_insert_select_pushdown_9.out

@@ -1,150 +0,0 @@
---
--- failure_insert_select_pushdown
---
--- performs failure/cancellation test for insert/select pushed down to shards.
---
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-CREATE SCHEMA insert_select_pushdown;
-SET SEARCH_PATH=insert_select_pushdown;
-SET citus.shard_count to 2;
-SET citus.shard_replication_factor to 1;
-SELECT pg_backend_pid() as pid \gset
-CREATE TABLE events_table(user_id int, event_id int, event_type int);
-CREATE TABLE events_summary(user_id int, event_id int, event_count int);
-SELECT create_distributed_table('events_table', 'user_id');
- create_distributed_table 
---------------------------
- 
-(1 row)
-
-SELECT create_distributed_table('events_summary', 'user_id');
- create_distributed_table 
---------------------------
- 
-(1 row)
-
-INSERT INTO events_table VALUES (1, 1, 3 ), (1, 2, 1), (1, 3, 2), (2, 4, 3), (3, 5, 1), (4, 7, 1), (4, 1, 9), (4, 3, 2);
-SELECT count(*) FROM events_summary; 
- count 
--------
-     0
-(1 row)
-
--- insert/select from one distributed table to another
--- kill worker query
-SELECT citus.mitmproxy('conn.onQuery(query="^INSERT INTO insert_select_pushdown").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-INSERT INTO events_summary SELECT user_id, event_id, count(*) FROM events_table GROUP BY 1,2;
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
---verify nothing is modified
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT count(*) FROM events_summary;
- count 
--------
-     0
-(1 row)
-
--- cancel worker query
-SELECT citus.mitmproxy('conn.onQuery(query="^INSERT INTO insert_select_pushdown").cancel(' || :pid || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-INSERT INTO events_summary SELECT user_id, event_id, count(*) FROM events_table GROUP BY 1,2;
-ERROR:  canceling statement due to user request
---verify nothing is modified
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT count(*) FROM events_summary;
- count 
--------
-     0
-(1 row)
-
--- test self insert/select
-SELECT count(*) FROM events_table;
- count 
--------
-     8
-(1 row)
-
--- kill worker query
-SELECT citus.mitmproxy('conn.onQuery(query="^INSERT INTO insert_select_pushdown").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-INSERT INTO events_table SELECT * FROM events_table;
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
---verify nothing is modified
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT count(*) FROM events_table;
- count 
--------
-     8
-(1 row)
-
--- cancel worker query
-SELECT citus.mitmproxy('conn.onQuery(query="^INSERT INTO insert_select_pushdown").cancel(' || :pid || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-INSERT INTO events_table SELECT * FROM events_table;
-ERROR:  canceling statement due to user request
---verify nothing is modified
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT count(*) FROM events_table;
- count 
--------
-     8
-(1 row)
-
-RESET SEARCH_PATH;
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-DROP SCHEMA insert_select_pushdown CASCADE;
-NOTICE:  drop cascades to 2 other objects
-DETAIL:  drop cascades to table insert_select_pushdown.events_table
-drop cascades to table insert_select_pushdown.events_summary

src/test/regress/expected/failure_insert_select_via_coordinator.out

@@ -53,10 +53,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="^COPY").kill()');
 (1 row)
 
 INSERT INTO events_summary SELECT event_id, event_type, count(*) FROM events_table GROUP BY 1,2;
-WARNING:  could not consume data from worker node
+ERROR:  server closed the connection unexpectedly
-WARNING:  could not consume data from worker node
+	This probably means the server terminated abnormally
-WARNING:  could not consume data from worker node
+	before or while processing the request.
-ERROR:  failed to execute task 1
+CONTEXT:  while executing command on localhost:9060
 -- kill data push
 SELECT citus.mitmproxy('conn.onQuery(query="^COPY coordinator_insert_select").kill()');
  mitmproxy 
@@ -109,10 +109,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="^COPY").kill()');
 (1 row)
 
 INSERT INTO events_reference SELECT event_type, count(*) FROM events_table GROUP BY 1;
-WARNING:  could not consume data from worker node
+ERROR:  server closed the connection unexpectedly
-WARNING:  could not consume data from worker node
+	This probably means the server terminated abnormally
-WARNING:  could not consume data from worker node
+	before or while processing the request.
-ERROR:  failed to execute task 1
+CONTEXT:  while executing command on localhost:9060
 -- kill data push
 SELECT citus.mitmproxy('conn.onQuery(query="^COPY coordinator_insert_select").kill()');
  mitmproxy 

src/test/regress/expected/failure_insert_select_via_coordinator_9.out

@@ -1,229 +0,0 @@
---
--- failure_insert_select_via_coordinator
---
--- performs failure/cancellation test for insert/select executed by coordinator.
--- test for insert using CTEs are done in failure_cte_subquery, not repeating them here
---
-CREATE SCHEMA coordinator_insert_select;
-SET SEARCH_PATH=coordinator_insert_select;
-SET citus.shard_count to 2;
-SET citus.shard_replication_factor to 1;
-SELECT pg_backend_pid() as pid \gset
-CREATE TABLE events_table(user_id int, event_id int, event_type int);
-CREATE TABLE events_summary(event_id int, event_type int, event_count int);
-CREATE TABLE events_reference(event_type int, event_count int);
-CREATE TABLE events_reference_distributed(event_type int, event_count int);
-SELECT create_distributed_table('events_table', 'user_id');
- create_distributed_table 
---------------------------
- 
-(1 row)
-
-SELECT create_distributed_table('events_summary', 'event_id');
- create_distributed_table 
---------------------------
- 
-(1 row)
-
-SELECT create_reference_table('events_reference');
- create_reference_table 
-------------------------
- 
-(1 row)
-
-SELECT create_distributed_table('events_reference_distributed', 'event_type');
- create_distributed_table 
---------------------------
- 
-(1 row)
-
-INSERT INTO events_table VALUES (1, 1, 3 ), (1, 2, 1), (1, 3, 2), (2, 4, 3), (3, 5, 1), (4, 7, 1), (4, 1, 9), (4, 3, 2);
-SELECT count(*) FROM events_summary; 
- count 
--------
-     0
-(1 row)
-
--- insert/select from one distributed table to another
--- kill coordinator pull query
-SELECT citus.mitmproxy('conn.onQuery(query="^COPY").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-INSERT INTO events_summary SELECT event_id, event_type, count(*) FROM events_table GROUP BY 1,2;
-ERROR:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
--- kill data push
-SELECT citus.mitmproxy('conn.onQuery(query="^COPY coordinator_insert_select").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-INSERT INTO events_summary SELECT event_id, event_type, count(*) FROM events_table GROUP BY 1,2;
-ERROR:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
--- cancel coordinator pull query
-SELECT citus.mitmproxy('conn.onQuery(query="^COPY").cancel(' || :pid || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-INSERT INTO events_summary SELECT event_id, event_type, count(*) FROM events_table GROUP BY 1,2;
-ERROR:  canceling statement due to user request
--- cancel data push
-SELECT citus.mitmproxy('conn.onQuery(query="^COPY coordinator_insert_select").cancel(' || :pid || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-INSERT INTO events_summary SELECT event_id, event_type, count(*) FROM events_table GROUP BY 1,2;
-ERROR:  canceling statement due to user request
---verify nothing is modified
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT count(*) FROM events_summary;
- count 
--------
-     0
-(1 row)
-
--- insert into reference table from a distributed table
--- kill coordinator pull query
-SELECT citus.mitmproxy('conn.onQuery(query="^COPY").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-INSERT INTO events_reference SELECT event_type, count(*) FROM events_table GROUP BY 1;
-ERROR:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
--- kill data push
-SELECT citus.mitmproxy('conn.onQuery(query="^COPY coordinator_insert_select").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-INSERT INTO events_reference SELECT event_type, count(*) FROM events_table GROUP BY 1;
-ERROR:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
--- cancel coordinator pull query
-SELECT citus.mitmproxy('conn.onQuery(query="^COPY").cancel(' || :pid || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-INSERT INTO events_reference SELECT event_type, count(*) FROM events_table GROUP BY 1;
-ERROR:  canceling statement due to user request
--- cancel data push
-SELECT citus.mitmproxy('conn.onQuery(query="^COPY coordinator_insert_select").cancel(' || :pid || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-INSERT INTO events_reference SELECT event_type, count(*) FROM events_table GROUP BY 1;
-ERROR:  canceling statement due to user request
---verify nothing is modified
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT count(*) FROM events_reference;
- count 
--------
-     0
-(1 row)
-
--- insert/select from reference table to distributed
--- fill up reference table first
-INSERT INTO events_reference SELECT event_type, count(*) FROM events_table GROUP BY 1;
--- kill coordinator pull query
-SELECT citus.mitmproxy('conn.onQuery(query="^COPY").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-INSERT INTO events_reference_distributed SELECT * FROM events_reference;
-ERROR:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
--- kill data push
-SELECT citus.mitmproxy('conn.onQuery(query="^COPY coordinator_insert_select").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-INSERT INTO events_reference_distributed SELECT * FROM events_reference;
-ERROR:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
--- cancel coordinator pull query
-SELECT citus.mitmproxy('conn.onQuery(query="^COPY").cancel(' || :pid || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-INSERT INTO events_reference_distributed SELECT * FROM events_reference;
-ERROR:  canceling statement due to user request
--- cancel data push
-SELECT citus.mitmproxy('conn.onQuery(query="^COPY coordinator_insert_select").cancel(' || :pid || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-INSERT INTO events_reference_distributed SELECT * FROM events_reference;
-ERROR:  canceling statement due to user request
---verify nothing is modified
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT count(*) FROM events_reference_distributed;
- count 
--------
-     0
-(1 row)
-
-RESET SEARCH_PATH;
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-DROP SCHEMA coordinator_insert_select CASCADE;
-NOTICE:  drop cascades to 4 other objects
-DETAIL:  drop cascades to table coordinator_insert_select.events_table
-drop cascades to table coordinator_insert_select.events_summary
-drop cascades to table coordinator_insert_select.events_reference
-drop cascades to table coordinator_insert_select.events_reference_distributed

src/test/regress/expected/failure_multi_dml.out

@@ -33,10 +33,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="^DELETE").kill()');
 
 BEGIN;
 DELETE FROM dml_test WHERE id = 1;
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 DELETE FROM dml_test WHERE id = 2;
 ERROR:  current transaction is aborted, commands ignored until end of transaction block
 INSERT INTO dml_test VALUES (5, 'Epsilon');
@@ -96,10 +96,10 @@ BEGIN;
 DELETE FROM dml_test WHERE id = 1;
 DELETE FROM dml_test WHERE id = 2;
 INSERT INTO dml_test VALUES (5, 'Epsilon');
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 UPDATE dml_test SET name = 'alpha' WHERE id = 1;
 ERROR:  current transaction is aborted, commands ignored until end of transaction block
 UPDATE dml_test SET name = 'gamma' WHERE id = 3;
@@ -154,10 +154,10 @@ DELETE FROM dml_test WHERE id = 1;
 DELETE FROM dml_test WHERE id = 2;
 INSERT INTO dml_test VALUES (5, 'Epsilon');
 UPDATE dml_test SET name = 'alpha' WHERE id = 1;
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 UPDATE dml_test SET name = 'gamma' WHERE id = 3;
 ERROR:  current transaction is aborted, commands ignored until end of transaction block
 COMMIT;

src/test/regress/expected/failure_multi_dml_9.out

@@ -1,490 +0,0 @@
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SET citus.shard_count = 2;
-SET citus.shard_replication_factor = 1; -- one shard per worker
-SET citus.next_shard_id TO 103400;
-ALTER SEQUENCE pg_catalog.pg_dist_placement_placementid_seq RESTART 100;
-CREATE TABLE dml_test (id integer, name text);
-SELECT create_distributed_table('dml_test', 'id');
- create_distributed_table 
---------------------------
- 
-(1 row)
-
-COPY dml_test FROM STDIN WITH CSV;
-SELECT citus.clear_network_traffic();
- clear_network_traffic 
------------------------
- 
-(1 row)
-
----- test multiple statements spanning multiple shards,
----- at each significant point. These transactions are 2pc
--- fail at DELETE
-SELECT citus.mitmproxy('conn.onQuery(query="^DELETE").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-DELETE FROM dml_test WHERE id = 1;
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-DELETE FROM dml_test WHERE id = 2;
-ERROR:  current transaction is aborted, commands ignored until end of transaction block
-INSERT INTO dml_test VALUES (5, 'Epsilon');
-ERROR:  current transaction is aborted, commands ignored until end of transaction block
-UPDATE dml_test SET name = 'alpha' WHERE id = 1;
-ERROR:  current transaction is aborted, commands ignored until end of transaction block
-UPDATE dml_test SET name = 'gamma' WHERE id = 3;
-ERROR:  current transaction is aborted, commands ignored until end of transaction block
-COMMIT;
---- shouldn't see any changes performed in failed transaction
-SELECT * FROM dml_test ORDER BY id ASC;
- id | name  
-----+-------
-  1 | Alpha
-  2 | Beta
-  3 | Gamma
-  4 | Delta
-(4 rows)
-
--- cancel at DELETE
-SELECT citus.mitmproxy('conn.onQuery(query="^DELETE").cancel(' ||  pg_backend_pid() || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-DELETE FROM dml_test WHERE id = 1;
-ERROR:  canceling statement due to user request
-DELETE FROM dml_test WHERE id = 2;
-ERROR:  current transaction is aborted, commands ignored until end of transaction block
-INSERT INTO dml_test VALUES (5, 'Epsilon');
-ERROR:  current transaction is aborted, commands ignored until end of transaction block
-UPDATE dml_test SET name = 'alpha' WHERE id = 1;
-ERROR:  current transaction is aborted, commands ignored until end of transaction block
-UPDATE dml_test SET name = 'gamma' WHERE id = 3;
-ERROR:  current transaction is aborted, commands ignored until end of transaction block
-COMMIT;
---- shouldn't see any changes performed in failed transaction
-SELECT * FROM dml_test ORDER BY id ASC;
- id | name  
-----+-------
-  1 | Alpha
-  2 | Beta
-  3 | Gamma
-  4 | Delta
-(4 rows)
-
--- fail at INSERT
-SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-DELETE FROM dml_test WHERE id = 1;
-DELETE FROM dml_test WHERE id = 2;
-INSERT INTO dml_test VALUES (5, 'Epsilon');
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-UPDATE dml_test SET name = 'alpha' WHERE id = 1;
-ERROR:  current transaction is aborted, commands ignored until end of transaction block
-UPDATE dml_test SET name = 'gamma' WHERE id = 3;
-ERROR:  current transaction is aborted, commands ignored until end of transaction block
-COMMIT;
---- shouldn't see any changes before failed INSERT
-SELECT * FROM dml_test ORDER BY id ASC;
- id | name  
-----+-------
-  1 | Alpha
-  2 | Beta
-  3 | Gamma
-  4 | Delta
-(4 rows)
-
--- cancel at INSERT
-SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").cancel(' ||  pg_backend_pid() || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-DELETE FROM dml_test WHERE id = 1;
-DELETE FROM dml_test WHERE id = 2;
-INSERT INTO dml_test VALUES (5, 'Epsilon');
-ERROR:  canceling statement due to user request
-UPDATE dml_test SET name = 'alpha' WHERE id = 1;
-ERROR:  current transaction is aborted, commands ignored until end of transaction block
-UPDATE dml_test SET name = 'gamma' WHERE id = 3;
-ERROR:  current transaction is aborted, commands ignored until end of transaction block
-COMMIT;
---- shouldn't see any changes before failed INSERT
-SELECT * FROM dml_test ORDER BY id ASC;
- id | name  
-----+-------
-  1 | Alpha
-  2 | Beta
-  3 | Gamma
-  4 | Delta
-(4 rows)
-
--- fail at UPDATE
-SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-DELETE FROM dml_test WHERE id = 1;
-DELETE FROM dml_test WHERE id = 2;
-INSERT INTO dml_test VALUES (5, 'Epsilon');
-UPDATE dml_test SET name = 'alpha' WHERE id = 1;
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-UPDATE dml_test SET name = 'gamma' WHERE id = 3;
-ERROR:  current transaction is aborted, commands ignored until end of transaction block
-COMMIT;
---- shouldn't see any changes after failed UPDATE
-SELECT * FROM dml_test ORDER BY id ASC;
- id | name  
-----+-------
-  1 | Alpha
-  2 | Beta
-  3 | Gamma
-  4 | Delta
-(4 rows)
-
--- cancel at UPDATE
-SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").cancel(' ||  pg_backend_pid() || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-DELETE FROM dml_test WHERE id = 1;
-DELETE FROM dml_test WHERE id = 2;
-INSERT INTO dml_test VALUES (5, 'Epsilon');
-UPDATE dml_test SET name = 'alpha' WHERE id = 1;
-ERROR:  canceling statement due to user request
-UPDATE dml_test SET name = 'gamma' WHERE id = 3;
-ERROR:  current transaction is aborted, commands ignored until end of transaction block
-COMMIT;
---- shouldn't see any changes after failed UPDATE
-SELECT * FROM dml_test ORDER BY id ASC;
- id | name  
-----+-------
-  1 | Alpha
-  2 | Beta
-  3 | Gamma
-  4 | Delta
-(4 rows)
-
--- fail at PREPARE TRANSACTION
-SELECT citus.mitmproxy('conn.onQuery(query="^PREPARE TRANSACTION").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
--- this transaction block will be sent to the coordinator as a remote command to hide the
--- error message that is caused during commit.
--- we'll test for the txn side-effects to ensure it didn't run
-SELECT master_run_on_worker(
-    ARRAY['localhost']::text[],
-    ARRAY[:master_port]::int[],
-    ARRAY['
-BEGIN;
-DELETE FROM dml_test WHERE id = 1;
-DELETE FROM dml_test WHERE id = 2;
-INSERT INTO dml_test VALUES (5, ''Epsilon'');
-UPDATE dml_test SET name = ''alpha'' WHERE id = 1;
-UPDATE dml_test SET name = ''gamma'' WHERE id = 3;
-COMMIT;
-    '],
-    false
-);
-   master_run_on_worker    
----------------------------
- (localhost,57636,t,BEGIN)
-(1 row)
-
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT shardid FROM pg_dist_shard_placement WHERE shardstate = 3;
- shardid 
----------
-(0 rows)
-
-SELECT recover_prepared_transactions();
- recover_prepared_transactions 
--------------------------------
-                             0
-(1 row)
-
--- shouldn't see any changes after failed PREPARE
-SELECT * FROM dml_test ORDER BY id ASC;
- id | name  
-----+-------
-  1 | Alpha
-  2 | Beta
-  3 | Gamma
-  4 | Delta
-(4 rows)
-
--- cancel at PREPARE TRANSACTION
-SELECT citus.mitmproxy('conn.onQuery(query="^PREPARE TRANSACTION").cancel(' ||  pg_backend_pid() || ')');
- mitmproxy 
------------
- 
-(1 row)
-
--- we'll test for the txn side-effects to ensure it didn't run
-BEGIN;
-DELETE FROM dml_test WHERE id = 1;
-DELETE FROM dml_test WHERE id = 2;
-INSERT INTO dml_test VALUES (5, 'Epsilon');
-UPDATE dml_test SET name = 'alpha' WHERE id = 1;
-UPDATE dml_test SET name = 'gamma' WHERE id = 3;
-COMMIT;
-ERROR:  canceling statement due to user request
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT shardid FROM pg_dist_shard_placement WHERE shardstate = 3;
- shardid 
----------
-(0 rows)
-
-SELECT recover_prepared_transactions();
- recover_prepared_transactions 
--------------------------------
-                             0
-(1 row)
-
--- shouldn't see any changes after failed PREPARE
-SELECT * FROM dml_test ORDER BY id ASC;
- id | name  
-----+-------
-  1 | Alpha
-  2 | Beta
-  3 | Gamma
-  4 | Delta
-(4 rows)
-
--- fail at COMMIT
-SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
--- hide the error message (it has the PID)...
--- we'll test for the txn side-effects to ensure it didn't run
-SET client_min_messages TO ERROR;
-BEGIN;
-DELETE FROM dml_test WHERE id = 1;
-DELETE FROM dml_test WHERE id = 2;
-INSERT INTO dml_test VALUES (5, 'Epsilon');
-UPDATE dml_test SET name = 'alpha' WHERE id = 1;
-UPDATE dml_test SET name = 'gamma' WHERE id = 3;
-COMMIT;
-SET client_min_messages TO DEFAULT;
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT shardid FROM pg_dist_shard_placement WHERE shardstate = 3;
- shardid 
----------
-(0 rows)
-
-SELECT recover_prepared_transactions();
- recover_prepared_transactions 
--------------------------------
-                             1
-(1 row)
-
--- should see changes, because of txn recovery
-SELECT * FROM dml_test ORDER BY id ASC;
- id |  name   
-----+---------
-  3 | gamma
-  4 | Delta
-  5 | Epsilon
-(3 rows)
-
--- cancel at COMMITs are ignored by Postgres
-SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT").cancel(' ||  pg_backend_pid() || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-DELETE FROM dml_test WHERE id = 1;
-DELETE FROM dml_test WHERE id = 2;
-INSERT INTO dml_test VALUES (5, 'Epsilon');
-UPDATE dml_test SET name = 'alpha' WHERE id = 1;
-UPDATE dml_test SET name = 'gamma' WHERE id = 3;
-COMMIT;
--- should see changes, because cancellation is ignored
-SELECT * FROM dml_test ORDER BY id ASC;
- id |  name   
-----+---------
-  3 | gamma
-  4 | Delta
-  5 | Epsilon
-  5 | Epsilon
-(4 rows)
-
--- drop table and recreate with different replication/sharding
-DROP TABLE dml_test;
-SET citus.shard_count = 1;
-SET citus.shard_replication_factor = 2; -- two placements
-CREATE TABLE dml_test (id integer, name text);
-SELECT create_distributed_table('dml_test', 'id');
- create_distributed_table 
---------------------------
- 
-(1 row)
-
-COPY dml_test FROM STDIN WITH CSV;
----- test multiple statements against a single shard, but with two placements
--- fail at COMMIT (actually COMMIT this time, as no 2pc in use)
-SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-DELETE FROM dml_test WHERE id = 1;
-DELETE FROM dml_test WHERE id = 2;
-INSERT INTO dml_test VALUES (5, 'Epsilon');
-UPDATE dml_test SET name = 'alpha' WHERE id = 1;
-UPDATE dml_test SET name = 'gamma' WHERE id = 3;
-COMMIT;
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
-WARNING:  failed to commit transaction on localhost:9060
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
---- should see all changes, but they only went to one placement (other is unhealthy)
-SELECT * FROM dml_test ORDER BY id ASC;
- id |  name   
-----+---------
-  3 | gamma
-  4 | Delta
-  5 | Epsilon
-(3 rows)
-
-SELECT shardid FROM pg_dist_shard_placement WHERE shardstate = 3;
- shardid 
----------
-  103402
-(1 row)
-
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
--- drop table and recreate as reference table
-DROP TABLE dml_test;
-SET citus.shard_count = 2;
-SET citus.shard_replication_factor = 1;
-CREATE TABLE dml_test (id integer, name text);
-SELECT create_reference_table('dml_test');
- create_reference_table 
-------------------------
- 
-(1 row)
-
-COPY dml_test FROM STDIN WITH CSV;
--- fail at COMMIT (by failing to PREPARE)
-SELECT citus.mitmproxy('conn.onQuery(query="^PREPARE").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-DELETE FROM dml_test WHERE id = 1;
-DELETE FROM dml_test WHERE id = 2;
-INSERT INTO dml_test VALUES (5, 'Epsilon');
-UPDATE dml_test SET name = 'alpha' WHERE id = 1;
-UPDATE dml_test SET name = 'gamma' WHERE id = 3;
-COMMIT;
-ERROR:  connection not open
-CONTEXT:  while executing command on localhost:9060
---- shouldn't see any changes after failed COMMIT
-SELECT * FROM dml_test ORDER BY id ASC;
- id | name  
-----+-------
-  1 | Alpha
-  2 | Beta
-  3 | Gamma
-  4 | Delta
-(4 rows)
-
--- cancel at COMMIT (by cancelling on PREPARE)
-SELECT citus.mitmproxy('conn.onQuery(query="^PREPARE").cancel(' ||  pg_backend_pid() || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-DELETE FROM dml_test WHERE id = 1;
-DELETE FROM dml_test WHERE id = 2;
-INSERT INTO dml_test VALUES (5, 'Epsilon');
-UPDATE dml_test SET name = 'alpha' WHERE id = 1;
-UPDATE dml_test SET name = 'gamma' WHERE id = 3;
-COMMIT;
-ERROR:  canceling statement due to user request
---- shouldn't see any changes after cancelled PREPARE
-SELECT * FROM dml_test ORDER BY id ASC;
- id | name  
-----+-------
-  1 | Alpha
-  2 | Beta
-  3 | Gamma
-  4 | Delta
-(4 rows)
-
--- allow connection to allow DROP
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-DROP TABLE dml_test;

src/test/regress/expected/failure_multi_row_insert.out

@@ -43,10 +43,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()');
 (1 row)
 
 INSERT INTO distributed_table VALUES (1,1), (1,2), (1,3);
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 -- this test is broken, see https://github.com/citusdata/citus/issues/2460
 -- SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").cancel(' || :pid || ')');
 -- INSERT INTO distributed_table VALUES (1,4), (1,5), (1,6);
@@ -58,10 +58,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()');
 (1 row)
 
 INSERT INTO distributed_table VALUES (1,7), (5,8);
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 -- this test is broken, see https://github.com/citusdata/citus/issues/2460
 -- SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").cancel(' || :pid || ')');
 -- INSERT INTO distributed_table VALUES (1,9), (5,10);
@@ -73,10 +73,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()');
 (1 row)
 
 INSERT INTO distributed_table VALUES (1,11), (6,12);
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").cancel(' || :pid || ')');
  mitmproxy 
 -----------
@@ -93,10 +93,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").after(1).kill()');
 (1 row)
 
 INSERT INTO distributed_table VALUES (1,15), (6,16);
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").after(1).cancel(' || :pid || ')');
  mitmproxy 
 -----------
@@ -113,10 +113,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()');
 (1 row)
 
 INSERT INTO distributed_table VALUES (2,19),(1,20);
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").cancel(' || :pid || ')');
  mitmproxy 
 -----------

src/test/regress/expected/failure_multi_row_insert_9.out

@@ -1,158 +0,0 @@
---
--- failure_multi_row_insert
---
-CREATE SCHEMA IF NOT EXISTS failure_multi_row_insert;
-SET SEARCH_PATH TO failure_multi_row_insert;
--- this test is dependent on the shard count, so do not change
--- whitout changing the test
-SET citus.shard_count TO 4;
-SET citus.next_shard_id TO 301000;
-SET citus.shard_replication_factor TO 1;
-SELECT pg_backend_pid() as pid \gset
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-CREATE TABLE distributed_table(key int, value int);
-CREATE TABLE reference_table(value int);
-SELECT create_distributed_table('distributed_table', 'key');
- create_distributed_table 
---------------------------
- 
-(1 row)
-
-SELECT create_reference_table('reference_table');
- create_reference_table 
-------------------------
- 
-(1 row)
-
--- we'll test failure cases of the following cases:
--- (a) multi-row INSERT that hits the same shard with the same value
--- (b) multi-row INSERT that hits the same shard with different values
--- (c) multi-row INSERT that hits multiple shards in a single worker
--- (d) multi-row INSERT that hits multiple shards in multiple workers
--- (e) multi-row INSERT to a reference table
---  Failure and cancellation on multi-row INSERT that hits the same shard with the same value
-SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-INSERT INTO distributed_table VALUES (1,1), (1,2), (1,3);
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
--- this test is broken, see https://github.com/citusdata/citus/issues/2460
--- SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").cancel(' || :pid || ')');
--- INSERT INTO distributed_table VALUES (1,4), (1,5), (1,6);
---  Failure and cancellation on multi-row INSERT that hits the same shard with different values
-SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-INSERT INTO distributed_table VALUES (1,7), (5,8);
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
--- this test is broken, see https://github.com/citusdata/citus/issues/2460
--- SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").cancel(' || :pid || ')');
--- INSERT INTO distributed_table VALUES (1,9), (5,10);
---  Failure and cancellation multi-row INSERT that hits multiple shards in a single worker
-SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-INSERT INTO distributed_table VALUES (1,11), (6,12);
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").cancel(' || :pid || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-INSERT INTO distributed_table VALUES (1,13), (6,14);
-ERROR:  canceling statement due to user request
---  Failure and cancellation multi-row INSERT that hits multiple shards in a single worker, happening on the second query
-SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").after(1).kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-INSERT INTO distributed_table VALUES (1,15), (6,16);
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").after(1).cancel(' || :pid || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-INSERT INTO distributed_table VALUES (1,17), (6,18);
-ERROR:  canceling statement due to user request
---  Failure and cancellation multi-row INSERT that hits multiple shards in multiple workers
-SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-INSERT INTO distributed_table VALUES (2,19),(1,20);
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").cancel(' || :pid || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-INSERT INTO distributed_table VALUES (2,21), (1,22);
-ERROR:  canceling statement due to user request
--- one test for the reference tables for completeness
-SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").cancel(' || :pid || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-INSERT INTO reference_table VALUES (1), (2), (3), (4);
-ERROR:  canceling statement due to user request
--- we've either failed or cancelled all queries, so should be empty
-SELECT * FROM distributed_table;
- key | value 
------+-------
-(0 rows)
-
-SELECT * FROM reference_table;
- value 
--------
-(0 rows)
-
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-RESET SEARCH_PATH;
-DROP SCHEMA failure_multi_row_insert CASCADE;
-NOTICE:  drop cascades to 2 other objects
-DETAIL:  drop cascades to table failure_multi_row_insert.distributed_table
-drop cascades to table failure_multi_row_insert.reference_table

src/test/regress/expected/failure_multi_shard_update_delete.out

@@ -63,10 +63,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM").kill()');
 
 -- issue a multi shard delete
 DELETE FROM t2 WHERE b = 2;
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 -- verify nothing is deleted
 SELECT count(*) FROM t2;
  count 
@@ -82,10 +82,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM multi_shard.t2_201005").
 (1 row)
 
 DELETE FROM t2 WHERE b = 2;
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 -- verify nothing is deleted
 SELECT count(*) FROM t2;
  count 
@@ -145,10 +145,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()');
 
 -- issue a multi shard update
 UPDATE t2 SET c = 4 WHERE b = 2;
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 -- verify nothing is updated
 SELECT count(*) FILTER (WHERE b = 2) AS b2, count(*) FILTER (WHERE c = 4) AS c4 FROM t2;
  b2 | c4 
@@ -164,10 +164,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="UPDATE multi_shard.t2_201005").kill(
 (1 row)
 
 UPDATE t2 SET c = 4 WHERE b = 2;
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 -- verify nothing is updated
 SELECT count(*) FILTER (WHERE b = 2) AS b2, count(*) FILTER (WHERE c = 4) AS c4 FROM t2;
  b2 | c4 
@@ -221,10 +221,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM").kill()');
 
 -- issue a multi shard delete
 DELETE FROM t2 WHERE b = 2;
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 -- verify nothing is deleted
 SELECT count(*) FROM t2;
  count 
@@ -240,10 +240,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM multi_shard.t2_201005").
 (1 row)
 
 DELETE FROM t2 WHERE b = 2;
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 -- verify nothing is deleted
 SELECT count(*) FROM t2;
  count 
@@ -303,10 +303,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()');
 
 -- issue a multi shard update
 UPDATE t2 SET c = 4 WHERE b = 2;
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 -- verify nothing is updated
 SELECT count(*) FILTER (WHERE b = 2) AS b2, count(*) FILTER (WHERE c = 4) AS c4 FROM t2;
  b2 | c4 
@@ -322,10 +322,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="UPDATE multi_shard.t2_201005").kill(
 (1 row)
 
 UPDATE t2 SET c = 4 WHERE b = 2;
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 -- verify nothing is updated
 SELECT count(*) FILTER (WHERE b = 2) AS b2, count(*) FILTER (WHERE c = 4) AS c4 FROM t2;
  b2 | c4 
@@ -396,10 +396,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM").kill()');
 (1 row)
 
 DELETE FROM r1 WHERE a = 2;
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 -- verify nothing is deleted
 SELECT count(*) FILTER (WHERE b = 2) AS b2 FROM t2;
  b2 
@@ -414,10 +414,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM").kill()');
 (1 row)
 
 DELETE FROM t2 WHERE b = 2;
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 -- verify nothing is deleted
 SELECT count(*) FILTER (WHERE b = 2) AS b2 FROM t2;
  b2 
@@ -496,10 +496,10 @@ UPDATE t3 SET c = q.c FROM (
 	SELECT b, max(c) as c FROM t2  GROUP BY b) q
 WHERE t3.b = q.b
 RETURNING *;
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 --- verify nothing is updated
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy 
@@ -552,10 +552,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="UPDATE multi_shard.t3_201013").kill(
 (1 row)
 
 UPDATE t3 SET b = 2 WHERE b = 1;
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 -- verify nothing is updated
 SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FROM t3;
  b1 | b2 
@@ -587,10 +587,10 @@ SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FRO
 
 -- following will fail
 UPDATE t3 SET b = 2 WHERE b = 1;
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 END;
 -- verify everything is rolled back
 SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FROM t2;
@@ -606,10 +606,10 @@ SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FRO
 (1 row)
 
 UPDATE t3 SET b = 1 WHERE b = 2 RETURNING *;
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 -- verify nothing is updated
 SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FROM t3;
  b1 | b2 
@@ -626,10 +626,10 @@ SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FRO
 (1 row)
 
 UPDATE t3 SET b = 2 WHERE b = 1;
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 -- verify nothing is updated
 SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FROM t3;
  b1 | b2 
@@ -661,10 +661,10 @@ SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FRO
 
 -- following will fail
 UPDATE t3 SET b = 2 WHERE b = 1;
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 END;
 -- verify everything is rolled back
 SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FROM t2;

src/test/regress/expected/failure_multi_shard_update_delete_9.out

@@ -1,694 +0,0 @@
---
--- failure_multi_shard_update_delete
---
-CREATE SCHEMA IF NOT EXISTS multi_shard;
-SET SEARCH_PATH = multi_shard;
-SET citus.shard_count TO 4;
-SET citus.next_shard_id TO 201000;
-SET citus.shard_replication_factor TO 1;
--- do not cache any connections
-SET citus.max_cached_conns_per_worker TO 0;
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-CREATE TABLE t1(a int PRIMARY KEY, b int, c int);
-CREATE TABLE r1(a int, b int PRIMARY KEY);
-CREATE TABLE t2(a int REFERENCES t1(a) ON DELETE CASCADE, b int REFERENCES r1(b) ON DELETE CASCADE, c int);
-SELECT create_distributed_table('t1', 'a');
- create_distributed_table 
---------------------------
- 
-(1 row)
-
-SELECT create_reference_table('r1');
- create_reference_table 
-------------------------
- 
-(1 row)
-
-SELECT create_distributed_table('t2', 'a');
- create_distributed_table 
---------------------------
- 
-(1 row)
-
--- insert some data
-INSERT INTO r1 VALUES (1, 1), (2, 2), (3, 3);
-INSERT INTO t1 VALUES (1, 1, 1), (2, 2, 2), (3, 3, 3);
-INSERT INTO t2 VALUES (1, 1, 1), (1, 2, 1), (2, 1, 2), (2, 2, 4), (3, 1, 3), (3, 2, 3), (3, 3, 3);
-SELECT pg_backend_pid() as pid \gset
-SELECT count(*) FROM t2;
- count 
--------
-     7
-(1 row)
-
-SHOW citus.multi_shard_commit_protocol ;
- citus.multi_shard_commit_protocol 
------------------------------------
- 2pc
-(1 row)
-
--- DELETION TESTS
--- delete using a filter on non-partition column filter
--- test both kill and cancellation
-SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
--- issue a multi shard delete
-DELETE FROM t2 WHERE b = 2;
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
--- verify nothing is deleted
-SELECT count(*) FROM t2;
- count 
--------
-     7
-(1 row)
-
--- kill just one connection
-SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM multi_shard.t2_201005").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-DELETE FROM t2 WHERE b = 2;
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
--- verify nothing is deleted
-SELECT count(*) FROM t2;
- count 
--------
-     7
-(1 row)
-
--- cancellation
-SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM").cancel(' || :pid || ')');
- mitmproxy 
------------
- 
-(1 row)
-
--- issue a multi shard delete
-DELETE FROM t2 WHERE b = 2;
-ERROR:  canceling statement due to user request
--- verify nothing is deleted
-SELECT count(*) FROM t2;
- count 
--------
-     7
-(1 row)
-
--- cancel just one connection
-SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM multi_shard.t2_201005").cancel(' || :pid || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-DELETE FROM t2 WHERE b = 2;
-ERROR:  canceling statement due to user request
--- verify nothing is deleted
-SELECT count(*) FROM t2;
- count 
--------
-     7
-(1 row)
-
--- UPDATE TESTS
--- update non-partition column based on a filter on another non-partition column
--- DELETION TESTS
--- delete using a filter on non-partition column filter
--- test both kill and cancellation
-SELECT count(*) FILTER (WHERE b = 2) AS b2, count(*) FILTER (WHERE c = 4) AS c4 FROM t2;
- b2 | c4 
-----+----
-  3 |  1
-(1 row)
-
-SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
--- issue a multi shard update
-UPDATE t2 SET c = 4 WHERE b = 2;
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
--- verify nothing is updated
-SELECT count(*) FILTER (WHERE b = 2) AS b2, count(*) FILTER (WHERE c = 4) AS c4 FROM t2;
- b2 | c4 
-----+----
-  3 |  1
-(1 row)
-
--- kill just one connection
-SELECT citus.mitmproxy('conn.onQuery(query="UPDATE multi_shard.t2_201005").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-UPDATE t2 SET c = 4 WHERE b = 2;
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
--- verify nothing is updated
-SELECT count(*) FILTER (WHERE b = 2) AS b2, count(*) FILTER (WHERE c = 4) AS c4 FROM t2;
- b2 | c4 
-----+----
-  3 |  1
-(1 row)
-
--- cancellation
-SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").cancel(' || :pid || ')');
- mitmproxy 
------------
- 
-(1 row)
-
--- issue a multi shard update
-UPDATE t2 SET c = 4 WHERE b = 2;
-ERROR:  canceling statement due to user request
--- verify nothing is updated
-SELECT count(*) FILTER (WHERE b = 2) AS b2, count(*) FILTER (WHERE c = 4) AS c4 FROM t2;
- b2 | c4 
-----+----
-  3 |  1
-(1 row)
-
--- cancel just one connection
-SELECT citus.mitmproxy('conn.onQuery(query="UPDATE multi_shard.t2_201005").cancel(' || :pid || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-UPDATE t2 SET c = 4 WHERE b = 2;
-ERROR:  canceling statement due to user request
--- verify nothing is updated
-SELECT count(*) FILTER (WHERE b = 2) AS b2, count(*) FILTER (WHERE c = 4) AS c4 FROM t2;
- b2 | c4 
-----+----
-  3 |  1
-(1 row)
-
--- switch to 1PC
-SET citus.multi_shard_commit_protocol TO '1PC';
--- DELETION TESTS
--- delete using a filter on non-partition column filter
--- test both kill and cancellation
-SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
--- issue a multi shard delete
-DELETE FROM t2 WHERE b = 2;
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
--- verify nothing is deleted
-SELECT count(*) FROM t2;
- count 
--------
-     7
-(1 row)
-
--- kill just one connection
-SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM multi_shard.t2_201005").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-DELETE FROM t2 WHERE b = 2;
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
--- verify nothing is deleted
-SELECT count(*) FROM t2;
- count 
--------
-     7
-(1 row)
-
--- cancellation
-SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM").cancel(' || :pid || ')');
- mitmproxy 
------------
- 
-(1 row)
-
--- issue a multi shard delete
-DELETE FROM t2 WHERE b = 2;
-ERROR:  canceling statement due to user request
--- verify nothing is deleted
-SELECT count(*) FROM t2;
- count 
--------
-     7
-(1 row)
-
--- cancel just one connection
-SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM multi_shard.t2_201005").cancel(' || :pid || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-DELETE FROM t2 WHERE b = 2;
-ERROR:  canceling statement due to user request
--- verify nothing is deleted
-SELECT count(*) FROM t2;
- count 
--------
-     7
-(1 row)
-
--- UPDATE TESTS
--- update non-partition column based on a filter on another non-partition column
--- DELETION TESTS
--- delete using a filter on non-partition column filter
--- test both kill and cancellation
-SELECT count(*) FILTER (WHERE b = 2) AS b2, count(*) FILTER (WHERE c = 4) AS c4 FROM t2;
- b2 | c4 
-----+----
-  3 |  1
-(1 row)
-
-SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
--- issue a multi shard update
-UPDATE t2 SET c = 4 WHERE b = 2;
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
--- verify nothing is updated
-SELECT count(*) FILTER (WHERE b = 2) AS b2, count(*) FILTER (WHERE c = 4) AS c4 FROM t2;
- b2 | c4 
-----+----
-  3 |  1
-(1 row)
-
--- kill just one connection
-SELECT citus.mitmproxy('conn.onQuery(query="UPDATE multi_shard.t2_201005").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-UPDATE t2 SET c = 4 WHERE b = 2;
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
--- verify nothing is updated
-SELECT count(*) FILTER (WHERE b = 2) AS b2, count(*) FILTER (WHERE c = 4) AS c4 FROM t2;
- b2 | c4 
-----+----
-  3 |  1
-(1 row)
-
--- cancellation
-SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").cancel(' || :pid || ')');
- mitmproxy 
------------
- 
-(1 row)
-
--- issue a multi shard update
-UPDATE t2 SET c = 4 WHERE b = 2;
-ERROR:  canceling statement due to user request
--- verify nothing is updated
-SELECT count(*) FILTER (WHERE b = 2) AS b2, count(*) FILTER (WHERE c = 4) AS c4 FROM t2;
- b2 | c4 
-----+----
-  3 |  1
-(1 row)
-
--- cancel just one connection
-SELECT citus.mitmproxy('conn.onQuery(query="UPDATE multi_shard.t2_201005").cancel(' || :pid || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-UPDATE t2 SET c = 4 WHERE b = 2;
-ERROR:  canceling statement due to user request
--- verify nothing is updated
-SELECT count(*) FILTER (WHERE b = 2) AS b2, count(*) FILTER (WHERE c = 4) AS c4 FROM t2;
- b2 | c4 
-----+----
-  3 |  1
-(1 row)
-
-RESET citus.multi_shard_commit_protocol;
---
--- fail when cascading deletes from foreign key
--- unfortunately cascading deletes from foreign keys
--- are done inside the worker only and do not
--- generate any network output
--- therefore we can't just fail cascade part
--- following tests are added for completeness purposes
--- it is safe to remove them without reducing any
--- test coverage
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
--- check counts before delete
-SELECT count(*) FILTER (WHERE b = 2) AS b2 FROM t2;
- b2 
-----
-  3
-(1 row)
-
-SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-DELETE FROM r1 WHERE a = 2;
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
--- verify nothing is deleted
-SELECT count(*) FILTER (WHERE b = 2) AS b2 FROM t2;
- b2 
-----
-  3
-(1 row)
-
-SELECT citus.mitmproxy('conn.onQuery(query="DELETE FROM").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-DELETE FROM t2 WHERE b = 2;
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
--- verify nothing is deleted
-SELECT count(*) FILTER (WHERE b = 2) AS b2 FROM t2;
- b2 
-----
-  3
-(1 row)
-
--- test update with subquery pull
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-CREATE TABLE t3 AS SELECT * FROM t2;
-SELECT create_distributed_table('t3', 'a');
-NOTICE:  Copying data from local table...
- create_distributed_table 
---------------------------
- 
-(1 row)
-
-SELECT * FROM t3 ORDER BY 1, 2, 3;
- a | b | c 
----+---+---
- 1 | 1 | 1
- 1 | 2 | 1
- 2 | 1 | 2
- 2 | 2 | 4
- 3 | 1 | 3
- 3 | 2 | 3
- 3 | 3 | 3
-(7 rows)
-
-SELECT citus.mitmproxy('conn.onQuery(query="^COPY").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-UPDATE t3 SET c = q.c FROM (
-	SELECT b, max(c) as c FROM t2  GROUP BY b) q
-WHERE t3.b = q.b
-RETURNING *;
-ERROR:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
---- verify nothing is updated
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT * FROM t3 ORDER BY 1, 2, 3;
- a | b | c 
----+---+---
- 1 | 1 | 1
- 1 | 2 | 1
- 2 | 1 | 2
- 2 | 2 | 4
- 3 | 1 | 3
- 3 | 2 | 3
- 3 | 3 | 3
-(7 rows)
-
--- kill update part
-SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE multi_shard.t3_201009").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-UPDATE t3 SET c = q.c FROM (
-	SELECT b, max(c) as c FROM t2  GROUP BY b) q
-WHERE t3.b = q.b
-RETURNING *;
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
---- verify nothing is updated
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT * FROM t3 ORDER BY 1, 2, 3;
- a | b | c 
----+---+---
- 1 | 1 | 1
- 1 | 2 | 1
- 2 | 1 | 2
- 2 | 2 | 4
- 3 | 1 | 3
- 3 | 2 | 3
- 3 | 3 | 3
-(7 rows)
-
--- test with replication_factor = 2
--- table can not have foreign reference with this setting so
--- use a different set of table
-SET citus.shard_replication_factor to 2;
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-DROP TABLE t3;
-CREATE TABLE t3 AS SELECT * FROM t2;
-SELECT create_distributed_table('t3', 'a');
-NOTICE:  Copying data from local table...
- create_distributed_table 
---------------------------
- 
-(1 row)
-
-SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FROM t3;
- b1 | b2 
-----+----
-  3 |  3
-(1 row)
-
--- prevent update of one replica of one shard
-SELECT citus.mitmproxy('conn.onQuery(query="UPDATE multi_shard.t3_201013").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-UPDATE t3 SET b = 2 WHERE b = 1;
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
--- verify nothing is updated
-SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FROM t3;
- b1 | b2 
-----+----
-  3 |  3
-(1 row)
-
--- fail only one update verify transaction is rolled back correctly
-BEGIN;
-SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FROM t2;
- b1 | b2 
-----+----
-  3 |  3
-(1 row)
-
-SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FROM t3;
- b1 | b2 
-----+----
-  3 |  3
-(1 row)
-
-UPDATE t2 SET b = 2 WHERE b = 1;
--- verify update is performed on t2
-SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FROM t2;
- b1 | b2 
-----+----
-  0 |  6
-(1 row)
-
--- following will fail
-UPDATE t3 SET b = 2 WHERE b = 1;
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-END;
--- verify everything is rolled back
-SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FROM t2;
- b1 | b2 
-----+----
-  3 |  3
-(1 row)
-
-SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FROM t3;
- b1 | b2 
-----+----
-  3 |  3
-(1 row)
-
-UPDATE t3 SET b = 1 WHERE b = 2 RETURNING *;
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
--- verify nothing is updated
-SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FROM t3;
- b1 | b2 
-----+----
-  3 |  3
-(1 row)
-
--- switch to 1PC
-SET citus.multi_shard_commit_protocol TO '1PC';
-SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FROM t3;
- b1 | b2 
-----+----
-  3 |  3
-(1 row)
-
-UPDATE t3 SET b = 2 WHERE b = 1;
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
--- verify nothing is updated
-SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FROM t3;
- b1 | b2 
-----+----
-  3 |  3
-(1 row)
-
--- fail only one update verify transaction is rolled back correctly
-BEGIN;
-SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FROM t2;
- b1 | b2 
-----+----
-  3 |  3
-(1 row)
-
-SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FROM t3;
- b1 | b2 
-----+----
-  3 |  3
-(1 row)
-
-UPDATE t2 SET b = 2 WHERE b = 1;
--- verify update is performed on t2
-SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FROM t2;
- b1 | b2 
-----+----
-  0 |  6
-(1 row)
-
--- following will fail
-UPDATE t3 SET b = 2 WHERE b = 1;
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-END;
--- verify everything is rolled back
-SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FROM t2;
- b1 | b2 
-----+----
-  3 |  3
-(1 row)
-
-SELECT count(*) FILTER (WHERE b = 1) b1, count(*) FILTER (WHERE b = 2) AS b2 FROM t3;
- b1 | b2 
-----+----
-  3 |  3
-(1 row)
-
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-RESET SEARCH_PATH;
-DROP SCHEMA multi_shard CASCADE;
-NOTICE:  drop cascades to 4 other objects
-DETAIL:  drop cascades to table multi_shard.t1
-drop cascades to table multi_shard.r1
-drop cascades to table multi_shard.t2
-drop cascades to table multi_shard.t3

src/test/regress/expected/failure_real_time_select.out

@@ -33,7 +33,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="SELECT|COPY").kill()');
 (1 row)
 
 SELECT count(*) FROM test_table;
-ERROR:  failed to execute task 1
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
+	This probably means the server terminated abnormally
+	before or while processing the request.
 SET client_min_messages TO DEFAULT;
 -- Kill the connection with a CTE
 SELECT citus.mitmproxy('conn.onQuery(query=".*SELECT.*test_table.*").kill()');
@@ -46,13 +49,10 @@ WITH
 results AS (SELECT * FROM test_table)
 SELECT * FROM test_table, results
 WHERE test_table.id = results.id;
-WARNING:  could not consume data from worker node
+ERROR:  connection error: localhost:9060
-WARNING:  could not consume data from worker node
+DETAIL:  server closed the connection unexpectedly
-WARNING:  could not consume data from worker node
+	This probably means the server terminated abnormally
-WARNING:  could not consume data from worker node
+	before or while processing the request.
-WARNING:  could not consume data from worker node
-WARNING:  could not consume data from worker node
-ERROR:  failed to execute task 1
 -- Since the outer query uses the connection opened by the CTE,
 -- killing connection after first successful query should break.
 SET client_min_messages TO ERROR;
@@ -141,12 +141,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="SELECT|COPY").after(1).kill()');
 
 BEGIN;
 SELECT count(*) FROM test_table;
-WARNING:  could not consume data from worker node
+ERROR:  connection error: localhost:9060
-WARNING:  could not consume data from worker node
+DETAIL:  server closed the connection unexpectedly
-WARNING:  could not consume data from worker node
+	This probably means the server terminated abnormally
-WARNING:  could not consume data from worker node
+	before or while processing the request.
-WARNING:  could not consume data from worker node
-ERROR:  failed to execute task 1
 COMMIT;
 -- Cancel a real-time executor query - in sequential mode
 SELECT citus.mitmproxy('conn.onQuery(query="SELECT|COPY").cancel(' || pg_backend_pid() || ')');
@@ -209,8 +207,14 @@ SELECT citus.mitmproxy('conn.onQuery(query="SELECT|COPY").kill()');
 (1 row)
 
 SELECT count(*) FROM test_table;
-WARNING:  could not consume data from worker node
+WARNING:  connection error: localhost:9060
-WARNING:  could not consume data from worker node
+DETAIL:  server closed the connection unexpectedly
+	This probably means the server terminated abnormally
+	before or while processing the request.
+WARNING:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
+	This probably means the server terminated abnormally
+	before or while processing the request.
  count 
 -------
      6
@@ -226,8 +230,14 @@ SELECT citus.mitmproxy('conn.onQuery(query="SELECT|COPY").kill()');
 
 BEGIN;
 SELECT count(*) FROM test_table;
-WARNING:  could not consume data from worker node
+WARNING:  connection error: localhost:9060
-WARNING:  could not consume data from worker node
+DETAIL:  server closed the connection unexpectedly
+	This probably means the server terminated abnormally
+	before or while processing the request.
+WARNING:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
+	This probably means the server terminated abnormally
+	before or while processing the request.
  count 
 -------
      6
@@ -289,8 +299,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="SELECT|COPY").after(1).kill()');
 
 BEGIN;
 SELECT count(*) FROM test_table;
-WARNING:  could not consume data from worker node
+WARNING:  connection error: localhost:9060
-WARNING:  could not consume data from worker node
+DETAIL:  server closed the connection unexpectedly
+	This probably means the server terminated abnormally
+	before or while processing the request.
  count 
 -------
      6

src/test/regress/expected/failure_real_time_select_9.out

@@ -1,314 +0,0 @@
--- 
---  Failure tests for real time select queries
--- 
-CREATE SCHEMA real_time_select_failure;
-SET search_path TO 'real_time_select_failure';
-SET citus.next_shard_id TO 190000;
--- Preparation
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SET citus.shard_count to 4;
-SET citus.shard_replication_factor to 1;
--- create tables
-CREATE TABLE test_table(id int, value_1 int, value_2 int);
-SELECT create_distributed_table('test_table','id');
- create_distributed_table 
---------------------------
- 
-(1 row)
-
--- Populate data to the table
-INSERT INTO test_table VALUES(1,1,1),(1,2,2),(2,1,1),(2,2,2),(3,1,1),(3,2,2);
--- Kill when the first COPY command arrived, since we have a single placement 
--- it is expected to error out.
-SET client_min_messages TO ERROR;
-SELECT citus.mitmproxy('conn.onQuery(query="SELECT|COPY").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT count(*) FROM test_table;
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-SET client_min_messages TO DEFAULT;
--- Kill the connection with a CTE
-SELECT citus.mitmproxy('conn.onQuery(query=".*SELECT.*test_table.*").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-WITH
-results AS (SELECT * FROM test_table)
-SELECT * FROM test_table, results
-WHERE test_table.id = results.id;
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
--- Since the outer query uses the connection opened by the CTE,
--- killing connection after first successful query should break.
-SET client_min_messages TO ERROR;
-SELECT citus.mitmproxy('conn.onQuery(query="SELECT|COPY").after(1).kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-WITH results AS (SELECT * FROM test_table)
-SELECT * FROM test_table, results
-WHERE test_table.id = results.id;
-ERROR:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
-SET client_min_messages TO DEFAULT;
--- In parallel execution mode Citus opens separate connections for each shard
--- so killing the connection after the first copy does not break it.
-SET citus.force_max_query_parallelization=ON;
-SELECT citus.mitmproxy('conn.onQuery(query="SELECT|COPY").after(1).kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT count(*) FROM test_table;
- count 
--------
-     6
-(1 row)
-
--- set back the force flag to original value
-SET citus.force_max_query_parallelization=OFF;
--- Cancel a real-time executor query
-SELECT citus.mitmproxy('conn.onQuery(query="SELECT|COPY").cancel(' || pg_backend_pid() || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT count(*) FROM test_table;
-ERROR:  canceling statement due to user request
--- Cancel a query within the transaction
-SELECT citus.mitmproxy('conn.onQuery(query="SELECT|COPY").cancel(' || pg_backend_pid() || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-SELECT count(*) FROM test_table;
-ERROR:  canceling statement due to user request
-COMMIT;
--- Cancel a query within the transaction after a multi-shard update
-SELECT citus.mitmproxy('conn.onQuery(query=".*SELECT.*test_table.*").cancel(' || pg_backend_pid() || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-UPDATE test_table SET value_1 = value_1 + 1;
-SELECT count(*) FROM test_table;
-ERROR:  canceling statement due to user request
-COMMIT;
--- Cancel a query with CTE
-SELECT citus.mitmproxy('conn.onQuery(query=".*SELECT.*test_table.*").cancel(' || pg_backend_pid() || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-WITH
-results AS (SELECT * FROM test_table)
-SELECT * FROM test_table
-WHERE test_table.id > (SELECT id FROM results);
-ERROR:  canceling statement due to user request
--- Citus fails if the connection that is already used fails afterwards
-SET citus.multi_shard_modify_mode to sequential;
-SELECT citus.mitmproxy('conn.onQuery(query="SELECT|COPY").after(1).kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-SELECT count(*) FROM test_table;
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-COMMIT;
--- Cancel a real-time executor query - in sequential mode
-SELECT citus.mitmproxy('conn.onQuery(query="SELECT|COPY").cancel(' || pg_backend_pid() || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT count(*) FROM test_table;
-ERROR:  canceling statement due to user request
--- Cancel a query within the transaction - in sequential mode
-SELECT citus.mitmproxy('conn.onQuery(query="SELECT|COPY").cancel(' || pg_backend_pid() || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-SELECT count(*) FROM test_table;
-ERROR:  canceling statement due to user request
-COMMIT;
--- Cancel the query within a transaction after a single succesful run
-SELECT citus.mitmproxy('conn.onQuery(query="SELECT|COPY").after(1).cancel(' || pg_backend_pid() || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-SELECT count(*) FROM test_table;
-ERROR:  canceling statement due to user request
-COMMIT;
--- Now, test with replication factor 2, tests are expected to pass
--- since we have two placements for the same shard
-DROP TABLE test_table;
-SET citus.multi_shard_modify_mode to default;
--- Create table with shard placements on each node
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SET citus.shard_replication_factor to 2;
-CREATE TABLE test_table(id int, value_1 int, value_2 int);
-SELECT create_distributed_table('test_table','id');
- create_distributed_table 
---------------------------
- 
-(1 row)
-
--- Populate data to the table
-INSERT INTO test_table VALUES(1,1,1),(1,2,2),(2,1,1),(2,2,2),(3,1,1),(3,2,2);
--- Kill when the first SELECT command arrived, since we have placements on each node
--- it shouldn't fail.
-SELECT citus.mitmproxy('conn.onQuery(query="SELECT|COPY").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT count(*) FROM test_table;
-WARNING:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-WARNING:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
- count 
--------
-     6
-(1 row)
-
--- Kill within the transaction, since we have placements on each node
--- it shouldn't fail.
-SELECT citus.mitmproxy('conn.onQuery(query="SELECT|COPY").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-SELECT count(*) FROM test_table;
-WARNING:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-WARNING:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
- count 
--------
-     6
-(1 row)
-
-COMMIT;
--- Cancel a real-time executor query
-SELECT citus.mitmproxy('conn.onQuery(query="SELECT|COPY").cancel(' || pg_backend_pid() || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT count(*) FROM test_table;
-ERROR:  canceling statement due to user request
--- Cancel a query within the transaction
-SELECT citus.mitmproxy('conn.onQuery(query="SELECT|COPY").cancel(' || pg_backend_pid() || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-SELECT count(*) FROM test_table;
-ERROR:  canceling statement due to user request
-COMMIT;
--- Cancel a query within the transaction after a multi-shard update
-SELECT citus.mitmproxy('conn.onQuery(query=".*SELECT.*test_table.*").cancel(' || pg_backend_pid() || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-UPDATE test_table SET value_1 = value_1 + 1;
-SELECT count(*) FROM test_table;
-ERROR:  canceling statement due to user request
-COMMIT;
--- Cancel a query with CTE
-SELECT citus.mitmproxy('conn.onQuery(query="SELECT|COPY").cancel(' || pg_backend_pid() || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-WITH
-results AS (SELECT * FROM test_table)
-SELECT * FROM test_table
-WHERE test_table.id > (SELECT id FROM results);
-ERROR:  canceling statement due to user request
--- Since we have the placement on each node, test with sequential mode
--- should pass as well.
-SET citus.multi_shard_modify_mode to sequential;
-SELECT citus.mitmproxy('conn.onQuery(query="SELECT|COPY").after(1).kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-SELECT count(*) FROM test_table;
-WARNING:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
- count 
--------
-     6
-(1 row)
-
-COMMIT;
-DROP SCHEMA real_time_select_failure CASCADE;
-NOTICE:  drop cascades to table test_table
-SET search_path TO default;

src/test/regress/expected/failure_ref_tables.out

@@ -33,10 +33,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()');
 (1 row)
 
 INSERT INTO ref_table VALUES (5, 6);
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 SELECT COUNT(*) FROM ref_table WHERE key=5;
  count 
 -------
@@ -51,10 +51,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()');
 (1 row)
 
 UPDATE ref_table SET key=7 RETURNING value;
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 SELECT COUNT(*) FROM ref_table WHERE key=7;
  count 
 -------
@@ -71,10 +71,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()');
 BEGIN;
 DELETE FROM ref_table WHERE key=5;
 UPDATE ref_table SET key=value;
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 COMMIT;
 SELECT COUNT(*) FROM ref_table WHERE key=value;
  count 

src/test/regress/expected/failure_ref_tables_9.out

@@ -1,99 +0,0 @@
-SET citus.next_shard_id TO 100500;
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-CREATE TABLE ref_table (key int, value int);
-SELECT create_reference_table('ref_table');
- create_reference_table 
-------------------------
- 
-(1 row)
-
-\copy ref_table FROM stdin delimiter ',';
-SELECT citus.clear_network_traffic();
- clear_network_traffic 
------------------------
- 
-(1 row)
-
-SELECT COUNT(*) FROM ref_table;
- count 
--------
-     4
-(1 row)
-
--- verify behavior of single INSERT; should fail to execute
-SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-INSERT INTO ref_table VALUES (5, 6);
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-SELECT COUNT(*) FROM ref_table WHERE key=5;
- count 
--------
-     0
-(1 row)
-
--- verify behavior of UPDATE ... RETURNING; should not execute
-SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-UPDATE ref_table SET key=7 RETURNING value;
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-SELECT COUNT(*) FROM ref_table WHERE key=7;
- count 
--------
-     0
-(1 row)
-
--- verify fix to #2214; should raise error and fail to execute
-SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-DELETE FROM ref_table WHERE key=5;
-UPDATE ref_table SET key=value;
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-COMMIT;
-SELECT COUNT(*) FROM ref_table WHERE key=value;
- count 
--------
-     0
-(1 row)
-
--- all shards should still be healthy
-SELECT COUNT(*) FROM pg_dist_shard_placement WHERE shardstate = 3;
- count 
--------
-     0
-(1 row)
-
--- ==== Clean up, we're done here ====
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-DROP TABLE ref_table;

src/test/regress/expected/failure_savepoints.out

@@ -40,18 +40,16 @@ INSERT INTO artists VALUES (5, 'Asher Lev');
 SAVEPOINT s1;
 WARNING:  connection not open
 CONTEXT:  while executing command on localhost:9060
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
-DELETE FROM artists WHERE id=4;
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
 WARNING:  connection error: localhost:9060
 DETAIL:  connection not open
 WARNING:  connection not open
 CONTEXT:  while executing command on localhost:9060
 WARNING:  connection not open
 CONTEXT:  while executing command on localhost:9060
-ERROR:  could not modify any active placements
+ERROR:  connection not open
+CONTEXT:  while executing command on localhost:9060
+DELETE FROM artists WHERE id=4;
+ERROR:  current transaction is aborted, commands ignored until end of transaction block
 RELEASE SAVEPOINT s1;
 ERROR:  current transaction is aborted, commands ignored until end of transaction block
 COMMIT;
@@ -131,12 +129,18 @@ RELEASE SAVEPOINT s1;
 SAVEPOINT s2;
 INSERT INTO artists VALUES (5, 'Jacob Kahn');
 RELEASE SAVEPOINT s2;
+WARNING:  AbortSubTransaction while in COMMIT state
+WARNING:  connection not open
+CONTEXT:  while executing command on localhost:9060
+WARNING:  connection error: localhost:9060
+DETAIL:  connection not open
 WARNING:  connection not open
 CONTEXT:  while executing command on localhost:9060
 WARNING:  connection not open
 CONTEXT:  while executing command on localhost:9060
+ERROR:  connection not open
+CONTEXT:  while executing command on localhost:9060
 COMMIT;
-ERROR:  could not make changes to shard 100950 on any node
 SELECT * FROM artists WHERE id IN (4, 5);
  id |      name       
 ----+-----------------
@@ -208,6 +212,10 @@ INSERT INTO artists VALUES (7, 'Emily Carr');
 ROLLBACK TO SAVEPOINT s1;
 WARNING:  connection not open
 WARNING:  connection not open
+WARNING:  connection not open
+WARNING:  connection error: localhost:9060
+WARNING:  connection not open
+WARNING:  connection not open
 COMMIT;
 ERROR:  could not make changes to shard 100950 on any node
 SELECT * FROM artists WHERE id=6;
@@ -240,24 +248,22 @@ BEGIN;
 INSERT INTO researchers VALUES (7, 4, 'Jan Plaza');
 SAVEPOINT s1;
 WARNING:  connection not open
-WARNING:  connection not open
-INSERT INTO researchers VALUES (8, 4, 'Alonzo Church');
-ROLLBACK TO s1;
-WARNING:  connection not open
 WARNING:  connection error: localhost:9060
 WARNING:  connection not open
 WARNING:  connection not open
+ERROR:  connection not open
+INSERT INTO researchers VALUES (8, 4, 'Alonzo Church');
+ERROR:  current transaction is aborted, commands ignored until end of transaction block
+ROLLBACK TO s1;
+ERROR:  savepoint "s1" does not exist
 RELEASE SAVEPOINT s1;
+ERROR:  current transaction is aborted, commands ignored until end of transaction block
 COMMIT;
-WARNING:  connection not open
-WARNING:  connection not open
-WARNING:  connection not open
 -- should see correct results from healthy placement and one bad placement
 SELECT * FROM researchers WHERE lab_id = 4;
- id | lab_id |   name    
+ id | lab_id | name 
-----+--------+-----------
+----+--------+------
-  7 |      4 | Jan Plaza
+(0 rows)
-(1 row)
 
 UPDATE pg_dist_shard_placement SET shardstate = 1
 WHERE shardstate = 3 AND shardid IN (
@@ -265,8 +271,7 @@ WHERE shardstate = 3 AND shardid IN (
 ) RETURNING placementid;
  placementid 
 -------------
-         152
+(0 rows)
-(1 row)
 
 TRUNCATE researchers;
 -- fail at rollback
@@ -285,15 +290,12 @@ WARNING:  connection not open
 WARNING:  connection not open
 RELEASE SAVEPOINT s1;
 COMMIT;
-WARNING:  connection not open
+ERROR:  failure on connection marked as essential: localhost:9060
-WARNING:  connection not open
-WARNING:  connection not open
 -- should see correct results from healthy placement and one bad placement
 SELECT * FROM researchers WHERE lab_id = 4;
- id | lab_id |   name    
+ id | lab_id | name 
-----+--------+-----------
+----+--------+------
-  7 |      4 | Jan Plaza
+(0 rows)
-(1 row)
 
 UPDATE pg_dist_shard_placement SET shardstate = 1
 WHERE shardstate = 3 AND shardid IN (
@@ -301,8 +303,7 @@ WHERE shardstate = 3 AND shardid IN (
 ) RETURNING placementid;
  placementid 
 -------------
-         152
+(0 rows)
-(1 row)
 
 TRUNCATE researchers;
 -- fail at release
@@ -318,18 +319,19 @@ SAVEPOINT s1;
 INSERT INTO researchers VALUES (8, 4, 'Alonzo Church');
 ROLLBACK TO s1;
 RELEASE SAVEPOINT s1;
+WARNING:  AbortSubTransaction while in COMMIT state
+WARNING:  connection not open
+WARNING:  connection error: localhost:9060
 WARNING:  connection not open
 WARNING:  connection not open
+WARNING:  savepoint "savepoint_3" does not exist
+ERROR:  connection not open
 COMMIT;
-WARNING:  connection not open
-WARNING:  connection not open
-WARNING:  connection not open
 -- should see correct results from healthy placement and one bad placement
 SELECT * FROM researchers WHERE lab_id = 4;
- id | lab_id |   name    
+ id | lab_id | name 
-----+--------+-----------
+----+--------+------
-  7 |      4 | Jan Plaza
+(0 rows)
-(1 row)
 
 UPDATE pg_dist_shard_placement SET shardstate = 1
 WHERE shardstate = 3 AND shardid IN (
@@ -337,8 +339,7 @@ WHERE shardstate = 3 AND shardid IN (
 ) RETURNING placementid;
  placementid 
 -------------
-         152
+(0 rows)
-(1 row)
 
 TRUNCATE researchers;
 -- clean up

src/test/regress/expected/failure_savepoints_9.out

@@ -1,353 +0,0 @@
--- We have two different output files for this failure test because the
--- failure behaviour of SAVEPOINT and RELEASE commands are different if
--- we use the executor. If we use it, these commands error out if any of
--- the placement commands fail. Otherwise, we might mark the placement
--- as invalid and continue with a WARNING.
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SET citus.shard_count = 2;
-SET citus.shard_replication_factor = 1; -- one shard per worker
-SET citus.next_shard_id TO 100950;
-ALTER SEQUENCE pg_catalog.pg_dist_placement_placementid_seq RESTART 150;
-CREATE TABLE artists (
-    id bigint NOT NULL,
-    name text NOT NULL
-);
-SELECT create_distributed_table('artists', 'id');
- create_distributed_table 
---------------------------
- 
-(1 row)
-
--- add some data
-INSERT INTO artists VALUES (1, 'Pablo Picasso');
-INSERT INTO artists VALUES (2, 'Vincent van Gogh');
-INSERT INTO artists VALUES (3, 'Claude Monet');
-INSERT INTO artists VALUES (4, 'William Kurelek');
--- simply fail at SAVEPOINT
-SELECT citus.mitmproxy('conn.onQuery(query="^SAVEPOINT").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-INSERT INTO artists VALUES (5, 'Asher Lev');
-SAVEPOINT s1;
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
-WARNING:  connection error: localhost:9060
-DETAIL:  connection not open
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
-ERROR:  connection not open
-CONTEXT:  while executing command on localhost:9060
-DELETE FROM artists WHERE id=4;
-ERROR:  current transaction is aborted, commands ignored until end of transaction block
-RELEASE SAVEPOINT s1;
-ERROR:  current transaction is aborted, commands ignored until end of transaction block
-COMMIT;
-SELECT * FROM artists WHERE id IN (4, 5);
- id |      name       
-----+-----------------
-  4 | William Kurelek
-(1 row)
-
--- fail at RELEASE
-SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-UPDATE artists SET name='a';
-SAVEPOINT s1;
-DELETE FROM artists WHERE id=4;
-RELEASE SAVEPOINT s1;
-WARNING:  AbortSubTransaction while in COMMIT state
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
-WARNING:  connection error: localhost:9060
-DETAIL:  connection not open
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
-WARNING:  savepoint "savepoint_2" does not exist
-CONTEXT:  while executing command on localhost:57637
-ERROR:  connection not open
-CONTEXT:  while executing command on localhost:9060
-ROLLBACK;
-SELECT * FROM artists WHERE id IN (4, 5);
- id |      name       
-----+-----------------
-  4 | William Kurelek
-(1 row)
-
--- fail at ROLLBACK
-SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-INSERT INTO artists VALUES (5, 'Asher Lev');
-SAVEPOINT s1;
-DELETE FROM artists WHERE id=4;
-ROLLBACK TO SAVEPOINT s1;
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
-COMMIT;
-ERROR:  could not make changes to shard 100950 on any node
-SELECT * FROM artists WHERE id IN (4, 5);
- id |      name       
-----+-----------------
-  4 | William Kurelek
-(1 row)
-
--- fail at second RELEASE
-SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").after(1).kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-SAVEPOINT s1;
-DELETE FROM artists WHERE id=4;
-RELEASE SAVEPOINT s1;
-SAVEPOINT s2;
-INSERT INTO artists VALUES (5, 'Jacob Kahn');
-RELEASE SAVEPOINT s2;
-WARNING:  AbortSubTransaction while in COMMIT state
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
-WARNING:  connection error: localhost:9060
-DETAIL:  connection not open
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
-ERROR:  connection not open
-CONTEXT:  while executing command on localhost:9060
-COMMIT;
-SELECT * FROM artists WHERE id IN (4, 5);
- id |      name       
-----+-----------------
-  4 | William Kurelek
-(1 row)
-
--- fail at second ROLLBACK
-SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").after(1).kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-SAVEPOINT s1;
-UPDATE artists SET name='A' WHERE id=4;
-ROLLBACK TO SAVEPOINT s1;
-SAVEPOINT s2;
-DELETE FROM artists WHERE id=5;
-ROLLBACK TO SAVEPOINT s2;
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
-COMMIT;
-ERROR:  could not make changes to shard 100950 on any node
-SELECT * FROM artists WHERE id IN (4, 5);
- id |      name       
-----+-----------------
-  4 | William Kurelek
-(1 row)
-
-SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").after(1).kill()');
- mitmproxy 
------------
- 
-(1 row)
-
--- Release after rollback
-BEGIN;
-SAVEPOINT s1;
-ROLLBACK TO s1;
-RELEASE SAVEPOINT s1;
-SAVEPOINT s2;
-INSERT INTO artists VALUES (6, 'John J. Audubon');
-INSERT INTO artists VALUES (7, 'Emily Carr');
-ROLLBACK TO s2;
-RELEASE SAVEPOINT s2;
-COMMIT;
-SELECT * FROM artists WHERE id=7;
- id | name 
-----+------
-(0 rows)
-
-SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
--- Recover from errors
-\set VERBOSITY terse
-BEGIN;
-SAVEPOINT s1;
-SAVEPOINT s2;
-INSERT INTO artists VALUES (6, 'John J. Audubon');
-INSERT INTO artists VALUES (7, 'Emily Carr');
-INSERT INTO artists VALUES (7, 'Emily Carr');
-ROLLBACK TO SAVEPOINT s1;
-WARNING:  connection not open
-WARNING:  connection not open
-WARNING:  connection not open
-WARNING:  connection error: localhost:9060
-WARNING:  connection not open
-WARNING:  connection not open
-COMMIT;
-ERROR:  could not make changes to shard 100950 on any node
-SELECT * FROM artists WHERE id=6;
- id | name 
-----+------
-(0 rows)
-
--- replication factor > 1
-CREATE TABLE researchers (
-  id bigint NOT NULL,
-  lab_id int NOT NULL,
-  name text NOT NULL
-);
-SET citus.shard_count = 1;
-SET citus.shard_replication_factor = 2; -- single shard, on both workers
-SELECT create_distributed_table('researchers', 'lab_id', 'hash');
- create_distributed_table 
---------------------------
- 
-(1 row)
-
--- simply fail at SAVEPOINT
-SELECT citus.mitmproxy('conn.onQuery(query="^SAVEPOINT").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-INSERT INTO researchers VALUES (7, 4, 'Jan Plaza');
-SAVEPOINT s1;
-WARNING:  connection not open
-WARNING:  connection error: localhost:9060
-WARNING:  connection not open
-WARNING:  connection not open
-ERROR:  connection not open
-INSERT INTO researchers VALUES (8, 4, 'Alonzo Church');
-ERROR:  current transaction is aborted, commands ignored until end of transaction block
-ROLLBACK TO s1;
-ERROR:  savepoint "s1" does not exist
-RELEASE SAVEPOINT s1;
-ERROR:  current transaction is aborted, commands ignored until end of transaction block
-COMMIT;
--- should see correct results from healthy placement and one bad placement
-SELECT * FROM researchers WHERE lab_id = 4;
- id | lab_id | name 
-----+--------+------
-(0 rows)
-
-UPDATE pg_dist_shard_placement SET shardstate = 1
-WHERE shardstate = 3 AND shardid IN (
-  SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'researchers'::regclass
-) RETURNING placementid;
- placementid 
--------------
-(0 rows)
-
-TRUNCATE researchers;
--- fail at rollback
-SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-INSERT INTO researchers VALUES (7, 4, 'Jan Plaza');
-SAVEPOINT s1;
-INSERT INTO researchers VALUES (8, 4, 'Alonzo Church');
-ROLLBACK TO s1;
-WARNING:  connection not open
-WARNING:  connection not open
-RELEASE SAVEPOINT s1;
-COMMIT;
-ERROR:  failure on connection marked as essential: localhost:9060
--- should see correct results from healthy placement and one bad placement
-SELECT * FROM researchers WHERE lab_id = 4;
- id | lab_id | name 
-----+--------+------
-(0 rows)
-
-UPDATE pg_dist_shard_placement SET shardstate = 1
-WHERE shardstate = 3 AND shardid IN (
-  SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'researchers'::regclass
-) RETURNING placementid;
- placementid 
--------------
-(0 rows)
-
-TRUNCATE researchers;
--- fail at release
-SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-INSERT INTO researchers VALUES (7, 4, 'Jan Plaza');
-SAVEPOINT s1;
-INSERT INTO researchers VALUES (8, 4, 'Alonzo Church');
-ROLLBACK TO s1;
-RELEASE SAVEPOINT s1;
-WARNING:  AbortSubTransaction while in COMMIT state
-WARNING:  connection not open
-WARNING:  connection error: localhost:9060
-WARNING:  connection not open
-WARNING:  connection not open
-WARNING:  savepoint "savepoint_3" does not exist
-ERROR:  connection not open
-COMMIT;
--- should see correct results from healthy placement and one bad placement
-SELECT * FROM researchers WHERE lab_id = 4;
- id | lab_id | name 
-----+--------+------
-(0 rows)
-
-UPDATE pg_dist_shard_placement SET shardstate = 1
-WHERE shardstate = 3 AND shardid IN (
-  SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'researchers'::regclass
-) RETURNING placementid;
- placementid 
--------------
-(0 rows)
-
-TRUNCATE researchers;
--- clean up
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-DROP TABLE artists;
-DROP TABLE researchers;

src/test/regress/expected/failure_single_mod.out

@@ -27,10 +27,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()');
 (1 row)
 
 INSERT INTO mod_test VALUES (2, 6);
-WARNING:  server closed the connection unexpectedly
+WARNING:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 SELECT COUNT(*) FROM mod_test WHERE key=2;
  count 
 -------
@@ -63,10 +63,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()');
 (1 row)
 
 UPDATE mod_test SET value='ok' WHERE key=2 RETURNING key;
-WARNING:  server closed the connection unexpectedly
+WARNING:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
  key 
 -----
    2
@@ -102,17 +102,11 @@ INSERT INTO mod_test VALUES (2, 6);
 INSERT INTO mod_test VALUES (2, 7);
 DELETE FROM mod_test WHERE key=2 AND value = '7';
 UPDATE mod_test SET value='ok' WHERE key=2;
-WARNING:  server closed the connection unexpectedly
+WARNING:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 COMMIT;
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
 SELECT COUNT(*) FROM mod_test WHERE key=2;
  count 
 -------

src/test/regress/expected/failure_single_mod_9.out

@@ -1,128 +0,0 @@
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT citus.clear_network_traffic();
- clear_network_traffic 
------------------------
- 
-(1 row)
-
-SET citus.shard_count = 2;
-SET citus.shard_replication_factor = 2;
-CREATE TABLE mod_test (key int, value text);
-SELECT create_distributed_table('mod_test', 'key');
- create_distributed_table 
---------------------------
- 
-(1 row)
-
--- verify behavior of single INSERT; should mark shard as failed
-SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-INSERT INTO mod_test VALUES (2, 6);
-WARNING:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-SELECT COUNT(*) FROM mod_test WHERE key=2;
- count 
--------
-     1
-(1 row)
-
--- some clean up
-UPDATE pg_dist_shard_placement SET shardstate = 1
-WHERE shardid IN (
-  SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'mod_test'::regclass
-) AND shardstate = 3 RETURNING placementid;
- placementid 
--------------
-         137
-(1 row)
-
-TRUNCATE mod_test;
--- verify behavior of UPDATE ... RETURNING; should mark as failed
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-INSERT INTO mod_test VALUES (2, 6);
-SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-UPDATE mod_test SET value='ok' WHERE key=2 RETURNING key;
-WARNING:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
- key 
------
-   2
-(1 row)
-
-SELECT COUNT(*) FROM mod_test WHERE value='ok';
- count 
--------
-     1
-(1 row)
-
--- some clean up
-UPDATE pg_dist_shard_placement SET shardstate = 1
-WHERE shardid IN (
-  SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'mod_test'::regclass
-) AND shardstate = 3 RETURNING placementid;
- placementid 
--------------
-         137
-(1 row)
-
-TRUNCATE mod_test;
--- verify behavior of multi-statement modifications to a single shard
--- should succeed but mark a placement as failed
-SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-INSERT INTO mod_test VALUES (2, 6);
-INSERT INTO mod_test VALUES (2, 7);
-DELETE FROM mod_test WHERE key=2 AND value = '7';
-UPDATE mod_test SET value='ok' WHERE key=2;
-WARNING:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-COMMIT;
-SELECT COUNT(*) FROM mod_test WHERE key=2;
- count 
--------
-     1
-(1 row)
-
--- some clean up
-UPDATE pg_dist_shard_placement SET shardstate = 1
-WHERE shardid IN (
-  SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'mod_test'::regclass
-) AND shardstate = 3 RETURNING placementid;
- placementid 
--------------
-         137
-(1 row)
-
-TRUNCATE mod_test;
--- ==== Clean up, we're done here ====
-DROP TABLE mod_test;

src/test/regress/expected/failure_single_select_9.out

@@ -1,247 +0,0 @@
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT citus.clear_network_traffic();
- clear_network_traffic 
------------------------
- 
-(1 row)
-
-SET citus.shard_count = 2;
-SET citus.shard_replication_factor = 2;
-CREATE TABLE select_test (key int, value text);
-SELECT create_distributed_table('select_test', 'key');
- create_distributed_table 
---------------------------
- 
-(1 row)
-
--- put data in shard for which mitm node is first placement
-INSERT INTO select_test VALUES (3, 'test data');
-SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT * FROM select_test WHERE key = 3;
-WARNING:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
- key |   value   
------+-----------
-   3 | test data
-(1 row)
-
-SELECT * FROM select_test WHERE key = 3;
-WARNING:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
- key |   value   
------+-----------
-   3 | test data
-(1 row)
-
--- kill after first SELECT; txn should work (though placement marked bad)
-SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-INSERT INTO select_test VALUES (3, 'more data');
-SELECT * FROM select_test WHERE key = 3;
-WARNING:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
- key |   value   
------+-----------
-   3 | test data
-   3 | more data
-(2 rows)
-
-INSERT INTO select_test VALUES (3, 'even more data');
-SELECT * FROM select_test WHERE key = 3;
- key |     value      
------+----------------
-   3 | test data
-   3 | more data
-   3 | even more data
-(3 rows)
-
-COMMIT;
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
--- some clean up
-UPDATE pg_dist_shard_placement SET shardstate = 1
-WHERE shardid IN (
-  SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'select_test'::regclass
-);
-TRUNCATE select_test;
--- now the same tests with query cancellation
--- put data in shard for which mitm node is first placement
-INSERT INTO select_test VALUES (3, 'test data');
-SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").cancel(' ||  pg_backend_pid() || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT * FROM select_test WHERE key = 3;
-ERROR:  canceling statement due to user request
-SELECT * FROM select_test WHERE key = 3;
-ERROR:  canceling statement due to user request
--- cancel after first SELECT; txn should fail and nothing should be marked as invalid
-SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").cancel(' ||  pg_backend_pid() || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-INSERT INTO select_test VALUES (3, 'more data');
-SELECT * FROM select_test WHERE key = 3;
-ERROR:  canceling statement due to user request
-COMMIT;
--- show that all placements are OK
-SELECT DISTINCT shardstate FROM  pg_dist_shard_placement
-WHERE shardid IN (
-  SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'select_test'::regclass
-);
- shardstate 
-------------
-          1
-(1 row)
-
-TRUNCATE select_test;
--- cancel the second query
--- error after second SELECT; txn should fail
-SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").after(1).cancel(' ||  pg_backend_pid() || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-INSERT INTO select_test VALUES (3, 'more data');
-SELECT * FROM select_test WHERE key = 3;
- key |   value   
------+-----------
-   3 | more data
-(1 row)
-
-INSERT INTO select_test VALUES (3, 'even more data');
-SELECT * FROM select_test WHERE key = 3;
-ERROR:  canceling statement due to user request
-COMMIT;
--- error after second SELECT; txn should work (though placement marked bad)
-SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").after(1).reset()');
- mitmproxy 
------------
- 
-(1 row)
-
-BEGIN;
-INSERT INTO select_test VALUES (3, 'more data');
-SELECT * FROM select_test WHERE key = 3;
- key |   value   
------+-----------
-   3 | more data
-(1 row)
-
-INSERT INTO select_test VALUES (3, 'even more data');
-SELECT * FROM select_test WHERE key = 3;
-WARNING:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
- key |     value      
------+----------------
-   3 | more data
-   3 | even more data
-(2 rows)
-
-COMMIT;
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
-SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").after(2).kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT recover_prepared_transactions();
- recover_prepared_transactions 
--------------------------------
-                             0
-(1 row)
-
-SELECT recover_prepared_transactions();
-ERROR:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
--- bug from https://github.com/citusdata/citus/issues/1926
-SET citus.max_cached_conns_per_worker TO 0; -- purge cache
-DROP TABLE select_test;
-SET citus.shard_count = 2;
-SET citus.shard_replication_factor = 1;
-CREATE TABLE select_test (key int, value text);
-SELECT create_distributed_table('select_test', 'key');
- create_distributed_table 
---------------------------
- 
-(1 row)
-
-SET citus.max_cached_conns_per_worker TO 1; -- allow connection to be cached
-INSERT INTO select_test VALUES (1, 'test data');
-SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").after(1).kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT * FROM select_test WHERE key = 1;
- key |   value   
------+-----------
-   1 | test data
-(1 row)
-
-SELECT * FROM select_test WHERE key = 1;
-WARNING:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
-ERROR:  could not receive query results
--- now the same test with query cancellation
-SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").after(1).cancel(' ||  pg_backend_pid() || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-SELECT * FROM select_test WHERE key = 1;
- key |   value   
------+-----------
-   1 | test data
-(1 row)
-
-SELECT * FROM select_test WHERE key = 1;
-ERROR:  canceling statement due to user request
--- ==== Clean up, we're done here ====
-DROP TABLE select_test;

src/test/regress/expected/failure_truncate.out

@@ -102,7 +102,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 TRUNCATE test_table;
-ERROR:  failure on connection marked as essential: localhost:9060
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
+	This probably means the server terminated abnormally
+	before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy 
 -----------
@@ -156,10 +159,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="TRUNCATE TABLE truncate_failure.test
 (1 row)
 
 TRUNCATE test_table;
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy 
 -----------
@@ -430,10 +433,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="^TRUNCATE TABLE").after(2).kill()');
 (1 row)
 
 TRUNCATE reference_table CASCADE;
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy 
 -----------
@@ -632,7 +635,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 TRUNCATE test_table;
-ERROR:  failure on connection marked as essential: localhost:9060
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
+	This probably means the server terminated abnormally
+	before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy 
 -----------
@@ -686,10 +692,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="^TRUNCATE TABLE truncate_failure.tes
 (1 row)
 
 TRUNCATE test_table;
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy 
 -----------
@@ -1008,7 +1014,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL R
 (1 row)
 
 TRUNCATE test_table;
-ERROR:  failure on connection marked as essential: localhost:9060
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
+	This probably means the server terminated abnormally
+	before or while processing the request.
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy 
 -----------
@@ -1062,10 +1071,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="TRUNCATE TABLE truncate_failure.test
 (1 row)
 
 TRUNCATE test_table;
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy 
 -----------

src/test/regress/expected/failure_vacuum.out

@@ -31,10 +31,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="^VACUUM").kill()');
 (1 row)
 
 VACUUM vacuum_test;
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 SELECT citus.mitmproxy('conn.onQuery(query="^ANALYZE").kill()');
  mitmproxy 
 -----------
@@ -42,10 +42,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="^ANALYZE").kill()');
 (1 row)
 
 ANALYZE vacuum_test;
-ERROR:  server closed the connection unexpectedly
+WARNING:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT").kill()');
  mitmproxy 
 -----------
@@ -53,11 +53,6 @@ SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT").kill()');
 (1 row)
 
 ANALYZE vacuum_test;
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
-WARNING:  failed to commit transaction on localhost:9060
-WARNING:  connection not open
-CONTEXT:  while executing command on localhost:9060
 -- ANALYZE transactions being critical is an open question, see #2430
 -- show that we marked as INVALID on COMMIT FAILURE
 SELECT shardid, shardstate FROM pg_dist_shard_placement where shardstate != 1 AND 
@@ -116,10 +111,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="^VACUUM.*other").kill()');
 (1 row)
 
 VACUUM vacuum_test, other_vacuum_test;
-ERROR:  server closed the connection unexpectedly
+ERROR:  connection error: localhost:9060
+DETAIL:  server closed the connection unexpectedly
 	This probably means the server terminated abnormally
 	before or while processing the request.
-CONTEXT:  while executing command on localhost:9060
 SELECT citus.mitmproxy('conn.onQuery(query="^VACUUM.*other").cancel(' ||  pg_backend_pid() || ')');
  mitmproxy 
 -----------

src/test/regress/expected/failure_vacuum_9.out

@@ -1,133 +0,0 @@
--- We have different output files for the executor. This is because
--- we don't mark transactions with ANALYZE as critical anymore, and
--- get WARNINGs instead of ERRORs.
-SET citus.next_shard_id TO 12000000;
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-SET citus.shard_count = 1;
-SET citus.shard_replication_factor = 2; -- one shard per worker
-SET citus.multi_shard_commit_protocol TO '1pc';
-CREATE TABLE vacuum_test (key int, value int);
-SELECT create_distributed_table('vacuum_test', 'key');
- create_distributed_table 
---------------------------
- 
-(1 row)
-
-SELECT citus.clear_network_traffic();
- clear_network_traffic 
------------------------
- 
-(1 row)
-
-SELECT citus.mitmproxy('conn.onQuery(query="^VACUUM").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-VACUUM vacuum_test;
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-SELECT citus.mitmproxy('conn.onQuery(query="^ANALYZE").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-ANALYZE vacuum_test;
-WARNING:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-ANALYZE vacuum_test;
--- ANALYZE transactions being critical is an open question, see #2430
--- show that we marked as INVALID on COMMIT FAILURE
-SELECT shardid, shardstate FROM pg_dist_shard_placement where shardstate != 1 AND 
-shardid in ( SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'vacuum_test'::regclass);
- shardid  | shardstate 
-----------+------------
- 12000000 |          3
-(1 row)
-
-UPDATE pg_dist_shard_placement SET shardstate = 1
-WHERE shardid IN (
-  SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'vacuum_test'::regclass
-);
--- the same tests with cancel
-SELECT citus.mitmproxy('conn.onQuery(query="^VACUUM").cancel(' ||  pg_backend_pid() || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-VACUUM vacuum_test;
-ERROR:  canceling statement due to user request
-SELECT citus.mitmproxy('conn.onQuery(query="^ANALYZE").cancel(' ||  pg_backend_pid() || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-ANALYZE vacuum_test;
-ERROR:  canceling statement due to user request
--- cancel during COMMIT should be ignored
-SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT").cancel(' ||  pg_backend_pid() || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-ANALYZE vacuum_test;
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-CREATE TABLE other_vacuum_test (key int, value int);
-SELECT create_distributed_table('other_vacuum_test', 'key');
- create_distributed_table 
---------------------------
- 
-(1 row)
-
-SELECT citus.mitmproxy('conn.onQuery(query="^VACUUM.*other").kill()');
- mitmproxy 
------------
- 
-(1 row)
-
-VACUUM vacuum_test, other_vacuum_test;
-ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-SELECT citus.mitmproxy('conn.onQuery(query="^VACUUM.*other").cancel(' ||  pg_backend_pid() || ')');
- mitmproxy 
------------
- 
-(1 row)
-
-VACUUM vacuum_test, other_vacuum_test;
-ERROR:  canceling statement due to user request
--- ==== Clean up, we're done here ====
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy 
------------
- 
-(1 row)
-
-DROP TABLE vacuum_test, other_vacuum_test;

src/test/regress/expected/isolation_citus_dist_activity_9.out

@@ -1,229 +0,0 @@
-Parsed test spec with 3 sessions
-
-starting permutation: s1-cache-connections s1-begin s2-begin s3-begin s1-alter-table s2-sleep s2-view-dist s3-view-worker s2-rollback s1-commit s3-rollback
-create_distributed_table
-
-               
-step s1-cache-connections: 
-    SET citus.max_cached_conns_per_worker TO 4;
-    SET citus.force_max_query_parallelization TO on;
-    UPDATE test_table SET column2 = 0;
-
-step s1-begin: 
-    BEGIN;
-
-step s2-begin: 
-	BEGIN;
-
-step s3-begin: 
-	BEGIN;
-
-step s1-alter-table: 
-    ALTER TABLE test_table ADD COLUMN x INT;
-
-step s2-sleep: 
-	SELECT pg_sleep(0.5);
-
-pg_sleep       
-
-               
-step s2-view-dist: 
-	SELECT query, query_hostname, query_hostport, master_query_host_name, master_query_host_port, state, wait_event_type, wait_event, usename, datname FROM citus_dist_stat_activity WHERE query NOT ILIKE '%pg_prepared_xacts%' AND query NOT ILIKE '%COMMIT%' ORDER BY query DESC;
-	
-
-query          query_hostname query_hostport master_query_host_namemaster_query_host_portstate          wait_event_typewait_event     usename        datname        
-
-
-    ALTER TABLE test_table ADD COLUMN x INT;
-coordinator_host57636          coordinator_host57636          idle in transactionClient         ClientRead     postgres       regression     
-step s3-view-worker: 
-	SELECT query, query_hostname, query_hostport, master_query_host_name, master_query_host_port, state, wait_event_type, wait_event, usename, datname FROM citus_worker_stat_activity WHERE query NOT ILIKE '%pg_prepared_xacts%' AND query NOT ILIKE '%COMMIT%' ORDER BY query DESC;
-
-query          query_hostname query_hostport master_query_host_namemaster_query_host_portstate          wait_event_typewait_event     usename        datname        
-
-SELECT worker_apply_shard_ddl_command (1300004, 'public', '
-    ALTER TABLE test_table ADD COLUMN x INT;
-')localhost      57638          coordinator_host57636          idle in transactionClient         ClientRead     postgres       regression     
-SELECT worker_apply_shard_ddl_command (1300003, 'public', '
-    ALTER TABLE test_table ADD COLUMN x INT;
-')localhost      57637          coordinator_host57636          idle in transactionClient         ClientRead     postgres       regression     
-SELECT worker_apply_shard_ddl_command (1300002, 'public', '
-    ALTER TABLE test_table ADD COLUMN x INT;
-')localhost      57638          coordinator_host57636          idle in transactionClient         ClientRead     postgres       regression     
-SELECT worker_apply_shard_ddl_command (1300001, 'public', '
-    ALTER TABLE test_table ADD COLUMN x INT;
-')localhost      57637          coordinator_host57636          idle in transactionClient         ClientRead     postgres       regression     
-step s2-rollback: 
-	ROLLBACK;
-
-step s1-commit: 
-    COMMIT;
-
-step s3-rollback: 
-	ROLLBACK;
-
-
-starting permutation: s1-cache-connections s1-begin s2-begin s3-begin s1-insert s2-sleep s2-view-dist s3-view-worker s2-rollback s1-commit s3-rollback
-create_distributed_table
-
-               
-step s1-cache-connections: 
-    SET citus.max_cached_conns_per_worker TO 4;
-    SET citus.force_max_query_parallelization TO on;
-    UPDATE test_table SET column2 = 0;
-
-step s1-begin: 
-    BEGIN;
-
-step s2-begin: 
-	BEGIN;
-
-step s3-begin: 
-	BEGIN;
-
-step s1-insert: 
- 	INSERT INTO test_table VALUES (100, 100);
-
-step s2-sleep: 
-	SELECT pg_sleep(0.5);
-
-pg_sleep       
-
-               
-step s2-view-dist: 
-	SELECT query, query_hostname, query_hostport, master_query_host_name, master_query_host_port, state, wait_event_type, wait_event, usename, datname FROM citus_dist_stat_activity WHERE query NOT ILIKE '%pg_prepared_xacts%' AND query NOT ILIKE '%COMMIT%' ORDER BY query DESC;
-	
-
-query          query_hostname query_hostport master_query_host_namemaster_query_host_portstate          wait_event_typewait_event     usename        datname        
-
-
- 	INSERT INTO test_table VALUES (100, 100);
-coordinator_host57636          coordinator_host57636          idle in transactionClient         ClientRead     postgres       regression     
-step s3-view-worker: 
-	SELECT query, query_hostname, query_hostport, master_query_host_name, master_query_host_port, state, wait_event_type, wait_event, usename, datname FROM citus_worker_stat_activity WHERE query NOT ILIKE '%pg_prepared_xacts%' AND query NOT ILIKE '%COMMIT%' ORDER BY query DESC;
-
-query          query_hostname query_hostport master_query_host_namemaster_query_host_portstate          wait_event_typewait_event     usename        datname        
-
-INSERT INTO public.test_table_1300008 (column1, column2) VALUES (100, 100)localhost      57637          coordinator_host57636          idle in transactionClient         ClientRead     postgres       regression     
-step s2-rollback: 
-	ROLLBACK;
-
-step s1-commit: 
-    COMMIT;
-
-step s3-rollback: 
-	ROLLBACK;
-
-
-starting permutation: s1-cache-connections s1-begin s2-begin s3-begin s1-select s2-sleep s2-view-dist s3-view-worker s2-rollback s1-commit s3-rollback
-create_distributed_table
-
-               
-step s1-cache-connections: 
-    SET citus.max_cached_conns_per_worker TO 4;
-    SET citus.force_max_query_parallelization TO on;
-    UPDATE test_table SET column2 = 0;
-
-step s1-begin: 
-    BEGIN;
-
-step s2-begin: 
-	BEGIN;
-
-step s3-begin: 
-	BEGIN;
-
-step s1-select: 
-   SELECT count(*) FROM test_table;
-
-count          
-
-0              
-step s2-sleep: 
-	SELECT pg_sleep(0.5);
-
-pg_sleep       
-
-               
-step s2-view-dist: 
-	SELECT query, query_hostname, query_hostport, master_query_host_name, master_query_host_port, state, wait_event_type, wait_event, usename, datname FROM citus_dist_stat_activity WHERE query NOT ILIKE '%pg_prepared_xacts%' AND query NOT ILIKE '%COMMIT%' ORDER BY query DESC;
-	
-
-query          query_hostname query_hostport master_query_host_namemaster_query_host_portstate          wait_event_typewait_event     usename        datname        
-
-
-   SELECT count(*) FROM test_table;
-coordinator_host57636          coordinator_host57636          idle in transactionClient         ClientRead     postgres       regression     
-step s3-view-worker: 
-	SELECT query, query_hostname, query_hostport, master_query_host_name, master_query_host_port, state, wait_event_type, wait_event, usename, datname FROM citus_worker_stat_activity WHERE query NOT ILIKE '%pg_prepared_xacts%' AND query NOT ILIKE '%COMMIT%' ORDER BY query DESC;
-
-query          query_hostname query_hostport master_query_host_namemaster_query_host_portstate          wait_event_typewait_event     usename        datname        
-
-COPY (SELECT count(*) AS count FROM test_table_1300014 test_table WHERE true) TO STDOUTlocalhost      57638          coordinator_host57636          idle in transactionClient         ClientRead     postgres       regression     
-COPY (SELECT count(*) AS count FROM test_table_1300013 test_table WHERE true) TO STDOUTlocalhost      57637          coordinator_host57636          idle in transactionClient         ClientRead     postgres       regression     
-COPY (SELECT count(*) AS count FROM test_table_1300012 test_table WHERE true) TO STDOUTlocalhost      57638          coordinator_host57636          idle in transactionClient         ClientRead     postgres       regression     
-COPY (SELECT count(*) AS count FROM test_table_1300011 test_table WHERE true) TO STDOUTlocalhost      57637          coordinator_host57636          idle in transactionClient         ClientRead     postgres       regression     
-step s2-rollback: 
-	ROLLBACK;
-
-step s1-commit: 
-    COMMIT;
-
-step s3-rollback: 
-	ROLLBACK;
-
-
-starting permutation: s1-cache-connections s1-begin s2-begin s3-begin s1-select-router s2-sleep s2-view-dist s3-view-worker s2-rollback s1-commit s3-rollback
-create_distributed_table
-
-               
-step s1-cache-connections: 
-    SET citus.max_cached_conns_per_worker TO 4;
-    SET citus.force_max_query_parallelization TO on;
-    UPDATE test_table SET column2 = 0;
-
-step s1-begin: 
-    BEGIN;
-
-step s2-begin: 
-	BEGIN;
-
-step s3-begin: 
-	BEGIN;
-
-step s1-select-router: 
-   SELECT count(*) FROM test_table WHERE column1 = 55;
-
-count          
-
-0              
-step s2-sleep: 
-	SELECT pg_sleep(0.5);
-
-pg_sleep       
-
-               
-step s2-view-dist: 
-	SELECT query, query_hostname, query_hostport, master_query_host_name, master_query_host_port, state, wait_event_type, wait_event, usename, datname FROM citus_dist_stat_activity WHERE query NOT ILIKE '%pg_prepared_xacts%' AND query NOT ILIKE '%COMMIT%' ORDER BY query DESC;
-	
-
-query          query_hostname query_hostport master_query_host_namemaster_query_host_portstate          wait_event_typewait_event     usename        datname        
-
-
-   SELECT count(*) FROM test_table WHERE column1 = 55;
-coordinator_host57636          coordinator_host57636          idle in transactionClient         ClientRead     postgres       regression     
-step s3-view-worker: 
-	SELECT query, query_hostname, query_hostport, master_query_host_name, master_query_host_port, state, wait_event_type, wait_event, usename, datname FROM citus_worker_stat_activity WHERE query NOT ILIKE '%pg_prepared_xacts%' AND query NOT ILIKE '%COMMIT%' ORDER BY query DESC;
-
-query          query_hostname query_hostport master_query_host_namemaster_query_host_portstate          wait_event_typewait_event     usename        datname        
-
-SELECT count(*) AS count FROM public.test_table_1300017 test_table WHERE (column1 OPERATOR(pg_catalog.=) 55)localhost      57638                         0              idle           Client         ClientRead     postgres       regression     
-step s2-rollback: 
-	ROLLBACK;
-
-step s1-commit: 
-    COMMIT;
-
-step s3-rollback: 
-	ROLLBACK;
-

src/test/regress/expected/isolation_dump_global_wait_edges_9.out

@@ -1,98 +0,0 @@
-Parsed test spec with 4 sessions
-
-starting permutation: s1-begin s2-begin s1-update s2-update detector-dump-wait-edges s1-abort s2-abort
-step s1-begin: 
-    BEGIN;
-
-step s2-begin: 
-    BEGIN;
-
-step s1-update: 
-    UPDATE distributed_table SET y = 1 WHERE x = 1;
-
-step s2-update: 
-    UPDATE distributed_table SET y = 2 WHERE x = 1;
- <waiting ...>
-step detector-dump-wait-edges: 
-    SELECT
-        waiting_transaction_num,
-        blocking_transaction_num,
-        blocking_transaction_waiting
-    FROM
-        dump_global_wait_edges()
-    ORDER BY
-        waiting_transaction_num,
-        blocking_transaction_num,
-        blocking_transaction_waiting;
-
-    SELECT * FROM get_adjacency_list_wait_graph() ORDER BY 1;
-
-waiting_transaction_numblocking_transaction_numblocking_transaction_waiting
-
-275            274            f              
-transactionnumberwaitingtransactionnumbers
-
-274                           
-275            274            
-step s1-abort: 
-    ABORT;
-
-step s2-update: <... completed>
-step s2-abort: 
-    ABORT;
-
-
-starting permutation: s1-begin s2-begin s3-begin s1-update s2-update s3-update detector-dump-wait-edges s1-abort s2-abort s3-abort
-step s1-begin: 
-    BEGIN;
-
-step s2-begin: 
-    BEGIN;
-
-step s3-begin: 
-    BEGIN;
-
-step s1-update: 
-    UPDATE distributed_table SET y = 1 WHERE x = 1;
-
-step s2-update: 
-    UPDATE distributed_table SET y = 2 WHERE x = 1;
- <waiting ...>
-step s3-update: 
-    UPDATE distributed_table SET y = 3 WHERE x = 1;
- <waiting ...>
-step detector-dump-wait-edges: 
-    SELECT
-        waiting_transaction_num,
-        blocking_transaction_num,
-        blocking_transaction_waiting
-    FROM
-        dump_global_wait_edges()
-    ORDER BY
-        waiting_transaction_num,
-        blocking_transaction_num,
-        blocking_transaction_waiting;
-
-    SELECT * FROM get_adjacency_list_wait_graph() ORDER BY 1;
-
-waiting_transaction_numblocking_transaction_numblocking_transaction_waiting
-
-279            278            f              
-280            278            f              
-280            279            t              
-transactionnumberwaitingtransactionnumbers
-
-278                           
-279            278            
-280            278,279        
-step s1-abort: 
-    ABORT;
-
-step s2-update: <... completed>
-step s2-abort: 
-    ABORT;
-
-step s3-update: <... completed>
-step s3-abort: 
-    ABORT;
-

src/test/regress/expected/local_shard_execution.out

@@ -959,68 +959,6 @@ LOG:  executing the command locally: DELETE FROM local_shard_execution.reference
 	DELETE FROM reference_table;
 LOG:  executing the command locally: DELETE FROM local_shard_execution.reference_table_1470000 reference_table
 ROLLBACK;
--- mix with other executors should fail
--- router modify execution should error
-BEGIN;
-	DELETE FROM distributed_table WHERE key = 500;
-LOG:  executing the command locally: DELETE FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE (key OPERATOR(pg_catalog.=) 500)
-	SET LOCAL citus.task_executor_type = 'real-time';
-	DELETE FROM distributed_table;
-ERROR:  cannot execute command because a local execution has already been done in the transaction
-DETAIL:  Some parallel commands cannot be executed if a previous command has already been executed locally
-HINT:  Try re-running the transaction with "SET LOCAL citus.enable_local_execution TO OFF;"
-ROLLBACK;
--- local execution should not be executed locally
--- becase a multi-shard router query has already been executed
-BEGIN;
-	SET LOCAL citus.task_executor_type = 'real-time';
-	DELETE FROM distributed_table;
-	DELETE FROM distributed_table WHERE key = 500;
-ROLLBACK;
--- router select execution
-BEGIN;
-	DELETE FROM distributed_table WHERE key = 500;
-LOG:  executing the command locally: DELETE FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE (key OPERATOR(pg_catalog.=) 500)
-	SET LOCAL citus.task_executor_type = 'real-time';
-	SELECT count(*) FROM distributed_table WHERE key = 500;
-ERROR:  cannot execute command because a local execution has already been done in the transaction
-DETAIL:  Some parallel commands cannot be executed if a previous command has already been executed locally
-HINT:  Try re-running the transaction with "SET LOCAL citus.enable_local_execution TO OFF;"
-ROLLBACK;
--- local execution should not be executed locally
--- becase a single-shard router query has already been executed
-BEGIN;
-	SET LOCAL citus.task_executor_type = 'real-time';
-	SELECT count(*) FROM distributed_table WHERE key = 500;
- count 
--------
-     1
-(1 row)
-
-	DELETE FROM distributed_table WHERE key = 500;
-ROLLBACK;
--- real-time select execution
-BEGIN;
-	DELETE FROM distributed_table WHERE key = 500;
-LOG:  executing the command locally: DELETE FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE (key OPERATOR(pg_catalog.=) 500)
-	SET LOCAL citus.task_executor_type = 'real-time';
-	SELECT count(*) FROM distributed_table;
-ERROR:  cannot execute command because a local execution has already been done in the transaction
-DETAIL:  Some parallel commands cannot be executed if a previous command has already been executed locally
-HINT:  Try re-running the transaction with "SET LOCAL citus.enable_local_execution TO OFF;"
-ROLLBACK;
--- local execution should not be executed locally
--- becase a real-time query has already been executed
-BEGIN;
-	SET LOCAL citus.task_executor_type = 'real-time';
-	SELECT count(*) FROM distributed_table;
- count 
--------
-   101
-(1 row)
-
-	DELETE FROM distributed_table WHERE key = 500;
-ROLLBACK;
 -- task-tracker select execution
 BEGIN;
 	DELETE FROM distributed_table WHERE key = 500;

src/test/regress/expected/multi_insert_select.out

@@ -143,7 +143,6 @@ FROM
   raw_events_first
 WHERE
   user_id = 0;
-WARNING:  function public.evaluate_on_master(integer) does not exist
 ERROR:  function public.evaluate_on_master(integer) does not exist
 -- add one more row
 INSERT INTO raw_events_first (user_id, time) VALUES
@@ -1748,7 +1747,6 @@ BEGIN;
 ALTER TABLE reference_table ADD COLUMN z int;
 INSERT INTO raw_events_first (user_id)
 SELECT user_id FROM raw_events_second JOIN reference_table USING (user_id);
-ERROR:  cannot establish a new connection for placement 13300025, since DDL has been executed on a connection that is in use
 ROLLBACK;
 -- the same test with sequential DDL should work fine
 BEGIN;

src/test/regress/expected/multi_join_pruning.out

@@ -74,10 +74,10 @@ EXPLAIN SELECT count(*)
 DEBUG:  Router planner does not support append-partitioned tables.
 DEBUG:  join prunable for intervals [{},{AZZXSP27F21T6,AZZXSP27F21T6}] and [{BA1000U2AMO4ZGX,BZZXSP27F21T6},{CA1000U2AMO4ZGX,CZZXSP27F21T6}]
 DEBUG:  join prunable for intervals [{BA1000U2AMO4ZGX,BZZXSP27F21T6},{CA1000U2AMO4ZGX,CZZXSP27F21T6}] and [{},{AZZXSP27F21T6,AZZXSP27F21T6}]
-                              QUERY PLAN                               
+                              QUERY PLAN                              
------------------------------------------------------------------------
+----------------------------------------------------------------------
  Aggregate  (cost=0.00..0.00 rows=0 width=0)
-   ->  Custom Scan (Citus Real-Time)  (cost=0.00..0.00 rows=0 width=0)
+   ->  Custom Scan (Citus Adaptive)  (cost=0.00..0.00 rows=0 width=0)
          explain statements for distributed queries are not enabled
 (3 rows)
 
@@ -87,10 +87,10 @@ EXPLAIN SELECT count(*)
 DEBUG:  Router planner does not support append-partitioned tables.
 DEBUG:  join prunable for intervals [(a,3,b),(b,4,c)] and [(c,5,d),(d,6,e)]
 DEBUG:  join prunable for intervals [(c,5,d),(d,6,e)] and [(a,3,b),(b,4,c)]
-                              QUERY PLAN                               
+                              QUERY PLAN                              
------------------------------------------------------------------------
+----------------------------------------------------------------------
  Aggregate  (cost=0.00..0.00 rows=0 width=0)
-   ->  Custom Scan (Citus Real-Time)  (cost=0.00..0.00 rows=0 width=0)
+   ->  Custom Scan (Citus Adaptive)  (cost=0.00..0.00 rows=0 width=0)
          explain statements for distributed queries are not enabled
 (3 rows)
 
@@ -101,10 +101,10 @@ EXPLAIN SELECT count(*)
 DEBUG:  Router planner does not support append-partitioned tables.
 DEBUG:  join prunable for intervals [AA1000U2AMO4ZGX,AZZXSP27F21T6] and [BA1000U2AMO4ZGX,BZZXSP27F21T6]
 DEBUG:  join prunable for intervals [BA1000U2AMO4ZGX,BZZXSP27F21T6] and [AA1000U2AMO4ZGX,AZZXSP27F21T6]
-                              QUERY PLAN                               
+                              QUERY PLAN                              
------------------------------------------------------------------------
+----------------------------------------------------------------------
  Aggregate  (cost=0.00..0.00 rows=0 width=0)
-   ->  Custom Scan (Citus Real-Time)  (cost=0.00..0.00 rows=0 width=0)
+   ->  Custom Scan (Citus Adaptive)  (cost=0.00..0.00 rows=0 width=0)
          explain statements for distributed queries are not enabled
 (3 rows)
 

src/test/regress/expected/multi_modifications.out

@@ -348,7 +348,7 @@ ALTER TABLE limit_orders_750000 RENAME TO renamed_orders;
 -- the whole transaction should fail
 \set VERBOSITY terse
 INSERT INTO limit_orders VALUES (276, 'ADR', 140, '2007-07-02 16:32:15', 'sell', 43.67);
-WARNING:  relation "public.limit_orders_750000" does not exist
+ERROR:  relation "public.limit_orders_750000" does not exist
 -- set the shard name back
 \c - - - :worker_2_port
 -- Second: Move aside limit_orders shard on the second worker node
@@ -359,7 +359,7 @@ ALTER TABLE renamed_orders RENAME TO limit_orders_750000;
 SELECT count(*) FROM limit_orders_750000 WHERE id = 276;
  count 
 -------
-     1
+     0
 (1 row)
 
 \c - - - :worker_2_port
@@ -373,7 +373,7 @@ SELECT count(*) FROM limit_orders_750000 WHERE id = 276;
 SELECT count(*) FROM limit_orders WHERE id = 276;
  count 
 -------
-     1
+     0
 (1 row)
 
 SELECT count(*)
@@ -384,7 +384,7 @@ AND    sp.shardstate = 3
 AND    s.logicalrelid = 'limit_orders'::regclass;
  count 
 -------
-     1
+     0
 (1 row)
 
 -- Test that if all shards miss a modification, no state change occurs

src/test/regress/expected/multi_modifying_xacts.out

@@ -433,7 +433,6 @@ SELECT lab_id FROM researchers WHERE lab_id = 2 AND id = 0;
 (0 rows)
 
 ALTER TABLE researchers ADD COLUMN motto text;
-ERROR:  cannot perform a parallel DDL command because multiple placements have been accessed over the same connection
 ROLLBACK;
 -- can perform sequential DDL once a connection is used for multiple shards
 BEGIN;
@@ -528,14 +527,13 @@ FOR EACH ROW EXECUTE PROCEDURE reject_bad();
 BEGIN;
 INSERT INTO objects VALUES (1, 'apple');
 INSERT INTO objects VALUES (2, 'BAD');
-WARNING:  illegal value
+ERROR:  illegal value
 COMMIT;
 -- so the data should noy be persisted
 SELECT * FROM objects WHERE id = 2;
  id | name 
 ----+------
-  2 | BAD
+(0 rows)
-(1 row)
 
 SELECT * FROM labs WHERE id = 7;
  id | name 
@@ -553,7 +551,7 @@ AND    sp.shardstate = 3
 AND    s.logicalrelid = 'objects'::regclass;
  count 
 -------
-     1
+     0
 (1 row)
 
 DELETE FROM objects;
@@ -577,9 +575,11 @@ FOR EACH ROW EXECUTE PROCEDURE reject_bad();
 BEGIN;
 INSERT INTO objects VALUES (1, 'apple');
 INSERT INTO objects VALUES (2, 'BAD');
-INSERT INTO labs VALUES (8, 'Aperture Science');
-INSERT INTO labs VALUES (2, 'BAD');
 ERROR:  illegal value
+INSERT INTO labs VALUES (8, 'Aperture Science');
+ERROR:  current transaction is aborted, commands ignored until end of transaction block
+INSERT INTO labs VALUES (2, 'BAD');
+ERROR:  current transaction is aborted, commands ignored until end of transaction block
 COMMIT;
 -- data should NOT be persisted
 SELECT * FROM objects WHERE id = 1;
@@ -602,7 +602,7 @@ AND    (s.logicalrelid = 'objects'::regclass OR
 	    s.logicalrelid = 'labs'::regclass);
  count 
 -------
-     2
+     3
 (1 row)
 
 -- what if the failures happen at COMMIT time?
@@ -619,6 +619,8 @@ INSERT INTO objects VALUES (1, 'apple');
 INSERT INTO objects VALUES (2, 'BAD');
 INSERT INTO labs VALUES (9, 'Umbrella Corporation');
 COMMIT;
+WARNING:  illegal value
+WARNING:  failed to commit transaction on localhost:57638
 -- data should be persisted
 SELECT * FROM objects WHERE id = 2;
  id | name 
@@ -1308,8 +1310,7 @@ ALTER USER test_user RENAME TO test_user_new;
 \c - test_user - :master_port
 -- fails on all shard placements
 INSERT INTO numbers_hash_failure_test VALUES (2,2);
-WARNING:  connection error: localhost:57638
+ERROR:  connection error: localhost:57638
-ERROR:  could not modify any active placements
 -- connect back to the master with the proper user to continue the tests 
 \c - :default_user - :master_port
 SET citus.next_shard_id TO 1200020;
@@ -1436,7 +1437,6 @@ SELECT id FROM users WHERE id = 6;
 (1 row)
 
 ALTER TABLE items ADD COLUMN last_update timestamptz;
-ERROR:  cannot perform a parallel DDL command because multiple placements have been accessed over the same connection
 ROLLBACK;
 -- can perform sequential DDL after a co-located table has been read over 1 connection
 BEGIN;

src/test/regress/expected/multi_partition_pruning.out

@@ -153,7 +153,7 @@ SELECT count(*) FROM varchar_partitioned_table WHERE varchar_column = 'BA2';
                    QUERY PLAN                    
 -------------------------------------------------
  Aggregate
-   ->  Custom Scan (Citus Real-Time)
+   ->  Custom Scan (Citus Adaptive)
          Task Count: 1
          Tasks Shown: All
          ->  Task
@@ -166,7 +166,7 @@ SELECT count(*) FROM array_partitioned_table
                    QUERY PLAN                    
 -------------------------------------------------
  Aggregate
-   ->  Custom Scan (Citus Real-Time)
+   ->  Custom Scan (Citus Adaptive)
          Task Count: 1
          Tasks Shown: All
          ->  Task
@@ -179,7 +179,7 @@ SELECT count(*) FROM composite_partitioned_table
                    QUERY PLAN                    
 -------------------------------------------------
  Aggregate
-   ->  Custom Scan (Citus Real-Time)
+   ->  Custom Scan (Citus Adaptive)
          Task Count: 1
          Tasks Shown: All
          ->  Task

src/test/regress/expected/multi_partitioning.out

@@ -412,7 +412,7 @@ SELECT * FROM partitioning_test WHERE id = 9 OR id = 10 ORDER BY 1;
 -- create default partition
 CREATE TABLE partitioning_test_default PARTITION OF partitioning_test DEFAULT;
 \d+ partitioning_test
-                             Table "public.partitioning_test"
+                       Partitioned table "public.partitioning_test"
  Column |  Type   | Collation | Nullable | Default | Storage | Stats target | Description 
 --------+---------+-----------+----------+---------+---------+--------------+-------------
  id     | integer |           |          |         | plain   |              | 
@@ -1475,21 +1475,13 @@ WHERE
     pid = pg_backend_pid()
 ORDER BY
     1, 2, 3;
-      logicalrelid       | locktype |           mode           
+      logicalrelid       | locktype |   mode    
--------------------------+----------+--------------------------
+-------------------------+----------+-----------
- partitioning_locks      | advisory | ShareUpdateExclusiveLock
- partitioning_locks      | advisory | ShareUpdateExclusiveLock
- partitioning_locks      | advisory | ShareUpdateExclusiveLock
- partitioning_locks      | advisory | ShareUpdateExclusiveLock
  partitioning_locks_2009 | advisory | ShareLock
  partitioning_locks_2009 | advisory | ShareLock
  partitioning_locks_2009 | advisory | ShareLock
  partitioning_locks_2009 | advisory | ShareLock
- partitioning_locks_2009 | advisory | ShareUpdateExclusiveLock
+(4 rows)
- partitioning_locks_2009 | advisory | ShareUpdateExclusiveLock
- partitioning_locks_2009 | advisory | ShareUpdateExclusiveLock
- partitioning_locks_2009 | advisory | ShareUpdateExclusiveLock
-(12 rows)
 
 COMMIT;
 -- test shard resource locks with INSERT/SELECT
@@ -1583,7 +1575,7 @@ EXPLAIN (COSTS OFF)
 SELECT * FROM partitioning_hash_test JOIN partitioning_hash_join_test USING (id, subid);
                                                                                 QUERY PLAN                                                                                
 --------------------------------------------------------------------------------------------------------------------------------------------------------------------------
- Custom Scan (Citus Real-Time)
+ Custom Scan (Citus Adaptive)
    Task Count: 4
    Tasks Shown: One of 4
    ->  Task
@@ -1623,7 +1615,7 @@ EXPLAIN (COSTS OFF)
 SELECT * FROM partitioning_hash_test JOIN partitioning_hash_join_test USING (id, subid);
                                                                                   QUERY PLAN                                                                                  
 ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
- Custom Scan (Citus Real-Time)
+ Custom Scan (Citus Adaptive)
    Task Count: 4
    Tasks Shown: One of 4
    ->  Task
@@ -1653,7 +1645,7 @@ EXPLAIN (COSTS OFF)
 SELECT * FROM partitioning_hash_test JOIN partitioning_hash_join_test USING (id);
                                                QUERY PLAN                                                
 ---------------------------------------------------------------------------------------------------------
- Custom Scan (Citus Real-Time)
+ Custom Scan (Citus Adaptive)
    Task Count: 4
    Tasks Shown: One of 4
    ->  Task

src/test/regress/expected/multi_subquery_window_functions.out

@@ -716,7 +716,7 @@ EXPLAIN (COSTS FALSE, VERBOSE TRUE)
    ->  Sort
          Output: remote_scan.user_id, remote_scan.sum
          Sort Key: remote_scan.sum DESC, remote_scan.user_id DESC
-         ->  Custom Scan (Citus Real-Time)
+         ->  Custom Scan (Citus Adaptive)
                Output: remote_scan.user_id, remote_scan.sum
                Task Count: 4
                Tasks Shown: One of 4

src/test/regress/expected/with_transactions.out

@@ -1,6 +1,7 @@
 CREATE SCHEMA with_transactions;
 SET search_path TO 	with_transactions, public;
 SET citus.shard_count TO 4;
+SET citus.shard_replication_factor TO 1; -- https://github.com/citusdata/citus/issues/3061
 SET citus.next_placement_id TO 800000;
 CREATE TABLE with_transactions.raw_table (tenant_id int, income float, created_at timestamptz);
 SELECT create_distributed_table('raw_table', 'tenant_id');
@@ -82,18 +83,17 @@ WITH ids_inserted AS
 UPDATE raw_table SET created_at = '2001-02-10 20:00:00' WHERE tenant_id IN (SELECT tenant_id FROM ids_inserted);
 DEBUG:  generating subplan 12_1 for CTE ids_inserted: INSERT INTO with_transactions.raw_table (tenant_id, income, created_at) VALUES (11,1000,now()), (12,1000,now()), (13,1000,now()) RETURNING raw_table.tenant_id
 DEBUG:  Plan 12 query after replacing subqueries and CTEs: UPDATE with_transactions.raw_table SET created_at = 'Sat Feb 10 20:00:00 2001 PST'::timestamp with time zone WHERE (tenant_id OPERATOR(pg_catalog.=) ANY (SELECT ids_inserted.tenant_id FROM (SELECT intermediate_result.tenant_id FROM read_intermediate_result('12_1'::text, 'binary'::citus_copy_format) intermediate_result(tenant_id integer)) ids_inserted))
-ERROR:  cannot establish a new connection for placement 800007, since DML has been executed on a connection that is in use
 -- make sure that everything committed
 SELECT count(*) FROM raw_table;
  count 
 -------
-   102
+   105
 (1 row)
 
 SELECT count(*) FROM raw_table WHERE created_at = '2001-02-10 20:00:00';
  count 
 -------
-     1
+     4
 (1 row)
 
 SELECT count(*) FROM second_raw_table;

src/test/regress/expected/with_transactions_9.out

@@ -1,126 +0,0 @@
-CREATE SCHEMA with_transactions;
-SET search_path TO 	with_transactions, public;
-SET citus.shard_count TO 4;
-SET citus.shard_replication_factor TO 1; -- https://github.com/citusdata/citus/issues/3061
-SET citus.next_placement_id TO 800000;
-CREATE TABLE with_transactions.raw_table (tenant_id int, income float, created_at timestamptz);
-SELECT create_distributed_table('raw_table', 'tenant_id');
- create_distributed_table 
---------------------------
- 
-(1 row)
-
-CREATE TABLE with_transactions.second_raw_table (tenant_id int, income float, created_at timestamptz);
-SELECT create_distributed_table('second_raw_table', 'tenant_id');
- create_distributed_table 
---------------------------
- 
-(1 row)
-
-INSERT INTO 
-	raw_table (tenant_id, income, created_at) 
-SELECT 
-	i % 10, i * 10.0, timestamp '2014-01-10 20:00:00' + i * interval '1 day' 
-FROM 
-	generate_series (0, 100) i;
-INSERT INTO second_raw_table SELECT * FROM raw_table;
-SET client_min_messages TO DEBUG1;
--- run a transaction which DELETE 
-BEGIN;
-	WITH ids_to_delete AS
-		(
-			SELECT tenant_id FROM raw_table WHERE income < 250
-		),
-		deleted_ids AS
-		(
-			DELETE FROM raw_table WHERE created_at  < '2014-02-10 20:00:00' AND tenant_id IN (SELECT * from ids_to_delete) RETURNING tenant_id
-		)
-		UPDATE raw_table SET income = income * 2 WHERE tenant_id IN (SELECT tenant_id FROM deleted_ids);
-DEBUG:  generating subplan 3_1 for CTE ids_to_delete: SELECT tenant_id FROM with_transactions.raw_table WHERE (income OPERATOR(pg_catalog.<) (250)::double precision)
-DEBUG:  generating subplan 3_2 for CTE deleted_ids: DELETE FROM with_transactions.raw_table WHERE ((created_at OPERATOR(pg_catalog.<) 'Mon Feb 10 20:00:00 2014 PST'::timestamp with time zone) AND (tenant_id OPERATOR(pg_catalog.=) ANY (SELECT ids_to_delete.tenant_id FROM (SELECT intermediate_result.tenant_id FROM read_intermediate_result('3_1'::text, 'binary'::citus_copy_format) intermediate_result(tenant_id integer)) ids_to_delete))) RETURNING tenant_id
-DEBUG:  Plan 3 query after replacing subqueries and CTEs: UPDATE with_transactions.raw_table SET income = (income OPERATOR(pg_catalog.*) (2)::double precision) WHERE (tenant_id OPERATOR(pg_catalog.=) ANY (SELECT deleted_ids.tenant_id FROM (SELECT intermediate_result.tenant_id FROM read_intermediate_result('3_2'::text, 'binary'::citus_copy_format) intermediate_result(tenant_id integer)) deleted_ids))
-ROLLBACK;
--- see that both UPDATE and DELETE commands are rollbacked
-SELECT count(*) FROM raw_table;
- count 
--------
-   101
-(1 row)
-
-SELECT max(income) FROM raw_table;
- max  
-------
- 1000
-(1 row)
-
--- multi-statement multi shard modifying statements should work
-BEGIN;
-	SELECT count (*) FROM second_raw_table;
- count 
--------
-   101
-(1 row)
-
-	WITH distinct_count AS (
-		SELECT count(DISTINCT created_at) FROM raw_table
-	),
-	ids_inserted AS
-	(
-		INSERT INTO raw_table VALUES (11, 1000, now()) RETURNING tenant_id
-	)
-	UPDATE raw_table SET created_at = '2001-02-10 20:00:00'
-	WHERE tenant_id IN (SELECT tenant_id FROM ids_inserted) AND tenant_id < (SELECT count FROM distinct_count);
-DEBUG:  generating subplan 9_1 for CTE distinct_count: SELECT count(DISTINCT created_at) AS count FROM with_transactions.raw_table
-DEBUG:  generating subplan 9_2 for CTE ids_inserted: INSERT INTO with_transactions.raw_table (tenant_id, income, created_at) VALUES (11, 1000, now()) RETURNING tenant_id
-DEBUG:  Plan 9 query after replacing subqueries and CTEs: UPDATE with_transactions.raw_table SET created_at = 'Sat Feb 10 20:00:00 2001 PST'::timestamp with time zone WHERE ((tenant_id OPERATOR(pg_catalog.=) ANY (SELECT ids_inserted.tenant_id FROM (SELECT intermediate_result.tenant_id FROM read_intermediate_result('9_2'::text, 'binary'::citus_copy_format) intermediate_result(tenant_id integer)) ids_inserted)) AND (tenant_id OPERATOR(pg_catalog.<) (SELECT distinct_count.count FROM (SELECT intermediate_result.count FROM read_intermediate_result('9_1'::text, 'binary'::citus_copy_format) intermediate_result(count bigint)) distinct_count)))
-	TRUNCATE second_raw_table;
-COMMIT;
--- sequential insert followed by parallel update works just fine
-WITH ids_inserted AS
-(
-  INSERT INTO raw_table VALUES (11, 1000, now()), (12, 1000, now()), (13, 1000, now()) RETURNING tenant_id 
-)
-UPDATE raw_table SET created_at = '2001-02-10 20:00:00' WHERE tenant_id IN (SELECT tenant_id FROM ids_inserted);
-DEBUG:  generating subplan 12_1 for CTE ids_inserted: INSERT INTO with_transactions.raw_table (tenant_id, income, created_at) VALUES (11,1000,now()), (12,1000,now()), (13,1000,now()) RETURNING raw_table.tenant_id
-DEBUG:  Plan 12 query after replacing subqueries and CTEs: UPDATE with_transactions.raw_table SET created_at = 'Sat Feb 10 20:00:00 2001 PST'::timestamp with time zone WHERE (tenant_id OPERATOR(pg_catalog.=) ANY (SELECT ids_inserted.tenant_id FROM (SELECT intermediate_result.tenant_id FROM read_intermediate_result('12_1'::text, 'binary'::citus_copy_format) intermediate_result(tenant_id integer)) ids_inserted))
--- make sure that everything committed
-SELECT count(*) FROM raw_table;
- count 
--------
-   105
-(1 row)
-
-SELECT count(*) FROM raw_table WHERE created_at = '2001-02-10 20:00:00';
- count 
--------
-     4
-(1 row)
-
-SELECT count(*) FROM second_raw_table;
- count 
--------
-     0
-(1 row)
-
--- sequential insert followed by a sequential real-time query should be fine
-BEGIN;
-SET LOCAL citus.multi_shard_modify_mode TO 'sequential';
-WITH ids_inserted AS
-(
-  INSERT INTO raw_table (tenant_id) VALUES (11), (12), (13), (14) RETURNING tenant_id
-)
-SELECT income FROM second_raw_table WHERE tenant_id IN (SELECT * FROM ids_inserted) ORDER BY 1 DESC LIMIT 3;
-DEBUG:  generating subplan 17_1 for CTE ids_inserted: INSERT INTO with_transactions.raw_table (tenant_id) VALUES (11), (12), (13), (14) RETURNING raw_table.tenant_id
-DEBUG:  Plan 17 query after replacing subqueries and CTEs: SELECT income FROM with_transactions.second_raw_table WHERE (tenant_id OPERATOR(pg_catalog.=) ANY (SELECT ids_inserted.tenant_id FROM (SELECT intermediate_result.tenant_id FROM read_intermediate_result('17_1'::text, 'binary'::citus_copy_format) intermediate_result(tenant_id integer)) ids_inserted)) ORDER BY income DESC LIMIT 3
-DEBUG:  push down of limit count: 3
- income 
---------
-(0 rows)
-
-ROLLBACK;
-RESET client_min_messages;
-RESET citus.shard_count;
-DROP SCHEMA with_transactions CASCADE;
-NOTICE:  drop cascades to 2 other objects
-DETAIL:  drop cascades to table raw_table
-drop cascades to table second_raw_table

src/test/regress/sql/local_shard_execution.sql

@@ -541,66 +541,6 @@ BEGIN;
 ROLLBACK;
 
 
--- mix with other executors should fail
-
--- router modify execution should error
-BEGIN;
-	DELETE FROM distributed_table WHERE key = 500;
-
-	SET LOCAL citus.task_executor_type = 'real-time';
-
-	DELETE FROM distributed_table;
-ROLLBACK;
-
--- local execution should not be executed locally
--- becase a multi-shard router query has already been executed
-BEGIN;
-
-	SET LOCAL citus.task_executor_type = 'real-time';
-
-	DELETE FROM distributed_table;
-
-	DELETE FROM distributed_table WHERE key = 500;
-ROLLBACK;
-
--- router select execution
-BEGIN;
-	DELETE FROM distributed_table WHERE key = 500;
-
-	SET LOCAL citus.task_executor_type = 'real-time';
-
-	SELECT count(*) FROM distributed_table WHERE key = 500;
-ROLLBACK;
-
--- local execution should not be executed locally
--- becase a single-shard router query has already been executed
-BEGIN;
-	SET LOCAL citus.task_executor_type = 'real-time';
-
-	SELECT count(*) FROM distributed_table WHERE key = 500;
-
-	DELETE FROM distributed_table WHERE key = 500;
-ROLLBACK;
-
--- real-time select execution
-BEGIN;
-	DELETE FROM distributed_table WHERE key = 500;
-
-	SET LOCAL citus.task_executor_type = 'real-time';
-
-	SELECT count(*) FROM distributed_table;
-ROLLBACK;
-
--- local execution should not be executed locally
--- becase a real-time query has already been executed
-BEGIN;
-	SET LOCAL citus.task_executor_type = 'real-time';
-
-	SELECT count(*) FROM distributed_table;
-
-	DELETE FROM distributed_table WHERE key = 500;
-ROLLBACK;
-
 -- task-tracker select execution
 BEGIN;
 	DELETE FROM distributed_table WHERE key = 500;