Fix failure and isolation tests

On top of citus.max_cached_conns_per_worker GUC, with this commit
we're updating the regression tests to comply with the new behaviour.

src/test/regress/expected/failure_1pc_copy_append.out

@@ -50,11 +50,9 @@ SELECT citus.dump_network_traffic();
  (0,worker,"[""RowDescription(fieldcount=2,fields=['F(name=min,tableoid=0,colattrnum=0,typoid=23,typlen=4,typmod=-1,format_code=0)', 'F(name=max,tableoid=0,colattrnum=0,typoid=23,typlen=4,typmod=-1,format_code=0)'])"", 'DataRow(columncount=2,columns=[""C(length=0,value=b\\'\\')"", ""C(length=1,value=b\\'0\\')""])', 'CommandComplete(command=SELECT 1)', 'ReadyForQuery(state=in_transaction_block)']")
  (0,coordinator,"['Query(query=COMMIT)']")
  (0,worker,"['CommandComplete(command=COMMIT)', 'ReadyForQuery(state=idle)']")
- (1,coordinator,"[initial message]")
+ (0,coordinator,"['Query(query=COPY (SELECT count(1) AS count FROM copy_test_100400 copy_test WHERE true) TO STDOUT)']")
- (1,worker,"['AuthenticationOk()', 'ParameterStatus(application_name=citus)', 'ParameterStatus(client_encoding=UTF8)', 'ParameterStatus(DateStyle=ISO, MDY)', 'ParameterStatus(integer_datetimes=on)', 'ParameterStatus(IntervalStyle=postgres)', 'ParameterStatus(is_superuser=on)', 'ParameterStatus(server_encoding=UTF8)', 'ParameterStatus(server_version=XXX)', 'ParameterStatus(session_authorization=postgres)', 'ParameterStatus(standard_conforming_strings=on)', 'ParameterStatus(TimeZone=XXX)', 'BackendKeyData(XXX)', 'ReadyForQuery(state=idle)']")
+ (0,worker,"[""CopyOutResponse(format=0,columncount=1,columns=['Anonymous(format=0)'])"", ""CopyData(data=b'4\\\\n')"", 'CopyDone()', 'CommandComplete(command=COPY 1)', 'ReadyForQuery(state=idle)']")
- (1,coordinator,"['Query(query=COPY (SELECT count(1) AS count FROM copy_test_100400 copy_test WHERE true) TO STDOUT)']")
+(20 rows)
- (1,worker,"[""CopyOutResponse(format=0,columncount=1,columns=['Anonymous(format=0)'])"", ""CopyData(data=b'4\\\\n')"", 'CopyDone()', 'CommandComplete(command=COPY 1)', 'ReadyForQuery(state=idle)']")
-(22 rows)
 
 ---- all of the following tests test behavior with 2 shard placements ----
 SHOW citus.shard_replication_factor;

src/test/regress/expected/failure_1pc_copy_hash.out

@@ -8,6 +8,7 @@ SET citus.shard_count = 1;
 SET citus.shard_replication_factor = 2; -- one shard per worker
 SET citus.multi_shard_commit_protocol TO '1pc';
 SET citus.next_shard_id TO 100400;
+SET citus.max_cached_conns_per_worker TO 0;
 ALTER SEQUENCE pg_catalog.pg_dist_placement_placementid_seq RESTART 100;
 CREATE TABLE copy_test (key int, value int);
 SELECT create_distributed_table('copy_test', 'key');
@@ -212,10 +213,12 @@ SELECT citus.mitmproxy('conn.killall()');
 (1 row)
 
 COPY copy_test FROM PROGRAM 'echo 0, 0 && echo 1, 1 && echo 2, 4 && echo 3, 9' WITH CSV;
-WARNING:  connection not open
+WARNING:  connection error: localhost:9060
-CONTEXT:  while executing command on localhost:9060
+DETAIL:  server closed the connection unexpectedly
-COPY copy_test, line 1: "0, 0"
+	This probably means the server terminated abnormally
-ERROR:  failure on connection marked as essential: localhost:9060
+	before or while processing the request.
+CONTEXT:  COPY copy_test, line 1: "0, 0"
+ERROR:  could not connect to any active placements
 CONTEXT:  COPY copy_test, line 1: "0, 0"
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy 

src/test/regress/expected/failure_connection_establishment.out

@@ -14,6 +14,7 @@ SELECT citus.mitmproxy('conn.allow()');
 CREATE SCHEMA fail_connect;
 SET search_path TO 'fail_connect';
 SET citus.shard_count TO 4;
+SET citus.max_cached_conns_per_worker TO 0;
 ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 1450000;
 ALTER SEQUENCE pg_catalog.pg_dist_placement_placementid_seq RESTART 1450000;
 CREATE TABLE products (

src/test/regress/expected/failure_copy_on_hash.out

@@ -13,6 +13,7 @@ SELECT citus.mitmproxy('conn.allow()');
 -- With one placement COPY should error out and placement should stay healthy.
 SET citus.shard_replication_factor TO 1;
 SET citus.shard_count to 4;
+SET citus.max_cached_conns_per_worker to 0;
 CREATE TABLE test_table(id int, value_1 int);
 SELECT create_distributed_table('test_table','id');
  create_distributed_table 

src/test/regress/expected/failure_copy_to_reference.out

@@ -35,10 +35,7 @@ SELECT citus.mitmproxy('conn.kill()');
 (1 row)
 
 \copy test_table FROM STDIN DELIMITER ','
-ERROR:  connection error: localhost:9060
+ERROR:  failure on connection marked as essential: localhost:9060
-DETAIL:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
 CONTEXT:  COPY test_table, line 1: "1,2"
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy 

src/test/regress/expected/failure_create_table.out

@@ -595,14 +595,10 @@ SELECT citus.mitmproxy('conn.kill()');
 (1 row)
 
 SELECT master_create_worker_shards('test_table_2', 4, 2);
-WARNING:  connection error: localhost:9060
+WARNING:  connection not open
-DETAIL:  server closed the connection unexpectedly
+CONTEXT:  while executing command on localhost:9060
-	This probably means the server terminated abnormally
-	before or while processing the request.
 ERROR:  connection error: localhost:9060
-DETAIL:  server closed the connection unexpectedly
+DETAIL:  connection not open
-	This probably means the server terminated abnormally
-	before or while processing the request.
 SELECT count(*) FROM pg_dist_shard;
  count 
 -------

src/test/regress/expected/failure_single_select.out

@@ -197,6 +197,7 @@ ERROR:  server closed the connection unexpectedly
 	before or while processing the request.
 CONTEXT:  while executing command on localhost:9060
 -- bug from https://github.com/citusdata/citus/issues/1926
+SET citus.max_cached_conns_per_worker TO 0; -- purge cache
 DROP TABLE select_test;
 SET citus.shard_count = 2;
 SET citus.shard_replication_factor = 1;
@@ -207,6 +208,7 @@ SELECT create_distributed_table('select_test', 'key');
  
 (1 row)
 
+SET citus.max_cached_conns_per_worker TO 1; -- allow connection to be cached
 INSERT INTO select_test VALUES (1, 'test data');
 SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").after(1).kill()');
  mitmproxy 

src/test/regress/expected/isolation_citus_dist_activity.out

@@ -7,12 +7,21 @@ create_distributed_table
 step s1-begin: 
     BEGIN;
 
+    -- we don't want to see any entries related to 2PC recovery
+    SET citus.max_cached_conns_per_worker TO 0;
+
 step s2-begin: 
 	BEGIN;
         
+        -- we don't want to see any entries related to 2PC recovery
+        SET citus.max_cached_conns_per_worker TO 0;
+
 step s3-begin: 
 	BEGIN;
 
+        -- we don't want to see any entries related to 2PC recovery
+        SET citus.max_cached_conns_per_worker TO 0;
+
 step s1-alter-table: 
     ALTER TABLE test_table ADD COLUMN x INT;
 
@@ -67,12 +76,21 @@ create_distributed_table
 step s1-begin: 
     BEGIN;
 
+    -- we don't want to see any entries related to 2PC recovery
+    SET citus.max_cached_conns_per_worker TO 0;
+
 step s2-begin: 
 	BEGIN;
         
+        -- we don't want to see any entries related to 2PC recovery
+        SET citus.max_cached_conns_per_worker TO 0;
+
 step s3-begin: 
 	BEGIN;
 
+        -- we don't want to see any entries related to 2PC recovery
+        SET citus.max_cached_conns_per_worker TO 0;
+
 step s1-insert: 
  	INSERT INTO test_table VALUES (100, 100);
 
@@ -116,12 +134,21 @@ create_distributed_table
 step s1-begin: 
     BEGIN;
 
+    -- we don't want to see any entries related to 2PC recovery
+    SET citus.max_cached_conns_per_worker TO 0;
+
 step s2-begin: 
 	BEGIN;
         
+        -- we don't want to see any entries related to 2PC recovery
+        SET citus.max_cached_conns_per_worker TO 0;
+
 step s3-begin: 
 	BEGIN;
 
+        -- we don't want to see any entries related to 2PC recovery
+        SET citus.max_cached_conns_per_worker TO 0;
+
 step s1-select: 
    SELECT count(*) FROM test_table;
 
@@ -171,12 +198,21 @@ create_distributed_table
 step s1-begin: 
     BEGIN;
 
+    -- we don't want to see any entries related to 2PC recovery
+    SET citus.max_cached_conns_per_worker TO 0;
+
 step s2-begin: 
 	BEGIN;
         
+        -- we don't want to see any entries related to 2PC recovery
+        SET citus.max_cached_conns_per_worker TO 0;
+
 step s3-begin: 
 	BEGIN;
 
+        -- we don't want to see any entries related to 2PC recovery
+        SET citus.max_cached_conns_per_worker TO 0;
+
 step s1-select-router: 
    SELECT count(*) FROM test_table WHERE column1 = 55;
 
@@ -206,7 +242,6 @@ query          query_hostname query_hostport master_query_host_namemaster_query_
 SELECT gid FROM pg_prepared_xacts WHERE gid LIKE 'citus\_0\_%'localhost      57638                         0              idle           Client         ClientRead     postgres       regression     
 SELECT gid FROM pg_prepared_xacts WHERE gid LIKE 'citus\_0\_%'localhost      57637                         0              idle           Client         ClientRead     postgres       regression     
 SELECT count(*) AS count FROM public.test_table_105951 test_table WHERE (column1 OPERATOR(pg_catalog.=) 55)localhost      57638                         0              idle           Client         ClientRead     postgres       regression     
-COMMIT         localhost      57637                         0              idle           Client         ClientRead     postgres       regression     
 step s2-rollback: 
 	ROLLBACK;
 

src/test/regress/specs/isolation_citus_dist_activity.spec

@@ -1,8 +1,9 @@
 setup
 {
-	SET citus.shard_replication_factor TO 1;
+    SET citus.shard_replication_factor TO 1;
-	SET citus.shard_count TO 4;
+    SET citus.shard_count TO 4;
-
+    -- we don't want to see any entries related to 2PC recovery
+    SET citus.max_cached_conns_per_worker TO 0;
     CREATE TABLE test_table(column1 int, column2 int);
     SELECT create_distributed_table('test_table', 'column1');
 }
@@ -17,6 +18,9 @@ session "s1"
 step "s1-begin"
 {
     BEGIN;
+
+    -- we don't want to see any entries related to 2PC recovery
+    SET citus.max_cached_conns_per_worker TO 0;
 }
 
 step "s1-alter-table"
@@ -49,6 +53,9 @@ session "s2"
 step "s2-begin"
 {
 	BEGIN;
+        
+        -- we don't want to see any entries related to 2PC recovery
+        SET citus.max_cached_conns_per_worker TO 0;
 }
 
 step "s2-rollback"
@@ -72,6 +79,9 @@ session "s3"
 step "s3-begin"
 {
 	BEGIN;
+
+        -- we don't want to see any entries related to 2PC recovery
+        SET citus.max_cached_conns_per_worker TO 0;
 }
 
 step "s3-rollback"

src/test/regress/sql/failure_1pc_copy_hash.sql

@@ -4,6 +4,7 @@ SET citus.shard_count = 1;
 SET citus.shard_replication_factor = 2; -- one shard per worker
 SET citus.multi_shard_commit_protocol TO '1pc';
 SET citus.next_shard_id TO 100400;
+SET citus.max_cached_conns_per_worker TO 0;
 ALTER SEQUENCE pg_catalog.pg_dist_placement_placementid_seq RESTART 100;
 
 CREATE TABLE copy_test (key int, value int);

src/test/regress/sql/failure_connection_establishment.sql

@@ -12,6 +12,7 @@ CREATE SCHEMA fail_connect;
 SET search_path TO 'fail_connect';
 
 SET citus.shard_count TO 4;
+SET citus.max_cached_conns_per_worker TO 0;
 ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 1450000;
 ALTER SEQUENCE pg_catalog.pg_dist_placement_placementid_seq RESTART 1450000;
 

src/test/regress/sql/failure_copy_on_hash.sql

@@ -11,6 +11,7 @@ SELECT citus.mitmproxy('conn.allow()');
 -- With one placement COPY should error out and placement should stay healthy.
 SET citus.shard_replication_factor TO 1;
 SET citus.shard_count to 4;
+SET citus.max_cached_conns_per_worker to 0;
 
 CREATE TABLE test_table(id int, value_1 int);
 SELECT create_distributed_table('test_table','id');

src/test/regress/sql/failure_single_select.sql

@@ -81,12 +81,15 @@ SELECT recover_prepared_transactions();
 SELECT recover_prepared_transactions();
 
 -- bug from https://github.com/citusdata/citus/issues/1926
+SET citus.max_cached_conns_per_worker TO 0; -- purge cache
 DROP TABLE select_test;
 SET citus.shard_count = 2;
 SET citus.shard_replication_factor = 1;
 
 CREATE TABLE select_test (key int, value text);
 SELECT create_distributed_table('select_test', 'key');
+
+SET citus.max_cached_conns_per_worker TO 1; -- allow connection to be cached
 INSERT INTO select_test VALUES (1, 'test data');
 
 SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").after(1).kill()');