Add failure tests for VACUUM/ANALYZE

VACUUM runs outside of a transaction, so the failure modes for it are
somewhat straightforward, though ANALYZE runs in a 1pc transaction and
multi-table VACUUM can fail between statements (PG 11 and higher).

src/test/regress/expected/failure_vacuum.out

@@ -0,0 +1,101 @@
+-- print whether we're using version > 10 to make version-specific tests clear
+SHOW server_version \gset
+SELECT substring(:'server_version', '\d+')::int > 10 AS version_above_ten;
+ version_above_ten 
+-------------------
+ t
+(1 row)
+
+SELECT citus.mitmproxy('conn.allow()');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+SET citus.shard_count = 1;
+SET citus.shard_replication_factor = 2; -- one shard per worker
+SET citus.multi_shard_commit_protocol TO '1pc';
+CREATE TABLE vacuum_test (key int, value int);
+SELECT create_distributed_table('vacuum_test', 'key');
+ create_distributed_table 
+--------------------------
+ 
+(1 row)
+
+SELECT citus.clear_network_traffic();
+ clear_network_traffic 
+-----------------------
+ 
+(1 row)
+
+SELECT citus.mitmproxy('conn.onQuery(query="^VACUUM").kill()');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+VACUUM vacuum_test;
+ERROR:  server closed the connection unexpectedly
+	This probably means the server terminated abnormally
+	before or while processing the request.
+CONTEXT:  while executing command on localhost:9060
+SELECT citus.mitmproxy('conn.onQuery(query="^ANALYZE").kill()');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+ANALYZE vacuum_test;
+ERROR:  server closed the connection unexpectedly
+	This probably means the server terminated abnormally
+	before or while processing the request.
+CONTEXT:  while executing command on localhost:9060
+SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT").kill()');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+ANALYZE vacuum_test;
+WARNING:  connection not open
+CONTEXT:  while executing command on localhost:9060
+WARNING:  failed to commit critical transaction on localhost:9060, metadata is likely out of sync
+WARNING:  connection not open
+CONTEXT:  while executing command on localhost:9060
+-- ANALYZE transactions being critical is an open question, see #2430
+UPDATE pg_dist_shard_placement SET shardstate = 1
+WHERE shardid IN (
+  SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'vacuum_test'::regclass
+);
+SELECT citus.mitmproxy('conn.allow()');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+CREATE TABLE other_vacuum_test (key int, value int);
+SELECT create_distributed_table('other_vacuum_test', 'key');
+ create_distributed_table 
+--------------------------
+ 
+(1 row)
+
+SELECT citus.mitmproxy('conn.onQuery(query="^VACUUM.*other").kill()');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+VACUUM vacuum_test, other_vacuum_test;
+ERROR:  server closed the connection unexpectedly
+	This probably means the server terminated abnormally
+	before or while processing the request.
+CONTEXT:  while executing command on localhost:9060
+-- ==== Clean up, we're done here ====
+SELECT citus.mitmproxy('conn.allow()');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+DROP TABLE vacuum_test, other_vacuum_test;

src/test/regress/expected/failure_vacuum_0.out

@@ -0,0 +1,100 @@
+-- print whether we're using version > 10 to make version-specific tests clear
+SHOW server_version \gset
+SELECT substring(:'server_version', '\d+')::int > 10 AS version_above_ten;
+ version_above_ten 
+-------------------
+ f
+(1 row)
+
+SELECT citus.mitmproxy('conn.allow()');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+SET citus.shard_count = 1;
+SET citus.shard_replication_factor = 2; -- one shard per worker
+SET citus.multi_shard_commit_protocol TO '1pc';
+CREATE TABLE vacuum_test (key int, value int);
+SELECT create_distributed_table('vacuum_test', 'key');
+ create_distributed_table 
+--------------------------
+ 
+(1 row)
+
+SELECT citus.clear_network_traffic();
+ clear_network_traffic 
+-----------------------
+ 
+(1 row)
+
+SELECT citus.mitmproxy('conn.onQuery(query="^VACUUM").kill()');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+VACUUM vacuum_test;
+ERROR:  server closed the connection unexpectedly
+	This probably means the server terminated abnormally
+	before or while processing the request.
+CONTEXT:  while executing command on localhost:9060
+SELECT citus.mitmproxy('conn.onQuery(query="^ANALYZE").kill()');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+ANALYZE vacuum_test;
+ERROR:  server closed the connection unexpectedly
+	This probably means the server terminated abnormally
+	before or while processing the request.
+CONTEXT:  while executing command on localhost:9060
+SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT").kill()');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+ANALYZE vacuum_test;
+WARNING:  connection not open
+CONTEXT:  while executing command on localhost:9060
+WARNING:  failed to commit critical transaction on localhost:9060, metadata is likely out of sync
+WARNING:  connection not open
+CONTEXT:  while executing command on localhost:9060
+-- ANALYZE transactions being critical is an open question, see #2430
+UPDATE pg_dist_shard_placement SET shardstate = 1
+WHERE shardid IN (
+  SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'vacuum_test'::regclass
+);
+SELECT citus.mitmproxy('conn.allow()');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+CREATE TABLE other_vacuum_test (key int, value int);
+SELECT create_distributed_table('other_vacuum_test', 'key');
+ create_distributed_table 
+--------------------------
+ 
+(1 row)
+
+SELECT citus.mitmproxy('conn.onQuery(query="^VACUUM.*other").kill()');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+VACUUM vacuum_test, other_vacuum_test;
+ERROR:  syntax error at or near ","
+LINE 1: VACUUM vacuum_test, other_vacuum_test;
+                          ^
+-- ==== Clean up, we're done here ====
+SELECT citus.mitmproxy('conn.allow()');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+DROP TABLE vacuum_test, other_vacuum_test;

src/test/regress/failure_schedule

@@ -21,3 +21,4 @@ test: failure_multi_shard_update_delete
 test: failure_cte_subquery
 test: failure_insert_select_via_coordinator
 test: failure_multi_dml
+test: failure_vacuum

src/test/regress/sql/failure_vacuum.sql

@@ -0,0 +1,43 @@
+-- print whether we're using version > 10 to make version-specific tests clear
+SHOW server_version \gset
+SELECT substring(:'server_version', '\d+')::int > 10 AS version_above_ten;
+
+SELECT citus.mitmproxy('conn.allow()');
+
+SET citus.shard_count = 1;
+SET citus.shard_replication_factor = 2; -- one shard per worker
+SET citus.multi_shard_commit_protocol TO '1pc';
+
+CREATE TABLE vacuum_test (key int, value int);
+SELECT create_distributed_table('vacuum_test', 'key');
+
+SELECT citus.clear_network_traffic();
+
+SELECT citus.mitmproxy('conn.onQuery(query="^VACUUM").kill()');
+VACUUM vacuum_test;
+
+SELECT citus.mitmproxy('conn.onQuery(query="^ANALYZE").kill()');
+ANALYZE vacuum_test;
+
+SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT").kill()');
+ANALYZE vacuum_test;
+
+-- ANALYZE transactions being critical is an open question, see #2430
+UPDATE pg_dist_shard_placement SET shardstate = 1
+WHERE shardid IN (
+  SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'vacuum_test'::regclass
+);
+
+SELECT citus.mitmproxy('conn.allow()');
+
+CREATE TABLE other_vacuum_test (key int, value int);
+SELECT create_distributed_table('other_vacuum_test', 'key');
+
+SELECT citus.mitmproxy('conn.onQuery(query="^VACUUM.*other").kill()');
+
+VACUUM vacuum_test, other_vacuum_test;
+
+-- ==== Clean up, we're done here ====
+
+SELECT citus.mitmproxy('conn.allow()');
+DROP TABLE vacuum_test, other_vacuum_test;