Merge pull request #2281 from citusdata/add_create_reference_table

Add create_reference_table failure tests

src/test/regress/expected/failure_create_reference_table.out

@@ -0,0 +1,247 @@
+--
+-- Failure tests for creating reference table
+--
+CREATE SCHEMA failure_reference_table;
+SET search_path TO 'failure_reference_table';
+SELECT citus.mitmproxy('conn.allow()');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+CREATE TABLE ref_table(id int);
+INSERT INTO ref_table VALUES(1),(2),(3);
+-- Kill on sending first query to worker node, should error
+-- out and not create any placement
+SELECT citus.mitmproxy('conn.onQuery().kill()');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+SELECT create_reference_table('ref_table');
+ERROR:  server closed the connection unexpectedly
+	This probably means the server terminated abnormally
+	before or while processing the request.
+CONTEXT:  while executing command on localhost:9060
+SELECT count(*) FROM pg_dist_shard_placement;
+ count 
+-------
+     0
+(1 row)
+
+-- Kill after creating transaction on worker node
+SELECT citus.mitmproxy('conn.onCommandComplete(command="BEGIN").kill()');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+SELECT create_reference_table('ref_table');
+WARNING:  connection not open
+CONTEXT:  while executing command on localhost:9060
+ERROR:  connection error: localhost:9060
+DETAIL:  connection not open
+SELECT count(*) FROM pg_dist_shard_placement;
+ count 
+-------
+     0
+(1 row)
+
+-- Cancel after creating transaction on worker node
+SELECT citus.mitmproxy('conn.onCommandComplete(command="BEGIN").cancel(' || pg_backend_pid() || ')');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+SELECT create_reference_table('ref_table');
+ERROR:  canceling statement due to user request
+SELECT count(*) FROM pg_dist_shard_placement;
+ count 
+-------
+     0
+(1 row)
+
+-- Kill after copying data to worker node
+SELECT citus.mitmproxy('conn.onCommandComplete(command="SELECT 1").kill()');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+SELECT create_reference_table('ref_table');
+WARNING:  connection not open
+CONTEXT:  while executing command on localhost:9060
+ERROR:  connection error: localhost:9060
+DETAIL:  connection not open
+SELECT count(*) FROM pg_dist_shard_placement;
+ count 
+-------
+     0
+(1 row)
+
+-- Cancel after copying data to worker node
+SELECT citus.mitmproxy('conn.onCommandComplete(command="SELECT 1").cancel(' || pg_backend_pid() || ')');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+SELECT create_reference_table('ref_table');
+ERROR:  canceling statement due to user request
+SELECT count(*) FROM pg_dist_shard_placement;
+ count 
+-------
+     0
+(1 row)
+
+-- Kill after copying data to worker node
+SELECT citus.mitmproxy('conn.onCommandComplete(command="COPY 3").kill()');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+SELECT create_reference_table('ref_table');
+NOTICE:  Copying data from local table...
+ERROR:  failed to COPY to shard 102030 on localhost:9060
+SELECT count(*) FROM pg_dist_shard_placement;
+ count 
+-------
+     0
+(1 row)
+
+-- Cancel after copying data to worker node
+SELECT citus.mitmproxy('conn.onCommandComplete(command="COPY 3").cancel(' || pg_backend_pid() || ')');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+SELECT create_reference_table('ref_table');
+NOTICE:  Copying data from local table...
+ERROR:  canceling statement due to user request
+SELECT count(*) FROM pg_dist_shard_placement;
+ count 
+-------
+     0
+(1 row)
+
+-- we don't want to see the prepared transaction numbers in the warnings
+SET client_min_messages TO ERROR;
+-- Kill after preparing transaction. Since we don't commit after preparing, we recover
+-- prepared transaction afterwards.
+SELECT citus.mitmproxy('conn.onCommandComplete(command="PREPARE TRANSACTION").kill()');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+SELECT create_reference_table('ref_table');
+ERROR:  connection not open
+CONTEXT:  while executing command on localhost:9060
+SELECT count(*) FROM pg_dist_shard_placement;
+ count 
+-------
+     0
+(1 row)
+
+SELECT recover_prepared_transactions();
+ recover_prepared_transactions 
+-------------------------------
+                             1
+(1 row)
+
+-- Kill after commiting prepared, this should succeed
+SELECT citus.mitmproxy('conn.onCommandComplete(command="COMMIT PREPARED").kill()');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+SELECT create_reference_table('ref_table');
+ create_reference_table 
+------------------------
+ 
+(1 row)
+
+SELECT * FROM pg_dist_shard_placement ORDER BY shardid, nodeport;
+ shardid | shardstate | shardlength | nodename  | nodeport | placementid 
+---------+------------+-------------+-----------+----------+-------------
+  102033 |          1 |           0 | localhost |     9060 |         123
+  102033 |          1 |           0 | localhost |    57637 |         124
+(2 rows)
+
+SET client_min_messages TO NOTICE;
+SELECT citus.mitmproxy('conn.allow()');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+DROP TABLE ref_table;
+CREATE TABLE ref_table(id int);
+INSERT INTO ref_table VALUES(1),(2),(3);
+-- Test in transaction
+SELECT citus.mitmproxy('conn.onQuery().kill()');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+BEGIN;
+SELECT create_reference_table('ref_table');
+ERROR:  server closed the connection unexpectedly
+	This probably means the server terminated abnormally
+	before or while processing the request.
+CONTEXT:  while executing command on localhost:9060
+COMMIT;
+-- kill on ROLLBACK, should be rollbacked
+SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+BEGIN;
+SELECT create_reference_table('ref_table');
+NOTICE:  Copying data from local table...
+ create_reference_table 
+------------------------
+ 
+(1 row)
+
+ROLLBACK;
+WARNING:  connection not open
+CONTEXT:  while executing command on localhost:9060
+SELECT * FROM pg_dist_shard_placement ORDER BY shardid, nodeport;
+ shardid | shardstate | shardlength | nodename | nodeport | placementid 
+---------+------------+-------------+----------+----------+-------------
+(0 rows)
+
+-- cancel when the coordinator send ROLLBACK, should be rollbacked. We ignore cancellations
+-- during the ROLLBACK.
+SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").cancel(' || pg_backend_pid() || ')');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+BEGIN;
+SELECT create_reference_table('ref_table');
+NOTICE:  Copying data from local table...
+ create_reference_table 
+------------------------
+ 
+(1 row)
+
+ROLLBACK;
+SELECT * FROM pg_dist_shard_placement ORDER BY shardid, nodeport;
+ shardid | shardstate | shardlength | nodename | nodeport | placementid 
+---------+------------+-------------+----------+----------+-------------
+(0 rows)
+
+DROP SCHEMA failure_reference_table CASCADE;
+NOTICE:  drop cascades to table ref_table
+SET search_path TO default;

src/test/regress/failure_schedule

@@ -10,6 +10,7 @@ test: failure_create_index_concurrently
 test: failure_add_disable_node
 test: failure_copy_to_reference
 test: failure_copy_on_hash
+test: failure_create_reference_table
 
 test: failure_1pc_copy_hash
 test: failure_1pc_copy_append

src/test/regress/sql/failure_create_reference_table.sql

@@ -0,0 +1,103 @@
+--
+-- Failure tests for creating reference table
+--
+
+CREATE SCHEMA failure_reference_table;
+SET search_path TO 'failure_reference_table';
+
+SELECT citus.mitmproxy('conn.allow()');
+
+CREATE TABLE ref_table(id int);
+INSERT INTO ref_table VALUES(1),(2),(3);
+
+-- Kill on sending first query to worker node, should error
+-- out and not create any placement
+SELECT citus.mitmproxy('conn.onQuery().kill()');
+SELECT create_reference_table('ref_table');
+
+SELECT count(*) FROM pg_dist_shard_placement;
+
+-- Kill after creating transaction on worker node
+SELECT citus.mitmproxy('conn.onCommandComplete(command="BEGIN").kill()');
+SELECT create_reference_table('ref_table');
+
+SELECT count(*) FROM pg_dist_shard_placement;
+
+-- Cancel after creating transaction on worker node
+SELECT citus.mitmproxy('conn.onCommandComplete(command="BEGIN").cancel(' || pg_backend_pid() || ')');
+SELECT create_reference_table('ref_table');
+
+SELECT count(*) FROM pg_dist_shard_placement;
+
+-- Kill after copying data to worker node
+SELECT citus.mitmproxy('conn.onCommandComplete(command="SELECT 1").kill()');
+SELECT create_reference_table('ref_table');
+
+SELECT count(*) FROM pg_dist_shard_placement;
+
+-- Cancel after copying data to worker node
+SELECT citus.mitmproxy('conn.onCommandComplete(command="SELECT 1").cancel(' || pg_backend_pid() || ')');
+SELECT create_reference_table('ref_table');
+
+SELECT count(*) FROM pg_dist_shard_placement;
+
+-- Kill after copying data to worker node
+SELECT citus.mitmproxy('conn.onCommandComplete(command="COPY 3").kill()');
+SELECT create_reference_table('ref_table');
+
+SELECT count(*) FROM pg_dist_shard_placement;
+
+-- Cancel after copying data to worker node
+SELECT citus.mitmproxy('conn.onCommandComplete(command="COPY 3").cancel(' || pg_backend_pid() || ')');
+SELECT create_reference_table('ref_table');
+
+SELECT count(*) FROM pg_dist_shard_placement;
+
+-- we don't want to see the prepared transaction numbers in the warnings
+SET client_min_messages TO ERROR;
+
+-- Kill after preparing transaction. Since we don't commit after preparing, we recover
+-- prepared transaction afterwards.
+SELECT citus.mitmproxy('conn.onCommandComplete(command="PREPARE TRANSACTION").kill()');
+SELECT create_reference_table('ref_table');
+
+SELECT count(*) FROM pg_dist_shard_placement;
+SELECT recover_prepared_transactions();
+
+-- Kill after commiting prepared, this should succeed
+SELECT citus.mitmproxy('conn.onCommandComplete(command="COMMIT PREPARED").kill()');
+SELECT create_reference_table('ref_table');
+
+SELECT * FROM pg_dist_shard_placement ORDER BY shardid, nodeport;
+SET client_min_messages TO NOTICE;
+
+SELECT citus.mitmproxy('conn.allow()');
+DROP TABLE ref_table;
+CREATE TABLE ref_table(id int);
+INSERT INTO ref_table VALUES(1),(2),(3);
+
+-- Test in transaction
+SELECT citus.mitmproxy('conn.onQuery().kill()');
+BEGIN;
+SELECT create_reference_table('ref_table');
+COMMIT;
+
+-- kill on ROLLBACK, should be rollbacked
+SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()');
+BEGIN;
+SELECT create_reference_table('ref_table');
+ROLLBACK;
+
+SELECT * FROM pg_dist_shard_placement ORDER BY shardid, nodeport;
+
+-- cancel when the coordinator send ROLLBACK, should be rollbacked. We ignore cancellations
+-- during the ROLLBACK.
+SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").cancel(' || pg_backend_pid() || ')');
+BEGIN;
+SELECT create_reference_table('ref_table');
+ROLLBACK;
+
+SELECT * FROM pg_dist_shard_placement ORDER BY shardid, nodeport;
+
+DROP SCHEMA failure_reference_table CASCADE;
+SET search_path TO default;