Merge pull request #2210 from citusdata/mt_failure_test

Added failure test for create index concurrently

src/test/regress/expected/failure_create_index_concurrently.out

@@ -0,0 +1,200 @@
+--
+-- failure_create_index_concurrently
+-- test create index concurrently command
+-- failure.
+SELECT citus.mitmproxy('conn.allow()');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+SET citus.shard_count = 4; -- two per worker
+CREATE SCHEMA index_schema;
+SET SEARCH_PATH=index_schema;
+CREATE TABLE index_test(id int, value_1 int, value_2 int);
+SELECT create_distributed_table('index_test', 'id');
+ create_distributed_table 
+--------------------------
+ 
+(1 row)
+
+-- kill the connection when create command is issued
+SELECT citus.mitmproxy('conn.onQuery(query="CREATE").kill()');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1);
+ERROR:  CONCURRENTLY-enabled index command failed
+DETAIL:  CONCURRENTLY-enabled index commands can fail partially, leaving behind an INVALID index.
+HINT:  Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index, then retry the original command.
+SELECT citus.mitmproxy('conn.allow()');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+-- verify index is not created
+SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test%' $$)
+WHERE nodeport = :worker_2_proxy_port;
+ nodename  | nodeport | success | result 
+-----------+----------+---------+--------
+ localhost |    57640 | t       | 0
+(1 row)
+
+DROP TABLE index_test;
+CREATE TABLE index_test(id int, value_1 int, value_2 int);
+SELECT create_distributed_table('index_test', 'id');
+ create_distributed_table 
+--------------------------
+ 
+(1 row)
+
+-- kill the connection at the second create command is issued
+SELECT citus.mitmproxy('conn.onQuery(query="CREATE").after(1).kill()');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1);
+ERROR:  CONCURRENTLY-enabled index command failed
+DETAIL:  CONCURRENTLY-enabled index commands can fail partially, leaving behind an INVALID index.
+HINT:  Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index, then retry the original command.
+SELECT citus.mitmproxy('conn.allow()');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+-- verify only one index is created
+SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test%' $$)
+WHERE nodeport = :worker_2_proxy_port;
+ nodename  | nodeport | success | result 
+-----------+----------+---------+--------
+ localhost |    57640 | t       | 1
+(1 row)
+
+DROP TABLE index_test;
+CREATE TABLE index_test(id int, value_1 int, value_2 int);
+SELECT create_reference_table('index_test');
+ create_reference_table 
+------------------------
+ 
+(1 row)
+
+-- kill the connection when create command is issued
+SELECT citus.mitmproxy('conn.onQuery(query="CREATE").kill()');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1);
+ERROR:  CONCURRENTLY-enabled index command failed
+DETAIL:  CONCURRENTLY-enabled index commands can fail partially, leaving behind an INVALID index.
+HINT:  Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index, then retry the original command.
+SELECT citus.mitmproxy('conn.allow()');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+DROP TABLE index_test;
+CREATE TABLE index_test(id int, value_1 int, value_2 int);
+SELECT create_distributed_table('index_test', 'id');
+ create_distributed_table 
+--------------------------
+ 
+(1 row)
+
+-- cancel the connection when create command is issued
+-- network traffic may differ between execution during cancellation
+-- therefore dump_network_traffic() calls are not made
+SELECT citus.mitmproxy('conn.onQuery(query="CREATE").cancel(' || pg_backend_pid() || ')');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1);
+ERROR:  CONCURRENTLY-enabled index command failed
+DETAIL:  CONCURRENTLY-enabled index commands can fail partially, leaving behind an INVALID index.
+HINT:  Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index, then retry the original command.
+SELECT citus.mitmproxy('conn.allow()');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+DROP TABLE index_test;
+CREATE TABLE index_test(id int, value_1 int, value_2 int);
+SELECT create_reference_table('index_test');
+ create_reference_table 
+------------------------
+ 
+(1 row)
+
+-- cancel the connection when create command is issued
+SELECT citus.mitmproxy('conn.onQuery(query="CREATE").cancel(' || pg_backend_pid() || ')');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1);
+ERROR:  CONCURRENTLY-enabled index command failed
+DETAIL:  CONCURRENTLY-enabled index commands can fail partially, leaving behind an INVALID index.
+HINT:  Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index, then retry the original command.
+SELECT citus.mitmproxy('conn.allow()');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+DROP TABLE index_test;
+CREATE TABLE index_test(id int, value_1 int, value_2 int);
+SELECT create_distributed_table('index_test', 'id');
+ create_distributed_table 
+--------------------------
+ 
+(1 row)
+
+CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1);
+-- kill the connection when create command is issued
+SELECT citus.mitmproxy('conn.onQuery(query="DROP INDEX CONCURRENTLY").kill()');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+DROP INDEX CONCURRENTLY IF EXISTS idx_index_test;
+ERROR:  CONCURRENTLY-enabled index command failed
+DETAIL:  CONCURRENTLY-enabled index commands can fail partially, leaving behind an INVALID index.
+HINT:  Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index, then retry the original command.
+SELECT citus.mitmproxy('conn.allow()');
+ mitmproxy 
+-----------
+ 
+(1 row)
+
+-- verify index is not dropped at worker 2
+SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test%' $$)
+WHERE nodeport = :worker_2_proxy_port;
+ nodename  | nodeport | success | result 
+-----------+----------+---------+--------
+ localhost |    57640 | t       | 4
+(1 row)
+
+RESET SEARCH_PATH;
+DROP SCHEMA index_schema CASCADE;
+NOTICE:  drop cascades to table index_schema.index_test
+-- verify index is not at worker 2 upon cleanup
+SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test%' $$)
+WHERE nodeport = :worker_2_proxy_port;
+ nodename  | nodeport | success | result 
+-----------+----------+---------+--------
+ localhost |    57640 | t       | 0
+(1 row)
+

src/test/regress/failure_schedule

@@ -6,3 +6,4 @@ test: failure_setup
 test: multi_test_helpers
 
 test: failure_ddl
+test: failure_create_index_concurrently

src/test/regress/pg_regress_multi.pl

@@ -269,6 +269,8 @@ for (my $workerIndex = 1; $workerIndex <= $workerCount; $workerIndex++) {
     push(@followerWorkerPorts, $workerPort);
 }
 
+my $workerBehindProxyPort = $workerPorts[1] + 2;
+
 my $host = "localhost";
 my $user = "postgres";
 my @pgOptions = ();
@@ -417,6 +419,7 @@ if ($usingWindows)
 }
 print $fh catfile($bindir, "psql")." ";
 print $fh "--variable=master_port=$masterPort ";
+print $fh "--variable=worker_2_proxy_port=$workerBehindProxyPort ";
 print $fh "--variable=follower_master_port=$followerCoordPort ";
 print $fh "--variable=default_user=$user ";
 print $fh "--variable=SHOW_CONTEXT=always ";

src/test/regress/sql/failure_create_index_concurrently.sql

@@ -0,0 +1,106 @@
+--
+-- failure_create_index_concurrently
+-- test create index concurrently command
+-- failure.
+
+SELECT citus.mitmproxy('conn.allow()');
+
+SET citus.shard_count = 4; -- two per worker
+
+CREATE SCHEMA index_schema;
+SET SEARCH_PATH=index_schema;
+
+CREATE TABLE index_test(id int, value_1 int, value_2 int);
+SELECT create_distributed_table('index_test', 'id');
+
+-- kill the connection when create command is issued
+SELECT citus.mitmproxy('conn.onQuery(query="CREATE").kill()');
+
+CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1);
+
+SELECT citus.mitmproxy('conn.allow()');
+-- verify index is not created
+SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test%' $$)
+WHERE nodeport = :worker_2_proxy_port;
+
+
+DROP TABLE index_test;
+
+
+CREATE TABLE index_test(id int, value_1 int, value_2 int);
+SELECT create_distributed_table('index_test', 'id');
+
+-- kill the connection at the second create command is issued
+SELECT citus.mitmproxy('conn.onQuery(query="CREATE").after(1).kill()');
+
+CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1);
+
+SELECT citus.mitmproxy('conn.allow()');
+
+-- verify only one index is created
+SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test%' $$)
+WHERE nodeport = :worker_2_proxy_port;
+
+DROP TABLE index_test;
+
+
+CREATE TABLE index_test(id int, value_1 int, value_2 int);
+SELECT create_reference_table('index_test');
+
+-- kill the connection when create command is issued
+SELECT citus.mitmproxy('conn.onQuery(query="CREATE").kill()');
+
+CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1);
+
+SELECT citus.mitmproxy('conn.allow()');
+
+DROP TABLE index_test;
+
+
+CREATE TABLE index_test(id int, value_1 int, value_2 int);
+SELECT create_distributed_table('index_test', 'id');
+
+-- cancel the connection when create command is issued
+-- network traffic may differ between execution during cancellation
+-- therefore dump_network_traffic() calls are not made
+SELECT citus.mitmproxy('conn.onQuery(query="CREATE").cancel(' || pg_backend_pid() || ')');
+
+CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1);
+
+SELECT citus.mitmproxy('conn.allow()');
+
+DROP TABLE index_test;
+
+CREATE TABLE index_test(id int, value_1 int, value_2 int);
+SELECT create_reference_table('index_test');
+
+-- cancel the connection when create command is issued
+SELECT citus.mitmproxy('conn.onQuery(query="CREATE").cancel(' || pg_backend_pid() || ')');
+
+CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1);
+
+SELECT citus.mitmproxy('conn.allow()');
+
+DROP TABLE index_test;
+
+
+CREATE TABLE index_test(id int, value_1 int, value_2 int);
+SELECT create_distributed_table('index_test', 'id');
+
+CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1);
+
+-- kill the connection when create command is issued
+SELECT citus.mitmproxy('conn.onQuery(query="DROP INDEX CONCURRENTLY").kill()');
+DROP INDEX CONCURRENTLY IF EXISTS idx_index_test;
+SELECT citus.mitmproxy('conn.allow()');
+
+-- verify index is not dropped at worker 2
+SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test%' $$)
+WHERE nodeport = :worker_2_proxy_port;
+
+RESET SEARCH_PATH;
+DROP SCHEMA index_schema CASCADE;
+
+-- verify index is not at worker 2 upon cleanup
+SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test%' $$)
+WHERE nodeport = :worker_2_proxy_port;