From 2795494758ff68afdccbe4ccb84ed11cb5d8f253 Mon Sep 17 00:00:00 2001 From: Murat Tuncer Date: Tue, 12 Jun 2018 09:17:58 +0300 Subject: [PATCH] Added failure test for create index concurrently --- .../failure_create_index_concurrently.out | 200 ++++++++++++++++++ src/test/regress/failure_schedule | 1 + src/test/regress/pg_regress_multi.pl | 3 + .../sql/failure_create_index_concurrently.sql | 106 ++++++++++ 4 files changed, 310 insertions(+) create mode 100644 src/test/regress/expected/failure_create_index_concurrently.out create mode 100644 src/test/regress/sql/failure_create_index_concurrently.sql diff --git a/src/test/regress/expected/failure_create_index_concurrently.out b/src/test/regress/expected/failure_create_index_concurrently.out new file mode 100644 index 000000000..e243fc0a2 --- /dev/null +++ b/src/test/regress/expected/failure_create_index_concurrently.out @@ -0,0 +1,200 @@ +-- +-- failure_create_index_concurrently +-- test create index concurrently command +-- failure. +SELECT citus.mitmproxy('conn.allow()'); + mitmproxy +----------- + +(1 row) + +SET citus.shard_count = 4; -- two per worker +CREATE SCHEMA index_schema; +SET SEARCH_PATH=index_schema; +CREATE TABLE index_test(id int, value_1 int, value_2 int); +SELECT create_distributed_table('index_test', 'id'); + create_distributed_table +-------------------------- + +(1 row) + +-- kill the connection when create command is issued +SELECT citus.mitmproxy('conn.onQuery(query="CREATE").kill()'); + mitmproxy +----------- + +(1 row) + +CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1); +ERROR: CONCURRENTLY-enabled index command failed +DETAIL: CONCURRENTLY-enabled index commands can fail partially, leaving behind an INVALID index. +HINT: Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index, then retry the original command. +SELECT citus.mitmproxy('conn.allow()'); + mitmproxy +----------- + +(1 row) + +-- verify index is not created +SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test%' $$) +WHERE nodeport = :worker_2_proxy_port; + nodename | nodeport | success | result +-----------+----------+---------+-------- + localhost | 57640 | t | 0 +(1 row) + +DROP TABLE index_test; +CREATE TABLE index_test(id int, value_1 int, value_2 int); +SELECT create_distributed_table('index_test', 'id'); + create_distributed_table +-------------------------- + +(1 row) + +-- kill the connection at the second create command is issued +SELECT citus.mitmproxy('conn.onQuery(query="CREATE").after(1).kill()'); + mitmproxy +----------- + +(1 row) + +CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1); +ERROR: CONCURRENTLY-enabled index command failed +DETAIL: CONCURRENTLY-enabled index commands can fail partially, leaving behind an INVALID index. +HINT: Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index, then retry the original command. +SELECT citus.mitmproxy('conn.allow()'); + mitmproxy +----------- + +(1 row) + +-- verify only one index is created +SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test%' $$) +WHERE nodeport = :worker_2_proxy_port; + nodename | nodeport | success | result +-----------+----------+---------+-------- + localhost | 57640 | t | 1 +(1 row) + +DROP TABLE index_test; +CREATE TABLE index_test(id int, value_1 int, value_2 int); +SELECT create_reference_table('index_test'); + create_reference_table +------------------------ + +(1 row) + +-- kill the connection when create command is issued +SELECT citus.mitmproxy('conn.onQuery(query="CREATE").kill()'); + mitmproxy +----------- + +(1 row) + +CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1); +ERROR: CONCURRENTLY-enabled index command failed +DETAIL: CONCURRENTLY-enabled index commands can fail partially, leaving behind an INVALID index. +HINT: Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index, then retry the original command. +SELECT citus.mitmproxy('conn.allow()'); + mitmproxy +----------- + +(1 row) + +DROP TABLE index_test; +CREATE TABLE index_test(id int, value_1 int, value_2 int); +SELECT create_distributed_table('index_test', 'id'); + create_distributed_table +-------------------------- + +(1 row) + +-- cancel the connection when create command is issued +-- network traffic may differ between execution during cancellation +-- therefore dump_network_traffic() calls are not made +SELECT citus.mitmproxy('conn.onQuery(query="CREATE").cancel(' || pg_backend_pid() || ')'); + mitmproxy +----------- + +(1 row) + +CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1); +ERROR: CONCURRENTLY-enabled index command failed +DETAIL: CONCURRENTLY-enabled index commands can fail partially, leaving behind an INVALID index. +HINT: Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index, then retry the original command. +SELECT citus.mitmproxy('conn.allow()'); + mitmproxy +----------- + +(1 row) + +DROP TABLE index_test; +CREATE TABLE index_test(id int, value_1 int, value_2 int); +SELECT create_reference_table('index_test'); + create_reference_table +------------------------ + +(1 row) + +-- cancel the connection when create command is issued +SELECT citus.mitmproxy('conn.onQuery(query="CREATE").cancel(' || pg_backend_pid() || ')'); + mitmproxy +----------- + +(1 row) + +CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1); +ERROR: CONCURRENTLY-enabled index command failed +DETAIL: CONCURRENTLY-enabled index commands can fail partially, leaving behind an INVALID index. +HINT: Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index, then retry the original command. +SELECT citus.mitmproxy('conn.allow()'); + mitmproxy +----------- + +(1 row) + +DROP TABLE index_test; +CREATE TABLE index_test(id int, value_1 int, value_2 int); +SELECT create_distributed_table('index_test', 'id'); + create_distributed_table +-------------------------- + +(1 row) + +CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1); +-- kill the connection when create command is issued +SELECT citus.mitmproxy('conn.onQuery(query="DROP INDEX CONCURRENTLY").kill()'); + mitmproxy +----------- + +(1 row) + +DROP INDEX CONCURRENTLY IF EXISTS idx_index_test; +ERROR: CONCURRENTLY-enabled index command failed +DETAIL: CONCURRENTLY-enabled index commands can fail partially, leaving behind an INVALID index. +HINT: Use DROP INDEX CONCURRENTLY IF EXISTS to remove the invalid index, then retry the original command. +SELECT citus.mitmproxy('conn.allow()'); + mitmproxy +----------- + +(1 row) + +-- verify index is not dropped at worker 2 +SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test%' $$) +WHERE nodeport = :worker_2_proxy_port; + nodename | nodeport | success | result +-----------+----------+---------+-------- + localhost | 57640 | t | 4 +(1 row) + +RESET SEARCH_PATH; +DROP SCHEMA index_schema CASCADE; +NOTICE: drop cascades to table index_schema.index_test +-- verify index is not at worker 2 upon cleanup +SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test%' $$) +WHERE nodeport = :worker_2_proxy_port; + nodename | nodeport | success | result +-----------+----------+---------+-------- + localhost | 57640 | t | 0 +(1 row) + diff --git a/src/test/regress/failure_schedule b/src/test/regress/failure_schedule index 1b96e6657..e9849bf6e 100644 --- a/src/test/regress/failure_schedule +++ b/src/test/regress/failure_schedule @@ -6,3 +6,4 @@ test: failure_setup test: multi_test_helpers test: failure_ddl +test: failure_create_index_concurrently diff --git a/src/test/regress/pg_regress_multi.pl b/src/test/regress/pg_regress_multi.pl index 6274a8363..aa23ad264 100755 --- a/src/test/regress/pg_regress_multi.pl +++ b/src/test/regress/pg_regress_multi.pl @@ -269,6 +269,8 @@ for (my $workerIndex = 1; $workerIndex <= $workerCount; $workerIndex++) { push(@followerWorkerPorts, $workerPort); } +my $workerBehindProxyPort = $workerPorts[1] + 2; + my $host = "localhost"; my $user = "postgres"; my @pgOptions = (); @@ -417,6 +419,7 @@ if ($usingWindows) } print $fh catfile($bindir, "psql")." "; print $fh "--variable=master_port=$masterPort "; +print $fh "--variable=worker_2_proxy_port=$workerBehindProxyPort "; print $fh "--variable=follower_master_port=$followerCoordPort "; print $fh "--variable=default_user=$user "; print $fh "--variable=SHOW_CONTEXT=always "; diff --git a/src/test/regress/sql/failure_create_index_concurrently.sql b/src/test/regress/sql/failure_create_index_concurrently.sql new file mode 100644 index 000000000..a5355f86e --- /dev/null +++ b/src/test/regress/sql/failure_create_index_concurrently.sql @@ -0,0 +1,106 @@ +-- +-- failure_create_index_concurrently +-- test create index concurrently command +-- failure. + +SELECT citus.mitmproxy('conn.allow()'); + +SET citus.shard_count = 4; -- two per worker + +CREATE SCHEMA index_schema; +SET SEARCH_PATH=index_schema; + +CREATE TABLE index_test(id int, value_1 int, value_2 int); +SELECT create_distributed_table('index_test', 'id'); + +-- kill the connection when create command is issued +SELECT citus.mitmproxy('conn.onQuery(query="CREATE").kill()'); + +CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1); + +SELECT citus.mitmproxy('conn.allow()'); +-- verify index is not created +SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test%' $$) +WHERE nodeport = :worker_2_proxy_port; + + +DROP TABLE index_test; + + +CREATE TABLE index_test(id int, value_1 int, value_2 int); +SELECT create_distributed_table('index_test', 'id'); + +-- kill the connection at the second create command is issued +SELECT citus.mitmproxy('conn.onQuery(query="CREATE").after(1).kill()'); + +CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1); + +SELECT citus.mitmproxy('conn.allow()'); + +-- verify only one index is created +SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test%' $$) +WHERE nodeport = :worker_2_proxy_port; + +DROP TABLE index_test; + + +CREATE TABLE index_test(id int, value_1 int, value_2 int); +SELECT create_reference_table('index_test'); + +-- kill the connection when create command is issued +SELECT citus.mitmproxy('conn.onQuery(query="CREATE").kill()'); + +CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1); + +SELECT citus.mitmproxy('conn.allow()'); + +DROP TABLE index_test; + + +CREATE TABLE index_test(id int, value_1 int, value_2 int); +SELECT create_distributed_table('index_test', 'id'); + +-- cancel the connection when create command is issued +-- network traffic may differ between execution during cancellation +-- therefore dump_network_traffic() calls are not made +SELECT citus.mitmproxy('conn.onQuery(query="CREATE").cancel(' || pg_backend_pid() || ')'); + +CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1); + +SELECT citus.mitmproxy('conn.allow()'); + +DROP TABLE index_test; + +CREATE TABLE index_test(id int, value_1 int, value_2 int); +SELECT create_reference_table('index_test'); + +-- cancel the connection when create command is issued +SELECT citus.mitmproxy('conn.onQuery(query="CREATE").cancel(' || pg_backend_pid() || ')'); + +CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1); + +SELECT citus.mitmproxy('conn.allow()'); + +DROP TABLE index_test; + + +CREATE TABLE index_test(id int, value_1 int, value_2 int); +SELECT create_distributed_table('index_test', 'id'); + +CREATE INDEX CONCURRENTLY idx_index_test ON index_test(id, value_1); + +-- kill the connection when create command is issued +SELECT citus.mitmproxy('conn.onQuery(query="DROP INDEX CONCURRENTLY").kill()'); +DROP INDEX CONCURRENTLY IF EXISTS idx_index_test; +SELECT citus.mitmproxy('conn.allow()'); + +-- verify index is not dropped at worker 2 +SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test%' $$) +WHERE nodeport = :worker_2_proxy_port; + +RESET SEARCH_PATH; +DROP SCHEMA index_schema CASCADE; + +-- verify index is not at worker 2 upon cleanup +SELECT * FROM run_command_on_workers($$SELECT count(*) FROM pg_indexes WHERE indexname LIKE 'idx_index_test%' $$) +WHERE nodeport = :worker_2_proxy_port;