From 44fc9f46fcdfb472ce886eb674254e1b3b202585 Mon Sep 17 00:00:00 2001 From: velioglu Date: Wed, 11 Jul 2018 17:59:10 +0300 Subject: [PATCH] Add create_distributed_table (without data) failure tests --- .../regress/expected/failure_create_table.out | 697 ++++++++++++++++++ src/test/regress/failure_schedule | 1 + src/test/regress/sql/failure_create_table.sql | 234 ++++++ 3 files changed, 932 insertions(+) create mode 100644 src/test/regress/expected/failure_create_table.out create mode 100644 src/test/regress/sql/failure_create_table.sql diff --git a/src/test/regress/expected/failure_create_table.out b/src/test/regress/expected/failure_create_table.out new file mode 100644 index 000000000..8990e691f --- /dev/null +++ b/src/test/regress/expected/failure_create_table.out @@ -0,0 +1,697 @@ +-- +-- failure_create_table adds failure tests for creating table without data. +-- +CREATE SCHEMA failure_create_table; +SET search_path TO 'failure_create_table'; +SELECT citus.mitmproxy('conn.allow()'); + mitmproxy +----------- + +(1 row) + +SET citus.shard_replication_factor TO 1; +SET citus.shard_count to 4; +CREATE TABLE test_table(id int, value_1 int); +-- Kill connection before sending query to the worker +SELECT citus.mitmproxy('conn.kill()'); + mitmproxy +----------- + +(1 row) + +SELECT create_distributed_table('test_table','id'); +ERROR: connection error: localhost:9060 +DETAIL: server closed the connection unexpectedly + This probably means the server terminated abnormally + before or while processing the request. +SELECT citus.mitmproxy('conn.allow()'); + mitmproxy +----------- + +(1 row) + +SELECT count(*) FROM pg_dist_shard; + count +------- + 0 +(1 row) + +SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$); + run_command_on_workers +------------------------ + (localhost,9060,t,0) + (localhost,57637,t,0) +(2 rows) + +-- kill as soon as the coordinator sends CREATE SCHEMA +-- Since schemas are created in separate transaction, schema will +-- be created only on the node which is not behind the proxy. +-- https://github.com/citusdata/citus/pull/1652 +SELECT citus.mitmproxy('conn.onQuery(query="^CREATE SCHEMA").kill()'); + mitmproxy +----------- + +(1 row) + +SELECT create_distributed_table('test_table', 'id'); +ERROR: server closed the connection unexpectedly + This probably means the server terminated abnormally + before or while processing the request. +CONTEXT: while executing command on localhost:9060 +SELECT citus.mitmproxy('conn.allow()'); + mitmproxy +----------- + +(1 row) + +SELECT count(*) FROM pg_dist_shard; + count +------- + 0 +(1 row) + +SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.schemata WHERE schema_name = 'failure_create_table'$$); + run_command_on_workers +------------------------ + (localhost,9060,t,0) + (localhost,57637,t,1) +(2 rows) + +-- Now, kill the connection while opening transaction on workers. +SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED").kill()'); + mitmproxy +----------- + +(1 row) + +SELECT create_distributed_table('test_table','id'); +WARNING: connection not open +CONTEXT: while executing command on localhost:9060 +ERROR: connection error: localhost:9060 +DETAIL: connection not open +SELECT citus.mitmproxy('conn.allow()'); + mitmproxy +----------- + +(1 row) + +SELECT count(*) FROM pg_dist_shard; + count +------- + 0 +(1 row) + +SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$); + run_command_on_workers +------------------------ + (localhost,9060,t,0) + (localhost,57637,t,0) +(2 rows) + +-- Now, kill the connection after sending create table command with worker_apply_shard_ddl_command UDF +SELECT citus.mitmproxy('conn.onQuery(query="^SELECT worker_apply_shard_ddl_command").after(2).kill()'); + mitmproxy +----------- + +(1 row) + +SELECT create_distributed_table('test_table','id'); +ERROR: server closed the connection unexpectedly + This probably means the server terminated abnormally + before or while processing the request. +CONTEXT: while executing command on localhost:9060 +SELECT citus.mitmproxy('conn.allow()'); + mitmproxy +----------- + +(1 row) + +SELECT count(*) FROM pg_dist_shard; + count +------- + 0 +(1 row) + +SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$); + run_command_on_workers +------------------------ + (localhost,9060,t,0) + (localhost,57637,t,0) +(2 rows) + +-- Kill the connection while creating a distributed table in sequential mode on sending create command +-- with worker_apply_shard_ddl_command UDF. +BEGIN; + SET LOCAL citus.multi_shard_modify_mode TO 'sequential'; + SELECT citus.mitmproxy('conn.onQuery(query="^SELECT worker_apply_shard_ddl_command").after(2).kill()'); + mitmproxy +----------- + +(1 row) + + SELECT create_distributed_table('test_table', 'id'); +ERROR: server closed the connection unexpectedly + This probably means the server terminated abnormally + before or while processing the request. +CONTEXT: while executing command on localhost:9060 +COMMIT; +SELECT citus.mitmproxy('conn.allow()'); + mitmproxy +----------- + +(1 row) + +SELECT count(*) FROM pg_dist_shard; + count +------- + 0 +(1 row) + +SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$); + run_command_on_workers +------------------------ + (localhost,9060,t,0) + (localhost,57637,t,0) +(2 rows) + +-- Now, cancel the connection while creating transaction +-- workers. Note that, cancel requests will be ignored during +-- shard creation. +SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED").cancel(' || pg_backend_pid() || ')'); + mitmproxy +----------- + +(1 row) + +SELECT create_distributed_table('test_table','id'); +WARNING: cancel requests are ignored during shard creation + create_distributed_table +-------------------------- + +(1 row) + +SELECT citus.mitmproxy('conn.allow()'); + mitmproxy +----------- + +(1 row) + +SELECT count(*) FROM pg_dist_shard; + count +------- + 4 +(1 row) + +SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$); + run_command_on_workers +------------------------ + (localhost,9060,t,2) + (localhost,57637,t,2) +(2 rows) + +DROP TABLE test_table; +CREATE TABLE test_table(id int, value_1 int); +-- Kill and cancel the connection with colocate_with option while sending the create table command +CREATE TABLE temp_table(id int, value_1 int); +SELECT create_distributed_table('temp_table','id'); + create_distributed_table +-------------------------- + +(1 row) + +SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").kill()'); + mitmproxy +----------- + +(1 row) + +SELECT create_distributed_table('test_table','id',colocate_with=>'temp_table'); +ERROR: server closed the connection unexpectedly + This probably means the server terminated abnormally + before or while processing the request. +CONTEXT: while executing command on localhost:9060 +SELECT citus.mitmproxy('conn.allow()'); + mitmproxy +----------- + +(1 row) + +SELECT count(*) FROM pg_dist_shard; + count +------- + 4 +(1 row) + +SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$); + run_command_on_workers +------------------------ + (localhost,9060,t,0) + (localhost,57637,t,0) +(2 rows) + +SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").cancel(' || pg_backend_pid() || ')'); + mitmproxy +----------- + +(1 row) + +SELECT create_distributed_table('test_table','id',colocate_with=>'temp_table'); +ERROR: canceling statement due to user request +SELECT citus.mitmproxy('conn.allow()'); + mitmproxy +----------- + +(1 row) + +SELECT count(*) FROM pg_dist_shard; + count +------- + 4 +(1 row) + +SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$); + run_command_on_workers +------------------------ + (localhost,9060,t,0) + (localhost,57637,t,0) +(2 rows) + +-- Kill and cancel the connection after worker sends "PREPARE TRANSACTION" ack with colocate_with option +SELECT citus.mitmproxy('conn.onCommandComplete(command="PREPARE TRANSACTION").kill()'); + mitmproxy +----------- + +(1 row) + +SELECT create_distributed_table('test_table','id',colocate_with=>'temp_table'); +ERROR: connection not open +CONTEXT: while executing command on localhost:9060 +SELECT citus.mitmproxy('conn.allow()'); + mitmproxy +----------- + +(1 row) + +SELECT count(*) FROM pg_dist_shard; + count +------- + 4 +(1 row) + +SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$); + run_command_on_workers +------------------------ + (localhost,9060,t,0) + (localhost,57637,t,0) +(2 rows) + +SELECT citus.mitmproxy('conn.onCommandComplete(command="PREPARE TRANSACTION").cancel(' || pg_backend_pid() || ')'); + mitmproxy +----------- + +(1 row) + +SELECT create_distributed_table('test_table','id',colocate_with=>'temp_table'); +ERROR: canceling statement due to user request +SELECT citus.mitmproxy('conn.allow()'); + mitmproxy +----------- + +(1 row) + +SELECT count(*) FROM pg_dist_shard; + count +------- + 4 +(1 row) + +SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$); + run_command_on_workers +------------------------ + (localhost,9060,t,0) + (localhost,57637,t,0) +(2 rows) + +DROP TABLE temp_table; +-- Test inside transaction +-- Kill connection before sending query to the worker +SELECT citus.mitmproxy('conn.kill()'); + mitmproxy +----------- + +(1 row) + +BEGIN; +SELECT create_distributed_table('test_table','id'); +ERROR: connection error: localhost:9060 +DETAIL: server closed the connection unexpectedly + This probably means the server terminated abnormally + before or while processing the request. +ROLLBACK; +SELECT citus.mitmproxy('conn.allow()'); + mitmproxy +----------- + +(1 row) + +SELECT count(*) FROM pg_dist_shard; + count +------- + 0 +(1 row) + +SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$); + run_command_on_workers +------------------------ + (localhost,9060,t,0) + (localhost,57637,t,0) +(2 rows) + +-- Now, kill the connection while creating transaction on workers in transaction. +SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED").kill()'); + mitmproxy +----------- + +(1 row) + +BEGIN; +SELECT create_distributed_table('test_table','id'); +WARNING: connection not open +CONTEXT: while executing command on localhost:9060 +ERROR: connection error: localhost:9060 +DETAIL: connection not open +ROLLBACK; +SELECT citus.mitmproxy('conn.allow()'); + mitmproxy +----------- + +(1 row) + +SELECT count(*) FROM pg_dist_shard; + count +------- + 0 +(1 row) + +SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$); + run_command_on_workers +------------------------ + (localhost,9060,t,0) + (localhost,57637,t,0) +(2 rows) + +-- Now, cancel the connection while creating the transaction on +-- workers. Note that, cancel requests will be ignored during +-- shard creation again in transaction. +SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED").cancel(' || pg_backend_pid() || ')'); + mitmproxy +----------- + +(1 row) + +BEGIN; +SELECT create_distributed_table('test_table','id'); +WARNING: cancel requests are ignored during shard creation + create_distributed_table +-------------------------- + +(1 row) + +COMMIT; +SELECT citus.mitmproxy('conn.allow()'); + mitmproxy +----------- + +(1 row) + +SELECT count(*) FROM pg_dist_shard; + count +------- + 4 +(1 row) + +SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$); + run_command_on_workers +------------------------ + (localhost,9060,t,2) + (localhost,57637,t,2) +(2 rows) + +DROP TABLE test_table; +CREATE TABLE test_table(id int, value_1 int); +-- Test inside transaction and with 1PC +SET citus.multi_shard_commit_protocol TO "1pc"; +-- Kill connection before sending query to the worker with 1pc. +SELECT citus.mitmproxy('conn.kill()'); + mitmproxy +----------- + +(1 row) + +BEGIN; +SELECT create_distributed_table('test_table','id'); +ERROR: connection error: localhost:9060 +DETAIL: server closed the connection unexpectedly + This probably means the server terminated abnormally + before or while processing the request. +ROLLBACK; +SELECT citus.mitmproxy('conn.allow()'); + mitmproxy +----------- + +(1 row) + +SELECT count(*) FROM pg_dist_shard; + count +------- + 0 +(1 row) + +SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$); + run_command_on_workers +------------------------ + (localhost,9060,t,0) + (localhost,57637,t,0) +(2 rows) + +-- Kill connection while sending create table command with 1pc. +SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").kill()'); + mitmproxy +----------- + +(1 row) + +BEGIN; +SELECT create_distributed_table('test_table','id'); +ERROR: server closed the connection unexpectedly + This probably means the server terminated abnormally + before or while processing the request. +CONTEXT: while executing command on localhost:9060 +ROLLBACK; +SELECT citus.mitmproxy('conn.allow()'); + mitmproxy +----------- + +(1 row) + +SELECT count(*) FROM pg_dist_shard; + count +------- + 0 +(1 row) + +SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$); + run_command_on_workers +------------------------ + (localhost,9060,t,0) + (localhost,57637,t,0) +(2 rows) + +-- Now, kill the connection while opening transactions on workers with 1pc. Transaction will be opened due to BEGIN. +SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED").kill()'); + mitmproxy +----------- + +(1 row) + +BEGIN; +SELECT create_distributed_table('test_table','id'); +WARNING: connection not open +CONTEXT: while executing command on localhost:9060 +ERROR: connection error: localhost:9060 +DETAIL: connection not open +ROLLBACK; +SELECT citus.mitmproxy('conn.allow()'); + mitmproxy +----------- + +(1 row) + +SELECT count(*) FROM pg_dist_shard; + count +------- + 0 +(1 row) + +SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$); + run_command_on_workers +------------------------ + (localhost,9060,t,0) + (localhost,57637,t,0) +(2 rows) + +-- Now, cancel the connection while creating transactions on +-- workers with 1pc. Note that, cancel requests will be ignored during +-- shard creation. +SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED").cancel(' || pg_backend_pid() || ')'); + mitmproxy +----------- + +(1 row) + +BEGIN; +SELECT create_distributed_table('test_table','id'); +WARNING: cancel requests are ignored during shard creation + create_distributed_table +-------------------------- + +(1 row) + +COMMIT; +SELECT citus.mitmproxy('conn.allow()'); + mitmproxy +----------- + +(1 row) + +SELECT count(*) FROM pg_dist_shard; + count +------- + 4 +(1 row) + +SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$); + run_command_on_workers +------------------------ + (localhost,9060,t,2) + (localhost,57637,t,2) +(2 rows) + +DROP TABLE test_table; +-- Test master_create_worker_shards with 2pc +SET citus.multi_shard_commit_protocol TO "2pc"; +CREATE TABLE test_table_2(id int, value_1 int); +SELECT master_create_distributed_table('test_table_2', 'id', 'hash'); + master_create_distributed_table +--------------------------------- + +(1 row) + +-- Kill connection before sending query to the worker +SELECT citus.mitmproxy('conn.kill()'); + mitmproxy +----------- + +(1 row) + +SELECT master_create_worker_shards('test_table_2', 4, 2); +WARNING: connection error: localhost:9060 +DETAIL: server closed the connection unexpectedly + This probably means the server terminated abnormally + before or while processing the request. +ERROR: connection error: localhost:9060 +DETAIL: server closed the connection unexpectedly + This probably means the server terminated abnormally + before or while processing the request. +SELECT count(*) FROM pg_dist_shard; + count +------- + 0 +(1 row) + +SELECT citus.mitmproxy('conn.allow()'); + mitmproxy +----------- + +(1 row) + +SELECT count(*) FROM pg_dist_shard; + count +------- + 0 +(1 row) + +SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$); + run_command_on_workers +------------------------ + (localhost,9060,t,0) + (localhost,57637,t,0) +(2 rows) + +-- Kill the connection after worker sends "PREPARE TRANSACTION" ack +SELECT citus.mitmproxy('conn.onCommandComplete(command="^PREPARE TRANSACTION").kill()'); + mitmproxy +----------- + +(1 row) + +SELECT master_create_worker_shards('test_table_2', 4, 2); +ERROR: connection not open +CONTEXT: while executing command on localhost:9060 +SELECT citus.mitmproxy('conn.allow()'); + mitmproxy +----------- + +(1 row) + +SELECT count(*) FROM pg_dist_shard; + count +------- + 0 +(1 row) + +SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$); + run_command_on_workers +------------------------ + (localhost,9060,t,0) + (localhost,57637,t,0) +(2 rows) + +-- Cancel the connection after sending prepare transaction in master_create_worker_shards +SELECT citus.mitmproxy('conn.onCommandComplete(command="PREPARE TRANSACTION").cancel(' || pg_backend_pid() || ')'); + mitmproxy +----------- + +(1 row) + +SELECT master_create_worker_shards('test_table_2', 4, 2); +ERROR: canceling statement due to user request +-- Show that there is no pending transaction +SELECT recover_prepared_transactions(); + recover_prepared_transactions +------------------------------- + 2 +(1 row) + +SELECT citus.mitmproxy('conn.allow()'); + mitmproxy +----------- + +(1 row) + +SELECT count(*) FROM pg_dist_shard; + count +------- + 0 +(1 row) + +SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$); + run_command_on_workers +------------------------ + (localhost,9060,t,0) + (localhost,57637,t,0) +(2 rows) + +DROP SCHEMA failure_create_table CASCADE; +NOTICE: drop cascades to table test_table_2 +SET search_path TO default; diff --git a/src/test/regress/failure_schedule b/src/test/regress/failure_schedule index 180897a0c..b84e795f3 100644 --- a/src/test/regress/failure_schedule +++ b/src/test/regress/failure_schedule @@ -13,6 +13,7 @@ test: failure_copy_to_reference test: failure_copy_on_hash test: failure_create_reference_table test: failure_create_distributed_table_non_empty +test: failure_create_table test: failure_1pc_copy_hash test: failure_1pc_copy_append diff --git a/src/test/regress/sql/failure_create_table.sql b/src/test/regress/sql/failure_create_table.sql new file mode 100644 index 000000000..35f4198eb --- /dev/null +++ b/src/test/regress/sql/failure_create_table.sql @@ -0,0 +1,234 @@ +-- +-- failure_create_table adds failure tests for creating table without data. +-- + +CREATE SCHEMA failure_create_table; +SET search_path TO 'failure_create_table'; + +SELECT citus.mitmproxy('conn.allow()'); +SET citus.shard_replication_factor TO 1; +SET citus.shard_count to 4; + +CREATE TABLE test_table(id int, value_1 int); + +-- Kill connection before sending query to the worker +SELECT citus.mitmproxy('conn.kill()'); +SELECT create_distributed_table('test_table','id'); + +SELECT citus.mitmproxy('conn.allow()'); +SELECT count(*) FROM pg_dist_shard; +SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$); + +-- kill as soon as the coordinator sends CREATE SCHEMA +-- Since schemas are created in separate transaction, schema will +-- be created only on the node which is not behind the proxy. +-- https://github.com/citusdata/citus/pull/1652 +SELECT citus.mitmproxy('conn.onQuery(query="^CREATE SCHEMA").kill()'); +SELECT create_distributed_table('test_table', 'id'); + +SELECT citus.mitmproxy('conn.allow()'); +SELECT count(*) FROM pg_dist_shard; +SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.schemata WHERE schema_name = 'failure_create_table'$$); + +-- Now, kill the connection while opening transaction on workers. +SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED").kill()'); +SELECT create_distributed_table('test_table','id'); + +SELECT citus.mitmproxy('conn.allow()'); +SELECT count(*) FROM pg_dist_shard; +SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$); + +-- Now, kill the connection after sending create table command with worker_apply_shard_ddl_command UDF +SELECT citus.mitmproxy('conn.onQuery(query="^SELECT worker_apply_shard_ddl_command").after(2).kill()'); +SELECT create_distributed_table('test_table','id'); + +SELECT citus.mitmproxy('conn.allow()'); +SELECT count(*) FROM pg_dist_shard; +SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$); + +-- Kill the connection while creating a distributed table in sequential mode on sending create command +-- with worker_apply_shard_ddl_command UDF. +BEGIN; + SET LOCAL citus.multi_shard_modify_mode TO 'sequential'; + SELECT citus.mitmproxy('conn.onQuery(query="^SELECT worker_apply_shard_ddl_command").after(2).kill()'); + SELECT create_distributed_table('test_table', 'id'); +COMMIT; + +SELECT citus.mitmproxy('conn.allow()'); +SELECT count(*) FROM pg_dist_shard; +SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$); + +-- Now, cancel the connection while creating transaction +-- workers. Note that, cancel requests will be ignored during +-- shard creation. +SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED").cancel(' || pg_backend_pid() || ')'); +SELECT create_distributed_table('test_table','id'); + +SELECT citus.mitmproxy('conn.allow()'); +SELECT count(*) FROM pg_dist_shard; +SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$); + +DROP TABLE test_table; +CREATE TABLE test_table(id int, value_1 int); + +-- Kill and cancel the connection with colocate_with option while sending the create table command +CREATE TABLE temp_table(id int, value_1 int); +SELECT create_distributed_table('temp_table','id'); + +SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").kill()'); +SELECT create_distributed_table('test_table','id',colocate_with=>'temp_table'); + +SELECT citus.mitmproxy('conn.allow()'); +SELECT count(*) FROM pg_dist_shard; +SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$); + +SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").cancel(' || pg_backend_pid() || ')'); +SELECT create_distributed_table('test_table','id',colocate_with=>'temp_table'); + +SELECT citus.mitmproxy('conn.allow()'); +SELECT count(*) FROM pg_dist_shard; +SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$); + +-- Kill and cancel the connection after worker sends "PREPARE TRANSACTION" ack with colocate_with option +SELECT citus.mitmproxy('conn.onCommandComplete(command="PREPARE TRANSACTION").kill()'); +SELECT create_distributed_table('test_table','id',colocate_with=>'temp_table'); + +SELECT citus.mitmproxy('conn.allow()'); +SELECT count(*) FROM pg_dist_shard; +SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$); + +SELECT citus.mitmproxy('conn.onCommandComplete(command="PREPARE TRANSACTION").cancel(' || pg_backend_pid() || ')'); +SELECT create_distributed_table('test_table','id',colocate_with=>'temp_table'); + +SELECT citus.mitmproxy('conn.allow()'); +SELECT count(*) FROM pg_dist_shard; +SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$); + +DROP TABLE temp_table; + +-- Test inside transaction +-- Kill connection before sending query to the worker +SELECT citus.mitmproxy('conn.kill()'); + +BEGIN; +SELECT create_distributed_table('test_table','id'); +ROLLBACK; + +SELECT citus.mitmproxy('conn.allow()'); +SELECT count(*) FROM pg_dist_shard; +SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$); + +-- Now, kill the connection while creating transaction on workers in transaction. +SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED").kill()'); + +BEGIN; +SELECT create_distributed_table('test_table','id'); +ROLLBACK; + +SELECT citus.mitmproxy('conn.allow()'); +SELECT count(*) FROM pg_dist_shard; +SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$); + +-- Now, cancel the connection while creating the transaction on +-- workers. Note that, cancel requests will be ignored during +-- shard creation again in transaction. +SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED").cancel(' || pg_backend_pid() || ')'); + +BEGIN; +SELECT create_distributed_table('test_table','id'); +COMMIT; + +SELECT citus.mitmproxy('conn.allow()'); +SELECT count(*) FROM pg_dist_shard; +SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$); + +DROP TABLE test_table; +CREATE TABLE test_table(id int, value_1 int); + +-- Test inside transaction and with 1PC +SET citus.multi_shard_commit_protocol TO "1pc"; + +-- Kill connection before sending query to the worker with 1pc. +SELECT citus.mitmproxy('conn.kill()'); + +BEGIN; +SELECT create_distributed_table('test_table','id'); +ROLLBACK; + +SELECT citus.mitmproxy('conn.allow()'); +SELECT count(*) FROM pg_dist_shard; +SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$); + +-- Kill connection while sending create table command with 1pc. +SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").kill()'); + +BEGIN; +SELECT create_distributed_table('test_table','id'); +ROLLBACK; + +SELECT citus.mitmproxy('conn.allow()'); +SELECT count(*) FROM pg_dist_shard; +SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$); + +-- Now, kill the connection while opening transactions on workers with 1pc. Transaction will be opened due to BEGIN. +SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED").kill()'); + +BEGIN; +SELECT create_distributed_table('test_table','id'); +ROLLBACK; + +SELECT citus.mitmproxy('conn.allow()'); +SELECT count(*) FROM pg_dist_shard; +SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$); + +-- Now, cancel the connection while creating transactions on +-- workers with 1pc. Note that, cancel requests will be ignored during +-- shard creation. +SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED").cancel(' || pg_backend_pid() || ')'); + +BEGIN; +SELECT create_distributed_table('test_table','id'); +COMMIT; + +SELECT citus.mitmproxy('conn.allow()'); +SELECT count(*) FROM pg_dist_shard; +SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$); + +DROP TABLE test_table; + +-- Test master_create_worker_shards with 2pc +SET citus.multi_shard_commit_protocol TO "2pc"; +CREATE TABLE test_table_2(id int, value_1 int); +SELECT master_create_distributed_table('test_table_2', 'id', 'hash'); + +-- Kill connection before sending query to the worker +SELECT citus.mitmproxy('conn.kill()'); +SELECT master_create_worker_shards('test_table_2', 4, 2); + +SELECT count(*) FROM pg_dist_shard; + +SELECT citus.mitmproxy('conn.allow()'); +SELECT count(*) FROM pg_dist_shard; +SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$); + +-- Kill the connection after worker sends "PREPARE TRANSACTION" ack +SELECT citus.mitmproxy('conn.onCommandComplete(command="^PREPARE TRANSACTION").kill()'); +SELECT master_create_worker_shards('test_table_2', 4, 2); + +SELECT citus.mitmproxy('conn.allow()'); +SELECT count(*) FROM pg_dist_shard; +SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$); + +-- Cancel the connection after sending prepare transaction in master_create_worker_shards +SELECT citus.mitmproxy('conn.onCommandComplete(command="PREPARE TRANSACTION").cancel(' || pg_backend_pid() || ')'); +SELECT master_create_worker_shards('test_table_2', 4, 2); + +-- Show that there is no pending transaction +SELECT recover_prepared_transactions(); + +SELECT citus.mitmproxy('conn.allow()'); +SELECT count(*) FROM pg_dist_shard; +SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$); + +DROP SCHEMA failure_create_table CASCADE; +SET search_path TO default;