From 43218eebf62612bbfbe250851b3222a200227f5c Mon Sep 17 00:00:00 2001 From: Hadi Moshayedi Date: Tue, 14 Jan 2020 14:55:05 -0800 Subject: [PATCH] Failure tests for INSERT/SELECT repartition --- .../failure_insert_select_repartition.out | 200 ++++++++++++++++++ src/test/regress/failure_schedule | 1 + .../sql/failure_insert_select_repartition.sql | 96 +++++++++ 3 files changed, 297 insertions(+) create mode 100644 src/test/regress/expected/failure_insert_select_repartition.out create mode 100644 src/test/regress/sql/failure_insert_select_repartition.sql diff --git a/src/test/regress/expected/failure_insert_select_repartition.out b/src/test/regress/expected/failure_insert_select_repartition.out new file mode 100644 index 000000000..a0d91e0ef --- /dev/null +++ b/src/test/regress/expected/failure_insert_select_repartition.out @@ -0,0 +1,200 @@ +-- +-- failure_insert_select_repartition +-- +-- performs failure/cancellation test for repartitioned insert/select. +-- +CREATE SCHEMA repartitioned_insert_select; +SET SEARCH_PATH=repartitioned_insert_select; +SELECT pg_backend_pid() as pid \gset +SET citus.next_shard_id TO 4213581; +SET citus.shard_count TO 4; +SET citus.shard_replication_factor TO 2; +CREATE TABLE replicated_source_table(a int, b int); +SELECT create_distributed_table('replicated_source_table', 'a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO replicated_source_table SELECT i, i*i FROM generate_series(1, 10) i; +SET citus.shard_count TO 3; +CREATE TABLE replicated_target_table(a int, b int); +SELECT create_distributed_table('replicated_target_table', 'a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SET citus.shard_replication_factor TO 1; +SET citus.shard_count TO 4; +CREATE TABLE source_table(a int, b int); +SELECT create_distributed_table('source_table', 'a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO source_table SELECT i, i*i FROM generate_series(1, 10) i; +SET citus.shard_count TO 3; +CREATE TABLE target_table(a int, b int); +SELECT create_distributed_table('target_table', 'a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- +-- kill worker_partition_query_result +-- this fails the query on source table, so replicated case should succeed +-- +SELECT citus.mitmproxy('conn.onQuery(query="worker_partition_query_result").kill()'); + mitmproxy +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO target_table SELECT * FROM source_table; +ERROR: connection error: localhost:xxxxx +DETAIL: server closed the connection unexpectedly + This probably means the server terminated abnormally + before or while processing the request. +SELECT * FROM target_table ORDER BY a; + a | b +--------------------------------------------------------------------- +(0 rows) + +SELECT citus.mitmproxy('conn.onQuery(query="worker_partition_query_result").kill()'); + mitmproxy +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO target_table SELECT * FROM replicated_source_table; +WARNING: connection error: localhost:xxxxx +DETAIL: server closed the connection unexpectedly + This probably means the server terminated abnormally + before or while processing the request. +WARNING: connection error: localhost:xxxxx +DETAIL: server closed the connection unexpectedly + This probably means the server terminated abnormally + before or while processing the request. +SELECT * FROM target_table ORDER BY a; + a | b +--------------------------------------------------------------------- + 1 | 1 + 2 | 4 + 3 | 9 + 4 | 16 + 5 | 25 + 6 | 36 + 7 | 49 + 8 | 64 + 9 | 81 + 10 | 100 +(10 rows) + +-- +-- kill fetch_intermediate_results +-- this fails the fetch into target, so source replication doesn't matter +-- and both should fail +-- +TRUNCATE target_table; +SELECT citus.mitmproxy('conn.onQuery(query="fetch_intermediate_results").kill()'); + mitmproxy +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO target_table SELECT * FROM source_table; +ERROR: connection error: localhost:xxxxx +DETAIL: server closed the connection unexpectedly + This probably means the server terminated abnormally + before or while processing the request. +SELECT * FROM target_table ORDER BY a; + a | b +--------------------------------------------------------------------- +(0 rows) + +SELECT citus.mitmproxy('conn.onQuery(query="fetch_intermediate_results").kill()'); + mitmproxy +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO target_table SELECT * FROM replicated_source_table; +ERROR: connection error: localhost:xxxxx +DETAIL: server closed the connection unexpectedly + This probably means the server terminated abnormally + before or while processing the request. +SELECT * FROM target_table ORDER BY a; + a | b +--------------------------------------------------------------------- +(0 rows) + +-- +-- kill read_intermediate_results +-- again, both should fail +-- +TRUNCATE target_table; +SELECT citus.mitmproxy('conn.onQuery(query="read_intermediate_results").kill()'); + mitmproxy +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO target_table SELECT * FROM source_table; +ERROR: connection error: localhost:xxxxx +DETAIL: server closed the connection unexpectedly + This probably means the server terminated abnormally + before or while processing the request. +SELECT * FROM target_table ORDER BY a; + a | b +--------------------------------------------------------------------- +(0 rows) + +SELECT citus.mitmproxy('conn.onQuery(query="read_intermediate_results").kill()'); + mitmproxy +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO target_table SELECT * FROM replicated_source_table; +ERROR: connection error: localhost:xxxxx +DETAIL: server closed the connection unexpectedly + This probably means the server terminated abnormally + before or while processing the request. +SELECT * FROM target_table ORDER BY a; + a | b +--------------------------------------------------------------------- +(0 rows) + +-- +-- We error out even if table is replicated and only one of the replicas +-- fail. +-- +SELECT citus.mitmproxy('conn.onQuery(query="read_intermediate_results").kill()'); + mitmproxy +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO replicated_target_table SELECT * FROM source_table; +ERROR: connection error: localhost:xxxxx +DETAIL: server closed the connection unexpectedly + This probably means the server terminated abnormally + before or while processing the request. +SELECT * FROM replicated_target_table; + a | b +--------------------------------------------------------------------- +(0 rows) + +RESET SEARCH_PATH; +SELECT citus.mitmproxy('conn.allow()'); + mitmproxy +--------------------------------------------------------------------- + +(1 row) + +\set VERBOSITY TERSE +DROP SCHEMA repartitioned_insert_select CASCADE; +NOTICE: drop cascades to 4 other objects diff --git a/src/test/regress/failure_schedule b/src/test/regress/failure_schedule index ddaff3a8c..0e2143370 100644 --- a/src/test/regress/failure_schedule +++ b/src/test/regress/failure_schedule @@ -6,6 +6,7 @@ test: failure_setup test: multi_test_helpers test: failure_replicated_partitions test: multi_test_catalog_views +test: failure_insert_select_repartition test: failure_distributed_results test: failure_ddl test: failure_truncate diff --git a/src/test/regress/sql/failure_insert_select_repartition.sql b/src/test/regress/sql/failure_insert_select_repartition.sql new file mode 100644 index 000000000..1f6eb1812 --- /dev/null +++ b/src/test/regress/sql/failure_insert_select_repartition.sql @@ -0,0 +1,96 @@ +-- +-- failure_insert_select_repartition +-- +-- performs failure/cancellation test for repartitioned insert/select. +-- + +CREATE SCHEMA repartitioned_insert_select; +SET SEARCH_PATH=repartitioned_insert_select; +SELECT pg_backend_pid() as pid \gset + +SET citus.next_shard_id TO 4213581; + +SET citus.shard_count TO 4; +SET citus.shard_replication_factor TO 2; +CREATE TABLE replicated_source_table(a int, b int); +SELECT create_distributed_table('replicated_source_table', 'a'); +INSERT INTO replicated_source_table SELECT i, i*i FROM generate_series(1, 10) i; + +SET citus.shard_count TO 3; +CREATE TABLE replicated_target_table(a int, b int); +SELECT create_distributed_table('replicated_target_table', 'a'); + +SET citus.shard_replication_factor TO 1; + +SET citus.shard_count TO 4; +CREATE TABLE source_table(a int, b int); +SELECT create_distributed_table('source_table', 'a'); +INSERT INTO source_table SELECT i, i*i FROM generate_series(1, 10) i; + +SET citus.shard_count TO 3; +CREATE TABLE target_table(a int, b int); +SELECT create_distributed_table('target_table', 'a'); + +-- +-- kill worker_partition_query_result +-- this fails the query on source table, so replicated case should succeed +-- +SELECT citus.mitmproxy('conn.onQuery(query="worker_partition_query_result").kill()'); +INSERT INTO target_table SELECT * FROM source_table; + +SELECT * FROM target_table ORDER BY a; + +SELECT citus.mitmproxy('conn.onQuery(query="worker_partition_query_result").kill()'); +INSERT INTO target_table SELECT * FROM replicated_source_table; + +SELECT * FROM target_table ORDER BY a; + +-- +-- kill fetch_intermediate_results +-- this fails the fetch into target, so source replication doesn't matter +-- and both should fail +-- + +TRUNCATE target_table; + +SELECT citus.mitmproxy('conn.onQuery(query="fetch_intermediate_results").kill()'); +INSERT INTO target_table SELECT * FROM source_table; + +SELECT * FROM target_table ORDER BY a; + +SELECT citus.mitmproxy('conn.onQuery(query="fetch_intermediate_results").kill()'); +INSERT INTO target_table SELECT * FROM replicated_source_table; + +SELECT * FROM target_table ORDER BY a; + +-- +-- kill read_intermediate_results +-- again, both should fail +-- + +TRUNCATE target_table; + +SELECT citus.mitmproxy('conn.onQuery(query="read_intermediate_results").kill()'); +INSERT INTO target_table SELECT * FROM source_table; + +SELECT * FROM target_table ORDER BY a; + +SELECT citus.mitmproxy('conn.onQuery(query="read_intermediate_results").kill()'); +INSERT INTO target_table SELECT * FROM replicated_source_table; + +SELECT * FROM target_table ORDER BY a; + +-- +-- We error out even if table is replicated and only one of the replicas +-- fail. +-- + +SELECT citus.mitmproxy('conn.onQuery(query="read_intermediate_results").kill()'); +INSERT INTO replicated_target_table SELECT * FROM source_table; + +SELECT * FROM replicated_target_table; + +RESET SEARCH_PATH; +SELECT citus.mitmproxy('conn.allow()'); +\set VERBOSITY TERSE +DROP SCHEMA repartitioned_insert_select CASCADE;