From 9bcf2873a70f559217ea41e3053862e21a3b5f07 Mon Sep 17 00:00:00 2001 From: Jason Petersen Date: Mon, 1 Oct 2018 22:00:20 -0600 Subject: [PATCH] Add single-shard router select failure tests Including several examples from #1926. I couldn't understand why the recover_prepared_transactions "should be an error", and EXPLAIN has changed since the original bug (so that it runs EXPLAINs in txns, I think for EXPLAIN ANALYZE to not have side effects); other than that, most of the reported bugs now error out rather than crash or return an empty result set. --- .../expected/failure_single_select.out | 174 ++++++++++++++++++ src/test/regress/failure_schedule | 3 +- .../regress/sql/failure_single_select.sql | 63 +++++++ 3 files changed, 239 insertions(+), 1 deletion(-) create mode 100644 src/test/regress/expected/failure_single_select.out create mode 100644 src/test/regress/sql/failure_single_select.sql diff --git a/src/test/regress/expected/failure_single_select.out b/src/test/regress/expected/failure_single_select.out new file mode 100644 index 000000000..fca3e36e9 --- /dev/null +++ b/src/test/regress/expected/failure_single_select.out @@ -0,0 +1,174 @@ +SELECT citus.mitmproxy('conn.allow()'); + mitmproxy +----------- + +(1 row) + +SELECT citus.clear_network_traffic(); + clear_network_traffic +----------------------- + +(1 row) + +SET citus.shard_count = 2; +SET citus.shard_replication_factor = 2; +CREATE TABLE select_test (key int, value text); +SELECT create_distributed_table('select_test', 'key'); + create_distributed_table +-------------------------- + +(1 row) + +-- put data in shard for which mitm node is first placement +INSERT INTO select_test VALUES (2, 'test data'); +SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").kill()'); + mitmproxy +----------- + +(1 row) + +SELECT * FROM select_test WHERE key = 2; +WARNING: server closed the connection unexpectedly + This probably means the server terminated abnormally + before or while processing the request. +CONTEXT: while executing command on localhost:9060 + key | value +-----+----------- + 2 | test data +(1 row) + +SELECT * FROM select_test WHERE key = 2; +WARNING: server closed the connection unexpectedly + This probably means the server terminated abnormally + before or while processing the request. +CONTEXT: while executing command on localhost:9060 + key | value +-----+----------- + 2 | test data +(1 row) + +-- kill after first SELECT; txn should work (though placement marked bad) +SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").kill()'); + mitmproxy +----------- + +(1 row) + +BEGIN; +INSERT INTO select_test VALUES (2, 'more data'); +SELECT * FROM select_test WHERE key = 2; +WARNING: server closed the connection unexpectedly + This probably means the server terminated abnormally + before or while processing the request. +CONTEXT: while executing command on localhost:9060 + key | value +-----+----------- + 2 | test data + 2 | more data +(2 rows) + +INSERT INTO select_test VALUES (2, 'even more data'); +SELECT * FROM select_test WHERE key = 2; + key | value +-----+---------------- + 2 | test data + 2 | more data + 2 | even more data +(3 rows) + +COMMIT; +WARNING: connection not open +CONTEXT: while executing command on localhost:9060 +WARNING: connection not open +CONTEXT: while executing command on localhost:9060 +WARNING: connection not open +CONTEXT: while executing command on localhost:9060 +-- some clean up +UPDATE pg_dist_shard_placement SET shardstate = 1 +WHERE shardid IN ( + SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'select_test'::regclass +); +TRUNCATE select_test; +-- error after second SELECT; txn should work (though placement marked bad) +SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").after(1).reset()'); + mitmproxy +----------- + +(1 row) + +BEGIN; +INSERT INTO select_test VALUES (2, 'more data'); +SELECT * FROM select_test WHERE key = 2; + key | value +-----+----------- + 2 | more data +(1 row) + +INSERT INTO select_test VALUES (2, 'even more data'); +SELECT * FROM select_test WHERE key = 2; +WARNING: server closed the connection unexpectedly + This probably means the server terminated abnormally + before or while processing the request. +CONTEXT: while executing command on localhost:9060 + key | value +-----+---------------- + 2 | more data + 2 | even more data +(2 rows) + +COMMIT; +WARNING: connection not open +CONTEXT: while executing command on localhost:9060 +WARNING: connection not open +CONTEXT: while executing command on localhost:9060 +WARNING: connection not open +CONTEXT: while executing command on localhost:9060 +SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").after(2).kill()'); + mitmproxy +----------- + +(1 row) + +SELECT recover_prepared_transactions(); + recover_prepared_transactions +------------------------------- + 0 +(1 row) + +SELECT recover_prepared_transactions(); +ERROR: server closed the connection unexpectedly + This probably means the server terminated abnormally + before or while processing the request. +CONTEXT: while executing command on localhost:9060 +-- bug from https://github.com/citusdata/citus/issues/1926 +DROP TABLE select_test; +SET citus.shard_count = 2; +SET citus.shard_replication_factor = 1; +CREATE TABLE select_test (key int, value text); +SELECT create_distributed_table('select_test', 'key'); + create_distributed_table +-------------------------- + +(1 row) + +INSERT INTO select_test VALUES (1, 'test data'); +SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").after(1).kill()'); + mitmproxy +----------- + +(1 row) + +SELECT * FROM select_test WHERE key = 1; + key | value +-----+----------- + 1 | test data +(1 row) + +SELECT * FROM select_test WHERE key = 1; +WARNING: server closed the connection unexpectedly + This probably means the server terminated abnormally + before or while processing the request. +CONTEXT: while executing command on localhost:9060 +ERROR: could not receive query results +-- ==== Clean up, we're done here ==== +DROP TABLE select_test; diff --git a/src/test/regress/failure_schedule b/src/test/regress/failure_schedule index 3edbcd334..b5312ced1 100644 --- a/src/test/regress/failure_schedule +++ b/src/test/regress/failure_schedule @@ -21,4 +21,5 @@ test: failure_multi_shard_update_delete test: failure_cte_subquery test: failure_insert_select_via_coordinator test: failure_multi_dml -test: failure_vacuum \ No newline at end of file +test: failure_vacuum +test: failure_single_select diff --git a/src/test/regress/sql/failure_single_select.sql b/src/test/regress/sql/failure_single_select.sql new file mode 100644 index 000000000..ebddb038e --- /dev/null +++ b/src/test/regress/sql/failure_single_select.sql @@ -0,0 +1,63 @@ +SELECT citus.mitmproxy('conn.allow()'); +SELECT citus.clear_network_traffic(); + +SET citus.shard_count = 2; +SET citus.shard_replication_factor = 2; + +CREATE TABLE select_test (key int, value text); +SELECT create_distributed_table('select_test', 'key'); + +-- put data in shard for which mitm node is first placement +INSERT INTO select_test VALUES (2, 'test data'); + +SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").kill()'); +SELECT * FROM select_test WHERE key = 2; +SELECT * FROM select_test WHERE key = 2; + +-- kill after first SELECT; txn should work (though placement marked bad) +SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").kill()'); + +BEGIN; +INSERT INTO select_test VALUES (2, 'more data'); +SELECT * FROM select_test WHERE key = 2; +INSERT INTO select_test VALUES (2, 'even more data'); +SELECT * FROM select_test WHERE key = 2; +COMMIT; + +-- some clean up +UPDATE pg_dist_shard_placement SET shardstate = 1 +WHERE shardid IN ( + SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'select_test'::regclass +); +TRUNCATE select_test; + +-- error after second SELECT; txn should work (though placement marked bad) +SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").after(1).reset()'); + +BEGIN; +INSERT INTO select_test VALUES (2, 'more data'); +SELECT * FROM select_test WHERE key = 2; +INSERT INTO select_test VALUES (2, 'even more data'); +SELECT * FROM select_test WHERE key = 2; +COMMIT; + +SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").after(2).kill()'); +SELECT recover_prepared_transactions(); +SELECT recover_prepared_transactions(); + +-- bug from https://github.com/citusdata/citus/issues/1926 +DROP TABLE select_test; +SET citus.shard_count = 2; +SET citus.shard_replication_factor = 1; + +CREATE TABLE select_test (key int, value text); +SELECT create_distributed_table('select_test', 'key'); +INSERT INTO select_test VALUES (1, 'test data'); + +SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").after(1).kill()'); +SELECT * FROM select_test WHERE key = 1; +SELECT * FROM select_test WHERE key = 1; + +-- ==== Clean up, we're done here ==== + +DROP TABLE select_test;