-- -- failure_multi_row_insert -- CREATE SCHEMA IF NOT EXISTS failure_multi_row_insert; SET SEARCH_PATH TO failure_multi_row_insert; -- this test is dependent on the shard count, so do not change -- whitout changing the test SET citus.shard_count TO 4; SET citus.next_shard_id TO 301000; SET citus.shard_replication_factor TO 1; SELECT pg_backend_pid() as pid \gset SELECT citus.mitmproxy('conn.allow()'); mitmproxy --------------------------------------------------------------------- (1 row) CREATE TABLE distributed_table(key int, value int); CREATE TABLE reference_table(value int); SELECT create_distributed_table('distributed_table', 'key'); create_distributed_table --------------------------------------------------------------------- (1 row) SELECT create_reference_table('reference_table'); create_reference_table --------------------------------------------------------------------- (1 row) -- we'll test failure cases of the following cases: -- (a) multi-row INSERT that hits the same shard with the same value -- (b) multi-row INSERT that hits the same shard with different values -- (c) multi-row INSERT that hits multiple shards in a single worker -- (d) multi-row INSERT that hits multiple shards in multiple workers -- (e) multi-row INSERT to a reference table -- Failure and cancellation on multi-row INSERT that hits the same shard with the same value SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()'); mitmproxy --------------------------------------------------------------------- (1 row) INSERT INTO distributed_table VALUES (1,1), (1,2), (1,3); ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. -- this test is broken, see https://github.com/citusdata/citus/issues/2460 -- SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").cancel(' || :pid || ')'); -- INSERT INTO distributed_table VALUES (1,4), (1,5), (1,6); -- Failure and cancellation on multi-row INSERT that hits the same shard with different values SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()'); mitmproxy --------------------------------------------------------------------- (1 row) INSERT INTO distributed_table VALUES (1,7), (5,8); ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. -- this test is broken, see https://github.com/citusdata/citus/issues/2460 -- SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").cancel(' || :pid || ')'); -- INSERT INTO distributed_table VALUES (1,9), (5,10); -- Failure and cancellation multi-row INSERT that hits multiple shards in a single worker SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()'); mitmproxy --------------------------------------------------------------------- (1 row) INSERT INTO distributed_table VALUES (1,11), (6,12); ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").cancel(' || :pid || ')'); mitmproxy --------------------------------------------------------------------- (1 row) INSERT INTO distributed_table VALUES (1,13), (6,14); ERROR: canceling statement due to user request -- Failure and cancellation multi-row INSERT that hits multiple shards in a single worker, happening on the second query SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").after(1).kill()'); mitmproxy --------------------------------------------------------------------- (1 row) INSERT INTO distributed_table VALUES (1,15), (6,16); ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").after(1).cancel(' || :pid || ')'); mitmproxy --------------------------------------------------------------------- (1 row) INSERT INTO distributed_table VALUES (1,17), (6,18); ERROR: canceling statement due to user request -- Failure and cancellation multi-row INSERT that hits multiple shards in multiple workers SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()'); mitmproxy --------------------------------------------------------------------- (1 row) INSERT INTO distributed_table VALUES (2,19),(1,20); ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").cancel(' || :pid || ')'); mitmproxy --------------------------------------------------------------------- (1 row) INSERT INTO distributed_table VALUES (2,21), (1,22); ERROR: canceling statement due to user request -- one test for the reference tables for completeness SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").cancel(' || :pid || ')'); mitmproxy --------------------------------------------------------------------- (1 row) INSERT INTO reference_table VALUES (1), (2), (3), (4); ERROR: canceling statement due to user request SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").cancel(' || :pid || ')'); mitmproxy --------------------------------------------------------------------- (1 row) INSERT INTO distributed_table VALUES (1,1), (2,2), (3,3), (4,2), (5,2), (6,2), (7,2); ERROR: canceling statement due to user request -- cancel the second insert over the same connection SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").after(1).cancel(' || :pid || ')'); mitmproxy --------------------------------------------------------------------- (1 row) INSERT INTO distributed_table VALUES (1,1), (2,2), (3,3), (4,2), (5,2), (6,2), (7,2); ERROR: canceling statement due to user request -- we've either failed or cancelled all queries, so should be empty SELECT * FROM distributed_table; key | value --------------------------------------------------------------------- (0 rows) SELECT * FROM reference_table; value --------------------------------------------------------------------- (0 rows) SELECT citus.mitmproxy('conn.allow()'); mitmproxy --------------------------------------------------------------------- (1 row) RESET SEARCH_PATH; DROP SCHEMA failure_multi_row_insert CASCADE; NOTICE: drop cascades to 2 other objects DETAIL: drop cascades to table failure_multi_row_insert.distributed_table drop cascades to table failure_multi_row_insert.reference_table