external
/
citus
mirror of https://github.com/citusdata/citus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Failure tests for modifying multiple shards in txn 

Tests various failure points during a multi-shard modification within
a transaction with multiple statements. Verifies three cases:

  * Reference tables (single shard, many placements)
  * Normal table with replication factor two
  * Multi-shard table with no replication

In the replication-factor case, we expect shard health to be affected
in some transactions; most others fail the transaction entirely and
all we need verify is that no effects of the transaction are visible.

Had trouble testing the final PREPARE/COMMIT/ROLLBACK phase of the 2pc,
in particular because the error message produced includes the PID of
the backend, which is unpredictable.
pull/2421/head
Jason Petersen 2018-09-27 01:19:53 -06:00 committed by Jason Petersen
parent b45754a4d0
commit ee4114bc7a
3 changed files with 484 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

304
src/test/regress/expected/failure_multi_dml.out Normal file
View File

@ -0,0 +1,304 @@
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
-----------
(1 row)
SET citus.shard_count = 2;
SET citus.shard_replication_factor = 1; -- one shard per worker
SET citus.next_shard_id TO 103400;
ALTER SEQUENCE pg_catalog.pg_dist_placement_placementid_seq RESTART 100;
CREATE TABLE dml_test (id integer, name text);
SELECT create_distributed_table('dml_test', 'id');
create_distributed_table
--------------------------
(1 row)
COPY dml_test FROM STDIN WITH CSV;
SELECT citus.clear_network_traffic();
clear_network_traffic
-----------------------
(1 row)
---- test multiple statements spanning multiple shards,
---- at each significant point. These transactions are 2pc
-- fail at DELETE
SELECT citus.mitmproxy('conn.onQuery(query="^DELETE").kill()');
mitmproxy
-----------
(1 row)
BEGIN;
DELETE FROM dml_test WHERE id = 1;
ERROR: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
CONTEXT: while executing command on localhost:9060
DELETE FROM dml_test WHERE id = 2;
ERROR: current transaction is aborted, commands ignored until end of transaction block
INSERT INTO dml_test VALUES (5, 'Epsilon');
ERROR: current transaction is aborted, commands ignored until end of transaction block
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
ERROR: current transaction is aborted, commands ignored until end of transaction block
UPDATE dml_test SET name = 'gamma' WHERE id = 3;
ERROR: current transaction is aborted, commands ignored until end of transaction block
COMMIT;
--- shouldn't see any changes performed in failed transaction
SELECT * FROM dml_test ORDER BY id ASC;
id | name
----+-------
1 | Alpha
2 | Beta
3 | Gamma
4 | Delta
(4 rows)
-- fail at INSERT
SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()');
mitmproxy
-----------
(1 row)
BEGIN;
DELETE FROM dml_test WHERE id = 1;
DELETE FROM dml_test WHERE id = 2;
INSERT INTO dml_test VALUES (5, 'Epsilon');
ERROR: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
CONTEXT: while executing command on localhost:9060
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
ERROR: current transaction is aborted, commands ignored until end of transaction block
UPDATE dml_test SET name = 'gamma' WHERE id = 3;
ERROR: current transaction is aborted, commands ignored until end of transaction block
COMMIT;
--- shouldn't see any changes before failed INSERT
SELECT * FROM dml_test ORDER BY id ASC;
id | name
----+-------
1 | Alpha
2 | Beta
3 | Gamma
4 | Delta
(4 rows)
-- fail at UPDATE
SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()');
mitmproxy
-----------
(1 row)
BEGIN;
DELETE FROM dml_test WHERE id = 1;
DELETE FROM dml_test WHERE id = 2;
INSERT INTO dml_test VALUES (5, 'Epsilon');
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
ERROR: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
CONTEXT: while executing command on localhost:9060
UPDATE dml_test SET name = 'gamma' WHERE id = 3;
ERROR: current transaction is aborted, commands ignored until end of transaction block
COMMIT;
--- shouldn't see any changes after failed UPDATE
SELECT * FROM dml_test ORDER BY id ASC;
id | name
----+-------
1 | Alpha
2 | Beta
3 | Gamma
4 | Delta
(4 rows)
-- fail at PREPARE TRANSACTION
SELECT citus.mitmproxy('conn.onQuery(query="^PREPARE TRANSACTION").kill()');
mitmproxy
-----------
(1 row)
-- hide the error message (it has the PID)...
-- we'll test for the txn side-effects to ensure it didn't run
SET client_min_messages TO FATAL;
BEGIN;
DELETE FROM dml_test WHERE id = 1;
DELETE FROM dml_test WHERE id = 2;
INSERT INTO dml_test VALUES (5, 'Epsilon');
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
UPDATE dml_test SET name = 'gamma' WHERE id = 3;
COMMIT;
SET client_min_messages TO DEFAULT;
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
-----------
(1 row)
SELECT shardid FROM pg_dist_shard_placement WHERE shardstate = 3;
shardid
---------
(0 rows)
SELECT recover_prepared_transactions();
recover_prepared_transactions
-------------------------------
0
(1 row)
-- shouldn't see any changes after failed PREPARE
SELECT * FROM dml_test ORDER BY id ASC;
id | name
----+-------
1 | Alpha
2 | Beta
3 | Gamma
4 | Delta
(4 rows)
-- fail at COMMIT
SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT").kill()');
mitmproxy
-----------
(1 row)
-- hide the error message (it has the PID)...
-- we'll test for the txn side-effects to ensure it didn't run
SET client_min_messages TO FATAL;
BEGIN;
DELETE FROM dml_test WHERE id = 1;
DELETE FROM dml_test WHERE id = 2;
INSERT INTO dml_test VALUES (5, 'Epsilon');
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
UPDATE dml_test SET name = 'gamma' WHERE id = 3;
COMMIT;
SET client_min_messages TO DEFAULT;
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
-----------
(1 row)
SELECT shardid FROM pg_dist_shard_placement WHERE shardstate = 3;
shardid
---------
(0 rows)
SELECT recover_prepared_transactions();
recover_prepared_transactions
-------------------------------
1
(1 row)
-- should see changes, because of txn recovery
SELECT * FROM dml_test ORDER BY id ASC;
id | name
----+---------
3 | gamma
4 | Delta
5 | Epsilon
(3 rows)
-- drop table and recreate with different replication/sharding
DROP TABLE dml_test;
SET citus.shard_count = 1;
SET citus.shard_replication_factor = 2; -- two placements
CREATE TABLE dml_test (id integer, name text);
SELECT create_distributed_table('dml_test', 'id');
create_distributed_table
--------------------------
(1 row)
COPY dml_test FROM STDIN WITH CSV;
---- test multiple statements against a single shard, but with two placements
-- fail at COMMIT (actually COMMIT this time, as no 2pc in use)
SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT").kill()');
mitmproxy
-----------
(1 row)
BEGIN;
DELETE FROM dml_test WHERE id = 1;
DELETE FROM dml_test WHERE id = 2;
INSERT INTO dml_test VALUES (5, 'Epsilon');
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
UPDATE dml_test SET name = 'gamma' WHERE id = 3;
COMMIT;
WARNING: connection not open
CONTEXT: while executing command on localhost:9060
WARNING: failed to commit transaction on localhost:9060
WARNING: connection not open
CONTEXT: while executing command on localhost:9060
--- should see all changes, but they only went to one placement (other is unhealthy)
SELECT * FROM dml_test ORDER BY id ASC;
id | name
----+---------
3 | gamma
4 | Delta
5 | Epsilon
(3 rows)
SELECT shardid FROM pg_dist_shard_placement WHERE shardstate = 3;
shardid
---------
103402
(1 row)
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
-----------
(1 row)
-- drop table and recreate as reference table
DROP TABLE dml_test;
SET citus.shard_count = 2;
SET citus.shard_replication_factor = 1;
CREATE TABLE dml_test (id integer, name text);
SELECT create_reference_table('dml_test');
create_reference_table
------------------------
(1 row)
COPY dml_test FROM STDIN WITH CSV;
-- fail at COMMIT (by failing to PREPARE)
SELECT citus.mitmproxy('conn.onQuery(query="^PREPARE").kill()');
mitmproxy
-----------
(1 row)
BEGIN;
DELETE FROM dml_test WHERE id = 1;
DELETE FROM dml_test WHERE id = 2;
INSERT INTO dml_test VALUES (5, 'Epsilon');
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
UPDATE dml_test SET name = 'gamma' WHERE id = 3;
COMMIT;
ERROR: connection not open
CONTEXT: while executing command on localhost:9060
--- shouldn't see any changes after failed COMMIT
SELECT * FROM dml_test ORDER BY id ASC;
id | name
----+-------
1 | Alpha
2 | Beta
3 | Gamma
4 | Delta
(4 rows)
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
-----------
(1 row)
DROP TABLE dml_test;

1
src/test/regress/failure_schedule
View File

@ -20,3 +20,4 @@ test: failure_1pc_copy_append
test: failure_multi_shard_update_delete test: failure_multi_shard_update_delete
test: failure_cte_subquery test: failure_cte_subquery
test: failure_insert_select_via_coordinator test: failure_insert_select_via_coordinator
test: failure_multi_dml

179
src/test/regress/sql/failure_multi_dml.sql Normal file
View File

@ -0,0 +1,179 @@
SELECT citus.mitmproxy('conn.allow()');
SET citus.shard_count = 2;
SET citus.shard_replication_factor = 1; -- one shard per worker
SET citus.next_shard_id TO 103400;
ALTER SEQUENCE pg_catalog.pg_dist_placement_placementid_seq RESTART 100;
CREATE TABLE dml_test (id integer, name text);
SELECT create_distributed_table('dml_test', 'id');
COPY dml_test FROM STDIN WITH CSV;
1,Alpha
2,Beta
3,Gamma
4,Delta
\.
SELECT citus.clear_network_traffic();
---- test multiple statements spanning multiple shards,
---- at each significant point. These transactions are 2pc
-- fail at DELETE
SELECT citus.mitmproxy('conn.onQuery(query="^DELETE").kill()');
BEGIN;
DELETE FROM dml_test WHERE id = 1;
DELETE FROM dml_test WHERE id = 2;
INSERT INTO dml_test VALUES (5, 'Epsilon');
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
UPDATE dml_test SET name = 'gamma' WHERE id = 3;
COMMIT;
--- shouldn't see any changes performed in failed transaction
SELECT * FROM dml_test ORDER BY id ASC;
-- fail at INSERT
SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()');
BEGIN;
DELETE FROM dml_test WHERE id = 1;
DELETE FROM dml_test WHERE id = 2;
INSERT INTO dml_test VALUES (5, 'Epsilon');
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
UPDATE dml_test SET name = 'gamma' WHERE id = 3;
COMMIT;
--- shouldn't see any changes before failed INSERT
SELECT * FROM dml_test ORDER BY id ASC;
-- fail at UPDATE
SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()');
BEGIN;
DELETE FROM dml_test WHERE id = 1;
DELETE FROM dml_test WHERE id = 2;
INSERT INTO dml_test VALUES (5, 'Epsilon');
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
UPDATE dml_test SET name = 'gamma' WHERE id = 3;
COMMIT;
--- shouldn't see any changes after failed UPDATE
SELECT * FROM dml_test ORDER BY id ASC;
-- fail at PREPARE TRANSACTION
SELECT citus.mitmproxy('conn.onQuery(query="^PREPARE TRANSACTION").kill()');
-- hide the error message (it has the PID)...
-- we'll test for the txn side-effects to ensure it didn't run
SET client_min_messages TO FATAL;
BEGIN;
DELETE FROM dml_test WHERE id = 1;
DELETE FROM dml_test WHERE id = 2;
INSERT INTO dml_test VALUES (5, 'Epsilon');
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
UPDATE dml_test SET name = 'gamma' WHERE id = 3;
COMMIT;
SET client_min_messages TO DEFAULT;
SELECT citus.mitmproxy('conn.allow()');
SELECT shardid FROM pg_dist_shard_placement WHERE shardstate = 3;
SELECT recover_prepared_transactions();
-- shouldn't see any changes after failed PREPARE
SELECT * FROM dml_test ORDER BY id ASC;
-- fail at COMMIT
SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT").kill()');
-- hide the error message (it has the PID)...
-- we'll test for the txn side-effects to ensure it didn't run
SET client_min_messages TO FATAL;
BEGIN;
DELETE FROM dml_test WHERE id = 1;
DELETE FROM dml_test WHERE id = 2;
INSERT INTO dml_test VALUES (5, 'Epsilon');
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
UPDATE dml_test SET name = 'gamma' WHERE id = 3;
COMMIT;
SET client_min_messages TO DEFAULT;
SELECT citus.mitmproxy('conn.allow()');
SELECT shardid FROM pg_dist_shard_placement WHERE shardstate = 3;
SELECT recover_prepared_transactions();
-- should see changes, because of txn recovery
SELECT * FROM dml_test ORDER BY id ASC;
-- drop table and recreate with different replication/sharding
DROP TABLE dml_test;
SET citus.shard_count = 1;
SET citus.shard_replication_factor = 2; -- two placements
CREATE TABLE dml_test (id integer, name text);
SELECT create_distributed_table('dml_test', 'id');
COPY dml_test FROM STDIN WITH CSV;
1,Alpha
2,Beta
3,Gamma
4,Delta
\.
---- test multiple statements against a single shard, but with two placements
-- fail at COMMIT (actually COMMIT this time, as no 2pc in use)
SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT").kill()');
BEGIN;
DELETE FROM dml_test WHERE id = 1;
DELETE FROM dml_test WHERE id = 2;
INSERT INTO dml_test VALUES (5, 'Epsilon');
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
UPDATE dml_test SET name = 'gamma' WHERE id = 3;
COMMIT;
--- should see all changes, but they only went to one placement (other is unhealthy)
SELECT * FROM dml_test ORDER BY id ASC;
SELECT shardid FROM pg_dist_shard_placement WHERE shardstate = 3;
SELECT citus.mitmproxy('conn.allow()');
-- drop table and recreate as reference table
DROP TABLE dml_test;
SET citus.shard_count = 2;
SET citus.shard_replication_factor = 1;
CREATE TABLE dml_test (id integer, name text);
SELECT create_reference_table('dml_test');
COPY dml_test FROM STDIN WITH CSV;
1,Alpha
2,Beta
3,Gamma
4,Delta
\.
-- fail at COMMIT (by failing to PREPARE)
SELECT citus.mitmproxy('conn.onQuery(query="^PREPARE").kill()');
BEGIN;
DELETE FROM dml_test WHERE id = 1;
DELETE FROM dml_test WHERE id = 2;
INSERT INTO dml_test VALUES (5, 'Epsilon');
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
UPDATE dml_test SET name = 'gamma' WHERE id = 3;
COMMIT;
--- shouldn't see any changes after failed COMMIT
SELECT * FROM dml_test ORDER BY id ASC;
SELECT citus.mitmproxy('conn.allow()');
DROP TABLE dml_test;