mirror of https://github.com/citusdata/citus.git
Failure tests for modifying multiple shards in txn
Tests various failure points during a multi-shard modification within a transaction with multiple statements. Verifies three cases: * Reference tables (single shard, many placements) * Normal table with replication factor two * Multi-shard table with no replication In the replication-factor case, we expect shard health to be affected in some transactions; most others fail the transaction entirely and all we need verify is that no effects of the transaction are visible. Had trouble testing the final PREPARE/COMMIT/ROLLBACK phase of the 2pc, in particular because the error message produced includes the PID of the backend, which is unpredictable.pull/2421/head
Jason Petersen
Jason Petersen
parent
b45754a4d0
commit
ee4114bc7a
|
|
@ -0,0 +1,304 @@
|
|||
SELECT citus.mitmproxy('conn.allow()');
|
||||
mitmproxy
|
||||
-----------
|
||||
|
||||
(1 row)
|
||||
|
||||
SET citus.shard_count = 2;
|
||||
SET citus.shard_replication_factor = 1; -- one shard per worker
|
||||
SET citus.next_shard_id TO 103400;
|
||||
ALTER SEQUENCE pg_catalog.pg_dist_placement_placementid_seq RESTART 100;
|
||||
CREATE TABLE dml_test (id integer, name text);
|
||||
SELECT create_distributed_table('dml_test', 'id');
|
||||
create_distributed_table
|
||||
--------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
COPY dml_test FROM STDIN WITH CSV;
|
||||
SELECT citus.clear_network_traffic();
|
||||
clear_network_traffic
|
||||
-----------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
---- test multiple statements spanning multiple shards,
|
||||
---- at each significant point. These transactions are 2pc
|
||||
-- fail at DELETE
|
||||
SELECT citus.mitmproxy('conn.onQuery(query="^DELETE").kill()');
|
||||
mitmproxy
|
||||
-----------
|
||||
|
||||
(1 row)
|
||||
|
||||
BEGIN;
|
||||
DELETE FROM dml_test WHERE id = 1;
|
||||
ERROR: server closed the connection unexpectedly
|
||||
This probably means the server terminated abnormally
|
||||
before or while processing the request.
|
||||
CONTEXT: while executing command on localhost:9060
|
||||
DELETE FROM dml_test WHERE id = 2;
|
||||
ERROR: current transaction is aborted, commands ignored until end of transaction block
|
||||
INSERT INTO dml_test VALUES (5, 'Epsilon');
|
||||
ERROR: current transaction is aborted, commands ignored until end of transaction block
|
||||
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
|
||||
ERROR: current transaction is aborted, commands ignored until end of transaction block
|
||||
UPDATE dml_test SET name = 'gamma' WHERE id = 3;
|
||||
ERROR: current transaction is aborted, commands ignored until end of transaction block
|
||||
COMMIT;
|
||||
--- shouldn't see any changes performed in failed transaction
|
||||
SELECT * FROM dml_test ORDER BY id ASC;
|
||||
id | name
|
||||
----+-------
|
||||
1 | Alpha
|
||||
2 | Beta
|
||||
3 | Gamma
|
||||
4 | Delta
|
||||
(4 rows)
|
||||
|
||||
-- fail at INSERT
|
||||
SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()');
|
||||
mitmproxy
|
||||
-----------
|
||||
|
||||
(1 row)
|
||||
|
||||
BEGIN;
|
||||
DELETE FROM dml_test WHERE id = 1;
|
||||
DELETE FROM dml_test WHERE id = 2;
|
||||
INSERT INTO dml_test VALUES (5, 'Epsilon');
|
||||
ERROR: server closed the connection unexpectedly
|
||||
This probably means the server terminated abnormally
|
||||
before or while processing the request.
|
||||
CONTEXT: while executing command on localhost:9060
|
||||
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
|
||||
ERROR: current transaction is aborted, commands ignored until end of transaction block
|
||||
UPDATE dml_test SET name = 'gamma' WHERE id = 3;
|
||||
ERROR: current transaction is aborted, commands ignored until end of transaction block
|
||||
COMMIT;
|
||||
--- shouldn't see any changes before failed INSERT
|
||||
SELECT * FROM dml_test ORDER BY id ASC;
|
||||
id | name
|
||||
----+-------
|
||||
1 | Alpha
|
||||
2 | Beta
|
||||
3 | Gamma
|
||||
4 | Delta
|
||||
(4 rows)
|
||||
|
||||
-- fail at UPDATE
|
||||
SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()');
|
||||
mitmproxy
|
||||
-----------
|
||||
|
||||
(1 row)
|
||||
|
||||
BEGIN;
|
||||
DELETE FROM dml_test WHERE id = 1;
|
||||
DELETE FROM dml_test WHERE id = 2;
|
||||
INSERT INTO dml_test VALUES (5, 'Epsilon');
|
||||
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
|
||||
ERROR: server closed the connection unexpectedly
|
||||
This probably means the server terminated abnormally
|
||||
before or while processing the request.
|
||||
CONTEXT: while executing command on localhost:9060
|
||||
UPDATE dml_test SET name = 'gamma' WHERE id = 3;
|
||||
ERROR: current transaction is aborted, commands ignored until end of transaction block
|
||||
COMMIT;
|
||||
--- shouldn't see any changes after failed UPDATE
|
||||
SELECT * FROM dml_test ORDER BY id ASC;
|
||||
id | name
|
||||
----+-------
|
||||
1 | Alpha
|
||||
2 | Beta
|
||||
3 | Gamma
|
||||
4 | Delta
|
||||
(4 rows)
|
||||
|
||||
-- fail at PREPARE TRANSACTION
|
||||
SELECT citus.mitmproxy('conn.onQuery(query="^PREPARE TRANSACTION").kill()');
|
||||
mitmproxy
|
||||
-----------
|
||||
|
||||
(1 row)
|
||||
|
||||
-- hide the error message (it has the PID)...
|
||||
-- we'll test for the txn side-effects to ensure it didn't run
|
||||
SET client_min_messages TO FATAL;
|
||||
BEGIN;
|
||||
DELETE FROM dml_test WHERE id = 1;
|
||||
DELETE FROM dml_test WHERE id = 2;
|
||||
INSERT INTO dml_test VALUES (5, 'Epsilon');
|
||||
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
|
||||
UPDATE dml_test SET name = 'gamma' WHERE id = 3;
|
||||
COMMIT;
|
||||
SET client_min_messages TO DEFAULT;
|
||||
SELECT citus.mitmproxy('conn.allow()');
|
||||
mitmproxy
|
||||
-----------
|
||||
|
||||
(1 row)
|
||||
|
||||
SELECT shardid FROM pg_dist_shard_placement WHERE shardstate = 3;
|
||||
shardid
|
||||
---------
|
||||
(0 rows)
|
||||
|
||||
SELECT recover_prepared_transactions();
|
||||
recover_prepared_transactions
|
||||
-------------------------------
|
||||
0
|
||||
(1 row)
|
||||
|
||||
-- shouldn't see any changes after failed PREPARE
|
||||
SELECT * FROM dml_test ORDER BY id ASC;
|
||||
id | name
|
||||
----+-------
|
||||
1 | Alpha
|
||||
2 | Beta
|
||||
3 | Gamma
|
||||
4 | Delta
|
||||
(4 rows)
|
||||
|
||||
-- fail at COMMIT
|
||||
SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT").kill()');
|
||||
mitmproxy
|
||||
-----------
|
||||
|
||||
(1 row)
|
||||
|
||||
-- hide the error message (it has the PID)...
|
||||
-- we'll test for the txn side-effects to ensure it didn't run
|
||||
SET client_min_messages TO FATAL;
|
||||
BEGIN;
|
||||
DELETE FROM dml_test WHERE id = 1;
|
||||
DELETE FROM dml_test WHERE id = 2;
|
||||
INSERT INTO dml_test VALUES (5, 'Epsilon');
|
||||
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
|
||||
UPDATE dml_test SET name = 'gamma' WHERE id = 3;
|
||||
COMMIT;
|
||||
SET client_min_messages TO DEFAULT;
|
||||
SELECT citus.mitmproxy('conn.allow()');
|
||||
mitmproxy
|
||||
-----------
|
||||
|
||||
(1 row)
|
||||
|
||||
SELECT shardid FROM pg_dist_shard_placement WHERE shardstate = 3;
|
||||
shardid
|
||||
---------
|
||||
(0 rows)
|
||||
|
||||
SELECT recover_prepared_transactions();
|
||||
recover_prepared_transactions
|
||||
-------------------------------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
-- should see changes, because of txn recovery
|
||||
SELECT * FROM dml_test ORDER BY id ASC;
|
||||
id | name
|
||||
----+---------
|
||||
3 | gamma
|
||||
4 | Delta
|
||||
5 | Epsilon
|
||||
(3 rows)
|
||||
|
||||
-- drop table and recreate with different replication/sharding
|
||||
DROP TABLE dml_test;
|
||||
SET citus.shard_count = 1;
|
||||
SET citus.shard_replication_factor = 2; -- two placements
|
||||
CREATE TABLE dml_test (id integer, name text);
|
||||
SELECT create_distributed_table('dml_test', 'id');
|
||||
create_distributed_table
|
||||
--------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
COPY dml_test FROM STDIN WITH CSV;
|
||||
---- test multiple statements against a single shard, but with two placements
|
||||
-- fail at COMMIT (actually COMMIT this time, as no 2pc in use)
|
||||
SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT").kill()');
|
||||
mitmproxy
|
||||
-----------
|
||||
|
||||
(1 row)
|
||||
|
||||
BEGIN;
|
||||
DELETE FROM dml_test WHERE id = 1;
|
||||
DELETE FROM dml_test WHERE id = 2;
|
||||
INSERT INTO dml_test VALUES (5, 'Epsilon');
|
||||
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
|
||||
UPDATE dml_test SET name = 'gamma' WHERE id = 3;
|
||||
COMMIT;
|
||||
WARNING: connection not open
|
||||
CONTEXT: while executing command on localhost:9060
|
||||
WARNING: failed to commit transaction on localhost:9060
|
||||
WARNING: connection not open
|
||||
CONTEXT: while executing command on localhost:9060
|
||||
--- should see all changes, but they only went to one placement (other is unhealthy)
|
||||
SELECT * FROM dml_test ORDER BY id ASC;
|
||||
id | name
|
||||
----+---------
|
||||
3 | gamma
|
||||
4 | Delta
|
||||
5 | Epsilon
|
||||
(3 rows)
|
||||
|
||||
SELECT shardid FROM pg_dist_shard_placement WHERE shardstate = 3;
|
||||
shardid
|
||||
---------
|
||||
103402
|
||||
(1 row)
|
||||
|
||||
SELECT citus.mitmproxy('conn.allow()');
|
||||
mitmproxy
|
||||
-----------
|
||||
|
||||
(1 row)
|
||||
|
||||
-- drop table and recreate as reference table
|
||||
DROP TABLE dml_test;
|
||||
SET citus.shard_count = 2;
|
||||
SET citus.shard_replication_factor = 1;
|
||||
CREATE TABLE dml_test (id integer, name text);
|
||||
SELECT create_reference_table('dml_test');
|
||||
create_reference_table
|
||||
------------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
COPY dml_test FROM STDIN WITH CSV;
|
||||
-- fail at COMMIT (by failing to PREPARE)
|
||||
SELECT citus.mitmproxy('conn.onQuery(query="^PREPARE").kill()');
|
||||
mitmproxy
|
||||
-----------
|
||||
|
||||
(1 row)
|
||||
|
||||
BEGIN;
|
||||
DELETE FROM dml_test WHERE id = 1;
|
||||
DELETE FROM dml_test WHERE id = 2;
|
||||
INSERT INTO dml_test VALUES (5, 'Epsilon');
|
||||
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
|
||||
UPDATE dml_test SET name = 'gamma' WHERE id = 3;
|
||||
COMMIT;
|
||||
ERROR: connection not open
|
||||
CONTEXT: while executing command on localhost:9060
|
||||
--- shouldn't see any changes after failed COMMIT
|
||||
SELECT * FROM dml_test ORDER BY id ASC;
|
||||
id | name
|
||||
----+-------
|
||||
1 | Alpha
|
||||
2 | Beta
|
||||
3 | Gamma
|
||||
4 | Delta
|
||||
(4 rows)
|
||||
|
||||
SELECT citus.mitmproxy('conn.allow()');
|
||||
mitmproxy
|
||||
-----------
|
||||
|
||||
(1 row)
|
||||
|
||||
DROP TABLE dml_test;
|
||||
|
|
@ -20,3 +20,4 @@ test: failure_1pc_copy_append
|
|||
test: failure_multi_shard_update_delete
|
||||
test: failure_cte_subquery
|
||||
test: failure_insert_select_via_coordinator
|
||||
test: failure_multi_dml
|
||||
|
|
|
|||
|
|
@ -0,0 +1,179 @@
|
|||
SELECT citus.mitmproxy('conn.allow()');
|
||||
|
||||
SET citus.shard_count = 2;
|
||||
SET citus.shard_replication_factor = 1; -- one shard per worker
|
||||
SET citus.next_shard_id TO 103400;
|
||||
ALTER SEQUENCE pg_catalog.pg_dist_placement_placementid_seq RESTART 100;
|
||||
|
||||
CREATE TABLE dml_test (id integer, name text);
|
||||
SELECT create_distributed_table('dml_test', 'id');
|
||||
|
||||
COPY dml_test FROM STDIN WITH CSV;
|
||||
1,Alpha
|
||||
2,Beta
|
||||
3,Gamma
|
||||
4,Delta
|
||||
\.
|
||||
|
||||
SELECT citus.clear_network_traffic();
|
||||
|
||||
---- test multiple statements spanning multiple shards,
|
||||
---- at each significant point. These transactions are 2pc
|
||||
|
||||
-- fail at DELETE
|
||||
SELECT citus.mitmproxy('conn.onQuery(query="^DELETE").kill()');
|
||||
|
||||
BEGIN;
|
||||
DELETE FROM dml_test WHERE id = 1;
|
||||
DELETE FROM dml_test WHERE id = 2;
|
||||
INSERT INTO dml_test VALUES (5, 'Epsilon');
|
||||
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
|
||||
UPDATE dml_test SET name = 'gamma' WHERE id = 3;
|
||||
COMMIT;
|
||||
|
||||
--- shouldn't see any changes performed in failed transaction
|
||||
SELECT * FROM dml_test ORDER BY id ASC;
|
||||
|
||||
-- fail at INSERT
|
||||
SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()');
|
||||
|
||||
BEGIN;
|
||||
DELETE FROM dml_test WHERE id = 1;
|
||||
DELETE FROM dml_test WHERE id = 2;
|
||||
INSERT INTO dml_test VALUES (5, 'Epsilon');
|
||||
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
|
||||
UPDATE dml_test SET name = 'gamma' WHERE id = 3;
|
||||
COMMIT;
|
||||
|
||||
--- shouldn't see any changes before failed INSERT
|
||||
SELECT * FROM dml_test ORDER BY id ASC;
|
||||
|
||||
-- fail at UPDATE
|
||||
SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()');
|
||||
|
||||
BEGIN;
|
||||
DELETE FROM dml_test WHERE id = 1;
|
||||
DELETE FROM dml_test WHERE id = 2;
|
||||
INSERT INTO dml_test VALUES (5, 'Epsilon');
|
||||
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
|
||||
UPDATE dml_test SET name = 'gamma' WHERE id = 3;
|
||||
COMMIT;
|
||||
|
||||
--- shouldn't see any changes after failed UPDATE
|
||||
SELECT * FROM dml_test ORDER BY id ASC;
|
||||
|
||||
-- fail at PREPARE TRANSACTION
|
||||
SELECT citus.mitmproxy('conn.onQuery(query="^PREPARE TRANSACTION").kill()');
|
||||
|
||||
-- hide the error message (it has the PID)...
|
||||
-- we'll test for the txn side-effects to ensure it didn't run
|
||||
SET client_min_messages TO FATAL;
|
||||
|
||||
BEGIN;
|
||||
DELETE FROM dml_test WHERE id = 1;
|
||||
DELETE FROM dml_test WHERE id = 2;
|
||||
INSERT INTO dml_test VALUES (5, 'Epsilon');
|
||||
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
|
||||
UPDATE dml_test SET name = 'gamma' WHERE id = 3;
|
||||
COMMIT;
|
||||
|
||||
SET client_min_messages TO DEFAULT;
|
||||
|
||||
SELECT citus.mitmproxy('conn.allow()');
|
||||
SELECT shardid FROM pg_dist_shard_placement WHERE shardstate = 3;
|
||||
SELECT recover_prepared_transactions();
|
||||
|
||||
-- shouldn't see any changes after failed PREPARE
|
||||
SELECT * FROM dml_test ORDER BY id ASC;
|
||||
|
||||
-- fail at COMMIT
|
||||
SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT").kill()');
|
||||
|
||||
-- hide the error message (it has the PID)...
|
||||
-- we'll test for the txn side-effects to ensure it didn't run
|
||||
SET client_min_messages TO FATAL;
|
||||
|
||||
BEGIN;
|
||||
DELETE FROM dml_test WHERE id = 1;
|
||||
DELETE FROM dml_test WHERE id = 2;
|
||||
INSERT INTO dml_test VALUES (5, 'Epsilon');
|
||||
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
|
||||
UPDATE dml_test SET name = 'gamma' WHERE id = 3;
|
||||
COMMIT;
|
||||
|
||||
SET client_min_messages TO DEFAULT;
|
||||
|
||||
SELECT citus.mitmproxy('conn.allow()');
|
||||
SELECT shardid FROM pg_dist_shard_placement WHERE shardstate = 3;
|
||||
SELECT recover_prepared_transactions();
|
||||
|
||||
-- should see changes, because of txn recovery
|
||||
SELECT * FROM dml_test ORDER BY id ASC;
|
||||
|
||||
-- drop table and recreate with different replication/sharding
|
||||
|
||||
DROP TABLE dml_test;
|
||||
SET citus.shard_count = 1;
|
||||
SET citus.shard_replication_factor = 2; -- two placements
|
||||
|
||||
CREATE TABLE dml_test (id integer, name text);
|
||||
SELECT create_distributed_table('dml_test', 'id');
|
||||
|
||||
COPY dml_test FROM STDIN WITH CSV;
|
||||
1,Alpha
|
||||
2,Beta
|
||||
3,Gamma
|
||||
4,Delta
|
||||
\.
|
||||
|
||||
---- test multiple statements against a single shard, but with two placements
|
||||
|
||||
-- fail at COMMIT (actually COMMIT this time, as no 2pc in use)
|
||||
SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT").kill()');
|
||||
|
||||
BEGIN;
|
||||
DELETE FROM dml_test WHERE id = 1;
|
||||
DELETE FROM dml_test WHERE id = 2;
|
||||
INSERT INTO dml_test VALUES (5, 'Epsilon');
|
||||
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
|
||||
UPDATE dml_test SET name = 'gamma' WHERE id = 3;
|
||||
COMMIT;
|
||||
|
||||
--- should see all changes, but they only went to one placement (other is unhealthy)
|
||||
SELECT * FROM dml_test ORDER BY id ASC;
|
||||
SELECT shardid FROM pg_dist_shard_placement WHERE shardstate = 3;
|
||||
|
||||
SELECT citus.mitmproxy('conn.allow()');
|
||||
|
||||
-- drop table and recreate as reference table
|
||||
|
||||
DROP TABLE dml_test;
|
||||
SET citus.shard_count = 2;
|
||||
SET citus.shard_replication_factor = 1;
|
||||
|
||||
CREATE TABLE dml_test (id integer, name text);
|
||||
SELECT create_reference_table('dml_test');
|
||||
|
||||
COPY dml_test FROM STDIN WITH CSV;
|
||||
1,Alpha
|
||||
2,Beta
|
||||
3,Gamma
|
||||
4,Delta
|
||||
\.
|
||||
|
||||
-- fail at COMMIT (by failing to PREPARE)
|
||||
SELECT citus.mitmproxy('conn.onQuery(query="^PREPARE").kill()');
|
||||
|
||||
BEGIN;
|
||||
DELETE FROM dml_test WHERE id = 1;
|
||||
DELETE FROM dml_test WHERE id = 2;
|
||||
INSERT INTO dml_test VALUES (5, 'Epsilon');
|
||||
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
|
||||
UPDATE dml_test SET name = 'gamma' WHERE id = 3;
|
||||
COMMIT;
|
||||
|
||||
--- shouldn't see any changes after failed COMMIT
|
||||
SELECT * FROM dml_test ORDER BY id ASC;
|
||||
|
||||
SELECT citus.mitmproxy('conn.allow()');
|
||||
DROP TABLE dml_test;
|
||||
Loading…
Reference in New Issue