-- We have two different output files for this failure test because the -- failure behaviour of SAVEPOINT and RELEASE commands are different if -- we use the executor. If we use it, these commands error out if any of -- the placement commands fail. Otherwise, we might mark the placement -- as invalid and continue with a WARNING. SELECT citus.mitmproxy('conn.allow()'); mitmproxy ----------- (1 row) SET citus.shard_count = 2; SET citus.shard_replication_factor = 1; -- one shard per worker SET citus.next_shard_id TO 100950; ALTER SEQUENCE pg_catalog.pg_dist_placement_placementid_seq RESTART 150; CREATE TABLE artists ( id bigint NOT NULL, name text NOT NULL ); SELECT create_distributed_table('artists', 'id'); create_distributed_table -------------------------- (1 row) -- add some data INSERT INTO artists VALUES (1, 'Pablo Picasso'); INSERT INTO artists VALUES (2, 'Vincent van Gogh'); INSERT INTO artists VALUES (3, 'Claude Monet'); INSERT INTO artists VALUES (4, 'William Kurelek'); -- simply fail at SAVEPOINT SELECT citus.mitmproxy('conn.onQuery(query="^SAVEPOINT").kill()'); mitmproxy ----------- (1 row) BEGIN; INSERT INTO artists VALUES (5, 'Asher Lev'); SAVEPOINT s1; WARNING: connection not open CONTEXT: while executing command on localhost:9060 WARNING: connection error: localhost:9060 DETAIL: connection not open WARNING: connection not open CONTEXT: while executing command on localhost:9060 WARNING: connection not open CONTEXT: while executing command on localhost:9060 ERROR: connection not open CONTEXT: while executing command on localhost:9060 DELETE FROM artists WHERE id=4; ERROR: current transaction is aborted, commands ignored until end of transaction block RELEASE SAVEPOINT s1; ERROR: current transaction is aborted, commands ignored until end of transaction block COMMIT; SELECT * FROM artists WHERE id IN (4, 5); id | name ----+----------------- 4 | William Kurelek (1 row) -- fail at RELEASE SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").kill()'); mitmproxy ----------- (1 row) BEGIN; UPDATE artists SET name='a'; SAVEPOINT s1; DELETE FROM artists WHERE id=4; RELEASE SAVEPOINT s1; WARNING: AbortSubTransaction while in COMMIT state WARNING: connection not open CONTEXT: while executing command on localhost:9060 WARNING: connection error: localhost:9060 DETAIL: connection not open WARNING: connection not open CONTEXT: while executing command on localhost:9060 WARNING: connection not open CONTEXT: while executing command on localhost:9060 WARNING: savepoint "savepoint_2" does not exist CONTEXT: while executing command on localhost:57637 ERROR: connection not open CONTEXT: while executing command on localhost:9060 ROLLBACK; SELECT * FROM artists WHERE id IN (4, 5); id | name ----+----------------- 4 | William Kurelek (1 row) -- fail at ROLLBACK SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()'); mitmproxy ----------- (1 row) BEGIN; INSERT INTO artists VALUES (5, 'Asher Lev'); SAVEPOINT s1; DELETE FROM artists WHERE id=4; ROLLBACK TO SAVEPOINT s1; WARNING: connection not open CONTEXT: while executing command on localhost:9060 WARNING: connection not open CONTEXT: while executing command on localhost:9060 COMMIT; ERROR: could not make changes to shard 100950 on any node SELECT * FROM artists WHERE id IN (4, 5); id | name ----+----------------- 4 | William Kurelek (1 row) -- fail at second RELEASE SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").after(1).kill()'); mitmproxy ----------- (1 row) BEGIN; SAVEPOINT s1; DELETE FROM artists WHERE id=4; RELEASE SAVEPOINT s1; SAVEPOINT s2; INSERT INTO artists VALUES (5, 'Jacob Kahn'); RELEASE SAVEPOINT s2; WARNING: AbortSubTransaction while in COMMIT state WARNING: connection not open CONTEXT: while executing command on localhost:9060 WARNING: connection error: localhost:9060 DETAIL: connection not open WARNING: connection not open CONTEXT: while executing command on localhost:9060 WARNING: connection not open CONTEXT: while executing command on localhost:9060 ERROR: connection not open CONTEXT: while executing command on localhost:9060 COMMIT; SELECT * FROM artists WHERE id IN (4, 5); id | name ----+----------------- 4 | William Kurelek (1 row) -- fail at second ROLLBACK SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").after(1).kill()'); mitmproxy ----------- (1 row) BEGIN; SAVEPOINT s1; UPDATE artists SET name='A' WHERE id=4; ROLLBACK TO SAVEPOINT s1; SAVEPOINT s2; DELETE FROM artists WHERE id=5; ROLLBACK TO SAVEPOINT s2; WARNING: connection not open CONTEXT: while executing command on localhost:9060 WARNING: connection not open CONTEXT: while executing command on localhost:9060 COMMIT; ERROR: could not make changes to shard 100950 on any node SELECT * FROM artists WHERE id IN (4, 5); id | name ----+----------------- 4 | William Kurelek (1 row) SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").after(1).kill()'); mitmproxy ----------- (1 row) -- Release after rollback BEGIN; SAVEPOINT s1; ROLLBACK TO s1; RELEASE SAVEPOINT s1; SAVEPOINT s2; INSERT INTO artists VALUES (6, 'John J. Audubon'); INSERT INTO artists VALUES (7, 'Emily Carr'); ROLLBACK TO s2; RELEASE SAVEPOINT s2; COMMIT; SELECT * FROM artists WHERE id=7; id | name ----+------ (0 rows) SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()'); mitmproxy ----------- (1 row) -- Recover from errors \set VERBOSITY terse BEGIN; SAVEPOINT s1; SAVEPOINT s2; INSERT INTO artists VALUES (6, 'John J. Audubon'); INSERT INTO artists VALUES (7, 'Emily Carr'); INSERT INTO artists VALUES (7, 'Emily Carr'); ROLLBACK TO SAVEPOINT s1; WARNING: connection not open WARNING: connection not open WARNING: connection not open WARNING: connection error: localhost:9060 WARNING: connection not open WARNING: connection not open COMMIT; ERROR: could not make changes to shard 100950 on any node SELECT * FROM artists WHERE id=6; id | name ----+------ (0 rows) -- replication factor > 1 CREATE TABLE researchers ( id bigint NOT NULL, lab_id int NOT NULL, name text NOT NULL ); SET citus.shard_count = 1; SET citus.shard_replication_factor = 2; -- single shard, on both workers SELECT create_distributed_table('researchers', 'lab_id', 'hash'); create_distributed_table -------------------------- (1 row) -- simply fail at SAVEPOINT SELECT citus.mitmproxy('conn.onQuery(query="^SAVEPOINT").kill()'); mitmproxy ----------- (1 row) BEGIN; INSERT INTO researchers VALUES (7, 4, 'Jan Plaza'); SAVEPOINT s1; WARNING: connection not open WARNING: connection error: localhost:9060 WARNING: connection not open WARNING: connection not open ERROR: connection not open INSERT INTO researchers VALUES (8, 4, 'Alonzo Church'); ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK TO s1; ERROR: savepoint "s1" does not exist RELEASE SAVEPOINT s1; ERROR: current transaction is aborted, commands ignored until end of transaction block COMMIT; -- should see correct results from healthy placement and one bad placement SELECT * FROM researchers WHERE lab_id = 4; id | lab_id | name ----+--------+------ (0 rows) UPDATE pg_dist_shard_placement SET shardstate = 1 WHERE shardstate = 3 AND shardid IN ( SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'researchers'::regclass ) RETURNING placementid; placementid ------------- (0 rows) TRUNCATE researchers; -- fail at rollback SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()'); mitmproxy ----------- (1 row) BEGIN; INSERT INTO researchers VALUES (7, 4, 'Jan Plaza'); SAVEPOINT s1; INSERT INTO researchers VALUES (8, 4, 'Alonzo Church'); ROLLBACK TO s1; WARNING: connection not open WARNING: connection not open RELEASE SAVEPOINT s1; COMMIT; ERROR: failure on connection marked as essential: localhost:9060 -- should see correct results from healthy placement and one bad placement SELECT * FROM researchers WHERE lab_id = 4; id | lab_id | name ----+--------+------ (0 rows) UPDATE pg_dist_shard_placement SET shardstate = 1 WHERE shardstate = 3 AND shardid IN ( SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'researchers'::regclass ) RETURNING placementid; placementid ------------- (0 rows) TRUNCATE researchers; -- fail at release SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").kill()'); mitmproxy ----------- (1 row) BEGIN; INSERT INTO researchers VALUES (7, 4, 'Jan Plaza'); SAVEPOINT s1; INSERT INTO researchers VALUES (8, 4, 'Alonzo Church'); ROLLBACK TO s1; RELEASE SAVEPOINT s1; WARNING: AbortSubTransaction while in COMMIT state WARNING: connection not open WARNING: connection error: localhost:9060 WARNING: connection not open WARNING: connection not open WARNING: savepoint "savepoint_3" does not exist ERROR: connection not open COMMIT; -- should see correct results from healthy placement and one bad placement SELECT * FROM researchers WHERE lab_id = 4; id | lab_id | name ----+--------+------ (0 rows) UPDATE pg_dist_shard_placement SET shardstate = 1 WHERE shardstate = 3 AND shardid IN ( SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'researchers'::regclass ) RETURNING placementid; placementid ------------- (0 rows) TRUNCATE researchers; -- clean up SELECT citus.mitmproxy('conn.allow()'); mitmproxy ----------- (1 row) DROP TABLE artists; DROP TABLE researchers;