-- We have two different output files for this failure test because the -- failure behaviour of SAVEPOINT and RELEASE commands are different if -- we use the executor. If we use it, these commands error out if any of -- the placement commands fail. Otherwise, we might mark the placement -- as invalid and continue with a WARNING. SELECT citus.mitmproxy('conn.allow()'); mitmproxy --------------------------------------------------------------------- (1 row) SET citus.shard_count = 2; SET citus.shard_replication_factor = 1; -- one shard per worker SET citus.next_shard_id TO 100950; ALTER SEQUENCE pg_catalog.pg_dist_placement_placementid_seq RESTART 150; CREATE TABLE artists ( id bigint NOT NULL, name text NOT NULL ); SELECT create_distributed_table('artists', 'id'); create_distributed_table --------------------------------------------------------------------- (1 row) -- add some data INSERT INTO artists VALUES (1, 'Pablo Picasso'); INSERT INTO artists VALUES (2, 'Vincent van Gogh'); INSERT INTO artists VALUES (3, 'Claude Monet'); INSERT INTO artists VALUES (4, 'William Kurelek'); -- simply fail at SAVEPOINT SELECT citus.mitmproxy('conn.onQuery(query="^SAVEPOINT").kill()'); mitmproxy --------------------------------------------------------------------- (1 row) BEGIN; INSERT INTO artists VALUES (5, 'Asher Lev'); SAVEPOINT s1; WARNING: connection not open CONTEXT: while executing command on localhost:xxxxx WARNING: connection error: localhost:xxxxx DETAIL: connection not open WARNING: connection not open CONTEXT: while executing command on localhost:xxxxx WARNING: connection not open CONTEXT: while executing command on localhost:xxxxx ERROR: connection not open CONTEXT: while executing command on localhost:xxxxx DELETE FROM artists WHERE id=4; ERROR: current transaction is aborted, commands ignored until end of transaction block RELEASE SAVEPOINT s1; ERROR: current transaction is aborted, commands ignored until end of transaction block COMMIT; SELECT * FROM artists WHERE id IN (4, 5); id | name --------------------------------------------------------------------- 4 | William Kurelek (1 row) -- fail at RELEASE SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").kill()'); mitmproxy --------------------------------------------------------------------- (1 row) BEGIN; UPDATE artists SET name='a'; SAVEPOINT s1; DELETE FROM artists WHERE id=4; RELEASE SAVEPOINT s1; WARNING: AbortSubTransaction while in COMMIT state WARNING: connection not open CONTEXT: while executing command on localhost:xxxxx WARNING: connection error: localhost:xxxxx DETAIL: connection not open WARNING: connection not open CONTEXT: while executing command on localhost:xxxxx WARNING: connection not open CONTEXT: while executing command on localhost:xxxxx WARNING: savepoint "savepoint_2" does not exist CONTEXT: while executing command on localhost:xxxxx ERROR: connection not open CONTEXT: while executing command on localhost:xxxxx ROLLBACK; SELECT * FROM artists WHERE id IN (4, 5); id | name --------------------------------------------------------------------- 4 | William Kurelek (1 row) -- fail at ROLLBACK SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()'); mitmproxy --------------------------------------------------------------------- (1 row) BEGIN; INSERT INTO artists VALUES (5, 'Asher Lev'); SAVEPOINT s1; DELETE FROM artists WHERE id=4; ROLLBACK TO SAVEPOINT s1; WARNING: connection not open CONTEXT: while executing command on localhost:xxxxx WARNING: connection not open CONTEXT: while executing command on localhost:xxxxx COMMIT; ERROR: could not make changes to shard xxxxx on any node SELECT * FROM artists WHERE id IN (4, 5); id | name --------------------------------------------------------------------- 4 | William Kurelek (1 row) -- fail at second RELEASE SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").after(1).kill()'); mitmproxy --------------------------------------------------------------------- (1 row) BEGIN; SAVEPOINT s1; DELETE FROM artists WHERE id=4; RELEASE SAVEPOINT s1; SAVEPOINT s2; INSERT INTO artists VALUES (5, 'Jacob Kahn'); RELEASE SAVEPOINT s2; WARNING: AbortSubTransaction while in COMMIT state WARNING: connection not open CONTEXT: while executing command on localhost:xxxxx WARNING: connection error: localhost:xxxxx DETAIL: connection not open WARNING: connection not open CONTEXT: while executing command on localhost:xxxxx WARNING: connection not open CONTEXT: while executing command on localhost:xxxxx ERROR: connection not open CONTEXT: while executing command on localhost:xxxxx COMMIT; SELECT * FROM artists WHERE id IN (4, 5); id | name --------------------------------------------------------------------- 4 | William Kurelek (1 row) -- fail at second ROLLBACK SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").after(1).kill()'); mitmproxy --------------------------------------------------------------------- (1 row) BEGIN; SAVEPOINT s1; UPDATE artists SET name='A' WHERE id=4; ROLLBACK TO SAVEPOINT s1; SAVEPOINT s2; DELETE FROM artists WHERE id=5; ROLLBACK TO SAVEPOINT s2; WARNING: connection not open CONTEXT: while executing command on localhost:xxxxx WARNING: connection not open CONTEXT: while executing command on localhost:xxxxx COMMIT; ERROR: could not make changes to shard xxxxx on any node SELECT * FROM artists WHERE id IN (4, 5); id | name --------------------------------------------------------------------- 4 | William Kurelek (1 row) SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").after(1).kill()'); mitmproxy --------------------------------------------------------------------- (1 row) -- Release after rollback BEGIN; SAVEPOINT s1; ROLLBACK TO s1; RELEASE SAVEPOINT s1; SAVEPOINT s2; INSERT INTO artists VALUES (6, 'John J. Audubon'); INSERT INTO artists VALUES (7, 'Emily Carr'); ROLLBACK TO s2; RELEASE SAVEPOINT s2; COMMIT; SELECT * FROM artists WHERE id=7; id | name --------------------------------------------------------------------- (0 rows) SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()'); mitmproxy --------------------------------------------------------------------- (1 row) -- Recover from errors \set VERBOSITY terse BEGIN; SAVEPOINT s1; SAVEPOINT s2; INSERT INTO artists VALUES (6, 'John J. Audubon'); INSERT INTO artists VALUES (7, 'Emily Carr'); INSERT INTO artists VALUES (7, 'Emily Carr'); ROLLBACK TO SAVEPOINT s1; WARNING: connection not open WARNING: connection not open WARNING: connection not open WARNING: connection error: localhost:xxxxx WARNING: connection not open WARNING: connection not open COMMIT; ERROR: could not make changes to shard xxxxx on any node SELECT * FROM artists WHERE id=6; id | name --------------------------------------------------------------------- (0 rows) -- replication factor > 1 CREATE TABLE researchers ( id bigint NOT NULL, lab_id int NOT NULL, name text NOT NULL ); SET citus.shard_count = 1; SET citus.shard_replication_factor = 2; -- single shard, on both workers SELECT create_distributed_table('researchers', 'lab_id', 'hash'); create_distributed_table --------------------------------------------------------------------- (1 row) -- simply fail at SAVEPOINT SELECT citus.mitmproxy('conn.onQuery(query="^SAVEPOINT").kill()'); mitmproxy --------------------------------------------------------------------- (1 row) BEGIN; INSERT INTO researchers VALUES (7, 4, 'Jan Plaza'); SAVEPOINT s1; WARNING: connection not open WARNING: connection error: localhost:xxxxx WARNING: connection not open WARNING: connection not open ERROR: connection not open INSERT INTO researchers VALUES (8, 4, 'Alonzo Church'); ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK TO s1; ERROR: savepoint "s1" does not exist RELEASE SAVEPOINT s1; ERROR: current transaction is aborted, commands ignored until end of transaction block COMMIT; -- should see correct results from healthy placement and one bad placement SELECT * FROM researchers WHERE lab_id = 4; id | lab_id | name --------------------------------------------------------------------- (0 rows) UPDATE pg_dist_shard_placement SET shardstate = 1 WHERE shardstate = 3 AND shardid IN ( SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'researchers'::regclass ) RETURNING placementid; placementid --------------------------------------------------------------------- (0 rows) TRUNCATE researchers; -- fail at rollback SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()'); mitmproxy --------------------------------------------------------------------- (1 row) BEGIN; INSERT INTO researchers VALUES (7, 4, 'Jan Plaza'); SAVEPOINT s1; INSERT INTO researchers VALUES (8, 4, 'Alonzo Church'); ROLLBACK TO s1; WARNING: connection not open WARNING: connection not open RELEASE SAVEPOINT s1; COMMIT; ERROR: failure on connection marked as essential: localhost:xxxxx -- should see correct results from healthy placement and one bad placement SELECT * FROM researchers WHERE lab_id = 4; id | lab_id | name --------------------------------------------------------------------- (0 rows) UPDATE pg_dist_shard_placement SET shardstate = 1 WHERE shardstate = 3 AND shardid IN ( SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'researchers'::regclass ) RETURNING placementid; placementid --------------------------------------------------------------------- (0 rows) TRUNCATE researchers; -- fail at release SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").kill()'); mitmproxy --------------------------------------------------------------------- (1 row) BEGIN; INSERT INTO researchers VALUES (7, 4, 'Jan Plaza'); SAVEPOINT s1; INSERT INTO researchers VALUES (8, 4, 'Alonzo Church'); ROLLBACK TO s1; RELEASE SAVEPOINT s1; WARNING: AbortSubTransaction while in COMMIT state WARNING: connection not open WARNING: connection error: localhost:xxxxx WARNING: connection not open WARNING: connection not open WARNING: savepoint "savepoint_3" does not exist ERROR: connection not open COMMIT; -- should see correct results from healthy placement and one bad placement SELECT * FROM researchers WHERE lab_id = 4; id | lab_id | name --------------------------------------------------------------------- (0 rows) UPDATE pg_dist_shard_placement SET shardstate = 1 WHERE shardstate = 3 AND shardid IN ( SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'researchers'::regclass ) RETURNING placementid; placementid --------------------------------------------------------------------- (0 rows) TRUNCATE researchers; -- test that we don't mark reference placements unhealthy CREATE TABLE ref(a int, b int); SELECT create_reference_table('ref'); create_reference_table --------------------------------------------------------------------- (1 row) SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()'); mitmproxy --------------------------------------------------------------------- (1 row) BEGIN; SAVEPOINT start; INSERT INTO ref VALUES (1001,2); SELECT * FROM ref; a | b --------------------------------------------------------------------- 1001 | 2 (1 row) ROLLBACK TO SAVEPOINT start; WARNING: connection not open WARNING: connection not open SELECT * FROM ref; WARNING: connection not open WARNING: connection error: localhost:xxxxx WARNING: connection not open WARNING: connection not open ERROR: connection error: localhost:xxxxx END; -- clean up SELECT citus.mitmproxy('conn.allow()'); mitmproxy --------------------------------------------------------------------- (1 row) DROP TABLE artists; DROP TABLE researchers; DROP TABLE ref;