-- We have two different output files for this failure test because the -- failure behaviour of SAVEPOINT and RELEASE commands are different if -- we use the executor. If we use it, these commands error out if any of -- the placement commands fail. Otherwise, we might mark the placement -- as invalid and continue with a WARNING. SELECT citus.mitmproxy('conn.allow()'); mitmproxy --------------------------------------------------------------------- (1 row) SET citus.shard_count = 2; SET citus.shard_replication_factor = 1; -- one shard per worker SET citus.next_shard_id TO 100950; ALTER SEQUENCE pg_catalog.pg_dist_placement_placementid_seq RESTART 150; CREATE TABLE artists ( id bigint NOT NULL, name text NOT NULL ); SELECT create_distributed_table('artists', 'id'); create_distributed_table --------------------------------------------------------------------- (1 row) -- add some data INSERT INTO artists VALUES (1, 'Pablo Picasso'); INSERT INTO artists VALUES (2, 'Vincent van Gogh'); INSERT INTO artists VALUES (3, 'Claude Monet'); INSERT INTO artists VALUES (4, 'William Kurelek'); -- simply fail at SAVEPOINT SELECT citus.mitmproxy('conn.onQuery(query="^SAVEPOINT").kill()'); mitmproxy --------------------------------------------------------------------- (1 row) BEGIN; INSERT INTO artists VALUES (5, 'Asher Lev'); SAVEPOINT s1; WARNING: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open connection not open connection not open CONTEXT: while executing command on localhost:xxxxx WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open connection not open connection not open WARNING: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open connection not open connection not open connection not open CONTEXT: while executing command on localhost:xxxxx WARNING: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open connection not open connection not open connection not open connection not open CONTEXT: while executing command on localhost:xxxxx ERROR: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open CONTEXT: while executing command on localhost:xxxxx DELETE FROM artists WHERE id=4; ERROR: current transaction is aborted, commands ignored until end of transaction block RELEASE SAVEPOINT s1; ERROR: current transaction is aborted, commands ignored until end of transaction block COMMIT; SELECT * FROM artists WHERE id IN (4, 5); id | name --------------------------------------------------------------------- 4 | William Kurelek (1 row) -- fail at RELEASE SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").kill()'); mitmproxy --------------------------------------------------------------------- (1 row) BEGIN; UPDATE artists SET name='a'; SAVEPOINT s1; DELETE FROM artists WHERE id=4; RELEASE SAVEPOINT s1; WARNING: AbortSubTransaction while in COMMIT state WARNING: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open connection not open connection not open CONTEXT: while executing command on localhost:xxxxx WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open connection not open connection not open WARNING: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open connection not open connection not open connection not open CONTEXT: while executing command on localhost:xxxxx WARNING: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open connection not open connection not open connection not open connection not open CONTEXT: while executing command on localhost:xxxxx WARNING: savepoint "savepoint_2" does not exist CONTEXT: while executing command on localhost:xxxxx ERROR: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open CONTEXT: while executing command on localhost:xxxxx ROLLBACK; SELECT * FROM artists WHERE id IN (4, 5); id | name --------------------------------------------------------------------- 4 | William Kurelek (1 row) -- fail at ROLLBACK SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()'); mitmproxy --------------------------------------------------------------------- (1 row) BEGIN; INSERT INTO artists VALUES (5, 'Asher Lev'); SAVEPOINT s1; DELETE FROM artists WHERE id=4; ROLLBACK TO SAVEPOINT s1; WARNING: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open CONTEXT: while executing command on localhost:xxxxx WARNING: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open connection not open CONTEXT: while executing command on localhost:xxxxx COMMIT; ERROR: could not make changes to shard xxxxx on any node SELECT * FROM artists WHERE id IN (4, 5); id | name --------------------------------------------------------------------- 4 | William Kurelek (1 row) -- fail at second RELEASE SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").after(1).kill()'); mitmproxy --------------------------------------------------------------------- (1 row) BEGIN; SAVEPOINT s1; DELETE FROM artists WHERE id=4; RELEASE SAVEPOINT s1; SAVEPOINT s2; INSERT INTO artists VALUES (5, 'Jacob Kahn'); RELEASE SAVEPOINT s2; WARNING: AbortSubTransaction while in COMMIT state WARNING: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open connection not open connection not open CONTEXT: while executing command on localhost:xxxxx WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open connection not open connection not open WARNING: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open connection not open connection not open connection not open CONTEXT: while executing command on localhost:xxxxx WARNING: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open connection not open connection not open connection not open connection not open CONTEXT: while executing command on localhost:xxxxx ERROR: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open CONTEXT: while executing command on localhost:xxxxx COMMIT; SELECT * FROM artists WHERE id IN (4, 5); id | name --------------------------------------------------------------------- 4 | William Kurelek (1 row) -- fail at second ROLLBACK SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").after(1).kill()'); mitmproxy --------------------------------------------------------------------- (1 row) BEGIN; SAVEPOINT s1; UPDATE artists SET name='A' WHERE id=4; ROLLBACK TO SAVEPOINT s1; SAVEPOINT s2; DELETE FROM artists WHERE id=5; ROLLBACK TO SAVEPOINT s2; WARNING: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open CONTEXT: while executing command on localhost:xxxxx WARNING: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open connection not open CONTEXT: while executing command on localhost:xxxxx COMMIT; ERROR: could not make changes to shard xxxxx on any node SELECT * FROM artists WHERE id IN (4, 5); id | name --------------------------------------------------------------------- 4 | William Kurelek (1 row) SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").after(1).kill()'); mitmproxy --------------------------------------------------------------------- (1 row) -- Release after rollback BEGIN; SAVEPOINT s1; ROLLBACK TO s1; RELEASE SAVEPOINT s1; SAVEPOINT s2; INSERT INTO artists VALUES (6, 'John J. Audubon'); INSERT INTO artists VALUES (7, 'Emily Carr'); ROLLBACK TO s2; RELEASE SAVEPOINT s2; COMMIT; SELECT * FROM artists WHERE id=7; id | name --------------------------------------------------------------------- (0 rows) SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()'); mitmproxy --------------------------------------------------------------------- (1 row) -- Recover from errors \set VERBOSITY terse BEGIN; SAVEPOINT s1; SAVEPOINT s2; INSERT INTO artists VALUES (6, 'John J. Audubon'); INSERT INTO artists VALUES (7, 'Emily Carr'); INSERT INTO artists VALUES (7, 'Emily Carr'); ROLLBACK TO SAVEPOINT s1; WARNING: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open WARNING: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open connection not open WARNING: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open connection not open connection not open connection not open WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open connection not open connection not open connection not open WARNING: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open connection not open connection not open connection not open connection not open WARNING: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open connection not open connection not open connection not open connection not open connection not open COMMIT; ERROR: could not make changes to shard xxxxx on any node SELECT * FROM artists WHERE id=6; id | name --------------------------------------------------------------------- (0 rows) -- replication factor > 1 CREATE TABLE researchers ( id bigint NOT NULL, lab_id int NOT NULL, name text NOT NULL ); SET citus.shard_count = 1; SET citus.shard_replication_factor = 2; -- single shard, on both workers SELECT create_distributed_table('researchers', 'lab_id', 'hash'); create_distributed_table --------------------------------------------------------------------- (1 row) -- simply fail at SAVEPOINT SELECT citus.mitmproxy('conn.onQuery(query="^SAVEPOINT").kill()'); mitmproxy --------------------------------------------------------------------- (1 row) BEGIN; INSERT INTO researchers VALUES (7, 4, 'Jan Plaza'); SAVEPOINT s1; WARNING: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open connection not open connection not open WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open connection not open connection not open WARNING: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open connection not open connection not open connection not open WARNING: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open connection not open connection not open connection not open connection not open ERROR: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open INSERT INTO researchers VALUES (8, 4, 'Alonzo Church'); ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK TO s1; ERROR: savepoint "s1" does not exist RELEASE SAVEPOINT s1; ERROR: current transaction is aborted, commands ignored until end of transaction block COMMIT; -- should see correct results from healthy placement and one bad placement SELECT * FROM researchers WHERE lab_id = 4; id | lab_id | name --------------------------------------------------------------------- (0 rows) UPDATE pg_dist_shard_placement SET shardstate = 1 WHERE shardstate = 3 AND shardid IN ( SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'researchers'::regclass ) RETURNING placementid; placementid --------------------------------------------------------------------- (0 rows) TRUNCATE researchers; -- fail at rollback SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()'); mitmproxy --------------------------------------------------------------------- (1 row) BEGIN; INSERT INTO researchers VALUES (7, 4, 'Jan Plaza'); SAVEPOINT s1; INSERT INTO researchers VALUES (8, 4, 'Alonzo Church'); ROLLBACK TO s1; WARNING: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open WARNING: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open connection not open RELEASE SAVEPOINT s1; COMMIT; ERROR: failure on connection marked as essential: localhost:xxxxx -- should see correct results from healthy placement and one bad placement SELECT * FROM researchers WHERE lab_id = 4; id | lab_id | name --------------------------------------------------------------------- (0 rows) UPDATE pg_dist_shard_placement SET shardstate = 1 WHERE shardstate = 3 AND shardid IN ( SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'researchers'::regclass ) RETURNING placementid; placementid --------------------------------------------------------------------- (0 rows) TRUNCATE researchers; -- fail at release SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").kill()'); mitmproxy --------------------------------------------------------------------- (1 row) BEGIN; INSERT INTO researchers VALUES (7, 4, 'Jan Plaza'); SAVEPOINT s1; INSERT INTO researchers VALUES (8, 4, 'Alonzo Church'); ROLLBACK TO s1; RELEASE SAVEPOINT s1; WARNING: AbortSubTransaction while in COMMIT state WARNING: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open connection not open connection not open WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open connection not open connection not open WARNING: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open connection not open connection not open connection not open WARNING: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open connection not open connection not open connection not open connection not open WARNING: savepoint "savepoint_3" does not exist ERROR: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open COMMIT; -- should see correct results from healthy placement and one bad placement SELECT * FROM researchers WHERE lab_id = 4; id | lab_id | name --------------------------------------------------------------------- (0 rows) UPDATE pg_dist_shard_placement SET shardstate = 1 WHERE shardstate = 3 AND shardid IN ( SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'researchers'::regclass ) RETURNING placementid; placementid --------------------------------------------------------------------- (0 rows) TRUNCATE researchers; -- test that we don't mark reference placements unhealthy CREATE TABLE ref(a int, b int); SELECT create_reference_table('ref'); create_reference_table --------------------------------------------------------------------- (1 row) SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()'); mitmproxy --------------------------------------------------------------------- (1 row) BEGIN; SAVEPOINT start; INSERT INTO ref VALUES (1001,2); SELECT * FROM ref; a | b --------------------------------------------------------------------- 1001 | 2 (1 row) ROLLBACK TO SAVEPOINT start; WARNING: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open WARNING: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open connection not open SELECT * FROM ref; WARNING: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open connection not open connection not open connection not open WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open connection not open connection not open connection not open WARNING: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open connection not open connection not open connection not open connection not open WARNING: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open connection not open connection not open connection not open connection not open connection not open ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection not open connection not open END; -- clean up SELECT citus.mitmproxy('conn.allow()'); mitmproxy --------------------------------------------------------------------- (1 row) DROP TABLE artists; DROP TABLE researchers; DROP TABLE ref;