diff --git a/src/test/regress/expected/failure_savepoints.out b/src/test/regress/expected/failure_savepoints.out new file mode 100644 index 000000000..ca8a968b1 --- /dev/null +++ b/src/test/regress/expected/failure_savepoints.out @@ -0,0 +1,339 @@ +SELECT citus.mitmproxy('conn.allow()'); + mitmproxy +----------- + +(1 row) + +SET citus.shard_count = 2; +SET citus.shard_replication_factor = 1; -- one shard per worker +SET citus.next_shard_id TO 100950; +ALTER SEQUENCE pg_catalog.pg_dist_placement_placementid_seq RESTART 150; +CREATE TABLE artists ( + id bigint NOT NULL, + name text NOT NULL +); +SELECT create_distributed_table('artists', 'id'); + create_distributed_table +-------------------------- + +(1 row) + +-- add some data +INSERT INTO artists VALUES (1, 'Pablo Picasso'); +INSERT INTO artists VALUES (2, 'Vincent van Gogh'); +INSERT INTO artists VALUES (3, 'Claude Monet'); +INSERT INTO artists VALUES (4, 'William Kurelek'); +-- simply fail at SAVEPOINT +SELECT citus.mitmproxy('conn.onQuery(query="^SAVEPOINT").kill()'); + mitmproxy +----------- + +(1 row) + +BEGIN; +INSERT INTO artists VALUES (5, 'Asher Lev'); +SAVEPOINT s1; +WARNING: connection not open +CONTEXT: while executing command on localhost:9060 +WARNING: connection not open +CONTEXT: while executing command on localhost:9060 +DELETE FROM artists WHERE id=4; +WARNING: connection not open +CONTEXT: while executing command on localhost:9060 +WARNING: connection error: localhost:9060 +DETAIL: connection not open +WARNING: connection not open +CONTEXT: while executing command on localhost:9060 +WARNING: connection not open +CONTEXT: while executing command on localhost:9060 +ERROR: could not modify any active placements +RELEASE SAVEPOINT s1; +ERROR: current transaction is aborted, commands ignored until end of transaction block +COMMIT; +SELECT * FROM artists WHERE id IN (4, 5); + id | name +----+----------------- + 4 | William Kurelek +(1 row) + +-- fail at RELEASE +SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").kill()'); + mitmproxy +----------- + +(1 row) + +BEGIN; +INSERT INTO artists VALUES (5, 'Asher Lev'); +SAVEPOINT s1; +DELETE FROM artists WHERE id=4; +RELEASE SAVEPOINT s1; +WARNING: connection not open +CONTEXT: while executing command on localhost:9060 +WARNING: connection not open +CONTEXT: while executing command on localhost:9060 +COMMIT; +ERROR: could not make changes to shard 100950 on any node +SELECT * FROM artists WHERE id IN (4, 5); + id | name +----+----------------- + 4 | William Kurelek +(1 row) + +-- fail at ROLLBACK +SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()'); + mitmproxy +----------- + +(1 row) + +BEGIN; +INSERT INTO artists VALUES (5, 'Asher Lev'); +SAVEPOINT s1; +DELETE FROM artists WHERE id=4; +ROLLBACK TO SAVEPOINT s1; +WARNING: connection not open +CONTEXT: while executing command on localhost:9060 +WARNING: connection not open +CONTEXT: while executing command on localhost:9060 +COMMIT; +ERROR: could not make changes to shard 100950 on any node +SELECT * FROM artists WHERE id IN (4, 5); + id | name +----+----------------- + 4 | William Kurelek +(1 row) + +-- fail at second RELEASE +SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").after(1).kill()'); + mitmproxy +----------- + +(1 row) + +BEGIN; +SAVEPOINT s1; +DELETE FROM artists WHERE id=4; +RELEASE SAVEPOINT s1; +SAVEPOINT s2; +INSERT INTO artists VALUES (5, 'Jacob Kahn'); +RELEASE SAVEPOINT s2; +WARNING: connection not open +CONTEXT: while executing command on localhost:9060 +WARNING: connection not open +CONTEXT: while executing command on localhost:9060 +COMMIT; +ERROR: could not make changes to shard 100950 on any node +SELECT * FROM artists WHERE id IN (4, 5); + id | name +----+----------------- + 4 | William Kurelek +(1 row) + +-- fail at second ROLLBACK +SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").after(1).kill()'); + mitmproxy +----------- + +(1 row) + +BEGIN; +SAVEPOINT s1; +UPDATE artists SET name='A' WHERE id=4; +ROLLBACK TO SAVEPOINT s1; +SAVEPOINT s2; +DELETE FROM artists WHERE id=5; +ROLLBACK TO SAVEPOINT s2; +WARNING: connection not open +CONTEXT: while executing command on localhost:9060 +WARNING: connection not open +CONTEXT: while executing command on localhost:9060 +COMMIT; +ERROR: could not make changes to shard 100950 on any node +SELECT * FROM artists WHERE id IN (4, 5); + id | name +----+----------------- + 4 | William Kurelek +(1 row) + +SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").after(1).kill()'); + mitmproxy +----------- + +(1 row) + +-- Release after rollback +BEGIN; +SAVEPOINT s1; +ROLLBACK TO s1; +RELEASE SAVEPOINT s1; +SAVEPOINT s2; +INSERT INTO artists VALUES (6, 'John J. Audubon'); +INSERT INTO artists VALUES (7, 'Emily Carr'); +ROLLBACK TO s2; +RELEASE SAVEPOINT s2; +COMMIT; +SELECT * FROM artists WHERE id=7; + id | name +----+------ +(0 rows) + +SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()'); + mitmproxy +----------- + +(1 row) + +-- Recover from errors +\set VERBOSITY terse +BEGIN; +SAVEPOINT s1; +SAVEPOINT s2; +INSERT INTO artists VALUES (6, 'John J. Audubon'); +INSERT INTO artists VALUES (7, 'Emily Carr'); +INSERT INTO artists VALUES (7, 'Emily Carr'); +ROLLBACK TO SAVEPOINT s1; +WARNING: connection not open +WARNING: connection not open +COMMIT; +ERROR: could not make changes to shard 100950 on any node +SELECT * FROM artists WHERE id=6; + id | name +----+------ +(0 rows) + +-- replication factor > 1 +CREATE TABLE researchers ( + id bigint NOT NULL, + lab_id int NOT NULL, + name text NOT NULL +); +SET citus.shard_count = 1; +SET citus.shard_replication_factor = 2; -- single shard, on both workers +SELECT create_distributed_table('researchers', 'lab_id', 'hash'); + create_distributed_table +-------------------------- + +(1 row) + +-- simply fail at SAVEPOINT +SELECT citus.mitmproxy('conn.onQuery(query="^SAVEPOINT").kill()'); + mitmproxy +----------- + +(1 row) + +BEGIN; +INSERT INTO researchers VALUES (7, 4, 'Jan Plaza'); +SAVEPOINT s1; +WARNING: connection not open +WARNING: connection not open +INSERT INTO researchers VALUES (8, 4, 'Alonzo Church'); +ROLLBACK TO s1; +WARNING: connection not open +WARNING: connection error: localhost:9060 +WARNING: connection not open +WARNING: connection not open +RELEASE SAVEPOINT s1; +COMMIT; +WARNING: connection not open +WARNING: connection not open +WARNING: connection not open +-- should see correct results from healthy placement and one bad placement +SELECT * FROM researchers WHERE lab_id = 4; + id | lab_id | name +----+--------+----------- + 7 | 4 | Jan Plaza +(1 row) + +UPDATE pg_dist_shard_placement SET shardstate = 1 +WHERE shardstate = 3 AND shardid IN ( + SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'researchers'::regclass +) RETURNING placementid; + placementid +------------- + 152 +(1 row) + +TRUNCATE researchers; +-- fail at rollback +SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()'); + mitmproxy +----------- + +(1 row) + +BEGIN; +INSERT INTO researchers VALUES (7, 4, 'Jan Plaza'); +SAVEPOINT s1; +INSERT INTO researchers VALUES (8, 4, 'Alonzo Church'); +ROLLBACK TO s1; +WARNING: connection not open +WARNING: connection not open +RELEASE SAVEPOINT s1; +COMMIT; +WARNING: connection not open +WARNING: connection not open +WARNING: connection not open +-- should see correct results from healthy placement and one bad placement +SELECT * FROM researchers WHERE lab_id = 4; + id | lab_id | name +----+--------+----------- + 7 | 4 | Jan Plaza +(1 row) + +UPDATE pg_dist_shard_placement SET shardstate = 1 +WHERE shardstate = 3 AND shardid IN ( + SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'researchers'::regclass +) RETURNING placementid; + placementid +------------- + 152 +(1 row) + +TRUNCATE researchers; +-- fail at release +SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").kill()'); + mitmproxy +----------- + +(1 row) + +BEGIN; +INSERT INTO researchers VALUES (7, 4, 'Jan Plaza'); +SAVEPOINT s1; +INSERT INTO researchers VALUES (8, 4, 'Alonzo Church'); +ROLLBACK TO s1; +RELEASE SAVEPOINT s1; +WARNING: connection not open +WARNING: connection not open +COMMIT; +WARNING: connection not open +WARNING: connection not open +WARNING: connection not open +-- should see correct results from healthy placement and one bad placement +SELECT * FROM researchers WHERE lab_id = 4; + id | lab_id | name +----+--------+----------- + 7 | 4 | Jan Plaza +(1 row) + +UPDATE pg_dist_shard_placement SET shardstate = 1 +WHERE shardstate = 3 AND shardid IN ( + SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'researchers'::regclass +) RETURNING placementid; + placementid +------------- + 152 +(1 row) + +TRUNCATE researchers; +-- clean up +SELECT citus.mitmproxy('conn.allow()'); + mitmproxy +----------- + +(1 row) + +DROP TABLE artists; +DROP TABLE researchers; diff --git a/src/test/regress/failure_schedule b/src/test/regress/failure_schedule index d32732b22..ca5b491f2 100644 --- a/src/test/regress/failure_schedule +++ b/src/test/regress/failure_schedule @@ -27,3 +27,4 @@ test: failure_ref_tables test: failure_real_time_select test: failure_insert_select_pushdown test: failure_single_mod +test: failure_savepoints diff --git a/src/test/regress/sql/failure_savepoints.sql b/src/test/regress/sql/failure_savepoints.sql new file mode 100644 index 000000000..6f7550911 --- /dev/null +++ b/src/test/regress/sql/failure_savepoints.sql @@ -0,0 +1,188 @@ +SELECT citus.mitmproxy('conn.allow()'); + +SET citus.shard_count = 2; +SET citus.shard_replication_factor = 1; -- one shard per worker +SET citus.next_shard_id TO 100950; +ALTER SEQUENCE pg_catalog.pg_dist_placement_placementid_seq RESTART 150; + +CREATE TABLE artists ( + id bigint NOT NULL, + name text NOT NULL +); +SELECT create_distributed_table('artists', 'id'); + +-- add some data +INSERT INTO artists VALUES (1, 'Pablo Picasso'); +INSERT INTO artists VALUES (2, 'Vincent van Gogh'); +INSERT INTO artists VALUES (3, 'Claude Monet'); +INSERT INTO artists VALUES (4, 'William Kurelek'); + +-- simply fail at SAVEPOINT +SELECT citus.mitmproxy('conn.onQuery(query="^SAVEPOINT").kill()'); + +BEGIN; +INSERT INTO artists VALUES (5, 'Asher Lev'); +SAVEPOINT s1; +DELETE FROM artists WHERE id=4; +RELEASE SAVEPOINT s1; +COMMIT; + +SELECT * FROM artists WHERE id IN (4, 5); + +-- fail at RELEASE +SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").kill()'); + +BEGIN; +INSERT INTO artists VALUES (5, 'Asher Lev'); +SAVEPOINT s1; +DELETE FROM artists WHERE id=4; +RELEASE SAVEPOINT s1; +COMMIT; + +SELECT * FROM artists WHERE id IN (4, 5); + +-- fail at ROLLBACK +SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()'); + +BEGIN; +INSERT INTO artists VALUES (5, 'Asher Lev'); +SAVEPOINT s1; +DELETE FROM artists WHERE id=4; +ROLLBACK TO SAVEPOINT s1; +COMMIT; + +SELECT * FROM artists WHERE id IN (4, 5); + +-- fail at second RELEASE +SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").after(1).kill()'); +BEGIN; +SAVEPOINT s1; +DELETE FROM artists WHERE id=4; +RELEASE SAVEPOINT s1; +SAVEPOINT s2; +INSERT INTO artists VALUES (5, 'Jacob Kahn'); +RELEASE SAVEPOINT s2; +COMMIT; + +SELECT * FROM artists WHERE id IN (4, 5); + +-- fail at second ROLLBACK +SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").after(1).kill()'); + +BEGIN; +SAVEPOINT s1; +UPDATE artists SET name='A' WHERE id=4; +ROLLBACK TO SAVEPOINT s1; +SAVEPOINT s2; +DELETE FROM artists WHERE id=5; +ROLLBACK TO SAVEPOINT s2; +COMMIT; + +SELECT * FROM artists WHERE id IN (4, 5); + +SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").after(1).kill()'); + +-- Release after rollback +BEGIN; +SAVEPOINT s1; +ROLLBACK TO s1; +RELEASE SAVEPOINT s1; +SAVEPOINT s2; +INSERT INTO artists VALUES (6, 'John J. Audubon'); +INSERT INTO artists VALUES (7, 'Emily Carr'); +ROLLBACK TO s2; +RELEASE SAVEPOINT s2; +COMMIT; + +SELECT * FROM artists WHERE id=7; + +SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()'); + +-- Recover from errors +\set VERBOSITY terse +BEGIN; +SAVEPOINT s1; +SAVEPOINT s2; +INSERT INTO artists VALUES (6, 'John J. Audubon'); +INSERT INTO artists VALUES (7, 'Emily Carr'); +INSERT INTO artists VALUES (7, 'Emily Carr'); +ROLLBACK TO SAVEPOINT s1; +COMMIT; + +SELECT * FROM artists WHERE id=6; + +-- replication factor > 1 +CREATE TABLE researchers ( + id bigint NOT NULL, + lab_id int NOT NULL, + name text NOT NULL +); +SET citus.shard_count = 1; +SET citus.shard_replication_factor = 2; -- single shard, on both workers +SELECT create_distributed_table('researchers', 'lab_id', 'hash'); + + +-- simply fail at SAVEPOINT +SELECT citus.mitmproxy('conn.onQuery(query="^SAVEPOINT").kill()'); + +BEGIN; +INSERT INTO researchers VALUES (7, 4, 'Jan Plaza'); +SAVEPOINT s1; +INSERT INTO researchers VALUES (8, 4, 'Alonzo Church'); +ROLLBACK TO s1; +RELEASE SAVEPOINT s1; +COMMIT; + +-- should see correct results from healthy placement and one bad placement +SELECT * FROM researchers WHERE lab_id = 4; + +UPDATE pg_dist_shard_placement SET shardstate = 1 +WHERE shardstate = 3 AND shardid IN ( + SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'researchers'::regclass +) RETURNING placementid; +TRUNCATE researchers; + +-- fail at rollback +SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()'); + +BEGIN; +INSERT INTO researchers VALUES (7, 4, 'Jan Plaza'); +SAVEPOINT s1; +INSERT INTO researchers VALUES (8, 4, 'Alonzo Church'); +ROLLBACK TO s1; +RELEASE SAVEPOINT s1; +COMMIT; + +-- should see correct results from healthy placement and one bad placement +SELECT * FROM researchers WHERE lab_id = 4; + +UPDATE pg_dist_shard_placement SET shardstate = 1 +WHERE shardstate = 3 AND shardid IN ( + SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'researchers'::regclass +) RETURNING placementid; +TRUNCATE researchers; + +-- fail at release +SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").kill()'); + +BEGIN; +INSERT INTO researchers VALUES (7, 4, 'Jan Plaza'); +SAVEPOINT s1; +INSERT INTO researchers VALUES (8, 4, 'Alonzo Church'); +ROLLBACK TO s1; +RELEASE SAVEPOINT s1; +COMMIT; + +-- should see correct results from healthy placement and one bad placement +SELECT * FROM researchers WHERE lab_id = 4; + +UPDATE pg_dist_shard_placement SET shardstate = 1 +WHERE shardstate = 3 AND shardid IN ( + SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'researchers'::regclass +) RETURNING placementid; +TRUNCATE researchers; + +-- clean up +SELECT citus.mitmproxy('conn.allow()'); +DROP TABLE artists; +DROP TABLE researchers;