external
/
citus
mirror of https://github.com/citusdata/citus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2440 from citusdata/savepoint_failures 

Add savepoint failure tests
pull/2458/head
Nils Dijk 2018-10-26 11:59:37 +01:00 committed by GitHub
parent 0f768349e0 a37a809d49
commit 13226a9a3b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 528 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

339
src/test/regress/expected/failure_savepoints.out Normal file
View File

@ -0,0 +1,339 @@
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
-----------
(1 row)
SET citus.shard_count = 2;
SET citus.shard_replication_factor = 1; -- one shard per worker
SET citus.next_shard_id TO 100950;
ALTER SEQUENCE pg_catalog.pg_dist_placement_placementid_seq RESTART 150;
CREATE TABLE artists (
id bigint NOT NULL,
name text NOT NULL
);
SELECT create_distributed_table('artists', 'id');
create_distributed_table
--------------------------
(1 row)
-- add some data
INSERT INTO artists VALUES (1, 'Pablo Picasso');
INSERT INTO artists VALUES (2, 'Vincent van Gogh');
INSERT INTO artists VALUES (3, 'Claude Monet');
INSERT INTO artists VALUES (4, 'William Kurelek');
-- simply fail at SAVEPOINT
SELECT citus.mitmproxy('conn.onQuery(query="^SAVEPOINT").kill()');
mitmproxy
-----------
(1 row)
BEGIN;
INSERT INTO artists VALUES (5, 'Asher Lev');
SAVEPOINT s1;
WARNING: connection not open
CONTEXT: while executing command on localhost:9060
WARNING: connection not open
CONTEXT: while executing command on localhost:9060
DELETE FROM artists WHERE id=4;
WARNING: connection not open
CONTEXT: while executing command on localhost:9060
WARNING: connection error: localhost:9060
DETAIL: connection not open
WARNING: connection not open
CONTEXT: while executing command on localhost:9060
WARNING: connection not open
CONTEXT: while executing command on localhost:9060
ERROR: could not modify any active placements
RELEASE SAVEPOINT s1;
ERROR: current transaction is aborted, commands ignored until end of transaction block
COMMIT;
SELECT * FROM artists WHERE id IN (4, 5);
id | name
----+-----------------
4 | William Kurelek
(1 row)
-- fail at RELEASE
SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").kill()');
mitmproxy
-----------
(1 row)
BEGIN;
INSERT INTO artists VALUES (5, 'Asher Lev');
SAVEPOINT s1;
DELETE FROM artists WHERE id=4;
RELEASE SAVEPOINT s1;
WARNING: connection not open
CONTEXT: while executing command on localhost:9060
WARNING: connection not open
CONTEXT: while executing command on localhost:9060
COMMIT;
ERROR: could not make changes to shard 100950 on any node
SELECT * FROM artists WHERE id IN (4, 5);
id | name
----+-----------------
4 | William Kurelek
(1 row)
-- fail at ROLLBACK
SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()');
mitmproxy
-----------
(1 row)
BEGIN;
INSERT INTO artists VALUES (5, 'Asher Lev');
SAVEPOINT s1;
DELETE FROM artists WHERE id=4;
ROLLBACK TO SAVEPOINT s1;
WARNING: connection not open
CONTEXT: while executing command on localhost:9060
WARNING: connection not open
CONTEXT: while executing command on localhost:9060
COMMIT;
ERROR: could not make changes to shard 100950 on any node
SELECT * FROM artists WHERE id IN (4, 5);
id | name
----+-----------------
4 | William Kurelek
(1 row)
-- fail at second RELEASE
SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").after(1).kill()');
mitmproxy
-----------
(1 row)
BEGIN;
SAVEPOINT s1;
DELETE FROM artists WHERE id=4;
RELEASE SAVEPOINT s1;
SAVEPOINT s2;
INSERT INTO artists VALUES (5, 'Jacob Kahn');
RELEASE SAVEPOINT s2;
WARNING: connection not open
CONTEXT: while executing command on localhost:9060
WARNING: connection not open
CONTEXT: while executing command on localhost:9060
COMMIT;
ERROR: could not make changes to shard 100950 on any node
SELECT * FROM artists WHERE id IN (4, 5);
id | name
----+-----------------
4 | William Kurelek
(1 row)
-- fail at second ROLLBACK
SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").after(1).kill()');
mitmproxy
-----------
(1 row)
BEGIN;
SAVEPOINT s1;
UPDATE artists SET name='A' WHERE id=4;
ROLLBACK TO SAVEPOINT s1;
SAVEPOINT s2;
DELETE FROM artists WHERE id=5;
ROLLBACK TO SAVEPOINT s2;
WARNING: connection not open
CONTEXT: while executing command on localhost:9060
WARNING: connection not open
CONTEXT: while executing command on localhost:9060
COMMIT;
ERROR: could not make changes to shard 100950 on any node
SELECT * FROM artists WHERE id IN (4, 5);
id | name
----+-----------------
4 | William Kurelek
(1 row)
SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").after(1).kill()');
mitmproxy
-----------
(1 row)
-- Release after rollback
BEGIN;
SAVEPOINT s1;
ROLLBACK TO s1;
RELEASE SAVEPOINT s1;
SAVEPOINT s2;
INSERT INTO artists VALUES (6, 'John J. Audubon');
INSERT INTO artists VALUES (7, 'Emily Carr');
ROLLBACK TO s2;
RELEASE SAVEPOINT s2;
COMMIT;
SELECT * FROM artists WHERE id=7;
id | name
----+------
(0 rows)
SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()');
mitmproxy
-----------
(1 row)
-- Recover from errors
\set VERBOSITY terse
BEGIN;
SAVEPOINT s1;
SAVEPOINT s2;
INSERT INTO artists VALUES (6, 'John J. Audubon');
INSERT INTO artists VALUES (7, 'Emily Carr');
INSERT INTO artists VALUES (7, 'Emily Carr');
ROLLBACK TO SAVEPOINT s1;
WARNING: connection not open
WARNING: connection not open
COMMIT;
ERROR: could not make changes to shard 100950 on any node
SELECT * FROM artists WHERE id=6;
id | name
----+------
(0 rows)
-- replication factor > 1
CREATE TABLE researchers (
id bigint NOT NULL,
lab_id int NOT NULL,
name text NOT NULL
);
SET citus.shard_count = 1;
SET citus.shard_replication_factor = 2; -- single shard, on both workers
SELECT create_distributed_table('researchers', 'lab_id', 'hash');
create_distributed_table
--------------------------
(1 row)
-- simply fail at SAVEPOINT
SELECT citus.mitmproxy('conn.onQuery(query="^SAVEPOINT").kill()');
mitmproxy
-----------
(1 row)
BEGIN;
INSERT INTO researchers VALUES (7, 4, 'Jan Plaza');
SAVEPOINT s1;
WARNING: connection not open
WARNING: connection not open
INSERT INTO researchers VALUES (8, 4, 'Alonzo Church');
ROLLBACK TO s1;
WARNING: connection not open
WARNING: connection error: localhost:9060
WARNING: connection not open
WARNING: connection not open
RELEASE SAVEPOINT s1;
COMMIT;
WARNING: connection not open
WARNING: connection not open
WARNING: connection not open
-- should see correct results from healthy placement and one bad placement
SELECT * FROM researchers WHERE lab_id = 4;
id | lab_id | name
----+--------+-----------
7 | 4 | Jan Plaza
(1 row)
UPDATE pg_dist_shard_placement SET shardstate = 1
WHERE shardstate = 3 AND shardid IN (
SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'researchers'::regclass
) RETURNING placementid;
placementid
-------------
152
(1 row)
TRUNCATE researchers;
-- fail at rollback
SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()');
mitmproxy
-----------
(1 row)
BEGIN;
INSERT INTO researchers VALUES (7, 4, 'Jan Plaza');
SAVEPOINT s1;
INSERT INTO researchers VALUES (8, 4, 'Alonzo Church');
ROLLBACK TO s1;
WARNING: connection not open
WARNING: connection not open
RELEASE SAVEPOINT s1;
COMMIT;
WARNING: connection not open
WARNING: connection not open
WARNING: connection not open
-- should see correct results from healthy placement and one bad placement
SELECT * FROM researchers WHERE lab_id = 4;
id | lab_id | name
----+--------+-----------
7 | 4 | Jan Plaza
(1 row)
UPDATE pg_dist_shard_placement SET shardstate = 1
WHERE shardstate = 3 AND shardid IN (
SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'researchers'::regclass
) RETURNING placementid;
placementid
-------------
152
(1 row)
TRUNCATE researchers;
-- fail at release
SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").kill()');
mitmproxy
-----------
(1 row)
BEGIN;
INSERT INTO researchers VALUES (7, 4, 'Jan Plaza');
SAVEPOINT s1;
INSERT INTO researchers VALUES (8, 4, 'Alonzo Church');
ROLLBACK TO s1;
RELEASE SAVEPOINT s1;
WARNING: connection not open
WARNING: connection not open
COMMIT;
WARNING: connection not open
WARNING: connection not open
WARNING: connection not open
-- should see correct results from healthy placement and one bad placement
SELECT * FROM researchers WHERE lab_id = 4;
id | lab_id | name
----+--------+-----------
7 | 4 | Jan Plaza
(1 row)
UPDATE pg_dist_shard_placement SET shardstate = 1
WHERE shardstate = 3 AND shardid IN (
SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'researchers'::regclass
) RETURNING placementid;
placementid
-------------
152
(1 row)
TRUNCATE researchers;
-- clean up
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
-----------
(1 row)
DROP TABLE artists;
DROP TABLE researchers;

1
src/test/regress/failure_schedule
View File

@ -27,3 +27,4 @@ test: failure_ref_tables
test: failure_real_time_select test: failure_real_time_select
test: failure_insert_select_pushdown test: failure_insert_select_pushdown
test: failure_single_mod test: failure_single_mod
test: failure_savepoints

188
src/test/regress/sql/failure_savepoints.sql Normal file
View File

@ -0,0 +1,188 @@
SELECT citus.mitmproxy('conn.allow()');
SET citus.shard_count = 2;
SET citus.shard_replication_factor = 1; -- one shard per worker
SET citus.next_shard_id TO 100950;
ALTER SEQUENCE pg_catalog.pg_dist_placement_placementid_seq RESTART 150;
CREATE TABLE artists (
id bigint NOT NULL,
name text NOT NULL
);
SELECT create_distributed_table('artists', 'id');
-- add some data
INSERT INTO artists VALUES (1, 'Pablo Picasso');
INSERT INTO artists VALUES (2, 'Vincent van Gogh');
INSERT INTO artists VALUES (3, 'Claude Monet');
INSERT INTO artists VALUES (4, 'William Kurelek');
-- simply fail at SAVEPOINT
SELECT citus.mitmproxy('conn.onQuery(query="^SAVEPOINT").kill()');
BEGIN;
INSERT INTO artists VALUES (5, 'Asher Lev');
SAVEPOINT s1;
DELETE FROM artists WHERE id=4;
RELEASE SAVEPOINT s1;
COMMIT;
SELECT * FROM artists WHERE id IN (4, 5);
-- fail at RELEASE
SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").kill()');
BEGIN;
INSERT INTO artists VALUES (5, 'Asher Lev');
SAVEPOINT s1;
DELETE FROM artists WHERE id=4;
RELEASE SAVEPOINT s1;
COMMIT;
SELECT * FROM artists WHERE id IN (4, 5);
-- fail at ROLLBACK
SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()');
BEGIN;
INSERT INTO artists VALUES (5, 'Asher Lev');
SAVEPOINT s1;
DELETE FROM artists WHERE id=4;
ROLLBACK TO SAVEPOINT s1;
COMMIT;
SELECT * FROM artists WHERE id IN (4, 5);
-- fail at second RELEASE
SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").after(1).kill()');
BEGIN;
SAVEPOINT s1;
DELETE FROM artists WHERE id=4;
RELEASE SAVEPOINT s1;
SAVEPOINT s2;
INSERT INTO artists VALUES (5, 'Jacob Kahn');
RELEASE SAVEPOINT s2;
COMMIT;
SELECT * FROM artists WHERE id IN (4, 5);
-- fail at second ROLLBACK
SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").after(1).kill()');
BEGIN;
SAVEPOINT s1;
UPDATE artists SET name='A' WHERE id=4;
ROLLBACK TO SAVEPOINT s1;
SAVEPOINT s2;
DELETE FROM artists WHERE id=5;
ROLLBACK TO SAVEPOINT s2;
COMMIT;
SELECT * FROM artists WHERE id IN (4, 5);
SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").after(1).kill()');
-- Release after rollback
BEGIN;
SAVEPOINT s1;
ROLLBACK TO s1;
RELEASE SAVEPOINT s1;
SAVEPOINT s2;
INSERT INTO artists VALUES (6, 'John J. Audubon');
INSERT INTO artists VALUES (7, 'Emily Carr');
ROLLBACK TO s2;
RELEASE SAVEPOINT s2;
COMMIT;
SELECT * FROM artists WHERE id=7;
SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()');
-- Recover from errors
\set VERBOSITY terse
BEGIN;
SAVEPOINT s1;
SAVEPOINT s2;
INSERT INTO artists VALUES (6, 'John J. Audubon');
INSERT INTO artists VALUES (7, 'Emily Carr');
INSERT INTO artists VALUES (7, 'Emily Carr');
ROLLBACK TO SAVEPOINT s1;
COMMIT;
SELECT * FROM artists WHERE id=6;
-- replication factor > 1
CREATE TABLE researchers (
id bigint NOT NULL,
lab_id int NOT NULL,
name text NOT NULL
);
SET citus.shard_count = 1;
SET citus.shard_replication_factor = 2; -- single shard, on both workers
SELECT create_distributed_table('researchers', 'lab_id', 'hash');
-- simply fail at SAVEPOINT
SELECT citus.mitmproxy('conn.onQuery(query="^SAVEPOINT").kill()');
BEGIN;
INSERT INTO researchers VALUES (7, 4, 'Jan Plaza');
SAVEPOINT s1;
INSERT INTO researchers VALUES (8, 4, 'Alonzo Church');
ROLLBACK TO s1;
RELEASE SAVEPOINT s1;
COMMIT;
-- should see correct results from healthy placement and one bad placement
SELECT * FROM researchers WHERE lab_id = 4;
UPDATE pg_dist_shard_placement SET shardstate = 1
WHERE shardstate = 3 AND shardid IN (
SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'researchers'::regclass
) RETURNING placementid;
TRUNCATE researchers;
-- fail at rollback
SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()');
BEGIN;
INSERT INTO researchers VALUES (7, 4, 'Jan Plaza');
SAVEPOINT s1;
INSERT INTO researchers VALUES (8, 4, 'Alonzo Church');
ROLLBACK TO s1;
RELEASE SAVEPOINT s1;
COMMIT;
-- should see correct results from healthy placement and one bad placement
SELECT * FROM researchers WHERE lab_id = 4;
UPDATE pg_dist_shard_placement SET shardstate = 1
WHERE shardstate = 3 AND shardid IN (
SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'researchers'::regclass
) RETURNING placementid;
TRUNCATE researchers;
-- fail at release
SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").kill()');
BEGIN;
INSERT INTO researchers VALUES (7, 4, 'Jan Plaza');
SAVEPOINT s1;
INSERT INTO researchers VALUES (8, 4, 'Alonzo Church');
ROLLBACK TO s1;
RELEASE SAVEPOINT s1;
COMMIT;
-- should see correct results from healthy placement and one bad placement
SELECT * FROM researchers WHERE lab_id = 4;
UPDATE pg_dist_shard_placement SET shardstate = 1
WHERE shardstate = 3 AND shardid IN (
SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'researchers'::regclass
) RETURNING placementid;
TRUNCATE researchers;
-- clean up
SELECT citus.mitmproxy('conn.allow()');
DROP TABLE artists;
DROP TABLE researchers;