mirror of https://github.com/citusdata/citus.git
608 lines
20 KiB
Plaintext
608 lines
20 KiB
Plaintext
-- We have two different output files for this failure test because the
|
|
-- failure behaviour of SAVEPOINT and RELEASE commands are different if
|
|
-- we use the executor. If we use it, these commands error out if any of
|
|
-- the placement commands fail. Otherwise, we might mark the placement
|
|
-- as invalid and continue with a WARNING.
|
|
SELECT citus.mitmproxy('conn.allow()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SET citus.shard_count = 2;
|
|
SET citus.shard_replication_factor = 1; -- one shard per worker
|
|
SET citus.next_shard_id TO 100950;
|
|
ALTER SEQUENCE pg_catalog.pg_dist_placement_placementid_seq RESTART 150;
|
|
CREATE TABLE artists (
|
|
id bigint NOT NULL,
|
|
name text NOT NULL
|
|
);
|
|
SELECT create_distributed_table('artists', 'id');
|
|
create_distributed_table
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
-- add some data
|
|
INSERT INTO artists VALUES (1, 'Pablo Picasso');
|
|
INSERT INTO artists VALUES (2, 'Vincent van Gogh');
|
|
INSERT INTO artists VALUES (3, 'Claude Monet');
|
|
INSERT INTO artists VALUES (4, 'William Kurelek');
|
|
-- simply fail at SAVEPOINT
|
|
SELECT citus.mitmproxy('conn.onQuery(query="^SAVEPOINT").kill()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
BEGIN;
|
|
INSERT INTO artists VALUES (5, 'Asher Lev');
|
|
SAVEPOINT s1;
|
|
WARNING: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
CONTEXT: while executing command on localhost:xxxxx
|
|
WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
WARNING: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
CONTEXT: while executing command on localhost:xxxxx
|
|
WARNING: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
CONTEXT: while executing command on localhost:xxxxx
|
|
ERROR: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
CONTEXT: while executing command on localhost:xxxxx
|
|
DELETE FROM artists WHERE id=4;
|
|
ERROR: current transaction is aborted, commands ignored until end of transaction block
|
|
RELEASE SAVEPOINT s1;
|
|
ERROR: current transaction is aborted, commands ignored until end of transaction block
|
|
COMMIT;
|
|
SELECT * FROM artists WHERE id IN (4, 5);
|
|
id | name
|
|
---------------------------------------------------------------------
|
|
4 | William Kurelek
|
|
(1 row)
|
|
|
|
-- fail at RELEASE
|
|
SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").kill()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
BEGIN;
|
|
UPDATE artists SET name='a';
|
|
SAVEPOINT s1;
|
|
DELETE FROM artists WHERE id=4;
|
|
RELEASE SAVEPOINT s1;
|
|
WARNING: AbortSubTransaction while in COMMIT state
|
|
WARNING: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
CONTEXT: while executing command on localhost:xxxxx
|
|
WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
WARNING: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
CONTEXT: while executing command on localhost:xxxxx
|
|
WARNING: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
CONTEXT: while executing command on localhost:xxxxx
|
|
WARNING: savepoint "savepoint_2" does not exist
|
|
CONTEXT: while executing command on localhost:xxxxx
|
|
ERROR: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
CONTEXT: while executing command on localhost:xxxxx
|
|
ROLLBACK;
|
|
SELECT * FROM artists WHERE id IN (4, 5);
|
|
id | name
|
|
---------------------------------------------------------------------
|
|
4 | William Kurelek
|
|
(1 row)
|
|
|
|
-- fail at ROLLBACK
|
|
SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
BEGIN;
|
|
INSERT INTO artists VALUES (5, 'Asher Lev');
|
|
SAVEPOINT s1;
|
|
DELETE FROM artists WHERE id=4;
|
|
ROLLBACK TO SAVEPOINT s1;
|
|
WARNING: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
CONTEXT: while executing command on localhost:xxxxx
|
|
WARNING: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
connection not open
|
|
CONTEXT: while executing command on localhost:xxxxx
|
|
COMMIT;
|
|
ERROR: could not make changes to shard xxxxx on any node
|
|
SELECT * FROM artists WHERE id IN (4, 5);
|
|
id | name
|
|
---------------------------------------------------------------------
|
|
4 | William Kurelek
|
|
(1 row)
|
|
|
|
-- fail at second RELEASE
|
|
SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").after(1).kill()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
BEGIN;
|
|
SAVEPOINT s1;
|
|
DELETE FROM artists WHERE id=4;
|
|
RELEASE SAVEPOINT s1;
|
|
SAVEPOINT s2;
|
|
INSERT INTO artists VALUES (5, 'Jacob Kahn');
|
|
RELEASE SAVEPOINT s2;
|
|
WARNING: AbortSubTransaction while in COMMIT state
|
|
WARNING: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
CONTEXT: while executing command on localhost:xxxxx
|
|
WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
WARNING: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
CONTEXT: while executing command on localhost:xxxxx
|
|
WARNING: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
CONTEXT: while executing command on localhost:xxxxx
|
|
ERROR: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
CONTEXT: while executing command on localhost:xxxxx
|
|
COMMIT;
|
|
SELECT * FROM artists WHERE id IN (4, 5);
|
|
id | name
|
|
---------------------------------------------------------------------
|
|
4 | William Kurelek
|
|
(1 row)
|
|
|
|
-- fail at second ROLLBACK
|
|
SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").after(1).kill()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
BEGIN;
|
|
SAVEPOINT s1;
|
|
UPDATE artists SET name='A' WHERE id=4;
|
|
ROLLBACK TO SAVEPOINT s1;
|
|
SAVEPOINT s2;
|
|
DELETE FROM artists WHERE id=5;
|
|
ROLLBACK TO SAVEPOINT s2;
|
|
WARNING: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
CONTEXT: while executing command on localhost:xxxxx
|
|
WARNING: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
connection not open
|
|
CONTEXT: while executing command on localhost:xxxxx
|
|
COMMIT;
|
|
ERROR: could not make changes to shard xxxxx on any node
|
|
SELECT * FROM artists WHERE id IN (4, 5);
|
|
id | name
|
|
---------------------------------------------------------------------
|
|
4 | William Kurelek
|
|
(1 row)
|
|
|
|
SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").after(1).kill()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
-- Release after rollback
|
|
BEGIN;
|
|
SAVEPOINT s1;
|
|
ROLLBACK TO s1;
|
|
RELEASE SAVEPOINT s1;
|
|
SAVEPOINT s2;
|
|
INSERT INTO artists VALUES (6, 'John J. Audubon');
|
|
INSERT INTO artists VALUES (7, 'Emily Carr');
|
|
ROLLBACK TO s2;
|
|
RELEASE SAVEPOINT s2;
|
|
COMMIT;
|
|
SELECT * FROM artists WHERE id=7;
|
|
id | name
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
-- Recover from errors
|
|
\set VERBOSITY terse
|
|
BEGIN;
|
|
SAVEPOINT s1;
|
|
SAVEPOINT s2;
|
|
INSERT INTO artists VALUES (6, 'John J. Audubon');
|
|
INSERT INTO artists VALUES (7, 'Emily Carr');
|
|
INSERT INTO artists VALUES (7, 'Emily Carr');
|
|
ROLLBACK TO SAVEPOINT s1;
|
|
WARNING: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
WARNING: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
connection not open
|
|
WARNING: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
WARNING: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
WARNING: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
COMMIT;
|
|
ERROR: could not make changes to shard xxxxx on any node
|
|
SELECT * FROM artists WHERE id=6;
|
|
id | name
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
-- replication factor > 1
|
|
CREATE TABLE researchers (
|
|
id bigint NOT NULL,
|
|
lab_id int NOT NULL,
|
|
name text NOT NULL
|
|
);
|
|
SET citus.shard_count = 1;
|
|
SET citus.shard_replication_factor = 2; -- single shard, on both workers
|
|
SELECT create_distributed_table('researchers', 'lab_id', 'hash');
|
|
create_distributed_table
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
-- simply fail at SAVEPOINT
|
|
SELECT citus.mitmproxy('conn.onQuery(query="^SAVEPOINT").kill()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
BEGIN;
|
|
INSERT INTO researchers VALUES (7, 4, 'Jan Plaza');
|
|
SAVEPOINT s1;
|
|
WARNING: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
WARNING: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
WARNING: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
ERROR: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
INSERT INTO researchers VALUES (8, 4, 'Alonzo Church');
|
|
ERROR: current transaction is aborted, commands ignored until end of transaction block
|
|
ROLLBACK TO s1;
|
|
ERROR: savepoint "s1" does not exist
|
|
RELEASE SAVEPOINT s1;
|
|
ERROR: current transaction is aborted, commands ignored until end of transaction block
|
|
COMMIT;
|
|
-- should see correct results from healthy placement and one bad placement
|
|
SELECT * FROM researchers WHERE lab_id = 4;
|
|
id | lab_id | name
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
UPDATE pg_dist_shard_placement SET shardstate = 1
|
|
WHERE shardstate = 3 AND shardid IN (
|
|
SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'researchers'::regclass
|
|
) RETURNING placementid;
|
|
placementid
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
TRUNCATE researchers;
|
|
-- fail at rollback
|
|
SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
BEGIN;
|
|
INSERT INTO researchers VALUES (7, 4, 'Jan Plaza');
|
|
SAVEPOINT s1;
|
|
INSERT INTO researchers VALUES (8, 4, 'Alonzo Church');
|
|
ROLLBACK TO s1;
|
|
WARNING: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
WARNING: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
connection not open
|
|
RELEASE SAVEPOINT s1;
|
|
COMMIT;
|
|
ERROR: failure on connection marked as essential: localhost:xxxxx
|
|
-- should see correct results from healthy placement and one bad placement
|
|
SELECT * FROM researchers WHERE lab_id = 4;
|
|
id | lab_id | name
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
UPDATE pg_dist_shard_placement SET shardstate = 1
|
|
WHERE shardstate = 3 AND shardid IN (
|
|
SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'researchers'::regclass
|
|
) RETURNING placementid;
|
|
placementid
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
TRUNCATE researchers;
|
|
-- fail at release
|
|
SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").kill()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
BEGIN;
|
|
INSERT INTO researchers VALUES (7, 4, 'Jan Plaza');
|
|
SAVEPOINT s1;
|
|
INSERT INTO researchers VALUES (8, 4, 'Alonzo Church');
|
|
ROLLBACK TO s1;
|
|
RELEASE SAVEPOINT s1;
|
|
WARNING: AbortSubTransaction while in COMMIT state
|
|
WARNING: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
WARNING: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
WARNING: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
WARNING: savepoint "savepoint_3" does not exist
|
|
ERROR: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
COMMIT;
|
|
-- should see correct results from healthy placement and one bad placement
|
|
SELECT * FROM researchers WHERE lab_id = 4;
|
|
id | lab_id | name
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
UPDATE pg_dist_shard_placement SET shardstate = 1
|
|
WHERE shardstate = 3 AND shardid IN (
|
|
SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'researchers'::regclass
|
|
) RETURNING placementid;
|
|
placementid
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
TRUNCATE researchers;
|
|
-- test that we don't mark reference placements unhealthy
|
|
CREATE TABLE ref(a int, b int);
|
|
SELECT create_reference_table('ref');
|
|
create_reference_table
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
BEGIN;
|
|
SAVEPOINT start;
|
|
INSERT INTO ref VALUES (1001,2);
|
|
SELECT * FROM ref;
|
|
a | b
|
|
---------------------------------------------------------------------
|
|
1001 | 2
|
|
(1 row)
|
|
|
|
ROLLBACK TO SAVEPOINT start;
|
|
WARNING: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
WARNING: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
connection not open
|
|
SELECT * FROM ref;
|
|
WARNING: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
WARNING: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
WARNING: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
WARNING: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
connection not open
|
|
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
|
|
This probably means the server terminated abnormally
|
|
before or while processing the request.
|
|
connection not open
|
|
connection not open
|
|
END;
|
|
-- clean up
|
|
SELECT citus.mitmproxy('conn.allow()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
DROP TABLE artists;
|
|
DROP TABLE researchers;
|
|
DROP TABLE ref;
|