external
/
citus
mirror of https://github.com/citusdata/citus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add output files for failure tests on libpq 13.4

pull/6049/head
Nils Dijk 2022-07-06 14:12:00 +02:00
parent e14c9d400a
commit 07c1170893
No known key found for this signature in database
GPG Key ID: CA1177EF9434F241
12 changed files with 6438 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.circleci/config.yml
View File

@ -6,7 +6,7 @@ orbs:
parameters: parameters:
image_suffix: image_suffix:
type: string type: string
default: '-dev-b07a192' default: '-dev-aa7ace1'
pg13_version: pg13_version:
type: string type: string
default: '13.4' default: '13.4'

396
src/test/regress/expected/failure_copy_on_hash_1.out Normal file
View File

@ -0,0 +1,396 @@
--
-- Failure tests for COPY to hash distributed tables
--
CREATE SCHEMA copy_distributed_table;
SET search_path TO 'copy_distributed_table';
SET citus.next_shard_id TO 1710000;
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
-- With one placement COPY should error out and placement should stay healthy.
SET citus.shard_replication_factor TO 1;
SET citus.shard_count to 4;
SET citus.max_cached_conns_per_worker to 0;
CREATE TABLE test_table(id int, value_1 int);
SELECT create_distributed_table('test_table','id');
create_distributed_table
---------------------------------------------------------------------
(1 row)
CREATE VIEW unhealthy_shard_count AS
SELECT count(*)
FROM pg_dist_shard_placement pdsp
JOIN
pg_dist_shard pds
ON pdsp.shardid=pds.shardid
WHERE logicalrelid='copy_distributed_table.test_table'::regclass AND shardstate != 1;
-- Just kill the connection after sending the first query to the worker.
SELECT citus.mitmproxy('conn.kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
\COPY test_table FROM stdin delimiter ',';
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
CONTEXT: COPY test_table, line 1: "1,2"
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT * FROM unhealthy_shard_count;
count
---------------------------------------------------------------------
0
(1 row)
SELECT count(*) FROM test_table;
count
---------------------------------------------------------------------
0
(1 row)
-- Now, kill the connection while copying the data
SELECT citus.mitmproxy('conn.onCopyData().kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
\COPY test_table FROM stdin delimiter ',';
ERROR: failed to COPY to shard xxxxx on localhost:xxxxx
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT * FROM unhealthy_shard_count;
count
---------------------------------------------------------------------
0
(1 row)
SELECT count(*) FROM test_table;
count
---------------------------------------------------------------------
0
(1 row)
-- Similar to the above one, but now cancel the connection
-- instead of killing it.
SELECT citus.mitmproxy('conn.onCopyData().cancel(' || pg_backend_pid() || ')');
mitmproxy
---------------------------------------------------------------------
(1 row)
\COPY test_table FROM stdin delimiter ',';
ERROR: canceling statement due to user request
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT * FROM unhealthy_shard_count;
count
---------------------------------------------------------------------
0
(1 row)
SELECT count(*) FROM test_table;
count
---------------------------------------------------------------------
0
(1 row)
-- kill the connection after worker sends command complete message
SELECT citus.mitmproxy('conn.onCommandComplete(command="COPY 1").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
\COPY test_table FROM stdin delimiter ',';
ERROR: failed to COPY to shard xxxxx on localhost:xxxxx
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT * FROM unhealthy_shard_count;
count
---------------------------------------------------------------------
0
(1 row)
SELECT count(*) FROM test_table;
count
---------------------------------------------------------------------
0
(1 row)
-- similar to above one, but cancel the connection on command complete
SELECT citus.mitmproxy('conn.onCommandComplete(command="COPY 1").cancel(' || pg_backend_pid() || ')');
mitmproxy
---------------------------------------------------------------------
(1 row)
\COPY test_table FROM stdin delimiter ',';
ERROR: canceling statement due to user request
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT * FROM unhealthy_shard_count;
count
---------------------------------------------------------------------
0
(1 row)
SELECT count(*) FROM test_table;
count
---------------------------------------------------------------------
0
(1 row)
-- kill the connection on PREPARE TRANSACTION
SELECT citus.mitmproxy('conn.onQuery(query="PREPARE TRANSACTION").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
\COPY test_table FROM stdin delimiter ',';
ERROR: connection not open
CONTEXT: while executing command on localhost:xxxxx
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT * FROM unhealthy_shard_count;
count
---------------------------------------------------------------------
0
(1 row)
SELECT count(*) FROM test_table;
count
---------------------------------------------------------------------
0
(1 row)
-- we don't want to see the prepared transaction numbers in the warnings
SET client_min_messages TO ERROR;
-- kill on command complete on COMMIT PREPARE, command should succeed
SELECT citus.mitmproxy('conn.onCommandComplete(command="COMMIT PREPARED").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
\COPY test_table FROM stdin delimiter ',';
SET client_min_messages TO NOTICE;
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT * FROM unhealthy_shard_count;
count
---------------------------------------------------------------------
0
(1 row)
SELECT count(*) FROM test_table;
count
---------------------------------------------------------------------
4
(1 row)
TRUNCATE TABLE test_table;
-- kill on ROLLBACK, command could be rollbacked
SELECT citus.mitmproxy('conn.onQuery(query="ROLLBACK").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
BEGIN;
\COPY test_table FROM stdin delimiter ',';
ROLLBACK;
WARNING: connection not open
CONTEXT: while executing command on localhost:xxxxx
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT * FROM unhealthy_shard_count;
count
---------------------------------------------------------------------
0
(1 row)
SELECT count(*) FROM test_table;
count
---------------------------------------------------------------------
0
(1 row)
DROP TABLE test_table CASCADE;
NOTICE: drop cascades to view unhealthy_shard_count
-- With two placement, should we error out or mark untouched shard placements as inactive?
SET citus.shard_replication_factor TO 2;
CREATE TABLE test_table_2(id int, value_1 int);
SELECT create_distributed_table('test_table_2','id');
create_distributed_table
---------------------------------------------------------------------
(1 row)
SELECT citus.mitmproxy('conn.kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
\COPY test_table_2 FROM stdin delimiter ',';
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT pds.logicalrelid, pdsd.shardid, pdsd.shardstate
FROM pg_dist_shard_placement as pdsd
INNER JOIN pg_dist_shard as pds
ON pdsd.shardid = pds.shardid
WHERE pds.logicalrelid = 'test_table_2'::regclass
ORDER BY shardid, nodeport;
logicalrelid | shardid | shardstate
---------------------------------------------------------------------
test_table_2 | 1710004 | 1
test_table_2 | 1710004 | 1
test_table_2 | 1710005 | 1
test_table_2 | 1710005 | 1
test_table_2 | 1710006 | 1
test_table_2 | 1710006 | 1
test_table_2 | 1710007 | 1
test_table_2 | 1710007 | 1
(8 rows)
-- Create test_table_2 again to have healthy one
DROP TABLE test_table_2;
CREATE TABLE test_table_2(id int, value_1 int);
SELECT create_distributed_table('test_table_2','id');
create_distributed_table
---------------------------------------------------------------------
(1 row)
-- Kill the connection when we try to start the COPY
-- The query should abort
SELECT citus.mitmproxy('conn.onQuery(query="FROM STDIN WITH").killall()');
mitmproxy
---------------------------------------------------------------------
(1 row)
\COPY test_table_2 FROM stdin delimiter ',';
ERROR: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
CONTEXT: while executing command on localhost:xxxxx
COPY test_table_2, line 1: "1,2"
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT pds.logicalrelid, pdsd.shardid, pdsd.shardstate
FROM pg_dist_shard_placement as pdsd
INNER JOIN pg_dist_shard as pds
ON pdsd.shardid = pds.shardid
WHERE pds.logicalrelid = 'test_table_2'::regclass
ORDER BY shardid, nodeport;
logicalrelid | shardid | shardstate
---------------------------------------------------------------------
test_table_2 | 1710008 | 1
test_table_2 | 1710008 | 1
test_table_2 | 1710009 | 1
test_table_2 | 1710009 | 1
test_table_2 | 1710010 | 1
test_table_2 | 1710010 | 1
test_table_2 | 1710011 | 1
test_table_2 | 1710011 | 1
(8 rows)
-- Create test_table_2 again to have healthy one
DROP TABLE test_table_2;
CREATE TABLE test_table_2(id int, value_1 int);
SELECT create_distributed_table('test_table_2','id');
create_distributed_table
---------------------------------------------------------------------
(1 row)
-- When kill on copying data, it will be rollbacked and placements won't be labaled as invalid.
-- Note that now we sent data to shard xxxxx, yet it is not marked as invalid.
-- You can check the issue about this behaviour: https://github.com/citusdata/citus/issues/1933
SELECT citus.mitmproxy('conn.onCopyData().kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
\COPY test_table_2 FROM stdin delimiter ',';
ERROR: failed to COPY to shard xxxxx on localhost:xxxxx
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT pds.logicalrelid, pdsd.shardid, pdsd.shardstate
FROM pg_dist_shard_placement as pdsd
INNER JOIN pg_dist_shard as pds
ON pdsd.shardid = pds.shardid
WHERE pds.logicalrelid = 'test_table_2'::regclass
ORDER BY shardid, nodeport;
logicalrelid | shardid | shardstate
---------------------------------------------------------------------
test_table_2 | 1710012 | 1
test_table_2 | 1710012 | 1
test_table_2 | 1710013 | 1
test_table_2 | 1710013 | 1
test_table_2 | 1710014 | 1
test_table_2 | 1710014 | 1
test_table_2 | 1710015 | 1
test_table_2 | 1710015 | 1
(8 rows)
DROP SCHEMA copy_distributed_table CASCADE;
NOTICE: drop cascades to table test_table_2
SET search_path TO default;

455
src/test/regress/expected/failure_copy_to_reference_1.out Normal file
View File

@ -0,0 +1,455 @@
--
-- Failure tests for COPY to reference tables
--
CREATE SCHEMA copy_reference_failure;
SET search_path TO 'copy_reference_failure';
SET citus.next_shard_id TO 130000;
-- we don't want to see the prepared transaction numbers in the warnings
SET client_min_messages TO ERROR;
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
CREATE TABLE test_table(id int, value_1 int);
SELECT create_reference_table('test_table');
create_reference_table
---------------------------------------------------------------------
(1 row)
CREATE VIEW unhealthy_shard_count AS
SELECT count(*)
FROM pg_dist_shard_placement pdsp
JOIN
pg_dist_shard pds
ON pdsp.shardid=pds.shardid
WHERE logicalrelid='copy_reference_failure.test_table'::regclass AND shardstate != 1;
-- in the first test, kill just in the first
-- response we get from the worker
SELECT citus.mitmproxy('conn.kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
\copy test_table FROM STDIN DELIMITER ','
ERROR: failure on connection marked as essential: localhost:xxxxx
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT * FROM unhealthy_shard_count;
count
---------------------------------------------------------------------
0
(1 row)
SELECT count(*) FROM test_table;
count
---------------------------------------------------------------------
0
(1 row)
-- kill as soon as the coordinator sends begin
SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
\copy test_table FROM STDIN DELIMITER ','
ERROR: failure on connection marked as essential: localhost:xxxxx
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT * FROM unhealthy_shard_count;
count
---------------------------------------------------------------------
0
(1 row)
SELECT count(*) FROM test_table;
count
---------------------------------------------------------------------
0
(1 row)
-- cancel as soon as the coordinator sends begin
SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED").cancel(' || pg_backend_pid() || ')');
mitmproxy
---------------------------------------------------------------------
(1 row)
\copy test_table FROM STDIN DELIMITER ','
ERROR: canceling statement due to user request
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT * FROM unhealthy_shard_count;
count
---------------------------------------------------------------------
0
(1 row)
SELECT count(*) FROM test_table;
count
---------------------------------------------------------------------
0
(1 row)
-- kill as soon as the coordinator sends COPY command
SELECT citus.mitmproxy('conn.onQuery(query="^COPY").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
\copy test_table FROM STDIN DELIMITER ','
ERROR: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
CONTEXT: while executing command on localhost:xxxxx
COPY test_table, line 1: "1,2"
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT * FROM unhealthy_shard_count;
count
---------------------------------------------------------------------
0
(1 row)
SELECT count(*) FROM test_table;
count
---------------------------------------------------------------------
0
(1 row)
-- cancel as soon as the coordinator sends COPY command
SELECT citus.mitmproxy('conn.onQuery(query="^COPY").cancel(' || pg_backend_pid() || ')');
mitmproxy
---------------------------------------------------------------------
(1 row)
\copy test_table FROM STDIN DELIMITER ','
ERROR: canceling statement due to user request
CONTEXT: COPY test_table, line 1: "1,2"
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT * FROM unhealthy_shard_count;
count
---------------------------------------------------------------------
0
(1 row)
SELECT count(*) FROM test_table;
count
---------------------------------------------------------------------
0
(1 row)
-- kill as soon as the worker sends CopyComplete
SELECT citus.mitmproxy('conn.onCommandComplete(command="^COPY 3").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
\copy test_table FROM STDIN DELIMITER ','
ERROR: failed to COPY to shard xxxxx on localhost:xxxxx
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT * FROM unhealthy_shard_count;
count
---------------------------------------------------------------------
0
(1 row)
SELECT count(*) FROM test_table;
count
---------------------------------------------------------------------
0
(1 row)
-- cancel as soon as the coordinator sends CopyData
SELECT citus.mitmproxy('conn.onCommandComplete(command="^COPY 3").cancel(' || pg_backend_pid() || ')');
mitmproxy
---------------------------------------------------------------------
(1 row)
\copy test_table FROM STDIN DELIMITER ','
ERROR: canceling statement due to user request
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT * FROM unhealthy_shard_count;
count
---------------------------------------------------------------------
0
(1 row)
SELECT count(*) FROM test_table;
count
---------------------------------------------------------------------
0
(1 row)
-- kill the connection when we try to start the COPY
-- the query should abort
SELECT citus.mitmproxy('conn.onQuery(query="FROM STDIN WITH").killall()');
mitmproxy
---------------------------------------------------------------------
(1 row)
\copy test_table FROM STDIN DELIMITER ','
ERROR: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
CONTEXT: while executing command on localhost:xxxxx
COPY test_table, line 1: "1,2"
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT * FROM unhealthy_shard_count;
count
---------------------------------------------------------------------
0
(1 row)
SELECT count(*) FROM test_table;
count
---------------------------------------------------------------------
0
(1 row)
-- killing on PREPARE should be fine, everything should be rollbacked
SELECT citus.mitmproxy('conn.onQuery(query="^PREPARE TRANSACTION").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
\copy test_table FROM STDIN DELIMITER ','
ERROR: connection not open
CONTEXT: while executing command on localhost:xxxxx
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT * FROM unhealthy_shard_count;
count
---------------------------------------------------------------------
0
(1 row)
SELECT count(*) FROM test_table;
count
---------------------------------------------------------------------
0
(1 row)
-- cancelling on PREPARE should be fine, everything should be rollbacked
SELECT citus.mitmproxy('conn.onQuery(query="^PREPARE TRANSACTION").cancel(' || pg_backend_pid() || ')');
mitmproxy
---------------------------------------------------------------------
(1 row)
\copy test_table FROM STDIN DELIMITER ','
ERROR: canceling statement due to user request
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT * FROM unhealthy_shard_count;
count
---------------------------------------------------------------------
0
(1 row)
SELECT count(*) FROM test_table;
count
---------------------------------------------------------------------
0
(1 row)
-- killing on command complete of COMMIT PREPARE, we should see that the command succeeds
-- and all the workers committed
SELECT citus.mitmproxy('conn.onCommandComplete(command="^COMMIT PREPARED").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
\copy test_table FROM STDIN DELIMITER ','
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
-- we shouldn't have any prepared transactions in the workers
SELECT recover_prepared_transactions();
recover_prepared_transactions
---------------------------------------------------------------------
0
(1 row)
SELECT * FROM unhealthy_shard_count;
count
---------------------------------------------------------------------
0
(1 row)
SELECT count(*) FROM test_table;
count
---------------------------------------------------------------------
3
(1 row)
TRUNCATE test_table;
-- kill as soon as the coordinator sends COMMIT
SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
\copy test_table FROM STDIN DELIMITER ','
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
-- Since we kill connections to one worker after commit arrives but the
-- other worker connections are healthy, we cannot commit on 1 worker
-- which has 1 active shard placements, but the other does. That's why
-- we expect to see 1 recovered prepared transactions.
SELECT recover_prepared_transactions();
recover_prepared_transactions
---------------------------------------------------------------------
1
(1 row)
SELECT * FROM unhealthy_shard_count;
count
---------------------------------------------------------------------
0
(1 row)
SELECT count(*) FROM test_table;
count
---------------------------------------------------------------------
3
(1 row)
TRUNCATE test_table;
-- finally, test failing on ROLLBACK just after the coordinator
-- sends the ROLLBACK so the command can be rollbacked
SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
BEGIN;
SET LOCAL client_min_messages TO WARNING;
\copy test_table FROM STDIN DELIMITER ','
ROLLBACK;
WARNING: connection not open
CONTEXT: while executing command on localhost:xxxxx
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT * FROM unhealthy_shard_count;
count
---------------------------------------------------------------------
0
(1 row)
SELECT count(*) FROM test_table;
count
---------------------------------------------------------------------
0
(1 row)
-- but now kill just after the worker sends response to
-- ROLLBACK command, command should have been rollbacked
-- both on the distributed table and the placements
SELECT citus.mitmproxy('conn.onCommandComplete(command="^ROLLBACK").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
BEGIN;
SET LOCAL client_min_messages TO WARNING;
\copy test_table FROM STDIN DELIMITER ','
ROLLBACK;
WARNING: connection not open
CONTEXT: while executing command on localhost:xxxxx
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT recover_prepared_transactions();
recover_prepared_transactions
---------------------------------------------------------------------
0
(1 row)
SELECT * FROM unhealthy_shard_count;
count
---------------------------------------------------------------------
0
(1 row)
SELECT count(*) FROM test_table;
count
---------------------------------------------------------------------
0
(1 row)
DROP SCHEMA copy_reference_failure CASCADE;
SET search_path TO default;

1006
src/test/regress/expected/failure_create_distributed_table_non_empty_1.out Normal file
View File

File diff suppressed because it is too large Load Diff

266
src/test/regress/expected/failure_create_reference_table_1.out Normal file
View File

@ -0,0 +1,266 @@
--
-- Failure tests for creating reference table
--
CREATE SCHEMA failure_reference_table;
SET search_path TO 'failure_reference_table';
SET citus.next_shard_id TO 10000000;
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
-- this is merely used to get the schema creation propagated. Without there are failures
-- not related to reference tables but schema creation due to dependency creation on workers
CREATE TYPE schema_proc AS (a int);
DROP TYPE schema_proc;
CREATE TABLE ref_table(id int);
INSERT INTO ref_table VALUES(1),(2),(3);
-- Kill on sending first query to worker node, should error
-- out and not create any placement
SELECT citus.mitmproxy('conn.onQuery().kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT create_reference_table('ref_table');
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT count(*) FROM pg_dist_shard_placement;
count
---------------------------------------------------------------------
0
(1 row)
-- Kill after creating transaction on worker node
SELECT citus.mitmproxy('conn.onCommandComplete(command="BEGIN").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT create_reference_table('ref_table');
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT count(*) FROM pg_dist_shard_placement;
count
---------------------------------------------------------------------
0
(1 row)
-- Cancel after creating transaction on worker node
SELECT citus.mitmproxy('conn.onCommandComplete(command="BEGIN").cancel(' || pg_backend_pid() || ')');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT create_reference_table('ref_table');
ERROR: canceling statement due to user request
SELECT count(*) FROM pg_dist_shard_placement;
count
---------------------------------------------------------------------
0
(1 row)
-- Kill after copying data to worker node
SELECT citus.mitmproxy('conn.onCommandComplete(command="SELECT 1").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT create_reference_table('ref_table');
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT count(*) FROM pg_dist_shard_placement;
count
---------------------------------------------------------------------
0
(1 row)
-- Cancel after copying data to worker node
SELECT citus.mitmproxy('conn.onCommandComplete(command="SELECT 1").cancel(' || pg_backend_pid() || ')');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT create_reference_table('ref_table');
ERROR: canceling statement due to user request
SELECT count(*) FROM pg_dist_shard_placement;
count
---------------------------------------------------------------------
0
(1 row)
-- Kill after copying data to worker node
SELECT citus.mitmproxy('conn.onCommandComplete(command="COPY 3").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT create_reference_table('ref_table');
NOTICE: Copying data from local table...
NOTICE: copying the data has completed
DETAIL: The local data in the table is no longer visible, but is still on disk.
HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$failure_reference_table.ref_table$$)
ERROR: failed to COPY to shard xxxxx on localhost:xxxxx
SELECT count(*) FROM pg_dist_shard_placement;
count
---------------------------------------------------------------------
0
(1 row)
-- Cancel after copying data to worker node
SELECT citus.mitmproxy('conn.onCommandComplete(command="COPY 3").cancel(' || pg_backend_pid() || ')');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT create_reference_table('ref_table');
NOTICE: Copying data from local table...
NOTICE: copying the data has completed
DETAIL: The local data in the table is no longer visible, but is still on disk.
HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$failure_reference_table.ref_table$$)
ERROR: canceling statement due to user request
SELECT count(*) FROM pg_dist_shard_placement;
count
---------------------------------------------------------------------
0
(1 row)
-- we don't want to see the prepared transaction numbers in the warnings
SET client_min_messages TO ERROR;
-- Kill after preparing transaction. Since we don't commit after preparing, we recover
-- prepared transaction afterwards.
SELECT citus.mitmproxy('conn.onCommandComplete(command="PREPARE TRANSACTION").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT create_reference_table('ref_table');
ERROR: connection not open
CONTEXT: while executing command on localhost:xxxxx
SELECT count(*) FROM pg_dist_shard_placement;
count
---------------------------------------------------------------------
0
(1 row)
SELECT recover_prepared_transactions();
recover_prepared_transactions
---------------------------------------------------------------------
1
(1 row)
-- Kill after commiting prepared, this should succeed
SELECT citus.mitmproxy('conn.onCommandComplete(command="COMMIT PREPARED").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT create_reference_table('ref_table');
create_reference_table
---------------------------------------------------------------------
(1 row)
SELECT shardid, nodeport, shardstate FROM pg_dist_shard_placement ORDER BY shardid, nodeport;
shardid | nodeport | shardstate
---------------------------------------------------------------------
10000008 | 9060 | 1
10000008 | 57637 | 1
(2 rows)
SET client_min_messages TO NOTICE;
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
DROP TABLE ref_table;
DROP SCHEMA failure_reference_table;
SET citus.enable_ddl_propagation TO OFF;
CREATE SCHEMA failure_reference_table;
SET citus.enable_ddl_propagation TO ON;
CREATE TABLE ref_table(id int);
INSERT INTO ref_table VALUES(1),(2),(3);
-- Test in transaction
SELECT citus.mitmproxy('conn.onQuery().kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
BEGIN;
SELECT create_reference_table('ref_table');
WARNING: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
CONTEXT: while executing command on localhost:xxxxx
ERROR: failure on connection marked as essential: localhost:xxxxx
COMMIT;
-- kill on ROLLBACK, should be rollbacked
SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
BEGIN;
SELECT create_reference_table('ref_table');
NOTICE: Copying data from local table...
NOTICE: copying the data has completed
DETAIL: The local data in the table is no longer visible, but is still on disk.
HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$failure_reference_table.ref_table$$)
create_reference_table
---------------------------------------------------------------------
(1 row)
ROLLBACK;
WARNING: connection not open
CONTEXT: while executing command on localhost:xxxxx
SELECT * FROM pg_dist_shard_placement ORDER BY shardid, nodeport;
shardid | shardstate | shardlength | nodename | nodeport | placementid
---------------------------------------------------------------------
(0 rows)
-- cancel when the coordinator send ROLLBACK, should be rollbacked. We ignore cancellations
-- during the ROLLBACK.
SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").cancel(' || pg_backend_pid() || ')');
mitmproxy
---------------------------------------------------------------------
(1 row)
BEGIN;
SELECT create_reference_table('ref_table');
NOTICE: Copying data from local table...
NOTICE: copying the data has completed
DETAIL: The local data in the table is no longer visible, but is still on disk.
HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$failure_reference_table.ref_table$$)
create_reference_table
---------------------------------------------------------------------
(1 row)
ROLLBACK;
SELECT * FROM pg_dist_shard_placement ORDER BY shardid, nodeport;
shardid | shardstate | shardlength | nodename | nodeport | placementid
---------------------------------------------------------------------
(0 rows)
DROP SCHEMA failure_reference_table CASCADE;
NOTICE: drop cascades to table ref_table
SET search_path TO default;

724
src/test/regress/expected/failure_create_table_1.out Normal file
View File

@ -0,0 +1,724 @@
--
-- failure_create_table adds failure tests for creating table without data.
--
SET citus.enable_ddl_propagation TO OFF;
CREATE SCHEMA failure_create_table;
SET citus.enable_ddl_propagation TO ON;
SET search_path TO 'failure_create_table';
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SET citus.shard_replication_factor TO 1;
SET citus.shard_count to 4;
CREATE TABLE test_table(id int, value_1 int);
-- Kill connection before sending query to the worker
SELECT citus.mitmproxy('conn.kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT create_distributed_table('test_table','id');
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT count(*) FROM pg_dist_shard;
count
---------------------------------------------------------------------
0
(1 row)
SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
run_command_on_workers
---------------------------------------------------------------------
(localhost,9060,t,0)
(localhost,57637,t,0)
(2 rows)
-- kill as soon as the coordinator sends CREATE SCHEMA
-- Since schemas are created in separate transaction, schema will
-- be created only on the node which is not behind the proxy.
-- https://github.com/citusdata/citus/pull/1652
SELECT citus.mitmproxy('conn.onQuery(query="^CREATE SCHEMA").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT create_distributed_table('test_table', 'id');
ERROR: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
CONTEXT: while executing command on localhost:xxxxx
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT count(*) FROM pg_dist_shard;
count
---------------------------------------------------------------------
0
(1 row)
SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.schemata WHERE schema_name = 'failure_create_table'$$);
run_command_on_workers
---------------------------------------------------------------------
(localhost,9060,t,0)
(localhost,57637,t,1)
(2 rows)
-- this is merely used to get the schema creation propagated. Without there are failures
-- not related to reference tables but schema creation due to dependency creation on workers
CREATE TYPE schema_proc AS (a int);
DROP TYPE schema_proc;
-- Now, kill the connection while opening transaction on workers.
SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT create_distributed_table('test_table','id');
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT count(*) FROM pg_dist_shard;
count
---------------------------------------------------------------------
0
(1 row)
SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
run_command_on_workers
---------------------------------------------------------------------
(localhost,9060,t,0)
(localhost,57637,t,0)
(2 rows)
-- Now, kill the connection after sending create table command with worker_apply_shard_ddl_command UDF
SELECT citus.mitmproxy('conn.onQuery(query="SELECT worker_apply_shard_ddl_command").after(1).kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT create_distributed_table('test_table','id');
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT count(*) FROM pg_dist_shard;
count
---------------------------------------------------------------------
0
(1 row)
SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
run_command_on_workers
---------------------------------------------------------------------
(localhost,9060,t,0)
(localhost,57637,t,0)
(2 rows)
-- Kill the connection while creating a distributed table in sequential mode on sending create command
-- with worker_apply_shard_ddl_command UDF.
BEGIN;
SET LOCAL citus.multi_shard_modify_mode TO 'sequential';
SELECT citus.mitmproxy('conn.onQuery(query="SELECT worker_apply_shard_ddl_command").after(1).kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT create_distributed_table('test_table', 'id');
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
COMMIT;
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT count(*) FROM pg_dist_shard;
count
---------------------------------------------------------------------
0
(1 row)
SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
run_command_on_workers
---------------------------------------------------------------------
(localhost,9060,t,0)
(localhost,57637,t,0)
(2 rows)
-- Now, cancel the connection while creating transaction
-- workers. Note that, cancel requests will be ignored during
-- shard creation.
SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED").cancel(' || pg_backend_pid() || ')');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT create_distributed_table('test_table','id');
ERROR: canceling statement due to user request
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT count(*) FROM pg_dist_shard;
count
---------------------------------------------------------------------
0
(1 row)
SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
run_command_on_workers
---------------------------------------------------------------------
(localhost,9060,t,0)
(localhost,57637,t,0)
(2 rows)
DROP TABLE test_table;
CREATE TABLE test_table(id int, value_1 int);
-- Kill and cancel the connection with colocate_with option while sending the create table command
CREATE TABLE temp_table(id int, value_1 int);
SELECT create_distributed_table('temp_table','id');
create_distributed_table
---------------------------------------------------------------------
(1 row)
SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT create_distributed_table('test_table','id',colocate_with=>'temp_table');
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT count(*) FROM pg_dist_shard;
count
---------------------------------------------------------------------
4
(1 row)
SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
run_command_on_workers
---------------------------------------------------------------------
(localhost,9060,t,0)
(localhost,57637,t,0)
(2 rows)
SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").cancel(' || pg_backend_pid() || ')');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT create_distributed_table('test_table','id',colocate_with=>'temp_table');
ERROR: canceling statement due to user request
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT count(*) FROM pg_dist_shard;
count
---------------------------------------------------------------------
4
(1 row)
SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
run_command_on_workers
---------------------------------------------------------------------
(localhost,9060,t,0)
(localhost,57637,t,0)
(2 rows)
-- Kill and cancel the connection after worker sends "PREPARE TRANSACTION" ack with colocate_with option
SELECT citus.mitmproxy('conn.onCommandComplete(command="PREPARE TRANSACTION").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT create_distributed_table('test_table','id',colocate_with=>'temp_table');
ERROR: connection not open
CONTEXT: while executing command on localhost:xxxxx
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT recover_prepared_transactions();
recover_prepared_transactions
---------------------------------------------------------------------
1
(1 row)
SELECT count(*) FROM pg_dist_shard;
count
---------------------------------------------------------------------
4
(1 row)
SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
run_command_on_workers
---------------------------------------------------------------------
(localhost,9060,t,0)
(localhost,57637,t,0)
(2 rows)
SELECT citus.mitmproxy('conn.onCommandComplete(command="PREPARE TRANSACTION").cancel(' || pg_backend_pid() || ')');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT create_distributed_table('test_table','id',colocate_with=>'temp_table');
ERROR: canceling statement due to user request
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT count(*) FROM pg_dist_shard;
count
---------------------------------------------------------------------
4
(1 row)
SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
run_command_on_workers
---------------------------------------------------------------------
(localhost,9060,t,0)
(localhost,57637,t,0)
(2 rows)
-- drop tables and schema and recreate to start from a non-distributed schema again
DROP TABLE temp_table;
DROP TABLE test_table;
DROP SCHEMA failure_create_table;
CREATE SCHEMA failure_create_table;
CREATE TABLE test_table(id int, value_1 int);
-- Test inside transaction
-- Kill connection before sending query to the worker
SELECT citus.mitmproxy('conn.kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
BEGIN;
SELECT create_distributed_table('test_table','id');
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
ROLLBACK;
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT count(*) FROM pg_dist_shard;
count
---------------------------------------------------------------------
0
(1 row)
SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
run_command_on_workers
---------------------------------------------------------------------
(localhost,9060,t,0)
(localhost,57637,t,0)
(2 rows)
-- this is merely used to get the schema creation propagated. Without there are failures
-- not related to reference tables but schema creation due to dependency creation on workers
CREATE TYPE schema_proc AS (a int);
DROP TYPE schema_proc;
-- Now, kill the connection while creating transaction on workers in transaction.
SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
BEGIN;
SELECT create_distributed_table('test_table','id');
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
ROLLBACK;
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT count(*) FROM pg_dist_shard;
count
---------------------------------------------------------------------
0
(1 row)
SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
run_command_on_workers
---------------------------------------------------------------------
(localhost,9060,t,0)
(localhost,57637,t,0)
(2 rows)
-- Now, cancel the connection while creating the transaction on
-- workers. Note that, cancel requests will be ignored during
-- shard creation again in transaction if we're not relying on the
-- executor. So, we'll have two output files
SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED").cancel(' || pg_backend_pid() || ')');
mitmproxy
---------------------------------------------------------------------
(1 row)
BEGIN;
SELECT create_distributed_table('test_table','id');
ERROR: canceling statement due to user request
COMMIT;
SELECT recover_prepared_transactions();
recover_prepared_transactions
---------------------------------------------------------------------
0
(1 row)
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT count(*) FROM pg_dist_shard;
count
---------------------------------------------------------------------
0
(1 row)
SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
run_command_on_workers
---------------------------------------------------------------------
(localhost,9060,t,0)
(localhost,57637,t,0)
(2 rows)
-- drop tables and schema and recreate to start from a non-distributed schema again
DROP TABLE test_table;
DROP SCHEMA failure_create_table;
CREATE SCHEMA failure_create_table;
CREATE TABLE test_table(id int, value_1 int);
-- Kill connection before sending query to the worker with 1pc.
SELECT citus.mitmproxy('conn.kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
BEGIN;
SELECT create_distributed_table('test_table','id');
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
ROLLBACK;
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT count(*) FROM pg_dist_shard;
count
---------------------------------------------------------------------
0
(1 row)
SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
run_command_on_workers
---------------------------------------------------------------------
(localhost,9060,t,0)
(localhost,57637,t,0)
(2 rows)
-- Kill connection while sending create table command with 1pc.
SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
BEGIN;
SELECT create_distributed_table('test_table','id');
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
ROLLBACK;
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT count(*) FROM pg_dist_shard;
count
---------------------------------------------------------------------
0
(1 row)
SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
run_command_on_workers
---------------------------------------------------------------------
(localhost,9060,t,0)
(localhost,57637,t,0)
(2 rows)
-- this is merely used to get the schema creation propagated. Without there are failures
-- not related to reference tables but schema creation due to dependency creation on workers
CREATE TYPE schema_proc AS (a int);
DROP TYPE schema_proc;
-- Now, kill the connection while opening transactions on workers with 1pc. Transaction will be opened due to BEGIN.
SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
BEGIN;
SELECT create_distributed_table('test_table','id');
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
ROLLBACK;
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT count(*) FROM pg_dist_shard;
count
---------------------------------------------------------------------
0
(1 row)
SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
run_command_on_workers
---------------------------------------------------------------------
(localhost,9060,t,0)
(localhost,57637,t,0)
(2 rows)
-- Now, cancel the connection while creating transactions on
-- workers with 1pc. Note that, cancel requests will be ignored during
-- shard creation unless the executor is used. So, we'll have two output files
SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED").cancel(' || pg_backend_pid() || ')');
mitmproxy
---------------------------------------------------------------------
(1 row)
BEGIN;
SELECT create_distributed_table('test_table','id');
ERROR: canceling statement due to user request
COMMIT;
SELECT recover_prepared_transactions();
recover_prepared_transactions
---------------------------------------------------------------------
0
(1 row)
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT count(*) FROM pg_dist_shard;
count
---------------------------------------------------------------------
0
(1 row)
SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
run_command_on_workers
---------------------------------------------------------------------
(localhost,9060,t,0)
(localhost,57637,t,0)
(2 rows)
DROP TABLE test_table;
DROP SCHEMA failure_create_table;
CREATE SCHEMA failure_create_table;
-- this function is dropped in Citus10, added here for tests
CREATE OR REPLACE FUNCTION pg_catalog.master_create_distributed_table(table_name regclass,
distribution_column text,
distribution_method citus.distribution_type)
RETURNS void
LANGUAGE C STRICT
AS 'citus', $$master_create_distributed_table$$;
COMMENT ON FUNCTION pg_catalog.master_create_distributed_table(table_name regclass,
distribution_column text,
distribution_method citus.distribution_type)
IS 'define the table distribution functions';
-- this function is dropped in Citus10, added here for tests
CREATE OR REPLACE FUNCTION pg_catalog.master_create_worker_shards(table_name text, shard_count integer,
replication_factor integer DEFAULT 2)
RETURNS void
AS 'citus', $$master_create_worker_shards$$
LANGUAGE C STRICT;
-- Test master_create_worker_shards with 2pc
CREATE TABLE test_table_2(id int, value_1 int);
SELECT master_create_distributed_table('test_table_2', 'id', 'hash');
master_create_distributed_table
---------------------------------------------------------------------
(1 row)
-- Kill connection before sending query to the worker
SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT master_create_worker_shards('test_table_2', 4, 2);
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT count(*) FROM pg_dist_shard;
count
---------------------------------------------------------------------
0
(1 row)
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT count(*) FROM pg_dist_shard;
count
---------------------------------------------------------------------
0
(1 row)
SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
run_command_on_workers
---------------------------------------------------------------------
(localhost,9060,t,0)
(localhost,57637,t,0)
(2 rows)
-- Kill the connection after worker sends "PREPARE TRANSACTION" ack
SELECT citus.mitmproxy('conn.onCommandComplete(command="^PREPARE TRANSACTION").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT master_create_worker_shards('test_table_2', 4, 2);
ERROR: connection not open
CONTEXT: while executing command on localhost:xxxxx
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT count(*) FROM pg_dist_shard;
count
---------------------------------------------------------------------
0
(1 row)
SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
run_command_on_workers
---------------------------------------------------------------------
(localhost,9060,t,0)
(localhost,57637,t,0)
(2 rows)
-- Cancel the connection after sending prepare transaction in master_create_worker_shards
SELECT citus.mitmproxy('conn.onCommandComplete(command="PREPARE TRANSACTION").cancel(' || pg_backend_pid() || ')');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT master_create_worker_shards('test_table_2', 4, 2);
ERROR: canceling statement due to user request
-- Show that there is no pending transaction
SELECT recover_prepared_transactions();
recover_prepared_transactions
---------------------------------------------------------------------
1
(1 row)
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT count(*) FROM pg_dist_shard;
count
---------------------------------------------------------------------
0
(1 row)
SELECT run_command_on_workers($$SELECT count(*) FROM information_schema.tables WHERE table_schema = 'failure_create_table' and table_name LIKE 'test_table%' ORDER BY 1$$);
run_command_on_workers
---------------------------------------------------------------------
(localhost,9060,t,0)
(localhost,57637,t,0)
(2 rows)
DROP SCHEMA failure_create_table CASCADE;
NOTICE: drop cascades to table test_table_2
SET search_path TO default;

1060
src/test/regress/expected/failure_ddl_1.out Normal file
View File

File diff suppressed because it is too large Load Diff

507
src/test/regress/expected/failure_multi_dml_1.out Normal file
View File

@ -0,0 +1,507 @@
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SET citus.shard_count = 2;
SET citus.shard_replication_factor = 1; -- one shard per worker
SET citus.next_shard_id TO 103400;
ALTER SEQUENCE pg_catalog.pg_dist_placement_placementid_seq RESTART 100;
CREATE TABLE dml_test (id integer, name text);
SELECT create_distributed_table('dml_test', 'id');
create_distributed_table
---------------------------------------------------------------------
(1 row)
COPY dml_test FROM STDIN WITH CSV;
SELECT citus.clear_network_traffic();
clear_network_traffic
---------------------------------------------------------------------
(1 row)
---- test multiple statements spanning multiple shards,
---- at each significant point. These transactions are 2pc
-- fail at DELETE
SELECT citus.mitmproxy('conn.onQuery(query="^DELETE").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
BEGIN;
DELETE FROM dml_test WHERE id = 1;
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
DELETE FROM dml_test WHERE id = 2;
ERROR: current transaction is aborted, commands ignored until end of transaction block
INSERT INTO dml_test VALUES (5, 'Epsilon');
ERROR: current transaction is aborted, commands ignored until end of transaction block
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
ERROR: current transaction is aborted, commands ignored until end of transaction block
UPDATE dml_test SET name = 'gamma' WHERE id = 3;
ERROR: current transaction is aborted, commands ignored until end of transaction block
COMMIT;
--- shouldn't see any changes performed in failed transaction
SELECT * FROM dml_test ORDER BY id ASC;
id | name
---------------------------------------------------------------------
1 | Alpha
2 | Beta
3 | Gamma
4 | Delta
(4 rows)
-- cancel at DELETE
SELECT citus.mitmproxy('conn.onQuery(query="^DELETE").cancel(' || pg_backend_pid() || ')');
mitmproxy
---------------------------------------------------------------------
(1 row)
BEGIN;
DELETE FROM dml_test WHERE id = 1;
ERROR: canceling statement due to user request
DELETE FROM dml_test WHERE id = 2;
ERROR: current transaction is aborted, commands ignored until end of transaction block
INSERT INTO dml_test VALUES (5, 'Epsilon');
ERROR: current transaction is aborted, commands ignored until end of transaction block
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
ERROR: current transaction is aborted, commands ignored until end of transaction block
UPDATE dml_test SET name = 'gamma' WHERE id = 3;
ERROR: current transaction is aborted, commands ignored until end of transaction block
COMMIT;
--- shouldn't see any changes performed in failed transaction
SELECT * FROM dml_test ORDER BY id ASC;
id | name
---------------------------------------------------------------------
1 | Alpha
2 | Beta
3 | Gamma
4 | Delta
(4 rows)
-- fail at INSERT
SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
BEGIN;
DELETE FROM dml_test WHERE id = 1;
DELETE FROM dml_test WHERE id = 2;
INSERT INTO dml_test VALUES (5, 'Epsilon');
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
ERROR: current transaction is aborted, commands ignored until end of transaction block
UPDATE dml_test SET name = 'gamma' WHERE id = 3;
ERROR: current transaction is aborted, commands ignored until end of transaction block
COMMIT;
--- shouldn't see any changes before failed INSERT
SELECT * FROM dml_test ORDER BY id ASC;
id | name
---------------------------------------------------------------------
1 | Alpha
2 | Beta
3 | Gamma
4 | Delta
(4 rows)
-- cancel at INSERT
SELECT citus.mitmproxy('conn.onQuery(query="^INSERT").cancel(' || pg_backend_pid() || ')');
mitmproxy
---------------------------------------------------------------------
(1 row)
BEGIN;
DELETE FROM dml_test WHERE id = 1;
DELETE FROM dml_test WHERE id = 2;
INSERT INTO dml_test VALUES (5, 'Epsilon');
ERROR: canceling statement due to user request
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
ERROR: current transaction is aborted, commands ignored until end of transaction block
UPDATE dml_test SET name = 'gamma' WHERE id = 3;
ERROR: current transaction is aborted, commands ignored until end of transaction block
COMMIT;
--- shouldn't see any changes before failed INSERT
SELECT * FROM dml_test ORDER BY id ASC;
id | name
---------------------------------------------------------------------
1 | Alpha
2 | Beta
3 | Gamma
4 | Delta
(4 rows)
-- fail at UPDATE
SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
BEGIN;
DELETE FROM dml_test WHERE id = 1;
DELETE FROM dml_test WHERE id = 2;
INSERT INTO dml_test VALUES (5, 'Epsilon');
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
UPDATE dml_test SET name = 'gamma' WHERE id = 3;
ERROR: current transaction is aborted, commands ignored until end of transaction block
COMMIT;
--- shouldn't see any changes after failed UPDATE
SELECT * FROM dml_test ORDER BY id ASC;
id | name
---------------------------------------------------------------------
1 | Alpha
2 | Beta
3 | Gamma
4 | Delta
(4 rows)
-- cancel at UPDATE
SELECT citus.mitmproxy('conn.onQuery(query="^UPDATE").cancel(' || pg_backend_pid() || ')');
mitmproxy
---------------------------------------------------------------------
(1 row)
BEGIN;
DELETE FROM dml_test WHERE id = 1;
DELETE FROM dml_test WHERE id = 2;
INSERT INTO dml_test VALUES (5, 'Epsilon');
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
ERROR: canceling statement due to user request
UPDATE dml_test SET name = 'gamma' WHERE id = 3;
ERROR: current transaction is aborted, commands ignored until end of transaction block
COMMIT;
--- shouldn't see any changes after failed UPDATE
SELECT * FROM dml_test ORDER BY id ASC;
id | name
---------------------------------------------------------------------
1 | Alpha
2 | Beta
3 | Gamma
4 | Delta
(4 rows)
-- fail at PREPARE TRANSACTION
SELECT citus.mitmproxy('conn.onQuery(query="^PREPARE TRANSACTION").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
-- this transaction block will be sent to the coordinator as a remote command to hide the
-- error message that is caused during commit.
-- we'll test for the txn side-effects to ensure it didn't run
SELECT master_run_on_worker(
ARRAY['localhost']::text[],
ARRAY[:master_port]::int[],
ARRAY['
BEGIN;
DELETE FROM dml_test WHERE id = 1;
DELETE FROM dml_test WHERE id = 2;
INSERT INTO dml_test VALUES (5, ''Epsilon'');
UPDATE dml_test SET name = ''alpha'' WHERE id = 1;
UPDATE dml_test SET name = ''gamma'' WHERE id = 3;
COMMIT;
'],
false
);
WARNING: connection not open
CONTEXT: while executing command on localhost:xxxxx
while executing command on localhost:xxxxx
master_run_on_worker
---------------------------------------------------------------------
(localhost,57636,t,BEGIN)
(1 row)
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT shardid FROM pg_dist_shard_placement WHERE shardstate = 3;
shardid
---------------------------------------------------------------------
(0 rows)
SELECT recover_prepared_transactions();
recover_prepared_transactions
---------------------------------------------------------------------
0
(1 row)
-- shouldn't see any changes after failed PREPARE
SELECT * FROM dml_test ORDER BY id ASC;
id | name
---------------------------------------------------------------------
1 | Alpha
2 | Beta
3 | Gamma
4 | Delta
(4 rows)
-- cancel at PREPARE TRANSACTION
SELECT citus.mitmproxy('conn.onQuery(query="^PREPARE TRANSACTION").cancel(' || pg_backend_pid() || ')');
mitmproxy
---------------------------------------------------------------------
(1 row)
-- we'll test for the txn side-effects to ensure it didn't run
BEGIN;
DELETE FROM dml_test WHERE id = 1;
DELETE FROM dml_test WHERE id = 2;
INSERT INTO dml_test VALUES (5, 'Epsilon');
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
UPDATE dml_test SET name = 'gamma' WHERE id = 3;
COMMIT;
ERROR: canceling statement due to user request
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT shardid FROM pg_dist_shard_placement WHERE shardstate = 3;
shardid
---------------------------------------------------------------------
(0 rows)
SELECT recover_prepared_transactions();
recover_prepared_transactions
---------------------------------------------------------------------
0
(1 row)
-- shouldn't see any changes after failed PREPARE
SELECT * FROM dml_test ORDER BY id ASC;
id | name
---------------------------------------------------------------------
1 | Alpha
2 | Beta
3 | Gamma
4 | Delta
(4 rows)
-- fail at COMMIT
SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
-- hide the error message (it has the PID)...
-- we'll test for the txn side-effects to ensure it didn't run
SET client_min_messages TO ERROR;
BEGIN;
DELETE FROM dml_test WHERE id = 1;
DELETE FROM dml_test WHERE id = 2;
INSERT INTO dml_test VALUES (5, 'Epsilon');
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
UPDATE dml_test SET name = 'gamma' WHERE id = 3;
COMMIT;
SET client_min_messages TO DEFAULT;
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT shardid FROM pg_dist_shard_placement WHERE shardstate = 3;
shardid
---------------------------------------------------------------------
(0 rows)
SELECT recover_prepared_transactions();
recover_prepared_transactions
---------------------------------------------------------------------
1
(1 row)
-- should see changes, because of txn recovery
SELECT * FROM dml_test ORDER BY id ASC;
id | name
---------------------------------------------------------------------
3 | gamma
4 | Delta
5 | Epsilon
(3 rows)
-- cancel at COMMITs are ignored by Postgres
SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT").cancel(' || pg_backend_pid() || ')');
mitmproxy
---------------------------------------------------------------------
(1 row)
BEGIN;
DELETE FROM dml_test WHERE id = 1;
DELETE FROM dml_test WHERE id = 2;
INSERT INTO dml_test VALUES (5, 'Epsilon');
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
UPDATE dml_test SET name = 'gamma' WHERE id = 3;
COMMIT;
-- should see changes, because cancellation is ignored
SELECT * FROM dml_test ORDER BY id ASC;
id | name
---------------------------------------------------------------------
3 | gamma
4 | Delta
5 | Epsilon
5 | Epsilon
(4 rows)
-- drop table and recreate with different replication/sharding
DROP TABLE dml_test;
SET citus.shard_count = 1;
SET citus.shard_replication_factor = 2; -- two placements
CREATE TABLE dml_test (id integer, name text);
SELECT create_distributed_table('dml_test', 'id');
create_distributed_table
---------------------------------------------------------------------
(1 row)
COPY dml_test FROM STDIN WITH CSV;
---- test multiple statements against a single shard, but with two placements
-- fail at PREPARED COMMIT as we use 2PC
SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
BEGIN;
DELETE FROM dml_test WHERE id = 1;
DELETE FROM dml_test WHERE id = 2;
INSERT INTO dml_test VALUES (5, 'Epsilon');
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
UPDATE dml_test SET name = 'gamma' WHERE id = 3;
COMMIT;
WARNING: connection not open
CONTEXT: while executing command on localhost:xxxxx
WARNING: failed to commit transaction on localhost:xxxxx
WARNING: connection not open
CONTEXT: while executing command on localhost:xxxxx
-- all changes should be committed because we injected
-- the failure on the COMMIT time. And, we should not
-- mark any placements as INVALID
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT recover_prepared_transactions();
recover_prepared_transactions
---------------------------------------------------------------------
1
(1 row)
SELECT shardid FROM pg_dist_shard_placement WHERE shardstate = 3;
shardid
---------------------------------------------------------------------
(0 rows)
SET citus.task_assignment_policy TO "round-robin";
SELECT * FROM dml_test ORDER BY id ASC;
id | name
---------------------------------------------------------------------
3 | gamma
4 | Delta
5 | Epsilon
(3 rows)
SELECT * FROM dml_test ORDER BY id ASC;
id | name
---------------------------------------------------------------------
3 | gamma
4 | Delta
5 | Epsilon
(3 rows)
RESET citus.task_assignment_policy;
-- drop table and recreate as reference table
DROP TABLE dml_test;
SET citus.shard_count = 2;
SET citus.shard_replication_factor = 1;
CREATE TABLE dml_test (id integer, name text);
SELECT create_reference_table('dml_test');
create_reference_table
---------------------------------------------------------------------
(1 row)
COPY dml_test FROM STDIN WITH CSV;
-- fail at COMMIT (by failing to PREPARE)
SELECT citus.mitmproxy('conn.onQuery(query="^PREPARE").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
BEGIN;
DELETE FROM dml_test WHERE id = 1;
DELETE FROM dml_test WHERE id = 2;
INSERT INTO dml_test VALUES (5, 'Epsilon');
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
UPDATE dml_test SET name = 'gamma' WHERE id = 3;
COMMIT;
ERROR: connection not open
CONTEXT: while executing command on localhost:xxxxx
--- shouldn't see any changes after failed COMMIT
SELECT * FROM dml_test ORDER BY id ASC;
id | name
---------------------------------------------------------------------
1 | Alpha
2 | Beta
3 | Gamma
4 | Delta
(4 rows)
-- cancel at COMMIT (by cancelling on PREPARE)
SELECT citus.mitmproxy('conn.onQuery(query="^PREPARE").cancel(' || pg_backend_pid() || ')');
mitmproxy
---------------------------------------------------------------------
(1 row)
BEGIN;
DELETE FROM dml_test WHERE id = 1;
DELETE FROM dml_test WHERE id = 2;
INSERT INTO dml_test VALUES (5, 'Epsilon');
UPDATE dml_test SET name = 'alpha' WHERE id = 1;
UPDATE dml_test SET name = 'gamma' WHERE id = 3;
COMMIT;
ERROR: canceling statement due to user request
--- shouldn't see any changes after cancelled PREPARE
SELECT * FROM dml_test ORDER BY id ASC;
id | name
---------------------------------------------------------------------
1 | Alpha
2 | Beta
3 | Gamma
4 | Delta
(4 rows)
-- allow connection to allow DROP
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
DROP TABLE dml_test;

330
src/test/regress/expected/failure_online_move_shard_placement_1.out Normal file
View File

@ -0,0 +1,330 @@
--
-- failure_online_move_shard_placement
--
-- The tests cover moving shard placements using logical replication.
CREATE SCHEMA IF NOT EXISTS move_shard;
SET SEARCH_PATH = move_shard;
SET citus.shard_count TO 4;
SET citus.next_shard_id TO 100;
SET citus.shard_replication_factor TO 1;
SET citus.max_adaptive_executor_pool_size TO 1;
SELECT pg_backend_pid() as pid \gset
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
CREATE TABLE t(id int PRIMARY KEY, int_data int, data text);
CREATE INDEX index_failure ON t(id);
SELECT create_distributed_table('t', 'id');
create_distributed_table
---------------------------------------------------------------------
(1 row)
CREATE VIEW shards_in_workers AS
SELECT shardid,
(CASE WHEN nodeport = :worker_1_port THEN 'worker1' ELSE 'worker2' END) AS worker
FROM pg_dist_placement NATURAL JOIN pg_dist_node
WHERE shardstate != 4
ORDER BY 1,2 ASC;
-- Insert some data
INSERT INTO t SELECT x, x+1, MD5(random()::text) FROM generate_series(1,100000) AS f(x);
-- Initial shard placements
SELECT * FROM shards_in_workers;
shardid | worker
---------------------------------------------------------------------
100 | worker2
101 | worker1
102 | worker2
103 | worker1
(4 rows)
-- failure on sanity checks
SELECT citus.mitmproxy('conn.onQuery(query="DROP TABLE IF EXISTS move_shard.t CASCADE").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT master_move_shard_placement(101, 'localhost', :worker_1_port, 'localhost', :worker_2_proxy_port);
ERROR: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
CONTEXT: while executing command on localhost:xxxxx
-- cancellation on sanity checks
SELECT citus.mitmproxy('conn.onQuery(query="DROP TABLE IF EXISTS move_shard.t CASCADE").cancel(' || :pid || ')');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT master_move_shard_placement(101, 'localhost', :worker_1_port, 'localhost', :worker_2_proxy_port);
ERROR: canceling statement due to user request
-- failure on move_shard table creation
SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE move_shard.t").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT master_move_shard_placement(101, 'localhost', :worker_1_port, 'localhost', :worker_2_proxy_port);
ERROR: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
CONTEXT: while executing command on localhost:xxxxx
-- cancellation on move_shard table creation
SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE move_shard.t").cancel(' || :pid || ')');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT master_move_shard_placement(101, 'localhost', :worker_1_port, 'localhost', :worker_2_proxy_port);
ERROR: canceling statement due to user request
-- failure on polling subscription state
SELECT citus.mitmproxy('conn.onQuery(query="^SELECT count\(\*\) FROM pg_subscription_rel").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT master_move_shard_placement(101, 'localhost', :worker_1_port, 'localhost', :worker_2_proxy_port);
ERROR: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
CONTEXT: while executing command on localhost:xxxxx
-- cancellation on polling subscription state
SELECT citus.mitmproxy('conn.onQuery(query="^SELECT count\(\*\) FROM pg_subscription_rel").cancel(' || :pid || ')');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT master_move_shard_placement(101, 'localhost', :worker_1_port, 'localhost', :worker_2_proxy_port);
ERROR: canceling statement due to user request
-- failure on getting subscriber state
SELECT citus.mitmproxy('conn.onQuery(query="^SELECT sum").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT master_move_shard_placement(101, 'localhost', :worker_1_port, 'localhost', :worker_2_proxy_port);
ERROR: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
CONTEXT: while executing command on localhost:xxxxx
-- cancellation on getting subscriber state
SELECT citus.mitmproxy('conn.onQuery(query="^SELECT sum").cancel(' || :pid || ')');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT master_move_shard_placement(101, 'localhost', :worker_1_port, 'localhost', :worker_2_proxy_port);
ERROR: canceling statement due to user request
-- failure on polling last write-ahead log location reported to origin WAL sender
SELECT citus.mitmproxy('conn.onQuery(query="^SELECT min\(latest_end_lsn").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT master_move_shard_placement(101, 'localhost', :worker_1_port, 'localhost', :worker_2_proxy_port);
ERROR: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
CONTEXT: while executing command on localhost:xxxxx
-- cancellation on polling last write-ahead log location reported to origin WAL sender
SELECT citus.mitmproxy('conn.onQuery(query="^SELECT min\(latest_end_lsn").cancel(' || :pid || ')');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT master_move_shard_placement(101, 'localhost', :worker_1_port, 'localhost', :worker_2_proxy_port);
ERROR: canceling statement due to user request
-- failure on dropping subscription
SELECT citus.mitmproxy('conn.onQuery(query="^DROP SUBSCRIPTION").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT master_move_shard_placement(101, 'localhost', :worker_1_port, 'localhost', :worker_2_proxy_port);
WARNING: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
CONTEXT: while executing command on localhost:xxxxx
WARNING: connection not open
CONTEXT: while executing command on localhost:xxxxx
WARNING: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
CONTEXT: while executing command on localhost:xxxxx
WARNING: connection not open
CONTEXT: while executing command on localhost:xxxxx
ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open
-- cancellation on dropping subscription
SELECT citus.mitmproxy('conn.onQuery(query="^DROP SUBSCRIPTION").cancel(' || :pid || ')');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT master_move_shard_placement(101, 'localhost', :worker_1_port, 'localhost', :worker_2_proxy_port);
ERROR: canceling statement due to user request
-- failure on creating the primary key
SELECT citus.mitmproxy('conn.onQuery(query="t_pkey").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT master_move_shard_placement(101, 'localhost', :worker_1_port, 'localhost', :worker_2_proxy_port);
ERROR: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
CONTEXT: while executing command on localhost:xxxxx
-- cancellation on creating the primary key
SELECT citus.mitmproxy('conn.onQuery(query="t_pkey").cancel(' || :pid || ')');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT master_move_shard_placement(101, 'localhost', :worker_1_port, 'localhost', :worker_2_proxy_port);
ERROR: canceling statement due to user request
-- failure on create index
SELECT citus.mitmproxy('conn.matches(b"CREATE INDEX").killall()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT master_move_shard_placement(101, 'localhost', :worker_1_port, 'localhost', :worker_2_proxy_port);
WARNING: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
CONTEXT: while executing command on localhost:xxxxx
WARNING: connection not open
CONTEXT: while executing command on localhost:xxxxx
ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
-- lets create few more indexes and fail with both
-- parallel mode and sequential mode
CREATE INDEX index_failure_2 ON t(id);
CREATE INDEX index_failure_3 ON t(id);
CREATE INDEX index_failure_4 ON t(id);
CREATE INDEX index_failure_5 ON t(id);
-- failure on the third create index
ALTER SYSTEM SET citus.max_adaptive_executor_pool_size TO 1;
SELECT pg_reload_conf();
pg_reload_conf
---------------------------------------------------------------------
t
(1 row)
SELECT citus.mitmproxy('conn.matches(b"CREATE INDEX").killall()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT master_move_shard_placement(101, 'localhost', :worker_1_port, 'localhost', :worker_2_proxy_port);
WARNING: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
CONTEXT: while executing command on localhost:xxxxx
WARNING: connection not open
CONTEXT: while executing command on localhost:xxxxx
ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
-- failure on parallel create index
ALTER SYSTEM RESET citus.max_adaptive_executor_pool_size;
SELECT pg_reload_conf();
pg_reload_conf
---------------------------------------------------------------------
t
(1 row)
SELECT citus.mitmproxy('conn.matches(b"CREATE INDEX").killall()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT master_move_shard_placement(101, 'localhost', :worker_1_port, 'localhost', :worker_2_proxy_port);
WARNING: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
CONTEXT: while executing command on localhost:xxxxx
WARNING: connection not open
CONTEXT: while executing command on localhost:xxxxx
ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open
-- Verify that the shard is not moved and the number of rows are still 100k
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT * FROM shards_in_workers;
shardid | worker
---------------------------------------------------------------------
100 | worker2
101 | worker1
102 | worker2
103 | worker1
(4 rows)
SELECT count(*) FROM t;
count
---------------------------------------------------------------------
100000
(1 row)
-- Verify that shard can be moved after a temporary failure
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SELECT master_move_shard_placement(101, 'localhost', :worker_1_port, 'localhost', :worker_2_proxy_port);
master_move_shard_placement
---------------------------------------------------------------------
(1 row)
SELECT * FROM shards_in_workers;
shardid | worker
---------------------------------------------------------------------
100 | worker2
101 | worker2
102 | worker2
103 | worker1
(4 rows)
SELECT count(*) FROM t;
count
---------------------------------------------------------------------
100000
(1 row)
DROP SCHEMA move_shard CASCADE ;
NOTICE: drop cascades to 2 other objects
DETAIL: drop cascades to table t
drop cascades to view shards_in_workers

384
src/test/regress/expected/failure_savepoints_1.out Normal file
View File

@ -0,0 +1,384 @@
-- We have two different output files for this failure test because the
-- failure behaviour of SAVEPOINT and RELEASE commands are different if
-- we use the executor. If we use it, these commands error out if any of
-- the placement commands fail. Otherwise, we might mark the placement
-- as invalid and continue with a WARNING.
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
SET citus.shard_count = 2;
SET citus.shard_replication_factor = 1; -- one shard per worker
SET citus.next_shard_id TO 100950;
ALTER SEQUENCE pg_catalog.pg_dist_placement_placementid_seq RESTART 150;
CREATE TABLE artists (
id bigint NOT NULL,
name text NOT NULL
);
SELECT create_distributed_table('artists', 'id');
create_distributed_table
---------------------------------------------------------------------
(1 row)
-- add some data
INSERT INTO artists VALUES (1, 'Pablo Picasso');
INSERT INTO artists VALUES (2, 'Vincent van Gogh');
INSERT INTO artists VALUES (3, 'Claude Monet');
INSERT INTO artists VALUES (4, 'William Kurelek');
-- simply fail at SAVEPOINT
SELECT citus.mitmproxy('conn.onQuery(query="^SAVEPOINT").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
BEGIN;
INSERT INTO artists VALUES (5, 'Asher Lev');
SAVEPOINT s1;
WARNING: connection not open
CONTEXT: while executing command on localhost:xxxxx
WARNING: connection to the remote node localhost:xxxxx failed with the following error: connection not open
WARNING: connection not open
CONTEXT: while executing command on localhost:xxxxx
WARNING: connection not open
CONTEXT: while executing command on localhost:xxxxx
ERROR: connection not open
CONTEXT: while executing command on localhost:xxxxx
DELETE FROM artists WHERE id=4;
ERROR: current transaction is aborted, commands ignored until end of transaction block
RELEASE SAVEPOINT s1;
ERROR: current transaction is aborted, commands ignored until end of transaction block
COMMIT;
SELECT * FROM artists WHERE id IN (4, 5);
id | name
---------------------------------------------------------------------
4 | William Kurelek
(1 row)
-- fail at RELEASE
SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
BEGIN;
UPDATE artists SET name='a';
SAVEPOINT s1;
DELETE FROM artists WHERE id=4;
RELEASE SAVEPOINT s1;
WARNING: AbortSubTransaction while in COMMIT state
WARNING: connection not open
CONTEXT: while executing command on localhost:xxxxx
WARNING: connection to the remote node localhost:xxxxx failed with the following error: connection not open
WARNING: connection not open
CONTEXT: while executing command on localhost:xxxxx
WARNING: connection not open
CONTEXT: while executing command on localhost:xxxxx
WARNING: savepoint "savepoint_2" does not exist
CONTEXT: while executing command on localhost:xxxxx
ERROR: connection not open
CONTEXT: while executing command on localhost:xxxxx
ROLLBACK;
SELECT * FROM artists WHERE id IN (4, 5);
id | name
---------------------------------------------------------------------
4 | William Kurelek
(1 row)
-- fail at ROLLBACK
SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
BEGIN;
INSERT INTO artists VALUES (5, 'Asher Lev');
SAVEPOINT s1;
DELETE FROM artists WHERE id=4;
ROLLBACK TO SAVEPOINT s1;
WARNING: connection not open
CONTEXT: while executing command on localhost:xxxxx
WARNING: connection not open
CONTEXT: while executing command on localhost:xxxxx
COMMIT;
ERROR: failure on connection marked as essential: localhost:xxxxx
SELECT * FROM artists WHERE id IN (4, 5);
id | name
---------------------------------------------------------------------
4 | William Kurelek
(1 row)
-- fail at second RELEASE
SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").after(1).kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
BEGIN;
SAVEPOINT s1;
DELETE FROM artists WHERE id=4;
RELEASE SAVEPOINT s1;
SAVEPOINT s2;
INSERT INTO artists VALUES (5, 'Jacob Kahn');
RELEASE SAVEPOINT s2;
WARNING: AbortSubTransaction while in COMMIT state
WARNING: connection not open
CONTEXT: while executing command on localhost:xxxxx
WARNING: connection to the remote node localhost:xxxxx failed with the following error: connection not open
WARNING: connection not open
CONTEXT: while executing command on localhost:xxxxx
WARNING: connection not open
CONTEXT: while executing command on localhost:xxxxx
ERROR: connection not open
CONTEXT: while executing command on localhost:xxxxx
COMMIT;
SELECT * FROM artists WHERE id IN (4, 5);
id | name
---------------------------------------------------------------------
4 | William Kurelek
(1 row)
-- fail at second ROLLBACK
SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").after(1).kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
BEGIN;
SAVEPOINT s1;
UPDATE artists SET name='A' WHERE id=4;
ROLLBACK TO SAVEPOINT s1;
SAVEPOINT s2;
DELETE FROM artists WHERE id=5;
ROLLBACK TO SAVEPOINT s2;
WARNING: connection not open
CONTEXT: while executing command on localhost:xxxxx
WARNING: connection not open
CONTEXT: while executing command on localhost:xxxxx
COMMIT;
ERROR: failure on connection marked as essential: localhost:xxxxx
SELECT * FROM artists WHERE id IN (4, 5);
id | name
---------------------------------------------------------------------
4 | William Kurelek
(1 row)
SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").after(1).kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
-- Release after rollback
BEGIN;
SAVEPOINT s1;
ROLLBACK TO s1;
RELEASE SAVEPOINT s1;
SAVEPOINT s2;
INSERT INTO artists VALUES (6, 'John J. Audubon');
INSERT INTO artists VALUES (7, 'Emily Carr');
ROLLBACK TO s2;
RELEASE SAVEPOINT s2;
COMMIT;
SELECT * FROM artists WHERE id=7;
id | name
---------------------------------------------------------------------
(0 rows)
SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
-- Recover from errors
\set VERBOSITY terse
BEGIN;
SAVEPOINT s1;
SAVEPOINT s2;
INSERT INTO artists VALUES (6, 'John J. Audubon');
INSERT INTO artists VALUES (7, 'Emily Carr');
INSERT INTO artists VALUES (7, 'Emily Carr');
ROLLBACK TO SAVEPOINT s1;
WARNING: connection not open
WARNING: connection not open
WARNING: connection not open
WARNING: connection to the remote node localhost:xxxxx failed with the following error: connection not open
WARNING: connection not open
WARNING: connection not open
COMMIT;
ERROR: failure on connection marked as essential: localhost:xxxxx
SELECT * FROM artists WHERE id=6;
id | name
---------------------------------------------------------------------
(0 rows)
-- replication factor > 1
CREATE TABLE researchers (
id bigint NOT NULL,
lab_id int NOT NULL,
name text NOT NULL
);
SET citus.shard_count = 1;
SET citus.shard_replication_factor = 2; -- single shard, on both workers
SELECT create_distributed_table('researchers', 'lab_id', 'hash');
create_distributed_table
---------------------------------------------------------------------
(1 row)
-- simply fail at SAVEPOINT
SELECT citus.mitmproxy('conn.onQuery(query="^SAVEPOINT").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
BEGIN;
INSERT INTO researchers VALUES (7, 4, 'Jan Plaza');
SAVEPOINT s1;
WARNING: connection not open
WARNING: connection to the remote node localhost:xxxxx failed with the following error: connection not open
WARNING: connection not open
WARNING: connection not open
ERROR: connection not open
INSERT INTO researchers VALUES (8, 4, 'Alonzo Church');
ERROR: current transaction is aborted, commands ignored until end of transaction block
ROLLBACK TO s1;
ERROR: savepoint "s1" does not exist
RELEASE SAVEPOINT s1;
ERROR: current transaction is aborted, commands ignored until end of transaction block
COMMIT;
-- should see correct results from healthy placement and one bad placement
SELECT * FROM researchers WHERE lab_id = 4;
id | lab_id | name
---------------------------------------------------------------------
(0 rows)
UPDATE pg_dist_shard_placement SET shardstate = 1
WHERE shardstate = 3 AND shardid IN (
SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'researchers'::regclass
) RETURNING placementid;
placementid
---------------------------------------------------------------------
(0 rows)
TRUNCATE researchers;
-- fail at rollback
SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
BEGIN;
INSERT INTO researchers VALUES (7, 4, 'Jan Plaza');
SAVEPOINT s1;
INSERT INTO researchers VALUES (8, 4, 'Alonzo Church');
ROLLBACK TO s1;
WARNING: connection not open
WARNING: connection not open
RELEASE SAVEPOINT s1;
COMMIT;
ERROR: failure on connection marked as essential: localhost:xxxxx
-- should see correct results from healthy placement and one bad placement
SELECT * FROM researchers WHERE lab_id = 4;
id | lab_id | name
---------------------------------------------------------------------
(0 rows)
UPDATE pg_dist_shard_placement SET shardstate = 1
WHERE shardstate = 3 AND shardid IN (
SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'researchers'::regclass
) RETURNING placementid;
placementid
---------------------------------------------------------------------
(0 rows)
TRUNCATE researchers;
-- fail at release
SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
BEGIN;
INSERT INTO researchers VALUES (7, 4, 'Jan Plaza');
SAVEPOINT s1;
INSERT INTO researchers VALUES (8, 4, 'Alonzo Church');
ROLLBACK TO s1;
RELEASE SAVEPOINT s1;
WARNING: AbortSubTransaction while in COMMIT state
WARNING: connection not open
WARNING: connection to the remote node localhost:xxxxx failed with the following error: connection not open
WARNING: connection not open
WARNING: connection not open
WARNING: savepoint "savepoint_3" does not exist
ERROR: connection not open
COMMIT;
-- should see correct results from healthy placement and one bad placement
SELECT * FROM researchers WHERE lab_id = 4;
id | lab_id | name
---------------------------------------------------------------------
(0 rows)
UPDATE pg_dist_shard_placement SET shardstate = 1
WHERE shardstate = 3 AND shardid IN (
SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'researchers'::regclass
) RETURNING placementid;
placementid
---------------------------------------------------------------------
(0 rows)
TRUNCATE researchers;
-- test that we don't mark reference placements unhealthy
CREATE TABLE ref(a int, b int);
SELECT create_reference_table('ref');
create_reference_table
---------------------------------------------------------------------
(1 row)
SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
BEGIN;
SAVEPOINT start;
INSERT INTO ref VALUES (1001,2);
SELECT * FROM ref;
a | b
---------------------------------------------------------------------
1001 | 2
(1 row)
ROLLBACK TO SAVEPOINT start;
WARNING: connection not open
WARNING: connection not open
SELECT * FROM ref;
WARNING: connection not open
WARNING: connection to the remote node localhost:xxxxx failed with the following error: connection not open
WARNING: connection not open
WARNING: connection not open
ERROR: connection to the remote node localhost:xxxxx failed with the following error: connection not open
END;
-- clean up
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
DROP TABLE artists;
DROP TABLE researchers;
DROP TABLE ref;

1281
src/test/regress/expected/failure_truncate_1.out Normal file
View File

File diff suppressed because it is too large Load Diff

45
src/test/regress/expected/failure_vacuum_1.out
View File

@ -10,7 +10,6 @@ SELECT citus.mitmproxy('conn.allow()');
SET citus.shard_count = 1; SET citus.shard_count = 1;
SET citus.shard_replication_factor = 2; -- one shard per worker SET citus.shard_replication_factor = 2; -- one shard per worker
SET citus.multi_shard_commit_protocol TO '1pc';
CREATE TABLE vacuum_test (key int, value int); CREATE TABLE vacuum_test (key int, value int);
SELECT create_distributed_table('vacuum_test', 'key'); SELECT create_distributed_table('vacuum_test', 'key');
create_distributed_table create_distributed_table
@ -31,8 +30,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^VACUUM").kill()');
(1 row) (1 row)
VACUUM vacuum_test; VACUUM vacuum_test;
ERROR: connection error: localhost:xxxxx ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
DETAIL: server closed the connection unexpectedly
This probably means the server terminated abnormally This probably means the server terminated abnormally
before or while processing the request. before or while processing the request.
SELECT citus.mitmproxy('conn.onQuery(query="^ANALYZE").kill()'); SELECT citus.mitmproxy('conn.onQuery(query="^ANALYZE").kill()');
@ -42,8 +40,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^ANALYZE").kill()');
(1 row) (1 row)
ANALYZE vacuum_test; ANALYZE vacuum_test;
WARNING: connection error: localhost:xxxxx ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
DETAIL: server closed the connection unexpectedly
This probably means the server terminated abnormally This probably means the server terminated abnormally
before or while processing the request. before or while processing the request.
SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT").kill()'); SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT").kill()');
@ -53,19 +50,31 @@ SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT").kill()');
(1 row) (1 row)
ANALYZE vacuum_test; ANALYZE vacuum_test;
-- ANALYZE transactions being critical is an open question, see #2430 WARNING: connection not open
-- show that we marked as INVALID on COMMIT FAILURE CONTEXT: while executing command on localhost:xxxxx
SELECT shardid, shardstate FROM pg_dist_shard_placement where shardstate != 1 AND WARNING: failed to commit transaction on localhost:xxxxx
shardid in ( SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'vacuum_test'::regclass); WARNING: connection not open
shardid | shardstate CONTEXT: while executing command on localhost:xxxxx
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
--------------------------------------------------------------------- ---------------------------------------------------------------------
12000000 | 3
(1 row) (1 row)
UPDATE pg_dist_shard_placement SET shardstate = 1 SELECT recover_prepared_transactions();
WHERE shardid IN ( recover_prepared_transactions
SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'vacuum_test'::regclass ---------------------------------------------------------------------
); 1
(1 row)
-- ANALYZE transactions being critical is an open question, see #2430
-- show that we never mark as INVALID on COMMIT FAILURE
SELECT shardid, shardstate FROM pg_dist_shard_placement where shardstate != 1 AND
shardid in ( SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'vacuum_test'::regclass);
shardid | shardstate
---------------------------------------------------------------------
(0 rows)
-- the same tests with cancel -- the same tests with cancel
SELECT citus.mitmproxy('conn.onQuery(query="^VACUUM").cancel(' || pg_backend_pid() || ')'); SELECT citus.mitmproxy('conn.onQuery(query="^VACUUM").cancel(' || pg_backend_pid() || ')');
mitmproxy mitmproxy
@ -111,7 +120,9 @@ SELECT citus.mitmproxy('conn.onQuery(query="^VACUUM.*other").kill()');
(1 row) (1 row)
VACUUM vacuum_test, other_vacuum_test; VACUUM vacuum_test, other_vacuum_test;
ERROR: syntax error at or near "," ERROR: connection to the remote node localhost:xxxxx failed with the following error: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
SELECT citus.mitmproxy('conn.onQuery(query="^VACUUM.*other").cancel(' || pg_backend_pid() || ')'); SELECT citus.mitmproxy('conn.onQuery(query="^VACUUM.*other").cancel(' || pg_backend_pid() || ')');
mitmproxy mitmproxy
--------------------------------------------------------------------- ---------------------------------------------------------------------
@ -119,7 +130,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^VACUUM.*other").cancel(' || pg_bac
(1 row) (1 row)
VACUUM vacuum_test, other_vacuum_test; VACUUM vacuum_test, other_vacuum_test;
ERROR: syntax error at or near "," ERROR: canceling statement due to user request
-- ==== Clean up, we're done here ==== -- ==== Clean up, we're done here ====
SELECT citus.mitmproxy('conn.allow()'); SELECT citus.mitmproxy('conn.allow()');
mitmproxy mitmproxy