mirror of https://github.com/citusdata/citus.git
742 lines
31 KiB
Plaintext
742 lines
31 KiB
Plaintext
-- The test excercises below failure scenarios
|
|
--1. Failure while creating publications
|
|
--2. Failure while creating shared memory segment
|
|
--3. Failure while creating replication slots
|
|
--4. Failure while enabling subscription
|
|
--5. Failure on polling last write-ahead log location reported to origin WAL sender
|
|
--6. Failure on dropping subscription
|
|
CREATE SCHEMA "citus_failure_split_cleanup_schema";
|
|
SET search_path TO "citus_failure_split_cleanup_schema";
|
|
SET citus.next_shard_id TO 8981000;
|
|
SET citus.next_placement_id TO 8610000;
|
|
SET citus.next_operation_id TO 777;
|
|
SET citus.next_cleanup_record_id TO 11;
|
|
SET citus.shard_count TO 2;
|
|
SET citus.shard_replication_factor TO 1;
|
|
SELECT pg_backend_pid() as pid \gset
|
|
-- cleanup any leftovers from previous tests so we get consistent output
|
|
SELECT public.wait_for_resource_cleanup();
|
|
wait_for_resource_cleanup
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
-- Disable defer shard delete to stop auto cleanup.
|
|
ALTER SYSTEM SET citus.defer_shard_delete_interval TO -1;
|
|
SELECT pg_reload_conf();
|
|
pg_reload_conf
|
|
---------------------------------------------------------------------
|
|
t
|
|
(1 row)
|
|
|
|
-- Connections on the proxy port(worker_2) are monitored
|
|
SELECT nodeid AS worker_1_node FROM pg_dist_node WHERE nodeport=:worker_1_port \gset
|
|
SELECT nodeid AS worker_2_node FROM pg_dist_node WHERE nodeport=:worker_2_proxy_port \gset
|
|
CREATE TABLE table_to_split(id int PRIMARY KEY, int_data int, data text);
|
|
SELECT create_distributed_table('table_to_split', 'id');
|
|
create_distributed_table
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
--1. Failure while creating publications
|
|
SELECT citus.mitmproxy('conn.onQuery(query="CREATE PUBLICATION .* FOR TABLE").killall()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT pg_catalog.citus_split_shard_by_split_points(
|
|
8981000,
|
|
ARRAY['-100000'],
|
|
ARRAY[:worker_1_node, :worker_2_node],
|
|
'force_logical');
|
|
ERROR: connection not open
|
|
CONTEXT: while executing command on localhost:xxxxx
|
|
SELECT operation_id, object_type, object_name, node_group_id, policy_type
|
|
FROM pg_dist_cleanup where operation_id = 777 ORDER BY object_name;
|
|
operation_id | object_type | object_name | node_group_id | policy_type
|
|
---------------------------------------------------------------------
|
|
777 | 1 | citus_failure_split_cleanup_schema.table_to_split_8981000 | 1 | 0
|
|
777 | 1 | citus_failure_split_cleanup_schema.table_to_split_8981002 | 1 | 1
|
|
777 | 1 | citus_failure_split_cleanup_schema.table_to_split_8981002 | 2 | 0
|
|
777 | 1 | citus_failure_split_cleanup_schema.table_to_split_8981003 | 2 | 1
|
|
777 | 4 | citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx | 2 | 0
|
|
(5 rows)
|
|
|
|
-- we need to allow connection so that we can connect to proxy
|
|
SELECT citus.mitmproxy('conn.allow()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
\c - - - :worker_2_proxy_port
|
|
SET search_path TO "citus_failure_split_cleanup_schema", public, pg_catalog;
|
|
SET citus.show_shards_for_app_name_prefixes = '*';
|
|
-- Left over child shards
|
|
SELECT relname FROM pg_class where relname LIKE '%table_to_split_%' AND relkind = 'r' order by relname;
|
|
relname
|
|
---------------------------------------------------------------------
|
|
table_to_split_8981000
|
|
table_to_split_8981002
|
|
table_to_split_8981003
|
|
(3 rows)
|
|
|
|
-- Left over publications
|
|
SELECT pubname FROM pg_publication;
|
|
pubname
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
-- Left over replication slots
|
|
SELECT slot_name FROM pg_replication_slots;
|
|
slot_name
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
-- Left over subscriptions
|
|
SELECT subname FROM pg_subscription;
|
|
subname
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
\c - postgres - :master_port
|
|
SELECT public.wait_for_resource_cleanup();
|
|
wait_for_resource_cleanup
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT operation_id, object_type, object_name, node_group_id, policy_type
|
|
FROM pg_dist_cleanup where operation_id = 777 ORDER BY object_name;
|
|
operation_id | object_type | object_name | node_group_id | policy_type
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
\c - - - :worker_2_proxy_port
|
|
SET search_path TO "citus_failure_split_cleanup_schema", public, pg_catalog;
|
|
SET citus.show_shards_for_app_name_prefixes = '*';
|
|
-- Empty child shards after cleanup
|
|
SELECT relname FROM pg_class where relname LIKE '%table_to_split_%' AND relkind = 'r' order by relname;
|
|
relname
|
|
---------------------------------------------------------------------
|
|
table_to_split_8981000
|
|
(1 row)
|
|
|
|
-- Empty publications
|
|
SELECT pubname FROM pg_publication;
|
|
pubname
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
-- Empty replication slot table
|
|
SELECT slot_name FROM pg_replication_slots;
|
|
slot_name
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
-- Empty subscriptions
|
|
SELECT subname FROM pg_subscription;
|
|
subname
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
--2. Failure while creating shared memory segment
|
|
\c - postgres - :master_port
|
|
SET citus.next_shard_id TO 8981002;
|
|
SET citus.next_operation_id TO 777;
|
|
SET citus.next_cleanup_record_id TO 11;
|
|
SELECT citus.mitmproxy('conn.onQuery(query="SELECT \* FROM pg_catalog.worker_split_shard_replication_setup\(.*").killall()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
-- set log level to prevent flakiness
|
|
SET client_min_messages TO ERROR;
|
|
SELECT pg_catalog.citus_split_shard_by_split_points(
|
|
8981000,
|
|
ARRAY['-100000'],
|
|
ARRAY[:worker_1_node, :worker_2_node],
|
|
'force_logical');
|
|
ERROR: Failed to run worker_split_shard_replication_setup UDF. It should successfully execute for splitting a shard in a non-blocking way. Please retry.
|
|
RESET client_min_messages;
|
|
SELECT operation_id, object_type, object_name, node_group_id, policy_type
|
|
FROM pg_dist_cleanup where operation_id = 777 ORDER BY object_name;
|
|
operation_id | object_type | object_name | node_group_id | policy_type
|
|
---------------------------------------------------------------------
|
|
777 | 1 | citus_failure_split_cleanup_schema.table_to_split_8981000 | 1 | 0
|
|
777 | 1 | citus_failure_split_cleanup_schema.table_to_split_8981002 | 1 | 1
|
|
777 | 1 | citus_failure_split_cleanup_schema.table_to_split_8981002 | 2 | 0
|
|
777 | 1 | citus_failure_split_cleanup_schema.table_to_split_8981003 | 2 | 1
|
|
777 | 4 | citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx | 2 | 0
|
|
777 | 4 | citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx | 2 | 0
|
|
(6 rows)
|
|
|
|
-- we need to allow connection so that we can connect to proxy
|
|
SELECT citus.mitmproxy('conn.allow()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
\c - - - :worker_2_proxy_port
|
|
SET search_path TO "citus_failure_split_cleanup_schema", public, pg_catalog;
|
|
SET citus.show_shards_for_app_name_prefixes = '*';
|
|
-- Left over child shards
|
|
SELECT relname FROM pg_class where relname LIKE '%table_to_split_%' AND relkind = 'r' order by relname;
|
|
relname
|
|
---------------------------------------------------------------------
|
|
table_to_split_8981000
|
|
table_to_split_8981002
|
|
table_to_split_8981003
|
|
(3 rows)
|
|
|
|
-- Left over publications
|
|
SELECT pubname FROM pg_publication;
|
|
pubname
|
|
---------------------------------------------------------------------
|
|
citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx
|
|
citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx
|
|
(2 rows)
|
|
|
|
-- Left over replication slots
|
|
SELECT slot_name FROM pg_replication_slots;
|
|
slot_name
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
-- Left over subscriptions
|
|
SELECT subname FROM pg_subscription;
|
|
subname
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
\c - postgres - :master_port
|
|
SELECT public.wait_for_resource_cleanup();
|
|
wait_for_resource_cleanup
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT operation_id, object_type, object_name, node_group_id, policy_type
|
|
FROM pg_dist_cleanup where operation_id = 777 ORDER BY object_name;
|
|
operation_id | object_type | object_name | node_group_id | policy_type
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
\c - - - :worker_2_proxy_port
|
|
SET search_path TO "citus_failure_split_cleanup_schema", public, pg_catalog;
|
|
SET citus.show_shards_for_app_name_prefixes = '*';
|
|
-- Empty child shards after cleanup
|
|
SELECT relname FROM pg_class where relname LIKE '%table_to_split_%' AND relkind = 'r' order by relname;
|
|
relname
|
|
---------------------------------------------------------------------
|
|
table_to_split_8981000
|
|
(1 row)
|
|
|
|
-- Empty publications
|
|
SELECT pubname FROM pg_publication;
|
|
pubname
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
-- Empty replication slot table
|
|
SELECT slot_name FROM pg_replication_slots;
|
|
slot_name
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
-- Empty subscriptions
|
|
SELECT subname FROM pg_subscription;
|
|
subname
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
--3. Failure while executing 'CREATE_REPLICATION_SLOT' for Snapshot.
|
|
\c - postgres - :master_port
|
|
SET citus.next_shard_id TO 8981002;
|
|
SET citus.next_operation_id TO 777;
|
|
SET citus.next_cleanup_record_id TO 11;
|
|
SELECT citus.mitmproxy('conn.onQuery(query="CREATE_REPLICATION_SLOT .* LOGICAL .* EXPORT_SNAPSHOT.*").killall()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT pg_catalog.citus_split_shard_by_split_points(
|
|
8981000,
|
|
ARRAY['-100000'],
|
|
ARRAY[:worker_1_node, :worker_2_node],
|
|
'force_logical');
|
|
WARNING: connection not open
|
|
CONTEXT: while executing command on localhost:xxxxx
|
|
WARNING: connection not open
|
|
CONTEXT: while executing command on localhost:xxxxx
|
|
ERROR: connection not open
|
|
CONTEXT: while executing command on localhost:xxxxx
|
|
SELECT operation_id, object_type, object_name, node_group_id, policy_type
|
|
FROM pg_dist_cleanup where operation_id = 777 ORDER BY object_name;
|
|
operation_id | object_type | object_name | node_group_id | policy_type
|
|
---------------------------------------------------------------------
|
|
777 | 1 | citus_failure_split_cleanup_schema.table_to_split_8981000 | 1 | 0
|
|
777 | 1 | citus_failure_split_cleanup_schema.table_to_split_8981002 | 2 | 0
|
|
777 | 1 | citus_failure_split_cleanup_schema.table_to_split_8981002 | 1 | 1
|
|
777 | 1 | citus_failure_split_cleanup_schema.table_to_split_8981003 | 2 | 1
|
|
777 | 4 | citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx | 2 | 0
|
|
777 | 4 | citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx | 2 | 0
|
|
777 | 3 | citus_shard_split_slot_xxxxxxx_xxxxxxx_xxxxxxx | 2 | 0
|
|
(7 rows)
|
|
|
|
-- we need to allow connection so that we can connect to proxy
|
|
SELECT citus.mitmproxy('conn.allow()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
\c - - - :worker_2_proxy_port
|
|
SET search_path TO "citus_failure_split_cleanup_schema", public, pg_catalog;
|
|
SET citus.show_shards_for_app_name_prefixes = '*';
|
|
-- Left over child shards
|
|
SELECT relname FROM pg_class where relname LIKE '%table_to_split_%' AND relkind = 'r' order by relname;
|
|
relname
|
|
---------------------------------------------------------------------
|
|
table_to_split_8981000
|
|
table_to_split_8981002
|
|
table_to_split_8981003
|
|
(3 rows)
|
|
|
|
-- Left over publications
|
|
SELECT pubname FROM pg_publication;
|
|
pubname
|
|
---------------------------------------------------------------------
|
|
citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx
|
|
citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx
|
|
(2 rows)
|
|
|
|
-- Left over replication slots
|
|
SELECT slot_name FROM pg_replication_slots;
|
|
slot_name
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
-- Left over subscriptions
|
|
SELECT subname FROM pg_subscription;
|
|
subname
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
\c - postgres - :master_port
|
|
SELECT public.wait_for_resource_cleanup();
|
|
wait_for_resource_cleanup
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT operation_id, object_type, object_name, node_group_id, policy_type
|
|
FROM pg_dist_cleanup where operation_id = 777 ORDER BY object_name;
|
|
operation_id | object_type | object_name | node_group_id | policy_type
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
\c - - - :worker_2_proxy_port
|
|
SET search_path TO "citus_failure_split_cleanup_schema", public, pg_catalog;
|
|
SET citus.show_shards_for_app_name_prefixes = '*';
|
|
-- Empty child shards after cleanup
|
|
SELECT relname FROM pg_class where relname LIKE '%table_to_split_%' AND relkind = 'r' order by relname;
|
|
relname
|
|
---------------------------------------------------------------------
|
|
table_to_split_8981000
|
|
(1 row)
|
|
|
|
-- Empty publications
|
|
SELECT pubname FROM pg_publication;
|
|
pubname
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
-- Empty replication slot table
|
|
SELECT slot_name FROM pg_replication_slots;
|
|
slot_name
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
-- Empty subscriptions
|
|
SELECT subname FROM pg_subscription;
|
|
subname
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
--4. Failure while enabling subscription
|
|
\c - postgres - :master_port
|
|
SET citus.next_shard_id TO 8981002;
|
|
SET citus.next_operation_id TO 777;
|
|
SET citus.next_cleanup_record_id TO 11;
|
|
SELECT citus.mitmproxy('conn.onQuery(query="ALTER SUBSCRIPTION .* ENABLE").killall()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT pg_catalog.citus_split_shard_by_split_points(
|
|
8981000,
|
|
ARRAY['-100000'],
|
|
ARRAY[:worker_1_node, :worker_2_node],
|
|
'force_logical');
|
|
ERROR: connection not open
|
|
CONTEXT: while executing command on localhost:xxxxx
|
|
SELECT operation_id, object_type, object_name, node_group_id, policy_type
|
|
FROM pg_dist_cleanup where operation_id = 777 ORDER BY object_name;
|
|
operation_id | object_type | object_name | node_group_id | policy_type
|
|
---------------------------------------------------------------------
|
|
777 | 1 | citus_failure_split_cleanup_schema.table_to_split_8981000 | 1 | 0
|
|
777 | 1 | citus_failure_split_cleanup_schema.table_to_split_8981002 | 1 | 1
|
|
777 | 1 | citus_failure_split_cleanup_schema.table_to_split_8981002 | 2 | 0
|
|
777 | 1 | citus_failure_split_cleanup_schema.table_to_split_8981003 | 2 | 1
|
|
777 | 4 | citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx | 2 | 0
|
|
777 | 4 | citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx | 2 | 0
|
|
777 | 3 | citus_shard_split_slot_xxxxxxx_xxxxxxx_xxxxxxx | 2 | 0
|
|
777 | 3 | citus_shard_split_slot_xxxxxxx_xxxxxxx_xxxxxxx | 2 | 0
|
|
777 | 2 | citus_shard_split_subscription_xxxxxxx_xxxxxxx | 1 | 0
|
|
777 | 2 | citus_shard_split_subscription_xxxxxxx_xxxxxxx | 2 | 0
|
|
777 | 5 | citus_shard_split_subscription_role_xxxxxxx_xxxxxxx | 1 | 0
|
|
777 | 5 | citus_shard_split_subscription_role_xxxxxxx_xxxxxxx | 2 | 0
|
|
(12 rows)
|
|
|
|
-- we need to allow connection so that we can connect to proxy
|
|
SELECT citus.mitmproxy('conn.allow()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
\c - - - :worker_2_proxy_port
|
|
SET search_path TO "citus_failure_split_cleanup_schema", public, pg_catalog;
|
|
SET citus.show_shards_for_app_name_prefixes = '*';
|
|
-- Left over child shards
|
|
SELECT relname FROM pg_class where relname LIKE '%table_to_split_%' AND relkind = 'r' order by relname;
|
|
relname
|
|
---------------------------------------------------------------------
|
|
table_to_split_8981000
|
|
table_to_split_8981002
|
|
table_to_split_8981003
|
|
(3 rows)
|
|
|
|
-- Left over publications
|
|
SELECT pubname FROM pg_publication;
|
|
pubname
|
|
---------------------------------------------------------------------
|
|
citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx
|
|
citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx
|
|
(2 rows)
|
|
|
|
-- Left over replication slots
|
|
SELECT slot_name FROM pg_replication_slots;
|
|
slot_name
|
|
---------------------------------------------------------------------
|
|
citus_shard_split_slot_xxxxxxx_xxxxxxx_xxxxxxx
|
|
citus_shard_split_slot_xxxxxxx_xxxxxxx_xxxxxxx
|
|
(2 rows)
|
|
|
|
-- Left over subscriptions
|
|
SELECT subname FROM pg_subscription;
|
|
subname
|
|
---------------------------------------------------------------------
|
|
citus_shard_split_subscription_xxxxxxx_xxxxxxx
|
|
(1 row)
|
|
|
|
\c - postgres - :master_port
|
|
SELECT public.wait_for_resource_cleanup();
|
|
wait_for_resource_cleanup
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT operation_id, object_type, object_name, node_group_id, policy_type
|
|
FROM pg_dist_cleanup where operation_id = 777 ORDER BY object_name;
|
|
operation_id | object_type | object_name | node_group_id | policy_type
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
\c - - - :worker_2_proxy_port
|
|
SET search_path TO "citus_failure_split_cleanup_schema", public, pg_catalog;
|
|
SET citus.show_shards_for_app_name_prefixes = '*';
|
|
-- Empty child shards after cleanup
|
|
SELECT relname FROM pg_class where relname LIKE '%table_to_split_%' AND relkind = 'r' order by relname;
|
|
relname
|
|
---------------------------------------------------------------------
|
|
table_to_split_8981000
|
|
(1 row)
|
|
|
|
-- Empty publications
|
|
SELECT pubname FROM pg_publication;
|
|
pubname
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
-- Empty replication slot table
|
|
SELECT slot_name FROM pg_replication_slots;
|
|
slot_name
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
-- Empty subscriptions
|
|
SELECT subname FROM pg_subscription;
|
|
subname
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
--5. Failure on polling last write-ahead log location reported to origin WAL sender
|
|
\c - postgres - :master_port
|
|
SET citus.next_shard_id TO 8981002;
|
|
SET citus.next_operation_id TO 777;
|
|
SET citus.next_cleanup_record_id TO 11;
|
|
SELECT citus.mitmproxy('conn.onQuery(query="^SELECT min\(latest_end_lsn").killall()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT pg_catalog.citus_split_shard_by_split_points(
|
|
8981000,
|
|
ARRAY['-100000'],
|
|
ARRAY[:worker_1_node, :worker_2_node],
|
|
'force_logical');
|
|
ERROR: connection not open
|
|
CONTEXT: while executing command on localhost:xxxxx
|
|
SELECT operation_id, object_type, object_name, node_group_id, policy_type
|
|
FROM pg_dist_cleanup where operation_id = 777 ORDER BY object_name;
|
|
operation_id | object_type | object_name | node_group_id | policy_type
|
|
---------------------------------------------------------------------
|
|
777 | 1 | citus_failure_split_cleanup_schema.table_to_split_8981000 | 1 | 0
|
|
777 | 1 | citus_failure_split_cleanup_schema.table_to_split_8981002 | 1 | 1
|
|
777 | 1 | citus_failure_split_cleanup_schema.table_to_split_8981002 | 2 | 0
|
|
777 | 1 | citus_failure_split_cleanup_schema.table_to_split_8981003 | 2 | 1
|
|
777 | 4 | citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx | 2 | 0
|
|
777 | 4 | citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx | 2 | 0
|
|
777 | 3 | citus_shard_split_slot_xxxxxxx_xxxxxxx_xxxxxxx | 2 | 0
|
|
777 | 3 | citus_shard_split_slot_xxxxxxx_xxxxxxx_xxxxxxx | 2 | 0
|
|
777 | 2 | citus_shard_split_subscription_xxxxxxx_xxxxxxx | 1 | 0
|
|
777 | 2 | citus_shard_split_subscription_xxxxxxx_xxxxxxx | 2 | 0
|
|
777 | 5 | citus_shard_split_subscription_role_xxxxxxx_xxxxxxx | 1 | 0
|
|
777 | 5 | citus_shard_split_subscription_role_xxxxxxx_xxxxxxx | 2 | 0
|
|
(12 rows)
|
|
|
|
-- we need to allow connection so that we can connect to proxy
|
|
SELECT citus.mitmproxy('conn.allow()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
\c - - - :worker_2_proxy_port
|
|
SET search_path TO "citus_failure_split_cleanup_schema", public, pg_catalog;
|
|
SET citus.show_shards_for_app_name_prefixes = '*';
|
|
-- Left over child shards
|
|
SELECT relname FROM pg_class where relname LIKE '%table_to_split_%' AND relkind = 'r' order by relname;
|
|
relname
|
|
---------------------------------------------------------------------
|
|
table_to_split_8981000
|
|
table_to_split_8981002
|
|
table_to_split_8981003
|
|
(3 rows)
|
|
|
|
-- Left over publications
|
|
SELECT pubname FROM pg_publication;
|
|
pubname
|
|
---------------------------------------------------------------------
|
|
citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx
|
|
citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx
|
|
(2 rows)
|
|
|
|
-- Left over replication slots
|
|
SELECT slot_name FROM pg_replication_slots;
|
|
slot_name
|
|
---------------------------------------------------------------------
|
|
citus_shard_split_slot_xxxxxxx_xxxxxxx_xxxxxxx
|
|
citus_shard_split_slot_xxxxxxx_xxxxxxx_xxxxxxx
|
|
(2 rows)
|
|
|
|
-- Left over subscriptions
|
|
SELECT subname FROM pg_subscription;
|
|
subname
|
|
---------------------------------------------------------------------
|
|
citus_shard_split_subscription_xxxxxxx_xxxxxxx
|
|
(1 row)
|
|
|
|
\c - postgres - :master_port
|
|
SELECT public.wait_for_resource_cleanup();
|
|
wait_for_resource_cleanup
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT operation_id, object_type, object_name, node_group_id, policy_type
|
|
FROM pg_dist_cleanup where operation_id = 777 ORDER BY object_name;
|
|
operation_id | object_type | object_name | node_group_id | policy_type
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
\c - - - :worker_2_proxy_port
|
|
SET search_path TO "citus_failure_split_cleanup_schema", public, pg_catalog;
|
|
SET citus.show_shards_for_app_name_prefixes = '*';
|
|
-- Empty child shards after cleanup
|
|
SELECT relname FROM pg_class where relname LIKE '%table_to_split_%' AND relkind = 'r' order by relname;
|
|
relname
|
|
---------------------------------------------------------------------
|
|
table_to_split_8981000
|
|
(1 row)
|
|
|
|
-- Empty publications
|
|
SELECT pubname FROM pg_publication;
|
|
pubname
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
-- Empty replication slot table
|
|
SELECT slot_name FROM pg_replication_slots;
|
|
slot_name
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
-- Empty subscriptions
|
|
SELECT subname FROM pg_subscription;
|
|
subname
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
--6. Failure on dropping subscription
|
|
\c - postgres - :master_port
|
|
SET citus.next_shard_id TO 8981002;
|
|
SET citus.next_operation_id TO 777;
|
|
SET citus.next_cleanup_record_id TO 11;
|
|
SELECT citus.mitmproxy('conn.onQuery(query="^DROP SUBSCRIPTION").killall()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT pg_catalog.citus_split_shard_by_split_points(
|
|
8981000,
|
|
ARRAY['-100000'],
|
|
ARRAY[:worker_1_node, :worker_2_node],
|
|
'force_logical');
|
|
WARNING: connection not open
|
|
CONTEXT: while executing command on localhost:xxxxx
|
|
WARNING: connection not open
|
|
CONTEXT: while executing command on localhost:xxxxx
|
|
WARNING: connection to the remote node localhost:xxxxx failed with the following error: connection not open
|
|
WARNING: connection to the remote node localhost:xxxxx failed with the following error: connection not open
|
|
WARNING: connection to the remote node localhost:xxxxx failed with the following error: connection not open
|
|
WARNING: connection to the remote node localhost:xxxxx failed with the following error: connection not open
|
|
ERROR: connection not open
|
|
CONTEXT: while executing command on localhost:xxxxx
|
|
SELECT operation_id, object_type, object_name, node_group_id, policy_type
|
|
FROM pg_dist_cleanup where operation_id = 777 ORDER BY object_name;
|
|
operation_id | object_type | object_name | node_group_id | policy_type
|
|
---------------------------------------------------------------------
|
|
777 | 1 | citus_failure_split_cleanup_schema.table_to_split_8981002 | 1 | 1
|
|
777 | 1 | citus_failure_split_cleanup_schema.table_to_split_8981003 | 2 | 1
|
|
777 | 4 | citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx | 2 | 0
|
|
777 | 4 | citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx | 2 | 0
|
|
777 | 3 | citus_shard_split_slot_xxxxxxx_xxxxxxx_xxxxxxx | 2 | 0
|
|
777 | 3 | citus_shard_split_slot_xxxxxxx_xxxxxxx_xxxxxxx | 2 | 0
|
|
777 | 2 | citus_shard_split_subscription_xxxxxxx_xxxxxxx | 2 | 0
|
|
777 | 5 | citus_shard_split_subscription_role_xxxxxxx_xxxxxxx | 2 | 0
|
|
(8 rows)
|
|
|
|
SELECT relname FROM pg_class where relname LIKE '%table_to_split_%' AND relkind = 'r' order by relname;
|
|
relname
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
-- we need to allow connection so that we can connect to proxy
|
|
SELECT citus.mitmproxy('conn.allow()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
\c - - - :worker_2_proxy_port
|
|
SET search_path TO "citus_failure_split_cleanup_schema", public, pg_catalog;
|
|
SET citus.show_shards_for_app_name_prefixes = '*';
|
|
-- Left over child shards
|
|
SELECT relname FROM pg_class where relname LIKE '%table_to_split_%' AND relkind = 'r' order by relname;
|
|
relname
|
|
---------------------------------------------------------------------
|
|
table_to_split_8981000
|
|
table_to_split_8981003
|
|
(2 rows)
|
|
|
|
-- Left over publications
|
|
SELECT pubname FROM pg_publication;
|
|
pubname
|
|
---------------------------------------------------------------------
|
|
citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx
|
|
citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx
|
|
(2 rows)
|
|
|
|
-- Left over replication slots
|
|
SELECT slot_name FROM pg_replication_slots;
|
|
slot_name
|
|
---------------------------------------------------------------------
|
|
citus_shard_split_slot_xxxxxxx_xxxxxxx_xxxxxxx
|
|
citus_shard_split_slot_xxxxxxx_xxxxxxx_xxxxxxx
|
|
(2 rows)
|
|
|
|
-- Left over subscriptions
|
|
SELECT subname FROM pg_subscription;
|
|
subname
|
|
---------------------------------------------------------------------
|
|
citus_shard_split_subscription_xxxxxxx_xxxxxxx
|
|
(1 row)
|
|
|
|
\c - postgres - :master_port
|
|
SELECT public.wait_for_resource_cleanup();
|
|
wait_for_resource_cleanup
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT operation_id, object_type, object_name, node_group_id, policy_type
|
|
FROM pg_dist_cleanup where operation_id = 777 ORDER BY object_name;
|
|
operation_id | object_type | object_name | node_group_id | policy_type
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
\c - - - :worker_2_proxy_port
|
|
SET search_path TO "citus_failure_split_cleanup_schema", public, pg_catalog;
|
|
SET citus.show_shards_for_app_name_prefixes = '*';
|
|
-- Empty child shards after cleanup
|
|
SELECT relname FROM pg_class where relname LIKE '%table_to_split_%' AND relkind = 'r' order by relname;
|
|
relname
|
|
---------------------------------------------------------------------
|
|
table_to_split_8981000
|
|
(1 row)
|
|
|
|
-- Empty publications
|
|
SELECT pubname FROM pg_publication;
|
|
pubname
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
-- Empty replication slot table
|
|
SELECT slot_name FROM pg_replication_slots;
|
|
slot_name
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
-- Empty subscriptions
|
|
SELECT subname FROM pg_subscription;
|
|
subname
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
-- Cleanup
|
|
\c - postgres - :master_port
|
|
SET client_min_messages TO WARNING;
|
|
DROP SCHEMA "citus_failure_split_cleanup_schema" CASCADE;
|
|
-- Cleanup
|