-- The test excercises below failure scenarios
--1. Failure while creating publications
--2. Failure while creating shared memory segment
--3. Failure while creating replication slots
--4. Failure while enabling subscription
--5. Failure on polling last write-ahead log location reported to origin WAL sender
--6. Failure on dropping subscription
CREATE SCHEMA "citus_failure_split_cleanup_schema";
SET search_path TO "citus_failure_split_cleanup_schema";
SET citus.next_shard_id TO 8981000;
SET citus.next_placement_id TO 8610000;
SET citus.next_operation_id TO 777;
SET citus.next_cleanup_record_id TO 11;
SET citus.shard_count TO 2;
SET citus.shard_replication_factor TO 1;
SELECT pg_backend_pid() as pid \gset
-- cleanup any leftovers from previous tests so we get consistent output
SELECT public.wait_for_resource_cleanup();
 wait_for_resource_cleanup
---------------------------------------------------------------------

(1 row)

-- Disable defer shard delete to stop auto cleanup.
ALTER SYSTEM SET citus.defer_shard_delete_interval TO -1;
SELECT pg_reload_conf();
 pg_reload_conf
---------------------------------------------------------------------
 t
(1 row)

-- Connections on the proxy port(worker_2) are monitored
SELECT nodeid AS worker_1_node FROM pg_dist_node WHERE nodeport=:worker_1_port \gset
SELECT nodeid AS worker_2_node FROM pg_dist_node WHERE nodeport=:worker_2_proxy_port \gset
CREATE TABLE table_to_split(id int PRIMARY KEY, int_data int, data text);
SELECT create_distributed_table('table_to_split', 'id');
 create_distributed_table
---------------------------------------------------------------------

(1 row)

--1. Failure while creating publications
    SELECT citus.mitmproxy('conn.onQuery(query="CREATE PUBLICATION .* FOR TABLE").killall()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

    SELECT pg_catalog.citus_split_shard_by_split_points(
        8981000,
        ARRAY['-100000'],
        ARRAY[:worker_1_node, :worker_2_node],
        'force_logical');
ERROR:  connection not open
CONTEXT:  while executing command on localhost:xxxxx
    SELECT operation_id, object_type, object_name, node_group_id, policy_type
    FROM pg_dist_cleanup where operation_id = 777 ORDER BY object_name, node_group_id;
 operation_id | object_type |                        object_name                        | node_group_id | policy_type
---------------------------------------------------------------------
          777 |           1 | citus_failure_split_cleanup_schema.table_to_split_8981000 |             1 |           0
          777 |           1 | citus_failure_split_cleanup_schema.table_to_split_8981002 |             1 |           1
          777 |           1 | citus_failure_split_cleanup_schema.table_to_split_8981002 |             2 |           0
          777 |           1 | citus_failure_split_cleanup_schema.table_to_split_8981003 |             2 |           1
          777 |           4 | citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx                    |             2 |           0
(5 rows)

    -- we need to allow connection so that we can connect to proxy
    SELECT citus.mitmproxy('conn.allow()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

    \c - - - :worker_2_proxy_port
    SET search_path TO "citus_failure_split_cleanup_schema", public, pg_catalog;
    SET citus.show_shards_for_app_name_prefixes = '*';
    -- Left over child shards
    SELECT relname FROM pg_class where relname LIKE '%table_to_split_%' AND relkind = 'r' order by relname;
        relname
---------------------------------------------------------------------
 table_to_split_8981000
 table_to_split_8981002
 table_to_split_8981003
(3 rows)

    -- Left over publications
    SELECT pubname FROM pg_publication;
 pubname
---------------------------------------------------------------------
(0 rows)

    -- Left over replication slots
    SELECT slot_name FROM pg_replication_slots;
 slot_name
---------------------------------------------------------------------
(0 rows)

    -- Left over subscriptions
    SELECT subname FROM pg_subscription;
 subname
---------------------------------------------------------------------
(0 rows)

    \c - postgres - :master_port
    SELECT public.wait_for_resource_cleanup();
 wait_for_resource_cleanup
---------------------------------------------------------------------

(1 row)

    SELECT operation_id, object_type, object_name, node_group_id, policy_type
    FROM pg_dist_cleanup where operation_id = 777 ORDER BY object_name, node_group_id;
 operation_id | object_type | object_name | node_group_id | policy_type
---------------------------------------------------------------------
(0 rows)

    \c - - - :worker_2_proxy_port
    SET search_path TO "citus_failure_split_cleanup_schema", public, pg_catalog;
    SET citus.show_shards_for_app_name_prefixes = '*';
    -- Empty child shards after cleanup
    SELECT relname FROM pg_class where relname LIKE '%table_to_split_%' AND relkind = 'r' order by relname;
        relname
---------------------------------------------------------------------
 table_to_split_8981000
(1 row)

    -- Empty publications
    SELECT pubname FROM pg_publication;
 pubname
---------------------------------------------------------------------
(0 rows)

    -- Empty replication slot table
    SELECT slot_name FROM pg_replication_slots;
 slot_name
---------------------------------------------------------------------
(0 rows)

    -- Empty subscriptions
    SELECT subname FROM pg_subscription;
 subname
---------------------------------------------------------------------
(0 rows)

--2. Failure while creating shared memory segment
    \c - postgres - :master_port
    SET citus.next_shard_id TO 8981002;
    SET citus.next_operation_id TO 777;
    SET citus.next_cleanup_record_id TO 11;
    SELECT citus.mitmproxy('conn.onQuery(query="SELECT \* FROM pg_catalog.worker_split_shard_replication_setup\(.*").killall()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

    -- set log level to prevent flakiness
    SET client_min_messages TO ERROR;
    SELECT pg_catalog.citus_split_shard_by_split_points(
        8981000,
        ARRAY['-100000'],
        ARRAY[:worker_1_node, :worker_2_node],
        'force_logical');
ERROR:  Failed to run worker_split_shard_replication_setup UDF. It should successfully execute  for splitting a shard in a non-blocking way. Please retry.
    RESET client_min_messages;
    SELECT operation_id, object_type, object_name, node_group_id, policy_type
    FROM pg_dist_cleanup where operation_id = 777 ORDER BY object_name, node_group_id;
 operation_id | object_type |                        object_name                        | node_group_id | policy_type
---------------------------------------------------------------------
          777 |           1 | citus_failure_split_cleanup_schema.table_to_split_8981000 |             1 |           0
          777 |           1 | citus_failure_split_cleanup_schema.table_to_split_8981002 |             1 |           1
          777 |           1 | citus_failure_split_cleanup_schema.table_to_split_8981002 |             2 |           0
          777 |           1 | citus_failure_split_cleanup_schema.table_to_split_8981003 |             2 |           1
          777 |           4 | citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx                    |             2 |           0
          777 |           4 | citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx                    |             2 |           0
(6 rows)

    -- we need to allow connection so that we can connect to proxy
    SELECT citus.mitmproxy('conn.allow()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

    \c - - - :worker_2_proxy_port
    SET search_path TO "citus_failure_split_cleanup_schema", public, pg_catalog;
    SET citus.show_shards_for_app_name_prefixes = '*';
    -- Left over child shards
    SELECT relname FROM pg_class where relname LIKE '%table_to_split_%' AND relkind = 'r' order by relname;
        relname
---------------------------------------------------------------------
 table_to_split_8981000
 table_to_split_8981002
 table_to_split_8981003
(3 rows)

    -- Left over publications
    SELECT pubname FROM pg_publication;
                pubname
---------------------------------------------------------------------
 citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx
 citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx
(2 rows)

    -- Left over replication slots
    SELECT slot_name FROM pg_replication_slots;
 slot_name
---------------------------------------------------------------------
(0 rows)

    -- Left over subscriptions
    SELECT subname FROM pg_subscription;
 subname
---------------------------------------------------------------------
(0 rows)

    \c - postgres - :master_port
    SELECT public.wait_for_resource_cleanup();
 wait_for_resource_cleanup
---------------------------------------------------------------------

(1 row)

    SELECT operation_id, object_type, object_name, node_group_id, policy_type
    FROM pg_dist_cleanup where operation_id = 777 ORDER BY object_name, node_group_id;
 operation_id | object_type | object_name | node_group_id | policy_type
---------------------------------------------------------------------
(0 rows)

    \c - - - :worker_2_proxy_port
    SET search_path TO "citus_failure_split_cleanup_schema", public, pg_catalog;
    SET citus.show_shards_for_app_name_prefixes = '*';
    -- Empty child shards after cleanup
    SELECT relname FROM pg_class where relname LIKE '%table_to_split_%' AND relkind = 'r' order by relname;
        relname
---------------------------------------------------------------------
 table_to_split_8981000
(1 row)

    -- Empty publications
    SELECT pubname FROM pg_publication;
 pubname
---------------------------------------------------------------------
(0 rows)

    -- Empty replication slot table
    SELECT slot_name FROM pg_replication_slots;
 slot_name
---------------------------------------------------------------------
(0 rows)

    -- Empty subscriptions
    SELECT subname FROM pg_subscription;
 subname
---------------------------------------------------------------------
(0 rows)

--3. Failure while executing 'CREATE_REPLICATION_SLOT' for Snapshot.
    \c - postgres - :master_port
    SET citus.next_shard_id TO 8981002;
    SET citus.next_operation_id TO 777;
    SET citus.next_cleanup_record_id TO 11;
    SELECT citus.mitmproxy('conn.onQuery(query="CREATE_REPLICATION_SLOT .* LOGICAL .* EXPORT_SNAPSHOT.*").killall()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

    SELECT pg_catalog.citus_split_shard_by_split_points(
        8981000,
        ARRAY['-100000'],
        ARRAY[:worker_1_node, :worker_2_node],
        'force_logical');
WARNING:  connection not open
CONTEXT:  while executing command on localhost:xxxxx
WARNING:  connection not open
CONTEXT:  while executing command on localhost:xxxxx
ERROR:  connection not open
CONTEXT:  while executing command on localhost:xxxxx
    SELECT operation_id, object_type, object_name, node_group_id, policy_type
    FROM pg_dist_cleanup where operation_id = 777 ORDER BY object_name, node_group_id;
 operation_id | object_type |                        object_name                        | node_group_id | policy_type
---------------------------------------------------------------------
          777 |           1 | citus_failure_split_cleanup_schema.table_to_split_8981000 |             1 |           0
          777 |           1 | citus_failure_split_cleanup_schema.table_to_split_8981002 |             1 |           1
          777 |           1 | citus_failure_split_cleanup_schema.table_to_split_8981002 |             2 |           0
          777 |           1 | citus_failure_split_cleanup_schema.table_to_split_8981003 |             2 |           1
          777 |           4 | citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx                    |             2 |           0
          777 |           4 | citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx                    |             2 |           0
          777 |           3 | citus_shard_split_slot_xxxxxxx_xxxxxxx_xxxxxxx                           |             2 |           0
(7 rows)

    -- we need to allow connection so that we can connect to proxy
    SELECT citus.mitmproxy('conn.allow()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

    \c - - - :worker_2_proxy_port
    SET search_path TO "citus_failure_split_cleanup_schema", public, pg_catalog;
    SET citus.show_shards_for_app_name_prefixes = '*';
    -- Left over child shards
    SELECT relname FROM pg_class where relname LIKE '%table_to_split_%' AND relkind = 'r' order by relname;
        relname
---------------------------------------------------------------------
 table_to_split_8981000
 table_to_split_8981002
 table_to_split_8981003
(3 rows)

    -- Left over publications
    SELECT pubname FROM pg_publication;
                pubname
---------------------------------------------------------------------
 citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx
 citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx
(2 rows)

    -- Left over replication slots
    SELECT slot_name FROM pg_replication_slots;
 slot_name
---------------------------------------------------------------------
(0 rows)

    -- Left over subscriptions
    SELECT subname FROM pg_subscription;
 subname
---------------------------------------------------------------------
(0 rows)

    \c - postgres - :master_port
    SELECT public.wait_for_resource_cleanup();
 wait_for_resource_cleanup
---------------------------------------------------------------------

(1 row)

    SELECT operation_id, object_type, object_name, node_group_id, policy_type
    FROM pg_dist_cleanup where operation_id = 777 ORDER BY object_name, node_group_id;
 operation_id | object_type | object_name | node_group_id | policy_type
---------------------------------------------------------------------
(0 rows)

    \c - - - :worker_2_proxy_port
    SET search_path TO "citus_failure_split_cleanup_schema", public, pg_catalog;
    SET citus.show_shards_for_app_name_prefixes = '*';
    -- Empty child shards after cleanup
    SELECT relname FROM pg_class where relname LIKE '%table_to_split_%' AND relkind = 'r' order by relname;
        relname
---------------------------------------------------------------------
 table_to_split_8981000
(1 row)

    -- Empty publications
    SELECT pubname FROM pg_publication;
 pubname
---------------------------------------------------------------------
(0 rows)

    -- Empty replication slot table
    SELECT slot_name FROM pg_replication_slots;
 slot_name
---------------------------------------------------------------------
(0 rows)

    -- Empty subscriptions
    SELECT subname FROM pg_subscription;
 subname
---------------------------------------------------------------------
(0 rows)

--4. Failure while enabling subscription
    \c - postgres - :master_port
    SET citus.next_shard_id TO 8981002;
    SET citus.next_operation_id TO 777;
    SET citus.next_cleanup_record_id TO 11;
    SELECT citus.mitmproxy('conn.onQuery(query="ALTER SUBSCRIPTION .* ENABLE").killall()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

    SELECT pg_catalog.citus_split_shard_by_split_points(
        8981000,
        ARRAY['-100000'],
        ARRAY[:worker_1_node, :worker_2_node],
        'force_logical');
ERROR:  connection not open
CONTEXT:  while executing command on localhost:xxxxx
    SELECT operation_id, object_type, object_name, node_group_id, policy_type
    FROM pg_dist_cleanup where operation_id = 777 ORDER BY object_name, node_group_id;
 operation_id | object_type |                        object_name                        | node_group_id | policy_type
---------------------------------------------------------------------
          777 |           1 | citus_failure_split_cleanup_schema.table_to_split_8981000 |             1 |           0
          777 |           1 | citus_failure_split_cleanup_schema.table_to_split_8981002 |             1 |           1
          777 |           1 | citus_failure_split_cleanup_schema.table_to_split_8981002 |             2 |           0
          777 |           1 | citus_failure_split_cleanup_schema.table_to_split_8981003 |             2 |           1
          777 |           4 | citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx                    |             2 |           0
          777 |           4 | citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx                    |             2 |           0
          777 |           3 | citus_shard_split_slot_xxxxxxx_xxxxxxx_xxxxxxx                           |             2 |           0
          777 |           3 | citus_shard_split_slot_xxxxxxx_xxxxxxx_xxxxxxx                           |             2 |           0
          777 |           2 | citus_shard_split_subscription_xxxxxxx_xxxxxxx                     |             1 |           0
          777 |           2 | citus_shard_split_subscription_xxxxxxx_xxxxxxx                     |             2 |           0
          777 |           5 | citus_shard_split_subscription_role_xxxxxxx_xxxxxxx                |             1 |           0
          777 |           5 | citus_shard_split_subscription_role_xxxxxxx_xxxxxxx                |             2 |           0
(12 rows)

    -- we need to allow connection so that we can connect to proxy
    SELECT citus.mitmproxy('conn.allow()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

    \c - - - :worker_2_proxy_port
    SET search_path TO "citus_failure_split_cleanup_schema", public, pg_catalog;
    SET citus.show_shards_for_app_name_prefixes = '*';
    -- Left over child shards
    SELECT relname FROM pg_class where relname LIKE '%table_to_split_%' AND relkind = 'r' order by relname;
        relname
---------------------------------------------------------------------
 table_to_split_8981000
 table_to_split_8981002
 table_to_split_8981003
(3 rows)

    -- Left over publications
    SELECT pubname FROM pg_publication;
                pubname
---------------------------------------------------------------------
 citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx
 citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx
(2 rows)

    -- Left over replication slots
    SELECT slot_name FROM pg_replication_slots;
            slot_name
---------------------------------------------------------------------
 citus_shard_split_slot_xxxxxxx_xxxxxxx_xxxxxxx
 citus_shard_split_slot_xxxxxxx_xxxxxxx_xxxxxxx
(2 rows)

    -- Left over subscriptions
    SELECT subname FROM pg_subscription;
                subname
---------------------------------------------------------------------
 citus_shard_split_subscription_xxxxxxx_xxxxxxx
(1 row)

    \c - postgres - :master_port
    SELECT public.wait_for_resource_cleanup();
 wait_for_resource_cleanup
---------------------------------------------------------------------

(1 row)

    SELECT operation_id, object_type, object_name, node_group_id, policy_type
    FROM pg_dist_cleanup where operation_id = 777 ORDER BY object_name, node_group_id;
 operation_id | object_type | object_name | node_group_id | policy_type
---------------------------------------------------------------------
(0 rows)

    \c - - - :worker_2_proxy_port
    SET search_path TO "citus_failure_split_cleanup_schema", public, pg_catalog;
    SET citus.show_shards_for_app_name_prefixes = '*';
    -- Empty child shards after cleanup
    SELECT relname FROM pg_class where relname LIKE '%table_to_split_%' AND relkind = 'r' order by relname;
        relname
---------------------------------------------------------------------
 table_to_split_8981000
(1 row)

    -- Empty publications
    SELECT pubname FROM pg_publication;
 pubname
---------------------------------------------------------------------
(0 rows)

    -- Empty replication slot table
    SELECT slot_name FROM pg_replication_slots;
 slot_name
---------------------------------------------------------------------
(0 rows)

    -- Empty subscriptions
    SELECT subname FROM pg_subscription;
 subname
---------------------------------------------------------------------
(0 rows)

--5. Failure on polling last write-ahead log location reported to origin WAL sender
    \c - postgres - :master_port
    SET citus.next_shard_id TO 8981002;
    SET citus.next_operation_id TO 777;
    SET citus.next_cleanup_record_id TO 11;
    SELECT citus.mitmproxy('conn.onQuery(query="^SELECT min\(latest_end_lsn").killall()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

    SELECT pg_catalog.citus_split_shard_by_split_points(
        8981000,
        ARRAY['-100000'],
        ARRAY[:worker_1_node, :worker_2_node],
        'force_logical');
ERROR:  connection not open
CONTEXT:  while executing command on localhost:xxxxx
    SELECT operation_id, object_type, object_name, node_group_id, policy_type
    FROM pg_dist_cleanup where operation_id = 777 ORDER BY object_name, node_group_id;
 operation_id | object_type |                        object_name                        | node_group_id | policy_type
---------------------------------------------------------------------
          777 |           1 | citus_failure_split_cleanup_schema.table_to_split_8981000 |             1 |           0
          777 |           1 | citus_failure_split_cleanup_schema.table_to_split_8981002 |             1 |           1
          777 |           1 | citus_failure_split_cleanup_schema.table_to_split_8981002 |             2 |           0
          777 |           1 | citus_failure_split_cleanup_schema.table_to_split_8981003 |             2 |           1
          777 |           4 | citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx                    |             2 |           0
          777 |           4 | citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx                    |             2 |           0
          777 |           3 | citus_shard_split_slot_xxxxxxx_xxxxxxx_xxxxxxx                           |             2 |           0
          777 |           3 | citus_shard_split_slot_xxxxxxx_xxxxxxx_xxxxxxx                           |             2 |           0
          777 |           2 | citus_shard_split_subscription_xxxxxxx_xxxxxxx                     |             1 |           0
          777 |           2 | citus_shard_split_subscription_xxxxxxx_xxxxxxx                     |             2 |           0
          777 |           5 | citus_shard_split_subscription_role_xxxxxxx_xxxxxxx                |             1 |           0
          777 |           5 | citus_shard_split_subscription_role_xxxxxxx_xxxxxxx                |             2 |           0
(12 rows)

    -- we need to allow connection so that we can connect to proxy
    SELECT citus.mitmproxy('conn.allow()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

    \c - - - :worker_2_proxy_port
    SET search_path TO "citus_failure_split_cleanup_schema", public, pg_catalog;
    SET citus.show_shards_for_app_name_prefixes = '*';
    -- Left over child shards
    SELECT relname FROM pg_class where relname LIKE '%table_to_split_%' AND relkind = 'r' order by relname;
        relname
---------------------------------------------------------------------
 table_to_split_8981000
 table_to_split_8981002
 table_to_split_8981003
(3 rows)

    -- Left over publications
    SELECT pubname FROM pg_publication;
                pubname
---------------------------------------------------------------------
 citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx
 citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx
(2 rows)

    -- Left over replication slots
    SELECT slot_name FROM pg_replication_slots;
            slot_name
---------------------------------------------------------------------
 citus_shard_split_slot_xxxxxxx_xxxxxxx_xxxxxxx
 citus_shard_split_slot_xxxxxxx_xxxxxxx_xxxxxxx
(2 rows)

    -- Left over subscriptions
    SELECT subname FROM pg_subscription;
                subname
---------------------------------------------------------------------
 citus_shard_split_subscription_xxxxxxx_xxxxxxx
(1 row)

    \c - postgres - :master_port
    SELECT public.wait_for_resource_cleanup();
 wait_for_resource_cleanup
---------------------------------------------------------------------

(1 row)

    SELECT operation_id, object_type, object_name, node_group_id, policy_type
    FROM pg_dist_cleanup where operation_id = 777 ORDER BY object_name, node_group_id;
 operation_id | object_type | object_name | node_group_id | policy_type
---------------------------------------------------------------------
(0 rows)

    \c - - - :worker_2_proxy_port
    SET search_path TO "citus_failure_split_cleanup_schema", public, pg_catalog;
    SET citus.show_shards_for_app_name_prefixes = '*';
    -- Empty child shards after cleanup
    SELECT relname FROM pg_class where relname LIKE '%table_to_split_%' AND relkind = 'r' order by relname;
        relname
---------------------------------------------------------------------
 table_to_split_8981000
(1 row)

    -- Empty publications
    SELECT pubname FROM pg_publication;
 pubname
---------------------------------------------------------------------
(0 rows)

    -- Empty replication slot table
    SELECT slot_name FROM pg_replication_slots;
 slot_name
---------------------------------------------------------------------
(0 rows)

    -- Empty subscriptions
    SELECT subname FROM pg_subscription;
 subname
---------------------------------------------------------------------
(0 rows)

--6. Failure on dropping subscription
    \c - postgres - :master_port
    SET citus.next_shard_id TO 8981002;
    SET citus.next_operation_id TO 777;
    SET citus.next_cleanup_record_id TO 11;
    SELECT citus.mitmproxy('conn.onQuery(query="^DROP SUBSCRIPTION").killall()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

    SELECT pg_catalog.citus_split_shard_by_split_points(
        8981000,
        ARRAY['-100000'],
        ARRAY[:worker_1_node, :worker_2_node],
        'force_logical');
WARNING:  connection not open
CONTEXT:  while executing command on localhost:xxxxx
WARNING:  connection not open
CONTEXT:  while executing command on localhost:xxxxx
WARNING:  connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open
WARNING:  connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open
WARNING:  connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open
WARNING:  connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open
ERROR:  connection not open
CONTEXT:  while executing command on localhost:xxxxx
    SELECT operation_id, object_type, object_name, node_group_id, policy_type
    FROM pg_dist_cleanup where operation_id = 777 ORDER BY object_name, node_group_id;
 operation_id | object_type |                        object_name                        | node_group_id | policy_type
---------------------------------------------------------------------
          777 |           1 | citus_failure_split_cleanup_schema.table_to_split_8981002 |             1 |           1
          777 |           1 | citus_failure_split_cleanup_schema.table_to_split_8981003 |             2 |           1
          777 |           4 | citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx                    |             2 |           0
          777 |           4 | citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx                    |             2 |           0
          777 |           3 | citus_shard_split_slot_xxxxxxx_xxxxxxx_xxxxxxx                           |             2 |           0
          777 |           3 | citus_shard_split_slot_xxxxxxx_xxxxxxx_xxxxxxx                           |             2 |           0
          777 |           2 | citus_shard_split_subscription_xxxxxxx_xxxxxxx                     |             2 |           0
          777 |           5 | citus_shard_split_subscription_role_xxxxxxx_xxxxxxx                |             2 |           0
(8 rows)

    SELECT relname FROM pg_class where relname LIKE '%table_to_split_%' AND relkind = 'r' order by relname;
 relname
---------------------------------------------------------------------
(0 rows)

    -- we need to allow connection so that we can connect to proxy
    SELECT citus.mitmproxy('conn.allow()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

    \c - - - :worker_2_proxy_port
    SET search_path TO "citus_failure_split_cleanup_schema", public, pg_catalog;
    SET citus.show_shards_for_app_name_prefixes = '*';
    -- Left over child shards
    SELECT relname FROM pg_class where relname LIKE '%table_to_split_%' AND relkind = 'r' order by relname;
        relname
---------------------------------------------------------------------
 table_to_split_8981000
 table_to_split_8981003
(2 rows)

    -- Left over publications
    SELECT pubname FROM pg_publication;
                pubname
---------------------------------------------------------------------
 citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx
 citus_shard_split_publication_xxxxxxx_xxxxxxx_xxxxxxx
(2 rows)

    -- Left over replication slots
    SELECT slot_name FROM pg_replication_slots;
            slot_name
---------------------------------------------------------------------
 citus_shard_split_slot_xxxxxxx_xxxxxxx_xxxxxxx
 citus_shard_split_slot_xxxxxxx_xxxxxxx_xxxxxxx
(2 rows)

    -- Left over subscriptions
    SELECT subname FROM pg_subscription;
                subname
---------------------------------------------------------------------
 citus_shard_split_subscription_xxxxxxx_xxxxxxx
(1 row)

    \c - postgres - :master_port
    SELECT public.wait_for_resource_cleanup();
 wait_for_resource_cleanup
---------------------------------------------------------------------

(1 row)

    SELECT operation_id, object_type, object_name, node_group_id, policy_type
    FROM pg_dist_cleanup where operation_id = 777 ORDER BY object_name, node_group_id;
 operation_id | object_type | object_name | node_group_id | policy_type
---------------------------------------------------------------------
(0 rows)

    \c - - - :worker_2_proxy_port
    SET search_path TO "citus_failure_split_cleanup_schema", public, pg_catalog;
    SET citus.show_shards_for_app_name_prefixes = '*';
    -- Empty child shards after cleanup
    SELECT relname FROM pg_class where relname LIKE '%table_to_split_%' AND relkind = 'r' order by relname;
        relname
---------------------------------------------------------------------
 table_to_split_8981000
(1 row)

    -- Empty publications
    SELECT pubname FROM pg_publication;
 pubname
---------------------------------------------------------------------
(0 rows)

    -- Empty replication slot table
    SELECT slot_name FROM pg_replication_slots;
 slot_name
---------------------------------------------------------------------
(0 rows)

    -- Empty subscriptions
    SELECT subname FROM pg_subscription;
 subname
---------------------------------------------------------------------
(0 rows)

-- Cleanup
\c - postgres - :master_port
SET client_min_messages TO WARNING;
DROP SCHEMA "citus_failure_split_cleanup_schema" CASCADE;
-- Cleanup
