SET citus.enable_create_database_propagation TO ON;
SET client_min_messages TO WARNING;
CREATE FUNCTION get_temp_databases_on_nodes()
RETURNS TEXT AS $func$
  SELECT array_agg(DISTINCT result ORDER BY result) AS temp_databases_on_nodes FROM run_command_on_all_nodes($$SELECT datname FROM pg_database WHERE datname LIKE 'citus_temp_database_%'$$) WHERE result != '';
$func$
LANGUAGE sql;
CREATE FUNCTION count_db_cleanup_records()
RETURNS TABLE(object_name TEXT, count INTEGER) AS $func$
  SELECT object_name, COUNT(*) FROM pg_dist_cleanup WHERE object_name LIKE 'citus_temp_database_%' GROUP BY object_name;
$func$
LANGUAGE sql;
CREATE FUNCTION ensure_no_temp_databases_on_any_nodes()
RETURNS BOOLEAN AS $func$
  SELECT bool_and(result::boolean) AS no_temp_databases_on_any_nodes FROM run_command_on_all_nodes($$SELECT COUNT(*)=0 FROM pg_database WHERE datname LIKE 'citus_temp_database_%'$$);
$func$
LANGUAGE sql;
-- cleanup any orphaned resources from previous runs
CALL citus_cleanup_orphaned_resources();
SET citus.next_operation_id TO 4000;
ALTER SYSTEM SET citus.defer_shard_delete_interval TO -1;
SELECT pg_reload_conf();
 pg_reload_conf
---------------------------------------------------------------------
 t
(1 row)

SELECT pg_sleep(0.1);
 pg_sleep
---------------------------------------------------------------------

(1 row)

SELECT citus.mitmproxy('conn.kill()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

CREATE DATABASE db1;
ERROR:  connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open
SELECT citus.mitmproxy('conn.allow()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT get_temp_databases_on_nodes();
 get_temp_databases_on_nodes
---------------------------------------------------------------------

(1 row)

SELECT * FROM count_db_cleanup_records();
 object_name | count
---------------------------------------------------------------------
(0 rows)

CALL citus_cleanup_orphaned_resources();
SELECT ensure_no_temp_databases_on_any_nodes();
 ensure_no_temp_databases_on_any_nodes
---------------------------------------------------------------------
 t
(1 row)

SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result;
      node_type       |                                                              result
---------------------------------------------------------------------
 worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
 worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
(2 rows)

SELECT citus.mitmproxy('conn.onQuery(query="^CREATE DATABASE").cancel(' || pg_backend_pid() || ')');
 mitmproxy
---------------------------------------------------------------------

(1 row)

CREATE DATABASE db1;
ERROR:  canceling statement due to user request
SELECT citus.mitmproxy('conn.allow()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT get_temp_databases_on_nodes();
 get_temp_databases_on_nodes
---------------------------------------------------------------------
 {citus_temp_database_4000_0}
(1 row)

SELECT * FROM count_db_cleanup_records();
        object_name         | count
---------------------------------------------------------------------
 citus_temp_database_4000_0 |     2
(1 row)

CALL citus_cleanup_orphaned_resources();
SELECT ensure_no_temp_databases_on_any_nodes();
 ensure_no_temp_databases_on_any_nodes
---------------------------------------------------------------------
 t
(1 row)

SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result;
      node_type       |                                                              result
---------------------------------------------------------------------
 worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
 worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
(2 rows)

SELECT citus.mitmproxy('conn.onQuery(query="^ALTER DATABASE").cancel(' || pg_backend_pid() || ')');
 mitmproxy
---------------------------------------------------------------------

(1 row)

CREATE DATABASE db1;
ERROR:  canceling statement due to user request
SELECT citus.mitmproxy('conn.allow()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT get_temp_databases_on_nodes();
 get_temp_databases_on_nodes
---------------------------------------------------------------------
 {citus_temp_database_4001_0}
(1 row)

SELECT * FROM count_db_cleanup_records();
        object_name         | count
---------------------------------------------------------------------
 citus_temp_database_4001_0 |     2
(1 row)

CALL citus_cleanup_orphaned_resources();
SELECT ensure_no_temp_databases_on_any_nodes();
 ensure_no_temp_databases_on_any_nodes
---------------------------------------------------------------------
 t
(1 row)

SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result;
      node_type       |                                                              result
---------------------------------------------------------------------
 worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
 worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
(2 rows)

SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED").kill()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

CREATE DATABASE db1;
ERROR:  connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open
SELECT citus.mitmproxy('conn.allow()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT get_temp_databases_on_nodes();
 get_temp_databases_on_nodes
---------------------------------------------------------------------

(1 row)

SELECT * FROM count_db_cleanup_records();
 object_name | count
---------------------------------------------------------------------
(0 rows)

CALL citus_cleanup_orphaned_resources();
SELECT ensure_no_temp_databases_on_any_nodes();
 ensure_no_temp_databases_on_any_nodes
---------------------------------------------------------------------
 t
(1 row)

SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result;
      node_type       |                                                              result
---------------------------------------------------------------------
 worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
 worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
(2 rows)

SELECT citus.mitmproxy('conn.onQuery(query="^PREPARE TRANSACTION").kill()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

CREATE DATABASE db1;
ERROR:  connection not open
CONTEXT:  while executing command on localhost:xxxxx
SELECT citus.mitmproxy('conn.allow()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT get_temp_databases_on_nodes();
 get_temp_databases_on_nodes
---------------------------------------------------------------------
 {citus_temp_database_4002_0}
(1 row)

SELECT * FROM count_db_cleanup_records();
        object_name         | count
---------------------------------------------------------------------
 citus_temp_database_4002_0 |     2
(1 row)

CALL citus_cleanup_orphaned_resources();
SELECT ensure_no_temp_databases_on_any_nodes();
 ensure_no_temp_databases_on_any_nodes
---------------------------------------------------------------------
 t
(1 row)

SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result;
      node_type       |                                                              result
---------------------------------------------------------------------
 worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
 worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
(2 rows)

SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT PREPARED").kill()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

CREATE DATABASE db1;
WARNING:  connection not open
CONTEXT:  while executing command on localhost:xxxxx
WARNING:  failed to commit transaction on localhost:xxxxx
SELECT citus.mitmproxy('conn.allow()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

-- not call citus_cleanup_orphaned_resources() but recover the prepared transactions this time
SELECT recover_prepared_transactions();
 recover_prepared_transactions
---------------------------------------------------------------------
                             1
(1 row)

SELECT ensure_no_temp_databases_on_any_nodes();
 ensure_no_temp_databases_on_any_nodes
---------------------------------------------------------------------
 t
(1 row)

SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result;
      node_type       |                                                                                                                                                                                                                    result
---------------------------------------------------------------------
 worker node (remote) | {"database_properties": {"datacl": null, "datname": "db1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
 worker node (remote) | {"database_properties": {"datacl": null, "datname": "db1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
(2 rows)

DROP DATABASE db1;
-- after recovering the prepared transactions, cleanup records should also be removed
SELECT * FROM count_db_cleanup_records();
 object_name | count
---------------------------------------------------------------------
(0 rows)

SELECT citus.mitmproxy('conn.onQuery(query="^SELECT citus_internal.acquire_citus_advisory_object_class_lock").kill()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

CREATE DATABASE db1;
ERROR:  connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open
SELECT citus.mitmproxy('conn.allow()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT get_temp_databases_on_nodes();
 get_temp_databases_on_nodes
---------------------------------------------------------------------

(1 row)

SELECT * FROM count_db_cleanup_records();
 object_name | count
---------------------------------------------------------------------
(0 rows)

CALL citus_cleanup_orphaned_resources();
SELECT ensure_no_temp_databases_on_any_nodes();
 ensure_no_temp_databases_on_any_nodes
---------------------------------------------------------------------
 t
(1 row)

SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result;
      node_type       |                                                              result
---------------------------------------------------------------------
 worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
 worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
(2 rows)

SELECT citus.mitmproxy('conn.onParse(query="^WITH distributed_object_data").kill()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

CREATE DATABASE db1;
ERROR:  connection not open
CONTEXT:  while executing command on localhost:xxxxx
SELECT citus.mitmproxy('conn.allow()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT get_temp_databases_on_nodes();
 get_temp_databases_on_nodes
---------------------------------------------------------------------
 {citus_temp_database_4004_0}
(1 row)

SELECT * FROM count_db_cleanup_records();
        object_name         | count
---------------------------------------------------------------------
 citus_temp_database_4004_0 |     2
(1 row)

CALL citus_cleanup_orphaned_resources();
SELECT ensure_no_temp_databases_on_any_nodes();
 ensure_no_temp_databases_on_any_nodes
---------------------------------------------------------------------
 t
(1 row)

SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result;
      node_type       |                                                              result
---------------------------------------------------------------------
 worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
 worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
(2 rows)

CREATE DATABASE db1;
-- show that a successful database creation doesn't leave any pg_dist_cleanup records behind
SELECT * FROM count_db_cleanup_records();
 object_name | count
---------------------------------------------------------------------
(0 rows)

DROP DATABASE db1;
DROP FUNCTION get_temp_databases_on_nodes();
DROP FUNCTION ensure_no_temp_databases_on_any_nodes();
DROP FUNCTION count_db_cleanup_records();