mirror of https://github.com/citusdata/citus.git
367 lines
15 KiB
Plaintext
367 lines
15 KiB
Plaintext
SET citus.enable_create_database_propagation TO ON;
|
|
SET client_min_messages TO WARNING;
|
|
CREATE FUNCTION get_temp_databases_on_nodes()
|
|
RETURNS TEXT AS $func$
|
|
SELECT array_agg(DISTINCT result ORDER BY result) AS temp_databases_on_nodes FROM run_command_on_all_nodes($$SELECT datname FROM pg_database WHERE datname LIKE 'citus_temp_database_%'$$) WHERE result != '';
|
|
$func$
|
|
LANGUAGE sql;
|
|
CREATE FUNCTION count_db_cleanup_records()
|
|
RETURNS TABLE(object_name TEXT, count INTEGER) AS $func$
|
|
SELECT object_name, COUNT(*) FROM pg_dist_cleanup WHERE object_name LIKE 'citus_temp_database_%' GROUP BY object_name;
|
|
$func$
|
|
LANGUAGE sql;
|
|
CREATE FUNCTION ensure_no_temp_databases_on_any_nodes()
|
|
RETURNS BOOLEAN AS $func$
|
|
SELECT bool_and(result::boolean) AS no_temp_databases_on_any_nodes FROM run_command_on_all_nodes($$SELECT COUNT(*)=0 FROM pg_database WHERE datname LIKE 'citus_temp_database_%'$$);
|
|
$func$
|
|
LANGUAGE sql;
|
|
-- cleanup any orphaned resources from previous runs
|
|
CALL citus_cleanup_orphaned_resources();
|
|
SET citus.next_operation_id TO 4000;
|
|
ALTER SYSTEM SET citus.defer_shard_delete_interval TO -1;
|
|
SELECT pg_reload_conf();
|
|
pg_reload_conf
|
|
---------------------------------------------------------------------
|
|
t
|
|
(1 row)
|
|
|
|
SELECT pg_sleep(0.1);
|
|
pg_sleep
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT citus.mitmproxy('conn.kill()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
CREATE DATABASE db1;
|
|
ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open
|
|
SELECT citus.mitmproxy('conn.allow()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT get_temp_databases_on_nodes();
|
|
get_temp_databases_on_nodes
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT * FROM count_db_cleanup_records();
|
|
object_name | count
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
CALL citus_cleanup_orphaned_resources();
|
|
SELECT ensure_no_temp_databases_on_any_nodes();
|
|
ensure_no_temp_databases_on_any_nodes
|
|
---------------------------------------------------------------------
|
|
t
|
|
(1 row)
|
|
|
|
SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result;
|
|
node_type | result
|
|
---------------------------------------------------------------------
|
|
worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
|
|
worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
|
|
(2 rows)
|
|
|
|
SELECT citus.mitmproxy('conn.onQuery(query="^CREATE DATABASE").cancel(' || pg_backend_pid() || ')');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
CREATE DATABASE db1;
|
|
ERROR: canceling statement due to user request
|
|
SELECT citus.mitmproxy('conn.allow()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT get_temp_databases_on_nodes();
|
|
get_temp_databases_on_nodes
|
|
---------------------------------------------------------------------
|
|
{citus_temp_database_4000_0}
|
|
(1 row)
|
|
|
|
SELECT * FROM count_db_cleanup_records();
|
|
object_name | count
|
|
---------------------------------------------------------------------
|
|
citus_temp_database_4000_0 | 2
|
|
(1 row)
|
|
|
|
CALL citus_cleanup_orphaned_resources();
|
|
SELECT ensure_no_temp_databases_on_any_nodes();
|
|
ensure_no_temp_databases_on_any_nodes
|
|
---------------------------------------------------------------------
|
|
t
|
|
(1 row)
|
|
|
|
SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result;
|
|
node_type | result
|
|
---------------------------------------------------------------------
|
|
worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
|
|
worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
|
|
(2 rows)
|
|
|
|
SELECT citus.mitmproxy('conn.onQuery(query="^ALTER DATABASE").cancel(' || pg_backend_pid() || ')');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
CREATE DATABASE db1;
|
|
ERROR: canceling statement due to user request
|
|
SELECT citus.mitmproxy('conn.allow()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT get_temp_databases_on_nodes();
|
|
get_temp_databases_on_nodes
|
|
---------------------------------------------------------------------
|
|
{citus_temp_database_4001_0}
|
|
(1 row)
|
|
|
|
SELECT * FROM count_db_cleanup_records();
|
|
object_name | count
|
|
---------------------------------------------------------------------
|
|
citus_temp_database_4001_0 | 2
|
|
(1 row)
|
|
|
|
CALL citus_cleanup_orphaned_resources();
|
|
SELECT ensure_no_temp_databases_on_any_nodes();
|
|
ensure_no_temp_databases_on_any_nodes
|
|
---------------------------------------------------------------------
|
|
t
|
|
(1 row)
|
|
|
|
SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result;
|
|
node_type | result
|
|
---------------------------------------------------------------------
|
|
worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
|
|
worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
|
|
(2 rows)
|
|
|
|
SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED").kill()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
CREATE DATABASE db1;
|
|
ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open
|
|
SELECT citus.mitmproxy('conn.allow()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT get_temp_databases_on_nodes();
|
|
get_temp_databases_on_nodes
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT * FROM count_db_cleanup_records();
|
|
object_name | count
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
CALL citus_cleanup_orphaned_resources();
|
|
SELECT ensure_no_temp_databases_on_any_nodes();
|
|
ensure_no_temp_databases_on_any_nodes
|
|
---------------------------------------------------------------------
|
|
t
|
|
(1 row)
|
|
|
|
SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result;
|
|
node_type | result
|
|
---------------------------------------------------------------------
|
|
worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
|
|
worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
|
|
(2 rows)
|
|
|
|
SELECT citus.mitmproxy('conn.onQuery(query="^PREPARE TRANSACTION").kill()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
CREATE DATABASE db1;
|
|
ERROR: connection not open
|
|
CONTEXT: while executing command on localhost:xxxxx
|
|
SELECT citus.mitmproxy('conn.allow()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT get_temp_databases_on_nodes();
|
|
get_temp_databases_on_nodes
|
|
---------------------------------------------------------------------
|
|
{citus_temp_database_4002_0}
|
|
(1 row)
|
|
|
|
SELECT * FROM count_db_cleanup_records();
|
|
object_name | count
|
|
---------------------------------------------------------------------
|
|
citus_temp_database_4002_0 | 2
|
|
(1 row)
|
|
|
|
CALL citus_cleanup_orphaned_resources();
|
|
SELECT ensure_no_temp_databases_on_any_nodes();
|
|
ensure_no_temp_databases_on_any_nodes
|
|
---------------------------------------------------------------------
|
|
t
|
|
(1 row)
|
|
|
|
SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result;
|
|
node_type | result
|
|
---------------------------------------------------------------------
|
|
worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
|
|
worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
|
|
(2 rows)
|
|
|
|
SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT PREPARED").kill()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
CREATE DATABASE db1;
|
|
WARNING: connection not open
|
|
CONTEXT: while executing command on localhost:xxxxx
|
|
WARNING: failed to commit transaction on localhost:xxxxx
|
|
SELECT citus.mitmproxy('conn.allow()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
-- not call citus_cleanup_orphaned_resources() but recover the prepared transactions this time
|
|
SELECT recover_prepared_transactions();
|
|
recover_prepared_transactions
|
|
---------------------------------------------------------------------
|
|
1
|
|
(1 row)
|
|
|
|
SELECT ensure_no_temp_databases_on_any_nodes();
|
|
ensure_no_temp_databases_on_any_nodes
|
|
---------------------------------------------------------------------
|
|
t
|
|
(1 row)
|
|
|
|
SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result;
|
|
node_type | result
|
|
---------------------------------------------------------------------
|
|
worker node (remote) | {"database_properties": {"datacl": null, "datname": "db1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
|
|
worker node (remote) | {"database_properties": {"datacl": null, "datname": "db1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
|
|
(2 rows)
|
|
|
|
DROP DATABASE db1;
|
|
-- after recovering the prepared transactions, cleanup records should also be removed
|
|
SELECT * FROM count_db_cleanup_records();
|
|
object_name | count
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
SELECT citus.mitmproxy('conn.onQuery(query="^SELECT citus_internal.acquire_citus_advisory_object_class_lock").kill()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
CREATE DATABASE db1;
|
|
ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open
|
|
SELECT citus.mitmproxy('conn.allow()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT get_temp_databases_on_nodes();
|
|
get_temp_databases_on_nodes
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT * FROM count_db_cleanup_records();
|
|
object_name | count
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
CALL citus_cleanup_orphaned_resources();
|
|
SELECT ensure_no_temp_databases_on_any_nodes();
|
|
ensure_no_temp_databases_on_any_nodes
|
|
---------------------------------------------------------------------
|
|
t
|
|
(1 row)
|
|
|
|
SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result;
|
|
node_type | result
|
|
---------------------------------------------------------------------
|
|
worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
|
|
worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
|
|
(2 rows)
|
|
|
|
SELECT citus.mitmproxy('conn.onParse(query="^WITH distributed_object_data").kill()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
CREATE DATABASE db1;
|
|
ERROR: connection not open
|
|
CONTEXT: while executing command on localhost:xxxxx
|
|
SELECT citus.mitmproxy('conn.allow()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT get_temp_databases_on_nodes();
|
|
get_temp_databases_on_nodes
|
|
---------------------------------------------------------------------
|
|
{citus_temp_database_4004_0}
|
|
(1 row)
|
|
|
|
SELECT * FROM count_db_cleanup_records();
|
|
object_name | count
|
|
---------------------------------------------------------------------
|
|
citus_temp_database_4004_0 | 2
|
|
(1 row)
|
|
|
|
CALL citus_cleanup_orphaned_resources();
|
|
SELECT ensure_no_temp_databases_on_any_nodes();
|
|
ensure_no_temp_databases_on_any_nodes
|
|
---------------------------------------------------------------------
|
|
t
|
|
(1 row)
|
|
|
|
SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result;
|
|
node_type | result
|
|
---------------------------------------------------------------------
|
|
worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
|
|
worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
|
|
(2 rows)
|
|
|
|
CREATE DATABASE db1;
|
|
-- show that a successful database creation doesn't leave any pg_dist_cleanup records behind
|
|
SELECT * FROM count_db_cleanup_records();
|
|
object_name | count
|
|
---------------------------------------------------------------------
|
|
(0 rows)
|
|
|
|
DROP DATABASE db1;
|
|
DROP FUNCTION get_temp_databases_on_nodes();
|
|
DROP FUNCTION ensure_no_temp_databases_on_any_nodes();
|
|
DROP FUNCTION count_db_cleanup_records();
|