add some tests

failure-handling-drop-db
Onur Tirtir 2024-02-09 16:43:51 +03:00
parent a5748fc9f3
commit a6a684ba3d
3 changed files with 240 additions and 0 deletions

View File

@ -0,0 +1,184 @@
SET citus.enable_create_database_propagation TO ON;
SET client_min_messages TO WARNING;
SELECT citus.mitmproxy('conn.kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
CREATE DATABASE db1;
ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
CALL citus_cleanup_orphaned_resources();
SELECT bool_and(result::boolean) AS no_temp_databases_on_any_nodes FROM run_command_on_all_nodes($$SELECT COUNT(*)=0 FROM pg_database WHERE datname LIKE 'citus_temp_database_%'$$);
no_temp_databases_on_any_nodes
---------------------------------------------------------------------
t
(1 row)
SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result;
node_type | result
---------------------------------------------------------------------
worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
(2 rows)
SELECT citus.mitmproxy('conn.onQuery(query="^CREATE DATABASE").cancel(' || pg_backend_pid() || ')');
mitmproxy
---------------------------------------------------------------------
(1 row)
CREATE DATABASE db1;
WARNING: Commands that are not transaction-safe may result in partial failure, potentially leading to an inconsistent state.
If the problematic command is a CREATE operation, consider using the 'IF EXISTS' syntax to drop the object,
if applicable, and then re-attempt the original command.
ERROR: canceling statement due to user request
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
CALL citus_cleanup_orphaned_resources();
SELECT bool_and(result::boolean) AS no_temp_databases_on_any_nodes FROM run_command_on_all_nodes($$SELECT COUNT(*)=0 FROM pg_database WHERE datname LIKE 'citus_temp_database_%'$$);
no_temp_databases_on_any_nodes
---------------------------------------------------------------------
t
(1 row)
SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result;
node_type | result
---------------------------------------------------------------------
worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
(2 rows)
SELECT citus.mitmproxy('conn.onQuery(query="^ALTER DATABASE").cancel(' || pg_backend_pid() || ')');
mitmproxy
---------------------------------------------------------------------
(1 row)
CREATE DATABASE db1;
ERROR: canceling statement due to user request
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
CALL citus_cleanup_orphaned_resources();
SELECT bool_and(result::boolean) AS no_temp_databases_on_any_nodes FROM run_command_on_all_nodes($$SELECT COUNT(*)=0 FROM pg_database WHERE datname LIKE 'citus_temp_database_%'$$);
no_temp_databases_on_any_nodes
---------------------------------------------------------------------
t
(1 row)
SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result;
node_type | result
---------------------------------------------------------------------
worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
(2 rows)
SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
CREATE DATABASE db1;
ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
CALL citus_cleanup_orphaned_resources();
SELECT bool_and(result::boolean) AS no_temp_databases_on_any_nodes FROM run_command_on_all_nodes($$SELECT COUNT(*)=0 FROM pg_database WHERE datname LIKE 'citus_temp_database_%'$$);
no_temp_databases_on_any_nodes
---------------------------------------------------------------------
t
(1 row)
SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result;
node_type | result
---------------------------------------------------------------------
worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
(2 rows)
SELECT citus.mitmproxy('conn.onQuery(query="^PREPARE TRANSACTION").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
CREATE DATABASE db1;
ERROR: connection not open
CONTEXT: while executing command on localhost:xxxxx
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
CALL citus_cleanup_orphaned_resources();
SELECT bool_and(result::boolean) AS no_temp_databases_on_any_nodes FROM run_command_on_all_nodes($$SELECT COUNT(*)=0 FROM pg_database WHERE datname LIKE 'citus_temp_database_%'$$);
no_temp_databases_on_any_nodes
---------------------------------------------------------------------
t
(1 row)
SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result;
node_type | result
---------------------------------------------------------------------
worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
(2 rows)
SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT PREPARED").kill()');
mitmproxy
---------------------------------------------------------------------
(1 row)
CREATE DATABASE db1;
WARNING: connection not open
CONTEXT: while executing command on localhost:xxxxx
WARNING: failed to commit transaction on localhost:xxxxx
SELECT citus.mitmproxy('conn.allow()');
mitmproxy
---------------------------------------------------------------------
(1 row)
-- not call citus_cleanup_orphaned_resources() but recover the prepared transactions this time
SELECT recover_prepared_transactions();
recover_prepared_transactions
---------------------------------------------------------------------
1
(1 row)
SELECT bool_and(result::boolean) AS no_temp_databases_on_any_nodes FROM run_command_on_all_nodes($$SELECT COUNT(*)=0 FROM pg_database WHERE datname LIKE 'citus_temp_database_%'$$);
no_temp_databases_on_any_nodes
---------------------------------------------------------------------
t
(1 row)
SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result;
node_type | result
---------------------------------------------------------------------
worker node (remote) | {"database_properties": {"datacl": null, "datname": "db1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
worker node (remote) | {"database_properties": {"datacl": null, "datname": "db1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
(2 rows)
DROP DATABASE db1;
RESET client_min_messages;

View File

@ -35,6 +35,7 @@ test: failure_mx_metadata_sync
test: failure_mx_metadata_sync_multi_trans
test: failure_connection_establishment
test: failure_non_main_db_2pc
test: failure_create_database
# this test syncs metadata to the workers
test: failure_failover_to_local_execution

View File

@ -0,0 +1,55 @@
SET citus.enable_create_database_propagation TO ON;
SET client_min_messages TO WARNING;
SELECT citus.mitmproxy('conn.kill()');
CREATE DATABASE db1;
SELECT citus.mitmproxy('conn.allow()');
CALL citus_cleanup_orphaned_resources();
SELECT bool_and(result::boolean) AS no_temp_databases_on_any_nodes FROM run_command_on_all_nodes($$SELECT COUNT(*)=0 FROM pg_database WHERE datname LIKE 'citus_temp_database_%'$$);
SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result;
SELECT citus.mitmproxy('conn.onQuery(query="^CREATE DATABASE").cancel(' || pg_backend_pid() || ')');
CREATE DATABASE db1;
SELECT citus.mitmproxy('conn.allow()');
CALL citus_cleanup_orphaned_resources();
SELECT bool_and(result::boolean) AS no_temp_databases_on_any_nodes FROM run_command_on_all_nodes($$SELECT COUNT(*)=0 FROM pg_database WHERE datname LIKE 'citus_temp_database_%'$$);
SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result;
SELECT citus.mitmproxy('conn.onQuery(query="^ALTER DATABASE").cancel(' || pg_backend_pid() || ')');
CREATE DATABASE db1;
SELECT citus.mitmproxy('conn.allow()');
CALL citus_cleanup_orphaned_resources();
SELECT bool_and(result::boolean) AS no_temp_databases_on_any_nodes FROM run_command_on_all_nodes($$SELECT COUNT(*)=0 FROM pg_database WHERE datname LIKE 'citus_temp_database_%'$$);
SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result;
SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED").kill()');
CREATE DATABASE db1;
SELECT citus.mitmproxy('conn.allow()');
CALL citus_cleanup_orphaned_resources();
SELECT bool_and(result::boolean) AS no_temp_databases_on_any_nodes FROM run_command_on_all_nodes($$SELECT COUNT(*)=0 FROM pg_database WHERE datname LIKE 'citus_temp_database_%'$$);
SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result;
SELECT citus.mitmproxy('conn.onQuery(query="^PREPARE TRANSACTION").kill()');
CREATE DATABASE db1;
SELECT citus.mitmproxy('conn.allow()');
CALL citus_cleanup_orphaned_resources();
SELECT bool_and(result::boolean) AS no_temp_databases_on_any_nodes FROM run_command_on_all_nodes($$SELECT COUNT(*)=0 FROM pg_database WHERE datname LIKE 'citus_temp_database_%'$$);
SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result;
SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT PREPARED").kill()');
CREATE DATABASE db1;
SELECT citus.mitmproxy('conn.allow()');
-- not call citus_cleanup_orphaned_resources() but recover the prepared transactions this time
SELECT recover_prepared_transactions();
SELECT bool_and(result::boolean) AS no_temp_databases_on_any_nodes FROM run_command_on_all_nodes($$SELECT COUNT(*)=0 FROM pg_database WHERE datname LIKE 'citus_temp_database_%'$$);
SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result;
DROP DATABASE db1;
RESET client_min_messages;