diff --git a/src/test/regress/expected/failure_create_database.out b/src/test/regress/expected/failure_create_database.out new file mode 100644 index 000000000..004c651de --- /dev/null +++ b/src/test/regress/expected/failure_create_database.out @@ -0,0 +1,184 @@ +SET citus.enable_create_database_propagation TO ON; +SET client_min_messages TO WARNING; +SELECT citus.mitmproxy('conn.kill()'); + mitmproxy +--------------------------------------------------------------------- + +(1 row) + +CREATE DATABASE db1; +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open +SELECT citus.mitmproxy('conn.allow()'); + mitmproxy +--------------------------------------------------------------------- + +(1 row) + +CALL citus_cleanup_orphaned_resources(); +SELECT bool_and(result::boolean) AS no_temp_databases_on_any_nodes FROM run_command_on_all_nodes($$SELECT COUNT(*)=0 FROM pg_database WHERE datname LIKE 'citus_temp_database_%'$$); + no_temp_databases_on_any_nodes +--------------------------------------------------------------------- + t +(1 row) + +SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result; + node_type | result +--------------------------------------------------------------------- + worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} +(2 rows) + +SELECT citus.mitmproxy('conn.onQuery(query="^CREATE DATABASE").cancel(' || pg_backend_pid() || ')'); + mitmproxy +--------------------------------------------------------------------- + +(1 row) + +CREATE DATABASE db1; +WARNING: Commands that are not transaction-safe may result in partial failure, potentially leading to an inconsistent state. +If the problematic command is a CREATE operation, consider using the 'IF EXISTS' syntax to drop the object, +if applicable, and then re-attempt the original command. +ERROR: canceling statement due to user request +SELECT citus.mitmproxy('conn.allow()'); + mitmproxy +--------------------------------------------------------------------- + +(1 row) + +CALL citus_cleanup_orphaned_resources(); +SELECT bool_and(result::boolean) AS no_temp_databases_on_any_nodes FROM run_command_on_all_nodes($$SELECT COUNT(*)=0 FROM pg_database WHERE datname LIKE 'citus_temp_database_%'$$); + no_temp_databases_on_any_nodes +--------------------------------------------------------------------- + t +(1 row) + +SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result; + node_type | result +--------------------------------------------------------------------- + worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} +(2 rows) + +SELECT citus.mitmproxy('conn.onQuery(query="^ALTER DATABASE").cancel(' || pg_backend_pid() || ')'); + mitmproxy +--------------------------------------------------------------------- + +(1 row) + +CREATE DATABASE db1; +ERROR: canceling statement due to user request +SELECT citus.mitmproxy('conn.allow()'); + mitmproxy +--------------------------------------------------------------------- + +(1 row) + +CALL citus_cleanup_orphaned_resources(); +SELECT bool_and(result::boolean) AS no_temp_databases_on_any_nodes FROM run_command_on_all_nodes($$SELECT COUNT(*)=0 FROM pg_database WHERE datname LIKE 'citus_temp_database_%'$$); + no_temp_databases_on_any_nodes +--------------------------------------------------------------------- + t +(1 row) + +SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result; + node_type | result +--------------------------------------------------------------------- + worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} +(2 rows) + +SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED").kill()'); + mitmproxy +--------------------------------------------------------------------- + +(1 row) + +CREATE DATABASE db1; +ERROR: connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open +SELECT citus.mitmproxy('conn.allow()'); + mitmproxy +--------------------------------------------------------------------- + +(1 row) + +CALL citus_cleanup_orphaned_resources(); +SELECT bool_and(result::boolean) AS no_temp_databases_on_any_nodes FROM run_command_on_all_nodes($$SELECT COUNT(*)=0 FROM pg_database WHERE datname LIKE 'citus_temp_database_%'$$); + no_temp_databases_on_any_nodes +--------------------------------------------------------------------- + t +(1 row) + +SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result; + node_type | result +--------------------------------------------------------------------- + worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} +(2 rows) + +SELECT citus.mitmproxy('conn.onQuery(query="^PREPARE TRANSACTION").kill()'); + mitmproxy +--------------------------------------------------------------------- + +(1 row) + +CREATE DATABASE db1; +ERROR: connection not open +CONTEXT: while executing command on localhost:xxxxx +SELECT citus.mitmproxy('conn.allow()'); + mitmproxy +--------------------------------------------------------------------- + +(1 row) + +CALL citus_cleanup_orphaned_resources(); +SELECT bool_and(result::boolean) AS no_temp_databases_on_any_nodes FROM run_command_on_all_nodes($$SELECT COUNT(*)=0 FROM pg_database WHERE datname LIKE 'citus_temp_database_%'$$); + no_temp_databases_on_any_nodes +--------------------------------------------------------------------- + t +(1 row) + +SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result; + node_type | result +--------------------------------------------------------------------- + worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false} +(2 rows) + +SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT PREPARED").kill()'); + mitmproxy +--------------------------------------------------------------------- + +(1 row) + +CREATE DATABASE db1; +WARNING: connection not open +CONTEXT: while executing command on localhost:xxxxx +WARNING: failed to commit transaction on localhost:xxxxx +SELECT citus.mitmproxy('conn.allow()'); + mitmproxy +--------------------------------------------------------------------- + +(1 row) + +-- not call citus_cleanup_orphaned_resources() but recover the prepared transactions this time +SELECT recover_prepared_transactions(); + recover_prepared_transactions +--------------------------------------------------------------------- + 1 +(1 row) + +SELECT bool_and(result::boolean) AS no_temp_databases_on_any_nodes FROM run_command_on_all_nodes($$SELECT COUNT(*)=0 FROM pg_database WHERE datname LIKE 'citus_temp_database_%'$$); + no_temp_databases_on_any_nodes +--------------------------------------------------------------------- + t +(1 row) + +SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result; + node_type | result +--------------------------------------------------------------------- + worker node (remote) | {"database_properties": {"datacl": null, "datname": "db1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} + worker node (remote) | {"database_properties": {"datacl": null, "datname": "db1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false} +(2 rows) + +DROP DATABASE db1; +RESET client_min_messages; diff --git a/src/test/regress/failure_schedule b/src/test/regress/failure_schedule index e1ad362b5..8b992422e 100644 --- a/src/test/regress/failure_schedule +++ b/src/test/regress/failure_schedule @@ -35,6 +35,7 @@ test: failure_mx_metadata_sync test: failure_mx_metadata_sync_multi_trans test: failure_connection_establishment test: failure_non_main_db_2pc +test: failure_create_database # this test syncs metadata to the workers test: failure_failover_to_local_execution diff --git a/src/test/regress/sql/failure_create_database.sql b/src/test/regress/sql/failure_create_database.sql new file mode 100644 index 000000000..3362c7a00 --- /dev/null +++ b/src/test/regress/sql/failure_create_database.sql @@ -0,0 +1,55 @@ +SET citus.enable_create_database_propagation TO ON; +SET client_min_messages TO WARNING; + +SELECT citus.mitmproxy('conn.kill()'); +CREATE DATABASE db1; +SELECT citus.mitmproxy('conn.allow()'); + +CALL citus_cleanup_orphaned_resources(); +SELECT bool_and(result::boolean) AS no_temp_databases_on_any_nodes FROM run_command_on_all_nodes($$SELECT COUNT(*)=0 FROM pg_database WHERE datname LIKE 'citus_temp_database_%'$$); +SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result; + +SELECT citus.mitmproxy('conn.onQuery(query="^CREATE DATABASE").cancel(' || pg_backend_pid() || ')'); +CREATE DATABASE db1; +SELECT citus.mitmproxy('conn.allow()'); + +CALL citus_cleanup_orphaned_resources(); +SELECT bool_and(result::boolean) AS no_temp_databases_on_any_nodes FROM run_command_on_all_nodes($$SELECT COUNT(*)=0 FROM pg_database WHERE datname LIKE 'citus_temp_database_%'$$); +SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result; + +SELECT citus.mitmproxy('conn.onQuery(query="^ALTER DATABASE").cancel(' || pg_backend_pid() || ')'); +CREATE DATABASE db1; +SELECT citus.mitmproxy('conn.allow()'); + +CALL citus_cleanup_orphaned_resources(); +SELECT bool_and(result::boolean) AS no_temp_databases_on_any_nodes FROM run_command_on_all_nodes($$SELECT COUNT(*)=0 FROM pg_database WHERE datname LIKE 'citus_temp_database_%'$$); +SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result; + +SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED").kill()'); +CREATE DATABASE db1; +SELECT citus.mitmproxy('conn.allow()'); + +CALL citus_cleanup_orphaned_resources(); +SELECT bool_and(result::boolean) AS no_temp_databases_on_any_nodes FROM run_command_on_all_nodes($$SELECT COUNT(*)=0 FROM pg_database WHERE datname LIKE 'citus_temp_database_%'$$); +SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result; + +SELECT citus.mitmproxy('conn.onQuery(query="^PREPARE TRANSACTION").kill()'); +CREATE DATABASE db1; +SELECT citus.mitmproxy('conn.allow()'); + +CALL citus_cleanup_orphaned_resources(); +SELECT bool_and(result::boolean) AS no_temp_databases_on_any_nodes FROM run_command_on_all_nodes($$SELECT COUNT(*)=0 FROM pg_database WHERE datname LIKE 'citus_temp_database_%'$$); +SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result; + +SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT PREPARED").kill()'); +CREATE DATABASE db1; +SELECT citus.mitmproxy('conn.allow()'); + +-- not call citus_cleanup_orphaned_resources() but recover the prepared transactions this time +SELECT recover_prepared_transactions(); +SELECT bool_and(result::boolean) AS no_temp_databases_on_any_nodes FROM run_command_on_all_nodes($$SELECT COUNT(*)=0 FROM pg_database WHERE datname LIKE 'citus_temp_database_%'$$); +SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result; + +DROP DATABASE db1; + +RESET client_min_messages;