From 2e5ca8ba2bc2e45076995cf3b1a2593e0ba8cc42 Mon Sep 17 00:00:00 2001 From: Hanefi Onaldi Date: Fri, 4 Feb 2022 06:17:31 +0300 Subject: [PATCH] Add isolation tests for metadata sync vs all This commit introduces several test cases for concurrent operations that change metadata, and a concurrent metadata sync operation. The overall structure is as follows: - Session#1 starts metadata syncing in a transaction block - Session#2 does an operation that change metadata - Both sessions are committed - Another session checks whether the metadata are the same accross all nodes in the cluster. --- .../isolation_metadata_sync_vs_all.out | 716 ++++++++++++++++++ src/test/regress/isolation_schedule | 1 + .../spec/isolation_metadata_sync_vs_all.spec | 199 +++++ 3 files changed, 916 insertions(+) create mode 100644 src/test/regress/expected/isolation_metadata_sync_vs_all.out create mode 100644 src/test/regress/spec/isolation_metadata_sync_vs_all.spec diff --git a/src/test/regress/expected/isolation_metadata_sync_vs_all.out b/src/test/regress/expected/isolation_metadata_sync_vs_all.out new file mode 100644 index 000000000..087dcbe93 --- /dev/null +++ b/src/test/regress/expected/isolation_metadata_sync_vs_all.out @@ -0,0 +1,716 @@ +unused step name: s3-debug +Parsed test spec with 3 sessions + +starting permutation: s3-compare-snapshot +step s3-compare-snapshot: + SELECT count(*) = 0 AS same_metadata_in_workers + FROM + ( + ( + SELECT unnest(activate_node_snapshot()) + EXCEPT + SELECT unnest(result::text[]) AS unnested_result + FROM run_command_on_workers($$SELECT activate_node_snapshot()$$) + ) + UNION + ( + SELECT unnest(result::text[]) AS unnested_result + FROM run_command_on_workers($$SELECT activate_node_snapshot()$$) + EXCEPT + SELECT unnest(activate_node_snapshot()) + ) + ) AS foo; + +same_metadata_in_workers +--------------------------------------------------------------------- +t +(1 row) + + +starting permutation: s1-begin s2-begin s1-start-metadata-sync s2-start-metadata-sync-to-same-node s1-commit s2-commit s3-compare-snapshot +step s1-begin: + BEGIN; + +step s2-begin: + BEGIN; + +step s1-start-metadata-sync: + SELECT start_metadata_sync_to_node('localhost', 57638); + +start_metadata_sync_to_node +--------------------------------------------------------------------- + +(1 row) + +step s2-start-metadata-sync-to-same-node: + SELECT start_metadata_sync_to_node('localhost', 57638); + +step s1-commit: + COMMIT; + +step s2-start-metadata-sync-to-same-node: <... completed> +start_metadata_sync_to_node +--------------------------------------------------------------------- + +(1 row) + +step s2-commit: + COMMIT; + +step s3-compare-snapshot: + SELECT count(*) = 0 AS same_metadata_in_workers + FROM + ( + ( + SELECT unnest(activate_node_snapshot()) + EXCEPT + SELECT unnest(result::text[]) AS unnested_result + FROM run_command_on_workers($$SELECT activate_node_snapshot()$$) + ) + UNION + ( + SELECT unnest(result::text[]) AS unnested_result + FROM run_command_on_workers($$SELECT activate_node_snapshot()$$) + EXCEPT + SELECT unnest(activate_node_snapshot()) + ) + ) AS foo; + +same_metadata_in_workers +--------------------------------------------------------------------- +t +(1 row) + + +starting permutation: s1-begin s2-begin s1-start-metadata-sync s2-start-metadata-sync-to-another-node s1-commit s2-commit s3-compare-snapshot +step s1-begin: + BEGIN; + +step s2-begin: + BEGIN; + +step s1-start-metadata-sync: + SELECT start_metadata_sync_to_node('localhost', 57638); + +start_metadata_sync_to_node +--------------------------------------------------------------------- + +(1 row) + +step s2-start-metadata-sync-to-another-node: + SELECT start_metadata_sync_to_node('localhost', 57637); + +step s1-commit: + COMMIT; + +step s2-start-metadata-sync-to-another-node: <... completed> +start_metadata_sync_to_node +--------------------------------------------------------------------- + +(1 row) + +step s2-commit: + COMMIT; + +step s3-compare-snapshot: + SELECT count(*) = 0 AS same_metadata_in_workers + FROM + ( + ( + SELECT unnest(activate_node_snapshot()) + EXCEPT + SELECT unnest(result::text[]) AS unnested_result + FROM run_command_on_workers($$SELECT activate_node_snapshot()$$) + ) + UNION + ( + SELECT unnest(result::text[]) AS unnested_result + FROM run_command_on_workers($$SELECT activate_node_snapshot()$$) + EXCEPT + SELECT unnest(activate_node_snapshot()) + ) + ) AS foo; + +same_metadata_in_workers +--------------------------------------------------------------------- +t +(1 row) + + +starting permutation: s1-begin s2-begin s1-start-metadata-sync s2-alter-table s1-commit s2-commit s3-compare-snapshot +step s1-begin: + BEGIN; + +step s2-begin: + BEGIN; + +step s1-start-metadata-sync: + SELECT start_metadata_sync_to_node('localhost', 57638); + +start_metadata_sync_to_node +--------------------------------------------------------------------- + +(1 row) + +step s2-alter-table: + ALTER TABLE dist_table ADD COLUMN z int; + +step s1-commit: + COMMIT; + +step s2-alter-table: <... completed> +step s2-commit: + COMMIT; + +step s3-compare-snapshot: + SELECT count(*) = 0 AS same_metadata_in_workers + FROM + ( + ( + SELECT unnest(activate_node_snapshot()) + EXCEPT + SELECT unnest(result::text[]) AS unnested_result + FROM run_command_on_workers($$SELECT activate_node_snapshot()$$) + ) + UNION + ( + SELECT unnest(result::text[]) AS unnested_result + FROM run_command_on_workers($$SELECT activate_node_snapshot()$$) + EXCEPT + SELECT unnest(activate_node_snapshot()) + ) + ) AS foo; + +same_metadata_in_workers +--------------------------------------------------------------------- +t +(1 row) + + +starting permutation: s1-begin s2-begin s1-start-metadata-sync s2-drop-table s1-commit s2-commit s3-compare-snapshot +step s1-begin: + BEGIN; + +step s2-begin: + BEGIN; + +step s1-start-metadata-sync: + SELECT start_metadata_sync_to_node('localhost', 57638); + +start_metadata_sync_to_node +--------------------------------------------------------------------- + +(1 row) + +step s2-drop-table: + DROP TABLE dist_table; + +step s1-commit: + COMMIT; + +step s2-drop-table: <... completed> +step s2-commit: + COMMIT; + +step s3-compare-snapshot: + SELECT count(*) = 0 AS same_metadata_in_workers + FROM + ( + ( + SELECT unnest(activate_node_snapshot()) + EXCEPT + SELECT unnest(result::text[]) AS unnested_result + FROM run_command_on_workers($$SELECT activate_node_snapshot()$$) + ) + UNION + ( + SELECT unnest(result::text[]) AS unnested_result + FROM run_command_on_workers($$SELECT activate_node_snapshot()$$) + EXCEPT + SELECT unnest(activate_node_snapshot()) + ) + ) AS foo; + +same_metadata_in_workers +--------------------------------------------------------------------- +t +(1 row) + + +starting permutation: s1-begin s2-begin s1-start-metadata-sync s2-create-dist-table s1-commit s2-commit s3-compare-snapshot +step s1-begin: + BEGIN; + +step s2-begin: + BEGIN; + +step s1-start-metadata-sync: + SELECT start_metadata_sync_to_node('localhost', 57638); + +start_metadata_sync_to_node +--------------------------------------------------------------------- + +(1 row) + +step s2-create-dist-table: + CREATE TABLE new_dist_table(id int, data int); + SELECT create_distributed_table('new_dist_table', 'id'); + +step s1-commit: + COMMIT; + +step s2-create-dist-table: <... completed> +create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +step s2-commit: + COMMIT; + +step s3-compare-snapshot: + SELECT count(*) = 0 AS same_metadata_in_workers + FROM + ( + ( + SELECT unnest(activate_node_snapshot()) + EXCEPT + SELECT unnest(result::text[]) AS unnested_result + FROM run_command_on_workers($$SELECT activate_node_snapshot()$$) + ) + UNION + ( + SELECT unnest(result::text[]) AS unnested_result + FROM run_command_on_workers($$SELECT activate_node_snapshot()$$) + EXCEPT + SELECT unnest(activate_node_snapshot()) + ) + ) AS foo; + +same_metadata_in_workers +--------------------------------------------------------------------- +t +(1 row) + + +starting permutation: s1-begin s2-begin s1-start-metadata-sync s2-create-ref-table s1-commit s2-commit s3-compare-snapshot +step s1-begin: + BEGIN; + +step s2-begin: + BEGIN; + +step s1-start-metadata-sync: + SELECT start_metadata_sync_to_node('localhost', 57638); + +start_metadata_sync_to_node +--------------------------------------------------------------------- + +(1 row) + +step s2-create-ref-table: + CREATE TABLE new_ref_table(id int, data int); + SELECT create_reference_table('new_ref_table'); + +step s1-commit: + COMMIT; + +step s2-create-ref-table: <... completed> +create_reference_table +--------------------------------------------------------------------- + +(1 row) + +step s2-commit: + COMMIT; + +step s3-compare-snapshot: + SELECT count(*) = 0 AS same_metadata_in_workers + FROM + ( + ( + SELECT unnest(activate_node_snapshot()) + EXCEPT + SELECT unnest(result::text[]) AS unnested_result + FROM run_command_on_workers($$SELECT activate_node_snapshot()$$) + ) + UNION + ( + SELECT unnest(result::text[]) AS unnested_result + FROM run_command_on_workers($$SELECT activate_node_snapshot()$$) + EXCEPT + SELECT unnest(activate_node_snapshot()) + ) + ) AS foo; + +same_metadata_in_workers +--------------------------------------------------------------------- +t +(1 row) + + +starting permutation: s1-begin s2-begin s1-start-metadata-sync s2-attach-partition s1-commit s2-commit s3-compare-snapshot +step s1-begin: + BEGIN; + +step s2-begin: + BEGIN; + +step s1-start-metadata-sync: + SELECT start_metadata_sync_to_node('localhost', 57638); + +start_metadata_sync_to_node +--------------------------------------------------------------------- + +(1 row) + +step s2-attach-partition: + ALTER TABLE dist_partitioned_table ATTACH PARTITION dist_partitioned_table_p1 FOR VALUES FROM (1) TO (9); + +step s1-commit: + COMMIT; + +step s2-attach-partition: <... completed> +step s2-commit: + COMMIT; + +step s3-compare-snapshot: + SELECT count(*) = 0 AS same_metadata_in_workers + FROM + ( + ( + SELECT unnest(activate_node_snapshot()) + EXCEPT + SELECT unnest(result::text[]) AS unnested_result + FROM run_command_on_workers($$SELECT activate_node_snapshot()$$) + ) + UNION + ( + SELECT unnest(result::text[]) AS unnested_result + FROM run_command_on_workers($$SELECT activate_node_snapshot()$$) + EXCEPT + SELECT unnest(activate_node_snapshot()) + ) + ) AS foo; + +same_metadata_in_workers +--------------------------------------------------------------------- +t +(1 row) + + +starting permutation: s2-attach-partition s1-begin s2-begin s1-start-metadata-sync s2-detach-partition s1-commit s2-commit s3-compare-snapshot +step s2-attach-partition: + ALTER TABLE dist_partitioned_table ATTACH PARTITION dist_partitioned_table_p1 FOR VALUES FROM (1) TO (9); + +step s1-begin: + BEGIN; + +step s2-begin: + BEGIN; + +step s1-start-metadata-sync: + SELECT start_metadata_sync_to_node('localhost', 57638); + +start_metadata_sync_to_node +--------------------------------------------------------------------- + +(1 row) + +step s2-detach-partition: + ALTER TABLE dist_partitioned_table DETACH PARTITION dist_partitioned_table_p1; + +step s1-commit: + COMMIT; + +step s2-detach-partition: <... completed> +step s2-commit: + COMMIT; + +step s3-compare-snapshot: + SELECT count(*) = 0 AS same_metadata_in_workers + FROM + ( + ( + SELECT unnest(activate_node_snapshot()) + EXCEPT + SELECT unnest(result::text[]) AS unnested_result + FROM run_command_on_workers($$SELECT activate_node_snapshot()$$) + ) + UNION + ( + SELECT unnest(result::text[]) AS unnested_result + FROM run_command_on_workers($$SELECT activate_node_snapshot()$$) + EXCEPT + SELECT unnest(activate_node_snapshot()) + ) + ) AS foo; + +same_metadata_in_workers +--------------------------------------------------------------------- +t +(1 row) + + +starting permutation: s2-attach-partition s1-begin s2-begin s1-start-metadata-sync s2-create-partition-of s1-commit s2-commit s3-compare-snapshot +step s2-attach-partition: + ALTER TABLE dist_partitioned_table ATTACH PARTITION dist_partitioned_table_p1 FOR VALUES FROM (1) TO (9); + +step s1-begin: + BEGIN; + +step s2-begin: + BEGIN; + +step s1-start-metadata-sync: + SELECT start_metadata_sync_to_node('localhost', 57638); + +start_metadata_sync_to_node +--------------------------------------------------------------------- + +(1 row) + +step s2-create-partition-of: + CREATE TABLE dist_partitioned_table_p2 PARTITION OF dist_partitioned_table FOR VALUES FROM (10) TO (20); + +step s1-commit: + COMMIT; + +step s2-create-partition-of: <... completed> +step s2-commit: + COMMIT; + +step s3-compare-snapshot: + SELECT count(*) = 0 AS same_metadata_in_workers + FROM + ( + ( + SELECT unnest(activate_node_snapshot()) + EXCEPT + SELECT unnest(result::text[]) AS unnested_result + FROM run_command_on_workers($$SELECT activate_node_snapshot()$$) + ) + UNION + ( + SELECT unnest(result::text[]) AS unnested_result + FROM run_command_on_workers($$SELECT activate_node_snapshot()$$) + EXCEPT + SELECT unnest(activate_node_snapshot()) + ) + ) AS foo; + +same_metadata_in_workers +--------------------------------------------------------------------- +t +(1 row) + + +starting permutation: s1-begin s2-begin s1-start-metadata-sync s2-add-fk s1-commit s2-commit s3-compare-snapshot +step s1-begin: + BEGIN; + +step s2-begin: + BEGIN; + +step s1-start-metadata-sync: + SELECT start_metadata_sync_to_node('localhost', 57638); + +start_metadata_sync_to_node +--------------------------------------------------------------------- + +(1 row) + +step s2-add-fk: + ALTER TABLE dist_table ADD CONSTRAINT y_fk FOREIGN KEY (y) REFERENCES ref_table(y); + +step s1-commit: + COMMIT; + +step s2-add-fk: <... completed> +step s2-commit: + COMMIT; + +step s3-compare-snapshot: + SELECT count(*) = 0 AS same_metadata_in_workers + FROM + ( + ( + SELECT unnest(activate_node_snapshot()) + EXCEPT + SELECT unnest(result::text[]) AS unnested_result + FROM run_command_on_workers($$SELECT activate_node_snapshot()$$) + ) + UNION + ( + SELECT unnest(result::text[]) AS unnested_result + FROM run_command_on_workers($$SELECT activate_node_snapshot()$$) + EXCEPT + SELECT unnest(activate_node_snapshot()) + ) + ) AS foo; + +same_metadata_in_workers +--------------------------------------------------------------------- +t +(1 row) + + +starting permutation: s2-add-fk s1-begin s2-begin s1-start-metadata-sync s2-drop-fk s1-commit s2-commit s3-compare-snapshot +step s2-add-fk: + ALTER TABLE dist_table ADD CONSTRAINT y_fk FOREIGN KEY (y) REFERENCES ref_table(y); + +step s1-begin: + BEGIN; + +step s2-begin: + BEGIN; + +step s1-start-metadata-sync: + SELECT start_metadata_sync_to_node('localhost', 57638); + +start_metadata_sync_to_node +--------------------------------------------------------------------- + +(1 row) + +step s2-drop-fk: + ALTER TABLE dist_table DROP CONSTRAINT y_fk; + +step s1-commit: + COMMIT; + +step s2-drop-fk: <... completed> +step s2-commit: + COMMIT; + +step s3-compare-snapshot: + SELECT count(*) = 0 AS same_metadata_in_workers + FROM + ( + ( + SELECT unnest(activate_node_snapshot()) + EXCEPT + SELECT unnest(result::text[]) AS unnested_result + FROM run_command_on_workers($$SELECT activate_node_snapshot()$$) + ) + UNION + ( + SELECT unnest(result::text[]) AS unnested_result + FROM run_command_on_workers($$SELECT activate_node_snapshot()$$) + EXCEPT + SELECT unnest(activate_node_snapshot()) + ) + ) AS foo; + +same_metadata_in_workers +--------------------------------------------------------------------- +t +(1 row) + + +starting permutation: s1-begin s2-begin s1-start-metadata-sync s2-create-type s1-commit s2-commit s3-compare-snapshot +step s1-begin: + BEGIN; + +step s2-begin: + BEGIN; + +step s1-start-metadata-sync: + SELECT start_metadata_sync_to_node('localhost', 57638); + +start_metadata_sync_to_node +--------------------------------------------------------------------- + +(1 row) + +step s2-create-type: + CREATE TYPE my_type AS (a int, b int); + +step s1-commit: + COMMIT; + +step s2-commit: + COMMIT; + +step s3-compare-snapshot: + SELECT count(*) = 0 AS same_metadata_in_workers + FROM + ( + ( + SELECT unnest(activate_node_snapshot()) + EXCEPT + SELECT unnest(result::text[]) AS unnested_result + FROM run_command_on_workers($$SELECT activate_node_snapshot()$$) + ) + UNION + ( + SELECT unnest(result::text[]) AS unnested_result + FROM run_command_on_workers($$SELECT activate_node_snapshot()$$) + EXCEPT + SELECT unnest(activate_node_snapshot()) + ) + ) AS foo; + +same_metadata_in_workers +--------------------------------------------------------------------- +t +(1 row) + + +starting permutation: s1-begin s2-begin s1-start-metadata-sync s2-create-dist-func s1-commit s2-commit s3-compare-snapshot +step s1-begin: + BEGIN; + +step s2-begin: + BEGIN; + +step s1-start-metadata-sync: + SELECT start_metadata_sync_to_node('localhost', 57638); + +start_metadata_sync_to_node +--------------------------------------------------------------------- + +(1 row) + +step s2-create-dist-func: + CREATE FUNCTION squares(int) RETURNS SETOF RECORD + AS $$ SELECT i, i * i FROM generate_series(1, $1) i $$ + LANGUAGE SQL; + SELECT create_distributed_function('squares(int)'); + +step s1-commit: + COMMIT; + +step s2-create-dist-func: <... completed> +create_distributed_function +--------------------------------------------------------------------- + +(1 row) + +step s2-commit: + COMMIT; + +step s3-compare-snapshot: + SELECT count(*) = 0 AS same_metadata_in_workers + FROM + ( + ( + SELECT unnest(activate_node_snapshot()) + EXCEPT + SELECT unnest(result::text[]) AS unnested_result + FROM run_command_on_workers($$SELECT activate_node_snapshot()$$) + ) + UNION + ( + SELECT unnest(result::text[]) AS unnested_result + FROM run_command_on_workers($$SELECT activate_node_snapshot()$$) + EXCEPT + SELECT unnest(activate_node_snapshot()) + ) + ) AS foo; + +same_metadata_in_workers +--------------------------------------------------------------------- +t +(1 row) + diff --git a/src/test/regress/isolation_schedule b/src/test/regress/isolation_schedule index 5df2e9a6d..280f5df28 100644 --- a/src/test/regress/isolation_schedule +++ b/src/test/regress/isolation_schedule @@ -92,6 +92,7 @@ test: isolation_metadata_sync_deadlock test: isolation_replicated_dist_on_mx test: isolation_replicate_reference_tables_to_coordinator test: isolation_multiuser_locking +test: isolation_metadata_sync_vs_all # MXless tests test: isolation_check_mx diff --git a/src/test/regress/spec/isolation_metadata_sync_vs_all.spec b/src/test/regress/spec/isolation_metadata_sync_vs_all.spec new file mode 100644 index 000000000..7ff246365 --- /dev/null +++ b/src/test/regress/spec/isolation_metadata_sync_vs_all.spec @@ -0,0 +1,199 @@ +setup +{ + SET citus.shard_replication_factor to 1; + + ALTER SEQUENCE pg_catalog.pg_dist_colocationid_seq RESTART 123000; + ALTER SEQUENCE pg_catalog.pg_dist_groupid_seq RESTART 123000; + ALTER SEQUENCE pg_catalog.pg_dist_node_nodeid_seq RESTART 123000; + ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 123000; + + -- Create the necessary test utility function + CREATE OR REPLACE FUNCTION activate_node_snapshot() + RETURNS text[] + LANGUAGE C STRICT + AS 'citus'; + SELECT create_distributed_function('activate_node_snapshot()'); + + -- Create distributed tables + CREATE TABLE ref_table (test_id integer, y int unique); + SELECT create_reference_table('ref_table'); + + CREATE TABLE dist_table (x int, y int); + SELECT create_distributed_table('dist_table', 'x'); + + CREATE TABLE dist_partitioned_table (x int, y int) PARTITION BY RANGE(y); + SELECT create_distributed_table('dist_partitioned_table', 'x'); + + CREATE TABLE dist_partitioned_table_p1(x int, y int); +} + +teardown +{ + // drop all distributed tables + DROP TABLE IF EXISTS ref_table, + dist_table, + dist_partitioned_table, + dist_partitioned_table_p1, + dist_partitioned_table_p2, + new_dist_table, + new_ref_table; + + + // drop all distributed objects + DROP FUNCTION activate_node_snapshot(); + DROP FUNCTION IF EXISTS squares(int); + DROP TYPE IF EXISTS my_type; +} + +session "s1" + +step "s1-begin" +{ + BEGIN; +} + +step "s1-commit" +{ + COMMIT; +} + +step "s1-start-metadata-sync" +{ + SELECT start_metadata_sync_to_node('localhost', 57638); +} + +session "s2" + +step "s2-begin" +{ + BEGIN; +} + +step "s2-commit" +{ + COMMIT; +} + +step "s2-start-metadata-sync-to-same-node" +{ + SELECT start_metadata_sync_to_node('localhost', 57638); +} + +step "s2-start-metadata-sync-to-another-node" +{ + SELECT start_metadata_sync_to_node('localhost', 57637); +} + +step "s2-alter-table" +{ + ALTER TABLE dist_table ADD COLUMN z int; +} + +step "s2-add-fk" +{ + ALTER TABLE dist_table ADD CONSTRAINT y_fk FOREIGN KEY (y) REFERENCES ref_table(y); +} + +step "s2-drop-fk" +{ + ALTER TABLE dist_table DROP CONSTRAINT y_fk; +} + +step "s2-drop-table" +{ + DROP TABLE dist_table; +} + +step "s2-create-dist-table" +{ + CREATE TABLE new_dist_table(id int, data int); + SELECT create_distributed_table('new_dist_table', 'id'); +} + +step "s2-create-ref-table" +{ + CREATE TABLE new_ref_table(id int, data int); + SELECT create_reference_table('new_ref_table'); +} + +step "s2-attach-partition" +{ + ALTER TABLE dist_partitioned_table ATTACH PARTITION dist_partitioned_table_p1 FOR VALUES FROM (1) TO (9); +} + +step "s2-detach-partition" +{ + ALTER TABLE dist_partitioned_table DETACH PARTITION dist_partitioned_table_p1; +} + +step "s2-create-partition-of" +{ + CREATE TABLE dist_partitioned_table_p2 PARTITION OF dist_partitioned_table FOR VALUES FROM (10) TO (20); +} + +step "s2-create-type" +{ + CREATE TYPE my_type AS (a int, b int); +} + +step "s2-create-dist-func" +{ + CREATE FUNCTION squares(int) RETURNS SETOF RECORD + AS $$ SELECT i, i * i FROM generate_series(1, $1) i $$ + LANGUAGE SQL; + + SELECT create_distributed_function('squares(int)'); +} + +session "s3" + +step "s3-compare-snapshot" +{ + SELECT count(*) = 0 AS same_metadata_in_workers + FROM + ( + ( + SELECT unnest(activate_node_snapshot()) + EXCEPT + SELECT unnest(result::text[]) AS unnested_result + FROM run_command_on_workers($$SELECT activate_node_snapshot()$$) + ) + UNION + ( + SELECT unnest(result::text[]) AS unnested_result + FROM run_command_on_workers($$SELECT activate_node_snapshot()$$) + EXCEPT + SELECT unnest(activate_node_snapshot()) + ) + ) AS foo; +} + +step "s3-debug" +{ + SELECT unnest(activate_node_snapshot()); + + SELECT unnest(result::text[]) + FROM run_command_on_workers('SELECT activate_node_snapshot()'); +} + +// before running any updates to metadata, make sure all nodes have same metadata in the cluster +permutation "s3-compare-snapshot" + +// concurrent metadata syncing operations get blocked +permutation "s1-begin" "s2-begin" "s1-start-metadata-sync" "s2-start-metadata-sync-to-same-node" "s1-commit" "s2-commit" "s3-compare-snapshot" +permutation "s1-begin" "s2-begin" "s1-start-metadata-sync" "s2-start-metadata-sync-to-another-node" "s1-commit" "s2-commit" "s3-compare-snapshot" + +// the following operations get blocked when a concurrent metadata sync is in progress +permutation "s1-begin" "s2-begin" "s1-start-metadata-sync" "s2-alter-table" "s1-commit" "s2-commit" "s3-compare-snapshot" +permutation "s1-begin" "s2-begin" "s1-start-metadata-sync" "s2-drop-table" "s1-commit" "s2-commit" "s3-compare-snapshot" +permutation "s1-begin" "s2-begin" "s1-start-metadata-sync" "s2-create-dist-table" "s1-commit" "s2-commit" "s3-compare-snapshot" +permutation "s1-begin" "s2-begin" "s1-start-metadata-sync" "s2-create-ref-table" "s1-commit" "s2-commit" "s3-compare-snapshot" +permutation "s1-begin" "s2-begin" "s1-start-metadata-sync" "s2-attach-partition" "s1-commit" "s2-commit" "s3-compare-snapshot" +permutation "s2-attach-partition" "s1-begin" "s2-begin" "s1-start-metadata-sync" "s2-detach-partition" "s1-commit" "s2-commit" "s3-compare-snapshot" +permutation "s2-attach-partition" "s1-begin" "s2-begin" "s1-start-metadata-sync" "s2-create-partition-of" "s1-commit" "s2-commit" "s3-compare-snapshot" +permutation "s1-begin" "s2-begin" "s1-start-metadata-sync" "s2-add-fk" "s1-commit" "s2-commit" "s3-compare-snapshot" +permutation "s2-add-fk" "s1-begin" "s2-begin" "s1-start-metadata-sync" "s2-drop-fk" "s1-commit" "s2-commit" "s3-compare-snapshot" + +// the following operations do not get blocked +permutation "s1-begin" "s2-begin" "s1-start-metadata-sync" "s2-create-type" "s1-commit" "s2-commit" "s3-compare-snapshot" +permutation "s1-begin" "s2-begin" "s1-start-metadata-sync" "s2-create-dist-func" "s1-commit" "s2-commit" "s3-compare-snapshot"