Fix flakyness in create index concurrently isolation tests (#6158)

This creates consistent test output for isolation tests that involve
`CREATE INDEX CONCURRENTLY`. `CREATE INDEX CONCURRENTLY` is sometimes
temporarily detected as blocking, even though it will complete without any other
queries needing to be run. This change makes sure that we wait until that happens
without running any other queries in the meantime. This way we always get consistent
output. The way we do that is addressed by using an empty step in the same
session as the `CREATE INDEX CONCURRENLTY` command. Doing so forces
the isolation tester to wait until the command is finished and not continue with
steps from other sessions. This is [the recommended approach by Postgres][1].

There's two separate cases which are addressed in slightly different ways:
1. If `CREATE INDEX CONCURRENTLY` is actually blocked on another session: Add an 
    empty step right after the commit of blocking session.
    e.g. `"s2-ddl-create-index-concurrently" "s1-commit" "s2-empty"`
2. If it's not actually blocked on another session: Add [an asterisk marker][2] to make 
    it look like it's blocked (because sometimes this happens randomly) and right 
    after that we add an empty step to trigger waiting.
    e.g. `"s2-ddl-create-index-concurrently"(*) "s2-empty" "s1-commit"`

In passing this also enables isolation tests that were disabled due to a
bug that has already been fixed for a while.

Fixes #5993
Related to #5910 and #2966

[1]: 5f0adec253/src/test/isolation/README (L197-L204)
[2]: 5f0adec253/src/test/isolation/README (L174-L179)

Co-authored-by: Hanefi Onaldi <Hanefi.Onaldi@microsoft.com>

src/test/regress/bin/normalize.sed

@@ -198,20 +198,6 @@ s/^(ERROR:  child table is missing constraint "\w+)_([0-9])+"/\1_xxxxxx"/g
     }
 }
 
-# normalize for random waits for CREATE INDEX CONCURRENTLY isolation tests.
-# <waiting ...> outputs can be in separate lines, or on the same line, and hence
-# we have a slightly more complex pattern.
-# All the flaky tests use a index name that starts with `flaky` so we limit the
-# normalization using that pattern.
-/CREATE INDEX CONCURRENTLY flaky/ {
-	N; s/ <waiting ...>//
-}
-
-# Remove completion lines in isolation tests for CREATE INDEX CONCURRENTLY commands.
-# This is needed because the commands that are executed on the shards can block each other
-# for a small window of time and we may see the completion output in different lines.
-/step s2-flaky.* <... completed>/d
-
 # normalize long table shard name errors for alter_table_set_access_method and alter_distributed_table
 s/^(ERROR:  child table is missing constraint "\w+)_([0-9])+"/\1_xxxxxx"/g
 s/^(DEBUG:  the name of the shard \(abcde_01234567890123456789012345678901234567890_f7ff6612)_([0-9])+/\1_xxxxxx/g

src/test/regress/expected/isolation_copy_vs_all_on_mx.out

@@ -235,3 +235,58 @@ count
     8
 (1 row)
 
+
+starting permutation: s1-start-session-level-connection s1-begin-on-worker s1-copy s2-coordinator-create-index-concurrently s1-commit-worker s2-empty s3-select-count s1-stop-connection
+step s1-start-session-level-connection:
+        SELECT start_session_level_connection_to_node('localhost', 57637);
+
+start_session_level_connection_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
+step s1-begin-on-worker:
+        SELECT run_commands_on_session_level_connection_to_node('BEGIN');
+
+run_commands_on_session_level_connection_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
+step s1-copy:
+ SELECT run_commands_on_session_level_connection_to_node('COPY copy_table FROM PROGRAM ''echo 5, 50 && echo 6, 60 && echo 7, 70''WITH CSV');
+
+run_commands_on_session_level_connection_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
+step s2-coordinator-create-index-concurrently:
+ CREATE INDEX CONCURRENTLY copy_table_index ON copy_table(id);
+ <waiting ...>
+step s1-commit-worker: 
+ SELECT run_commands_on_session_level_connection_to_node('COMMIT');
+
+run_commands_on_session_level_connection_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
+step s2-coordinator-create-index-concurrently: <... completed>
+step s2-empty:
+step s3-select-count:
+ SELECT COUNT(*) FROM copy_table;
+
+count
+---------------------------------------------------------------------
+    8
+(1 row)
+
+step s1-stop-connection:
+ SELECT stop_session_level_connection_to_node();
+
+stop_session_level_connection_to_node
+---------------------------------------------------------------------
+
+(1 row)
+

src/test/regress/expected/isolation_ddl_vs_all.out

@@ -23,7 +23,7 @@ run_command_on_workers
 (2 rows)
 
 
-starting permutation: s1-initialize s1-begin s1-ddl-create-index s2-ddl-create-index-concurrently s1-commit s1-show-indexes
+starting permutation: s1-initialize s1-begin s1-ddl-create-index s2-ddl-create-index-concurrently s1-commit s2-empty s1-show-indexes
 create_distributed_table
 ---------------------------------------------------------------------
 
@@ -36,6 +36,7 @@ step s2-ddl-create-index-concurrently: CREATE INDEX CONCURRENTLY ddl_hash_index
 step s1-commit: COMMIT;
 step s2-ddl-create-index-concurrently: <... completed>
 ERROR:  relation "ddl_hash_index" already exists
+step s2-empty:
 step s1-show-indexes: SELECT run_command_on_workers('SELECT COUNT(*) FROM pg_indexes WHERE tablename LIKE ''ddl_hash\_%''');
 run_command_on_workers
 ---------------------------------------------------------------------
@@ -131,7 +132,7 @@ run_command_on_workers
 (2 rows)
 
 
-starting permutation: s1-initialize s1-begin s1-ddl-add-column s2-ddl-create-index-concurrently s1-commit s1-show-columns s1-show-indexes
+starting permutation: s1-initialize s1-begin s1-ddl-add-column s2-ddl-create-index-concurrently s1-commit s2-empty s1-show-columns s1-show-indexes
 create_distributed_table
 ---------------------------------------------------------------------
 
@@ -143,6 +144,7 @@ step s1-ddl-add-column: ALTER TABLE ddl_hash ADD new_column_1 int DEFAULT 0;
 step s2-ddl-create-index-concurrently: CREATE INDEX CONCURRENTLY ddl_hash_index ON ddl_hash(id); <waiting ...>
 step s1-commit: COMMIT;
 step s2-ddl-create-index-concurrently: <... completed>
+step s2-empty:
 step s1-show-columns: SELECT run_command_on_workers('SELECT column_name FROM information_schema.columns WHERE table_name LIKE ''ddl_hash%'' AND column_name = ''new_column'' ORDER BY 1 LIMIT 1');
 run_command_on_workers
 ---------------------------------------------------------------------
@@ -231,7 +233,7 @@ run_command_on_workers
 (2 rows)
 
 
-starting permutation: s1-initialize s1-begin s1-ddl-rename-column s2-ddl-create-index-concurrently s1-commit s1-show-columns s1-show-indexes
+starting permutation: s1-initialize s1-begin s1-ddl-rename-column s2-ddl-create-index-concurrently s1-commit s2-empty s1-show-columns s1-show-indexes
 create_distributed_table
 ---------------------------------------------------------------------
 
@@ -243,6 +245,7 @@ step s1-ddl-rename-column: ALTER TABLE ddl_hash RENAME data TO new_column;
 step s2-ddl-create-index-concurrently: CREATE INDEX CONCURRENTLY ddl_hash_index ON ddl_hash(id); <waiting ...>
 step s1-commit: COMMIT;
 step s2-ddl-create-index-concurrently: <... completed>
+step s2-empty:
 step s1-show-columns: SELECT run_command_on_workers('SELECT column_name FROM information_schema.columns WHERE table_name LIKE ''ddl_hash%'' AND column_name = ''new_column'' ORDER BY 1 LIMIT 1');
 run_command_on_workers
 ---------------------------------------------------------------------
@@ -613,7 +616,7 @@ run_command_on_workers
 (2 rows)
 
 
-starting permutation: s1-initialize s1-begin s1-table-size s2-ddl-create-index-concurrently s1-commit s1-show-indexes
+starting permutation: s1-initialize s1-begin s1-table-size s2-ddl-create-index-concurrently s2-empty s1-commit s1-show-indexes
 create_distributed_table
 ---------------------------------------------------------------------
 
@@ -627,7 +630,9 @@ citus_total_relation_size
                     57344
 (1 row)
 
-step s2-ddl-create-index-concurrently: CREATE INDEX CONCURRENTLY ddl_hash_index ON ddl_hash(id);
+step s2-ddl-create-index-concurrently: CREATE INDEX CONCURRENTLY ddl_hash_index ON ddl_hash(id); <waiting ...>
+step s2-ddl-create-index-concurrently: <... completed>
+step s2-empty:
 step s1-commit: COMMIT;
 step s1-show-indexes: SELECT run_command_on_workers('SELECT COUNT(*) FROM pg_indexes WHERE tablename LIKE ''ddl_hash\_%''');
 run_command_on_workers
@@ -637,7 +642,7 @@ run_command_on_workers
 (2 rows)
 
 
-starting permutation: s1-initialize s1-begin s1-master-modify-multiple-shards s2-ddl-create-index-concurrently s1-commit s1-show-indexes
+starting permutation: s1-initialize s1-begin s1-master-modify-multiple-shards s2-ddl-create-index-concurrently s1-commit s2-empty s1-show-indexes
 create_distributed_table
 ---------------------------------------------------------------------
 
@@ -649,6 +654,7 @@ step s1-master-modify-multiple-shards: DELETE FROM ddl_hash;
 step s2-ddl-create-index-concurrently: CREATE INDEX CONCURRENTLY ddl_hash_index ON ddl_hash(id); <waiting ...>
 step s1-commit: COMMIT;
 step s2-ddl-create-index-concurrently: <... completed>
+step s2-empty:
 step s1-show-indexes: SELECT run_command_on_workers('SELECT COUNT(*) FROM pg_indexes WHERE tablename LIKE ''ddl_hash\_%''');
 run_command_on_workers
 ---------------------------------------------------------------------
@@ -657,7 +663,7 @@ run_command_on_workers
 (2 rows)
 
 
-starting permutation: s1-drop s1-create-non-distributed-table s1-initialize s1-begin s1-distribute-table s2-ddl-create-index-concurrently s1-commit s1-show-indexes
+starting permutation: s1-drop s1-create-non-distributed-table s1-initialize s1-begin s1-distribute-table s2-ddl-create-index-concurrently s1-commit s2-empty s1-show-indexes
 create_distributed_table
 ---------------------------------------------------------------------
 
@@ -676,6 +682,7 @@ create_distributed_table
 step s2-ddl-create-index-concurrently: CREATE INDEX CONCURRENTLY ddl_hash_index ON ddl_hash(id); <waiting ...>
 step s1-commit: COMMIT;
 step s2-ddl-create-index-concurrently: <... completed>
+step s2-empty:
 step s1-show-indexes: SELECT run_command_on_workers('SELECT COUNT(*) FROM pg_indexes WHERE tablename LIKE ''ddl_hash\_%''');
 run_command_on_workers
 ---------------------------------------------------------------------

src/test/regress/expected/isolation_delete_vs_all.out

@@ -116,7 +116,7 @@ run_command_on_workers
 (2 rows)
 
 
-starting permutation: s1-initialize s1-begin s1-delete s2-ddl-create-index-concurrently s1-commit s1-select-count s1-show-indexes
+starting permutation: s1-initialize s1-begin s1-delete s2-ddl-create-index-concurrently s1-commit s2-empty s1-select-count s1-show-indexes
 create_distributed_table
 ---------------------------------------------------------------------
 
@@ -128,6 +128,7 @@ step s1-delete: DELETE FROM delete_hash WHERE id = 4;
 step s2-ddl-create-index-concurrently: CREATE INDEX CONCURRENTLY delete_hash_index ON delete_hash(id); <waiting ...>
 step s1-commit: COMMIT;
 step s2-ddl-create-index-concurrently: <... completed>
+step s2-empty:
 step s1-select-count: SELECT COUNT(*) FROM delete_hash;
 count
 ---------------------------------------------------------------------

src/test/regress/expected/isolation_dis2ref_foreign_keys_on_mx.out

@@ -689,3 +689,66 @@ id|value
  2|    2
 (1 row)
 
+
+starting permutation: s1-start-session-level-connection s1-begin-on-worker s1-update s2-coordinator-create-index-concurrently s1-commit-worker s2-empty s1-stop-connection s3-display
+step s1-start-session-level-connection:
+ SELECT start_session_level_connection_to_node('localhost', 57637);
+
+start_session_level_connection_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
+step s1-begin-on-worker:
+ SELECT run_commands_on_session_level_connection_to_node('BEGIN');
+
+run_commands_on_session_level_connection_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
+step s1-update:
+ SELECT run_commands_on_session_level_connection_to_node('UPDATE ref_table SET id=id+2 WHERE id=1');
+
+run_commands_on_session_level_connection_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
+step s2-coordinator-create-index-concurrently:
+ CREATE INDEX CONCURRENTLY dist_table_index ON dist_table(id);
+ <waiting ...>
+step s1-commit-worker: 
+ SELECT run_commands_on_session_level_connection_to_node('COMMIT');
+
+run_commands_on_session_level_connection_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
+step s2-coordinator-create-index-concurrently: <... completed>
+step s2-empty:
+step s1-stop-connection:
+ SELECT stop_session_level_connection_to_node();
+
+stop_session_level_connection_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
+step s3-display:
+ SELECT * FROM ref_table ORDER BY id, value;
+ SELECT * FROM dist_table ORDER BY id, value;
+
+id|value
+---------------------------------------------------------------------
+ 2|   20
+ 3|   10
+(2 rows)
+
+id|value
+---------------------------------------------------------------------
+ 1|    3
+ 2|    2
+(2 rows)
+

src/test/regress/expected/isolation_drop_alter_index_select_for_update_on_mx.out

@@ -220,7 +220,7 @@ stop_session_level_connection_to_node
 (1 row)
 
 
-starting permutation: s1-start-session-level-connection s1-begin-on-worker s1-select-for-update s2-flaky-coordinator-create-index-concurrently s1-commit-worker s1-stop-connection
+starting permutation: s1-start-session-level-connection s1-begin-on-worker s1-select-for-update s2-coordinator-create-index-concurrently s2-empty s1-commit-worker s1-stop-connection
 step s1-start-session-level-connection:
         SELECT start_session_level_connection_to_node('localhost', 57637);
 
@@ -245,9 +245,11 @@ run_commands_on_session_level_connection_to_node
 
 (1 row)
 
-step s2-flaky-coordinator-create-index-concurrently:
- CREATE INDEX CONCURRENTLY flaky_dist_table_index_conc ON dist_table(id);
-
+step s2-coordinator-create-index-concurrently:
+ CREATE INDEX CONCURRENTLY dist_table_index_conc ON dist_table(id);
+ <waiting ...>
+step s2-coordinator-create-index-concurrently: <... completed>
+step s2-empty:
 step s1-commit-worker:
  SELECT run_commands_on_session_level_connection_to_node('COMMIT');
 

src/test/regress/expected/isolation_drop_vs_all.out

@@ -108,7 +108,7 @@ run_command_on_workers
 (2 rows)
 
 
-starting permutation: s1-initialize s2-initialize s1-begin s1-drop s2-ddl-create-index-concurrently s1-commit s1-select-count s1-show-indexes
+starting permutation: s1-initialize s2-initialize s1-begin s1-drop s2-ddl-create-index-concurrently s1-commit s2-empty s1-select-count s1-show-indexes
 create_distributed_table
 ---------------------------------------------------------------------
 
@@ -122,6 +122,7 @@ step s2-ddl-create-index-concurrently: CREATE INDEX CONCURRENTLY drop_hash_index
 step s1-commit: COMMIT;
 step s2-ddl-create-index-concurrently: <... completed>
 ERROR:  relation "drop_hash" does not exist
+step s2-empty:
 step s1-select-count: SELECT COUNT(*) FROM drop_hash;
 ERROR:  relation "drop_hash" does not exist
 step s1-show-indexes: SELECT run_command_on_workers('SELECT COUNT(*) FROM pg_indexes WHERE tablename LIKE ''drop_hash%''');

src/test/regress/expected/isolation_hash_copy_vs_all.out

@@ -257,7 +257,7 @@ run_command_on_workers
 (2 rows)
 
 
-starting permutation: s1-initialize s1-begin s1-copy s2-flaky-ddl-create-index-concurrently s1-commit s1-select-count s1-show-indexes
+starting permutation: s1-initialize s1-begin s1-copy s2-ddl-create-index-concurrently s1-commit s2-empty s1-select-count s1-show-indexes
 create_distributed_table
 ---------------------------------------------------------------------
 
@@ -266,8 +266,10 @@ create_distributed_table
 step s1-initialize: COPY hash_copy FROM PROGRAM 'echo 0, a, 0 && echo 1, b, 1 && echo 2, c, 2 && echo 3, d, 3 && echo 4, e, 4' WITH CSV;
 step s1-begin: BEGIN;
 step s1-copy: COPY hash_copy FROM PROGRAM 'echo 5, f, 5 && echo 6, g, 6 && echo 7, h, 7 && echo 8, i, 8 && echo 9, j, 9' WITH CSV;
-step s2-flaky-ddl-create-index-concurrently: CREATE INDEX CONCURRENTLY flaky_hash_copy_index ON hash_copy(id);
+step s2-ddl-create-index-concurrently: CREATE INDEX CONCURRENTLY hash_copy_index ON hash_copy(id); <waiting ...>
 step s1-commit: COMMIT;
+step s2-ddl-create-index-concurrently: <... completed>
+step s2-empty:
 step s1-select-count: SELECT COUNT(*) FROM hash_copy;
 count
 ---------------------------------------------------------------------

src/test/regress/expected/isolation_insert_select_vs_all.out

@@ -180,7 +180,7 @@ run_command_on_workers
 (2 rows)
 
 
-starting permutation: s1-initialize s1-begin s1-insert-select s2-ddl-create-index-concurrently-on-inserted s1-commit s1-select-count s1-show-indexes-inserted
+starting permutation: s1-initialize s1-begin s1-insert-select s2-ddl-create-index-concurrently-on-inserted s1-commit s2-empty s1-select-count s1-show-indexes-inserted
 create_distributed_table
 ---------------------------------------------------------------------
 
@@ -197,6 +197,7 @@ step s1-insert-select: INSERT INTO insert_of_insert_select_hash SELECT * FROM se
 step s2-ddl-create-index-concurrently-on-inserted: CREATE INDEX CONCURRENTLY insert_of_insert_select_hash_index ON insert_of_insert_select_hash(id); <waiting ...>
 step s1-commit: COMMIT;
 step s2-ddl-create-index-concurrently-on-inserted: <... completed>
+step s2-empty:
 step s1-select-count: SELECT COUNT(*) FROM select_of_insert_select_hash;
 count
 ---------------------------------------------------------------------
@@ -575,7 +576,7 @@ run_command_on_workers
 (2 rows)
 
 
-starting permutation: s1-initialize s1-begin s1-insert-select s2-ddl-create-index-concurrently-on-selected s1-commit s1-select-count s1-show-indexes-selected
+starting permutation: s1-initialize s1-begin s1-insert-select s2-ddl-create-index-concurrently-on-selected s1-commit s2-empty s1-select-count s1-show-indexes-selected
 create_distributed_table
 ---------------------------------------------------------------------
 
@@ -592,6 +593,7 @@ step s1-insert-select: INSERT INTO insert_of_insert_select_hash SELECT * FROM se
 step s2-ddl-create-index-concurrently-on-selected: CREATE INDEX CONCURRENTLY select_of_insert_select_hash_index ON insert_of_insert_select_hash(id); <waiting ...>
 step s1-commit: COMMIT;
 step s2-ddl-create-index-concurrently-on-selected: <... completed>
+step s2-empty:
 step s1-select-count: SELECT COUNT(*) FROM select_of_insert_select_hash;
 count
 ---------------------------------------------------------------------

src/test/regress/expected/isolation_insert_select_vs_all_on_mx.out

@@ -1207,3 +1207,113 @@ count
    10
 (1 row)
 
+
+starting permutation: s1-start-session-level-connection s1-begin-on-worker s1-colocated-insert-select s2-coordinator-create-index-concurrently s1-commit-worker s2-empty s3-select-count s1-stop-connection
+step s1-start-session-level-connection:
+        SELECT start_session_level_connection_to_node('localhost', 57637);
+
+start_session_level_connection_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
+step s1-begin-on-worker:
+        SELECT run_commands_on_session_level_connection_to_node('BEGIN');
+
+run_commands_on_session_level_connection_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
+step s1-colocated-insert-select:
+ SELECT run_commands_on_session_level_connection_to_node('INSERT INTO dist_table SELECT * FROM dist_table');
+
+run_commands_on_session_level_connection_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
+step s2-coordinator-create-index-concurrently:
+ CREATE INDEX CONCURRENTLY dist_table_index ON dist_table(id);
+ <waiting ...>
+step s1-commit-worker: 
+ SELECT run_commands_on_session_level_connection_to_node('COMMIT');
+
+run_commands_on_session_level_connection_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
+step s2-coordinator-create-index-concurrently: <... completed>
+step s2-empty:
+step s3-select-count:
+ SELECT COUNT(*) FROM dist_table;
+
+count
+---------------------------------------------------------------------
+   10
+(1 row)
+
+step s1-stop-connection:
+ SELECT stop_session_level_connection_to_node();
+
+stop_session_level_connection_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
+
+starting permutation: s1-start-session-level-connection s1-begin-on-worker s1-insert-select-via-coordinator s2-coordinator-create-index-concurrently s1-commit-worker s2-empty s3-select-count s1-stop-connection
+step s1-start-session-level-connection:
+        SELECT start_session_level_connection_to_node('localhost', 57637);
+
+start_session_level_connection_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
+step s1-begin-on-worker:
+        SELECT run_commands_on_session_level_connection_to_node('BEGIN');
+
+run_commands_on_session_level_connection_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
+step s1-insert-select-via-coordinator:
+ SELECT run_commands_on_session_level_connection_to_node('INSERT INTO dist_table SELECT value, id FROM dist_table');
+
+run_commands_on_session_level_connection_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
+step s2-coordinator-create-index-concurrently:
+ CREATE INDEX CONCURRENTLY dist_table_index ON dist_table(id);
+ <waiting ...>
+step s1-commit-worker: 
+ SELECT run_commands_on_session_level_connection_to_node('COMMIT');
+
+run_commands_on_session_level_connection_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
+step s2-coordinator-create-index-concurrently: <... completed>
+step s2-empty:
+step s3-select-count:
+ SELECT COUNT(*) FROM dist_table;
+
+count
+---------------------------------------------------------------------
+   10
+(1 row)
+
+step s1-stop-connection:
+ SELECT stop_session_level_connection_to_node();
+
+stop_session_level_connection_to_node
+---------------------------------------------------------------------
+
+(1 row)
+

src/test/regress/expected/isolation_insert_vs_all.out

@@ -213,7 +213,7 @@ run_command_on_workers
 (2 rows)
 
 
-starting permutation: s1-initialize s1-begin s1-insert s2-ddl-create-index-concurrently s1-commit s1-select-count s1-show-indexes
+starting permutation: s1-initialize s1-begin s1-insert s2-ddl-create-index-concurrently s1-commit s2-empty s1-select-count s1-show-indexes
 create_distributed_table
 ---------------------------------------------------------------------
 
@@ -225,6 +225,7 @@ step s1-insert: INSERT INTO insert_hash VALUES(7, 'k');
 step s2-ddl-create-index-concurrently: CREATE INDEX CONCURRENTLY insert_hash_index ON insert_hash(id); <waiting ...>
 step s1-commit: COMMIT;
 step s2-ddl-create-index-concurrently: <... completed>
+step s2-empty:
 step s1-select-count: SELECT COUNT(*) FROM insert_hash;
 count
 ---------------------------------------------------------------------
@@ -814,7 +815,7 @@ run_command_on_workers
 (2 rows)
 
 
-starting permutation: s1-initialize s1-begin s1-insert-multi-row s2-ddl-create-index-concurrently s1-commit s1-select-count s1-show-indexes
+starting permutation: s1-initialize s1-begin s1-insert-multi-row s2-ddl-create-index-concurrently s1-commit s2-empty s1-select-count s1-show-indexes
 create_distributed_table
 ---------------------------------------------------------------------
 
@@ -826,6 +827,7 @@ step s1-insert-multi-row: INSERT INTO insert_hash VALUES(7, 'k'), (8, 'l'), (9,
 step s2-ddl-create-index-concurrently: CREATE INDEX CONCURRENTLY insert_hash_index ON insert_hash(id); <waiting ...>
 step s1-commit: COMMIT;
 step s2-ddl-create-index-concurrently: <... completed>
+step s2-empty:
 step s1-select-count: SELECT COUNT(*) FROM insert_hash;
 count
 ---------------------------------------------------------------------

src/test/regress/expected/isolation_insert_vs_all_on_mx.out

@@ -990,3 +990,58 @@ stop_session_level_connection_to_node
 
 (1 row)
 
+
+starting permutation: s1-start-session-level-connection s1-begin-on-worker s1-insert s2-coordinator-create-index-concurrently s1-commit-worker s2-empty s3-select-count s1-stop-connection
+step s1-start-session-level-connection:
+        SELECT start_session_level_connection_to_node('localhost', 57637);
+
+start_session_level_connection_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
+step s1-begin-on-worker:
+        SELECT run_commands_on_session_level_connection_to_node('BEGIN');
+
+run_commands_on_session_level_connection_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
+step s1-insert:
+ SELECT run_commands_on_session_level_connection_to_node('INSERT INTO insert_table VALUES(6, 60)');
+
+run_commands_on_session_level_connection_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
+step s2-coordinator-create-index-concurrently:
+ CREATE INDEX CONCURRENTLY insert_table_index ON insert_table(id);
+ <waiting ...>
+step s1-commit-worker: 
+ SELECT run_commands_on_session_level_connection_to_node('COMMIT');
+
+run_commands_on_session_level_connection_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
+step s2-coordinator-create-index-concurrently: <... completed>
+step s2-empty:
+step s3-select-count:
+ SELECT COUNT(*) FROM insert_table;
+
+count
+---------------------------------------------------------------------
+    6
+(1 row)
+
+step s1-stop-connection:
+ SELECT stop_session_level_connection_to_node();
+
+stop_session_level_connection_to_node
+---------------------------------------------------------------------
+
+(1 row)
+

src/test/regress/expected/isolation_range_copy_vs_all.out

@@ -197,13 +197,14 @@ run_command_on_workers
 (2 rows)
 
 
-starting permutation: s1-initialize s1-begin s1-copy s2-ddl-create-index-concurrently s1-commit s1-select-count s1-show-indexes
+starting permutation: s1-initialize s1-begin s1-copy s2-ddl-create-index-concurrently s1-commit s2-empty s1-select-count s1-show-indexes
 step s1-initialize: COPY range_copy FROM PROGRAM 'echo 0, a, 0 && echo 1, b, 1 && echo 2, c, 2 && echo 3, d, 3 && echo 4, e, 4' WITH CSV;
 step s1-begin: BEGIN;
 step s1-copy: COPY range_copy FROM PROGRAM 'echo 5, f, 5 && echo 6, g, 6 && echo 7, h, 7 && echo 8, i, 8 && echo 9, j, 9' WITH CSV;
 step s2-ddl-create-index-concurrently: CREATE INDEX CONCURRENTLY range_copy_index ON range_copy(id); <waiting ...>
 step s1-commit: COMMIT;
 step s2-ddl-create-index-concurrently: <... completed>
+step s2-empty:
 step s1-select-count: SELECT COUNT(*) FROM range_copy;
 count
 ---------------------------------------------------------------------

src/test/regress/expected/isolation_ref_select_for_update_vs_all_on_mx.out

@@ -626,7 +626,7 @@ stop_session_level_connection_to_node
 (1 row)
 
 
-starting permutation: s1-start-session-level-connection s1-begin-on-worker s1-select-for-update s2-coordinator-create-index-concurrently s1-commit-worker s1-stop-connection
+starting permutation: s1-start-session-level-connection s1-begin-on-worker s1-select-for-update s2-coordinator-create-index-concurrently s2-empty s1-commit-worker s1-stop-connection
 create_reference_table
 ---------------------------------------------------------------------
 
@@ -658,7 +658,9 @@ run_commands_on_session_level_connection_to_node
 
 step s2-coordinator-create-index-concurrently:
  CREATE INDEX CONCURRENTLY ref_table_index ON ref_table(id);
-
+ <waiting ...>
+step s2-coordinator-create-index-concurrently: <... completed>
+step s2-empty:
 step s1-commit-worker:
  SELECT run_commands_on_session_level_connection_to_node('COMMIT');
 

src/test/regress/expected/isolation_ref_update_delete_upsert_vs_all_on_mx.out

@@ -379,3 +379,63 @@ count
     0
 (1 row)
 
+
+starting permutation: s1-start-session-level-connection s1-begin-on-worker s1-delete s2-coordinator-create-index-concurrently s1-commit-worker s2-empty s3-select-count s1-stop-connection
+create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+step s1-start-session-level-connection:
+        SELECT start_session_level_connection_to_node('localhost', 57637);
+
+start_session_level_connection_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
+step s1-begin-on-worker:
+        SELECT run_commands_on_session_level_connection_to_node('BEGIN');
+
+run_commands_on_session_level_connection_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
+step s1-delete:
+ SELECT run_commands_on_session_level_connection_to_node('DELETE FROM ref_table WHERE id=1 OR id=2');
+
+run_commands_on_session_level_connection_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
+step s2-coordinator-create-index-concurrently:
+ CREATE INDEX CONCURRENTLY ref_table_index ON ref_table(id);
+ <waiting ...>
+step s1-commit-worker: 
+ SELECT run_commands_on_session_level_connection_to_node('COMMIT');
+
+run_commands_on_session_level_connection_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
+step s2-coordinator-create-index-concurrently: <... completed>
+step s2-empty:
+step s3-select-count:
+ SELECT COUNT(*) FROM ref_table;
+
+count
+---------------------------------------------------------------------
+    0
+(1 row)
+
+step s1-stop-connection:
+ SELECT stop_session_level_connection_to_node();
+
+stop_session_level_connection_to_node
+---------------------------------------------------------------------
+
+(1 row)
+

src/test/regress/expected/isolation_reference_copy_vs_all.out

@@ -259,7 +259,7 @@ run_command_on_workers
 (2 rows)
 
 
-starting permutation: s1-initialize s1-begin s1-copy s2-flaky-ddl-create-index-concurrently s1-commit s1-select-count s1-show-indexes
+starting permutation: s1-initialize s1-begin s1-copy s2-ddl-create-index-concurrently s1-commit s2-empty s1-select-count s1-show-indexes
 create_reference_table
 ---------------------------------------------------------------------
 
@@ -268,8 +268,10 @@ create_reference_table
 step s1-initialize: COPY reference_copy FROM PROGRAM 'echo 0, a, 0 && echo 1, b, 1 && echo 2, c, 2 && echo 3, d, 3 && echo 4, e, 4' WITH CSV;
 step s1-begin: BEGIN;
 step s1-copy: COPY reference_copy FROM PROGRAM 'echo 5, f, 5  && echo 6, g, 6 && echo 7, h, 7 && echo 8, i, 8 && echo 9, j, 9' WITH CSV;
-step s2-flaky-ddl-create-index-concurrently: CREATE INDEX CONCURRENTLY flaky_reference_copy_index ON reference_copy(id);
+step s2-ddl-create-index-concurrently: CREATE INDEX CONCURRENTLY reference_copy_index ON reference_copy(id); <waiting ...>
 step s1-commit: COMMIT;
+step s2-ddl-create-index-concurrently: <... completed>
+step s2-empty:
 step s1-select-count: SELECT COUNT(*) FROM reference_copy;
 count
 ---------------------------------------------------------------------

src/test/regress/expected/isolation_select_vs_all.out

@@ -516,7 +516,7 @@ run_command_on_workers
 (2 rows)
 
 
-starting permutation: s1-initialize s1-begin s1-disable-binary-protocol s1-router-select s2-ddl-create-index-concurrently s1-commit s1-select-count s1-show-indexes
+starting permutation: s1-initialize s1-begin s1-disable-binary-protocol s1-router-select s2-ddl-create-index-concurrently s2-empty s1-commit s1-select-count s1-show-indexes
 master_create_empty_shard
 ---------------------------------------------------------------------
                   6780300
@@ -534,7 +534,9 @@ id|data|int_data
  1| b  |       1
 (1 row)
 
-step s2-ddl-create-index-concurrently: CREATE INDEX CONCURRENTLY select_append_index ON select_append(id);
+step s2-ddl-create-index-concurrently: CREATE INDEX CONCURRENTLY select_append_index ON select_append(id); <waiting ...>
+step s2-ddl-create-index-concurrently: <... completed>
+step s2-empty:
 step s1-commit: COMMIT;
 step s1-select-count: SELECT COUNT(*) FROM select_append;
 count
@@ -1363,7 +1365,7 @@ run_command_on_workers
 (2 rows)
 
 
-starting permutation: s1-initialize s1-begin s1-real-time-select s2-ddl-create-index-concurrently s1-commit s1-select-count s1-show-indexes
+starting permutation: s1-initialize s1-begin s1-real-time-select s2-ddl-create-index-concurrently s2-empty s1-commit s1-select-count s1-show-indexes
 master_create_empty_shard
 ---------------------------------------------------------------------
                   6780300
@@ -1381,7 +1383,9 @@ id|data|int_data
  4| e  |       4
 (5 rows)
 
-step s2-ddl-create-index-concurrently: CREATE INDEX CONCURRENTLY select_append_index ON select_append(id);
+step s2-ddl-create-index-concurrently: CREATE INDEX CONCURRENTLY select_append_index ON select_append(id); <waiting ...>
+step s2-ddl-create-index-concurrently: <... completed>
+step s2-empty:
 step s1-commit: COMMIT;
 step s1-select-count: SELECT COUNT(*) FROM select_append;
 count
@@ -2254,7 +2258,7 @@ run_command_on_workers
 (2 rows)
 
 
-starting permutation: s1-initialize s1-begin s1-adaptive-select s2-ddl-create-index-concurrently s1-commit s1-select-count s1-show-indexes
+starting permutation: s1-initialize s1-begin s1-adaptive-select s2-ddl-create-index-concurrently s2-empty s1-commit s1-select-count s1-show-indexes
 master_create_empty_shard
 ---------------------------------------------------------------------
                   6780300
@@ -2275,7 +2279,9 @@ id|data|int_data|id|data|int_data
  4| e  |       4| 4| e  |       4
 (5 rows)
 
-step s2-ddl-create-index-concurrently: CREATE INDEX CONCURRENTLY select_append_index ON select_append(id);
+step s2-ddl-create-index-concurrently: CREATE INDEX CONCURRENTLY select_append_index ON select_append(id); <waiting ...>
+step s2-ddl-create-index-concurrently: <... completed>
+step s2-empty:
 step s1-commit: COMMIT;
 step s1-select-count: SELECT COUNT(*) FROM select_append;
 count

src/test/regress/expected/isolation_select_vs_all_on_mx.out

@@ -485,7 +485,7 @@ stop_session_level_connection_to_node
 (1 row)
 
 
-starting permutation: s1-start-session-level-connection s1-begin-on-worker s1-disable-binary-protocol-on-worker s1-select s2-flaky-coordinator-create-index-concurrently s1-commit-worker s1-stop-connection
+starting permutation: s1-start-session-level-connection s1-begin-on-worker s1-disable-binary-protocol-on-worker s1-select s2-coordinator-create-index-concurrently s2-empty s1-commit-worker s1-stop-connection
 step s1-start-session-level-connection:
         SELECT start_session_level_connection_to_node('localhost', 57637);
 
@@ -519,9 +519,11 @@ run_commands_on_session_level_connection_to_node
 
 (1 row)
 
-step s2-flaky-coordinator-create-index-concurrently:
- CREATE INDEX CONCURRENTLY flaky_select_table_index ON select_table(id);
-
+step s2-coordinator-create-index-concurrently:
+ CREATE INDEX CONCURRENTLY select_table_index ON select_table(id);
+ <waiting ...>
+step s2-coordinator-create-index-concurrently: <... completed>
+step s2-empty:
 step s1-commit-worker:
  SELECT run_commands_on_session_level_connection_to_node('COMMIT');
 

src/test/regress/expected/isolation_truncate_vs_all.out

@@ -116,7 +116,7 @@ run_command_on_workers
 (2 rows)
 
 
-starting permutation: s1-initialize s1-begin s1-truncate s2-ddl-create-index-concurrently s1-commit s1-select-count s1-show-indexes
+starting permutation: s1-initialize s1-begin s1-truncate s2-ddl-create-index-concurrently s1-commit s2-empty s1-select-count s1-show-indexes
 master_create_empty_shard
 ---------------------------------------------------------------------
                   5990340
@@ -128,6 +128,7 @@ step s1-truncate: TRUNCATE truncate_append;
 step s2-ddl-create-index-concurrently: CREATE INDEX CONCURRENTLY truncate_append_index ON truncate_append(id); <waiting ...>
 step s1-commit: COMMIT;
 step s2-ddl-create-index-concurrently: <... completed>
+step s2-empty:
 step s1-select-count: SELECT COUNT(*) FROM truncate_append;
 count
 ---------------------------------------------------------------------

src/test/regress/expected/isolation_update_delete_upsert_vs_all_on_mx.out

@@ -367,3 +367,63 @@ stop_session_level_connection_to_node
 
 (1 row)
 
+
+starting permutation: s1-start-session-level-connection s1-begin-on-worker s1-delete s2-coordinator-create-index-concurrently s1-commit-worker s2-empty s3-select-count s1-stop-connection
+create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+step s1-start-session-level-connection:
+        SELECT start_session_level_connection_to_node('localhost', 57637);
+
+start_session_level_connection_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
+step s1-begin-on-worker:
+        SELECT run_commands_on_session_level_connection_to_node('BEGIN');
+
+run_commands_on_session_level_connection_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
+step s1-delete:
+ SELECT run_commands_on_session_level_connection_to_node('DELETE FROM dist_table WHERE id=5');
+
+run_commands_on_session_level_connection_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
+step s2-coordinator-create-index-concurrently:
+ CREATE INDEX CONCURRENTLY dist_table_index ON dist_table(id);
+ <waiting ...>
+step s1-commit-worker: 
+ SELECT run_commands_on_session_level_connection_to_node('COMMIT');
+
+run_commands_on_session_level_connection_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
+step s2-coordinator-create-index-concurrently: <... completed>
+step s2-empty:
+step s3-select-count:
+ SELECT COUNT(*) FROM dist_table;
+
+count
+---------------------------------------------------------------------
+    0
+(1 row)
+
+step s1-stop-connection:
+ SELECT stop_session_level_connection_to_node();
+
+stop_session_level_connection_to_node
+---------------------------------------------------------------------
+
+(1 row)
+

src/test/regress/expected/isolation_update_vs_all.out

@@ -168,7 +168,7 @@ run_command_on_workers
 (2 rows)
 
 
-starting permutation: s1-initialize s1-begin s1-update s2-ddl-create-index-concurrently s1-commit s1-select-count s1-show-indexes
+starting permutation: s1-initialize s1-begin s1-update s2-ddl-create-index-concurrently s1-commit s2-empty s1-select-count s1-show-indexes
 create_distributed_table
 ---------------------------------------------------------------------
 
@@ -180,6 +180,7 @@ step s1-update: UPDATE update_hash SET data = 'l' WHERE id = 4;
 step s2-ddl-create-index-concurrently: CREATE INDEX CONCURRENTLY update_hash_index ON update_hash(id); <waiting ...>
 step s1-commit: COMMIT;
 step s2-ddl-create-index-concurrently: <... completed>
+step s2-empty:
 step s1-select-count: SELECT COUNT(*) FROM update_hash;
 count
 ---------------------------------------------------------------------
@@ -494,7 +495,7 @@ run_command_on_workers
 (2 rows)
 
 
-starting permutation: s1-initialize s1-begin s1-update-cte s2-ddl-create-index-concurrently s1-commit s1-select-count s1-show-indexes
+starting permutation: s1-initialize s1-begin s1-update-cte s2-ddl-create-index-concurrently s1-commit s2-empty s1-select-count s1-show-indexes
 create_distributed_table
 ---------------------------------------------------------------------
 
@@ -511,6 +512,7 @@ id|data
 step s2-ddl-create-index-concurrently: CREATE INDEX CONCURRENTLY update_hash_index ON update_hash(id); <waiting ...>
 step s1-commit: COMMIT;
 step s2-ddl-create-index-concurrently: <... completed>
+step s2-empty:
 step s1-select-count: SELECT COUNT(*) FROM update_hash;
 count
 ---------------------------------------------------------------------

src/test/regress/expected/isolation_upsert_vs_all.out

@@ -158,7 +158,7 @@ run_command_on_workers
 (2 rows)
 
 
-starting permutation: s1-initialize s1-begin s1-upsert s2-ddl-create-index-concurrently s1-commit s1-select-count s1-show-indexes
+starting permutation: s1-initialize s1-begin s1-upsert s2-ddl-create-index-concurrently s1-commit s2-empty s1-select-count s1-show-indexes
 create_distributed_table
 ---------------------------------------------------------------------
 
@@ -170,6 +170,7 @@ step s1-upsert: INSERT INTO upsert_hash VALUES(4, 'k') ON CONFLICT(id) DO UPDATE
 step s2-ddl-create-index-concurrently: CREATE INDEX CONCURRENTLY upsert_hash_index ON upsert_hash(id); <waiting ...>
 step s1-commit: COMMIT;
 step s2-ddl-create-index-concurrently: <... completed>
+step s2-empty:
 step s1-select-count: SELECT COUNT(*) FROM upsert_hash;
 count
 ---------------------------------------------------------------------

src/test/regress/spec/isolation_copy_vs_all_on_mx.spec

@@ -78,6 +78,11 @@ step "s2-select-for-update"
 	SELECT run_commands_on_session_level_connection_to_node('SELECT * FROM copy_table WHERE id=5 FOR UPDATE');
 }
 
+step "s2-coordinator-create-index-concurrently"
+{
+	CREATE INDEX CONCURRENTLY copy_table_index ON copy_table(id);
+}
+
 step "s2-commit-worker"
 {
         SELECT run_commands_on_session_level_connection_to_node('COMMIT');
@@ -92,6 +97,10 @@ step "s2-commit"
 {
 	COMMIT;
 }
+// We use this as a way to wait for s2-ddl-create-index-concurrently to
+// complete. We know it can complete after s1-commit has succeeded, this way
+// we make sure we get consistent output.
+step "s2-empty" {}
 
 
 session "s3"
@@ -106,5 +115,4 @@ step "s3-select-count"
 permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-copy" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-copy" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection" "s3-select-count"
 permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-copy" "s2-begin" "s2-coordinator-drop" "s1-commit-worker" "s2-commit" "s1-stop-connection" "s3-select-count"
 permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-copy" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-select-for-update" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection" "s3-select-count"
-//Not able to test the next permutation, until issue with CREATE INDEX CONCURRENTLY's locks is resolved. Issue #2966
-//permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-copy" "s2-coordinator-create-index-concurrently" "s1-commit-worker" "s3-select-count" "s1-stop-connection"
+permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-copy" "s2-coordinator-create-index-concurrently" "s1-commit-worker" "s2-empty" "s3-select-count" "s1-stop-connection"

src/test/regress/spec/isolation_ddl_vs_all.spec

@@ -43,18 +43,23 @@ step "s2-table-size" { SELECT citus_total_relation_size('ddl_hash'); }
 step "s2-master-modify-multiple-shards" { DELETE FROM ddl_hash; }
 step "s2-distribute-table" { SELECT create_distributed_table('ddl_hash', 'id'); }
 step "s2-commit" { COMMIT; }
+// This s2-empty step is used as a way to wait for the create index
+// concurrently command to complete before running s1-show-xxx. This is only
+// necessary for permutations where we don't run s2-commit, since that one
+// implicitly takes on that same function.
+step "s2-empty" {}
 
 // permutations - DDL vs DDL
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-ddl-create-index" "s2-ddl-create-index" "s1-commit" "s2-commit" "s1-show-indexes"
-permutation "s1-initialize" "s1-begin" "s1-ddl-create-index" "s2-ddl-create-index-concurrently" "s1-commit" "s1-show-indexes"
+permutation "s1-initialize" "s1-begin" "s1-ddl-create-index" "s2-ddl-create-index-concurrently" "s1-commit" "s2-empty" "s1-show-indexes"
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-ddl-create-index" "s2-ddl-add-column" "s1-commit" "s2-commit" "s1-show-indexes" "s1-show-columns"
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-ddl-create-index" "s2-ddl-rename-column" "s1-commit" "s2-commit" "s1-show-indexes" "s1-show-columns"
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-ddl-add-column" "s2-ddl-create-index" "s1-commit" "s2-commit" "s1-show-columns" "s1-show-indexes"
-permutation "s1-initialize" "s1-begin" "s1-ddl-add-column" "s2-ddl-create-index-concurrently" "s1-commit" "s1-show-columns" "s1-show-indexes"
+permutation "s1-initialize" "s1-begin" "s1-ddl-add-column" "s2-ddl-create-index-concurrently" "s1-commit" "s2-empty" "s1-show-columns" "s1-show-indexes"
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-ddl-add-column" "s2-ddl-add-column" "s1-commit" "s2-commit" "s1-show-columns"
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-ddl-add-column" "s2-ddl-rename-column" "s1-commit" "s2-commit" "s1-show-columns"
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-ddl-rename-column" "s2-ddl-create-index" "s1-commit" "s2-commit" "s1-show-columns" "s1-show-indexes"
-permutation "s1-initialize" "s1-begin" "s1-ddl-rename-column" "s2-ddl-create-index-concurrently" "s1-commit" "s1-show-columns" "s1-show-indexes"
+permutation "s1-initialize" "s1-begin" "s1-ddl-rename-column" "s2-ddl-create-index-concurrently" "s1-commit" "s2-empty" "s1-show-columns" "s1-show-indexes"
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-ddl-rename-column" "s2-ddl-add-column" "s1-commit" "s2-commit" "s1-show-columns"
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-ddl-rename-column" "s2-ddl-rename-column" "s1-commit" "s2-commit" "s1-show-columns"
 
@@ -76,9 +81,15 @@ permutation "s1-initialize" "s1-begin" "s2-begin" "s1-table-size" "s2-ddl-create
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-master-modify-multiple-shards" "s2-ddl-create-index" "s1-commit" "s2-commit" "s1-show-indexes"
 permutation "s1-drop" "s1-create-non-distributed-table" "s1-initialize" "s1-begin" "s2-begin" "s1-distribute-table" "s2-ddl-create-index" "s1-commit" "s2-commit" "s1-show-indexes"
 
-permutation "s1-initialize" "s1-begin" "s1-table-size" "s2-ddl-create-index-concurrently" "s1-commit" "s1-show-indexes"
-permutation "s1-initialize" "s1-begin" "s1-master-modify-multiple-shards" "s2-ddl-create-index-concurrently" "s1-commit" "s1-show-indexes"
-permutation "s1-drop" "s1-create-non-distributed-table" "s1-initialize" "s1-begin" "s1-distribute-table" "s2-ddl-create-index-concurrently" "s1-commit" "s1-show-indexes"
+// We use s2-empty slightly differently for this permutation than in the rest
+// of the permutations: We know create-index-concurrently doesn't have to wait
+// for s1-commit here, but the isolationtester sometimes detects it temporarily
+// as blocking. To get consistent test output we use a (*) marker to always
+// show create index concurrently as blocking. Then right after we put
+// s2-empty, to wait for it to complete.
+permutation "s1-initialize" "s1-begin" "s1-table-size" "s2-ddl-create-index-concurrently"(*) "s2-empty" "s1-commit" "s1-show-indexes"
+permutation "s1-initialize" "s1-begin" "s1-master-modify-multiple-shards" "s2-ddl-create-index-concurrently" "s1-commit" "s2-empty" "s1-show-indexes"
+permutation "s1-drop" "s1-create-non-distributed-table" "s1-initialize" "s1-begin" "s1-distribute-table" "s2-ddl-create-index-concurrently" "s1-commit" "s2-empty" "s1-show-indexes"
 
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-table-size" "s2-ddl-add-column" "s1-commit" "s2-commit" "s1-show-columns"
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-master-modify-multiple-shards" "s2-ddl-add-column" "s1-commit" "s2-commit" "s1-show-columns"

src/test/regress/spec/isolation_delete_vs_all.spec

@@ -51,6 +51,10 @@ step "s2-ddl-rename-column" { ALTER TABLE delete_hash RENAME data TO new_column;
 step "s2-table-size" { SELECT citus_total_relation_size('delete_hash'); }
 step "s2-distribute-table" { SELECT create_distributed_table('delete_hash', 'id'); }
 step "s2-commit" { COMMIT; }
+// We use this as a way to wait for s2-ddl-create-index-concurrently to
+// complete. We know it can complete after s1-commit has succeeded, this way we
+// make sure no other query is run over session s1 before that happens.
+step "s2-empty" {}
 
 // permutations - DELETE vs DELETE
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-delete" "s2-delete" "s1-commit" "s2-commit" "s1-select-count"
@@ -60,7 +64,7 @@ permutation "s1-initialize" "s1-begin" "s2-begin" "s1-delete" "s2-truncate" "s1-
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-delete" "s2-drop" "s1-commit" "s2-commit" "s1-select-count"
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-delete" "s2-ddl-create-index" "s1-commit" "s2-commit" "s1-select-count" "s1-show-indexes"
 permutation "s1-initialize" "s1-ddl-create-index" "s1-begin" "s2-begin" "s1-delete" "s2-ddl-drop-index" "s1-commit" "s2-commit" "s1-select-count" "s1-show-indexes"
-permutation "s1-initialize" "s1-begin" "s1-delete" "s2-ddl-create-index-concurrently" "s1-commit" "s1-select-count" "s1-show-indexes"
+permutation "s1-initialize" "s1-begin" "s1-delete" "s2-ddl-create-index-concurrently" "s1-commit" "s2-empty" "s1-select-count" "s1-show-indexes"
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-delete" "s2-ddl-add-column" "s1-commit" "s2-commit" "s1-select-count" "s1-show-columns"
 permutation "s1-initialize" "s1-ddl-add-column" "s1-begin" "s2-begin" "s1-delete" "s2-ddl-drop-column" "s1-commit" "s2-commit" "s1-select-count" "s1-show-columns"
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-delete" "s2-ddl-rename-column" "s1-commit" "s2-commit" "s1-select-count" "s1-show-columns"

src/test/regress/spec/isolation_dis2ref_foreign_keys_on_mx.spec

@@ -101,6 +101,11 @@ step "s2-select-for-udpate"
         SELECT run_commands_on_session_level_connection_to_node('SELECT * FROM dist_table WHERE id=1 FOR UPDATE');
 }
 
+step "s2-coordinator-create-index-concurrently"
+{
+	CREATE INDEX CONCURRENTLY dist_table_index ON dist_table(id);
+}
+
 step "s2-commit-worker"
 {
 	SELECT run_commands_on_session_level_connection_to_node('COMMIT');
@@ -111,6 +116,12 @@ step "s2-stop-connection"
 	SELECT stop_session_level_connection_to_node();
 }
 
+// We use this as a way to wait for s2-ddl-create-index-concurrently to
+// complete. We know it can complete after s1-commit has succeeded, this way
+// we make sure we get consistent output.
+step "s2-empty" {}
+
+
 session "s3"
 
 step "s3-display"
@@ -127,4 +138,4 @@ permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-update"
 permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-update" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-copy" "s1-rollback-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection" "s3-display"
 permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-update" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-truncate" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection" "s3-display"
 permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-delete" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-select-for-udpate" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection" "s3-display"
-//permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-update" "s2-coordinator-create-index-concurrently" "s1-commit-worker" "s1-stop-connection" "s3-display"
+permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-update" "s2-coordinator-create-index-concurrently" "s1-commit-worker" "s2-empty" "s1-stop-connection" "s3-display"

src/test/regress/spec/isolation_drop_alter_index_select_for_update_on_mx.spec

@@ -88,9 +88,9 @@ step "s2-select-for-update"
 	SELECT run_commands_on_session_level_connection_to_node('SELECT * FROM dist_table WHERE id = 5 FOR UPDATE');
 }
 
-step "s2-flaky-coordinator-create-index-concurrently"
+step "s2-coordinator-create-index-concurrently"
 {
-	CREATE INDEX CONCURRENTLY flaky_dist_table_index_conc ON dist_table(id);
+	CREATE INDEX CONCURRENTLY dist_table_index_conc ON dist_table(id);
 }
 
 step "s2-commit-worker"
@@ -103,6 +103,13 @@ step "s2-stop-connection"
         SELECT stop_session_level_connection_to_node();
 }
 
+// We use this as a way to wait for s2-coordinator-create-index-concurrently to
+// complete. We know create-index-concurrently doesn't have to wait for
+// s1-commit-worker, but the isolationtester sometimes detects it temporarily
+// as blocking. To get consistent test output we use a (*) marker to always
+// show create index concurrently as blocking. Then right after we put
+// s2-empty, to wait for it to complete.
+step "s2-empty" {}
 
 session "s3"
 
@@ -116,4 +123,4 @@ step "s3-select-count"
 permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-insert" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-alter" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection" "s3-select-count"
 permutation "s1-begin" "s1-index" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-select-for-update" "s1-commit" "s2-commit-worker" "s2-stop-connection"
 permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-select-for-update" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-select-for-update" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection"
-permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-select-for-update" "s2-flaky-coordinator-create-index-concurrently" "s1-commit-worker" "s1-stop-connection"
+permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-select-for-update" "s2-coordinator-create-index-concurrently"(*) "s2-empty" "s1-commit-worker" "s1-stop-connection"

src/test/regress/spec/isolation_drop_vs_all.spec

@@ -57,6 +57,10 @@ step "s2-ddl-rename-column" { ALTER TABLE drop_hash RENAME data TO new_column; }
 step "s2-table-size" { SELECT citus_total_relation_size('drop_hash'); }
 step "s2-distribute-table" { SELECT create_distributed_table('drop_hash', 'id'); }
 step "s2-commit" { COMMIT; }
+// We use this as a way to wait for s2-ddl-create-index-concurrently to
+// complete. We know it can complete after s1-commit has succeeded, this way we
+// make sure no other query is run over session s1 before that happens.
+step "s2-empty" {}
 
 // permutations - DROP vs DROP
 permutation "s1-initialize" "s2-initialize" "s1-begin" "s2-begin" "s1-drop" "s2-drop" "s1-commit" "s2-commit" "s1-select-count"
@@ -66,7 +70,7 @@ permutation "s1-initialize" "s2-initialize" "s1-begin" "s2-begin" "s1-drop-schem
 // permutations - DROP first
 permutation "s1-initialize" "s2-initialize" "s1-begin" "s2-begin" "s1-drop" "s2-ddl-create-index" "s1-commit" "s2-commit" "s1-select-count" "s1-show-indexes"
 permutation "s1-initialize" "s2-initialize" "s1-ddl-create-index" "s1-begin" "s2-begin" "s1-drop" "s2-ddl-drop-index" "s1-commit" "s2-commit" "s1-select-count" "s1-show-indexes"
-permutation "s1-initialize" "s2-initialize" "s1-begin" "s1-drop" "s2-ddl-create-index-concurrently" "s1-commit" "s1-select-count" "s1-show-indexes"
+permutation "s1-initialize" "s2-initialize" "s1-begin" "s1-drop" "s2-ddl-create-index-concurrently" "s1-commit" "s2-empty" "s1-select-count" "s1-show-indexes"
 permutation "s1-initialize" "s2-initialize" "s1-begin" "s2-begin" "s1-drop" "s2-ddl-add-column" "s1-commit" "s2-commit" "s1-select-count" "s1-show-columns"
 permutation "s1-initialize" "s2-initialize" "s1-ddl-add-column" "s1-begin" "s2-begin" "s1-drop" "s2-ddl-drop-column" "s1-commit" "s2-commit" "s1-select-count" "s1-show-columns"
 permutation "s1-initialize" "s2-initialize" "s1-begin" "s2-begin" "s1-drop" "s2-ddl-rename-column" "s1-commit" "s2-commit" "s1-select-count" "s1-show-columns"

src/test/regress/spec/isolation_hash_copy_vs_all.spec

@@ -75,7 +75,7 @@ step "s2-truncate" { TRUNCATE hash_copy; }
 step "s2-drop" { DROP TABLE hash_copy; }
 step "s2-ddl-create-index" { CREATE INDEX hash_copy_index ON hash_copy(id); }
 step "s2-ddl-drop-index" { DROP INDEX hash_copy_index; }
-step "s2-flaky-ddl-create-index-concurrently" { CREATE INDEX CONCURRENTLY flaky_hash_copy_index ON hash_copy(id); }
+step "s2-ddl-create-index-concurrently" { CREATE INDEX CONCURRENTLY hash_copy_index ON hash_copy(id); }
 step "s2-ddl-add-column" { ALTER TABLE hash_copy ADD new_column int DEFAULT 0; }
 step "s2-ddl-drop-column" { ALTER TABLE hash_copy DROP new_column; }
 step "s2-ddl-rename-column" { ALTER TABLE hash_copy RENAME data TO new_column; }
@@ -83,6 +83,10 @@ step "s2-table-size" { SELECT citus_total_relation_size('hash_copy'); }
 step "s2-master-modify-multiple-shards" { DELETE FROM hash_copy; }
 step "s2-master-drop-all-shards" { SELECT citus_drop_all_shards('hash_copy'::regclass, 'public', 'hash_copy'); }
 step "s2-distribute-table" { SELECT create_distributed_table('hash_copy', 'id'); }
+// We use this as a way to wait for s2-ddl-create-index-concurrently to
+// complete. We know it can complete after s1-commit has succeeded, this way we
+// make sure no other query is run over session s1 before that happens.
+step "s2-empty" {}
 
 // permutations - COPY vs COPY
 permutation "s1-initialize" "s1-begin" "s1-copy" "s2-copy" "s1-commit" "s1-select-count"
@@ -99,7 +103,7 @@ permutation "s1-initialize" "s1-begin" "s1-copy" "s2-truncate" "s1-commit" "s1-s
 permutation "s1-initialize" "s1-begin" "s1-copy" "s2-drop" "s1-commit" "s1-select-count"
 permutation "s1-initialize" "s1-begin" "s1-copy" "s2-ddl-create-index" "s1-commit" "s1-select-count" "s1-show-indexes"
 permutation "s1-initialize" "s1-ddl-create-index" "s1-begin" "s1-copy" "s2-ddl-drop-index" "s1-commit" "s1-select-count" "s1-show-indexes"
-permutation "s1-initialize" "s1-begin" "s1-copy" "s2-flaky-ddl-create-index-concurrently" "s1-commit" "s1-select-count" "s1-show-indexes"
+permutation "s1-initialize" "s1-begin" "s1-copy" "s2-ddl-create-index-concurrently" "s1-commit" "s2-empty" "s1-select-count" "s1-show-indexes"
 permutation "s1-initialize" "s1-begin" "s1-copy" "s2-ddl-add-column" "s1-commit" "s1-select-count" "s1-show-columns"
 permutation "s1-initialize" "s1-ddl-add-column" "s1-begin" "s1-copy-additional-column" "s2-ddl-drop-column" "s1-commit" "s1-select-count" "s1-show-columns"
 permutation "s1-initialize" "s1-begin" "s1-copy" "s2-ddl-rename-column" "s1-commit" "s1-select-count" "s1-show-columns"

src/test/regress/spec/isolation_insert_select_vs_all.spec

@@ -96,6 +96,10 @@ step "s2-table-size-on-selected" { SELECT citus_total_relation_size('select_of_i
 step "s2-master-modify-multiple-shards-on-selected" { DELETE FROM select_of_insert_select_hash; }
 step "s2-master-drop-all-shards-on-selected" { SELECT citus_drop_all_shards('select_of_insert_select_hash'::regclass, 'public', 'select_of_insert_select_hash'); }
 step "s2-distribute-table-on-selected" { SELECT create_distributed_table('select_of_insert_select_hash', 'id'); }
+// We use this as a way to wait for s2-ddl-create-index-concurrently to
+// complete. We know it can complete after s1-commit has succeeded, this way we
+// make sure no other query is run over session s1 before that happens.
+step "s2-empty" {}
 
 // permutations - INSERT/SELECT vs INSERT/SELECT
 permutation "s1-initialize" "s1-begin" "s1-insert-select" "s2-insert-select" "s1-commit" "s1-select-count"
@@ -107,7 +111,7 @@ permutation "s1-initialize" "s1-begin" "s1-insert-select" "s2-truncate-on-insert
 permutation "s1-initialize" "s1-begin" "s1-insert-select" "s2-drop-on-inserted" "s1-commit" "s1-select-count"
 permutation "s1-initialize" "s1-begin" "s1-insert-select" "s2-ddl-create-index-on-inserted" "s1-commit" "s1-select-count" "s1-show-indexes-inserted"
 permutation "s1-initialize" "s1-ddl-create-index-on-inserted" "s1-begin" "s1-insert-select" "s2-ddl-drop-index-on-inserted" "s1-commit" "s1-select-count" "s1-show-indexes-inserted"
-permutation "s1-initialize" "s1-begin" "s1-insert-select" "s2-ddl-create-index-concurrently-on-inserted" "s1-commit" "s1-select-count" "s1-show-indexes-inserted"
+permutation "s1-initialize" "s1-begin" "s1-insert-select" "s2-ddl-create-index-concurrently-on-inserted" "s1-commit" "s2-empty" "s1-select-count" "s1-show-indexes-inserted"
 permutation "s1-initialize" "s1-begin" "s1-insert-select" "s2-ddl-add-column-on-inserted" "s1-commit" "s1-select-count" "s1-show-columns-inserted"
 permutation "s1-initialize" "s1-ddl-add-column-on-inserted" "s1-begin" "s1-insert-select" "s2-ddl-drop-column-on-inserted" "s1-commit" "s1-select-count" "s1-show-columns-inserted"
 permutation "s1-initialize" "s1-begin" "s1-insert-select" "s2-ddl-rename-column-on-inserted" "s1-commit" "s1-select-count" "s1-show-columns-inserted" "s1-show-columns-inserted"
@@ -123,7 +127,7 @@ permutation "s1-initialize" "s1-begin" "s1-insert-select" "s2-truncate-on-select
 permutation "s1-initialize" "s1-begin" "s1-insert-select" "s2-drop-on-selected" "s1-commit" "s1-select-count"
 permutation "s1-initialize" "s1-begin" "s1-insert-select" "s2-ddl-create-index-on-selected" "s1-commit" "s1-select-count" "s1-show-indexes-selected"
 permutation "s1-initialize" "s1-ddl-create-index-on-selected" "s1-begin" "s1-insert-select" "s2-ddl-drop-index-on-selected" "s1-commit" "s1-select-count" "s1-show-indexes-selected"
-permutation "s1-initialize" "s1-begin" "s1-insert-select" "s2-ddl-create-index-concurrently-on-selected" "s1-commit" "s1-select-count" "s1-show-indexes-selected"
+permutation "s1-initialize" "s1-begin" "s1-insert-select" "s2-ddl-create-index-concurrently-on-selected" "s1-commit" "s2-empty" "s1-select-count" "s1-show-indexes-selected"
 permutation "s1-initialize" "s1-begin" "s1-insert-select" "s2-ddl-add-column-on-selected" "s1-commit" "s1-select-count" "s1-show-columns-selected"
 permutation "s1-initialize" "s1-ddl-add-column-on-selected" "s1-begin" "s1-insert-select" "s2-ddl-drop-column-on-selected" "s1-commit" "s1-select-count" "s1-show-columns-selected"
 permutation "s1-initialize" "s1-begin" "s1-insert-select" "s2-ddl-rename-column-on-selected" "s1-commit" "s1-select-count" "s1-show-columns-selected"

src/test/regress/spec/isolation_insert_select_vs_all_on_mx.spec

@@ -108,6 +108,11 @@ step "s2-select-for-update"
 	SELECT run_commands_on_session_level_connection_to_node('SELECT * FROM dist_table WHERE id = 5 FOR UPDATE');
 }
 
+step "s2-coordinator-create-index-concurrently"
+{
+	CREATE INDEX CONCURRENTLY dist_table_index ON dist_table(id);
+}
+
 step "s2-commit-worker"
 {
         SELECT run_commands_on_session_level_connection_to_node('COMMIT');
@@ -123,6 +128,11 @@ step "s2-commit"
 	COMMIT;
 }
 
+// We use this as a way to wait for s2-ddl-create-index-concurrently to
+// complete. We know it can complete after s1-commit has succeeded, this way
+// we make sure we get consistent output.
+step "s2-empty" {}
+
 
 session "s3"
 
@@ -149,6 +159,5 @@ permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-colocat
 permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-insert-select-via-coordinator" "s2-begin" "s2-coordinator-drop" "s1-commit-worker" "s2-commit" "s1-stop-connection" "s2-stop-connection" "s3-select-count"
 permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-colocated-insert-select" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-select-for-update" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection" "s3-select-count"
 permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-insert-select-via-coordinator" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-select-for-update" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection" "s3-select-count"
-//Not able to test the next permutations, until issue with CREATE INDEX CONCURRENTLY's locks is resolved. Issue #2966
-//permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-colocated-insert-select" "s2-coordinator-create-index-concurrently" "s1-commit-worker" "s3-select-count" "s1-stop-connection"
-//permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-insert-select-via-coordinator" "s2-coordinator-create-index-concurrently" "s1-commit-worker" "s3-select-count" "s1-stop-connection"
+permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-colocated-insert-select" "s2-coordinator-create-index-concurrently" "s1-commit-worker" "s2-empty" "s3-select-count" "s1-stop-connection"
+permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-insert-select-via-coordinator" "s2-coordinator-create-index-concurrently" "s1-commit-worker" "s2-empty" "s3-select-count" "s1-stop-connection"

src/test/regress/spec/isolation_insert_vs_all.spec

@@ -59,6 +59,10 @@ step "s2-ddl-rename-column" { ALTER TABLE insert_hash RENAME data TO new_column;
 step "s2-table-size" { SELECT citus_total_relation_size('insert_hash'); }
 step "s2-master-modify-multiple-shards" { DELETE FROM insert_hash; }
 step "s2-distribute-table" { SELECT create_distributed_table('insert_hash', 'id'); }
+// We use this as a way to wait for s2-ddl-create-index-concurrently to
+// complete. We know it can complete after s1-commit has succeeded, this way we
+// make sure no other query is run over session s1 before that happens.
+step "s2-empty" {}
 
 // permutations - INSERT vs INSERT
 permutation "s1-initialize" "s1-begin" "s1-insert" "s2-insert" "s1-commit" "s1-select-count"
@@ -74,7 +78,7 @@ permutation "s1-initialize" "s1-begin" "s1-insert" "s2-truncate" "s1-commit" "s1
 permutation "s1-initialize" "s1-begin" "s1-insert" "s2-drop" "s1-commit" "s1-select-count"
 permutation "s1-initialize" "s1-begin" "s1-insert" "s2-ddl-create-index" "s1-commit" "s1-select-count" "s1-show-indexes"
 permutation "s1-initialize" "s1-ddl-create-index" "s1-begin" "s1-insert" "s2-ddl-drop-index" "s1-commit" "s1-select-count" "s1-show-indexes"
-permutation "s1-initialize" "s1-begin" "s1-insert" "s2-ddl-create-index-concurrently" "s1-commit" "s1-select-count" "s1-show-indexes"
+permutation "s1-initialize" "s1-begin" "s1-insert" "s2-ddl-create-index-concurrently" "s1-commit" "s2-empty" "s1-select-count" "s1-show-indexes"
 permutation "s1-initialize" "s1-begin" "s1-insert" "s2-ddl-add-column" "s1-commit" "s1-select-count" "s1-show-columns"
 permutation "s1-initialize" "s1-ddl-add-column" "s1-begin" "s1-insert" "s2-ddl-drop-column" "s1-commit" "s1-select-count" "s1-show-columns"
 permutation "s1-initialize" "s1-begin" "s1-insert" "s2-ddl-rename-column" "s1-commit" "s1-select-count" "s1-show-columns"
@@ -105,7 +109,7 @@ permutation "s1-initialize" "s1-begin" "s1-insert-multi-row" "s2-truncate" "s1-c
 permutation "s1-initialize" "s1-begin" "s1-insert-multi-row" "s2-drop" "s1-commit" "s1-select-count"
 permutation "s1-initialize" "s1-begin" "s1-insert-multi-row" "s2-ddl-create-index" "s1-commit" "s1-select-count" "s1-show-indexes"
 permutation "s1-initialize" "s1-ddl-create-index" "s1-begin" "s1-insert-multi-row" "s2-ddl-drop-index" "s1-commit" "s1-select-count" "s1-show-indexes"
-permutation "s1-initialize" "s1-begin" "s1-insert-multi-row" "s2-ddl-create-index-concurrently" "s1-commit" "s1-select-count" "s1-show-indexes"
+permutation "s1-initialize" "s1-begin" "s1-insert-multi-row" "s2-ddl-create-index-concurrently" "s1-commit" "s2-empty" "s1-select-count" "s1-show-indexes"
 permutation "s1-initialize" "s1-begin" "s1-insert-multi-row" "s2-ddl-add-column" "s1-commit" "s1-select-count" "s1-show-columns"
 permutation "s1-initialize" "s1-ddl-add-column" "s1-begin" "s1-insert-multi-row" "s2-ddl-drop-column" "s1-commit" "s1-select-count" "s1-show-columns"
 permutation "s1-initialize" "s1-begin" "s1-insert-multi-row" "s2-ddl-rename-column" "s1-commit" "s1-select-count" "s1-show-columns"

src/test/regress/spec/isolation_insert_vs_all_on_mx.spec

@@ -109,6 +109,11 @@ step "s2-select-for-update"
 	SELECT run_commands_on_session_level_connection_to_node('SELECT * FROM insert_table WHERE id = 6 FOR UPDATE');
 }
 
+step "s2-coordinator-create-index-concurrently"
+{
+	CREATE INDEX CONCURRENTLY insert_table_index ON insert_table(id);
+}
+
 step "s2-commit-worker"
 {
         SELECT run_commands_on_session_level_connection_to_node('COMMIT');
@@ -119,6 +124,11 @@ step "s2-stop-connection"
         SELECT stop_session_level_connection_to_node();
 }
 
+// We use this as a way to wait for s2-coordinator-create-index-concurrently to
+// complete. We know it can complete after s1-commit has succeeded, this way
+// we make sure we get consistent output.
+step "s2-empty" {}
+
 
 session "s3"
 
@@ -141,5 +151,4 @@ permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-insert-
 permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-insert" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-copy" "s1-commit-worker" "s2-commit-worker" "s3-select-count" "s1-stop-connection" "s2-stop-connection"
 permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-insert" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-truncate" "s1-commit-worker" "s2-commit-worker" "s3-select-count" "s1-stop-connection" "s2-stop-connection"
 permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-insert" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-select-for-update" "s1-commit-worker" "s2-commit-worker" "s3-select-count" "s1-stop-connection" "s2-stop-connection"
-//Not able to test the next permutation, until issue with CREATE INDEX CONCURRENTLY's locks is resolved. Issue #2966
-//permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-insert" "s2-coordinator-create-index-concurrently" "s1-commit-worker" "s3-select-count" "s1-stop-connection"
+permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-insert" "s2-coordinator-create-index-concurrently" "s1-commit-worker" "s2-empty" "s3-select-count" "s1-stop-connection"

src/test/regress/spec/isolation_range_copy_vs_all.spec

@@ -88,6 +88,10 @@ step "s2-distribute-table" {
   UPDATE pg_dist_shard SET shardminvalue = '0', shardmaxvalue = '4' WHERE shardid = 3004005;
   UPDATE pg_dist_shard SET shardminvalue = '5', shardmaxvalue = '9' WHERE shardid = 3004006;
  }
+// We use this as a way to wait for s2-ddl-create-index-concurrently to
+// complete. We know it can complete after s1-commit has succeeded, this way we
+// make sure no other query is run over session s1 before that happens.
+step "s2-empty" {}
 
 // permutations - COPY vs COPY
 permutation "s1-initialize" "s1-begin" "s1-copy" "s2-copy" "s1-commit" "s1-select-count"
@@ -104,7 +108,7 @@ permutation "s1-initialize" "s1-begin" "s1-copy" "s2-truncate" "s1-commit" "s1-s
 permutation "s1-initialize" "s1-begin" "s1-copy" "s2-drop" "s1-commit" "s1-select-count"
 permutation "s1-initialize" "s1-begin" "s1-copy" "s2-ddl-create-index" "s1-commit" "s1-select-count" "s1-show-indexes"
 permutation "s1-initialize" "s1-ddl-create-index" "s1-begin" "s1-copy" "s2-ddl-drop-index" "s1-commit" "s1-select-count" "s1-show-indexes"
-permutation "s1-initialize" "s1-begin" "s1-copy" "s2-ddl-create-index-concurrently" "s1-commit" "s1-select-count" "s1-show-indexes"
+permutation "s1-initialize" "s1-begin" "s1-copy" "s2-ddl-create-index-concurrently" "s1-commit" "s2-empty" "s1-select-count" "s1-show-indexes"
 permutation "s1-initialize" "s1-begin" "s1-copy" "s2-ddl-add-column" "s1-commit" "s1-select-count" "s1-show-columns"
 permutation "s1-initialize" "s1-ddl-add-column" "s1-begin" "s1-copy-additional-column" "s2-ddl-drop-column" "s1-commit" "s1-select-count" "s1-show-columns"
 permutation "s1-initialize" "s1-begin" "s1-copy" "s2-ddl-rename-column" "s1-commit" "s1-select-count" "s1-show-columns"

src/test/regress/spec/isolation_ref_select_for_update_vs_all_on_mx.spec

@@ -107,6 +107,13 @@ step "s2-stop-connection"
         SELECT stop_session_level_connection_to_node();
 }
 
+// We use this as a way to wait for s2-coordinator-create-index-concurrently to
+// complete. We know create-index-concurrently doesn't have to wait for
+// s1-commit-worker, but the isolationtester sometimes detects it temporarily
+// as blocking. To get consistent test output we use a (*) marker to always
+// show create index concurrently as blocking. Then right after we put
+// s2-empty, to wait for it to complete.
+step "s2-empty" {}
 
 session "s3"
 
@@ -125,4 +132,4 @@ permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-select-
 permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-select-for-update" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-copy" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection" "s3-select-count"
 permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-select-for-update" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-alter" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection"
 permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-select-for-update" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-truncate" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection"
-permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-select-for-update" "s2-coordinator-create-index-concurrently" "s1-commit-worker" "s1-stop-connection"
+permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-select-for-update" "s2-coordinator-create-index-concurrently"(*) "s2-empty" "s1-commit-worker" "s1-stop-connection"

src/test/regress/spec/isolation_ref_update_delete_upsert_vs_all_on_mx.spec

@@ -87,6 +87,11 @@ step "s2-truncate"
 	SELECT run_commands_on_session_level_connection_to_node('TRUNCATE ref_table');
 }
 
+step "s2-coordinator-create-index-concurrently"
+{
+	CREATE INDEX CONCURRENTLY ref_table_index ON ref_table(id);
+}
+
 step "s2-commit-worker"
 {
         SELECT run_commands_on_session_level_connection_to_node('COMMIT');
@@ -96,6 +101,10 @@ step "s2-stop-connection"
 {
         SELECT stop_session_level_connection_to_node();
 }
+// We use this as a way to wait for s2-ddl-create-index-concurrently to
+// complete. We know it can complete after s1-commit has succeeded, this way
+// we make sure we get consistent output.
+step "s2-empty" {}
 
 
 session "s3"
@@ -111,5 +120,4 @@ permutation "s1-add-primary-key" "s1-start-session-level-connection" "s1-begin-o
 permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-delete" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-insert-select-ref-table" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection" "s3-select-count"
 permutation "s1-add-primary-key" "s1-start-session-level-connection" "s1-begin-on-worker" "s1-upsert" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-drop" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection" "s3-select-count"
 permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-delete" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-truncate" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection" "s3-select-count"
-//Not able to test the next permutation, until issue with CREATE INDEX CONCURRENTLY's locks is resolved. Issue #2966
-//permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-update" "s2-coordinator-create-index-concurrently" "s1-commit-worker" "s3-select-count" "s1-stop-connection"
+permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-delete" "s2-coordinator-create-index-concurrently" "s1-commit-worker" "s2-empty" "s3-select-count" "s1-stop-connection"

src/test/regress/spec/isolation_reference_copy_vs_all.spec

@@ -65,13 +65,17 @@ step "s2-truncate" { TRUNCATE reference_copy; }
 step "s2-drop" { DROP TABLE reference_copy; }
 step "s2-ddl-create-index" { CREATE INDEX reference_copy_index ON reference_copy(id); }
 step "s2-ddl-drop-index" { DROP INDEX reference_copy_index; }
-step "s2-flaky-ddl-create-index-concurrently" { CREATE INDEX CONCURRENTLY flaky_reference_copy_index ON reference_copy(id); }
+step "s2-ddl-create-index-concurrently" { CREATE INDEX CONCURRENTLY reference_copy_index ON reference_copy(id); }
 step "s2-ddl-add-column" { ALTER TABLE reference_copy ADD new_column int DEFAULT 0; }
 step "s2-ddl-drop-column" { ALTER TABLE reference_copy DROP new_column; }
 step "s2-ddl-rename-column" { ALTER TABLE reference_copy RENAME data TO new_column; }
 step "s2-table-size" { SELECT citus_total_relation_size('reference_copy'); }
 step "s2-master-modify-multiple-shards" { DELETE FROM reference_copy; }
 step "s2-distribute-table" { SELECT create_reference_table('reference_copy'); }
+// We use this as a way to wait for s2-ddl-create-index-concurrently to
+// complete. We know it can complete after s1-commit has succeeded, this way we
+// make sure no other query is run over session s1 before that happens.
+step "s2-empty" {}
 
 // permutations - COPY vs COPY
 permutation "s1-initialize" "s1-begin" "s1-copy" "s2-copy" "s1-commit" "s1-select-count"
@@ -88,7 +92,7 @@ permutation "s1-initialize" "s1-begin" "s1-copy" "s2-truncate" "s1-commit" "s1-s
 permutation "s1-initialize" "s1-begin" "s1-copy" "s2-drop" "s1-commit" "s1-select-count"
 permutation "s1-initialize" "s1-begin" "s1-copy" "s2-ddl-create-index" "s1-commit" "s1-select-count" "s1-show-indexes"
 permutation "s1-initialize" "s1-ddl-create-index" "s1-begin" "s1-copy" "s2-ddl-drop-index" "s1-commit" "s1-select-count" "s1-show-indexes"
-permutation "s1-initialize" "s1-begin" "s1-copy" "s2-flaky-ddl-create-index-concurrently" "s1-commit" "s1-select-count" "s1-show-indexes"
+permutation "s1-initialize" "s1-begin" "s1-copy" "s2-ddl-create-index-concurrently" "s1-commit" "s2-empty" "s1-select-count" "s1-show-indexes"
 permutation "s1-initialize" "s1-begin" "s1-copy" "s2-ddl-add-column" "s1-commit" "s1-select-count" "s1-show-columns"
 permutation "s1-initialize" "s1-ddl-add-column" "s1-begin" "s1-copy-additional-column" "s2-ddl-drop-column" "s1-commit" "s1-select-count" "s1-show-columns"
 permutation "s1-initialize" "s1-begin" "s1-copy" "s2-ddl-rename-column" "s1-commit" "s1-select-count" "s1-show-columns"

src/test/regress/spec/isolation_select_vs_all.spec

@@ -80,6 +80,13 @@ step "s2-table-size" { SELECT citus_total_relation_size('select_append'); }
 step "s2-master-modify-multiple-shards" { DELETE FROM select_append; }
 step "s2-master-drop-all-shards" { SELECT citus_drop_all_shards('select_append'::regclass, 'public', 'append_copy'); }
 step "s2-distribute-table" { SELECT create_distributed_table('select_append', 'id', 'append'); }
+// We use this as a way to wait for s2-ddl-create-index-concurrently to
+// complete. We know create-index-concurrently doesn't have to wait for
+// s1-commit, but the isolationtester sometimes detects it temporarily as
+// blocking. To get consistent test output we use a (*) marker to always show
+// create index concurrently as blocking. Then right after we put s2-empty, to
+// wait for it to complete.
+step "s2-empty" {}
 
 // permutations - SELECT vs SELECT
 permutation "s1-initialize" "s1-begin" "s1-router-select" "s2-router-select" "s1-commit" "s1-select-count"
@@ -101,7 +108,7 @@ permutation "s1-initialize" "s1-begin" "s1-router-select" "s2-truncate" "s1-comm
 permutation "s1-initialize" "s1-begin" "s1-router-select" "s2-drop" "s1-commit" "s1-select-count"
 permutation "s1-initialize" "s1-begin" "s1-router-select" "s2-ddl-create-index" "s1-commit" "s1-select-count" "s1-show-indexes"
 permutation "s1-initialize" "s1-ddl-create-index" "s1-begin" "s1-router-select" "s2-ddl-drop-index" "s1-commit" "s1-select-count" "s1-show-indexes"
-permutation "s1-initialize" "s1-begin" "s1-disable-binary-protocol" "s1-router-select" "s2-ddl-create-index-concurrently" "s1-commit" "s1-select-count" "s1-show-indexes"
+permutation "s1-initialize" "s1-begin" "s1-disable-binary-protocol" "s1-router-select" "s2-ddl-create-index-concurrently"(*) "s2-empty" "s1-commit" "s1-select-count" "s1-show-indexes"
 permutation "s1-initialize" "s1-begin" "s1-router-select" "s2-ddl-add-column" "s1-commit" "s1-select-count" "s1-show-columns"
 permutation "s1-initialize" "s1-ddl-add-column" "s1-begin" "s1-router-select" "s2-ddl-drop-column" "s1-commit" "s1-select-count" "s1-show-columns"
 permutation "s1-initialize" "s1-begin" "s1-router-select" "s2-ddl-rename-column" "s1-commit" "s1-select-count" "s1-show-columns"
@@ -136,7 +143,7 @@ permutation "s1-initialize" "s1-begin" "s1-real-time-select" "s2-truncate" "s1-c
 permutation "s1-initialize" "s1-begin" "s1-real-time-select" "s2-drop" "s1-commit" "s1-select-count"
 permutation "s1-initialize" "s1-begin" "s1-real-time-select" "s2-ddl-create-index" "s1-commit" "s1-select-count" "s1-show-indexes"
 permutation "s1-initialize" "s1-ddl-create-index" "s1-begin" "s1-real-time-select" "s2-ddl-drop-index" "s1-commit" "s1-select-count" "s1-show-indexes"
-permutation "s1-initialize" "s1-begin" "s1-real-time-select" "s2-ddl-create-index-concurrently" "s1-commit" "s1-select-count" "s1-show-indexes"
+permutation "s1-initialize" "s1-begin" "s1-real-time-select" "s2-ddl-create-index-concurrently"(*) "s2-empty"  "s1-commit" "s1-select-count" "s1-show-indexes"
 permutation "s1-initialize" "s1-begin" "s1-real-time-select" "s2-ddl-add-column" "s1-commit" "s1-select-count" "s1-show-columns"
 permutation "s1-initialize" "s1-ddl-add-column" "s1-begin" "s1-real-time-select" "s2-ddl-drop-column" "s1-commit" "s1-select-count" "s1-show-columns"
 permutation "s1-initialize" "s1-begin" "s1-real-time-select" "s2-ddl-rename-column" "s1-commit" "s1-select-count" "s1-show-columns"
@@ -169,7 +176,7 @@ permutation "s1-initialize" "s1-begin" "s1-adaptive-select" "s2-truncate" "s1-co
 permutation "s1-initialize" "s1-begin" "s1-adaptive-select" "s2-drop" "s1-commit" "s1-select-count"
 permutation "s1-initialize" "s1-begin" "s1-adaptive-select" "s2-ddl-create-index" "s1-commit" "s1-select-count" "s1-show-indexes"
 permutation "s1-initialize" "s1-ddl-create-index" "s1-begin" "s1-adaptive-select" "s2-ddl-drop-index" "s1-commit" "s1-select-count" "s1-show-indexes"
-permutation "s1-initialize" "s1-begin" "s1-adaptive-select" "s2-ddl-create-index-concurrently" "s1-commit" "s1-select-count" "s1-show-indexes"
+permutation "s1-initialize" "s1-begin" "s1-adaptive-select" "s2-ddl-create-index-concurrently"(*) "s2-empty"  "s1-commit" "s1-select-count" "s1-show-indexes"
 permutation "s1-initialize" "s1-begin" "s1-adaptive-select" "s2-ddl-add-column" "s1-commit" "s1-select-count" "s1-show-columns"
 permutation "s1-initialize" "s1-ddl-add-column" "s1-begin" "s1-adaptive-select" "s2-ddl-drop-column" "s1-commit" "s1-select-count" "s1-show-columns"
 permutation "s1-initialize" "s1-begin" "s1-adaptive-select" "s2-ddl-rename-column" "s1-commit" "s1-select-count" "s1-show-columns"

src/test/regress/spec/isolation_select_vs_all_on_mx.spec

@@ -98,9 +98,9 @@ step "s2-select-for-update"
 	SELECT run_commands_on_session_level_connection_to_node('SELECT * FROM select_table WHERE id = 6 FOR UPDATE');
 }
 
-step "s2-flaky-coordinator-create-index-concurrently"
+step "s2-coordinator-create-index-concurrently"
 {
-	CREATE INDEX CONCURRENTLY flaky_select_table_index ON select_table(id);
+	CREATE INDEX CONCURRENTLY select_table_index ON select_table(id);
 }
 
 step "s2-commit-worker"
@@ -118,6 +118,14 @@ step "s2-commit"
 	COMMIT;
 }
 
+// We use this as a way to wait for s2-coordinator-create-index-concurrently to
+// complete. We know create-index-concurrently doesn't have to wait for
+// s1-commit-worker, but the isolationtester sometimes detects it temporarily
+// as blocking. To get consistent test output we use a (*) marker to always
+// show create index concurrently as blocking. Then right after we put
+// s2-empty, to wait for it to complete.
+step "s2-empty" {}
+
 
 session "s3"
 
@@ -134,4 +142,4 @@ permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-select"
 permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-select" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-copy" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection" "s3-select-count"
 permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-select" "s2-begin" "s2-index" "s1-commit-worker" "s2-commit" "s1-stop-connection"
 permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-select" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-select-for-update" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection"
-permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-disable-binary-protocol-on-worker" "s1-select" "s2-flaky-coordinator-create-index-concurrently" "s1-commit-worker" "s1-stop-connection"
+permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-disable-binary-protocol-on-worker" "s1-select" "s2-coordinator-create-index-concurrently"(*) "s2-empty" "s1-commit-worker" "s1-stop-connection"

src/test/regress/spec/isolation_truncate_vs_all.spec

@@ -55,6 +55,10 @@ step "s2-master-modify-multiple-shards" { DELETE FROM truncate_append; }
 step "s2-master-drop-all-shards" { SELECT citus_drop_all_shards('truncate_append'::regclass, 'public', 'truncate_append'); }
 step "s2-distribute-table" { SELECT create_distributed_table('truncate_append', 'id', 'append'); }
 step "s2-commit" { COMMIT; }
+// We use this as a way to wait for s2-ddl-create-index-concurrently to
+// complete. We know it can complete after s1-commit has succeeded, this way
+// we make sure we get consistent output.
+step "s2-empty" {}
 
 // permutations - TRUNCATE vs TRUNCATE
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-truncate" "s2-truncate" "s1-commit" "s2-commit" "s1-select-count"
@@ -64,7 +68,7 @@ permutation "s1-initialize" "s1-begin" "s2-begin" "s1-truncate" "s2-truncate" "s
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-truncate" "s2-drop" "s1-commit" "s2-commit" "s1-select-count"
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-truncate" "s2-ddl-create-index" "s1-commit" "s2-commit" "s1-select-count" "s1-show-indexes"
 permutation "s1-initialize" "s1-ddl-create-index" "s1-begin" "s2-begin" "s1-truncate" "s2-ddl-drop-index" "s1-commit" "s2-commit" "s1-select-count" "s1-show-indexes"
-permutation "s1-initialize" "s1-begin" "s1-truncate" "s2-ddl-create-index-concurrently" "s1-commit" "s1-select-count" "s1-show-indexes"
+permutation "s1-initialize" "s1-begin" "s1-truncate" "s2-ddl-create-index-concurrently" "s1-commit" "s2-empty" "s1-select-count" "s1-show-indexes"
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-truncate" "s2-ddl-add-column" "s1-commit" "s2-commit" "s1-select-count" "s1-show-columns"
 permutation "s1-initialize" "s1-ddl-add-column" "s1-begin" "s2-begin" "s1-truncate" "s2-ddl-drop-column" "s1-commit" "s2-commit" "s1-select-count" "s1-show-columns"
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-truncate" "s2-ddl-rename-column" "s1-commit" "s2-commit" "s1-select-count" "s1-show-columns"

src/test/regress/spec/isolation_update_delete_upsert_vs_all_on_mx.spec

@@ -82,6 +82,11 @@ step "s2-select-for-update"
 	SELECT run_commands_on_session_level_connection_to_node('SELECT * FROM dist_table WHERE id=5 FOR UPDATE');
 }
 
+step "s2-coordinator-create-index-concurrently"
+{
+	CREATE INDEX CONCURRENTLY dist_table_index ON dist_table(id);
+}
+
 step "s2-commit-worker"
 {
         SELECT run_commands_on_session_level_connection_to_node('COMMIT');
@@ -92,6 +97,10 @@ step "s2-stop-connection"
         SELECT stop_session_level_connection_to_node();
 }
 
+// We use this as a way to wait for s2-ddl-create-index-concurrently to
+// complete. We know it can complete after s1-commit has succeeded, this way
+// we make sure we get consistent output.
+step "s2-empty" {}
 
 session "s3"
 
@@ -105,5 +114,4 @@ permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-update"
 permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-delete" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-copy" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection" "s3-select-count"
 permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-update" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-alter-table" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection" "s3-select-count"
 permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-update" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-select-for-update" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection"
-//Not able to test the next permutation, until issue with CREATE INDEX CONCURRENTLY's locks is resolved. Issue #2966
-//permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-delete" "s2-coordinator-create-index-concurrently" "s1-commit-worker" "s3-select-count" "s1-stop-connection"
+permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-delete" "s2-coordinator-create-index-concurrently" "s1-commit-worker" "s2-empty" "s3-select-count" "s1-stop-connection"

src/test/regress/spec/isolation_update_vs_all.spec

@@ -57,6 +57,10 @@ step "s2-table-size" { SELECT citus_total_relation_size('update_hash'); }
 step "s2-master-modify-multiple-shards" { DELETE FROM update_hash; }
 step "s2-distribute-table" { SELECT create_distributed_table('update_hash', 'id'); }
 step "s2-commit" { COMMIT; }
+// We use this as a way to wait for s2-ddl-create-index-concurrently to
+// complete. We know it can complete after s1-commit has succeeded, this way we
+// make sure no other query is run over session s1 before that happens.
+step "s2-empty" {}
 
 // permutations - UPDATE vs UPDATE
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-update" "s2-update" "s1-commit" "s2-commit" "s1-select-count"
@@ -68,7 +72,7 @@ permutation "s1-initialize" "s1-begin" "s2-begin" "s1-update" "s2-truncate" "s1-
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-update" "s2-drop" "s1-commit" "s2-commit" "s1-select-count"
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-update" "s2-ddl-create-index" "s1-commit" "s2-commit" "s1-select-count" "s1-show-indexes"
 permutation "s1-initialize" "s1-ddl-create-index" "s1-begin" "s2-begin" "s1-update" "s2-ddl-drop-index" "s1-commit" "s2-commit" "s1-select-count" "s1-show-indexes"
-permutation "s1-initialize" "s1-begin" "s1-update" "s2-ddl-create-index-concurrently" "s1-commit" "s1-select-count" "s1-show-indexes"
+permutation "s1-initialize" "s1-begin" "s1-update" "s2-ddl-create-index-concurrently" "s1-commit" "s2-empty" "s1-select-count" "s1-show-indexes"
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-update" "s2-ddl-add-column" "s1-commit" "s2-commit" "s1-select-count" "s1-show-columns"
 permutation "s1-initialize" "s1-ddl-add-column" "s1-begin" "s2-begin" "s1-update" "s2-ddl-drop-column" "s1-commit" "s2-commit" "s1-select-count" "s1-show-columns"
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-update" "s2-ddl-rename-column" "s1-commit" "s2-commit" "s1-select-count" "s1-show-columns"
@@ -81,7 +85,7 @@ permutation "s1-initialize" "s1-begin" "s2-begin" "s1-update-cte" "s2-truncate"
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-update-cte" "s2-drop" "s1-commit" "s2-commit" "s1-select-count"
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-update-cte" "s2-ddl-create-index" "s1-commit" "s2-commit" "s1-select-count" "s1-show-indexes"
 permutation "s1-initialize" "s1-ddl-create-index" "s1-begin" "s2-begin" "s1-update-cte" "s2-ddl-drop-index" "s1-commit" "s2-commit" "s1-select-count" "s1-show-indexes"
-permutation "s1-initialize" "s1-begin" "s1-update-cte" "s2-ddl-create-index-concurrently" "s1-commit" "s1-select-count" "s1-show-indexes"
+permutation "s1-initialize" "s1-begin" "s1-update-cte" "s2-ddl-create-index-concurrently" "s1-commit" "s2-empty" "s1-select-count" "s1-show-indexes"
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-update-cte" "s2-ddl-add-column" "s1-commit" "s2-commit" "s1-select-count" "s1-show-columns"
 permutation "s1-initialize" "s1-ddl-add-column" "s1-begin" "s2-begin" "s1-update-cte" "s2-ddl-drop-column" "s1-commit" "s2-commit" "s1-select-count" "s1-show-columns"
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-update-cte" "s2-ddl-rename-column" "s1-commit" "s2-commit" "s1-select-count" "s1-show-columns"

src/test/regress/spec/isolation_upsert_vs_all.spec

@@ -56,6 +56,10 @@ step "s2-table-size" { SELECT citus_total_relation_size('upsert_hash'); }
 step "s2-master-modify-multiple-shards" { DELETE FROM upsert_hash; }
 step "s2-distribute-table" { SELECT create_distributed_table('upsert_hash', 'id'); }
 step "s2-commit" { COMMIT; }
+// We use this as a way to wait for s2-ddl-create-index-concurrently to
+// complete. We know it can complete after s1-commit has succeeded, this way we
+// make sure no other query is run over session s1 before that happens.
+step "s2-empty" {}
 
 // permutations - UPSERT vs UPSERT
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-upsert" "s2-upsert" "s1-commit" "s2-commit" "s1-select-count"
@@ -67,7 +71,7 @@ permutation "s1-initialize" "s1-begin" "s2-begin" "s1-upsert" "s2-truncate" "s1-
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-upsert" "s2-drop" "s1-commit" "s2-commit" "s1-select-count"
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-upsert" "s2-ddl-create-index" "s1-commit" "s2-commit" "s1-select-count" "s1-show-indexes"
 permutation "s1-initialize" "s1-ddl-create-index" "s1-begin" "s2-begin" "s1-upsert" "s2-ddl-drop-index" "s1-commit" "s2-commit" "s1-select-count" "s1-show-indexes"
-permutation "s1-initialize" "s1-begin" "s1-upsert" "s2-ddl-create-index-concurrently" "s1-commit" "s1-select-count" "s1-show-indexes"
+permutation "s1-initialize" "s1-begin" "s1-upsert" "s2-ddl-create-index-concurrently" "s1-commit" "s2-empty" "s1-select-count" "s1-show-indexes"
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-upsert" "s2-ddl-add-column" "s1-commit" "s2-commit" "s1-select-count" "s1-show-columns"
 permutation "s1-initialize" "s1-ddl-add-column" "s1-begin" "s2-begin" "s1-upsert" "s2-ddl-drop-column" "s1-commit" "s2-commit" "s1-select-count" "s1-show-columns"
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-upsert" "s2-ddl-rename-column" "s1-commit" "s2-commit" "s1-select-count" "s1-show-columns"