Add normalization rules for flaky isolation tests

We remove `<waiting ...>` and `<... completed>` outputs for some CREATE
INDEX CONCURRENTLY commands since they can cause flakiness in some scenarios.

Postgres calls WaitForOlderSnapshots() and this can cause CREATE INDEX
CONCURRENTLY commands for shards to get blocked by each other for brief
periods of time. The extra waits can pop-up, or they can get completed
at different lines in the output files. To remedy that, we rename those
indexes to be captured by the new normalization rule.

src/test/regress/bin/normalize.sed

@@ -219,6 +219,20 @@ s/^(ERROR:  child table is missing constraint "\w+)_([0-9])+"/\1_xxxxxx"/g
     }
 }
 
+# normalize for random waits for CREATE INDEX CONCURRENTLY isolation tests.
+# <waiting ...> outputs can be in separate lines, or on the same line, and hence
+# we have a slightly more complex pattern.
+# All the flaky tests use a index name that starts with `flaky` so we limit the
+# normalization using that pattern.
+/CREATE INDEX CONCURRENTLY flaky/ {
+	N; s/ <waiting ...>//
+}
+
+# Remove completion lines in isolation tests for CREATE INDEX CONCURRENTLY commands.
+# This is needed because the commands that are executed on the shards can block each other
+# for a small window of time and we may see the completion output in different lines.
+/step s2-flaky.* <... completed>/d
+
 # normalize long table shard name errors for alter_table_set_access_method and alter_distributed_table
 s/^(ERROR:  child table is missing constraint "\w+)_([0-9])+"/\1_xxxxxx"/g
 s/^(DEBUG:  the name of the shard \(abcde_01234567890123456789012345678901234567890_f7ff6612)_([0-9])+/\1_xxxxxx/g

src/test/regress/expected/isolation_drop_alter_index_select_for_update_on_mx.out

@@ -235,7 +235,7 @@ restore_isolation_tester_func
 (1 row)
 
 
-starting permutation: s1-start-session-level-connection s1-begin-on-worker s1-select-for-update s2-coordinator-create-index-concurrently s1-commit-worker s1-stop-connection
+starting permutation: s1-start-session-level-connection s1-begin-on-worker s1-select-for-update s2-flaky-coordinator-create-index-concurrently s1-commit-worker s1-stop-connection
 step s1-start-session-level-connection:
         SELECT start_session_level_connection_to_node('localhost', 57637);
 
@@ -260,8 +260,8 @@ run_commands_on_session_level_connection_to_node
 
 (1 row)
 
-step s2-coordinator-create-index-concurrently:
- CREATE INDEX CONCURRENTLY dist_table_index_conc ON dist_table(id);
+step s2-flaky-coordinator-create-index-concurrently:
+ CREATE INDEX CONCURRENTLY flaky_dist_table_index_conc ON dist_table(id);
 
 step s1-commit-worker:
  SELECT run_commands_on_session_level_connection_to_node('COMMIT');

src/test/regress/expected/isolation_hash_copy_vs_all.out

@@ -317,7 +317,7 @@ restore_isolation_tester_func
 (1 row)
 
 
-starting permutation: s1-initialize s1-begin s1-copy s2-ddl-create-index-concurrently s1-commit s1-select-count s1-show-indexes
+starting permutation: s1-initialize s1-begin s1-copy s2-flaky-ddl-create-index-concurrently s1-commit s1-select-count s1-show-indexes
 create_distributed_table
 ---------------------------------------------------------------------
 
@@ -326,9 +326,8 @@ create_distributed_table
 step s1-initialize: COPY hash_copy FROM PROGRAM 'echo 0, a, 0 && echo 1, b, 1 && echo 2, c, 2 && echo 3, d, 3 && echo 4, e, 4' WITH CSV;
 step s1-begin: BEGIN;
 step s1-copy: COPY hash_copy FROM PROGRAM 'echo 5, f, 5 && echo 6, g, 6 && echo 7, h, 7 && echo 8, i, 8 && echo 9, j, 9' WITH CSV;
-step s2-ddl-create-index-concurrently: CREATE INDEX CONCURRENTLY hash_copy_index ON hash_copy(id); <waiting ...>
+step s2-flaky-ddl-create-index-concurrently: CREATE INDEX CONCURRENTLY flaky_hash_copy_index ON hash_copy(id);
 step s1-commit: COMMIT;
-step s2-ddl-create-index-concurrently: <... completed>
 step s1-select-count: SELECT COUNT(*) FROM hash_copy;
 count
 ---------------------------------------------------------------------

src/test/regress/expected/isolation_reference_copy_vs_all.out

@@ -319,7 +319,7 @@ restore_isolation_tester_func
 (1 row)
 
 
-starting permutation: s1-initialize s1-begin s1-copy s2-ddl-create-index-concurrently s1-commit s1-select-count s1-show-indexes
+starting permutation: s1-initialize s1-begin s1-copy s2-flaky-ddl-create-index-concurrently s1-commit s1-select-count s1-show-indexes
 create_reference_table
 ---------------------------------------------------------------------
 
@@ -328,9 +328,8 @@ create_reference_table
 step s1-initialize: COPY reference_copy FROM PROGRAM 'echo 0, a, 0 && echo 1, b, 1 && echo 2, c, 2 && echo 3, d, 3 && echo 4, e, 4' WITH CSV;
 step s1-begin: BEGIN;
 step s1-copy: COPY reference_copy FROM PROGRAM 'echo 5, f, 5  && echo 6, g, 6 && echo 7, h, 7 && echo 8, i, 8 && echo 9, j, 9' WITH CSV;
-step s2-ddl-create-index-concurrently: CREATE INDEX CONCURRENTLY reference_copy_index ON reference_copy(id); <waiting ...>
+step s2-flaky-ddl-create-index-concurrently: CREATE INDEX CONCURRENTLY flaky_reference_copy_index ON reference_copy(id);
 step s1-commit: COMMIT;
-step s2-ddl-create-index-concurrently: <... completed>
 step s1-select-count: SELECT COUNT(*) FROM reference_copy;
 count
 ---------------------------------------------------------------------

src/test/regress/expected/isolation_select_vs_all_on_mx.out

@@ -515,7 +515,7 @@ restore_isolation_tester_func
 (1 row)
 
 
-starting permutation: s1-start-session-level-connection s1-begin-on-worker s1-disable-binary-protocol-on-worker s1-select s2-coordinator-create-index-concurrently s1-commit-worker s1-stop-connection
+starting permutation: s1-start-session-level-connection s1-begin-on-worker s1-disable-binary-protocol-on-worker s1-select s2-flaky-coordinator-create-index-concurrently s1-commit-worker s1-stop-connection
 step s1-start-session-level-connection:
         SELECT start_session_level_connection_to_node('localhost', 57637);
 
@@ -549,8 +549,8 @@ run_commands_on_session_level_connection_to_node
 
 (1 row)
 
-step s2-coordinator-create-index-concurrently:
- CREATE INDEX CONCURRENTLY select_table_index ON select_table(id);
+step s2-flaky-coordinator-create-index-concurrently:
+ CREATE INDEX CONCURRENTLY flaky_select_table_index ON select_table(id);
 
 step s1-commit-worker:
  SELECT run_commands_on_session_level_connection_to_node('COMMIT');

src/test/regress/spec/isolation_drop_alter_index_select_for_update_on_mx.spec

@@ -89,9 +89,9 @@ step "s2-select-for-update"
 	SELECT run_commands_on_session_level_connection_to_node('SELECT * FROM dist_table WHERE id = 5 FOR UPDATE');
 }
 
-step "s2-coordinator-create-index-concurrently"
+step "s2-flaky-coordinator-create-index-concurrently"
 {
-	CREATE INDEX CONCURRENTLY dist_table_index_conc ON dist_table(id);
+	CREATE INDEX CONCURRENTLY flaky_dist_table_index_conc ON dist_table(id);
 }
 
 step "s2-commit-worker"
@@ -117,4 +117,4 @@ step "s3-select-count"
 permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-insert" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-alter" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection" "s3-select-count"
 permutation "s1-begin" "s1-index" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-select-for-update" "s1-commit" "s2-commit-worker" "s2-stop-connection"
 permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-select-for-update" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-select-for-update" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection"
-permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-select-for-update" "s2-coordinator-create-index-concurrently" "s1-commit-worker" "s1-stop-connection"
+permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-select-for-update" "s2-flaky-coordinator-create-index-concurrently" "s1-commit-worker" "s1-stop-connection"

src/test/regress/spec/isolation_hash_copy_vs_all.spec

@@ -78,7 +78,7 @@ step "s2-truncate" { TRUNCATE hash_copy; }
 step "s2-drop" { DROP TABLE hash_copy; }
 step "s2-ddl-create-index" { CREATE INDEX hash_copy_index ON hash_copy(id); }
 step "s2-ddl-drop-index" { DROP INDEX hash_copy_index; }
-step "s2-ddl-create-index-concurrently" { CREATE INDEX CONCURRENTLY hash_copy_index ON hash_copy(id); }
+step "s2-flaky-ddl-create-index-concurrently" { CREATE INDEX CONCURRENTLY flaky_hash_copy_index ON hash_copy(id); }
 step "s2-ddl-add-column" { ALTER TABLE hash_copy ADD new_column int DEFAULT 0; }
 step "s2-ddl-drop-column" { ALTER TABLE hash_copy DROP new_column; }
 step "s2-ddl-rename-column" { ALTER TABLE hash_copy RENAME data TO new_column; }
@@ -102,7 +102,7 @@ permutation "s1-initialize" "s1-begin" "s1-copy" "s2-truncate" "s1-commit" "s1-s
 permutation "s1-initialize" "s1-begin" "s1-copy" "s2-drop" "s1-commit" "s1-select-count"
 permutation "s1-initialize" "s1-begin" "s1-copy" "s2-ddl-create-index" "s1-commit" "s1-select-count" "s1-show-indexes"
 permutation "s1-initialize" "s1-ddl-create-index" "s1-begin" "s1-copy" "s2-ddl-drop-index" "s1-commit" "s1-select-count" "s1-show-indexes"
-permutation "s1-initialize" "s1-begin" "s1-copy" "s2-ddl-create-index-concurrently" "s1-commit" "s1-select-count" "s1-show-indexes"
+permutation "s1-initialize" "s1-begin" "s1-copy" "s2-flaky-ddl-create-index-concurrently" "s1-commit" "s1-select-count" "s1-show-indexes"
 permutation "s1-initialize" "s1-begin" "s1-copy" "s2-ddl-add-column" "s1-commit" "s1-select-count" "s1-show-columns"
 permutation "s1-initialize" "s1-ddl-add-column" "s1-begin" "s1-copy-additional-column" "s2-ddl-drop-column" "s1-commit" "s1-select-count" "s1-show-columns"
 permutation "s1-initialize" "s1-begin" "s1-copy" "s2-ddl-rename-column" "s1-commit" "s1-select-count" "s1-show-columns"

src/test/regress/spec/isolation_reference_copy_vs_all.spec

@@ -68,7 +68,7 @@ step "s2-truncate" { TRUNCATE reference_copy; }
 step "s2-drop" { DROP TABLE reference_copy; }
 step "s2-ddl-create-index" { CREATE INDEX reference_copy_index ON reference_copy(id); }
 step "s2-ddl-drop-index" { DROP INDEX reference_copy_index; }
-step "s2-ddl-create-index-concurrently" { CREATE INDEX CONCURRENTLY reference_copy_index ON reference_copy(id); }
+step "s2-flaky-ddl-create-index-concurrently" { CREATE INDEX CONCURRENTLY flaky_reference_copy_index ON reference_copy(id); }
 step "s2-ddl-add-column" { ALTER TABLE reference_copy ADD new_column int DEFAULT 0; }
 step "s2-ddl-drop-column" { ALTER TABLE reference_copy DROP new_column; }
 step "s2-ddl-rename-column" { ALTER TABLE reference_copy RENAME data TO new_column; }
@@ -91,7 +91,7 @@ permutation "s1-initialize" "s1-begin" "s1-copy" "s2-truncate" "s1-commit" "s1-s
 permutation "s1-initialize" "s1-begin" "s1-copy" "s2-drop" "s1-commit" "s1-select-count"
 permutation "s1-initialize" "s1-begin" "s1-copy" "s2-ddl-create-index" "s1-commit" "s1-select-count" "s1-show-indexes"
 permutation "s1-initialize" "s1-ddl-create-index" "s1-begin" "s1-copy" "s2-ddl-drop-index" "s1-commit" "s1-select-count" "s1-show-indexes"
-permutation "s1-initialize" "s1-begin" "s1-copy" "s2-ddl-create-index-concurrently" "s1-commit" "s1-select-count" "s1-show-indexes"
+permutation "s1-initialize" "s1-begin" "s1-copy" "s2-flaky-ddl-create-index-concurrently" "s1-commit" "s1-select-count" "s1-show-indexes"
 permutation "s1-initialize" "s1-begin" "s1-copy" "s2-ddl-add-column" "s1-commit" "s1-select-count" "s1-show-columns"
 permutation "s1-initialize" "s1-ddl-add-column" "s1-begin" "s1-copy-additional-column" "s2-ddl-drop-column" "s1-commit" "s1-select-count" "s1-show-columns"
 permutation "s1-initialize" "s1-begin" "s1-copy" "s2-ddl-rename-column" "s1-commit" "s1-select-count" "s1-show-columns"

src/test/regress/spec/isolation_select_vs_all_on_mx.spec

@@ -99,9 +99,9 @@ step "s2-select-for-update"
 	SELECT run_commands_on_session_level_connection_to_node('SELECT * FROM select_table WHERE id = 6 FOR UPDATE');
 }
 
-step "s2-coordinator-create-index-concurrently"
+step "s2-flaky-coordinator-create-index-concurrently"
 {
-	CREATE INDEX CONCURRENTLY select_table_index ON select_table(id);
+	CREATE INDEX CONCURRENTLY flaky_select_table_index ON select_table(id);
 }
 
 step "s2-commit-worker"
@@ -135,4 +135,4 @@ permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-select"
 permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-select" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-copy" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection" "s3-select-count"
 permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-select" "s2-begin" "s2-index" "s1-commit-worker" "s2-commit" "s1-stop-connection"
 permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-select" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-select-for-update" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection"
-permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-disable-binary-protocol-on-worker" "s1-select" "s2-coordinator-create-index-concurrently" "s1-commit-worker" "s1-stop-connection"
+permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-disable-binary-protocol-on-worker" "s1-select" "s2-flaky-coordinator-create-index-concurrently" "s1-commit-worker" "s1-stop-connection"