Properly add / remove coordinator for isolation tests (#6181)

We used to rely on a seperate session to add the coordinator.
However, that might prevent the existing sessions to get
assigned proper gpids, which causes flaky tests.

(cherry picked from commit 961fcff5db)

src/test/regress/expected/isolation_add_coordinator.out

@@ -0,0 +1,21 @@
+Parsed test spec with 1 sessions
+
+starting permutation: s1-begin
+?column?
+---------------------------------------------------------------------
+       1
+(1 row)
+
+step s1-begin:
+    SELECT 1;
+
+?column?
+---------------------------------------------------------------------
+       1
+(1 row)
+
+?column?
+---------------------------------------------------------------------
+       1
+(1 row)
+

src/test/regress/expected/isolation_citus_dist_activity.out

@@ -1,25 +1,4 @@
-Parsed test spec with 4 sessions
+Parsed test spec with 3 sessions
-
-starting permutation: add-coordinator-to-metadata
-create_distributed_table
----------------------------------------------------------------------
-
-(1 row)
-
-step add-coordinator-to-metadata:
-    SELECT 1 FROM citus_add_node('localhost', 57636, groupid:=0);
-    SELECT test_assign_global_pid();
-
-?column?
----------------------------------------------------------------------
-       1
-(1 row)
-
-test_assign_global_pid
----------------------------------------------------------------------
-
-(1 row)
-
 
 starting permutation: s1-cache-connections s1-begin s2-begin s3-begin s1-alter-table s2-sleep s2-view-dist s3-view-worker s2-rollback s1-commit s3-rollback
 create_distributed_table
@@ -67,16 +46,16 @@ step s3-view-worker:
 
 query                                                                                                      |citus_nodename_for_nodeid|citus_nodeport_for_nodeid|state              |wait_event_type|wait_event|usename |datname
 ---------------------------------------------------------------------
-SELECT worker_apply_shard_ddl_command (1300009, 'public', '
+SELECT worker_apply_shard_ddl_command (1300004, 'public', '
     ALTER TABLE test_table ADD COLUMN x INT;
 ')|localhost                |                    57636|idle in transaction|Client         |ClientRead|postgres|regression
-SELECT worker_apply_shard_ddl_command (1300008, 'public', '
+SELECT worker_apply_shard_ddl_command (1300003, 'public', '
     ALTER TABLE test_table ADD COLUMN x INT;
 ')|localhost                |                    57636|idle in transaction|Client         |ClientRead|postgres|regression
-SELECT worker_apply_shard_ddl_command (1300007, 'public', '
+SELECT worker_apply_shard_ddl_command (1300002, 'public', '
     ALTER TABLE test_table ADD COLUMN x INT;
 ')|localhost                |                    57636|idle in transaction|Client         |ClientRead|postgres|regression
-SELECT worker_apply_shard_ddl_command (1300006, 'public', '
+SELECT worker_apply_shard_ddl_command (1300001, 'public', '
     ALTER TABLE test_table ADD COLUMN x INT;
 ')|localhost                |                    57636|idle in transaction|Client         |ClientRead|postgres|regression
 (4 rows)
@@ -137,7 +116,7 @@ step s3-view-worker:
 
 query                                                                     |citus_nodename_for_nodeid|citus_nodeport_for_nodeid|state              |wait_event_type|wait_event|usename |datname
 ---------------------------------------------------------------------
-INSERT INTO public.test_table_1300013 (column1, column2) VALUES (100, 100)|localhost                |                    57636|idle in transaction|Client         |ClientRead|postgres|regression
+INSERT INTO public.test_table_1300008 (column1, column2) VALUES (100, 100)|localhost                |                    57636|idle in transaction|Client         |ClientRead|postgres|regression
 (1 row)
 
 step s2-rollback:
@@ -201,10 +180,10 @@ step s3-view-worker:
 
 query                                                                        |citus_nodename_for_nodeid|citus_nodeport_for_nodeid|state              |wait_event_type|wait_event|usename |datname
 ---------------------------------------------------------------------
-SELECT count(*) AS count FROM public.test_table_1300019 test_table WHERE true|localhost                |                    57636|idle in transaction|Client         |ClientRead|postgres|regression
+SELECT count(*) AS count FROM public.test_table_1300014 test_table WHERE true|localhost                |                    57636|idle in transaction|Client         |ClientRead|postgres|regression
-SELECT count(*) AS count FROM public.test_table_1300018 test_table WHERE true|localhost                |                    57636|idle in transaction|Client         |ClientRead|postgres|regression
+SELECT count(*) AS count FROM public.test_table_1300013 test_table WHERE true|localhost                |                    57636|idle in transaction|Client         |ClientRead|postgres|regression
-SELECT count(*) AS count FROM public.test_table_1300017 test_table WHERE true|localhost                |                    57636|idle in transaction|Client         |ClientRead|postgres|regression
+SELECT count(*) AS count FROM public.test_table_1300012 test_table WHERE true|localhost                |                    57636|idle in transaction|Client         |ClientRead|postgres|regression
-SELECT count(*) AS count FROM public.test_table_1300016 test_table WHERE true|localhost                |                    57636|idle in transaction|Client         |ClientRead|postgres|regression
+SELECT count(*) AS count FROM public.test_table_1300011 test_table WHERE true|localhost                |                    57636|idle in transaction|Client         |ClientRead|postgres|regression
 (4 rows)
 
 step s2-rollback:
@@ -268,7 +247,7 @@ step s3-view-worker:
 
 query                                                                                                       |citus_nodename_for_nodeid|citus_nodeport_for_nodeid|state              |wait_event_type|wait_event|usename |datname
 ---------------------------------------------------------------------
-SELECT count(*) AS count FROM public.test_table_1300022 test_table WHERE (column1 OPERATOR(pg_catalog.=) 55)|localhost                |                    57636|idle in transaction|Client         |ClientRead|postgres|regression
+SELECT count(*) AS count FROM public.test_table_1300017 test_table WHERE (column1 OPERATOR(pg_catalog.=) 55)|localhost                |                    57636|idle in transaction|Client         |ClientRead|postgres|regression
 (1 row)
 
 step s2-rollback:
@@ -280,18 +259,3 @@ step s1-commit:
 step s3-rollback:
 	ROLLBACK;
 
-
-starting permutation: remove-coordinator-from-metadata
-create_distributed_table
----------------------------------------------------------------------
-
-(1 row)
-
-step remove-coordinator-from-metadata:
-    SELECT citus_remove_node('localhost', 57636);
-
-citus_remove_node
----------------------------------------------------------------------
-
-(1 row)
-

src/test/regress/expected/isolation_remove_coordinator.out

@@ -0,0 +1,21 @@
+Parsed test spec with 1 sessions
+
+starting permutation: s1-begin
+?column?
+---------------------------------------------------------------------
+       1
+(1 row)
+
+step s1-begin:
+    SELECT 1;
+
+?column?
+---------------------------------------------------------------------
+       1
+(1 row)
+
+?column?
+---------------------------------------------------------------------
+       1
+(1 row)
+

src/test/regress/isolation_schedule

@@ -21,7 +21,9 @@ test: isolation_cluster_management
 test: isolation_metadata_sync_vs_all
 test: isolation_distributed_transaction_id
 test: isolation_dump_global_wait_edges
+test: isolation_add_coordinator
 test: isolation_citus_dist_activity
+test: isolation_remove_coordinator
 
 test: isolation_insert_select_repartition
 test: isolation_dml_vs_repair isolation_copy_placement_vs_copy_placement

src/test/regress/spec/isolation_add_coordinator.spec

@@ -0,0 +1,19 @@
+setup
+{
+    SELECT 1 FROM citus_set_coordinator_host('localhost', 57636);
+
+}
+
+teardown
+{
+	SELECT 1;
+}
+
+session "s1"
+
+step "s1-begin"
+{
+    SELECT 1;
+}
+
+permutation "s1-begin"

src/test/regress/spec/isolation_citus_dist_activity.spec

@@ -101,25 +101,9 @@ step "s3-view-worker"
 	SELECT query, citus_nodename_for_nodeid(citus_nodeid_for_gpid(global_pid)), citus_nodeport_for_nodeid(citus_nodeid_for_gpid(global_pid)), state, wait_event_type, wait_event, usename, datname FROM citus_stat_activity WHERE query NOT ILIKE ALL(VALUES('%pg_prepared_xacts%'), ('%COMMIT%'), ('%csa_from_one_node%')) AND is_worker_query = true AND backend_type = 'client backend' ORDER BY query DESC;
 }
 
-session "s4"
-step "add-coordinator-to-metadata"
-{
-    SELECT 1 FROM citus_add_node('localhost', 57636, groupid:=0);
-    SELECT test_assign_global_pid();
-}
-
-step "remove-coordinator-from-metadata"
-{
-    SELECT citus_remove_node('localhost', 57636);
-}
-
-permutation "add-coordinator-to-metadata"
-
 // we prefer to sleep before "s2-view-dist" so that we can ensure
 // the "wait_event" in the output doesn't change randomly (e.g., NULL to CliendRead etc.)
 permutation "s1-cache-connections" "s1-begin" "s2-begin" "s3-begin" "s1-alter-table" "s2-sleep" "s2-view-dist" "s3-view-worker" "s2-rollback" "s1-commit" "s3-rollback"
 permutation "s1-cache-connections" "s1-begin" "s2-begin" "s3-begin" "s1-insert" "s2-sleep" "s2-view-dist" "s3-view-worker" "s2-rollback" "s1-commit" "s3-rollback"
 permutation "s1-cache-connections" "s1-begin" "s2-begin" "s3-begin" "s1-select" "s2-sleep" "s2-view-dist" "s3-view-worker" "s2-rollback" "s1-commit" "s3-rollback"
 permutation "s1-cache-connections" "s1-begin" "s2-begin" "s3-begin" "s1-select-router" "s2-sleep" "s2-view-dist" "s3-view-worker" "s2-rollback" "s1-commit" "s3-rollback"
-
-permutation "remove-coordinator-from-metadata"

src/test/regress/spec/isolation_remove_coordinator.spec

@@ -0,0 +1,18 @@
+setup
+{
+	SELECT 1;
+}
+
+teardown
+{
+	SELECT 1 FROM citus_remove_node('localhost', 57636);
+}
+
+session "s1"
+
+step "s1-begin"
+{
+    SELECT 1;
+}
+
+permutation "s1-begin"