Reduce isolation flakyness by improving blocked process detection

Sometimes our CI randomly fails on a test in a way similar to this:
```diff
 step s2-drop:
     DROP TABLE cancel_table;
-
+ <waiting ...>
+step s2-drop: <... completed>

 starting permutation: s1-timeout s1-begin s1-sleep10000 s1-rollback s1-reset s1-drop
```
Source: https://app.circleci.com/pipelines/github/citusdata/citus/26524/workflows/5415b84f-13a3-482f-bef9-648314c79a67/jobs/756377

I tried to fix that already in #6252 by disabling the maintenance daemon
during isolation tests. But it seems that hasn't fixed all cases of
these errors. This is another attempt at fixing these issues that seems
to have better results.

What it does is that it starts using the pInterestingPids parameter that
citus_isolation_test_session_is_blocked receives. With this change we
start filter out processes that are not children of any of the global
pids.

This approach works for almost all of our isolation tests, but not for
all of them. For the ones where it doesn't work I use a GUC to fall back
to the old behaviour which checks all blocks.

src/backend/distributed/shared_library_init.c

@@ -148,6 +148,7 @@ DEFINE_COLUMNAR_PASSTHROUGH_FUNC(test_columnar_storage_write_new_page)
 static char *CitusVersion = CITUS_VERSION;
 static char *DeprecatedEmptyString = "";
 static char *MitmfifoEmptyString = "";
+static bool IsolationTestCheckAllBlocks = false;
 
 /* deprecated GUC value that should not be used anywhere outside this file */
 static int ReplicationModel = REPLICATION_MODEL_STREAMING;
@@ -1508,6 +1509,16 @@ RegisterCitusConfigVariables(void)
 		GUC_NO_SHOW_ALL,
 		NULL, NULL, NULL);
 
+	DefineCustomBoolVariable(
+		"citus.isolation_test_check_all_blocks",
+		NULL,
+		NULL,
+		&IsolationTestCheckAllBlocks,
+		false,
+		PGC_USERSET,
+		GUC_NO_SHOW_ALL,
+		NULL, NULL, NULL);
+
 	DefineCustomIntVariable(
 		"citus.isolation_test_session_process_id",
 		NULL,

src/backend/distributed/sql/citus--11.1-1--11.2-1.sql

@@ -1 +1,3 @@
 #include "udfs/get_rebalance_progress/11.2-1.sql"
+#include "udfs/citus_backend_gpid/11.2-1.sql"
+#include "udfs/citus_isolation_test_session_is_blocked/11.2-1.sql"

src/backend/distributed/sql/downgrades/citus--11.2-1--11.1-1.sql

@@ -1 +1,4 @@
+#include "../udfs/citus_isolation_test_session_is_blocked/11.1-1.sql"
+DROP FUNCTION pg_catalog.citus_backend_gpid(int);
+#include "../udfs/citus_backend_gpid/11.0-1.sql"
 #include "../udfs/get_rebalance_progress/11.1-1.sql"

src/backend/distributed/sql/udfs/citus_backend_gpid/11.2-1.sql

@@ -0,0 +1,9 @@
+DROP FUNCTION pg_catalog.citus_backend_gpid;
+CREATE FUNCTION pg_catalog.citus_backend_gpid(pid int default NULL)
+    RETURNS BIGINT
+    LANGUAGE C
+    AS 'MODULE_PATHNAME',$$citus_backend_gpid$$;
+COMMENT ON FUNCTION pg_catalog.citus_backend_gpid(int)
+    IS 'returns gpid of the current backend or of a specific backend if pid is passed';
+
+GRANT EXECUTE ON FUNCTION pg_catalog.citus_backend_gpid(int) TO PUBLIC;

src/backend/distributed/sql/udfs/citus_backend_gpid/latest.sql

@@ -1,8 +1,9 @@
-CREATE FUNCTION pg_catalog.citus_backend_gpid()
+DROP FUNCTION pg_catalog.citus_backend_gpid;
+CREATE FUNCTION pg_catalog.citus_backend_gpid(pid int default NULL)
     RETURNS BIGINT
-    LANGUAGE C STRICT
+    LANGUAGE C
     AS 'MODULE_PATHNAME',$$citus_backend_gpid$$;
-COMMENT ON FUNCTION pg_catalog.citus_backend_gpid()
+COMMENT ON FUNCTION pg_catalog.citus_backend_gpid(int)
-    IS 'returns gpid of the current backend';
+    IS 'returns gpid of the current backend or of a specific backend if pid is passed';
 
-GRANT EXECUTE ON FUNCTION pg_catalog.citus_backend_gpid() TO PUBLIC;
+GRANT EXECUTE ON FUNCTION pg_catalog.citus_backend_gpid(int) TO PUBLIC;

src/backend/distributed/sql/udfs/citus_isolation_test_session_is_blocked/11.2-1.sql

@@ -0,0 +1,43 @@
+CREATE OR REPLACE FUNCTION pg_catalog.citus_isolation_test_session_is_blocked(pBlockedPid integer, pInterestingPids integer[])
+RETURNS boolean AS $$
+  DECLARE
+    mBlockedGlobalPid int8;
+    workerProcessId integer := current_setting('citus.isolation_test_session_remote_process_id');
+    coordinatorProcessId integer := current_setting('citus.isolation_test_session_process_id');
+  BEGIN
+    IF pg_catalog.old_pg_isolation_test_session_is_blocked(pBlockedPid, pInterestingPids) THEN
+      RETURN true;
+    END IF;
+
+    -- pg says we're not blocked locally; check whether we're blocked globally.
+    -- Note that worker process may be blocked or waiting for a lock. So we need to
+    -- get transaction number for both of them. Following IF provides the transaction
+    -- number when the worker process waiting for other session.
+    IF EXISTS (SELECT 1 FROM get_global_active_transactions()
+               WHERE process_id = workerProcessId AND pBlockedPid = coordinatorProcessId) THEN
+      SELECT global_pid INTO mBlockedGlobalPid FROM get_global_active_transactions()
+      WHERE process_id = workerProcessId AND pBlockedPid = coordinatorProcessId;
+    ELSE
+      -- Check whether transactions initiated from the coordinator get locked
+      SELECT global_pid INTO mBlockedGlobalPid
+        FROM get_all_active_transactions() WHERE process_id = pBlockedPid;
+    END IF;
+
+    IF current_setting('citus.isolation_test_check_all_blocks') = 'off' THEN
+      RETURN EXISTS (
+        SELECT 1 FROM citus_internal_global_blocked_processes()
+          WHERE waiting_global_pid = mBlockedGlobalPid
+          AND blocking_global_pid in (
+              SELECT citus_backend_gpid(pid) FROM unnest(pInterestingPids) pid
+          )
+      );
+    ELSE
+      RETURN EXISTS (
+        SELECT 1 FROM citus_internal_global_blocked_processes()
+          WHERE waiting_global_pid = mBlockedGlobalPid
+      );
+    END IF;
+  END;
+$$ LANGUAGE plpgsql;
+
+REVOKE ALL ON FUNCTION citus_isolation_test_session_is_blocked(integer,integer[]) FROM PUBLIC;

src/backend/distributed/sql/udfs/citus_isolation_test_session_is_blocked/latest.sql

@@ -23,10 +23,20 @@ RETURNS boolean AS $$
         FROM get_all_active_transactions() WHERE process_id = pBlockedPid;
     END IF;
 
+    IF current_setting('citus.isolation_test_check_all_blocks') = 'off' THEN
+      RETURN EXISTS (
+        SELECT 1 FROM citus_internal_global_blocked_processes()
+          WHERE waiting_global_pid = mBlockedGlobalPid
+          AND blocking_global_pid in (
+              SELECT citus_backend_gpid(pid) FROM unnest(pInterestingPids) pid
+          )
+      );
+    ELSE
       RETURN EXISTS (
         SELECT 1 FROM citus_internal_global_blocked_processes()
           WHERE waiting_global_pid = mBlockedGlobalPid
       );
+    END IF;
   END;
 $$ LANGUAGE plpgsql;
 

src/backend/distributed/transaction/backend_data.c

@@ -994,7 +994,12 @@ citus_backend_gpid(PG_FUNCTION_ARGS)
 {
 	CheckCitusVersion(ERROR);
 
+	if (PG_ARGISNULL(0))
+	{
 		PG_RETURN_UINT64(GetGlobalPID());
+	}
+
+	PG_RETURN_UINT64(CalculateGlobalPID(GetLocalNodeId(), PG_GETARG_INT32(0)));
 }
 
 

src/test/regress/expected/isolation_acquire_distributed_locks.out

@@ -1,6 +1,6 @@
-Parsed test spec with 3 sessions
+Parsed test spec with 4 sessions
 
-starting permutation: coor-begin coor-acquire-aggresive-lock-on-dist-table w1-start-session-level-connection w1-begin w1-read-dist-table coor-rollback w1-rollback w1-stop-connection
+starting permutation: coor-begin coor-acquire-aggresive-lock-on-dist-table w1-start-session-level-connection w1-begin w1-read-dist-table coor-rollback w1-rollback w1-stop-connection teardown
 step coor-begin:
  BEGIN;
 
@@ -51,13 +51,16 @@ stop_session_level_connection_to_node
 
 (1 row)
 
+step teardown:
+ ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+
 citus_remove_node
 ---------------------------------------------------------------------
 
 (1 row)
 
 
-starting permutation: coor-begin coor-acquire-aggresive-lock-on-dist-table w1-start-session-level-connection w1-begin w1-acquire-aggressive-lock-dist-table coor-rollback coor-read-dist-table w1-rollback w1-stop-connection
+starting permutation: coor-begin coor-acquire-aggresive-lock-on-dist-table w1-start-session-level-connection w1-begin w1-acquire-aggressive-lock-dist-table coor-rollback coor-read-dist-table w1-rollback w1-stop-connection teardown
 step coor-begin:
  BEGIN;
 
@@ -117,13 +120,16 @@ stop_session_level_connection_to_node
 
 (1 row)
 
+step teardown:
+ ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+
 citus_remove_node
 ---------------------------------------------------------------------
 
 (1 row)
 
 
-starting permutation: w1-start-session-level-connection w1-begin w1-acquire-aggressive-lock-dist-table coor-begin coor-acquire-aggresive-lock-on-dist-table-nowait coor-rollback w1-rollback w1-stop-connection
+starting permutation: w1-start-session-level-connection w1-begin w1-acquire-aggressive-lock-dist-table coor-begin coor-acquire-aggresive-lock-on-dist-table-nowait coor-rollback w1-rollback w1-stop-connection teardown
 step w1-start-session-level-connection:
     SELECT start_session_level_connection_to_node('localhost', 57637);
 
@@ -174,13 +180,16 @@ stop_session_level_connection_to_node
 
 (1 row)
 
+step teardown:
+ ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+
 citus_remove_node
 ---------------------------------------------------------------------
 
 (1 row)
 
 
-starting permutation: w1-start-session-level-connection w1-begin w2-start-session-level-connection w2-begin w1-acquire-aggressive-lock-dist-table w2-acquire-aggressive-lock-dist-table w1-rollback w1-read-dist-table w2-rollback w1-stop-connection w2-stop-connection
+starting permutation: w1-start-session-level-connection w1-begin w2-start-session-level-connection w2-begin w1-acquire-aggressive-lock-dist-table w2-acquire-aggressive-lock-dist-table w1-rollback w1-read-dist-table w2-rollback w1-stop-connection w2-stop-connection teardown
 step w1-start-session-level-connection:
     SELECT start_session_level_connection_to_node('localhost', 57637);
 
@@ -271,13 +280,16 @@ stop_session_level_connection_to_node
 
 (1 row)
 
+step teardown:
+ ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+
 citus_remove_node
 ---------------------------------------------------------------------
 
 (1 row)
 
 
-starting permutation: coor-begin coor-acquire-weak-lock-on-dist-table w1-start-session-level-connection w1-begin w1-read-dist-table w1-acquire-aggressive-lock-dist-table coor-rollback w1-rollback w1-stop-connection
+starting permutation: coor-begin coor-acquire-weak-lock-on-dist-table w1-start-session-level-connection w1-begin w1-read-dist-table w1-acquire-aggressive-lock-dist-table coor-rollback w1-rollback w1-stop-connection teardown
 step coor-begin:
  BEGIN;
 
@@ -336,13 +348,16 @@ stop_session_level_connection_to_node
 
 (1 row)
 
+step teardown:
+ ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+
 citus_remove_node
 ---------------------------------------------------------------------
 
 (1 row)
 
 
-starting permutation: w1-start-session-level-connection w1-begin w1-lock-reference-table coor-begin coor-read-ref-table w1-rollback coor-rollback w1-stop-connection
+starting permutation: w1-start-session-level-connection w1-begin w1-lock-reference-table coor-begin coor-read-ref-table w1-rollback coor-rollback w1-stop-connection teardown
 step w1-start-session-level-connection:
     SELECT start_session_level_connection_to_node('localhost', 57637);
 
@@ -398,13 +413,16 @@ stop_session_level_connection_to_node
 
 (1 row)
 
+step teardown:
+ ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+
 citus_remove_node
 ---------------------------------------------------------------------
 
 (1 row)
 
 
-starting permutation: coor-begin coor-acquire-aggresive-lock-on-view w1-start-session-level-connection w1-begin w1-read-dist-table coor-rollback w1-rollback w1-stop-connection
+starting permutation: coor-begin coor-acquire-aggresive-lock-on-view w1-start-session-level-connection w1-begin w1-read-dist-table coor-rollback w1-rollback w1-stop-connection teardown
 step coor-begin:
  BEGIN;
 
@@ -455,13 +473,16 @@ stop_session_level_connection_to_node
 
 (1 row)
 
+step teardown:
+ ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+
 citus_remove_node
 ---------------------------------------------------------------------
 
 (1 row)
 
 
-starting permutation: coor-begin coor-acquire-aggresive-lock-on-view w1-start-session-level-connection w1-begin w1-acquire-aggressive-lock-dist-table coor-rollback w1-rollback w1-stop-connection
+starting permutation: coor-begin coor-acquire-aggresive-lock-on-view w1-start-session-level-connection w1-begin w1-acquire-aggressive-lock-dist-table coor-rollback w1-rollback w1-stop-connection teardown
 step coor-begin:
  BEGIN;
 
@@ -512,13 +533,16 @@ stop_session_level_connection_to_node
 
 (1 row)
 
+step teardown:
+ ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+
 citus_remove_node
 ---------------------------------------------------------------------
 
 (1 row)
 
 
-starting permutation: coor-begin coor-acquire-aggresive-lock-on-view w1-start-session-level-connection w1-begin w1-read-ref-table coor-rollback w1-rollback w1-stop-connection
+starting permutation: coor-begin coor-acquire-aggresive-lock-on-view w1-start-session-level-connection w1-begin w1-read-ref-table coor-rollback w1-rollback w1-stop-connection teardown
 step coor-begin:
  BEGIN;
 
@@ -569,13 +593,16 @@ stop_session_level_connection_to_node
 
 (1 row)
 
+step teardown:
+ ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+
 citus_remove_node
 ---------------------------------------------------------------------
 
 (1 row)
 
 
-starting permutation: coor-begin coor-acquire-aggresive-lock-on-only-view w1-start-session-level-connection w1-begin w1-read-ref-table coor-rollback w1-rollback w1-stop-connection
+starting permutation: coor-begin coor-acquire-aggresive-lock-on-only-view w1-start-session-level-connection w1-begin w1-read-ref-table coor-rollback w1-rollback w1-stop-connection teardown
 step coor-begin:
  BEGIN;
 
@@ -626,13 +653,16 @@ stop_session_level_connection_to_node
 
 (1 row)
 
+step teardown:
+ ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+
 citus_remove_node
 ---------------------------------------------------------------------
 
 (1 row)
 
 
-starting permutation: w1-start-session-level-connection w1-begin w1-acquire-aggressive-lock-dist-table coor-begin coor-acquire-aggresive-lock-on-view-nowait coor-rollback w1-rollback w1-stop-connection
+starting permutation: w1-start-session-level-connection w1-begin w1-acquire-aggressive-lock-dist-table coor-begin coor-acquire-aggresive-lock-on-view-nowait coor-rollback w1-rollback w1-stop-connection teardown
 step w1-start-session-level-connection:
     SELECT start_session_level_connection_to_node('localhost', 57637);
 
@@ -683,13 +713,16 @@ stop_session_level_connection_to_node
 
 (1 row)
 
+step teardown:
+ ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+
 citus_remove_node
 ---------------------------------------------------------------------
 
 (1 row)
 
 
-starting permutation: coor-begin coor-lock-all w1-start-session-level-connection w1-begin w1-read-citus-local-table coor-rollback w1-rollback w1-stop-connection
+starting permutation: coor-begin coor-lock-all w1-start-session-level-connection w1-begin w1-read-citus-local-table coor-rollback w1-rollback w1-stop-connection teardown
 step coor-begin:
  BEGIN;
 
@@ -740,13 +773,16 @@ stop_session_level_connection_to_node
 
 (1 row)
 
+step teardown:
+ ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+
 citus_remove_node
 ---------------------------------------------------------------------
 
 (1 row)
 
 
-starting permutation: coor-begin coor-acquire-aggresive-lock-on-partitioned-table w1-start-session-level-connection w1-begin w1-read-partitioned-table coor-rollback w1-rollback w1-stop-connection
+starting permutation: coor-begin coor-acquire-aggresive-lock-on-partitioned-table w1-start-session-level-connection w1-begin w1-read-partitioned-table coor-rollback w1-rollback w1-stop-connection teardown
 step coor-begin:
  BEGIN;
 
@@ -797,13 +833,16 @@ stop_session_level_connection_to_node
 
 (1 row)
 
+step teardown:
+ ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+
 citus_remove_node
 ---------------------------------------------------------------------
 
 (1 row)
 
 
-starting permutation: coor-begin coor-acquire-aggresive-lock-on-partitioned-table w1-start-session-level-connection w1-begin w1-read-partition-of-partitioned-table coor-rollback w1-rollback w1-stop-connection
+starting permutation: coor-begin coor-acquire-aggresive-lock-on-partitioned-table w1-start-session-level-connection w1-begin w1-read-partition-of-partitioned-table coor-rollback w1-rollback w1-stop-connection teardown
 step coor-begin:
  BEGIN;
 
@@ -854,13 +893,16 @@ stop_session_level_connection_to_node
 
 (1 row)
 
+step teardown:
+ ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+
 citus_remove_node
 ---------------------------------------------------------------------
 
 (1 row)
 
 
-starting permutation: coor-begin coor-acquire-aggresive-lock-on-partitioned-table-with-*-syntax w1-start-session-level-connection w1-begin w1-read-partition-of-partitioned-table coor-rollback w1-rollback w1-stop-connection
+starting permutation: coor-begin coor-acquire-aggresive-lock-on-partitioned-table-with-*-syntax w1-start-session-level-connection w1-begin w1-read-partition-of-partitioned-table coor-rollback w1-rollback w1-stop-connection teardown
 step coor-begin:
  BEGIN;
 
@@ -911,13 +953,16 @@ stop_session_level_connection_to_node
 
 (1 row)
 
+step teardown:
+ ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+
 citus_remove_node
 ---------------------------------------------------------------------
 
 (1 row)
 
 
-starting permutation: coor-begin coor-acquire-aggresive-lock-on-only-partitioned-table w1-start-session-level-connection w1-begin w1-read-partitioned-table coor-rollback w1-rollback w1-stop-connection
+starting permutation: coor-begin coor-acquire-aggresive-lock-on-only-partitioned-table w1-start-session-level-connection w1-begin w1-read-partitioned-table coor-rollback w1-rollback w1-stop-connection teardown
 step coor-begin:
  BEGIN;
 
@@ -968,13 +1013,16 @@ stop_session_level_connection_to_node
 
 (1 row)
 
+step teardown:
+ ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+
 citus_remove_node
 ---------------------------------------------------------------------
 
 (1 row)
 
 
-starting permutation: coor-begin coor-acquire-aggresive-lock-on-only-partitioned-table w1-start-session-level-connection w1-begin w1-read-partition-of-partitioned-table coor-rollback w1-rollback w1-stop-connection
+starting permutation: coor-begin coor-acquire-aggresive-lock-on-only-partitioned-table w1-start-session-level-connection w1-begin w1-read-partition-of-partitioned-table coor-rollback w1-rollback w1-stop-connection teardown
 step coor-begin:
  BEGIN;
 
@@ -1024,13 +1072,16 @@ stop_session_level_connection_to_node
 
 (1 row)
 
+step teardown:
+ ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+
 citus_remove_node
 ---------------------------------------------------------------------
 
 (1 row)
 
 
-starting permutation: coor-begin coor-acquire-aggresive-lock-on-ref-table w1-start-session-level-connection w1-begin w1-read-main-view coor-rollback w1-rollback w1-stop-connection
+starting permutation: coor-begin coor-acquire-aggresive-lock-on-ref-table w1-start-session-level-connection w1-begin w1-read-main-view coor-rollback w1-rollback w1-stop-connection teardown
 step coor-begin:
  BEGIN;
 
@@ -1081,13 +1132,16 @@ stop_session_level_connection_to_node
 
 (1 row)
 
+step teardown:
+ ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+
 citus_remove_node
 ---------------------------------------------------------------------
 
 (1 row)
 
 
-starting permutation: coor-begin coor-read-dist-table w2-start-session-level-connection w2-begin w1-start-session-level-connection w1-begin w2-acquire-aggressive-lock-dist-table w1-acquire-aggressive-lock-dist-table coor-rollback w2-rollback w1-rollback w1-stop-connection w2-stop-connection
+starting permutation: coor-begin coor-read-dist-table w2-start-session-level-connection w2-begin w1-start-session-level-connection w1-begin w2-acquire-aggressive-lock-dist-table w1-acquire-aggressive-lock-dist-table coor-rollback w2-rollback w1-rollback w1-stop-connection w2-stop-connection teardown
 step coor-begin:
  BEGIN;
 
@@ -1184,6 +1238,9 @@ stop_session_level_connection_to_node
 
 (1 row)
 
+step teardown:
+ ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+
 citus_remove_node
 ---------------------------------------------------------------------
 

src/test/regress/expected/isolation_create_distributed_table_concurrently.out

@@ -1,5 +1,5 @@
 unused step name: s1-create-concurrently-table_2
-Parsed test spec with 4 sessions
+Parsed test spec with 5 sessions
 
 starting permutation: s1-truncate s3-acquire-split-advisory-lock s1-settings s2-settings s1-create-concurrently-table_1 s2-begin s2-insert s2-commit s3-release-split-advisory-lock s2-print-status
 step s1-truncate:
@@ -665,7 +665,18 @@ citus_remove_node
 (1 row)
 
 
-starting permutation: s2-begin s2-create-table_2 s1-create-concurrently-table_none_colocated s4-print-waiting-advisory-locks s2-commit s4-print-colocations
+starting permutation: show-all-blocks reload-conf s2-begin s2-create-table_2 s1-create-concurrently-table_none_colocated s4-print-waiting-advisory-locks s2-commit s4-print-colocations reset-show-all-blocks reload-conf
+step show-all-blocks:
+ ALTER SYSTEM SET citus.isolation_test_check_all_blocks = true;
+
+step reload-conf:
+ SELECT pg_reload_conf();
+
+pg_reload_conf
+---------------------------------------------------------------------
+t
+(1 row)
+
 step s2-begin:
  BEGIN;
 
@@ -708,13 +719,35 @@ shardcount|replicationfactor|distributioncolumntype|distributioncolumncollation
          4|                1|                    23|                          0
 (2 rows)
 
+step reset-show-all-blocks:
+ ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+
+step reload-conf:
+ SELECT pg_reload_conf();
+
+pg_reload_conf
+---------------------------------------------------------------------
+t
+(1 row)
+
 citus_remove_node
 ---------------------------------------------------------------------
 
 (1 row)
 
 
-starting permutation: s2-begin s2-create-table_2-none s1-create-concurrently-table_none_colocated s4-print-waiting-advisory-locks s2-commit s4-print-colocations
+starting permutation: show-all-blocks reload-conf s2-begin s2-create-table_2-none s1-create-concurrently-table_none_colocated s4-print-waiting-advisory-locks s2-commit s4-print-colocations reset-show-all-blocks reload-conf
+step show-all-blocks:
+ ALTER SYSTEM SET citus.isolation_test_check_all_blocks = true;
+
+step reload-conf:
+ SELECT pg_reload_conf();
+
+pg_reload_conf
+---------------------------------------------------------------------
+t
+(1 row)
+
 step s2-begin:
  BEGIN;
 
@@ -756,6 +789,17 @@ shardcount|replicationfactor|distributioncolumntype|distributioncolumncollation
          4|                1|                    23|                          0
 (2 rows)
 
+step reset-show-all-blocks:
+ ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+
+step reload-conf:
+ SELECT pg_reload_conf();
+
+pg_reload_conf
+---------------------------------------------------------------------
+t
+(1 row)
+
 citus_remove_node
 ---------------------------------------------------------------------
 

src/test/regress/expected/isolation_get_distributed_wait_queries_mx.out

@@ -1,6 +1,6 @@
-Parsed test spec with 8 sessions
+Parsed test spec with 9 sessions
 
-starting permutation: s1-begin s1-update-ref-table-from-coordinator s2-start-session-level-connection s2-begin-on-worker s2-update-ref-table s3-select-distributed-waiting-queries s1-commit s2-commit-worker s2-stop-connection
+starting permutation: s1-begin s1-update-ref-table-from-coordinator s2-start-session-level-connection s2-begin-on-worker s2-update-ref-table s3-select-distributed-waiting-queries s1-commit s2-commit-worker s2-stop-connection teardown
 step s1-begin:
  BEGIN;
 
@@ -61,13 +61,16 @@ stop_session_level_connection_to_node
 
 (1 row)
 
+step teardown:
+ ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+
 citus_remove_node
 ---------------------------------------------------------------------
 
 (1 row)
 
 
-starting permutation: s1-start-session-level-connection s1-begin-on-worker s1-update-ref-table s2-start-session-level-connection s2-begin-on-worker s2-update-ref-table s3-select-distributed-waiting-queries s1-commit-worker s2-commit-worker s1-stop-connection s2-stop-connection
+starting permutation: s1-start-session-level-connection s1-begin-on-worker s1-update-ref-table s2-start-session-level-connection s2-begin-on-worker s2-update-ref-table s3-select-distributed-waiting-queries s1-commit-worker s2-commit-worker s1-stop-connection s2-stop-connection teardown
 step s1-start-session-level-connection:
  SELECT start_session_level_connection_to_node('localhost', 57637);
 
@@ -157,13 +160,16 @@ stop_session_level_connection_to_node
 
 (1 row)
 
+step teardown:
+ ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+
 citus_remove_node
 ---------------------------------------------------------------------
 
 (1 row)
 
 
-starting permutation: s1-start-session-level-connection s1-begin-on-worker s1-update-dist-table s2-start-session-level-connection s2-begin-on-worker s2-update-dist-table s3-select-distributed-waiting-queries s1-commit-worker s2-commit-worker s1-stop-connection s2-stop-connection
+starting permutation: s1-start-session-level-connection s1-begin-on-worker s1-update-dist-table s2-start-session-level-connection s2-begin-on-worker s2-update-dist-table s3-select-distributed-waiting-queries s1-commit-worker s2-commit-worker s1-stop-connection s2-stop-connection teardown
 step s1-start-session-level-connection:
  SELECT start_session_level_connection_to_node('localhost', 57637);
 
@@ -253,13 +259,16 @@ stop_session_level_connection_to_node
 
 (1 row)
 
+step teardown:
+ ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+
 citus_remove_node
 ---------------------------------------------------------------------
 
 (1 row)
 
 
-starting permutation: s1-start-session-level-connection s1-begin-on-worker s1-delete-from-ref-table s2-start-session-level-connection s2-begin-on-worker s2-update-ref-table s3-select-distributed-waiting-queries s1-commit-worker s2-commit-worker s1-stop-connection s2-stop-connection
+starting permutation: s1-start-session-level-connection s1-begin-on-worker s1-delete-from-ref-table s2-start-session-level-connection s2-begin-on-worker s2-update-ref-table s3-select-distributed-waiting-queries s1-commit-worker s2-commit-worker s1-stop-connection s2-stop-connection teardown
 step s1-start-session-level-connection:
  SELECT start_session_level_connection_to_node('localhost', 57637);
 
@@ -349,13 +358,16 @@ stop_session_level_connection_to_node
 
 (1 row)
 
+step teardown:
+ ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+
 citus_remove_node
 ---------------------------------------------------------------------
 
 (1 row)
 
 
-starting permutation: s1-start-session-level-connection s1-begin-on-worker s1-insert-into-ref-table s2-start-session-level-connection s2-begin-on-worker s2-update-ref-table s3-select-distributed-waiting-queries s1-commit-worker s2-commit-worker s1-stop-connection s2-stop-connection
+starting permutation: s1-start-session-level-connection s1-begin-on-worker s1-insert-into-ref-table s2-start-session-level-connection s2-begin-on-worker s2-update-ref-table s3-select-distributed-waiting-queries s1-commit-worker s2-commit-worker s1-stop-connection s2-stop-connection teardown
 step s1-start-session-level-connection:
  SELECT start_session_level_connection_to_node('localhost', 57637);
 
@@ -445,13 +457,16 @@ stop_session_level_connection_to_node
 
 (1 row)
 
+step teardown:
+ ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+
 citus_remove_node
 ---------------------------------------------------------------------
 
 (1 row)
 
 
-starting permutation: s1-start-session-level-connection s1-begin-on-worker s1-insert-into-ref-table s2-start-session-level-connection s2-begin-on-worker s2-insert-into-ref-table s3-select-distributed-waiting-queries s1-commit-worker s2-commit-worker s1-stop-connection s2-stop-connection
+starting permutation: s1-start-session-level-connection s1-begin-on-worker s1-insert-into-ref-table s2-start-session-level-connection s2-begin-on-worker s2-insert-into-ref-table s3-select-distributed-waiting-queries s1-commit-worker s2-commit-worker s1-stop-connection s2-stop-connection teardown
 step s1-start-session-level-connection:
  SELECT start_session_level_connection_to_node('localhost', 57637);
 
@@ -539,13 +554,16 @@ stop_session_level_connection_to_node
 
 (1 row)
 
+step teardown:
+ ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+
 citus_remove_node
 ---------------------------------------------------------------------
 
 (1 row)
 
 
-starting permutation: s1-start-session-level-connection s1-begin-on-worker s1-copy-to-ref-table s2-start-session-level-connection s2-begin-on-worker s2-update-ref-table s3-select-distributed-waiting-queries s1-commit-worker s2-commit-worker s1-stop-connection s2-stop-connection
+starting permutation: s1-start-session-level-connection s1-begin-on-worker s1-copy-to-ref-table s2-start-session-level-connection s2-begin-on-worker s2-update-ref-table s3-select-distributed-waiting-queries s1-commit-worker s2-commit-worker s1-stop-connection s2-stop-connection teardown
 step s1-start-session-level-connection:
  SELECT start_session_level_connection_to_node('localhost', 57637);
 
@@ -635,13 +653,16 @@ stop_session_level_connection_to_node
 
 (1 row)
 
+step teardown:
+ ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+
 citus_remove_node
 ---------------------------------------------------------------------
 
 (1 row)
 
 
-starting permutation: s1-start-session-level-connection s1-begin-on-worker s1-copy-to-ref-table s2-start-session-level-connection s2-begin-on-worker s2-insert-into-ref-table s3-select-distributed-waiting-queries s1-commit-worker s2-commit-worker s1-stop-connection s2-stop-connection
+starting permutation: s1-start-session-level-connection s1-begin-on-worker s1-copy-to-ref-table s2-start-session-level-connection s2-begin-on-worker s2-insert-into-ref-table s3-select-distributed-waiting-queries s1-commit-worker s2-commit-worker s1-stop-connection s2-stop-connection teardown
 step s1-start-session-level-connection:
  SELECT start_session_level_connection_to_node('localhost', 57637);
 
@@ -729,13 +750,16 @@ stop_session_level_connection_to_node
 
 (1 row)
 
+step teardown:
+ ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+
 citus_remove_node
 ---------------------------------------------------------------------
 
 (1 row)
 
 
-starting permutation: s1-start-session-level-connection s1-begin-on-worker s1-copy-to-ref-table s2-start-session-level-connection s2-begin-on-worker s2-copy-to-ref-table s3-select-distributed-waiting-queries s1-commit-worker s2-commit-worker s1-stop-connection s2-stop-connection
+starting permutation: s1-start-session-level-connection s1-begin-on-worker s1-copy-to-ref-table s2-start-session-level-connection s2-begin-on-worker s2-copy-to-ref-table s3-select-distributed-waiting-queries s1-commit-worker s2-commit-worker s1-stop-connection s2-stop-connection teardown
 step s1-start-session-level-connection:
  SELECT start_session_level_connection_to_node('localhost', 57637);
 
@@ -823,13 +847,16 @@ stop_session_level_connection_to_node
 
 (1 row)
 
+step teardown:
+ ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+
 citus_remove_node
 ---------------------------------------------------------------------
 
 (1 row)
 
 
-starting permutation: s1-start-session-level-connection s1-begin-on-worker s1-select-for-update s2-start-session-level-connection s2-begin-on-worker s2-update-ref-table s3-select-distributed-waiting-queries s1-commit-worker s2-commit-worker s1-stop-connection s2-stop-connection
+starting permutation: s1-start-session-level-connection s1-begin-on-worker s1-select-for-update s2-start-session-level-connection s2-begin-on-worker s2-update-ref-table s3-select-distributed-waiting-queries s1-commit-worker s2-commit-worker s1-stop-connection s2-stop-connection teardown
 step s1-start-session-level-connection:
  SELECT start_session_level_connection_to_node('localhost', 57637);
 
@@ -919,13 +946,16 @@ stop_session_level_connection_to_node
 
 (1 row)
 
+step teardown:
+ ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+
 citus_remove_node
 ---------------------------------------------------------------------
 
 (1 row)
 
 
-starting permutation: s2-start-session-level-connection s2-begin-on-worker s2-insert-into-ref-table s1-begin s1-alter-table s3-select-distributed-waiting-queries s2-commit-worker s1-commit s2-stop-connection
+starting permutation: s2-start-session-level-connection s2-begin-on-worker s2-insert-into-ref-table s1-begin s1-alter-table s3-select-distributed-waiting-queries s2-commit-worker s1-commit s2-stop-connection teardown
 step s2-start-session-level-connection:
  SELECT start_session_level_connection_to_node('localhost', 57638);
 
@@ -986,13 +1016,16 @@ stop_session_level_connection_to_node
 
 (1 row)
 
+step teardown:
+ ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+
 citus_remove_node
 ---------------------------------------------------------------------
 
 (1 row)
 
 
-starting permutation: s1-begin s1-update-on-the-coordinator s2-update-on-the-coordinator s3-select-distributed-waiting-queries s1-commit
+starting permutation: s1-begin s1-update-on-the-coordinator s2-update-on-the-coordinator s3-select-distributed-waiting-queries s1-commit teardown
 step s1-begin:
  BEGIN;
 
@@ -1018,13 +1051,16 @@ step s1-commit:
  COMMIT;
 
 step s2-update-on-the-coordinator: <... completed>
+step teardown:
+ ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+
 citus_remove_node
 ---------------------------------------------------------------------
 
 (1 row)
 
 
-starting permutation: s1-start-session-level-connection s1-begin-on-worker s1-update-dist-table s4-start-session-level-connection s4-begin-on-worker s4-update-dist-table s3-select-distributed-waiting-queries s1-commit-worker s4-commit-worker s1-stop-connection s4-stop-connection
+starting permutation: s1-start-session-level-connection s1-begin-on-worker s1-update-dist-table s4-start-session-level-connection s4-begin-on-worker s4-update-dist-table s3-select-distributed-waiting-queries s1-commit-worker s4-commit-worker s1-stop-connection s4-stop-connection teardown
 step s1-start-session-level-connection:
  SELECT start_session_level_connection_to_node('localhost', 57637);
 
@@ -1114,13 +1150,16 @@ stop_session_level_connection_to_node
 
 (1 row)
 
+step teardown:
+ ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+
 citus_remove_node
 ---------------------------------------------------------------------
 
 (1 row)
 
 
-starting permutation: s1-start-session-level-connection s1-begin-on-worker s1-update-dist-table-id-1 s2-start-session-level-connection s2-update-dist-table-id-1 s3-select-distributed-waiting-queries s1-commit-worker s1-stop-connection s2-stop-connection
+starting permutation: s1-start-session-level-connection s1-begin-on-worker s1-update-dist-table-id-1 s2-start-session-level-connection s2-update-dist-table-id-1 s3-select-distributed-waiting-queries s1-commit-worker s1-stop-connection s2-stop-connection teardown
 step s1-start-session-level-connection:
  SELECT start_session_level_connection_to_node('localhost', 57637);
 
@@ -1194,13 +1233,16 @@ stop_session_level_connection_to_node
 
 (1 row)
 
+step teardown:
+ ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+
 citus_remove_node
 ---------------------------------------------------------------------
 
 (1 row)
 
 
-starting permutation: s1-begin s1-update-ref-table-from-coordinator s2-start-session-level-connection s2-update-ref-table s3-select-distributed-waiting-queries s1-commit s2-stop-connection
+starting permutation: s1-begin s1-update-ref-table-from-coordinator s2-start-session-level-connection s2-update-ref-table s3-select-distributed-waiting-queries s1-commit s2-stop-connection teardown
 step s1-begin:
  BEGIN;
 
@@ -1245,13 +1287,16 @@ stop_session_level_connection_to_node
 
 (1 row)
 
+step teardown:
+ ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+
 citus_remove_node
 ---------------------------------------------------------------------
 
 (1 row)
 
 
-starting permutation: s5-begin s5-alter s6-select s3-select-distributed-waiting-queries s5-rollback
+starting permutation: s5-begin s5-alter s6-select s3-select-distributed-waiting-queries s5-rollback teardown
 step s5-begin:
  BEGIN;
 
@@ -1282,13 +1327,16 @@ user_id
       7
 (1 row)
 
+step teardown:
+ ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+
 citus_remove_node
 ---------------------------------------------------------------------
 
 (1 row)
 
 
-starting permutation: s8-begin s8-select s7-alter s3-select-distributed-waiting-queries s3-show-actual-gpids s8-rollback
+starting permutation: s8-begin s8-select s7-alter s3-select-distributed-waiting-queries s3-show-actual-gpids s8-rollback teardown
 step s8-begin:
  BEGIN;
 
@@ -1329,6 +1377,9 @@ step s8-rollback:
  ROLLBACK;
 
 step s7-alter: <... completed>
+step teardown:
+ ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+
 citus_remove_node
 ---------------------------------------------------------------------
 

src/test/regress/expected/isolation_replicate_reference_tables_to_coordinator.out

@@ -1,6 +1,6 @@
 Parsed test spec with 4 sessions
 
-starting permutation: add-node s1-begin s2-begin s1-update-dist-table s2-lock-ref-table-placement-on-coordinator s1-lock-ref-table-placement-on-coordinator s2-update-dist-table deadlock-checker-call s1-end s2-end
+starting permutation: add-node s1-begin s2-begin s1-update-dist-table s2-lock-ref-table-placement-on-coordinator s1-lock-ref-table-placement-on-coordinator show-all-blocks reload-conf s2-update-dist-table deadlock-checker-call s1-end s2-end reset-show-all-blocks reload-conf
 create_distributed_table
 ---------------------------------------------------------------------
 
@@ -41,6 +41,17 @@ step s1-lock-ref-table-placement-on-coordinator:
       END
     $$;
  <waiting ...>
+step show-all-blocks: 
+ ALTER SYSTEM SET citus.isolation_test_check_all_blocks = true;
+
+step reload-conf:
+ SELECT pg_reload_conf();
+
+pg_reload_conf
+---------------------------------------------------------------------
+t
+(1 row)
+
 step s2-update-dist-table:
     update dist_table set b = 2 where a = 1;
  <waiting ...>
@@ -61,6 +72,17 @@ step s1-end:
 step s2-end:
     END;
 
+step reset-show-all-blocks:
+ ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+
+step reload-conf:
+ SELECT pg_reload_conf();
+
+pg_reload_conf
+---------------------------------------------------------------------
+t
+(1 row)
+
 master_remove_node
 ---------------------------------------------------------------------
 
@@ -104,11 +126,11 @@ step s2-view-dist:
 query                                |state              |wait_event_type|wait_event|usename |datname
 ---------------------------------------------------------------------
 
-  SELECT check_distributed_deadlocks();
+ SELECT pg_reload_conf();
-|idle               |Client         |ClientRead|postgres|regression
+          |idle               |Client         |ClientRead|postgres|regression
 
     update ref_table set a = a + 1;
-    |idle in transaction|Client         |ClientRead|postgres|regression
+|idle in transaction|Client         |ClientRead|postgres|regression
 (2 rows)
 
 step s2-view-worker:

src/test/regress/expected/multi_extension.out

@@ -1201,9 +1201,11 @@ ALTER EXTENSION citus UPDATE TO '11.2-1';
 SELECT * FROM multi_extension.print_extension_changes();
                                                                                                                                     previous_object                                                                                                                                     |                                                                                                                                                        current_object
 ---------------------------------------------------------------------
+ function citus_backend_gpid() bigint                                                                                                                                                                                                                                                   |
  function get_rebalance_progress() TABLE(sessionid integer, table_name regclass, shardid bigint, shard_size bigint, sourcename text, sourceport integer, targetname text, targetport integer, progress bigint, source_shard_size bigint, target_shard_size bigint, operation_type text) |
+                                                                                                                                                                                                                                                                                        | function citus_backend_gpid(integer) bigint
                                                                                                                                                                                                                                                                                         | function get_rebalance_progress() TABLE(sessionid integer, table_name regclass, shardid bigint, shard_size bigint, sourcename text, sourceport integer, targetname text, targetport integer, progress bigint, source_shard_size bigint, target_shard_size bigint, operation_type text, source_lsn pg_lsn, target_lsn pg_lsn)
-(2 rows)
+(4 rows)
 
 DROP TABLE multi_extension.prev_objects, multi_extension.extension_diff;
 -- show running version

src/test/regress/expected/upgrade_list_citus_objects.out

@@ -26,7 +26,7 @@ ORDER BY 1;
  function citus_add_node(text,integer,integer,noderole,name)
  function citus_add_rebalance_strategy(name,regproc,regproc,regproc,real,real,real)
  function citus_add_secondary_node(text,integer,text,integer,name)
- function citus_backend_gpid()
+ function citus_backend_gpid(integer)
  function citus_blocking_pids(integer)
  function citus_calculate_gpid(integer,integer)
  function citus_check_cluster_node_health()

src/test/regress/spec/isolation_acquire_distributed_locks.spec

@@ -2,6 +2,11 @@
 
 setup
 {
+	ALTER SYSTEM SET citus.isolation_test_check_all_blocks = true;
+}
+setup
+{
+	SELECT pg_reload_conf();
 	SELECT citus_set_coordinator_host('localhost', 57636);
 
 	CREATE TABLE dist_table(a int);
@@ -42,6 +47,7 @@ setup
 
 teardown
 {
+	SELECT pg_reload_conf();
 	DROP VIEW main_view;
 	DROP VIEW sub_view;
 	DROP TABLE dist_table;
@@ -220,22 +226,29 @@ step "w2-stop-connection"
 	SELECT stop_session_level_connection_to_node();
 }
 
-permutation "coor-begin" "coor-acquire-aggresive-lock-on-dist-table" "w1-start-session-level-connection" "w1-begin" "w1-read-dist-table" "coor-rollback" "w1-rollback" "w1-stop-connection"
+session "admin"
-permutation "coor-begin" "coor-acquire-aggresive-lock-on-dist-table" "w1-start-session-level-connection" "w1-begin" "w1-acquire-aggressive-lock-dist-table" "coor-rollback" "coor-read-dist-table" "w1-rollback" "w1-stop-connection"
+
-permutation "w1-start-session-level-connection" "w1-begin" "w1-acquire-aggressive-lock-dist-table" "coor-begin" "coor-acquire-aggresive-lock-on-dist-table-nowait" "coor-rollback" "w1-rollback" "w1-stop-connection"
+step "teardown"
-permutation "w1-start-session-level-connection" "w1-begin" "w2-start-session-level-connection" "w2-begin" "w1-acquire-aggressive-lock-dist-table" "w2-acquire-aggressive-lock-dist-table" "w1-rollback" "w1-read-dist-table" "w2-rollback" "w1-stop-connection" "w2-stop-connection"
+{
-permutation "coor-begin" "coor-acquire-weak-lock-on-dist-table" "w1-start-session-level-connection" "w1-begin" "w1-read-dist-table" "w1-acquire-aggressive-lock-dist-table" "coor-rollback" "w1-rollback" "w1-stop-connection"
+	ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
-permutation "w1-start-session-level-connection" "w1-begin" "w1-lock-reference-table" "coor-begin" "coor-read-ref-table" "w1-rollback" "coor-rollback" "w1-stop-connection"
+}
-permutation "coor-begin" "coor-acquire-aggresive-lock-on-view" "w1-start-session-level-connection" "w1-begin" "w1-read-dist-table" "coor-rollback" "w1-rollback" "w1-stop-connection"
+
-permutation "coor-begin" "coor-acquire-aggresive-lock-on-view" "w1-start-session-level-connection" "w1-begin" "w1-acquire-aggressive-lock-dist-table" "coor-rollback" "w1-rollback" "w1-stop-connection"
+permutation "coor-begin" "coor-acquire-aggresive-lock-on-dist-table" "w1-start-session-level-connection" "w1-begin" "w1-read-dist-table" "coor-rollback" "w1-rollback" "w1-stop-connection" "teardown"
-permutation "coor-begin" "coor-acquire-aggresive-lock-on-view" "w1-start-session-level-connection" "w1-begin" "w1-read-ref-table" "coor-rollback" "w1-rollback" "w1-stop-connection"
+permutation "coor-begin" "coor-acquire-aggresive-lock-on-dist-table" "w1-start-session-level-connection" "w1-begin" "w1-acquire-aggressive-lock-dist-table" "coor-rollback" "coor-read-dist-table" "w1-rollback" "w1-stop-connection" "teardown"
-permutation "coor-begin" "coor-acquire-aggresive-lock-on-only-view" "w1-start-session-level-connection" "w1-begin" "w1-read-ref-table" "coor-rollback" "w1-rollback" "w1-stop-connection"
+permutation "w1-start-session-level-connection" "w1-begin" "w1-acquire-aggressive-lock-dist-table" "coor-begin" "coor-acquire-aggresive-lock-on-dist-table-nowait" "coor-rollback" "w1-rollback" "w1-stop-connection" "teardown"
-permutation "w1-start-session-level-connection" "w1-begin" "w1-acquire-aggressive-lock-dist-table" "coor-begin" "coor-acquire-aggresive-lock-on-view-nowait" "coor-rollback" "w1-rollback" "w1-stop-connection"
+permutation "w1-start-session-level-connection" "w1-begin" "w2-start-session-level-connection" "w2-begin" "w1-acquire-aggressive-lock-dist-table" "w2-acquire-aggressive-lock-dist-table" "w1-rollback" "w1-read-dist-table" "w2-rollback" "w1-stop-connection" "w2-stop-connection" "teardown"
-permutation "coor-begin" "coor-lock-all" "w1-start-session-level-connection" "w1-begin" "w1-read-citus-local-table" "coor-rollback" "w1-rollback" "w1-stop-connection"
+permutation "coor-begin" "coor-acquire-weak-lock-on-dist-table" "w1-start-session-level-connection" "w1-begin" "w1-read-dist-table" "w1-acquire-aggressive-lock-dist-table" "coor-rollback" "w1-rollback" "w1-stop-connection" "teardown"
-permutation "coor-begin" "coor-acquire-aggresive-lock-on-partitioned-table" "w1-start-session-level-connection" "w1-begin" "w1-read-partitioned-table" "coor-rollback" "w1-rollback" "w1-stop-connection"
+permutation "w1-start-session-level-connection" "w1-begin" "w1-lock-reference-table" "coor-begin" "coor-read-ref-table" "w1-rollback" "coor-rollback" "w1-stop-connection" "teardown"
-permutation "coor-begin" "coor-acquire-aggresive-lock-on-partitioned-table" "w1-start-session-level-connection" "w1-begin" "w1-read-partition-of-partitioned-table" "coor-rollback" "w1-rollback" "w1-stop-connection"
+permutation "coor-begin" "coor-acquire-aggresive-lock-on-view" "w1-start-session-level-connection" "w1-begin" "w1-read-dist-table" "coor-rollback" "w1-rollback" "w1-stop-connection" "teardown"
-permutation "coor-begin" "coor-acquire-aggresive-lock-on-partitioned-table-with-*-syntax" "w1-start-session-level-connection" "w1-begin" "w1-read-partition-of-partitioned-table" "coor-rollback" "w1-rollback" "w1-stop-connection"
+permutation "coor-begin" "coor-acquire-aggresive-lock-on-view" "w1-start-session-level-connection" "w1-begin" "w1-acquire-aggressive-lock-dist-table" "coor-rollback" "w1-rollback" "w1-stop-connection" "teardown"
-permutation "coor-begin" "coor-acquire-aggresive-lock-on-only-partitioned-table" "w1-start-session-level-connection" "w1-begin" "w1-read-partitioned-table" "coor-rollback" "w1-rollback" "w1-stop-connection"
+permutation "coor-begin" "coor-acquire-aggresive-lock-on-view" "w1-start-session-level-connection" "w1-begin" "w1-read-ref-table" "coor-rollback" "w1-rollback" "w1-stop-connection" "teardown"
-permutation "coor-begin" "coor-acquire-aggresive-lock-on-only-partitioned-table" "w1-start-session-level-connection" "w1-begin" "w1-read-partition-of-partitioned-table" "coor-rollback" "w1-rollback" "w1-stop-connection"
+permutation "coor-begin" "coor-acquire-aggresive-lock-on-only-view" "w1-start-session-level-connection" "w1-begin" "w1-read-ref-table" "coor-rollback" "w1-rollback" "w1-stop-connection" "teardown"
-permutation "coor-begin" "coor-acquire-aggresive-lock-on-ref-table" "w1-start-session-level-connection" "w1-begin" "w1-read-main-view" "coor-rollback" "w1-rollback" "w1-stop-connection"
+permutation "w1-start-session-level-connection" "w1-begin" "w1-acquire-aggressive-lock-dist-table" "coor-begin" "coor-acquire-aggresive-lock-on-view-nowait" "coor-rollback" "w1-rollback" "w1-stop-connection" "teardown"
-permutation "coor-begin" "coor-read-dist-table" "w2-start-session-level-connection" "w2-begin" "w1-start-session-level-connection" "w1-begin" "w2-acquire-aggressive-lock-dist-table" "w1-acquire-aggressive-lock-dist-table" "coor-rollback" "w2-rollback" "w1-rollback" "w1-stop-connection" "w2-stop-connection"
+permutation "coor-begin" "coor-lock-all" "w1-start-session-level-connection" "w1-begin" "w1-read-citus-local-table" "coor-rollback" "w1-rollback" "w1-stop-connection" "teardown"
+permutation "coor-begin" "coor-acquire-aggresive-lock-on-partitioned-table" "w1-start-session-level-connection" "w1-begin" "w1-read-partitioned-table" "coor-rollback" "w1-rollback" "w1-stop-connection" "teardown"
+permutation "coor-begin" "coor-acquire-aggresive-lock-on-partitioned-table" "w1-start-session-level-connection" "w1-begin" "w1-read-partition-of-partitioned-table" "coor-rollback" "w1-rollback" "w1-stop-connection" "teardown"
+permutation "coor-begin" "coor-acquire-aggresive-lock-on-partitioned-table-with-*-syntax" "w1-start-session-level-connection" "w1-begin" "w1-read-partition-of-partitioned-table" "coor-rollback" "w1-rollback" "w1-stop-connection" "teardown"
+permutation "coor-begin" "coor-acquire-aggresive-lock-on-only-partitioned-table" "w1-start-session-level-connection" "w1-begin" "w1-read-partitioned-table" "coor-rollback" "w1-rollback" "w1-stop-connection" "teardown"
+permutation "coor-begin" "coor-acquire-aggresive-lock-on-only-partitioned-table" "w1-start-session-level-connection" "w1-begin" "w1-read-partition-of-partitioned-table" "coor-rollback" "w1-rollback" "w1-stop-connection" "teardown"
+permutation "coor-begin" "coor-acquire-aggresive-lock-on-ref-table" "w1-start-session-level-connection" "w1-begin" "w1-read-main-view" "coor-rollback" "w1-rollback" "w1-stop-connection" "teardown"
+permutation "coor-begin" "coor-read-dist-table" "w2-start-session-level-connection" "w2-begin" "w1-start-session-level-connection" "w1-begin" "w2-acquire-aggressive-lock-dist-table" "w1-acquire-aggressive-lock-dist-table" "coor-rollback" "w2-rollback" "w1-rollback" "w1-stop-connection" "w2-stop-connection" "teardown"

src/test/regress/spec/isolation_create_distributed_table_concurrently.spec

@@ -1,5 +1,6 @@
 setup
 {
+	select setval('pg_dist_shardid_seq', GREATEST(1400292, nextval('pg_dist_shardid_seq')-1));
 	-- make sure coordinator is in metadata
 	SELECT citus_set_coordinator_host('localhost', 57636);
   	CREATE TABLE table_1(id int PRIMARY KEY);
@@ -172,6 +173,21 @@ step "s4-print-colocations"
 	SELECT shardcount, replicationfactor, distributioncolumntype, distributioncolumncollation FROM pg_dist_colocation ORDER BY colocationid;
 }
 
+session "admin"
+step "show-all-blocks"
+{
+	ALTER SYSTEM SET citus.isolation_test_check_all_blocks = true;
+}
+step "reset-show-all-blocks"
+{
+	ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+}
+step "reload-conf"
+{
+	SELECT pg_reload_conf();
+}
+
+
 // show concurrent insert is NOT blocked by create_distributed_table_concurrently
 permutation "s1-truncate" "s3-acquire-split-advisory-lock" "s1-settings" "s2-settings" "s1-create-concurrently-table_1" "s2-begin" "s2-insert" "s2-commit" "s3-release-split-advisory-lock" "s2-print-status"
 
@@ -215,7 +231,7 @@ permutation "s2-begin" "s2-create-table_2" "s1-create-concurrently-table_default
 permutation "s1-create-concurrently-table_default_colocated" "s3-acquire-split-advisory-lock" "s1-create-concurrently-table_1" "s2-create-table_2" "s4-print-waiting-advisory-locks" "s3-release-split-advisory-lock" "s4-print-colocations"
 
 // show concurrent colocate_with => 'default' and colocate_with => 'none' are allowed.
-permutation "s2-begin" "s2-create-table_2" "s1-create-concurrently-table_none_colocated" "s4-print-waiting-advisory-locks" "s2-commit" "s4-print-colocations"
+permutation "show-all-blocks" "reload-conf" "s2-begin" "s2-create-table_2" "s1-create-concurrently-table_none_colocated" "s4-print-waiting-advisory-locks" "s2-commit" "s4-print-colocations" "reset-show-all-blocks" "reload-conf"
 
 // show concurrent colocate_with => 'none' and colocate_with => 'none' are allowed.
-permutation "s2-begin" "s2-create-table_2-none" "s1-create-concurrently-table_none_colocated" "s4-print-waiting-advisory-locks" "s2-commit" "s4-print-colocations"
+permutation "show-all-blocks" "reload-conf" "s2-begin" "s2-create-table_2-none" "s1-create-concurrently-table_none_colocated" "s4-print-waiting-advisory-locks" "s2-commit" "s4-print-colocations" "reset-show-all-blocks" "reload-conf"

src/test/regress/spec/isolation_get_distributed_wait_queries_mx.spec

@@ -1,6 +1,11 @@
 #include "isolation_mx_common.include.spec"
 
+setup
+{
+	ALTER SYSTEM SET citus.isolation_test_check_all_blocks = true;
+}
 setup {
+	SELECT pg_reload_conf();
 	SELECT citus_add_node('localhost', 57636, groupid:=0);
 	CREATE TABLE ref_table(user_id int, value_1 int);
 	SELECT create_reference_table('ref_table');
@@ -15,6 +20,7 @@ setup {
 // back to the initial state.
 teardown
 {
+	SELECT pg_reload_conf();
 	DROP TABLE ref_table;
 	DROP TABLE tt1;
 	SELECT citus_remove_node('localhost', 57636);
@@ -249,34 +255,42 @@ step "s8-rollback"
 	ROLLBACK;
 }
 
-permutation "s1-begin" "s1-update-ref-table-from-coordinator" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-update-ref-table" "s3-select-distributed-waiting-queries" "s1-commit" "s2-commit-worker" "s2-stop-connection"
+session "admin"
-permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-update-ref-table" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-update-ref-table" "s3-select-distributed-waiting-queries" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection"
+
-permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-update-dist-table" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-update-dist-table" "s3-select-distributed-waiting-queries" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection"
+step "teardown"
-permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-delete-from-ref-table" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-update-ref-table" "s3-select-distributed-waiting-queries" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection"
+{
-permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-insert-into-ref-table" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-update-ref-table" "s3-select-distributed-waiting-queries" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection"
+	ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
-permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-insert-into-ref-table" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-insert-into-ref-table" "s3-select-distributed-waiting-queries" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection"
+}
-permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-copy-to-ref-table" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-update-ref-table" "s3-select-distributed-waiting-queries" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection"
+
-permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-copy-to-ref-table" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-insert-into-ref-table" "s3-select-distributed-waiting-queries" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection"
+
-permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-copy-to-ref-table" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-copy-to-ref-table" "s3-select-distributed-waiting-queries" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection"
+permutation "s1-begin" "s1-update-ref-table-from-coordinator" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-update-ref-table" "s3-select-distributed-waiting-queries" "s1-commit" "s2-commit-worker" "s2-stop-connection" "teardown"
-permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-select-for-update" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-update-ref-table" "s3-select-distributed-waiting-queries" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection"
+permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-update-ref-table" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-update-ref-table" "s3-select-distributed-waiting-queries" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection" "teardown"
-permutation "s2-start-session-level-connection" "s2-begin-on-worker" "s2-insert-into-ref-table" "s1-begin" "s1-alter-table" "s3-select-distributed-waiting-queries" "s2-commit-worker" "s1-commit" "s2-stop-connection"
+permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-update-dist-table" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-update-dist-table" "s3-select-distributed-waiting-queries" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection" "teardown"
+permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-delete-from-ref-table" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-update-ref-table" "s3-select-distributed-waiting-queries" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection" "teardown"
+permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-insert-into-ref-table" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-update-ref-table" "s3-select-distributed-waiting-queries" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection" "teardown"
+permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-insert-into-ref-table" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-insert-into-ref-table" "s3-select-distributed-waiting-queries" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection" "teardown"
+permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-copy-to-ref-table" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-update-ref-table" "s3-select-distributed-waiting-queries" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection" "teardown"
+permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-copy-to-ref-table" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-insert-into-ref-table" "s3-select-distributed-waiting-queries" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection" "teardown"
+permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-copy-to-ref-table" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-copy-to-ref-table" "s3-select-distributed-waiting-queries" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection" "teardown"
+permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-select-for-update" "s2-start-session-level-connection" "s2-begin-on-worker" "s2-update-ref-table" "s3-select-distributed-waiting-queries" "s1-commit-worker" "s2-commit-worker" "s1-stop-connection" "s2-stop-connection" "teardown"
+permutation "s2-start-session-level-connection" "s2-begin-on-worker" "s2-insert-into-ref-table" "s1-begin" "s1-alter-table" "s3-select-distributed-waiting-queries" "s2-commit-worker" "s1-commit" "s2-stop-connection" "teardown"
 
 // make sure that multi-shard modification queries
 // show up in the waiting processes even if they are
 // blocked on the same node
-permutation "s1-begin" "s1-update-on-the-coordinator" "s2-update-on-the-coordinator" "s3-select-distributed-waiting-queries" "s1-commit"
+permutation "s1-begin" "s1-update-on-the-coordinator" "s2-update-on-the-coordinator" "s3-select-distributed-waiting-queries" "s1-commit" "teardown"
-permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-update-dist-table" "s4-start-session-level-connection" "s4-begin-on-worker" "s4-update-dist-table" "s3-select-distributed-waiting-queries" "s1-commit-worker" "s4-commit-worker" "s1-stop-connection" "s4-stop-connection"
+permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-update-dist-table" "s4-start-session-level-connection" "s4-begin-on-worker" "s4-update-dist-table" "s3-select-distributed-waiting-queries" "s1-commit-worker" "s4-commit-worker" "s1-stop-connection" "s4-stop-connection" "teardown"
 
 
 // show that even if the commands are not in a transaction block
 // we can find the blocking relationship
-permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-update-dist-table-id-1" "s2-start-session-level-connection" "s2-update-dist-table-id-1" "s3-select-distributed-waiting-queries" "s1-commit-worker" "s1-stop-connection" "s2-stop-connection"
+permutation "s1-start-session-level-connection" "s1-begin-on-worker" "s1-update-dist-table-id-1" "s2-start-session-level-connection" "s2-update-dist-table-id-1" "s3-select-distributed-waiting-queries" "s1-commit-worker" "s1-stop-connection" "s2-stop-connection" "teardown"
-permutation "s1-begin" "s1-update-ref-table-from-coordinator" "s2-start-session-level-connection" "s2-update-ref-table" "s3-select-distributed-waiting-queries" "s1-commit" "s2-stop-connection"
+permutation "s1-begin" "s1-update-ref-table-from-coordinator" "s2-start-session-level-connection" "s2-update-ref-table" "s3-select-distributed-waiting-queries" "s1-commit" "s2-stop-connection" "teardown"
 
 // show that we can see blocking activity even if these are the first commands in the sessions
 // such that global_pids have not been assigned
 // in the first permutation, we would normally have s3-show-actual-gpids and it'd show the gpid has NOT been assigned
 // however, as of PG commit 3f323956128ff8589ce4d3a14e8b950837831803, isolation tester sends set application_name command
 // even before we can do anything on the session. That's why we removed s3-show-actual-gpids, but still useful to show waiting queries
-permutation "s5-begin" "s5-alter" "s6-select" "s3-select-distributed-waiting-queries" "s5-rollback"
+permutation "s5-begin" "s5-alter" "s6-select" "s3-select-distributed-waiting-queries" "s5-rollback" "teardown"
-permutation "s8-begin" "s8-select" "s7-alter" "s3-select-distributed-waiting-queries" "s3-show-actual-gpids" "s8-rollback"
+permutation "s8-begin" "s8-select" "s7-alter" "s3-select-distributed-waiting-queries" "s3-show-actual-gpids" "s8-rollback" "teardown"

src/test/regress/spec/isolation_replicate_reference_tables_to_coordinator.spec

@@ -125,6 +125,20 @@ step "deadlock-checker-call"
   SELECT check_distributed_deadlocks();
 }
 
+step "show-all-blocks"
+{
+	ALTER SYSTEM SET citus.isolation_test_check_all_blocks = true;
+}
+step "reset-show-all-blocks"
+{
+	ALTER SYSTEM RESET citus.isolation_test_check_all_blocks;
+}
+step "reload-conf"
+{
+	SELECT pg_reload_conf();
+}
+
+
 
 // adding node in setup stage prevents getting a gpid with proper nodeid
 session "add-node"
@@ -138,7 +152,7 @@ step "add-node"
 
 // verify that locks on the placement of the reference table on the coordinator is
 // taken into account when looking for distributed deadlocks
-permutation "add-node" "s1-begin" "s2-begin" "s1-update-dist-table" "s2-lock-ref-table-placement-on-coordinator" "s1-lock-ref-table-placement-on-coordinator" "s2-update-dist-table" ("s1-lock-ref-table-placement-on-coordinator") "deadlock-checker-call" "s1-end" "s2-end"
+permutation "add-node" "s1-begin" "s2-begin" "s1-update-dist-table" "s2-lock-ref-table-placement-on-coordinator" "s1-lock-ref-table-placement-on-coordinator" "show-all-blocks" "reload-conf" "s2-update-dist-table" ("s1-lock-ref-table-placement-on-coordinator") "deadlock-checker-call" "s1-end" "s2-end" "reset-show-all-blocks" "reload-conf"
 
 // verify that *_dist_stat_activity() functions return the correct result when query
 // has a task on the coordinator.