Turn metadata sync on in isolation_update_node and isolation_update_node_lock_writes tests (#5779)

src/test/regress/expected/isolation_metadata_sync_vs_all.out

@@ -290,7 +290,7 @@ step s3-compare-snapshot:
 
 same_metadata_in_workers
 ---------------------------------------------------------------------
-f
+t
 (1 row)
 
 step s2-drop-schema:

src/test/regress/expected/isolation_turn_mx_off.out

@@ -1,21 +0,0 @@
-Parsed test spec with 1 sessions
-
-starting permutation: disable-mx-by-default reload stop-metadata-sync
-step disable-mx-by-default:
-  ALTER SYSTEM SET citus.enable_metadata_sync TO OFF;
-
-step reload:
-  SELECT pg_reload_conf();
-
-pg_reload_conf
----------------------------------------------------------------------
-t
-(1 row)
-
-step stop-metadata-sync:
-  SELECT stop_metadata_sync_to_node(nodename, nodeport) FROM pg_dist_node WHERE isactive = 't' and noderole = 'primary';
-
-stop_metadata_sync_to_node
----------------------------------------------------------------------
-(0 rows)
-

src/test/regress/expected/isolation_turn_mx_off_0.out

@@ -1,23 +0,0 @@
-Parsed test spec with 1 sessions
-
-starting permutation: disable-mx-by-default reload stop-metadata-sync
-step disable-mx-by-default:
-  ALTER SYSTEM SET citus.enable_metadata_sync TO OFF;
-
-step reload:
-  SELECT pg_reload_conf();
-
-pg_reload_conf
----------------------------------------------------------------------
-t
-(1 row)
-
-step stop-metadata-sync:
-  SELECT stop_metadata_sync_to_node(nodename, nodeport) FROM pg_dist_node WHERE isactive = 't' and noderole = 'primary';
-
-stop_metadata_sync_to_node
----------------------------------------------------------------------
-
-
-(2 rows)
-

src/test/regress/expected/isolation_turn_mx_on.out

@@ -1,21 +0,0 @@
-Parsed test spec with 1 sessions
-
-starting permutation: enable-mx-by-default reload start-metadata-sync
-step enable-mx-by-default:
-  ALTER SYSTEM SET citus.enable_metadata_sync TO ON;
-
-step reload:
-  SELECT pg_reload_conf();
-
-pg_reload_conf
----------------------------------------------------------------------
-t
-(1 row)
-
-step start-metadata-sync:
-  SELECT start_metadata_sync_to_node(nodename, nodeport) FROM pg_dist_node WHERE isactive = 't' and noderole = 'primary';
-
-start_metadata_sync_to_node
----------------------------------------------------------------------
-(0 rows)
-

src/test/regress/expected/isolation_turn_mx_on_0.out

@@ -1,23 +0,0 @@
-Parsed test spec with 1 sessions
-
-starting permutation: enable-mx-by-default reload start-metadata-sync
-step enable-mx-by-default:
-  ALTER SYSTEM SET citus.enable_metadata_sync TO ON;
-
-step reload:
-  SELECT pg_reload_conf();
-
-pg_reload_conf
----------------------------------------------------------------------
-t
-(1 row)
-
-step start-metadata-sync:
-  SELECT start_metadata_sync_to_node(nodename, nodeport) FROM pg_dist_node WHERE isactive = 't' and noderole = 'primary';
-
-start_metadata_sync_to_node
----------------------------------------------------------------------
-
-
-(2 rows)
-

src/test/regress/expected/isolation_update_node.out

@@ -1,6 +1,6 @@
-Parsed test spec with 2 sessions
+Parsed test spec with 3 sessions
 
-starting permutation: s1-begin s1-update-node-1 s2-update-node-2 s1-commit s1-show-nodes
+starting permutation: s1-begin s1-update-node-1 s2-update-node-2 s1-commit s1-show-nodes s3-update-node-1-back s3-update-node-2-back s3-manually-fix-metadata
 nodeid|nodename |nodeport
 ---------------------------------------------------------------------
     22|localhost|   57638
@@ -47,12 +47,50 @@ nodeid|nodename |nodeport|isactive
     22|localhost|   58638|t
 (2 rows)
 
+step s3-update-node-1-back:
+    SELECT 1 FROM master_update_node(
+        (select nodeid from pg_dist_node where nodeport = 58637),
+        'localhost',
+        57637);
+
+?column?
+---------------------------------------------------------------------
+       1
+(1 row)
+
+step s3-update-node-2-back:
+    SELECT 1 FROM master_update_node(
+        (select nodeid from pg_dist_node where nodeport = 58638),
+        'localhost',
+        57638);
+
+?column?
+---------------------------------------------------------------------
+       1
+(1 row)
+
+step s3-manually-fix-metadata:
+    UPDATE pg_dist_node SET metadatasynced = 't' WHERE nodeport = 57637;
+    UPDATE pg_dist_node SET metadatasynced = 't' WHERE nodeport = 57638;
+    SELECT start_metadata_sync_to_node('localhost', 57637);
+    SELECT start_metadata_sync_to_node('localhost', 57638);
+
+start_metadata_sync_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
+start_metadata_sync_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
 nodeid|nodename|nodeport
 ---------------------------------------------------------------------
 (0 rows)
 
 
-starting permutation: s1-begin s1-update-node-1 s2-begin s2-update-node-1 s1-commit s2-abort s1-show-nodes
+starting permutation: s1-begin s1-update-node-1 s2-begin s2-update-node-1 s1-commit s2-abort s1-show-nodes s3-update-node-1-back s3-manually-fix-metadata
 nodeid|nodename |nodeport
 ---------------------------------------------------------------------
     24|localhost|   57638
@@ -105,60 +143,31 @@ nodeid|nodename |nodeport|isactive
     23|localhost|   58637|t
 (2 rows)
 
-nodeid|nodename|nodeport
+step s3-update-node-1-back:
----------------------------------------------------------------------
-(0 rows)
-
-
-starting permutation: s1-begin s1-update-node-1 s2-start-metadata-sync-node-2 s1-commit s2-verify-metadata
-nodeid|nodename |nodeport
----------------------------------------------------------------------
-    26|localhost|   57638
-    25|localhost|   57637
-(2 rows)
-
-step s1-begin:
-	BEGIN;
-
-step s1-update-node-1:
     SELECT 1 FROM master_update_node(
-        (select nodeid from pg_dist_node where nodeport = 57637),
+        (select nodeid from pg_dist_node where nodeport = 58637),
         'localhost',
-        58637);
+        57637);
 
 ?column?
 ---------------------------------------------------------------------
        1
 (1 row)
 
-step s2-start-metadata-sync-node-2:
+step s3-manually-fix-metadata:
+    UPDATE pg_dist_node SET metadatasynced = 't' WHERE nodeport = 57637;
+    UPDATE pg_dist_node SET metadatasynced = 't' WHERE nodeport = 57638;
+    SELECT start_metadata_sync_to_node('localhost', 57637);
     SELECT start_metadata_sync_to_node('localhost', 57638);
- <waiting ...>
-step s1-commit: 
-	COMMIT;
 
-step s2-start-metadata-sync-node-2: <... completed>
 start_metadata_sync_to_node
 ---------------------------------------------------------------------
 
 (1 row)
 
-step s2-verify-metadata:
+start_metadata_sync_to_node
-    SELECT nodeid, groupid, nodename, nodeport FROM pg_dist_node ORDER BY nodeid;
-    SELECT master_run_on_worker(
-        ARRAY['localhost'], ARRAY[57638],
-        ARRAY['SELECT jsonb_agg(ROW(nodeid, groupid, nodename, nodeport) ORDER BY nodeid) FROM  pg_dist_node'],
-        false);
-
-nodeid|groupid|nodename |nodeport
 ---------------------------------------------------------------------
-    25|     25|localhost|   58637
-    26|     26|localhost|   57638
-(2 rows)
 
-master_run_on_worker
----------------------------------------------------------------------
-(localhost,57638,t,"[{""f1"": 10, ""f2"": 10, ""f3"": ""localhost"", ""f4"": 57638}]")
 (1 row)
 
 nodeid|nodename|nodeport
@@ -166,11 +175,11 @@ nodeid|nodename|nodeport
 (0 rows)
 
 
-starting permutation: s2-create-table s1-begin s1-update-node-nonexistent s1-prepare-transaction s2-cache-prepared-statement s1-commit-prepared s2-execute-prepared s1-update-node-existent s2-drop-table
+starting permutation: s2-create-table s1-begin s1-update-node-nonexistent s1-prepare-transaction s2-cache-prepared-statement s1-commit-prepared s2-execute-prepared s1-update-node-existent s3-manually-fix-metadata
 nodeid|nodename |nodeport
 ---------------------------------------------------------------------
-    28|localhost|   57638
+    26|localhost|   57638
-    27|localhost|   57637
+    25|localhost|   57637
 (2 rows)
 
 step s2-create-table:
@@ -261,8 +270,21 @@ step s1-update-node-existent:
        1
 (1 row)
 
-step s2-drop-table:
+step s3-manually-fix-metadata:
-    DROP TABLE test;
+    UPDATE pg_dist_node SET metadatasynced = 't' WHERE nodeport = 57637;
+    UPDATE pg_dist_node SET metadatasynced = 't' WHERE nodeport = 57638;
+    SELECT start_metadata_sync_to_node('localhost', 57637);
+    SELECT start_metadata_sync_to_node('localhost', 57638);
+
+start_metadata_sync_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
+start_metadata_sync_to_node
+---------------------------------------------------------------------
+
+(1 row)
 
 nodeid|nodename|nodeport
 ---------------------------------------------------------------------

src/test/regress/isolation_schedule

@@ -1,8 +1,6 @@
 test: isolation_add_remove_node
-test: isolation_turn_mx_off
 test: isolation_update_node
 test: isolation_update_node_lock_writes
-test: isolation_turn_mx_on
 test: isolation_ensure_dependency_activate_node
 test: isolation_add_node_vs_reference_table_operations
 test: isolation_create_table_vs_add_remove_node

src/test/regress/spec/isolation_turn_mx_off.spec

@@ -1,18 +0,0 @@
-session "s1"
-
-step "disable-mx-by-default"
-{
-  ALTER SYSTEM SET citus.enable_metadata_sync TO OFF;
-}
-
-step "reload"
-{
-  SELECT pg_reload_conf();
-}
-
-step "stop-metadata-sync"
-{
-  SELECT stop_metadata_sync_to_node(nodename, nodeport) FROM pg_dist_node WHERE isactive = 't' and noderole = 'primary';
-}
-
-permutation "disable-mx-by-default" "reload" "stop-metadata-sync"

src/test/regress/spec/isolation_turn_mx_on.spec

@@ -1,18 +0,0 @@
-session "s1"
-
-step "enable-mx-by-default"
-{
-  ALTER SYSTEM SET citus.enable_metadata_sync TO ON;
-}
-
-step "reload"
-{
-  SELECT pg_reload_conf();
-}
-
-step "start-metadata-sync"
-{
-  SELECT start_metadata_sync_to_node(nodename, nodeport) FROM pg_dist_node WHERE isactive = 't' and noderole = 'primary';
-}
-
-permutation "enable-mx-by-default" "reload" "start-metadata-sync"

src/test/regress/spec/isolation_update_node.spec

@@ -8,6 +8,7 @@ setup
 
 teardown
 {
+    DROP TABLE IF EXISTS test;
     SELECT master_remove_node(nodename, nodeport) FROM pg_dist_node;
     SELECT nodeid, nodename, nodeport from pg_dist_node ORDER BY 1 DESC;
 }
@@ -103,43 +104,47 @@ step "s2-execute-prepared" {
     EXECUTE foo;
 }
 
-step "s2-verify-metadata"
-{
-    SELECT nodeid, groupid, nodename, nodeport FROM pg_dist_node ORDER BY nodeid;
-    SELECT master_run_on_worker(
-        ARRAY['localhost'], ARRAY[57638],
-        ARRAY['SELECT jsonb_agg(ROW(nodeid, groupid, nodename, nodeport) ORDER BY nodeid) FROM  pg_dist_node'],
-        false);
-}
-
-step "s2-start-metadata-sync-node-2"
-{
-    SELECT start_metadata_sync_to_node('localhost', 57638);
-}
-
-step "s2-drop-table" {
-    DROP TABLE test;
-}
-
 step "s2-abort"
 {
 	ABORT;
 }
 
+session "s3"
+
+step "s3-update-node-1-back"
+{
+    SELECT 1 FROM master_update_node(
+        (select nodeid from pg_dist_node where nodeport = 58637),
+        'localhost',
+        57637);
+}
+
+step "s3-update-node-2-back"
+{
+    SELECT 1 FROM master_update_node(
+        (select nodeid from pg_dist_node where nodeport = 58638),
+        'localhost',
+        57638);
+}
+
+
+// since we update the nodes to unexistent nodes we break metadata, so here we fix it manually
+step "s3-manually-fix-metadata"
+{
+    UPDATE pg_dist_node SET metadatasynced = 't' WHERE nodeport = 57637;
+    UPDATE pg_dist_node SET metadatasynced = 't' WHERE nodeport = 57638;
+    SELECT start_metadata_sync_to_node('localhost', 57637);
+    SELECT start_metadata_sync_to_node('localhost', 57638);
+}
+
+
 // session 1 updates node 1, session 2 updates node 2, should be ok
-permutation "s1-begin" "s1-update-node-1" "s2-update-node-2" "s1-commit" "s1-show-nodes"
+permutation "s1-begin" "s1-update-node-1" "s2-update-node-2" "s1-commit" "s1-show-nodes" "s3-update-node-1-back" "s3-update-node-2-back" "s3-manually-fix-metadata"
 
 // sessions 1 updates node 1, session 2 tries to do the same
-permutation "s1-begin" "s1-update-node-1" "s2-begin" "s2-update-node-1" "s1-commit" "s2-abort" "s1-show-nodes"
+permutation "s1-begin" "s1-update-node-1" "s2-begin" "s2-update-node-1" "s1-commit" "s2-abort" "s1-show-nodes" "s3-update-node-1-back" "s3-manually-fix-metadata"
-
-// master_update_node should block start_metadata_sync_to_node. Note that we
-// cannot run start_metadata_sync_to_node in a transaction, so we're not
-// testing the reverse order here.
-// Having different result on coordinator and worker is expected for now since
-// we run test after disabling mx.
-permutation "s1-begin" "s1-update-node-1" "s2-start-metadata-sync-node-2" "s1-commit" "s2-verify-metadata"
 
 // make sure we have entries in prepared statement cache
 // then make sure that after we update pg_dist_node, the changes are visible to
 // the prepared statement
-permutation "s2-create-table" "s1-begin" "s1-update-node-nonexistent" "s1-prepare-transaction" "s2-cache-prepared-statement" "s1-commit-prepared" "s2-execute-prepared" "s1-update-node-existent" "s2-drop-table"
+permutation "s2-create-table" "s1-begin" "s1-update-node-nonexistent" "s1-prepare-transaction" "s2-cache-prepared-statement" "s1-commit-prepared" "s2-execute-prepared" "s1-update-node-existent" "s3-manually-fix-metadata"

src/test/regress/spec/isolation_update_node_lock_writes.spec

@@ -7,8 +7,12 @@ setup
     SELECT create_distributed_table('update_node', 'id');
 }
 
+// we sleep 2 seconds to let isolation test sync metadata
+// which is longer than citus.metadata_sync_interval, 1 second
 teardown
 {
+    SELECT pg_sleep(2);
+
     RESET citus.shard_replication_factor;
     DROP TABLE update_node CASCADE;