Turn MX on by default

src/backend/distributed/metadata/node_metadata.c

@@ -75,6 +75,8 @@ bool ReplicateReferenceTablesOnActivate = true;
 /* did current transaction modify pg_dist_node? */
 bool TransactionModifiedNodeMetadata = false;
 
+bool EnableMetadataSyncByDefault = true;
+
 typedef struct NodeMetadata
 {
 	int32 groupId;
@@ -835,6 +837,12 @@ ActivateNode(char *nodeName, int nodePort)
 	WorkerNode *newWorkerNode = SetNodeState(nodeName, nodePort, isActive);
 
 	SetUpDistributedTableDependencies(newWorkerNode);
+
+	if (EnableMetadataSyncByDefault)
+	{
+		StartMetadataSyncToNode(nodeName, nodePort);
+	}
+
 	return newWorkerNode->nodeId;
 }
 

src/backend/distributed/shared_library_init.c

@@ -876,6 +876,16 @@ RegisterCitusConfigVariables(void)
 		GUC_SUPERUSER_ONLY | GUC_NO_SHOW_ALL,
 		NULL, NULL, NULL);
 
+	DefineCustomBoolVariable(
+		"citus.enable_metadata_sync_by_default",
+		gettext_noop("Enables MX in the new nodes by default"),
+		NULL,
+		&EnableMetadataSyncByDefault,
+		true,
+		PGC_USERSET,
+		GUC_SUPERUSER_ONLY | GUC_NO_SHOW_ALL,
+		NULL, NULL, NULL);
+
 	DefineCustomBoolVariable(
 		"citus.enable_object_propagation",
 		gettext_noop("Enables propagating object creation for more complex objects, "

src/include/distributed/metadata_sync.h

@@ -83,5 +83,6 @@ extern Oid GetAttributeTypeOid(Oid relationId, AttrNumber attnum);
 
 /* controlled via GUC */
 extern char *EnableManualMetadataChangesForUser;
+extern bool EnableMetadataSyncByDefault;
 
 #endif /* METADATA_SYNC_H */

src/test/regress/after_citus_upgrade_coord_schedule

@@ -1,5 +1,6 @@
 # this schedule is to be run only on coordinators
 
+test: turn_mx_off
 test: upgrade_basic_after
 test: upgrade_partition_constraints_after
 test: upgrade_pg_dist_object_test_after

src/test/regress/base_schedule

@@ -1,6 +1,7 @@
 # ----------
 # Only run few basic tests to set up a testing environment
 # ----------
+test: turn_mx_off
 test: multi_cluster_management
 test: multi_test_helpers multi_test_helpers_superuser multi_create_fdw columnar_test_helpers
 test: multi_test_catalog_views

src/test/regress/before_pg_upgrade_schedule

@@ -1,4 +1,5 @@
 # The basic tests runs analyze which depends on shard numbers
+test: turn_mx_off
 test: multi_test_helpers multi_test_helpers_superuser
 test: multi_test_catalog_views
 test: upgrade_basic_before

src/test/regress/columnar_schedule

@@ -1,3 +1,4 @@
+test: turn_mx_off
 test: multi_cluster_management
 test: multi_test_helpers multi_test_helpers_superuser columnar_test_helpers
 test: multi_test_catalog_views

src/test/regress/expected/check_mx.out

@@ -0,0 +1,12 @@
+SHOW citus.enable_metadata_sync_by_default;
+ citus.enable_metadata_sync_by_default
+---------------------------------------------------------------------
+ on
+(1 row)
+
+SELECT bool_and(metadatasynced) FROM pg_dist_node WHERE noderole = 'primary';
+ bool_and
+---------------------------------------------------------------------
+ t
+(1 row)
+

src/test/regress/expected/isolation_turn_mx_off.out

@@ -0,0 +1,21 @@
+Parsed test spec with 1 sessions
+
+starting permutation: disable-mx-by-default reload stop-metadata-sync
+step disable-mx-by-default:
+  ALTER SYSTEM SET citus.enable_metadata_sync_by_default TO OFF;
+
+step reload:
+  SELECT pg_reload_conf();
+
+pg_reload_conf
+---------------------------------------------------------------------
+t
+(1 row)
+
+step stop-metadata-sync:
+  SELECT stop_metadata_sync_to_node(nodename, nodeport) FROM pg_dist_node WHERE isactive = 't' and noderole = 'primary';
+
+stop_metadata_sync_to_node
+---------------------------------------------------------------------
+(0 rows)
+

src/test/regress/expected/multi_fix_partition_shard_index_names.out

@@ -306,6 +306,7 @@ SELECT tablename, indexname FROM pg_indexes WHERE schemaname = 'fix_idx_names' O
 
 \c - - - :master_port
 SET search_path TO fix_idx_names, public;
+SET citus.next_shard_id TO 361176;
 ALTER TABLE dist_partitioned_table DROP CONSTRAINT constraint1 CASCADE;
 NOTICE:  drop cascades to constraint fk_table_id_fk_column_fkey on table fk_table
 DROP INDEX dist_partitioned_table_dist_col_partition_col_idx;

src/test/regress/expected/multi_partitioning.out

@@ -3914,6 +3914,7 @@ CONTEXT:  PL/pgSQL function drop_old_time_partitions(regclass,timestamp with tim
 DROP TABLE non_partitioned_table;
 -- https://github.com/citusdata/citus/issues/4962
 SET citus.shard_replication_factor TO 1;
+SET citus.next_shard_id TO 361168;
 CREATE TABLE part_table_with_very_long_name (
     dist_col integer,
     long_named_integer_col integer,

src/test/regress/expected/turn_mx_off.out

@@ -0,0 +1,13 @@
+ALTER SYSTEM SET citus.enable_metadata_sync_by_default TO OFF;
+SELECT pg_reload_conf();
+ pg_reload_conf
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SET client_min_messages TO ERROR;
+SELECT stop_metadata_sync_to_node(nodename, nodeport) FROM pg_dist_node WHERE isactive = 't' and noderole = 'primary';
+ stop_metadata_sync_to_node
+---------------------------------------------------------------------
+(0 rows)
+

src/test/regress/expected/turn_mx_off_0.out

@@ -0,0 +1,15 @@
+ALTER SYSTEM SET citus.enable_metadata_sync_by_default TO OFF;
+SELECT pg_reload_conf();
+ pg_reload_conf
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SET client_min_messages TO ERROR;
+SELECT stop_metadata_sync_to_node(nodename, nodeport) FROM pg_dist_node WHERE isactive = 't' and noderole = 'primary';
+ stop_metadata_sync_to_node
+---------------------------------------------------------------------
+
+
+(2 rows)
+

src/test/regress/expected/turn_mx_on.out

@@ -0,0 +1,15 @@
+ALTER SYSTEM SET citus.enable_metadata_sync_by_default TO ON;
+SELECT pg_reload_conf();
+ pg_reload_conf
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SET client_min_messages TO ERROR;
+SELECT start_metadata_sync_to_node(nodename, nodeport) FROM pg_dist_node WHERE isactive = 't' and noderole = 'primary';
+ start_metadata_sync_to_node
+---------------------------------------------------------------------
+
+
+(2 rows)
+

src/test/regress/failure_base_schedule

@@ -1,4 +1,5 @@
 # import this file (from psql you can use \i) to use mitmproxy manually
+test: turn_mx_off
 test: failure_test_helpers
 
 # this should only be run by pg_regress_multi, you don't need it

src/test/regress/failure_schedule

@@ -1,4 +1,5 @@
 # import this file (from psql you can use \i) to use mitmproxy manually
+test: turn_mx_off
 test: failure_test_helpers
 
 # this should only be run by pg_regress_multi, you don't need it

src/test/regress/isolation_schedule

@@ -1,3 +1,4 @@
+test: isolation_turn_mx_off
 test: isolation_add_remove_node
 test: isolation_update_node
 test: isolation_update_node_lock_writes

src/test/regress/minimal_schedule

@@ -1,3 +1,4 @@
+test: turn_mx_off
 test: multi_cluster_management
 test: multi_test_helpers multi_test_helpers_superuser columnar_test_helpers
 test: multi_test_catalog_views

src/test/regress/mixed_after_citus_upgrade_schedule

@@ -1,2 +1,3 @@
+test: turn_mx_off
 test: upgrade_basic_after
 test: upgrade_pg_dist_object_test_after

src/test/regress/multi_1_schedule

@@ -15,6 +15,7 @@
 # ---
 # Tests around schema changes, these are run first, so there's no preexisting objects.
 # ---
+test: turn_mx_off
 test: multi_extension
 test: single_node
 test: single_node_truncate

src/test/regress/multi_follower_schedule

@@ -1,3 +1,4 @@
+test: turn_mx_off
 test: multi_follower_sanity_check
 test: follower_single_node
 test: multi_follower_select_statements

src/test/regress/multi_mx_schedule

@@ -13,6 +13,7 @@
 # ---
 # Tests around schema changes, these are run first, so there's no preexisting objects.
 # ---
+test: turn_mx_off
 test: multi_extension
 test: multi_test_helpers multi_test_helpers_superuser
 test: multi_mx_node_metadata

src/test/regress/multi_schedule

@@ -1,4 +1,6 @@
+test: turn_mx_off
 test: multi_cluster_management
+test: turn_mx_on
 test: multi_test_helpers multi_test_helpers_superuser multi_create_fdw
 test: multi_test_catalog_views
 
@@ -30,7 +32,7 @@ test: ensure_no_intermediate_data_leak
 # ----------
 # Tests for partitioning support
 # ----------
-test: multi_partitioning_utils multi_partitioning replicated_partitioned_table
+test: multi_partitioning_utils replicated_partitioned_table
 
 
 # ----------
@@ -52,19 +54,31 @@ test: subquery_in_targetlist subquery_in_where subquery_complex_target_list
 test: subquery_prepared_statements
 test: non_colocated_leaf_subquery_joins non_colocated_subquery_joins non_colocated_join_order
 test: cte_inline recursive_view_local_table values
-test: pg13 pg12
+test: pg13
+test: tableam
+
+# mx-less tests
+test: check_mx
+test: turn_mx_off
+test: multi_partitioning
 # run pg14 sequentially as it syncs metadata
 test: pg14
-test: tableam drop_column_partitioned_table
+test: pg12
+test: drop_column_partitioned_table
+test: multi_real_time_transaction
+test: undistribute_table
+test: alter_table_set_access_method
+test: alter_distributed_table
+test: turn_mx_on
 
 # ----------
 # Miscellaneous tests to check our query planning behavior
 # ----------
 test: multi_deparse_shard_query multi_distributed_transaction_id intermediate_results limit_intermediate_size rollback_to_savepoint
-test: multi_explain hyperscale_tutorial partitioned_intermediate_results distributed_intermediate_results multi_real_time_transaction
+test: multi_explain hyperscale_tutorial partitioned_intermediate_results distributed_intermediate_results
 test: multi_basic_queries cross_join multi_complex_expressions multi_subquery multi_subquery_complex_queries multi_subquery_behavioral_analytics
 test: multi_subquery_complex_reference_clause multi_subquery_window_functions multi_view multi_sql_function multi_prepare_sql
-test: sql_procedure multi_function_in_join row_types materialized_view undistribute_table
+test: sql_procedure multi_function_in_join row_types materialized_view
 test: multi_subquery_in_where_reference_clause adaptive_executor propagate_set_commands geqo
 # this should be run alone as it gets too many clients
 test: join_pushdown
@@ -86,8 +100,6 @@ test: multi_task_assignment_policy multi_cross_shard
 test: multi_utility_statements
 test: multi_dropped_column_aliases foreign_key_restriction_enforcement
 test: binary_protocol
-test: alter_table_set_access_method
-test: alter_distributed_table
 test: issue_5248
 
 # ---------
@@ -100,3 +112,4 @@ test: ensure_no_intermediate_data_leak
 # in the shared memory
 # --------
 test: ensure_no_shared_connection_leak
+test: check_mx

src/test/regress/multi_schedule_hyperscale

@@ -19,6 +19,7 @@
 # remove / add node operations, we do not want any preexisting objects before
 # propagate_extension_commands
 # ---
+test: turn_mx_off
 test: multi_test_helpers
 test: multi_test_catalog_views
 test: multi_name_resolution

src/test/regress/multi_schedule_hyperscale_superuser

@@ -19,6 +19,7 @@
 # remove / add node operations, we do not want any preexisting objects before
 # propagate_extension_commands
 # ---
+test: turn_mx_off
 test: multi_test_helpers multi_test_helpers_superuser
 test: multi_test_catalog_views
 test: multi_name_lengths

src/test/regress/mx_base_schedule

@@ -1,6 +1,7 @@
 # ----------
 # Only run few basic tests to set up a testing environment
 # ----------
+test: turn_mx_off
 test: multi_cluster_management
 test: multi_test_helpers multi_test_helpers_superuser
 test: multi_test_catalog_views

src/test/regress/mx_minimal_schedule

@@ -1,6 +1,7 @@
 # ----------
 # Only run few basic tests to set up a testing environment
 # ----------
+test: turn_mx_off
 test: multi_cluster_management
 test: multi_test_helpers multi_test_helpers_superuser
 test: multi_test_catalog_views

src/test/regress/operations_schedule

@@ -1,3 +1,4 @@
+test: turn_mx_off
 test: multi_cluster_management
 test: multi_test_helpers multi_test_helpers_superuser
 test: multi_test_catalog_views

src/test/regress/spec/isolation_turn_mx_off.spec

@@ -0,0 +1,18 @@
+session "s1"
+
+step "disable-mx-by-default"
+{
+  ALTER SYSTEM SET citus.enable_metadata_sync_by_default TO OFF;
+}
+
+step "reload"
+{
+  SELECT pg_reload_conf();
+}
+
+step "stop-metadata-sync"
+{
+  SELECT stop_metadata_sync_to_node(nodename, nodeport) FROM pg_dist_node WHERE isactive = 't' and noderole = 'primary';
+}
+
+permutation "disable-mx-by-default" "reload" "stop-metadata-sync"

src/test/regress/sql/check_mx.sql

@@ -0,0 +1,3 @@
+SHOW citus.enable_metadata_sync_by_default;
+
+SELECT bool_and(metadatasynced) FROM pg_dist_node WHERE noderole = 'primary';

src/test/regress/sql/multi_fix_partition_shard_index_names.sql

@@ -135,6 +135,7 @@ SELECT tablename, indexname FROM pg_indexes WHERE schemaname = 'fix_idx_names' O
 
 \c - - - :master_port
 SET search_path TO fix_idx_names, public;
+SET citus.next_shard_id TO 361176;
 
 ALTER TABLE dist_partitioned_table DROP CONSTRAINT constraint1 CASCADE;
 DROP INDEX dist_partitioned_table_dist_col_partition_col_idx;

src/test/regress/sql/multi_partitioning.sql

@@ -1888,6 +1888,7 @@ DROP TABLE non_partitioned_table;
 
 -- https://github.com/citusdata/citus/issues/4962
 SET citus.shard_replication_factor TO 1;
+SET citus.next_shard_id TO 361168;
 CREATE TABLE part_table_with_very_long_name (
     dist_col integer,
     long_named_integer_col integer,

src/test/regress/sql/turn_mx_off.sql

@@ -0,0 +1,5 @@
+ALTER SYSTEM SET citus.enable_metadata_sync_by_default TO OFF;
+SELECT pg_reload_conf();
+
+SET client_min_messages TO ERROR;
+SELECT stop_metadata_sync_to_node(nodename, nodeport) FROM pg_dist_node WHERE isactive = 't' and noderole = 'primary';

src/test/regress/sql/turn_mx_on.sql

@@ -0,0 +1,5 @@
+ALTER SYSTEM SET citus.enable_metadata_sync_by_default TO ON;
+SELECT pg_reload_conf();
+
+SET client_min_messages TO ERROR;
+SELECT start_metadata_sync_to_node(nodename, nodeport) FROM pg_dist_node WHERE isactive = 't' and noderole = 'primary';