From 924cd7343aaed477596c49f3d5a73773f960d2e0 Mon Sep 17 00:00:00 2001
From: Marco Slot <marco.slot@gmail.com>
Date: Sun, 8 Mar 2020 00:49:06 +0100
Subject: [PATCH 1/5] Defer reference table replication to shard creation time

---
 .../commands/create_distributed_table.c       |   6 +
 .../distributed/master/master_create_shards.c |   2 +
 .../distributed/master/master_repair_shards.c |  14 +
 .../distributed/metadata/node_metadata.c      |  15 +-
 .../distributed/sql/citus--9.2-4--9.3-2.sql   |   1 +
 .../udfs/replicate_reference_tables/9.3-1.sql |   7 +
 .../replicate_reference_tables/latest.sql     |   7 +
 .../transaction/transaction_management.c      |   1 +
 .../distributed/utils/reference_table_utils.c | 278 +++++++++++---
 src/backend/distributed/utils/resource_lock.c |  30 ++
 .../distributed/reference_table_utils.h       |   3 +-
 src/include/distributed/resource_lock.h       |   4 +
 .../distributed/transaction_management.h      |   3 +
 .../expected/failure_add_disable_node.out     | 144 +------
 .../expected/insert_select_repartition.out    |   1 -
 ...add_node_vs_reference_table_operations.out | 350 ++++++++----------
 .../regress/expected/local_shard_copy.out     |   4 -
 .../regress/expected/multi_metadata_sync.out  |   7 +-
 .../expected/multi_mx_node_metadata.out       |   4 -
 .../multi_remove_node_reference_table.out     |  76 ++--
 .../multi_replicate_reference_table.out       |  49 +--
 .../multi_transactional_drop_shards.out       |   1 -
 .../expected/propagate_extension_commands.out |   2 -
 src/test/regress/input/multi_copy.source      |   2 +
 src/test/regress/output/multi_copy.source     |   9 +-
 .../regress/sql/failure_add_disable_node.sql  |  60 ---
 .../sql/multi_replicate_reference_table.sql   |   5 +-
 27 files changed, 528 insertions(+), 557 deletions(-)
 create mode 100644 src/backend/distributed/sql/udfs/replicate_reference_tables/9.3-1.sql
 create mode 100644 src/backend/distributed/sql/udfs/replicate_reference_tables/latest.sql

diff --git a/src/backend/distributed/commands/create_distributed_table.c b/src/backend/distributed/commands/create_distributed_table.c
index aaa95af78..28d6795f5 100644
--- a/src/backend/distributed/commands/create_distributed_table.c
+++ b/src/backend/distributed/commands/create_distributed_table.c
@@ -346,6 +346,12 @@ CreateDistributedTable(Oid relationId, Var *distributionColumn, char distributio
 	EnsureRelationCanBeDistributed(relationId, distributionColumn, distributionMethod,
 								   colocationId, replicationModel, viaDeprecatedAPI);
 
+	/*
+	 * Make sure that existing reference tables have been replicated to all the nodes
+	 * such that we can create foreign keys and joins work immediately after creation.
+	 */
+	EnsureReferenceTablesExistOnAllNodes();
+
 	/* we need to calculate these variables before creating distributed metadata */
 	bool localTableEmpty = LocalTableEmpty(relationId);
 	Oid colocatedTableId = ColocatedTableId(colocationId);
diff --git a/src/backend/distributed/master/master_create_shards.c b/src/backend/distributed/master/master_create_shards.c
index 128957c18..ce2239c5a 100644
--- a/src/backend/distributed/master/master_create_shards.c
+++ b/src/backend/distributed/master/master_create_shards.c
@@ -86,6 +86,8 @@ master_create_worker_shards(PG_FUNCTION_ARGS)
 	ObjectAddressSet(tableAddress, RelationRelationId, distributedTableId);
 	EnsureDependenciesExistOnAllNodes(&tableAddress);
 
+	EnsureReferenceTablesExistOnAllNodes();
+
 	CreateShardsWithRoundRobinPolicy(distributedTableId, shardCount, replicationFactor,
 									 useExclusiveConnections);
 
diff --git a/src/backend/distributed/master/master_repair_shards.c b/src/backend/distributed/master/master_repair_shards.c
index ed6f3f897..c3b24f50f 100644
--- a/src/backend/distributed/master/master_repair_shards.c
+++ b/src/backend/distributed/master/master_repair_shards.c
@@ -28,6 +28,7 @@
 #include "distributed/metadata_sync.h"
 #include "distributed/multi_join_order.h"
 #include "distributed/multi_partitioning_utils.h"
+#include "distributed/reference_table_utils.h"
 #include "distributed/resource_lock.h"
 #include "distributed/worker_manager.h"
 #include "distributed/worker_protocol.h"
@@ -449,6 +450,19 @@ ReplicateColocatedShardPlacement(int64 shardId, char *sourceNodeName,
 							   targetNodeName, targetNodePort);
 	}
 
+	if (!IsReferenceTable(distributedTableId))
+	{
+		/*
+		 * When copying a shard to a new node, we should first ensure that reference
+		 * tables are present such that joins work immediately after copying the shard.
+		 * When copying a reference table, we are probably trying to achieve just that.
+		 *
+		 * Since this a long-running operation we do this after the error checks, but
+		 * before taking metadata locks.
+		 */
+		EnsureReferenceTablesExistOnAllNodes();
+	}
+
 	/*
 	 * CopyColocatedShardPlacement function copies given shard with its co-located
 	 * shards.
diff --git a/src/backend/distributed/metadata/node_metadata.c b/src/backend/distributed/metadata/node_metadata.c
index 2b869d4f9..4cc9f5d17 100644
--- a/src/backend/distributed/metadata/node_metadata.c
+++ b/src/backend/distributed/metadata/node_metadata.c
@@ -60,6 +60,9 @@ int GroupSize = 1;
 /* config variable managed via guc.c */
 char *CurrentCluster = "default";
 
+/* did current transaction modify pg_dist_node? */
+bool TransactionModifiedNodeMetadata = false;
+
 typedef struct NodeMetadata
 {
 	int32 groupId;
@@ -158,6 +161,7 @@ master_add_node(PG_FUNCTION_ARGS)
 
 	int nodeId = AddNodeMetadata(nodeNameString, nodePort, &nodeMetadata,
 								 &nodeAlreadyExists);
+	TransactionModifiedNodeMetadata = true;
 
 	/*
 	 * After adding new node, if the node did not already exist, we will activate
@@ -196,6 +200,7 @@ master_add_inactive_node(PG_FUNCTION_ARGS)
 
 	int nodeId = AddNodeMetadata(nodeNameString, nodePort, &nodeMetadata,
 								 &nodeAlreadyExists);
+	TransactionModifiedNodeMetadata = true;
 
 	PG_RETURN_INT32(nodeId);
 }
@@ -229,6 +234,7 @@ master_add_secondary_node(PG_FUNCTION_ARGS)
 
 	int nodeId = AddNodeMetadata(nodeNameString, nodePort, &nodeMetadata,
 								 &nodeAlreadyExists);
+	TransactionModifiedNodeMetadata = true;
 
 	PG_RETURN_INT32(nodeId);
 }
@@ -252,6 +258,7 @@ master_remove_node(PG_FUNCTION_ARGS)
 	CheckCitusVersion(ERROR);
 
 	RemoveNodeFromCluster(text_to_cstring(nodeNameText), nodePort);
+	TransactionModifiedNodeMetadata = true;
 
 	PG_RETURN_VOID();
 }
@@ -305,6 +312,7 @@ master_disable_node(PG_FUNCTION_ARGS)
 		}
 
 		SetNodeState(nodeName, nodePort, isActive);
+		TransactionModifiedNodeMetadata = true;
 	}
 	PG_CATCH();
 	{
@@ -351,6 +359,7 @@ master_set_node_property(PG_FUNCTION_ARGS)
 							)));
 	}
 
+	TransactionModifiedNodeMetadata = true;
 
 	PG_RETURN_VOID();
 }
@@ -371,8 +380,6 @@ SetUpDistributedTableDependencies(WorkerNode *newWorkerNode)
 		EnsureNoModificationsHaveBeenDone();
 		ReplicateAllDependenciesToNode(newWorkerNode->workerName,
 									   newWorkerNode->workerPort);
-		ReplicateAllReferenceTablesToNode(newWorkerNode->workerName,
-										  newWorkerNode->workerPort);
 
 		/*
 		 * Let the maintenance daemon do the hard work of syncing the metadata.
@@ -452,6 +459,8 @@ master_activate_node(PG_FUNCTION_ARGS)
 												  nodePort);
 	ActivateNode(workerNode->workerName, workerNode->workerPort);
 
+	TransactionModifiedNodeMetadata = true;
+
 	PG_RETURN_INT32(workerNode->nodeId);
 }
 
@@ -723,6 +732,8 @@ master_update_node(PG_FUNCTION_ARGS)
 		TerminateBackgroundWorker(handle);
 	}
 
+	TransactionModifiedNodeMetadata = true;
+
 	PG_RETURN_VOID();
 }
 
diff --git a/src/backend/distributed/sql/citus--9.2-4--9.3-2.sql b/src/backend/distributed/sql/citus--9.2-4--9.3-2.sql
index 0f763f4d7..25ba9cba0 100644
--- a/src/backend/distributed/sql/citus--9.2-4--9.3-2.sql
+++ b/src/backend/distributed/sql/citus--9.2-4--9.3-2.sql
@@ -4,3 +4,4 @@
 
 #include "udfs/citus_extradata_container/9.3-2.sql"
 #include "udfs/update_distributed_table_colocation/9.3-2.sql"
+#include "udfs/replicate_reference_tables/9.3-1.sql"
diff --git a/src/backend/distributed/sql/udfs/replicate_reference_tables/9.3-1.sql b/src/backend/distributed/sql/udfs/replicate_reference_tables/9.3-1.sql
new file mode 100644
index 000000000..556899eb2
--- /dev/null
+++ b/src/backend/distributed/sql/udfs/replicate_reference_tables/9.3-1.sql
@@ -0,0 +1,7 @@
+CREATE FUNCTION pg_catalog.replicate_reference_tables()
+  RETURNS VOID
+  LANGUAGE C STRICT
+  AS 'MODULE_PATHNAME', $$replicate_reference_tables$$;
+COMMENT ON FUNCTION pg_catalog.replicate_reference_tables()
+  IS 'replicate reference tables to all nodes';
+REVOKE ALL ON FUNCTION pg_catalog.replicate_reference_tables() FROM PUBLIC;
diff --git a/src/backend/distributed/sql/udfs/replicate_reference_tables/latest.sql b/src/backend/distributed/sql/udfs/replicate_reference_tables/latest.sql
new file mode 100644
index 000000000..556899eb2
--- /dev/null
+++ b/src/backend/distributed/sql/udfs/replicate_reference_tables/latest.sql
@@ -0,0 +1,7 @@
+CREATE FUNCTION pg_catalog.replicate_reference_tables()
+  RETURNS VOID
+  LANGUAGE C STRICT
+  AS 'MODULE_PATHNAME', $$replicate_reference_tables$$;
+COMMENT ON FUNCTION pg_catalog.replicate_reference_tables()
+  IS 'replicate reference tables to all nodes';
+REVOKE ALL ON FUNCTION pg_catalog.replicate_reference_tables() FROM PUBLIC;
diff --git a/src/backend/distributed/transaction/transaction_management.c b/src/backend/distributed/transaction/transaction_management.c
index f28a43571..803851ff2 100644
--- a/src/backend/distributed/transaction/transaction_management.c
+++ b/src/backend/distributed/transaction/transaction_management.c
@@ -449,6 +449,7 @@ ResetGlobalVariables()
 	dlist_init(&InProgressTransactions);
 	activeSetStmts = NULL;
 	CoordinatedTransactionUses2PC = false;
+	TransactionModifiedNodeMetadata = false;
 }
 
 
diff --git a/src/backend/distributed/utils/reference_table_utils.c b/src/backend/distributed/utils/reference_table_utils.c
index ebf5350dd..ea5265e00 100644
--- a/src/backend/distributed/utils/reference_table_utils.c
+++ b/src/backend/distributed/utils/reference_table_utils.c
@@ -24,26 +24,243 @@
 #include "distributed/metadata_sync.h"
 #include "distributed/multi_logical_planner.h"
 #include "distributed/reference_table_utils.h"
+#include "distributed/relation_access_tracking.h"
+#include "distributed/remote_commands.h"
 #include "distributed/resource_lock.h"
 #include "distributed/shardinterval_utils.h"
 #include "distributed/transaction_management.h"
 #include "distributed/worker_manager.h"
 #include "distributed/worker_transaction.h"
+#include "postmaster/postmaster.h"
 #include "storage/lmgr.h"
+#include "utils/builtins.h"
 #include "utils/fmgroids.h"
 #include "utils/lsyscache.h"
 #include "utils/rel.h"
 
 
 /* local function forward declarations */
+static List * WorkersWithoutReferenceTablePlacement(uint64 shardId);
+static void CopyShardPlacementToNewWorkerNode(ShardPlacement *sourceShardPlacement,
+											  WorkerNode *newWorkerNode);
 static void ReplicateSingleShardTableToAllNodes(Oid relationId);
 static void ReplicateShardToAllNodes(ShardInterval *shardInterval);
 static void ReplicateShardToNode(ShardInterval *shardInterval, char *nodeName,
 								 int nodePort);
 static void ConvertToReferenceTableMetadata(Oid relationId, uint64 shardId);
+static bool AnyRelationsModifiedInTransaction(List *relationIdList);
 
 /* exports for SQL callable functions */
 PG_FUNCTION_INFO_V1(upgrade_to_reference_table);
+PG_FUNCTION_INFO_V1(replicate_reference_tables);
+
+
+/*
+ * IsReferenceTable returns whether the given relation ID identifies a reference
+ * table.
+ */
+bool
+IsReferenceTable(Oid relationId)
+{
+	CitusTableCacheEntry *tableEntry = GetCitusTableCacheEntry(relationId);
+
+	if (!tableEntry->isCitusTable)
+	{
+		return false;
+	}
+
+	if (tableEntry->partitionMethod != DISTRIBUTE_BY_NONE)
+	{
+		return false;
+	}
+
+	return true;
+}
+
+
+/*
+ * replicate_reference_tables is a UDF to ensure that allreference tables are
+ * replicated to all nodes.
+ */
+Datum
+replicate_reference_tables(PG_FUNCTION_ARGS)
+{
+	EnsureReferenceTablesExistOnAllNodes();
+
+	PG_RETURN_VOID();
+}
+
+
+/*
+ * EnsureReferenceTablesExistOnAllNodes ensures that a shard placement for every
+ * reference table exists on all nodes. If a node does not have a set of shard
+ * placements, then master_copy_shard_placement is called in a subtransaction
+ * to pull the data to the new node.
+ */
+void
+EnsureReferenceTablesExistOnAllNodes(void)
+{
+	List *referenceTableIdList = ReferenceTableOidList();
+	if (list_length(referenceTableIdList) == 0)
+	{
+		/* no reference tables exist */
+		return;
+	}
+
+	Oid referenceTableId = linitial_oid(referenceTableIdList);
+	List *shardIntervalList = LoadShardIntervalList(referenceTableId);
+	if (list_length(shardIntervalList) == 0)
+	{
+		/* check for corrupt metadata */
+		ereport(ERROR, (errmsg("reference table \"%s\" does not have a shard",
+							   get_rel_name(referenceTableId))));
+	}
+
+	ShardInterval *shardInterval = (ShardInterval *) linitial(shardIntervalList);
+	uint64 shardId = shardInterval->shardId;
+
+	/* prevent this function from running concurrently with itself */
+	int colocationId = TableColocationId(referenceTableId);
+	LockColocationId(colocationId, ExclusiveLock);
+
+	List *newWorkersList = WorkersWithoutReferenceTablePlacement(shardId);
+	if (list_length(newWorkersList) == 0)
+	{
+		/* nothing to do, no need for lock */
+		UnlockColocationId(colocationId, ExclusiveLock);
+		return;
+	}
+
+	/*
+	 * master_copy_shard_placement triggers metadata sync-up, which tries to
+	 * acquire a ShareLock on pg_dist_node. We do master_copy_shad_placement
+	 * in a separate connection. If we have modified pg_dist_node in the
+	 * current backend, this will cause a deadlock.
+	 */
+	if (TransactionModifiedNodeMetadata)
+	{
+		ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+						errmsg("cannot replicate reference tables in a transaction "
+							   "that modified node metadata")));
+	}
+
+	/*
+	 * Modifications to reference tables in current transaction are not visible
+	 * to master_copy_shard_placement, since it is done in a separate backend.
+	 */
+	if (AnyRelationsModifiedInTransaction(referenceTableIdList))
+	{
+		ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+						errmsg("cannot replicate reference tables in a transaction "
+							   "that modified a reference table")));
+	}
+
+	bool missingOk = false;
+	ShardPlacement *sourceShardPlacement = ActiveShardPlacement(shardId, missingOk);
+	if (sourceShardPlacement == NULL)
+	{
+		/* check for corrupt metadata */
+		ereport(ERROR, (errmsg("reference table shard " UINT64_FORMAT " does not "
+																	  "have an active shard placement",
+							   shardId)));
+	}
+
+	WorkerNode *newWorkerNode = NULL;
+	foreach_ptr(newWorkerNode, newWorkersList)
+	{
+		CopyShardPlacementToNewWorkerNode(sourceShardPlacement, newWorkerNode);
+	}
+
+	/*
+	 * Unblock other backends, they will probably observe that there are no
+	 * more worker nodes without placements, unless nodes were added concurrently
+	 */
+	UnlockColocationId(colocationId, ExclusiveLock);
+}
+
+
+/*
+ * AnyRelationsModifiedInTransaction returns true if any of the given relations
+ * were modified in the current transaction.
+ */
+static bool
+AnyRelationsModifiedInTransaction(List *relationIdList)
+{
+	Oid relationId = InvalidOid;
+
+	foreach_oid(relationId, relationIdList)
+	{
+		if (GetRelationDDLAccessMode(relationId) != RELATION_NOT_ACCESSED ||
+			GetRelationDMLAccessMode(relationId) != RELATION_NOT_ACCESSED)
+		{
+			return true;
+		}
+	}
+
+	return false;
+}
+
+
+/*
+ * WorkersWithoutReferenceTablePlacement returns a list of workers (WorkerNode) that
+ * do not yet have a placement for the given reference table shard ID, but are
+ * supposed to.
+ */
+static List *
+WorkersWithoutReferenceTablePlacement(uint64 shardId)
+{
+	List *workersWithoutPlacements = NIL;
+
+	List *shardPlacementList = ActiveShardPlacementList(shardId);
+
+	/* we only take an access share lock, otherwise we'll hold up master_add_node */
+	List *workerNodeList = ReferenceTablePlacementNodeList(AccessShareLock);
+	workerNodeList = SortList(workerNodeList, CompareWorkerNodes);
+
+	WorkerNode *workerNode = NULL;
+	foreach_ptr(workerNode, workerNodeList)
+	{
+		char *nodeName = workerNode->workerName;
+		uint32 nodePort = workerNode->workerPort;
+		ShardPlacement *targetPlacement = SearchShardPlacementInList(shardPlacementList,
+																	 nodeName, nodePort);
+		if (targetPlacement == NULL)
+		{
+			workersWithoutPlacements = lappend(workersWithoutPlacements, workerNode);
+		}
+	}
+
+	return workersWithoutPlacements;
+}
+
+
+/*
+ * CopyShardPlacementToNewWorkerNode runs master_copy_shard_placement in a
+ * subtransaction by connecting to localhost.
+ */
+static void
+CopyShardPlacementToNewWorkerNode(ShardPlacement *sourceShardPlacement,
+								  WorkerNode *newWorkerNode)
+{
+	int connectionFlags = OUTSIDE_TRANSACTION;
+	StringInfo queryString = makeStringInfo();
+	const char *userName = CitusExtensionOwnerName();
+
+	MultiConnection *connection = GetNodeUserDatabaseConnection(
+		connectionFlags, "localhost", PostPortNumber,
+		userName, NULL);
+
+	appendStringInfo(queryString,
+					 "SELECT master_copy_shard_placement("
+					 UINT64_FORMAT ", %s, %d, %s, %d, do_repair := false)",
+					 sourceShardPlacement->shardId,
+					 quote_literal_cstr(sourceShardPlacement->nodeName),
+					 sourceShardPlacement->nodePort,
+					 quote_literal_cstr(newWorkerNode->workerName),
+					 newWorkerNode->workerPort);
+
+	ExecuteCriticalRemoteCommand(connection, queryString->data);
+}
 
 
 /*
@@ -104,72 +321,13 @@ upgrade_to_reference_table(PG_FUNCTION_ARGS)
 								  "reference tables.", relationName)));
 	}
 
+	EnsureReferenceTablesExistOnAllNodes();
 	ReplicateSingleShardTableToAllNodes(relationId);
 
 	PG_RETURN_VOID();
 }
 
 
-/*
- * ReplicateAllReferenceTablesToNode function finds all reference tables and
- * replicates them to the given worker node. It also modifies pg_dist_colocation
- * table to update the replication factor column when necessary. This function
- * skips reference tables if that node already has healthy placement of that
- * reference table to prevent unnecessary data transfer.
- */
-void
-ReplicateAllReferenceTablesToNode(char *nodeName, int nodePort)
-{
-	List *referenceTableList = ReferenceTableOidList();
-
-	/* if there is no reference table, we do not need to replicate anything */
-	if (list_length(referenceTableList) > 0)
-	{
-		List *referenceShardIntervalList = NIL;
-
-		/*
-		 * We sort the reference table list to prevent deadlocks in concurrent
-		 * ReplicateAllReferenceTablesToAllNodes calls.
-		 */
-		referenceTableList = SortList(referenceTableList, CompareOids);
-		Oid referenceTableId = InvalidOid;
-		foreach_oid(referenceTableId, referenceTableList)
-		{
-			List *shardIntervalList = LoadShardIntervalList(referenceTableId);
-			ShardInterval *shardInterval = (ShardInterval *) linitial(shardIntervalList);
-
-			referenceShardIntervalList = lappend(referenceShardIntervalList,
-												 shardInterval);
-		}
-
-		if (ClusterHasKnownMetadataWorkers())
-		{
-			BlockWritesToShardList(referenceShardIntervalList);
-		}
-
-		ShardInterval *shardInterval = NULL;
-		foreach_ptr(shardInterval, referenceShardIntervalList)
-		{
-			uint64 shardId = shardInterval->shardId;
-
-			LockShardDistributionMetadata(shardId, ExclusiveLock);
-
-			ReplicateShardToNode(shardInterval, nodeName, nodePort);
-		}
-
-		/* create foreign constraints between reference tables */
-		foreach_ptr(shardInterval, referenceShardIntervalList)
-		{
-			char *tableOwner = TableOwner(shardInterval->relationId);
-			List *commandList = CopyShardForeignConstraintCommandList(shardInterval);
-
-			SendCommandListToWorkerInSingleTransaction(nodeName, nodePort, tableOwner,
-													   commandList);
-		}
-	}
-}
-
-
 /*
  * ReplicateSingleShardTableToAllNodes accepts a broadcast table and replicates
  * it to all worker nodes, and the coordinator if it has been added by the user
diff --git a/src/backend/distributed/utils/resource_lock.c b/src/backend/distributed/utils/resource_lock.c
index 71f5221c5..5bdf2e570 100644
--- a/src/backend/distributed/utils/resource_lock.c
+++ b/src/backend/distributed/utils/resource_lock.c
@@ -319,6 +319,36 @@ IntToLockMode(int mode)
 }
 
 
+/*
+ * LockColocationId returns after acquiring a co-location ID lock, typically used
+ * for rebalancing and replication.
+ */
+void
+LockColocationId(int colocationId, LOCKMODE lockMode)
+{
+	LOCKTAG tag;
+	const bool sessionLock = false;
+	const bool dontWait = false;
+
+	SET_LOCKTAG_REBALANCE_COLOCATION(tag, (int64) colocationId);
+	(void) LockAcquire(&tag, lockMode, sessionLock, dontWait);
+}
+
+
+/*
+ * UnlockColocationId releases a co-location ID lock.
+ */
+void
+UnlockColocationId(int colocationId, LOCKMODE lockMode)
+{
+	LOCKTAG tag;
+	const bool sessionLock = false;
+
+	SET_LOCKTAG_REBALANCE_COLOCATION(tag, (int64) colocationId);
+	LockRelease(&tag, lockMode, sessionLock);
+}
+
+
 /*
  * LockShardDistributionMetadata returns after grabbing a lock for distribution
  * metadata related to the specified shard, blocking if required. Any locks
diff --git a/src/include/distributed/reference_table_utils.h b/src/include/distributed/reference_table_utils.h
index 6bf552511..073aa731a 100644
--- a/src/include/distributed/reference_table_utils.h
+++ b/src/include/distributed/reference_table_utils.h
@@ -16,8 +16,9 @@
 
 #include "listutils.h"
 
+extern bool IsReferenceTable(Oid relationId);
+extern void EnsureReferenceTablesExistOnAllNodes(void);
 extern uint32 CreateReferenceTableColocationId(void);
-extern void ReplicateAllReferenceTablesToNode(char *nodeName, int nodePort);
 extern void DeleteAllReferenceTablePlacementsFromNodeGroup(int32 groupId);
 extern List * ReferenceTableOidList(void);
 extern int CompareOids(const void *leftElement, const void *rightElement);
diff --git a/src/include/distributed/resource_lock.h b/src/include/distributed/resource_lock.h
index 25f50f0c8..bb1e6ccdd 100644
--- a/src/include/distributed/resource_lock.h
+++ b/src/include/distributed/resource_lock.h
@@ -102,6 +102,10 @@ extern void UnlockShardResource(uint64 shardId, LOCKMODE lockmode);
 extern void LockJobResource(uint64 jobId, LOCKMODE lockmode);
 extern void UnlockJobResource(uint64 jobId, LOCKMODE lockmode);
 
+/* Lock a co-location group */
+extern void LockColocationId(int colocationId, LOCKMODE lockMode);
+extern void UnlockColocationId(int colocationId, LOCKMODE lockMode);
+
 /* Lock multiple shards for safe modification */
 extern void LockShardListMetadata(List *shardIntervalList, LOCKMODE lockMode);
 extern void LockShardsInPlacementListMetadata(List *shardPlacementList,
diff --git a/src/include/distributed/transaction_management.h b/src/include/distributed/transaction_management.h
index 43c1e4d02..317e5df69 100644
--- a/src/include/distributed/transaction_management.h
+++ b/src/include/distributed/transaction_management.h
@@ -103,6 +103,9 @@ extern int DoBlockLevel;
 /* SET LOCAL statements active in the current (sub-)transaction. */
 extern StringInfo activeSetStmts;
 
+/* did current transaction modify pg_dist_node? */
+extern bool TransactionModifiedNodeMetadata;
+
 /*
  * Coordinated transaction management.
  */
diff --git a/src/test/regress/expected/failure_add_disable_node.out b/src/test/regress/expected/failure_add_disable_node.out
index 473df0f4a..c37f9a0ef 100644
--- a/src/test/regress/expected/failure_add_disable_node.out
+++ b/src/test/regress/expected/failure_add_disable_node.out
@@ -75,25 +75,6 @@ ORDER BY placementid;
   200000 |          1
 (1 row)
 
--- fail activate node by failing reference table creation
-SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").kill()');
- mitmproxy
----------------------------------------------------------------------
-
-(1 row)
-
-SELECT master_activate_node('localhost', :worker_2_proxy_port);
-NOTICE:  Replicating reference table "user_table" to the node localhost:xxxxx
-ERROR:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-CONTEXT:  while executing command on localhost:xxxxx
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy
----------------------------------------------------------------------
-
-(1 row)
-
 -- verify node is not activated
 SELECT * FROM master_get_active_worker_nodes()
 ORDER BY 1, 2;
@@ -140,39 +121,6 @@ ORDER BY placementid;
   200000 |          1
 (1 row)
 
--- fail activate node by failing reference table creation
-SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").cancel(' || pg_backend_pid() || ')');
- mitmproxy
----------------------------------------------------------------------
-
-(1 row)
-
-SELECT master_activate_node('localhost', :worker_2_proxy_port);
-NOTICE:  Replicating reference table "user_table" to the node localhost:xxxxx
-ERROR:  canceling statement due to user request
--- verify node is not activated
-SELECT * FROM master_get_active_worker_nodes()
-ORDER BY 1, 2;
- node_name | node_port
----------------------------------------------------------------------
- localhost |     57637
-(1 row)
-
-SELECT shardid, shardstate
-FROM pg_dist_placement p JOIN pg_dist_shard s USING (shardid)
-WHERE s.logicalrelid = 'user_table'::regclass
-ORDER BY placementid;
- shardid | shardstate
----------------------------------------------------------------------
-  200000 |          1
-(1 row)
-
-SELECT citus.mitmproxy('conn.allow()');
- mitmproxy
----------------------------------------------------------------------
-
-(1 row)
-
 -- master_remove_node fails when there are shards on that worker
 SELECT master_remove_node('localhost', :worker_2_proxy_port);
 ERROR:  you cannot remove the primary node of a node group which has shard placements
@@ -225,63 +173,6 @@ ORDER BY placementid;
   200000 |          1
 (1 row)
 
--- test master_add_node replicated a reference table
--- to newly added node.
-SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").kill()');
- mitmproxy
----------------------------------------------------------------------
-
-(1 row)
-
-SELECT master_add_node('localhost', :worker_2_proxy_port);
-NOTICE:  Replicating reference table "user_table" to the node localhost:xxxxx
-ERROR:  server closed the connection unexpectedly
-	This probably means the server terminated abnormally
-	before or while processing the request.
-CONTEXT:  while executing command on localhost:xxxxx
--- verify node is not added
-SELECT * FROM master_get_active_worker_nodes()
-ORDER BY 1, 2;
- node_name | node_port
----------------------------------------------------------------------
- localhost |     57637
-(1 row)
-
-SELECT shardid, shardstate
-FROM pg_dist_placement p JOIN pg_dist_shard s USING (shardid)
-WHERE s.logicalrelid = 'user_table'::regclass
-ORDER BY placementid;
- shardid | shardstate
----------------------------------------------------------------------
-  200000 |          1
-(1 row)
-
-SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").cancel(' || pg_backend_pid() || ')');
- mitmproxy
----------------------------------------------------------------------
-
-(1 row)
-
-SELECT master_add_node('localhost', :worker_2_proxy_port);
-NOTICE:  Replicating reference table "user_table" to the node localhost:xxxxx
-ERROR:  canceling statement due to user request
--- verify node is not added
-SELECT * FROM master_get_active_worker_nodes()
-ORDER BY 1, 2;
- node_name | node_port
----------------------------------------------------------------------
- localhost |     57637
-(1 row)
-
-SELECT shardid, shardstate
-FROM pg_dist_placement p JOIN pg_dist_shard s USING (shardid)
-WHERE s.logicalrelid = 'user_table'::regclass
-ORDER BY placementid;
- shardid | shardstate
----------------------------------------------------------------------
-  200000 |          1
-(1 row)
-
 -- reset cluster to original state
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy
@@ -290,10 +181,9 @@ SELECT citus.mitmproxy('conn.allow()');
 (1 row)
 
 SELECT master_add_node('localhost', :worker_2_proxy_port);
-NOTICE:  Replicating reference table "user_table" to the node localhost:xxxxx
  master_add_node
 ---------------------------------------------------------------------
-               6
+               4
 (1 row)
 
 -- verify node is added
@@ -312,32 +202,6 @@ ORDER BY placementid;
  shardid | shardstate
 ---------------------------------------------------------------------
   200000 |          1
-  200000 |          1
-(2 rows)
-
--- fail master_add_node by failing copy out operation
-SELECT master_remove_node('localhost', :worker_1_port);
- master_remove_node
----------------------------------------------------------------------
-
-(1 row)
-
-SELECT citus.mitmproxy('conn.onQuery(query="COPY").kill()');
- mitmproxy
----------------------------------------------------------------------
-
-(1 row)
-
-SELECT master_add_node('localhost', :worker_1_port);
-NOTICE:  Replicating reference table "user_table" to the node localhost:xxxxx
-ERROR:  could not copy table "user_table_200000" from "localhost:xxxxx"
-CONTEXT:  while executing command on localhost:xxxxx
--- verify node is not added
-SELECT * FROM master_get_active_worker_nodes()
-ORDER BY 1, 2;
- node_name | node_port
----------------------------------------------------------------------
- localhost |      9060
 (1 row)
 
 SELECT citus.mitmproxy('conn.allow()');
@@ -347,10 +211,9 @@ SELECT citus.mitmproxy('conn.allow()');
 (1 row)
 
 SELECT master_add_node('localhost', :worker_1_port);
-NOTICE:  Replicating reference table "user_table" to the node localhost:xxxxx
  master_add_node
 ---------------------------------------------------------------------
-               8
+               1
 (1 row)
 
 -- verify node is added
@@ -369,8 +232,7 @@ ORDER BY placementid;
  shardid | shardstate
 ---------------------------------------------------------------------
   200000 |          1
-  200000 |          1
-(2 rows)
+(1 row)
 
 RESET SEARCH_PATH;
 DROP SCHEMA add_remove_node CASCADE;
diff --git a/src/test/regress/expected/insert_select_repartition.out b/src/test/regress/expected/insert_select_repartition.out
index 1e3b95a7b..87d4ac185 100644
--- a/src/test/regress/expected/insert_select_repartition.out
+++ b/src/test/regress/expected/insert_select_repartition.out
@@ -1122,7 +1122,6 @@ ON CONFLICT(c1, c2, c3, c4, c5, c6)
 DO UPDATE SET
  cardinality = enriched.cardinality + excluded.cardinality,
  sum = enriched.sum + excluded.sum;
-DEBUG:  rehashing catalog cache id 14 for pg_opclass; 17 tups, 8 buckets at character 224
 DEBUG:  INSERT target table and the source relation of the SELECT partition column value must be colocated in distributed INSERT ... SELECT
 DEBUG:  Router planner cannot handle multi-shard select queries
 DEBUG:  performing repartitioned INSERT ... SELECT
diff --git a/src/test/regress/expected/isolation_add_node_vs_reference_table_operations.out b/src/test/regress/expected/isolation_add_node_vs_reference_table_operations.out
index 4fa0f754a..7049099db 100644
--- a/src/test/regress/expected/isolation_add_node_vs_reference_table_operations.out
+++ b/src/test/regress/expected/isolation_add_node_vs_reference_table_operations.out
@@ -5,36 +5,34 @@ create_distributed_table
 
 
 step s2-load-metadata-cache:
-	COPY test_reference_table FROM PROGRAM 'echo 1 && echo 2 && echo 3 && echo 4 && echo 5';
+ COPY test_reference_table FROM PROGRAM 'echo 1 && echo 2 && echo 3 && echo 4 && echo 5';
 
 step s1-begin:
     BEGIN;
 
 step s1-add-second-worker:
-	SELECT 1 FROM master_add_node('localhost', 57638);
+ SELECT 1 FROM master_add_node('localhost', 57638);
 
 ?column?
 
 1
 step s2-copy-to-reference-table:
-	COPY test_reference_table FROM PROGRAM 'echo 1 && echo 2 && echo 3 && echo 4 && echo 5';
- <waiting ...>
+ COPY test_reference_table FROM PROGRAM 'echo 1 && echo 2 && echo 3 && echo 4 && echo 5';
+
 step s1-commit:
     COMMIT;
 
-step s2-copy-to-reference-table: <... completed>
 step s2-print-content:
-	SELECT
-		nodeport, success, result
-	FROM
-		run_command_on_placements('test_reference_table', 'select count(*) from %s')
-	ORDER BY
-		nodeport;
+ SELECT
+  nodeport, success, result
+ FROM
+  run_command_on_placements('test_reference_table', 'select count(*) from %s')
+ ORDER BY
+  nodeport;
 
 nodeport       success        result
 
 57637          t              10
-57638          t              10
 master_remove_node
 
 
@@ -45,36 +43,34 @@ create_distributed_table
 
 
 step s2-load-metadata-cache:
-	COPY test_reference_table FROM PROGRAM 'echo 1 && echo 2 && echo 3 && echo 4 && echo 5';
+ COPY test_reference_table FROM PROGRAM 'echo 1 && echo 2 && echo 3 && echo 4 && echo 5';
 
 step s2-begin:
-	BEGIN;
+ BEGIN;
 
 step s2-copy-to-reference-table:
-	COPY test_reference_table FROM PROGRAM 'echo 1 && echo 2 && echo 3 && echo 4 && echo 5';
+ COPY test_reference_table FROM PROGRAM 'echo 1 && echo 2 && echo 3 && echo 4 && echo 5';
 
 step s1-add-second-worker:
-	SELECT 1 FROM master_add_node('localhost', 57638);
- <waiting ...>
-step s2-commit:
-	COMMIT;
+ SELECT 1 FROM master_add_node('localhost', 57638);
 
-step s1-add-second-worker: <... completed>
 ?column?
 
 1
+step s2-commit:
+ COMMIT;
+
 step s2-print-content:
-	SELECT
-		nodeport, success, result
-	FROM
-		run_command_on_placements('test_reference_table', 'select count(*) from %s')
-	ORDER BY
-		nodeport;
+ SELECT
+  nodeport, success, result
+ FROM
+  run_command_on_placements('test_reference_table', 'select count(*) from %s')
+ ORDER BY
+  nodeport;
 
 nodeport       success        result
 
 57637          t              10
-57638          t              10
 master_remove_node
 
 
@@ -85,36 +81,34 @@ create_distributed_table
 
 
 step s2-load-metadata-cache:
-	COPY test_reference_table FROM PROGRAM 'echo 1 && echo 2 && echo 3 && echo 4 && echo 5';
+ COPY test_reference_table FROM PROGRAM 'echo 1 && echo 2 && echo 3 && echo 4 && echo 5';
 
 step s1-begin:
     BEGIN;
 
 step s1-add-second-worker:
-	SELECT 1 FROM master_add_node('localhost', 57638);
+ SELECT 1 FROM master_add_node('localhost', 57638);
 
 ?column?
 
 1
 step s2-insert-to-reference-table:
-	INSERT INTO test_reference_table VALUES (6);
- <waiting ...>
+ INSERT INTO test_reference_table VALUES (6);
+
 step s1-commit:
     COMMIT;
 
-step s2-insert-to-reference-table: <... completed>
 step s2-print-content:
-	SELECT
-		nodeport, success, result
-	FROM
-		run_command_on_placements('test_reference_table', 'select count(*) from %s')
-	ORDER BY
-		nodeport;
+ SELECT
+  nodeport, success, result
+ FROM
+  run_command_on_placements('test_reference_table', 'select count(*) from %s')
+ ORDER BY
+  nodeport;
 
 nodeport       success        result
 
 57637          t              6
-57638          t              6
 master_remove_node
 
 
@@ -125,36 +119,34 @@ create_distributed_table
 
 
 step s2-load-metadata-cache:
-	COPY test_reference_table FROM PROGRAM 'echo 1 && echo 2 && echo 3 && echo 4 && echo 5';
+ COPY test_reference_table FROM PROGRAM 'echo 1 && echo 2 && echo 3 && echo 4 && echo 5';
 
 step s2-begin:
-	BEGIN;
+ BEGIN;
 
 step s2-insert-to-reference-table:
-	INSERT INTO test_reference_table VALUES (6);
+ INSERT INTO test_reference_table VALUES (6);
 
 step s1-add-second-worker:
-	SELECT 1 FROM master_add_node('localhost', 57638);
- <waiting ...>
-step s2-commit:
-	COMMIT;
+ SELECT 1 FROM master_add_node('localhost', 57638);
 
-step s1-add-second-worker: <... completed>
 ?column?
 
 1
+step s2-commit:
+ COMMIT;
+
 step s2-print-content:
-	SELECT
-		nodeport, success, result
-	FROM
-		run_command_on_placements('test_reference_table', 'select count(*) from %s')
-	ORDER BY
-		nodeport;
+ SELECT
+  nodeport, success, result
+ FROM
+  run_command_on_placements('test_reference_table', 'select count(*) from %s')
+ ORDER BY
+  nodeport;
 
 nodeport       success        result
 
 57637          t              6
-57638          t              6
 master_remove_node
 
 
@@ -165,36 +157,35 @@ create_distributed_table
 
 
 step s2-load-metadata-cache:
-	COPY test_reference_table FROM PROGRAM 'echo 1 && echo 2 && echo 3 && echo 4 && echo 5';
+ COPY test_reference_table FROM PROGRAM 'echo 1 && echo 2 && echo 3 && echo 4 && echo 5';
 
 step s1-begin:
     BEGIN;
 
 step s1-add-second-worker:
-	SELECT 1 FROM master_add_node('localhost', 57638);
+ SELECT 1 FROM master_add_node('localhost', 57638);
 
 ?column?
 
 1
 step s2-ddl-on-reference-table:
-	CREATE INDEX reference_index ON test_reference_table(test_id);
+ CREATE INDEX reference_index ON test_reference_table(test_id);
  <waiting ...>
 step s1-commit:
     COMMIT;
 
 step s2-ddl-on-reference-table: <... completed>
 step s2-print-index-count:
-	SELECT
-		nodeport, success, result
-	FROM
-		run_command_on_placements('test_reference_table', 'select count(*) from pg_indexes WHERE tablename = ''%s''')
-	ORDER BY
-		nodeport;
+ SELECT
+  nodeport, success, result
+ FROM
+  run_command_on_placements('test_reference_table', 'select count(*) from pg_indexes WHERE tablename = ''%s''')
+ ORDER BY
+  nodeport;
 
 nodeport       success        result
 
 57637          t              1
-57638          t              1
 master_remove_node
 
 
@@ -205,36 +196,35 @@ create_distributed_table
 
 
 step s2-load-metadata-cache:
-	COPY test_reference_table FROM PROGRAM 'echo 1 && echo 2 && echo 3 && echo 4 && echo 5';
+ COPY test_reference_table FROM PROGRAM 'echo 1 && echo 2 && echo 3 && echo 4 && echo 5';
 
 step s2-begin:
-	BEGIN;
+ BEGIN;
 
 step s2-ddl-on-reference-table:
-	CREATE INDEX reference_index ON test_reference_table(test_id);
+ CREATE INDEX reference_index ON test_reference_table(test_id);
 
 step s1-add-second-worker:
-	SELECT 1 FROM master_add_node('localhost', 57638);
+ SELECT 1 FROM master_add_node('localhost', 57638);
  <waiting ...>
 step s2-commit:
-	COMMIT;
+ COMMIT;
 
 step s1-add-second-worker: <... completed>
 ?column?
 
 1
 step s2-print-index-count:
-	SELECT
-		nodeport, success, result
-	FROM
-		run_command_on_placements('test_reference_table', 'select count(*) from pg_indexes WHERE tablename = ''%s''')
-	ORDER BY
-		nodeport;
+ SELECT
+  nodeport, success, result
+ FROM
+  run_command_on_placements('test_reference_table', 'select count(*) from pg_indexes WHERE tablename = ''%s''')
+ ORDER BY
+  nodeport;
 
 nodeport       success        result
 
 57637          t              1
-57638          t              1
 master_remove_node
 
 
@@ -245,19 +235,19 @@ create_distributed_table
 
 
 step s2-load-metadata-cache:
-	COPY test_reference_table FROM PROGRAM 'echo 1 && echo 2 && echo 3 && echo 4 && echo 5';
+ COPY test_reference_table FROM PROGRAM 'echo 1 && echo 2 && echo 3 && echo 4 && echo 5';
 
 step s1-begin:
     BEGIN;
 
 step s1-add-second-worker:
-	SELECT 1 FROM master_add_node('localhost', 57638);
+ SELECT 1 FROM master_add_node('localhost', 57638);
 
 ?column?
 
 1
 step s2-create-reference-table-2:
-	SELECT create_reference_table('test_reference_table_2');
+ SELECT create_reference_table('test_reference_table_2');
  <waiting ...>
 step s1-commit:
     COMMIT;
@@ -267,12 +257,12 @@ create_reference_table
 
 
 step s2-print-content-2:
-	SELECT
-		nodeport, success, result
-	FROM
-		run_command_on_placements('test_reference_table_2', 'select count(*) from %s')
-	ORDER BY
-		nodeport;
+ SELECT
+  nodeport, success, result
+ FROM
+  run_command_on_placements('test_reference_table_2', 'select count(*) from %s')
+ ORDER BY
+  nodeport;
 
 nodeport       success        result
 
@@ -288,39 +278,38 @@ create_distributed_table
 
 
 step s2-load-metadata-cache:
-	COPY test_reference_table FROM PROGRAM 'echo 1 && echo 2 && echo 3 && echo 4 && echo 5';
+ COPY test_reference_table FROM PROGRAM 'echo 1 && echo 2 && echo 3 && echo 4 && echo 5';
 
 step s2-begin:
-	BEGIN;
+ BEGIN;
 
 step s2-create-reference-table-2:
-	SELECT create_reference_table('test_reference_table_2');
+ SELECT create_reference_table('test_reference_table_2');
 
 create_reference_table
 
 
 step s1-add-second-worker:
-	SELECT 1 FROM master_add_node('localhost', 57638);
+ SELECT 1 FROM master_add_node('localhost', 57638);
  <waiting ...>
 step s2-commit:
-	COMMIT;
+ COMMIT;
 
 step s1-add-second-worker: <... completed>
 ?column?
 
 1
 step s2-print-content-2:
-	SELECT
-		nodeport, success, result
-	FROM
-		run_command_on_placements('test_reference_table_2', 'select count(*) from %s')
-	ORDER BY
-		nodeport;
+ SELECT
+  nodeport, success, result
+ FROM
+  run_command_on_placements('test_reference_table_2', 'select count(*) from %s')
+ ORDER BY
+  nodeport;
 
 nodeport       success        result
 
 57637          t              1
-57638          t              1
 master_remove_node
 
 
@@ -334,30 +323,28 @@ step s1-begin:
     BEGIN;
 
 step s1-add-second-worker:
-	SELECT 1 FROM master_add_node('localhost', 57638);
+ SELECT 1 FROM master_add_node('localhost', 57638);
 
 ?column?
 
 1
 step s2-copy-to-reference-table:
-	COPY test_reference_table FROM PROGRAM 'echo 1 && echo 2 && echo 3 && echo 4 && echo 5';
- <waiting ...>
+ COPY test_reference_table FROM PROGRAM 'echo 1 && echo 2 && echo 3 && echo 4 && echo 5';
+
 step s1-commit:
     COMMIT;
 
-step s2-copy-to-reference-table: <... completed>
 step s2-print-content:
-	SELECT
-		nodeport, success, result
-	FROM
-		run_command_on_placements('test_reference_table', 'select count(*) from %s')
-	ORDER BY
-		nodeport;
+ SELECT
+  nodeport, success, result
+ FROM
+  run_command_on_placements('test_reference_table', 'select count(*) from %s')
+ ORDER BY
+  nodeport;
 
 nodeport       success        result
 
 57637          t              5
-57638          t              5
 master_remove_node
 
 
@@ -368,33 +355,31 @@ create_distributed_table
 
 
 step s2-begin:
-	BEGIN;
+ BEGIN;
 
 step s2-copy-to-reference-table:
-	COPY test_reference_table FROM PROGRAM 'echo 1 && echo 2 && echo 3 && echo 4 && echo 5';
+ COPY test_reference_table FROM PROGRAM 'echo 1 && echo 2 && echo 3 && echo 4 && echo 5';
 
 step s1-add-second-worker:
-	SELECT 1 FROM master_add_node('localhost', 57638);
- <waiting ...>
-step s2-commit:
-	COMMIT;
+ SELECT 1 FROM master_add_node('localhost', 57638);
 
-step s1-add-second-worker: <... completed>
 ?column?
 
 1
+step s2-commit:
+ COMMIT;
+
 step s2-print-content:
-	SELECT
-		nodeport, success, result
-	FROM
-		run_command_on_placements('test_reference_table', 'select count(*) from %s')
-	ORDER BY
-		nodeport;
+ SELECT
+  nodeport, success, result
+ FROM
+  run_command_on_placements('test_reference_table', 'select count(*) from %s')
+ ORDER BY
+  nodeport;
 
 nodeport       success        result
 
 57637          t              5
-57638          t              5
 master_remove_node
 
 
@@ -408,30 +393,28 @@ step s1-begin:
     BEGIN;
 
 step s1-add-second-worker:
-	SELECT 1 FROM master_add_node('localhost', 57638);
+ SELECT 1 FROM master_add_node('localhost', 57638);
 
 ?column?
 
 1
 step s2-insert-to-reference-table:
-	INSERT INTO test_reference_table VALUES (6);
- <waiting ...>
+ INSERT INTO test_reference_table VALUES (6);
+
 step s1-commit:
     COMMIT;
 
-step s2-insert-to-reference-table: <... completed>
 step s2-print-content:
-	SELECT
-		nodeport, success, result
-	FROM
-		run_command_on_placements('test_reference_table', 'select count(*) from %s')
-	ORDER BY
-		nodeport;
+ SELECT
+  nodeport, success, result
+ FROM
+  run_command_on_placements('test_reference_table', 'select count(*) from %s')
+ ORDER BY
+  nodeport;
 
 nodeport       success        result
 
 57637          t              1
-57638          t              1
 master_remove_node
 
 
@@ -442,33 +425,31 @@ create_distributed_table
 
 
 step s2-begin:
-	BEGIN;
+ BEGIN;
 
 step s2-insert-to-reference-table:
-	INSERT INTO test_reference_table VALUES (6);
+ INSERT INTO test_reference_table VALUES (6);
 
 step s1-add-second-worker:
-	SELECT 1 FROM master_add_node('localhost', 57638);
- <waiting ...>
-step s2-commit:
-	COMMIT;
+ SELECT 1 FROM master_add_node('localhost', 57638);
 
-step s1-add-second-worker: <... completed>
 ?column?
 
 1
+step s2-commit:
+ COMMIT;
+
 step s2-print-content:
-	SELECT
-		nodeport, success, result
-	FROM
-		run_command_on_placements('test_reference_table', 'select count(*) from %s')
-	ORDER BY
-		nodeport;
+ SELECT
+  nodeport, success, result
+ FROM
+  run_command_on_placements('test_reference_table', 'select count(*) from %s')
+ ORDER BY
+  nodeport;
 
 nodeport       success        result
 
 57637          t              1
-57638          t              1
 master_remove_node
 
 
@@ -482,30 +463,29 @@ step s1-begin:
     BEGIN;
 
 step s1-add-second-worker:
-	SELECT 1 FROM master_add_node('localhost', 57638);
+ SELECT 1 FROM master_add_node('localhost', 57638);
 
 ?column?
 
 1
 step s2-ddl-on-reference-table:
-	CREATE INDEX reference_index ON test_reference_table(test_id);
+ CREATE INDEX reference_index ON test_reference_table(test_id);
  <waiting ...>
 step s1-commit:
     COMMIT;
 
 step s2-ddl-on-reference-table: <... completed>
 step s2-print-index-count:
-	SELECT
-		nodeport, success, result
-	FROM
-		run_command_on_placements('test_reference_table', 'select count(*) from pg_indexes WHERE tablename = ''%s''')
-	ORDER BY
-		nodeport;
+ SELECT
+  nodeport, success, result
+ FROM
+  run_command_on_placements('test_reference_table', 'select count(*) from pg_indexes WHERE tablename = ''%s''')
+ ORDER BY
+  nodeport;
 
 nodeport       success        result
 
 57637          t              1
-57638          t              1
 master_remove_node
 
 
@@ -516,33 +496,32 @@ create_distributed_table
 
 
 step s2-begin:
-	BEGIN;
+ BEGIN;
 
 step s2-ddl-on-reference-table:
-	CREATE INDEX reference_index ON test_reference_table(test_id);
+ CREATE INDEX reference_index ON test_reference_table(test_id);
 
 step s1-add-second-worker:
-	SELECT 1 FROM master_add_node('localhost', 57638);
+ SELECT 1 FROM master_add_node('localhost', 57638);
  <waiting ...>
 step s2-commit:
-	COMMIT;
+ COMMIT;
 
 step s1-add-second-worker: <... completed>
 ?column?
 
 1
 step s2-print-index-count:
-	SELECT
-		nodeport, success, result
-	FROM
-		run_command_on_placements('test_reference_table', 'select count(*) from pg_indexes WHERE tablename = ''%s''')
-	ORDER BY
-		nodeport;
+ SELECT
+  nodeport, success, result
+ FROM
+  run_command_on_placements('test_reference_table', 'select count(*) from pg_indexes WHERE tablename = ''%s''')
+ ORDER BY
+  nodeport;
 
 nodeport       success        result
 
 57637          t              1
-57638          t              1
 master_remove_node
 
 
@@ -556,13 +535,13 @@ step s1-begin:
     BEGIN;
 
 step s1-add-second-worker:
-	SELECT 1 FROM master_add_node('localhost', 57638);
+ SELECT 1 FROM master_add_node('localhost', 57638);
 
 ?column?
 
 1
 step s2-create-reference-table-2:
-	SELECT create_reference_table('test_reference_table_2');
+ SELECT create_reference_table('test_reference_table_2');
  <waiting ...>
 step s1-commit:
     COMMIT;
@@ -572,12 +551,12 @@ create_reference_table
 
 
 step s2-print-content-2:
-	SELECT
-		nodeport, success, result
-	FROM
-		run_command_on_placements('test_reference_table_2', 'select count(*) from %s')
-	ORDER BY
-		nodeport;
+ SELECT
+  nodeport, success, result
+ FROM
+  run_command_on_placements('test_reference_table_2', 'select count(*) from %s')
+ ORDER BY
+  nodeport;
 
 nodeport       success        result
 
@@ -593,36 +572,35 @@ create_distributed_table
 
 
 step s2-begin:
-	BEGIN;
+ BEGIN;
 
 step s2-create-reference-table-2:
-	SELECT create_reference_table('test_reference_table_2');
+ SELECT create_reference_table('test_reference_table_2');
 
 create_reference_table
 
 
 step s1-add-second-worker:
-	SELECT 1 FROM master_add_node('localhost', 57638);
+ SELECT 1 FROM master_add_node('localhost', 57638);
  <waiting ...>
 step s2-commit:
-	COMMIT;
+ COMMIT;
 
 step s1-add-second-worker: <... completed>
 ?column?
 
 1
 step s2-print-content-2:
-	SELECT
-		nodeport, success, result
-	FROM
-		run_command_on_placements('test_reference_table_2', 'select count(*) from %s')
-	ORDER BY
-		nodeport;
+ SELECT
+  nodeport, success, result
+ FROM
+  run_command_on_placements('test_reference_table_2', 'select count(*) from %s')
+ ORDER BY
+  nodeport;
 
 nodeport       success        result
 
 57637          t              1
-57638          t              1
 master_remove_node
 
 
diff --git a/src/test/regress/expected/local_shard_copy.out b/src/test/regress/expected/local_shard_copy.out
index 5feeb5935..e07f595a8 100644
--- a/src/test/regress/expected/local_shard_copy.out
+++ b/src/test/regress/expected/local_shard_copy.out
@@ -15,10 +15,6 @@ DEBUG:  schema "citus_mx_test_schema_join_2" already exists, skipping
 DETAIL:  NOTICE from localhost:xxxxx
 DEBUG:  schema "citus_mx_schema_for_xacts" already exists, skipping
 DETAIL:  NOTICE from localhost:xxxxx
-NOTICE:  Replicating reference table "customer_mx" to the node localhost:xxxxx
-NOTICE:  Replicating reference table "nation_mx" to the node localhost:xxxxx
-NOTICE:  Replicating reference table "part_mx" to the node localhost:xxxxx
-NOTICE:  Replicating reference table "supplier_mx" to the node localhost:xxxxx
  master_add_node
 ---------------------------------------------------------------------
               32
diff --git a/src/test/regress/expected/multi_metadata_sync.out b/src/test/regress/expected/multi_metadata_sync.out
index 31cd05e43..99557f0b6 100644
--- a/src/test/regress/expected/multi_metadata_sync.out
+++ b/src/test/regress/expected/multi_metadata_sync.out
@@ -1394,7 +1394,6 @@ WHERE logicalrelid='mx_ref'::regclass;
 
 \c - - - :master_port
 SELECT master_add_node('localhost', :worker_2_port);
-NOTICE:  Replicating reference table "mx_ref" to the node localhost:xxxxx
  master_add_node
 ---------------------------------------------------------------------
                7
@@ -1407,8 +1406,7 @@ ORDER BY shardid, nodeport;
  shardid | nodename  | nodeport
 ---------------------------------------------------------------------
  1310073 | localhost |    57637
- 1310073 | localhost |    57638
-(2 rows)
+(1 row)
 
 \c - - - :worker_1_port
 SELECT shardid, nodename, nodeport
@@ -1418,8 +1416,7 @@ ORDER BY shardid, nodeport;
  shardid | nodename  | nodeport
 ---------------------------------------------------------------------
  1310073 | localhost |    57637
- 1310073 | localhost |    57638
-(2 rows)
+(1 row)
 
 -- Get the metadata back into a consistent state
 \c - - - :master_port
diff --git a/src/test/regress/expected/multi_mx_node_metadata.out b/src/test/regress/expected/multi_mx_node_metadata.out
index b44d69a84..8f6720887 100644
--- a/src/test/regress/expected/multi_mx_node_metadata.out
+++ b/src/test/regress/expected/multi_mx_node_metadata.out
@@ -173,7 +173,6 @@ SELECT nodeid, hasmetadata, metadatasynced FROM pg_dist_node;
 -- Test updating a node when another node is in readonly-mode
 ---------------------------------------------------------------------
 SELECT master_add_node('localhost', :worker_2_port) AS nodeid_2 \gset
-NOTICE:  Replicating reference table "ref_table" to the node localhost:xxxxx
 SELECT 1 FROM start_metadata_sync_to_node('localhost', :worker_2_port);
  ?column?
 ---------------------------------------------------------------------
@@ -383,7 +382,6 @@ SELECT verify_metadata('localhost', :worker_1_port);
 (1 row)
 
 SELECT 1 FROM master_activate_node('localhost', :worker_2_port);
-NOTICE:  Replicating reference table "ref_table" to the node localhost:xxxxx
  ?column?
 ---------------------------------------------------------------------
         1
@@ -449,7 +447,6 @@ SELECT wait_until_metadata_sync();
 (1 row)
 
 SELECT 1 FROM master_activate_node('localhost', :worker_2_port);
-NOTICE:  Replicating reference table "ref_table" to the node localhost:xxxxx
  ?column?
 ---------------------------------------------------------------------
         1
@@ -511,7 +508,6 @@ SELECT wait_until_metadata_sync();
 (1 row)
 
 SELECT 1 FROM master_activate_node('localhost', :worker_2_port);
-NOTICE:  Replicating reference table "ref_table" to the node localhost:xxxxx
  ?column?
 ---------------------------------------------------------------------
         1
diff --git a/src/test/regress/expected/multi_remove_node_reference_table.out b/src/test/regress/expected/multi_remove_node_reference_table.out
index 28fd7f502..97b0d17d6 100644
--- a/src/test/regress/expected/multi_remove_node_reference_table.out
+++ b/src/test/regress/expected/multi_remove_node_reference_table.out
@@ -221,7 +221,6 @@ SELECT master_remove_node('localhost', :worker_2_port);
 ERROR:  node at "localhost:xxxxx" does not exist
 -- re-add the node for next tests
 SELECT 1 FROM master_add_node('localhost', :worker_2_port);
-NOTICE:  Replicating reference table "remove_node_reference_table" to the node localhost:xxxxx
  ?column?
 ---------------------------------------------------------------------
         1
@@ -242,7 +241,6 @@ SELECT master_remove_node('localhost', :worker_2_port);
 
 -- re-add the node for the next test
 SELECT 1 FROM master_add_node('localhost', :worker_2_port);
-NOTICE:  Replicating reference table "remove_node_reference_table" to the node localhost:xxxxx
  ?column?
 ---------------------------------------------------------------------
         1
@@ -262,10 +260,9 @@ FROM
     pg_dist_shard_placement
 WHERE
     nodeport = :worker_2_port;
- shardid | shardstate | shardlength | nodename  | nodeport
+ shardid | shardstate | shardlength | nodename | nodeport
 ---------------------------------------------------------------------
- 1380000 |          1 |           0 | localhost |    57638
-(1 row)
+(0 rows)
 
 SELECT *
 FROM pg_dist_colocation
@@ -291,10 +288,9 @@ FROM
     pg_dist_shard_placement
 WHERE
     nodeport = :worker_2_port;
- shardid | shardstate | shardlength | nodename  | nodeport
+ shardid | shardstate | shardlength | nodename | nodeport
 ---------------------------------------------------------------------
- 1380000 |          1 |           0 | localhost |    57638
-(1 row)
+(0 rows)
 
 \c - - - :master_port
 BEGIN;
@@ -318,10 +314,9 @@ FROM
     pg_dist_shard_placement
 WHERE
     nodeport = :worker_2_port;
- shardid | shardstate | shardlength | nodename  | nodeport
+ shardid | shardstate | shardlength | nodename | nodeport
 ---------------------------------------------------------------------
- 1380000 |          1 |           0 | localhost |    57638
-(1 row)
+(0 rows)
 
 SELECT *
 FROM pg_dist_colocation
@@ -347,10 +342,9 @@ FROM
     pg_dist_shard_placement
 WHERE
     nodeport = :worker_2_port;
- shardid | shardstate | shardlength | nodename  | nodeport
+ shardid | shardstate | shardlength | nodename | nodeport
 ---------------------------------------------------------------------
- 1380000 |          1 |           0 | localhost |    57638
-(1 row)
+(0 rows)
 
 \c - - - :master_port
 -- remove node in a transaction and COMMIT
@@ -367,10 +361,9 @@ FROM
     pg_dist_shard_placement
 WHERE
     nodeport = :worker_2_port;
- shardid | shardstate | shardlength | nodename  | nodeport
+ shardid | shardstate | shardlength | nodename | nodeport
 ---------------------------------------------------------------------
- 1380000 |          1 |           0 | localhost |    57638
-(1 row)
+(0 rows)
 
 SELECT *
 FROM pg_dist_colocation
@@ -396,10 +389,9 @@ FROM
     pg_dist_shard_placement
 WHERE
     nodeport = :worker_2_port;
- shardid | shardstate | shardlength | nodename  | nodeport
+ shardid | shardstate | shardlength | nodename | nodeport
 ---------------------------------------------------------------------
- 1380000 |          1 |           0 | localhost |    57638
-(1 row)
+(0 rows)
 
 \c - - - :master_port
 BEGIN;
@@ -458,7 +450,6 @@ WHERE
 \c - - - :master_port
 -- re-add the node for next tests
 SELECT 1 FROM master_add_node('localhost', :worker_2_port);
-NOTICE:  Replicating reference table "remove_node_reference_table" to the node localhost:xxxxx
  ?column?
 ---------------------------------------------------------------------
         1
@@ -478,10 +469,9 @@ FROM
     pg_dist_shard_placement
 WHERE
     nodeport = :worker_2_port;
- shardid | shardstate | shardlength | nodename  | nodeport
+ shardid | shardstate | shardlength | nodename | nodeport
 ---------------------------------------------------------------------
- 1380000 |          1 |           0 | localhost |    57638
-(1 row)
+(0 rows)
 
 SELECT *
 FROM pg_dist_colocation
@@ -507,10 +497,9 @@ FROM
     pg_dist_shard_placement
 WHERE
     nodeport = :worker_2_port;
- shardid | shardstate | shardlength | nodename  | nodeport
+ shardid | shardstate | shardlength | nodename | nodeport
 ---------------------------------------------------------------------
- 1380000 |          1 |           0 | localhost |    57638
-(1 row)
+(0 rows)
 
 \c - - - :master_port
 BEGIN;
@@ -583,7 +572,6 @@ SELECT * FROM remove_node_reference_table;
 \c - - - :master_port
 -- re-add the node for next tests
 SELECT 1 FROM master_add_node('localhost', :worker_2_port);
-NOTICE:  Replicating reference table "remove_node_reference_table" to the node localhost:xxxxx
  ?column?
 ---------------------------------------------------------------------
         1
@@ -603,10 +591,9 @@ FROM
     pg_dist_shard_placement
 WHERE
     nodeport = :worker_2_port;
- shardid | shardstate | shardlength | nodename  | nodeport
+ shardid | shardstate | shardlength | nodename | nodeport
 ---------------------------------------------------------------------
- 1380000 |          1 |           0 | localhost |    57638
-(1 row)
+(0 rows)
 
 SELECT *
 FROM pg_dist_colocation
@@ -632,10 +619,9 @@ FROM
     pg_dist_shard_placement
 WHERE
     nodeport = :worker_2_port;
- shardid | shardstate | shardlength | nodename  | nodeport
+ shardid | shardstate | shardlength | nodename | nodeport
 ---------------------------------------------------------------------
- 1380000 |          1 |           0 | localhost |    57638
-(1 row)
+(0 rows)
 
 \c - - - :master_port
 BEGIN;
@@ -704,7 +690,6 @@ SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.remove_
 
 -- re-add the node for next tests
 SELECT 1 FROM master_add_node('localhost', :worker_2_port);
-NOTICE:  Replicating reference table "remove_node_reference_table" to the node localhost:xxxxx
  ?column?
 ---------------------------------------------------------------------
         1
@@ -724,10 +709,9 @@ FROM
     pg_dist_shard_placement
 WHERE
     nodeport = :worker_2_port;
- shardid | shardstate | shardlength | nodename  | nodeport
+ shardid | shardstate | shardlength | nodename | nodeport
 ---------------------------------------------------------------------
- 1380000 |          1 |           0 | localhost |    57638
-(1 row)
+(0 rows)
 
 SELECT *
 FROM pg_dist_colocation
@@ -903,8 +887,6 @@ WHERE
 \c - - - :master_port
 -- re-add the node for next tests
 SELECT 1 FROM master_add_node('localhost', :worker_2_port);
-NOTICE:  Replicating reference table "remove_node_reference_table" to the node localhost:xxxxx
-NOTICE:  Replicating reference table "table1" to the node localhost:xxxxx
  ?column?
 ---------------------------------------------------------------------
         1
@@ -926,11 +908,9 @@ WHERE
     nodeport = :worker_2_port
 ORDER BY
     shardid;
- shardid | shardstate | shardlength | nodename  | nodeport
+ shardid | shardstate | shardlength | nodename | nodeport
 ---------------------------------------------------------------------
- 1380001 |          1 |           0 | localhost |    57638
- 1380002 |          1 |           0 | localhost |    57638
-(2 rows)
+(0 rows)
 
 SELECT *
 FROM pg_dist_colocation
@@ -957,11 +937,9 @@ FROM
 WHERE
     nodeport = :worker_2_port
 ORDER BY shardid ASC;
- shardid | shardstate | shardlength | nodename  | nodeport
+ shardid | shardstate | shardlength | nodename | nodeport
 ---------------------------------------------------------------------
- 1380001 |          1 |           0 | localhost |    57638
- 1380002 |          1 |           0 | localhost |    57638
-(2 rows)
+(0 rows)
 
 \c - - - :master_port
 SELECT master_disable_node('localhost', :worker_2_port);
@@ -1018,8 +996,6 @@ WHERE
 \c - - - :master_port
 -- re-add the node for next tests
 SELECT 1 FROM master_activate_node('localhost', :worker_2_port);
-NOTICE:  Replicating reference table "remove_node_reference_table" to the node localhost:xxxxx
-NOTICE:  Replicating reference table "table1" to the node localhost:xxxxx
  ?column?
 ---------------------------------------------------------------------
         1
diff --git a/src/test/regress/expected/multi_replicate_reference_table.out b/src/test/regress/expected/multi_replicate_reference_table.out
index e23342e00..9162abc2b 100644
--- a/src/test/regress/expected/multi_replicate_reference_table.out
+++ b/src/test/regress/expected/multi_replicate_reference_table.out
@@ -70,12 +70,16 @@ SELECT create_reference_table('replicate_reference_table_unhealthy');
 
 UPDATE pg_dist_shard_placement SET shardstate = 3 WHERE shardid = 1370000;
 SELECT 1 FROM master_add_node('localhost', :worker_2_port);
-ERROR:  could not find any healthy placement for shard xxxxx
+ ?column?
+---------------------------------------------------------------------
+        1
+(1 row)
+
 -- verify node is not added
 SELECT COUNT(*) FROM pg_dist_node WHERE nodeport = :worker_2_port;
  count
 ---------------------------------------------------------------------
-     0
+     1
 (1 row)
 
 -- verify nothing is replicated to the new node
@@ -106,9 +110,10 @@ FROM
 WHERE
     nodeport = :worker_2_port
 ORDER BY shardid, nodeport;
- shardid | shardstate | shardlength | nodename | nodeport
+ shardid | shardstate | shardlength | nodename  | nodeport
 ---------------------------------------------------------------------
-(0 rows)
+ 1370001 |          1 |           0 | localhost |    57638
+(1 row)
 
 SELECT *
 FROM pg_dist_colocation
@@ -122,7 +127,6 @@ WHERE colocationid IN
 (1 row)
 
 SELECT 1 FROM master_add_node('localhost', :worker_2_port);
-NOTICE:  Replicating reference table "replicate_reference_table_valid" to the node localhost:xxxxx
  ?column?
 ---------------------------------------------------------------------
         1
@@ -247,7 +251,6 @@ WHERE colocationid IN
 
 BEGIN;
 SELECT 1 FROM master_add_node('localhost', :worker_2_port);
-NOTICE:  Replicating reference table "replicate_reference_table_rollback" to the node localhost:xxxxx
  ?column?
 ---------------------------------------------------------------------
         1
@@ -311,7 +314,6 @@ WHERE colocationid IN
 
 BEGIN;
 SELECT 1 FROM master_add_node('localhost', :worker_2_port);
-NOTICE:  Replicating reference table "replicate_reference_table_commit" to the node localhost:xxxxx
  ?column?
 ---------------------------------------------------------------------
         1
@@ -326,10 +328,9 @@ FROM
 WHERE
     nodeport = :worker_2_port
 ORDER BY shardid, nodeport;
- shardid | shardstate | shardlength | nodename  | nodeport
+ shardid | shardstate | shardlength | nodename | nodeport
 ---------------------------------------------------------------------
- 1370003 |          1 |           0 | localhost |    57638
-(1 row)
+(0 rows)
 
 SELECT *
 FROM pg_dist_colocation
@@ -406,8 +407,7 @@ ORDER BY logicalrelid;
  replicate_reference_table_hash          | h          | f        | c
 (2 rows)
 
-BEGIN;
-SET LOCAL client_min_messages TO ERROR;
+SET client_min_messages TO WARNING;
 SELECT 1 FROM master_add_node('localhost', :worker_2_port);
  ?column?
 ---------------------------------------------------------------------
@@ -426,7 +426,7 @@ SELECT create_reference_table('replicate_reference_table_reference_two');
 
 (1 row)
 
-COMMIT;
+RESET client_min_messages;
 -- status after master_add_node
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
@@ -554,7 +554,6 @@ WHERE colocationid IN
 
 BEGIN;
 SELECT 1 FROM master_add_node('localhost', :worker_2_port);
-NOTICE:  Replicating reference table "replicate_reference_table_drop" to the node localhost:xxxxx
  ?column?
 ---------------------------------------------------------------------
         1
@@ -618,7 +617,6 @@ WHERE colocationid IN
 (1 row)
 
 SELECT 1 FROM master_add_node('localhost', :worker_2_port);
-NOTICE:  Replicating reference table "table1" to the node localhost:xxxxx
  ?column?
 ---------------------------------------------------------------------
         1
@@ -632,10 +630,9 @@ FROM
 WHERE
     nodeport = :worker_2_port
 ORDER BY shardid, nodeport;
- shardid | shardstate | shardlength | nodename  | nodeport
+ shardid | shardstate | shardlength | nodename | nodeport
 ---------------------------------------------------------------------
- 1370011 |          1 |           0 | localhost |    57638
-(1 row)
+(0 rows)
 
 SELECT *
 FROM pg_dist_colocation
@@ -681,9 +678,6 @@ ORDER BY shardid, nodeport;
 (0 rows)
 
 SELECT 1 FROM master_add_node('localhost', :worker_2_port);
-NOTICE:  Replicating reference table "ref_table_1" to the node localhost:xxxxx
-NOTICE:  Replicating reference table "ref_table_2" to the node localhost:xxxxx
-NOTICE:  Replicating reference table "ref_table_3" to the node localhost:xxxxx
  ?column?
 ---------------------------------------------------------------------
         1
@@ -697,19 +691,16 @@ FROM
 WHERE
     nodeport = :worker_2_port
 ORDER BY shardid, nodeport;
- shardid | shardstate | shardlength | nodename  | nodeport
+ shardid | shardstate | shardlength | nodename | nodeport
 ---------------------------------------------------------------------
- 1370012 |          1 |           0 | localhost |    57638
- 1370013 |          1 |           0 | localhost |    57638
- 1370014 |          1 |           0 | localhost |    57638
-(3 rows)
+(0 rows)
 
 -- verify constraints have been created on the new node
 SELECT run_command_on_workers('select count(*) from pg_constraint where contype=''f'' AND conname like ''ref_table%'';');
  run_command_on_workers
 ---------------------------------------------------------------------
  (localhost,57637,t,2)
- (localhost,57638,t,2)
+ (localhost,57638,t,0)
 (2 rows)
 
 DROP TABLE ref_table_1, ref_table_2, ref_table_3;
@@ -754,7 +745,6 @@ ORDER BY 1,4,5;
 
 -- we should see the two shard placements after activation
 SELECT 1 FROM master_activate_node('localhost', :worker_2_port);
-NOTICE:  Replicating reference table "initially_not_replicated_reference_table" to the node localhost:xxxxx
  ?column?
 ---------------------------------------------------------------------
         1
@@ -776,8 +766,7 @@ ORDER BY 1,4,5;
  shardid | shardstate | shardlength | nodename  | nodeport
 ---------------------------------------------------------------------
  1370015 |          1 |           0 | localhost |    57637
- 1370015 |          1 |           0 | localhost |    57638
-(2 rows)
+(1 row)
 
 -- this should have no effect
 SELECT 1 FROM master_add_node('localhost', :worker_2_port);
diff --git a/src/test/regress/expected/multi_transactional_drop_shards.out b/src/test/regress/expected/multi_transactional_drop_shards.out
index 31094b237..93043aa92 100644
--- a/src/test/regress/expected/multi_transactional_drop_shards.out
+++ b/src/test/regress/expected/multi_transactional_drop_shards.out
@@ -655,7 +655,6 @@ ORDER BY
 \c - - - :master_port
 -- try using the coordinator as a worker and then dropping the table
 SELECT 1 FROM master_add_node('localhost', :master_port);
-NOTICE:  Replicating reference table "transactional_drop_reference" to the node localhost:xxxxx
  ?column?
 ---------------------------------------------------------------------
         1
diff --git a/src/test/regress/expected/propagate_extension_commands.out b/src/test/regress/expected/propagate_extension_commands.out
index de6ded967..8138a6e32 100644
--- a/src/test/regress/expected/propagate_extension_commands.out
+++ b/src/test/regress/expected/propagate_extension_commands.out
@@ -247,7 +247,6 @@ SELECT run_command_on_workers($$SELECT extversion FROM pg_extension WHERE extnam
 
 -- and add the other node
 SELECT 1 from master_add_node('localhost', :worker_2_port);
-NOTICE:  Replicating reference table "ref_table_2" to the node localhost:xxxxx
  ?column?
 ---------------------------------------------------------------------
         1
@@ -443,7 +442,6 @@ BEGIN;
 COMMIT;
 -- add the node back
 SELECT 1 from master_add_node('localhost', :worker_2_port);
-NOTICE:  Replicating reference table "t3" to the node localhost:xxxxx
  ?column?
 ---------------------------------------------------------------------
         1
diff --git a/src/test/regress/input/multi_copy.source b/src/test/regress/input/multi_copy.source
index 16b7a0e1b..d58494383 100644
--- a/src/test/regress/input/multi_copy.source
+++ b/src/test/regress/input/multi_copy.source
@@ -502,7 +502,9 @@ SELECT shardid, nodename, nodeport
 	WHERE logicalrelid = 'numbers_append'::regclass order by placementid;
 
 -- add the node back
+SET client_min_messages TO ERROR;
 SELECT 1 FROM master_activate_node('localhost', :worker_1_port);
+RESET client_min_messages;
 RESET citus.shard_replication_factor;
 -- add two new shards and verify they are created at both workers
 COPY numbers_append FROM STDIN WITH (FORMAT 'csv');
diff --git a/src/test/regress/output/multi_copy.source b/src/test/regress/output/multi_copy.source
index 99b3aa124..fce99bc54 100644
--- a/src/test/regress/output/multi_copy.source
+++ b/src/test/regress/output/multi_copy.source
@@ -674,19 +674,14 @@ SELECT shardid, nodename, nodeport
 (4 rows)
 
 -- add the node back
+SET client_min_messages TO ERROR;
 SELECT 1 FROM master_activate_node('localhost', :worker_1_port);
-NOTICE:  Replicating reference table "orders_reference" to the node localhost:57637
-NOTICE:  Replicating reference table "customer" to the node localhost:57637
-NOTICE:  Replicating reference table "nation" to the node localhost:57637
-NOTICE:  Replicating reference table "part" to the node localhost:57637
-NOTICE:  Replicating reference table "supplier" to the node localhost:57637
-NOTICE:  Replicating reference table "multi_outer_join_right_reference" to the node localhost:57637
-NOTICE:  Replicating reference table "multi_outer_join_third_reference" to the node localhost:57637
  ?column? 
 ----------
         1
 (1 row)
 
+RESET client_min_messages;
 RESET citus.shard_replication_factor;
 -- add two new shards and verify they are created at both workers
 COPY numbers_append FROM STDIN WITH (FORMAT 'csv');
diff --git a/src/test/regress/sql/failure_add_disable_node.sql b/src/test/regress/sql/failure_add_disable_node.sql
index 716710d84..4cb8cd860 100644
--- a/src/test/regress/sql/failure_add_disable_node.sql
+++ b/src/test/regress/sql/failure_add_disable_node.sql
@@ -39,13 +39,6 @@ FROM pg_dist_placement p JOIN pg_dist_shard s USING (shardid)
 WHERE s.logicalrelid = 'user_table'::regclass
 ORDER BY placementid;
 
--- fail activate node by failing reference table creation
-SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").kill()');
-
-SELECT master_activate_node('localhost', :worker_2_proxy_port);
-
-SELECT citus.mitmproxy('conn.allow()');
-
 -- verify node is not activated
 SELECT * FROM master_get_active_worker_nodes()
 ORDER BY 1, 2;
@@ -69,22 +62,6 @@ FROM pg_dist_placement p JOIN pg_dist_shard s USING (shardid)
 WHERE s.logicalrelid = 'user_table'::regclass
 ORDER BY placementid;
 
--- fail activate node by failing reference table creation
-SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").cancel(' || pg_backend_pid() || ')');
-
-SELECT master_activate_node('localhost', :worker_2_proxy_port);
-
--- verify node is not activated
-SELECT * FROM master_get_active_worker_nodes()
-ORDER BY 1, 2;
-
-SELECT shardid, shardstate
-FROM pg_dist_placement p JOIN pg_dist_shard s USING (shardid)
-WHERE s.logicalrelid = 'user_table'::regclass
-ORDER BY placementid;
-
-SELECT citus.mitmproxy('conn.allow()');
-
 -- master_remove_node fails when there are shards on that worker
 SELECT master_remove_node('localhost', :worker_2_proxy_port);
 
@@ -113,34 +90,6 @@ FROM pg_dist_placement p JOIN pg_dist_shard s USING (shardid)
 WHERE s.logicalrelid = 'user_table'::regclass
 ORDER BY placementid;
 
--- test master_add_node replicated a reference table
--- to newly added node.
-SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").kill()');
-
-SELECT master_add_node('localhost', :worker_2_proxy_port);
-
--- verify node is not added
-SELECT * FROM master_get_active_worker_nodes()
-ORDER BY 1, 2;
-
-SELECT shardid, shardstate
-FROM pg_dist_placement p JOIN pg_dist_shard s USING (shardid)
-WHERE s.logicalrelid = 'user_table'::regclass
-ORDER BY placementid;
-
-SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE").cancel(' || pg_backend_pid() || ')');
-
-SELECT master_add_node('localhost', :worker_2_proxy_port);
-
--- verify node is not added
-SELECT * FROM master_get_active_worker_nodes()
-ORDER BY 1, 2;
-
-SELECT shardid, shardstate
-FROM pg_dist_placement p JOIN pg_dist_shard s USING (shardid)
-WHERE s.logicalrelid = 'user_table'::regclass
-ORDER BY placementid;
-
 -- reset cluster to original state
 SELECT citus.mitmproxy('conn.allow()');
 SELECT master_add_node('localhost', :worker_2_proxy_port);
@@ -154,15 +103,6 @@ FROM pg_dist_placement p JOIN pg_dist_shard s USING (shardid)
 WHERE s.logicalrelid = 'user_table'::regclass
 ORDER BY placementid;
 
--- fail master_add_node by failing copy out operation
-SELECT master_remove_node('localhost', :worker_1_port);
-SELECT citus.mitmproxy('conn.onQuery(query="COPY").kill()');
-SELECT master_add_node('localhost', :worker_1_port);
-
--- verify node is not added
-SELECT * FROM master_get_active_worker_nodes()
-ORDER BY 1, 2;
-
 SELECT citus.mitmproxy('conn.allow()');
 SELECT master_add_node('localhost', :worker_1_port);
 
diff --git a/src/test/regress/sql/multi_replicate_reference_table.sql b/src/test/regress/sql/multi_replicate_reference_table.sql
index 8edfa40b7..ea80a3761 100644
--- a/src/test/regress/sql/multi_replicate_reference_table.sql
+++ b/src/test/regress/sql/multi_replicate_reference_table.sql
@@ -268,12 +268,11 @@ WHERE
     logicalrelid IN ('replicate_reference_table_reference_one', 'replicate_reference_table_hash', 'replicate_reference_table_reference_two')
 ORDER BY logicalrelid;
 
-BEGIN;
-SET LOCAL client_min_messages TO ERROR;
+SET client_min_messages TO WARNING;
 SELECT 1 FROM master_add_node('localhost', :worker_2_port);
 SELECT upgrade_to_reference_table('replicate_reference_table_hash');
 SELECT create_reference_table('replicate_reference_table_reference_two');
-COMMIT;
+RESET client_min_messages;
 
 -- status after master_add_node
 SELECT

From 0758a81287a401906685fa169d54d5b70e3e34dd Mon Sep 17 00:00:00 2001
From: Hadi Moshayedi <hadi@moshayedi.net>
Date: Fri, 3 Apr 2020 10:28:18 -0700
Subject: [PATCH 2/5] Prevent reference tables being dropped when replicating
 reference tables

---
 src/backend/distributed/commands/schema.c     |   9 +
 src/backend/distributed/commands/table.c      |   8 +
 .../distributed/metadata/metadata_cache.c     |  49 ++++
 .../distributed/utils/reference_table_utils.c |  77 +++--
 src/include/distributed/metadata_cache.h      |   1 +
 .../distributed/reference_table_utils.h       |   1 -
 .../expected/insert_select_repartition.out    |   1 +
 ...add_node_vs_reference_table_operations.out |  39 +++
 ...lation_create_table_vs_add_remove_node.out | 275 +++++++++---------
 .../isolation_dump_global_wait_edges.out      |  18 +-
 ...lation_ensure_dependency_activate_node.out |   6 +-
 ...dd_node_vs_reference_table_operations.spec |  20 +-
 ...ation_ensure_dependency_activate_node.spec |   2 +-
 13 files changed, 308 insertions(+), 198 deletions(-)

diff --git a/src/backend/distributed/commands/schema.c b/src/backend/distributed/commands/schema.c
index 774a99272..a0ba540bf 100644
--- a/src/backend/distributed/commands/schema.c
+++ b/src/backend/distributed/commands/schema.c
@@ -26,6 +26,8 @@
 #include "distributed/metadata/distobject.h"
 #include "distributed/metadata_cache.h"
 #include <distributed/metadata_sync.h>
+#include "distributed/reference_table_utils.h"
+#include "distributed/resource_lock.h"
 #include <distributed/remote_commands.h>
 #include <distributed/remote_commands.h>
 #include "nodes/parsenodes.h"
@@ -90,6 +92,13 @@ PreprocessDropSchemaStmt(Node *node, const char *queryString)
 				continue;
 			}
 
+			if (IsReferenceTable(relationId))
+			{
+				/* prevent concurrent EnsureReferenceTablesExistOnAllNodes */
+				int colocationId = CreateReferenceTableColocationId();
+				LockColocationId(colocationId, ExclusiveLock);
+			}
+
 			/* invalidate foreign key cache if the table involved in any foreign key */
 			if (TableReferenced(relationId) || TableReferencing(relationId))
 			{
diff --git a/src/backend/distributed/commands/table.c b/src/backend/distributed/commands/table.c
index c2fe4504c..f8f397b47 100644
--- a/src/backend/distributed/commands/table.c
+++ b/src/backend/distributed/commands/table.c
@@ -29,6 +29,7 @@
 #include "distributed/metadata_sync.h"
 #include "distributed/multi_executor.h"
 #include "distributed/multi_partitioning_utils.h"
+#include "distributed/reference_table_utils.h"
 #include "distributed/relation_access_tracking.h"
 #include "distributed/resource_lock.h"
 #include "distributed/version_compat.h"
@@ -85,6 +86,13 @@ PreprocessDropTableStmt(Node *node, const char *queryString)
 			continue;
 		}
 
+		if (IsReferenceTable(relationId))
+		{
+			/* prevent concurrent EnsureReferenceTablesExistOnAllNodes */
+			int colocationId = CreateReferenceTableColocationId();
+			LockColocationId(colocationId, ExclusiveLock);
+		}
+
 		/* invalidate foreign key cache if the table involved in any foreign key */
 		if ((TableReferenced(relationId) || TableReferencing(relationId)))
 		{
diff --git a/src/backend/distributed/metadata/metadata_cache.c b/src/backend/distributed/metadata/metadata_cache.c
index 1c4b4673b..fa5022d88 100644
--- a/src/backend/distributed/metadata/metadata_cache.c
+++ b/src/backend/distributed/metadata/metadata_cache.c
@@ -3591,6 +3591,55 @@ DistTableOidList(void)
 }
 
 
+/*
+ * ReferenceTableOidList function scans pg_dist_partition to create a list of all
+ * reference tables. To create the list, it performs sequential scan. Since it is not
+ * expected that this function will be called frequently, it is OK not to use index scan.
+ * If this function becomes performance bottleneck, it is possible to modify this function
+ * to perform index scan.
+ */
+List *
+ReferenceTableOidList()
+{
+	ScanKeyData scanKey[1];
+	int scanKeyCount = 0;
+	List *referenceTableOidList = NIL;
+
+	Relation pgDistPartition = heap_open(DistPartitionRelationId(), AccessShareLock);
+
+	SysScanDesc scanDescriptor = systable_beginscan(pgDistPartition,
+													InvalidOid, false,
+													NULL, scanKeyCount, scanKey);
+
+	TupleDesc tupleDescriptor = RelationGetDescr(pgDistPartition);
+
+	HeapTuple heapTuple = systable_getnext(scanDescriptor);
+	while (HeapTupleIsValid(heapTuple))
+	{
+		bool isNull = false;
+		Datum relationIdDatum = heap_getattr(heapTuple,
+											 Anum_pg_dist_partition_logicalrelid,
+											 tupleDescriptor, &isNull);
+		Oid relationId = DatumGetObjectId(relationIdDatum);
+		char partitionMethod = heap_getattr(heapTuple,
+											Anum_pg_dist_partition_partmethod,
+											tupleDescriptor, &isNull);
+
+		if (partitionMethod == DISTRIBUTE_BY_NONE)
+		{
+			referenceTableOidList = lappend_oid(referenceTableOidList, relationId);
+		}
+
+		heapTuple = systable_getnext(scanDescriptor);
+	}
+
+	systable_endscan(scanDescriptor);
+	heap_close(pgDistPartition, AccessShareLock);
+
+	return referenceTableOidList;
+}
+
+
 /*
  * InvalidateNodeRelationCacheCallback destroys the WorkerNodeHash when
  * any change happens on pg_dist_node table. It also set WorkerNodeHash to
diff --git a/src/backend/distributed/utils/reference_table_utils.c b/src/backend/distributed/utils/reference_table_utils.c
index ea5265e00..923a008ee 100644
--- a/src/backend/distributed/utils/reference_table_utils.c
+++ b/src/backend/distributed/utils/reference_table_utils.c
@@ -41,8 +41,8 @@
 
 /* local function forward declarations */
 static List * WorkersWithoutReferenceTablePlacement(uint64 shardId);
-static void CopyShardPlacementToNewWorkerNode(ShardPlacement *sourceShardPlacement,
-											  WorkerNode *newWorkerNode);
+static void CopyShardPlacementToWorkerNode(ShardPlacement *sourceShardPlacement,
+										   WorkerNode *workerNode, const char *userName);
 static void ReplicateSingleShardTableToAllNodes(Oid relationId);
 static void ReplicateShardToAllNodes(ShardInterval *shardInterval);
 static void ReplicateShardToNode(ShardInterval *shardInterval, char *nodeName,
@@ -100,10 +100,25 @@ replicate_reference_tables(PG_FUNCTION_ARGS)
 void
 EnsureReferenceTablesExistOnAllNodes(void)
 {
+	/*
+	 * TODO: remove this. This is here so we don't have to update regression test
+	 * outputs because of reference table colocation id being changed. Specially
+	 * multi_colocation_utils which requires the test itself being updated.
+	 */
+	if (ReferenceTableOidList() == NIL)
+	{
+		return;
+	}
+
+	/* prevent this function from running concurrently with itself */
+	int colocationId = CreateReferenceTableColocationId();
+	LockColocationId(colocationId, ExclusiveLock);
+
 	List *referenceTableIdList = ReferenceTableOidList();
-	if (list_length(referenceTableIdList) == 0)
+	if (referenceTableIdList == NIL)
 	{
 		/* no reference tables exist */
+		UnlockColocationId(colocationId, ExclusiveLock);
 		return;
 	}
 
@@ -119,10 +134,6 @@ EnsureReferenceTablesExistOnAllNodes(void)
 	ShardInterval *shardInterval = (ShardInterval *) linitial(shardIntervalList);
 	uint64 shardId = shardInterval->shardId;
 
-	/* prevent this function from running concurrently with itself */
-	int colocationId = TableColocationId(referenceTableId);
-	LockColocationId(colocationId, ExclusiveLock);
-
 	List *newWorkersList = WorkersWithoutReferenceTablePlacement(shardId);
 	if (list_length(newWorkersList) == 0)
 	{
@@ -168,7 +179,13 @@ EnsureReferenceTablesExistOnAllNodes(void)
 	WorkerNode *newWorkerNode = NULL;
 	foreach_ptr(newWorkerNode, newWorkersList)
 	{
-		CopyShardPlacementToNewWorkerNode(sourceShardPlacement, newWorkerNode);
+		/*
+		 * Call master_copy_shard_placement using citus extension owner. Current
+		 * user might not have permissions to do the copy.
+		 */
+		const char *userName = CitusExtensionOwnerName();
+		CopyShardPlacementToWorkerNode(sourceShardPlacement, newWorkerNode,
+									   userName);
 	}
 
 	/*
@@ -235,16 +252,16 @@ WorkersWithoutReferenceTablePlacement(uint64 shardId)
 
 
 /*
- * CopyShardPlacementToNewWorkerNode runs master_copy_shard_placement in a
- * subtransaction by connecting to localhost.
+ * CopyShardPlacementToWorkerNode runs master_copy_shard_placement
+ * using the given username by connecting to localhost.
  */
 static void
-CopyShardPlacementToNewWorkerNode(ShardPlacement *sourceShardPlacement,
-								  WorkerNode *newWorkerNode)
+CopyShardPlacementToWorkerNode(ShardPlacement *sourceShardPlacement,
+							   WorkerNode *workerNode,
+							   const char *userName)
 {
 	int connectionFlags = OUTSIDE_TRANSACTION;
 	StringInfo queryString = makeStringInfo();
-	const char *userName = CitusExtensionOwnerName();
 
 	MultiConnection *connection = GetNodeUserDatabaseConnection(
 		connectionFlags, "localhost", PostPortNumber,
@@ -256,8 +273,10 @@ CopyShardPlacementToNewWorkerNode(ShardPlacement *sourceShardPlacement,
 					 sourceShardPlacement->shardId,
 					 quote_literal_cstr(sourceShardPlacement->nodeName),
 					 sourceShardPlacement->nodePort,
-					 quote_literal_cstr(newWorkerNode->workerName),
-					 newWorkerNode->workerPort);
+					 quote_literal_cstr(workerNode->workerName),
+					 workerNode->workerPort);
+
+	elog(DEBUG3, "%s", queryString->data);
 
 	ExecuteCriticalRemoteCommand(connection, queryString->data);
 }
@@ -605,34 +624,6 @@ DeleteAllReferenceTablePlacementsFromNodeGroup(int32 groupId)
 }
 
 
-/*
- * ReferenceTableOidList function scans pg_dist_partition to create a list of all
- * reference tables. To create the list, it performs sequential scan. Since it is not
- * expected that this function will be called frequently, it is OK not to use index scan.
- * If this function becomes performance bottleneck, it is possible to modify this function
- * to perform index scan.
- */
-List *
-ReferenceTableOidList()
-{
-	List *referenceTableList = NIL;
-
-	List *distTableOidList = DistTableOidList();
-	Oid relationId = InvalidOid;
-	foreach_oid(relationId, distTableOidList)
-	{
-		CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(relationId);
-
-		if (cacheEntry->partitionMethod == DISTRIBUTE_BY_NONE)
-		{
-			referenceTableList = lappend_oid(referenceTableList, relationId);
-		}
-	}
-
-	return referenceTableList;
-}
-
-
 /* CompareOids is a comparison function for sort shard oids */
 int
 CompareOids(const void *leftElement, const void *rightElement)
diff --git a/src/include/distributed/metadata_cache.h b/src/include/distributed/metadata_cache.h
index ec2475c2b..1b042c7e3 100644
--- a/src/include/distributed/metadata_cache.h
+++ b/src/include/distributed/metadata_cache.h
@@ -130,6 +130,7 @@ extern DistObjectCacheEntry * LookupDistObjectCacheEntry(Oid classid, Oid objid,
 														 objsubid);
 extern int32 GetLocalGroupId(void);
 extern List * DistTableOidList(void);
+extern List * ReferenceTableOidList(void);
 extern Oid LookupShardRelation(int64 shardId, bool missing_ok);
 extern List * ShardPlacementList(uint64 shardId);
 extern void CitusInvalidateRelcacheByRelid(Oid relationId);
diff --git a/src/include/distributed/reference_table_utils.h b/src/include/distributed/reference_table_utils.h
index 073aa731a..4484e4411 100644
--- a/src/include/distributed/reference_table_utils.h
+++ b/src/include/distributed/reference_table_utils.h
@@ -20,7 +20,6 @@ extern bool IsReferenceTable(Oid relationId);
 extern void EnsureReferenceTablesExistOnAllNodes(void);
 extern uint32 CreateReferenceTableColocationId(void);
 extern void DeleteAllReferenceTablePlacementsFromNodeGroup(int32 groupId);
-extern List * ReferenceTableOidList(void);
 extern int CompareOids(const void *leftElement, const void *rightElement);
 extern int ReferenceTableReplicationFactor(void);
 
diff --git a/src/test/regress/expected/insert_select_repartition.out b/src/test/regress/expected/insert_select_repartition.out
index 87d4ac185..1e3b95a7b 100644
--- a/src/test/regress/expected/insert_select_repartition.out
+++ b/src/test/regress/expected/insert_select_repartition.out
@@ -1122,6 +1122,7 @@ ON CONFLICT(c1, c2, c3, c4, c5, c6)
 DO UPDATE SET
  cardinality = enriched.cardinality + excluded.cardinality,
  sum = enriched.sum + excluded.sum;
+DEBUG:  rehashing catalog cache id 14 for pg_opclass; 17 tups, 8 buckets at character 224
 DEBUG:  INSERT target table and the source relation of the SELECT partition column value must be colocated in distributed INSERT ... SELECT
 DEBUG:  Router planner cannot handle multi-shard select queries
 DEBUG:  performing repartitioned INSERT ... SELECT
diff --git a/src/test/regress/expected/isolation_add_node_vs_reference_table_operations.out b/src/test/regress/expected/isolation_add_node_vs_reference_table_operations.out
index 7049099db..a7a3eb3a2 100644
--- a/src/test/regress/expected/isolation_add_node_vs_reference_table_operations.out
+++ b/src/test/regress/expected/isolation_add_node_vs_reference_table_operations.out
@@ -605,3 +605,42 @@ master_remove_node
 
 
 
+
+starting permutation: s1-add-second-worker s2-begin s1-begin s1-drop-reference-table s2-replicate-reference-tables s1-commit s2-commit
+create_distributed_table
+
+
+step s1-add-second-worker:
+ SELECT 1 FROM master_add_node('localhost', 57638);
+
+?column?
+
+1
+step s2-begin:
+ BEGIN;
+
+step s1-begin:
+    BEGIN;
+
+step s1-drop-reference-table:
+ DROP TABLE test_reference_table;
+
+step s2-replicate-reference-tables:
+ SET client_min_messages TO DEBUG2;
+ SELECT replicate_reference_tables();
+ RESET client_min_messages;
+ <waiting ...>
+step s1-commit:
+    COMMIT;
+
+step s2-replicate-reference-tables: <... completed>
+replicate_reference_tables
+
+
+step s2-commit:
+ COMMIT;
+
+master_remove_node
+
+
+
diff --git a/src/test/regress/expected/isolation_create_table_vs_add_remove_node.out b/src/test/regress/expected/isolation_create_table_vs_add_remove_node.out
index ed61cf589..cdfcca4e6 100644
--- a/src/test/regress/expected/isolation_create_table_vs_add_remove_node.out
+++ b/src/test/regress/expected/isolation_create_table_vs_add_remove_node.out
@@ -5,36 +5,36 @@ node_name      node_port
 
 localhost      57637
 step s1-begin:
-	BEGIN;
+ BEGIN;
 
 step s1-add-node-2:
-	SELECT 1 FROM master_add_node('localhost', 57638);
+ SELECT 1 FROM master_add_node('localhost', 57638);
 
 ?column?
 
 1
 step s2-create-table-1:
-	SET citus.shard_count TO 4;
-	SET citus.shard_replication_factor TO 1;
-	CREATE TABLE dist_table (x int, y int);
-	SELECT create_distributed_table('dist_table', 'x');
+ SET citus.shard_count TO 4;
+ SET citus.shard_replication_factor TO 1;
+ CREATE TABLE dist_table (x int, y int);
+ SELECT create_distributed_table('dist_table', 'x');
  <waiting ...>
 step s1-commit:
-	COMMIT;
+ COMMIT;
 
 step s2-create-table-1: <... completed>
 create_distributed_table
 
 
 step s1-show-placements:
-	SELECT
-		nodename, nodeport
-	FROM
-		pg_dist_shard_placement JOIN pg_dist_shard USING (shardid)
-	WHERE
-		logicalrelid = 'dist_table'::regclass
-	ORDER BY
-		nodename, nodeport;
+ SELECT
+  nodename, nodeport
+ FROM
+  pg_dist_shard_placement JOIN pg_dist_shard USING (shardid)
+ WHERE
+  logicalrelid = 'dist_table'::regclass
+ ORDER BY
+  nodename, nodeport;
 
 nodename       nodeport
 
@@ -43,7 +43,7 @@ localhost      57637
 localhost      57638
 localhost      57638
 step s2-select:
-	SELECT * FROM dist_table;
+ SELECT * FROM dist_table;
 
 x              y
 
@@ -57,36 +57,36 @@ node_name      node_port
 
 localhost      57637
 step s1-begin:
-	BEGIN;
+ BEGIN;
 
 step s1-add-node-2:
-	SELECT 1 FROM master_add_node('localhost', 57638);
+ SELECT 1 FROM master_add_node('localhost', 57638);
 
 ?column?
 
 1
 step s2-create-table-1:
-	SET citus.shard_count TO 4;
-	SET citus.shard_replication_factor TO 1;
-	CREATE TABLE dist_table (x int, y int);
-	SELECT create_distributed_table('dist_table', 'x');
+ SET citus.shard_count TO 4;
+ SET citus.shard_replication_factor TO 1;
+ CREATE TABLE dist_table (x int, y int);
+ SELECT create_distributed_table('dist_table', 'x');
  <waiting ...>
 step s1-abort:
-	ABORT;
+ ABORT;
 
 step s2-create-table-1: <... completed>
 create_distributed_table
 
 
 step s1-show-placements:
-	SELECT
-		nodename, nodeport
-	FROM
-		pg_dist_shard_placement JOIN pg_dist_shard USING (shardid)
-	WHERE
-		logicalrelid = 'dist_table'::regclass
-	ORDER BY
-		nodename, nodeport;
+ SELECT
+  nodename, nodeport
+ FROM
+  pg_dist_shard_placement JOIN pg_dist_shard USING (shardid)
+ WHERE
+  logicalrelid = 'dist_table'::regclass
+ ORDER BY
+  nodename, nodeport;
 
 nodename       nodeport
 
@@ -95,7 +95,7 @@ localhost      57637
 localhost      57637
 localhost      57637
 step s2-select:
-	SELECT * FROM dist_table;
+ SELECT * FROM dist_table;
 
 x              y
 
@@ -108,36 +108,36 @@ node_name      node_port
 
 localhost      57637
 step s2-begin:
-	BEGIN;
+ BEGIN;
 
 step s2-create-table-1:
-	SET citus.shard_count TO 4;
-	SET citus.shard_replication_factor TO 1;
-	CREATE TABLE dist_table (x int, y int);
-	SELECT create_distributed_table('dist_table', 'x');
+ SET citus.shard_count TO 4;
+ SET citus.shard_replication_factor TO 1;
+ CREATE TABLE dist_table (x int, y int);
+ SELECT create_distributed_table('dist_table', 'x');
 
 create_distributed_table
 
 
 step s1-add-node-2:
-	SELECT 1 FROM master_add_node('localhost', 57638);
+ SELECT 1 FROM master_add_node('localhost', 57638);
  <waiting ...>
 step s2-commit:
-	COMMIT;
+ COMMIT;
 
 step s1-add-node-2: <... completed>
 ?column?
 
 1
 step s1-show-placements:
-	SELECT
-		nodename, nodeport
-	FROM
-		pg_dist_shard_placement JOIN pg_dist_shard USING (shardid)
-	WHERE
-		logicalrelid = 'dist_table'::regclass
-	ORDER BY
-		nodename, nodeport;
+ SELECT
+  nodename, nodeport
+ FROM
+  pg_dist_shard_placement JOIN pg_dist_shard USING (shardid)
+ WHERE
+  logicalrelid = 'dist_table'::regclass
+ ORDER BY
+  nodename, nodeport;
 
 nodename       nodeport
 
@@ -146,7 +146,7 @@ localhost      57637
 localhost      57637
 localhost      57637
 step s2-select:
-	SELECT * FROM dist_table;
+ SELECT * FROM dist_table;
 
 x              y
 
@@ -160,42 +160,42 @@ node_name      node_port
 
 localhost      57637
 step s1-add-node-2:
-	SELECT 1 FROM master_add_node('localhost', 57638);
+ SELECT 1 FROM master_add_node('localhost', 57638);
 
 ?column?
 
 1
 step s1-begin:
-	BEGIN;
+ BEGIN;
 
 step s1-remove-node-2:
-	SELECT * FROM master_remove_node('localhost', 57638);
+ SELECT * FROM master_remove_node('localhost', 57638);
 
 master_remove_node
 
 
 step s2-create-table-1:
-	SET citus.shard_count TO 4;
-	SET citus.shard_replication_factor TO 1;
-	CREATE TABLE dist_table (x int, y int);
-	SELECT create_distributed_table('dist_table', 'x');
+ SET citus.shard_count TO 4;
+ SET citus.shard_replication_factor TO 1;
+ CREATE TABLE dist_table (x int, y int);
+ SELECT create_distributed_table('dist_table', 'x');
  <waiting ...>
 step s1-commit:
-	COMMIT;
+ COMMIT;
 
 step s2-create-table-1: <... completed>
 create_distributed_table
 
 
 step s1-show-placements:
-	SELECT
-		nodename, nodeport
-	FROM
-		pg_dist_shard_placement JOIN pg_dist_shard USING (shardid)
-	WHERE
-		logicalrelid = 'dist_table'::regclass
-	ORDER BY
-		nodename, nodeport;
+ SELECT
+  nodename, nodeport
+ FROM
+  pg_dist_shard_placement JOIN pg_dist_shard USING (shardid)
+ WHERE
+  logicalrelid = 'dist_table'::regclass
+ ORDER BY
+  nodename, nodeport;
 
 nodename       nodeport
 
@@ -204,7 +204,7 @@ localhost      57637
 localhost      57637
 localhost      57637
 step s2-select:
-	SELECT * FROM dist_table;
+ SELECT * FROM dist_table;
 
 x              y
 
@@ -217,42 +217,42 @@ node_name      node_port
 
 localhost      57637
 step s1-add-node-2:
-	SELECT 1 FROM master_add_node('localhost', 57638);
+ SELECT 1 FROM master_add_node('localhost', 57638);
 
 ?column?
 
 1
 step s1-begin:
-	BEGIN;
+ BEGIN;
 
 step s1-remove-node-2:
-	SELECT * FROM master_remove_node('localhost', 57638);
+ SELECT * FROM master_remove_node('localhost', 57638);
 
 master_remove_node
 
 
 step s2-create-table-1:
-	SET citus.shard_count TO 4;
-	SET citus.shard_replication_factor TO 1;
-	CREATE TABLE dist_table (x int, y int);
-	SELECT create_distributed_table('dist_table', 'x');
+ SET citus.shard_count TO 4;
+ SET citus.shard_replication_factor TO 1;
+ CREATE TABLE dist_table (x int, y int);
+ SELECT create_distributed_table('dist_table', 'x');
  <waiting ...>
 step s1-abort:
-	ABORT;
+ ABORT;
 
 step s2-create-table-1: <... completed>
 create_distributed_table
 
 
 step s1-show-placements:
-	SELECT
-		nodename, nodeport
-	FROM
-		pg_dist_shard_placement JOIN pg_dist_shard USING (shardid)
-	WHERE
-		logicalrelid = 'dist_table'::regclass
-	ORDER BY
-		nodename, nodeport;
+ SELECT
+  nodename, nodeport
+ FROM
+  pg_dist_shard_placement JOIN pg_dist_shard USING (shardid)
+ WHERE
+  logicalrelid = 'dist_table'::regclass
+ ORDER BY
+  nodename, nodeport;
 
 nodename       nodeport
 
@@ -261,7 +261,7 @@ localhost      57637
 localhost      57638
 localhost      57638
 step s2-select:
-	SELECT * FROM dist_table;
+ SELECT * FROM dist_table;
 
 x              y
 
@@ -275,40 +275,40 @@ node_name      node_port
 
 localhost      57637
 step s1-add-node-2:
-	SELECT 1 FROM master_add_node('localhost', 57638);
+ SELECT 1 FROM master_add_node('localhost', 57638);
 
 ?column?
 
 1
 step s2-begin:
-	BEGIN;
+ BEGIN;
 
 step s2-create-table-1:
-	SET citus.shard_count TO 4;
-	SET citus.shard_replication_factor TO 1;
-	CREATE TABLE dist_table (x int, y int);
-	SELECT create_distributed_table('dist_table', 'x');
+ SET citus.shard_count TO 4;
+ SET citus.shard_replication_factor TO 1;
+ CREATE TABLE dist_table (x int, y int);
+ SELECT create_distributed_table('dist_table', 'x');
 
 create_distributed_table
 
 
 step s1-remove-node-2:
-	SELECT * FROM master_remove_node('localhost', 57638);
+ SELECT * FROM master_remove_node('localhost', 57638);
  <waiting ...>
 step s2-commit:
-	COMMIT;
+ COMMIT;
 
 step s1-remove-node-2: <... completed>
 error in steps s2-commit s1-remove-node-2: ERROR:  you cannot remove the primary node of a node group which has shard placements
 step s1-show-placements:
-	SELECT
-		nodename, nodeport
-	FROM
-		pg_dist_shard_placement JOIN pg_dist_shard USING (shardid)
-	WHERE
-		logicalrelid = 'dist_table'::regclass
-	ORDER BY
-		nodename, nodeport;
+ SELECT
+  nodename, nodeport
+ FROM
+  pg_dist_shard_placement JOIN pg_dist_shard USING (shardid)
+ WHERE
+  logicalrelid = 'dist_table'::regclass
+ ORDER BY
+  nodename, nodeport;
 
 nodename       nodeport
 
@@ -317,7 +317,7 @@ localhost      57637
 localhost      57638
 localhost      57638
 step s2-select:
-	SELECT * FROM dist_table;
+ SELECT * FROM dist_table;
 
 x              y
 
@@ -331,33 +331,33 @@ node_name      node_port
 
 localhost      57637
 step s1-add-node-2:
-	SELECT 1 FROM master_add_node('localhost', 57638);
+ SELECT 1 FROM master_add_node('localhost', 57638);
 
 ?column?
 
 1
 step s1-begin:
-	BEGIN;
+ BEGIN;
 
 step s1-remove-node-2:
-	SELECT * FROM master_remove_node('localhost', 57638);
+ SELECT * FROM master_remove_node('localhost', 57638);
 
 master_remove_node
 
 
 step s2-create-table-2:
-	SET citus.shard_count TO 4;
-	SET citus.shard_replication_factor TO 2;
-	CREATE TABLE dist_table (x int, y int);
-	SELECT create_distributed_table('dist_table', 'x');
+ SET citus.shard_count TO 4;
+ SET citus.shard_replication_factor TO 2;
+ CREATE TABLE dist_table (x int, y int);
+ SELECT create_distributed_table('dist_table', 'x');
  <waiting ...>
 step s1-commit:
-	COMMIT;
+ COMMIT;
 
 step s2-create-table-2: <... completed>
 error in steps s1-commit s2-create-table-2: ERROR:  replication_factor (2) exceeds number of worker nodes (1)
 step s2-select:
-	SELECT * FROM dist_table;
+ SELECT * FROM dist_table;
 
 ERROR:  relation "dist_table" does not exist
 master_remove_node
@@ -369,33 +369,33 @@ node_name      node_port
 
 localhost      57637
 step s1-add-node-2:
-	SELECT 1 FROM master_add_node('localhost', 57638);
+ SELECT 1 FROM master_add_node('localhost', 57638);
 
 ?column?
 
 1
 step s2-begin:
-	BEGIN;
+ BEGIN;
 
 step s2-create-table-2:
-	SET citus.shard_count TO 4;
-	SET citus.shard_replication_factor TO 2;
-	CREATE TABLE dist_table (x int, y int);
-	SELECT create_distributed_table('dist_table', 'x');
+ SET citus.shard_count TO 4;
+ SET citus.shard_replication_factor TO 2;
+ CREATE TABLE dist_table (x int, y int);
+ SELECT create_distributed_table('dist_table', 'x');
 
 create_distributed_table
 
 
 step s1-remove-node-2:
-	SELECT * FROM master_remove_node('localhost', 57638);
+ SELECT * FROM master_remove_node('localhost', 57638);
  <waiting ...>
 step s2-commit:
-	COMMIT;
+ COMMIT;
 
 step s1-remove-node-2: <... completed>
 error in steps s2-commit s1-remove-node-2: ERROR:  you cannot remove the primary node of a node group which has shard placements
 step s2-select:
-	SELECT * FROM dist_table;
+ SELECT * FROM dist_table;
 
 x              y
 
@@ -409,28 +409,28 @@ node_name      node_port
 
 localhost      57637
 step s1-add-node-2:
-	SELECT 1 FROM master_add_node('localhost', 57638);
+ SELECT 1 FROM master_add_node('localhost', 57638);
 
 ?column?
 
 1
 step s1-begin:
-	BEGIN;
+ BEGIN;
 
 step s1-remove-node-2:
-	SELECT * FROM master_remove_node('localhost', 57638);
+ SELECT * FROM master_remove_node('localhost', 57638);
 
 master_remove_node
 
 
 step s2-create-append-table:
-	SET citus.shard_replication_factor TO 1;
-	CREATE TABLE dist_table (x int, y int);
-	SELECT create_distributed_table('dist_table', 'x', 'append');
-	SELECT 1 FROM master_create_empty_shard('dist_table');
+ SET citus.shard_replication_factor TO 1;
+ CREATE TABLE dist_table (x int, y int);
+ SELECT create_distributed_table('dist_table', 'x', 'append');
+ SELECT 1 FROM master_create_empty_shard('dist_table');
  <waiting ...>
 step s1-commit:
-	COMMIT;
+ COMMIT;
 
 step s2-create-append-table: <... completed>
 create_distributed_table
@@ -440,7 +440,7 @@ create_distributed_table
 
 1
 step s2-select:
-	SELECT * FROM dist_table;
+ SELECT * FROM dist_table;
 
 x              y
 
@@ -453,19 +453,19 @@ node_name      node_port
 
 localhost      57637
 step s1-add-node-2:
-	SELECT 1 FROM master_add_node('localhost', 57638);
+ SELECT 1 FROM master_add_node('localhost', 57638);
 
 ?column?
 
 1
 step s2-begin:
-	BEGIN;
+ BEGIN;
 
 step s2-create-append-table:
-	SET citus.shard_replication_factor TO 1;
-	CREATE TABLE dist_table (x int, y int);
-	SELECT create_distributed_table('dist_table', 'x', 'append');
-	SELECT 1 FROM master_create_empty_shard('dist_table');
+ SET citus.shard_replication_factor TO 1;
+ CREATE TABLE dist_table (x int, y int);
+ SELECT create_distributed_table('dist_table', 'x', 'append');
+ SELECT 1 FROM master_create_empty_shard('dist_table');
 
 create_distributed_table
 
@@ -474,20 +474,19 @@ create_distributed_table
 
 1
 step s1-remove-node-2:
-	SELECT * FROM master_remove_node('localhost', 57638);
+ SELECT * FROM master_remove_node('localhost', 57638);
  <waiting ...>
 step s2-commit:
-	COMMIT;
+ COMMIT;
 
 step s1-remove-node-2: <... completed>
-master_remove_node
-
-
+error in steps s2-commit s1-remove-node-2: ERROR:  you cannot remove the primary node of a node group which has shard placements
 step s2-select:
-	SELECT * FROM dist_table;
+ SELECT * FROM dist_table;
 
 x              y
 
 master_remove_node
 
 
+
diff --git a/src/test/regress/expected/isolation_dump_global_wait_edges.out b/src/test/regress/expected/isolation_dump_global_wait_edges.out
index 037be6803..6e26a07a7 100644
--- a/src/test/regress/expected/isolation_dump_global_wait_edges.out
+++ b/src/test/regress/expected/isolation_dump_global_wait_edges.out
@@ -28,11 +28,11 @@ step detector-dump-wait-edges:
 
 waiting_transaction_numblocking_transaction_numblocking_transaction_waiting
 
-357            356            f
+360            359            f
 transactionnumberwaitingtransactionnumbers
 
-356
-357            356
+359
+360            359
 step s1-abort:
     ABORT;
 
@@ -75,14 +75,14 @@ step detector-dump-wait-edges:
 
 waiting_transaction_numblocking_transaction_numblocking_transaction_waiting
 
-361            360            f
-362            360            f
-362            361            t
+364            363            f
+365            363            f
+365            364            t
 transactionnumberwaitingtransactionnumbers
 
-360
-361            360
-362            360,361
+363
+364            363
+365            363,364
 step s1-abort:
     ABORT;
 
diff --git a/src/test/regress/expected/isolation_ensure_dependency_activate_node.out b/src/test/regress/expected/isolation_ensure_dependency_activate_node.out
index a17b47667..357ca66fd 100644
--- a/src/test/regress/expected/isolation_ensure_dependency_activate_node.out
+++ b/src/test/regress/expected/isolation_ensure_dependency_activate_node.out
@@ -1013,7 +1013,7 @@ master_remove_node
 
 
 
-starting permutation: s1-print-distributed-objects s1-begin s2-begin s3-begin s1-add-worker s2-create-schema s3-create-schema2 s2-create-table s3-create-table s1-commit s2-commit s3-commit s2-print-distributed-objects
+starting permutation: s1-print-distributed-objects s1-begin s2-begin s3-begin s1-add-worker s2-create-schema s3-create-schema2 s2-create-table s3-create-table s1-commit s3-commit s2-commit s2-print-distributed-objects
 ?column?
 
 1
@@ -1109,10 +1109,10 @@ step s3-create-table: <... completed>
 create_distributed_table
 
 
-step s2-commit:
+step s3-commit:
  COMMIT;
 
-step s3-commit:
+step s2-commit:
  COMMIT;
 
 step s2-print-distributed-objects:
diff --git a/src/test/regress/spec/isolation_add_node_vs_reference_table_operations.spec b/src/test/regress/spec/isolation_add_node_vs_reference_table_operations.spec
index 424c25326..9ff681f63 100644
--- a/src/test/regress/spec/isolation_add_node_vs_reference_table_operations.spec
+++ b/src/test/regress/spec/isolation_add_node_vs_reference_table_operations.spec
@@ -16,9 +16,9 @@ setup
 // ensure neither node's added for the remaining of the isolation tests
 teardown
 {
-	DROP TABLE test_reference_table;
-	DROP TABLE test_reference_table_2;
-	DROP TABLE test_table;
+	DROP TABLE IF EXISTS test_reference_table;
+	DROP TABLE IF EXISTS test_reference_table_2;
+	DROP TABLE IF EXISTS test_table;
 	SELECT master_remove_node(nodename, nodeport) FROM pg_dist_node;
 }
 
@@ -39,6 +39,11 @@ step "s1-remove-second-worker"
 	SELECT master_remove_node('localhost', 57638);
 }
 
+step "s1-drop-reference-table"
+{
+	DROP TABLE test_reference_table;
+}
+
 step "s1-commit"
 {
     COMMIT;
@@ -58,6 +63,13 @@ step "s2-copy-to-reference-table"
 	COPY test_reference_table FROM PROGRAM 'echo 1 && echo 2 && echo 3 && echo 4 && echo 5';
 }
 
+step "s2-replicate-reference-tables"
+{
+	SET client_min_messages TO DEBUG2;
+	SELECT replicate_reference_tables();
+	RESET client_min_messages;
+}
+
 step "s2-insert-to-reference-table"
 {
 	INSERT INTO test_reference_table VALUES (6);
@@ -137,3 +149,5 @@ permutation "s2-begin" "s2-ddl-on-reference-table" "s1-add-second-worker" "s2-co
 permutation "s1-begin" "s1-add-second-worker" "s2-create-reference-table-2" "s1-commit" "s2-print-content-2"
 permutation "s2-begin" "s2-create-reference-table-2" "s1-add-second-worker" "s2-commit" "s2-print-content-2"
 
+// verify drop table blocks replicate reference tables
+permutation "s1-add-second-worker" "s2-begin" "s1-begin" "s1-drop-reference-table" "s2-replicate-reference-tables" "s1-commit" "s2-commit"
diff --git a/src/test/regress/spec/isolation_ensure_dependency_activate_node.spec b/src/test/regress/spec/isolation_ensure_dependency_activate_node.spec
index a95624a7e..f36b1cbb5 100644
--- a/src/test/regress/spec/isolation_ensure_dependency_activate_node.spec
+++ b/src/test/regress/spec/isolation_ensure_dependency_activate_node.spec
@@ -206,7 +206,7 @@ permutation "s1-print-distributed-objects" "s1-begin" "s2-begin" "s2-create-sche
 // concurrency tests with multi schema distribution
 permutation "s1-print-distributed-objects" "s2-create-schema" "s1-begin" "s2-begin" "s3-begin" "s1-add-worker" "s2-create-table" "s3-use-schema" "s3-create-table" "s1-commit" "s2-commit" "s3-commit" "s2-print-distributed-objects"
 permutation "s1-print-distributed-objects" "s1-add-worker" "s2-create-schema" "s2-begin" "s3-begin" "s3-use-schema" "s2-create-table" "s3-create-table" "s2-commit" "s3-commit" "s2-print-distributed-objects"
-permutation "s1-print-distributed-objects" "s1-begin" "s2-begin" "s3-begin" "s1-add-worker" "s2-create-schema" "s3-create-schema2" "s2-create-table" "s3-create-table" "s1-commit" "s2-commit" "s3-commit" "s2-print-distributed-objects"
+permutation "s1-print-distributed-objects" "s1-begin" "s2-begin" "s3-begin" "s1-add-worker" "s2-create-schema" "s3-create-schema2" "s2-create-table" "s3-create-table" "s1-commit" "s3-commit" "s2-commit" "s2-print-distributed-objects"
 
 // type and schema tests
 permutation "s1-print-distributed-objects" "s1-begin" "s1-add-worker" "s2-public-schema" "s2-create-type" "s1-commit" "s2-print-distributed-objects"

From acfa850c38db4f663a539c0863c99fcd6a79debc Mon Sep 17 00:00:00 2001
From: Hadi Moshayedi <hadi@moshayedi.net>
Date: Fri, 3 Apr 2020 15:36:43 -0700
Subject: [PATCH 3/5] Make multi_replicate_reference_table check-base friendly

---
 .../multi_replicate_reference_table.out       | 141 ++++++++----------
 .../sql/multi_replicate_reference_table.sql   |  91 +++++------
 2 files changed, 105 insertions(+), 127 deletions(-)

diff --git a/src/test/regress/expected/multi_replicate_reference_table.out b/src/test/regress/expected/multi_replicate_reference_table.out
index 9162abc2b..e17901906 100644
--- a/src/test/regress/expected/multi_replicate_reference_table.out
+++ b/src/test/regress/expected/multi_replicate_reference_table.out
@@ -2,10 +2,15 @@
 -- MULTI_REPLICATE_REFERENCE_TABLE
 --
 -- Tests that check that reference tables are replicated when adding new nodes.
+CREATE SCHEMA replicate_reference_table;
+SET search_path TO replicate_reference_table;
 SET citus.next_shard_id TO 1370000;
 ALTER SEQUENCE pg_catalog.pg_dist_colocationid_seq RESTART 1370000;
 ALTER SEQUENCE pg_catalog.pg_dist_groupid_seq RESTART 1370000;
 ALTER SEQUENCE pg_catalog.pg_dist_node_nodeid_seq RESTART 1370000;
+-- only query shards created in this test
+CREATE VIEW pg_dist_shard_placement_view AS
+SELECT * FROM pg_dist_shard_placement WHERE shardid BETWEEN 1370000 AND 1380000;
 -- remove a node for testing purposes
 CREATE TABLE tmp_shard_placement AS SELECT * FROM pg_dist_shard_placement WHERE nodeport = :worker_2_port;
 DELETE FROM pg_dist_shard_placement WHERE nodeport = :worker_2_port;
@@ -40,7 +45,7 @@ SELECT COUNT(*) FROM pg_dist_node WHERE nodeport = :worker_2_port;
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
 FROM
-    pg_dist_shard_placement
+    pg_dist_shard_placement_view
 WHERE
     nodeport = :worker_2_port;
  shardid | shardstate | shardlength | nodename | nodeport
@@ -86,7 +91,7 @@ SELECT COUNT(*) FROM pg_dist_node WHERE nodeport = :worker_2_port;
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
 FROM
-    pg_dist_shard_placement
+    pg_dist_shard_placement_view
 WHERE
     nodeport = :worker_2_port;
  shardid | shardstate | shardlength | nodename | nodeport
@@ -106,7 +111,7 @@ SELECT create_reference_table('replicate_reference_table_valid');
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
 FROM
-    pg_dist_shard_placement
+    pg_dist_shard_placement_view
 WHERE
     nodeport = :worker_2_port
 ORDER BY shardid, nodeport;
@@ -115,15 +120,14 @@ ORDER BY shardid, nodeport;
  1370001 |          1 |           0 | localhost |    57638
 (1 row)
 
-SELECT *
-FROM pg_dist_colocation
+SELECT shardcount, replicationfactor, distributioncolumntype, distributioncolumncollation FROM pg_dist_colocation
 WHERE colocationid IN
     (SELECT colocationid
      FROM pg_dist_partition
      WHERE logicalrelid = 'replicate_reference_table_valid'::regclass);
- colocationid | shardcount | replicationfactor | distributioncolumntype | distributioncolumncollation
+ shardcount | replicationfactor | distributioncolumntype | distributioncolumncollation
 ---------------------------------------------------------------------
-        10004 |          1 |                -1 |                      0 |                           0
+          1 |                -1 |                      0 |                           0
 (1 row)
 
 SELECT 1 FROM master_add_node('localhost', :worker_2_port);
@@ -136,7 +140,7 @@ SELECT 1 FROM master_add_node('localhost', :worker_2_port);
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
 FROM
-    pg_dist_shard_placement
+    pg_dist_shard_placement_view
 WHERE
     nodeport = :worker_2_port
 ORDER BY shardid, nodeport;
@@ -145,15 +149,14 @@ ORDER BY shardid, nodeport;
  1370001 |          1 |           0 | localhost |    57638
 (1 row)
 
-SELECT *
-FROM pg_dist_colocation
+SELECT shardcount, replicationfactor, distributioncolumntype, distributioncolumncollation FROM pg_dist_colocation
 WHERE colocationid IN
     (SELECT colocationid
      FROM pg_dist_partition
      WHERE logicalrelid = 'replicate_reference_table_valid'::regclass);
- colocationid | shardcount | replicationfactor | distributioncolumntype | distributioncolumncollation
+ shardcount | replicationfactor | distributioncolumntype | distributioncolumncollation
 ---------------------------------------------------------------------
-        10004 |          1 |                -1 |                      0 |                           0
+          1 |                -1 |                      0 |                           0
 (1 row)
 
 -- test add same node twice
@@ -161,7 +164,7 @@ WHERE colocationid IN
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
 FROM
-    pg_dist_shard_placement
+    pg_dist_shard_placement_view
 WHERE
     nodeport = :worker_2_port
 ORDER BY shardid, nodeport;
@@ -170,15 +173,14 @@ ORDER BY shardid, nodeport;
  1370001 |          1 |           0 | localhost |    57638
 (1 row)
 
-SELECT *
-FROM pg_dist_colocation
+SELECT shardcount, replicationfactor, distributioncolumntype, distributioncolumncollation FROM pg_dist_colocation
 WHERE colocationid IN
     (SELECT colocationid
      FROM pg_dist_partition
      WHERE logicalrelid = 'replicate_reference_table_valid'::regclass);
- colocationid | shardcount | replicationfactor | distributioncolumntype | distributioncolumncollation
+ shardcount | replicationfactor | distributioncolumntype | distributioncolumncollation
 ---------------------------------------------------------------------
-        10004 |          1 |                -1 |                      0 |                           0
+          1 |                -1 |                      0 |                           0
 (1 row)
 
 SELECT 1 FROM master_add_node('localhost', :worker_2_port);
@@ -191,7 +193,7 @@ SELECT 1 FROM master_add_node('localhost', :worker_2_port);
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
 FROM
-    pg_dist_shard_placement
+    pg_dist_shard_placement_view
 WHERE
     nodeport = :worker_2_port
 ORDER BY shardid, nodeport;
@@ -200,15 +202,14 @@ ORDER BY shardid, nodeport;
  1370001 |          1 |           0 | localhost |    57638
 (1 row)
 
-SELECT *
-FROM pg_dist_colocation
+SELECT shardcount, replicationfactor, distributioncolumntype, distributioncolumncollation FROM pg_dist_colocation
 WHERE colocationid IN
     (SELECT colocationid
      FROM pg_dist_partition
      WHERE logicalrelid = 'replicate_reference_table_valid'::regclass);
- colocationid | shardcount | replicationfactor | distributioncolumntype | distributioncolumncollation
+ shardcount | replicationfactor | distributioncolumntype | distributioncolumncollation
 ---------------------------------------------------------------------
-        10004 |          1 |                -1 |                      0 |                           0
+          1 |                -1 |                      0 |                           0
 (1 row)
 
 DROP TABLE replicate_reference_table_valid;
@@ -230,7 +231,7 @@ SELECT create_reference_table('replicate_reference_table_rollback');
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
 FROM
-    pg_dist_shard_placement
+    pg_dist_shard_placement_view
 WHERE
     nodeport = :worker_2_port
 ORDER BY shardid, nodeport;
@@ -238,15 +239,14 @@ ORDER BY shardid, nodeport;
 ---------------------------------------------------------------------
 (0 rows)
 
-SELECT *
-FROM pg_dist_colocation
+SELECT shardcount, replicationfactor, distributioncolumntype, distributioncolumncollation FROM pg_dist_colocation
 WHERE colocationid IN
     (SELECT colocationid
      FROM pg_dist_partition
      WHERE logicalrelid = 'replicate_reference_table_rollback'::regclass);
- colocationid | shardcount | replicationfactor | distributioncolumntype | distributioncolumncollation
+ shardcount | replicationfactor | distributioncolumntype | distributioncolumncollation
 ---------------------------------------------------------------------
-        10004 |          1 |                -1 |                      0 |                           0
+          1 |                -1 |                      0 |                           0
 (1 row)
 
 BEGIN;
@@ -261,7 +261,7 @@ ROLLBACK;
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
 FROM
-    pg_dist_shard_placement
+    pg_dist_shard_placement_view
 WHERE
     nodeport = :worker_2_port
 ORDER BY shardid, nodeport;
@@ -269,15 +269,14 @@ ORDER BY shardid, nodeport;
 ---------------------------------------------------------------------
 (0 rows)
 
-SELECT *
-FROM pg_dist_colocation
+SELECT shardcount, replicationfactor, distributioncolumntype, distributioncolumncollation FROM pg_dist_colocation
 WHERE colocationid IN
     (SELECT colocationid
      FROM pg_dist_partition
      WHERE logicalrelid = 'replicate_reference_table_rollback'::regclass);
- colocationid | shardcount | replicationfactor | distributioncolumntype | distributioncolumncollation
+ shardcount | replicationfactor | distributioncolumntype | distributioncolumncollation
 ---------------------------------------------------------------------
-        10004 |          1 |                -1 |                      0 |                           0
+          1 |                -1 |                      0 |                           0
 (1 row)
 
 DROP TABLE replicate_reference_table_rollback;
@@ -293,7 +292,7 @@ SELECT create_reference_table('replicate_reference_table_commit');
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
 FROM
-    pg_dist_shard_placement
+    pg_dist_shard_placement_view
 WHERE
     nodeport = :worker_2_port
 ORDER BY shardid, nodeport;
@@ -301,15 +300,14 @@ ORDER BY shardid, nodeport;
 ---------------------------------------------------------------------
 (0 rows)
 
-SELECT *
-FROM pg_dist_colocation
+SELECT shardcount, replicationfactor, distributioncolumntype, distributioncolumncollation FROM pg_dist_colocation
 WHERE colocationid IN
     (SELECT colocationid
      FROM pg_dist_partition
      WHERE logicalrelid = 'replicate_reference_table_commit'::regclass);
- colocationid | shardcount | replicationfactor | distributioncolumntype | distributioncolumncollation
+ shardcount | replicationfactor | distributioncolumntype | distributioncolumncollation
 ---------------------------------------------------------------------
-        10004 |          1 |                -1 |                      0 |                           0
+          1 |                -1 |                      0 |                           0
 (1 row)
 
 BEGIN;
@@ -324,7 +322,7 @@ COMMIT;
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
 FROM
-    pg_dist_shard_placement
+    pg_dist_shard_placement_view
 WHERE
     nodeport = :worker_2_port
 ORDER BY shardid, nodeport;
@@ -332,15 +330,14 @@ ORDER BY shardid, nodeport;
 ---------------------------------------------------------------------
 (0 rows)
 
-SELECT *
-FROM pg_dist_colocation
+SELECT shardcount, replicationfactor, distributioncolumntype, distributioncolumncollation FROM pg_dist_colocation
 WHERE colocationid IN
     (SELECT colocationid
      FROM pg_dist_partition
      WHERE logicalrelid = 'replicate_reference_table_commit'::regclass);
- colocationid | shardcount | replicationfactor | distributioncolumntype | distributioncolumncollation
+ shardcount | replicationfactor | distributioncolumntype | distributioncolumncollation
 ---------------------------------------------------------------------
-        10004 |          1 |                -1 |                      0 |                           0
+          1 |                -1 |                      0 |                           0
 (1 row)
 
 DROP TABLE replicate_reference_table_commit;
@@ -375,22 +372,21 @@ CREATE TABLE replicate_reference_table_reference_two(column1 int);
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
 FROM
-    pg_dist_shard_placement
+    pg_dist_shard_placement_view
 WHERE
     nodeport = :worker_2_port;
  shardid | shardstate | shardlength | nodename | nodeport
 ---------------------------------------------------------------------
 (0 rows)
 
-SELECT *
-FROM pg_dist_colocation
+SELECT shardcount, replicationfactor, distributioncolumntype, distributioncolumncollation FROM pg_dist_colocation
 WHERE colocationid IN
     (SELECT colocationid
      FROM pg_dist_partition
      WHERE logicalrelid = 'replicate_reference_table_reference_one'::regclass);
- colocationid | shardcount | replicationfactor | distributioncolumntype | distributioncolumncollation
+ shardcount | replicationfactor | distributioncolumntype | distributioncolumncollation
 ---------------------------------------------------------------------
-        10004 |          1 |                -1 |                      0 |                           0
+          1 |                -1 |                      0 |                           0
 (1 row)
 
 SELECT colocationid AS reference_table_colocationid FROM pg_dist_colocation WHERE distributioncolumntype = 0 \gset
@@ -431,7 +427,7 @@ RESET client_min_messages;
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
 FROM
-    pg_dist_shard_placement
+    pg_dist_shard_placement_view
 WHERE
     nodeport = :worker_2_port
 ORDER BY shardid, nodeport;
@@ -442,15 +438,14 @@ ORDER BY shardid, nodeport;
  1370006 |          1 |           0 | localhost |    57638
 (3 rows)
 
-SELECT *
-FROM pg_dist_colocation
+SELECT shardcount, replicationfactor, distributioncolumntype, distributioncolumncollation FROM pg_dist_colocation
 WHERE colocationid IN
     (SELECT colocationid
      FROM pg_dist_partition
      WHERE logicalrelid = 'replicate_reference_table_reference_one'::regclass);
- colocationid | shardcount | replicationfactor | distributioncolumntype | distributioncolumncollation
+ shardcount | replicationfactor | distributioncolumntype | distributioncolumncollation
 ---------------------------------------------------------------------
-        10004 |          1 |                -1 |                      0 |                           0
+          1 |                -1 |                      0 |                           0
 (1 row)
 
 SELECT
@@ -533,7 +528,7 @@ SELECT create_reference_table('replicate_reference_table_drop');
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
 FROM
-    pg_dist_shard_placement
+    pg_dist_shard_placement_view
 WHERE
     nodeport = :worker_2_port
 ORDER BY shardid, nodeport;
@@ -541,15 +536,14 @@ ORDER BY shardid, nodeport;
 ---------------------------------------------------------------------
 (0 rows)
 
-SELECT *
-FROM pg_dist_colocation
+SELECT shardcount, replicationfactor, distributioncolumntype, distributioncolumncollation FROM pg_dist_colocation
 WHERE colocationid IN
     (SELECT colocationid
      FROM pg_dist_partition
      WHERE logicalrelid = 'replicate_reference_table_drop'::regclass);
- colocationid | shardcount | replicationfactor | distributioncolumntype | distributioncolumncollation
+ shardcount | replicationfactor | distributioncolumntype | distributioncolumncollation
 ---------------------------------------------------------------------
-        10004 |          1 |                -1 |                      0 |                           0
+          1 |                -1 |                      0 |                           0
 (1 row)
 
 BEGIN;
@@ -565,7 +559,7 @@ COMMIT;
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
 FROM
-    pg_dist_shard_placement
+    pg_dist_shard_placement_view
 WHERE
     nodeport = :worker_2_port
 ORDER BY shardid, nodeport;
@@ -597,7 +591,7 @@ SELECT create_reference_table('replicate_reference_table_schema.table1');
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
 FROM
-    pg_dist_shard_placement
+    pg_dist_shard_placement_view
 WHERE
     nodeport = :worker_2_port
 ORDER BY shardid, nodeport;
@@ -605,15 +599,14 @@ ORDER BY shardid, nodeport;
 ---------------------------------------------------------------------
 (0 rows)
 
-SELECT *
-FROM pg_dist_colocation
+SELECT shardcount, replicationfactor, distributioncolumntype, distributioncolumncollation FROM pg_dist_colocation
 WHERE colocationid IN
     (SELECT colocationid
      FROM pg_dist_partition
      WHERE logicalrelid = 'replicate_reference_table_schema.table1'::regclass);
- colocationid | shardcount | replicationfactor | distributioncolumntype | distributioncolumncollation
+ shardcount | replicationfactor | distributioncolumntype | distributioncolumncollation
 ---------------------------------------------------------------------
-        10004 |          1 |                -1 |                      0 |                           0
+          1 |                -1 |                      0 |                           0
 (1 row)
 
 SELECT 1 FROM master_add_node('localhost', :worker_2_port);
@@ -626,7 +619,7 @@ SELECT 1 FROM master_add_node('localhost', :worker_2_port);
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
 FROM
-    pg_dist_shard_placement
+    pg_dist_shard_placement_view
 WHERE
     nodeport = :worker_2_port
 ORDER BY shardid, nodeport;
@@ -634,15 +627,14 @@ ORDER BY shardid, nodeport;
 ---------------------------------------------------------------------
 (0 rows)
 
-SELECT *
-FROM pg_dist_colocation
+SELECT shardcount, replicationfactor, distributioncolumntype, distributioncolumncollation FROM pg_dist_colocation
 WHERE colocationid IN
     (SELECT colocationid
      FROM pg_dist_partition
      WHERE logicalrelid = 'replicate_reference_table_schema.table1'::regclass);
- colocationid | shardcount | replicationfactor | distributioncolumntype | distributioncolumncollation
+ shardcount | replicationfactor | distributioncolumntype | distributioncolumncollation
 ---------------------------------------------------------------------
-        10004 |          1 |                -1 |                      0 |                           0
+          1 |                -1 |                      0 |                           0
 (1 row)
 
 DROP TABLE replicate_reference_table_schema.table1;
@@ -669,7 +661,7 @@ SELECT create_reference_table('ref_table_1'),
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
 FROM
-    pg_dist_shard_placement
+    pg_dist_shard_placement_view
 WHERE
     nodeport = :worker_2_port
 ORDER BY shardid, nodeport;
@@ -687,7 +679,7 @@ SELECT 1 FROM master_add_node('localhost', :worker_2_port);
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
 FROM
-    pg_dist_shard_placement
+    pg_dist_shard_placement_view
 WHERE
     nodeport = :worker_2_port
 ORDER BY shardid, nodeport;
@@ -728,7 +720,7 @@ SELECT 1 FROM master_add_inactive_node('localhost', :worker_2_port);
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
 FROM
-    pg_dist_shard_placement
+    pg_dist_shard_placement_view
 WHERE
     shardid IN (SELECT
                     shardid
@@ -753,7 +745,7 @@ SELECT 1 FROM master_activate_node('localhost', :worker_2_port);
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
 FROM
-    pg_dist_shard_placement
+    pg_dist_shard_placement_view
 WHERE
     shardid IN (SELECT
                     shardid
@@ -782,12 +774,9 @@ SET client_min_messages to ERROR;
 \set VERBOSITY terse
 SELECT master_add_node('invalid-node-name', 9999);
 ERROR:  connection error: invalid-node-name:9999
-SET client_min_messages to DEFAULT;
-\set VERBOSITY default
 -- drop unnecassary tables
 DROP TABLE initially_not_replicated_reference_table;
 -- reload pg_dist_shard_placement table
 INSERT INTO pg_dist_shard_placement (SELECT * FROM tmp_shard_placement);
 DROP TABLE tmp_shard_placement;
-RESET citus.shard_replication_factor;
-RESET citus.replication_model;
+DROP SCHEMA replicate_reference_table CASCADE;
diff --git a/src/test/regress/sql/multi_replicate_reference_table.sql b/src/test/regress/sql/multi_replicate_reference_table.sql
index ea80a3761..2cc1634d7 100644
--- a/src/test/regress/sql/multi_replicate_reference_table.sql
+++ b/src/test/regress/sql/multi_replicate_reference_table.sql
@@ -3,11 +3,17 @@
 --
 -- Tests that check that reference tables are replicated when adding new nodes.
 
+CREATE SCHEMA replicate_reference_table;
+SET search_path TO replicate_reference_table;
+
 SET citus.next_shard_id TO 1370000;
 ALTER SEQUENCE pg_catalog.pg_dist_colocationid_seq RESTART 1370000;
 ALTER SEQUENCE pg_catalog.pg_dist_groupid_seq RESTART 1370000;
 ALTER SEQUENCE pg_catalog.pg_dist_node_nodeid_seq RESTART 1370000;
 
+-- only query shards created in this test
+CREATE VIEW pg_dist_shard_placement_view AS
+SELECT * FROM pg_dist_shard_placement WHERE shardid BETWEEN 1370000 AND 1380000;
 
 -- remove a node for testing purposes
 CREATE TABLE tmp_shard_placement AS SELECT * FROM pg_dist_shard_placement WHERE nodeport = :worker_2_port;
@@ -29,7 +35,7 @@ SELECT COUNT(*) FROM pg_dist_node WHERE nodeport = :worker_2_port;
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
 FROM
-    pg_dist_shard_placement
+    pg_dist_shard_placement_view
 WHERE
     nodeport = :worker_2_port;
 
@@ -53,7 +59,7 @@ SELECT COUNT(*) FROM pg_dist_node WHERE nodeport = :worker_2_port;
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
 FROM
-    pg_dist_shard_placement
+    pg_dist_shard_placement_view
 WHERE
     nodeport = :worker_2_port;
 
@@ -68,13 +74,12 @@ SELECT create_reference_table('replicate_reference_table_valid');
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
 FROM
-    pg_dist_shard_placement
+    pg_dist_shard_placement_view
 WHERE
     nodeport = :worker_2_port
 ORDER BY shardid, nodeport;
 
-SELECT *
-FROM pg_dist_colocation
+SELECT shardcount, replicationfactor, distributioncolumntype, distributioncolumncollation FROM pg_dist_colocation
 WHERE colocationid IN
     (SELECT colocationid
      FROM pg_dist_partition
@@ -86,13 +91,12 @@ SELECT 1 FROM master_add_node('localhost', :worker_2_port);
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
 FROM
-    pg_dist_shard_placement
+    pg_dist_shard_placement_view
 WHERE
     nodeport = :worker_2_port
 ORDER BY shardid, nodeport;
 
-SELECT *
-FROM pg_dist_colocation
+SELECT shardcount, replicationfactor, distributioncolumntype, distributioncolumncollation FROM pg_dist_colocation
 WHERE colocationid IN
     (SELECT colocationid
      FROM pg_dist_partition
@@ -105,13 +109,12 @@ WHERE colocationid IN
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
 FROM
-    pg_dist_shard_placement
+    pg_dist_shard_placement_view
 WHERE
     nodeport = :worker_2_port
 ORDER BY shardid, nodeport;
 
-SELECT *
-FROM pg_dist_colocation
+SELECT shardcount, replicationfactor, distributioncolumntype, distributioncolumncollation FROM pg_dist_colocation
 WHERE colocationid IN
     (SELECT colocationid
      FROM pg_dist_partition
@@ -123,13 +126,12 @@ SELECT 1 FROM master_add_node('localhost', :worker_2_port);
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
 FROM
-    pg_dist_shard_placement
+    pg_dist_shard_placement_view
 WHERE
     nodeport = :worker_2_port
 ORDER BY shardid, nodeport;
 
-SELECT *
-FROM pg_dist_colocation
+SELECT shardcount, replicationfactor, distributioncolumntype, distributioncolumncollation FROM pg_dist_colocation
 WHERE colocationid IN
     (SELECT colocationid
      FROM pg_dist_partition
@@ -148,13 +150,12 @@ SELECT create_reference_table('replicate_reference_table_rollback');
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
 FROM
-    pg_dist_shard_placement
+    pg_dist_shard_placement_view
 WHERE
     nodeport = :worker_2_port
 ORDER BY shardid, nodeport;
 
-SELECT *
-FROM pg_dist_colocation
+SELECT shardcount, replicationfactor, distributioncolumntype, distributioncolumncollation FROM pg_dist_colocation
 WHERE colocationid IN
     (SELECT colocationid
      FROM pg_dist_partition
@@ -168,13 +169,12 @@ ROLLBACK;
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
 FROM
-    pg_dist_shard_placement
+    pg_dist_shard_placement_view
 WHERE
     nodeport = :worker_2_port
 ORDER BY shardid, nodeport;
 
-SELECT *
-FROM pg_dist_colocation
+SELECT shardcount, replicationfactor, distributioncolumntype, distributioncolumncollation FROM pg_dist_colocation
 WHERE colocationid IN
     (SELECT colocationid
      FROM pg_dist_partition
@@ -191,13 +191,12 @@ SELECT create_reference_table('replicate_reference_table_commit');
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
 FROM
-    pg_dist_shard_placement
+    pg_dist_shard_placement_view
 WHERE
     nodeport = :worker_2_port
 ORDER BY shardid, nodeport;
 
-SELECT *
-FROM pg_dist_colocation
+SELECT shardcount, replicationfactor, distributioncolumntype, distributioncolumncollation FROM pg_dist_colocation
 WHERE colocationid IN
     (SELECT colocationid
      FROM pg_dist_partition
@@ -211,13 +210,12 @@ COMMIT;
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
 FROM
-    pg_dist_shard_placement
+    pg_dist_shard_placement_view
 WHERE
     nodeport = :worker_2_port
 ORDER BY shardid, nodeport;
 
-SELECT *
-FROM pg_dist_colocation
+SELECT shardcount, replicationfactor, distributioncolumntype, distributioncolumncollation FROM pg_dist_colocation
 WHERE colocationid IN
     (SELECT colocationid
      FROM pg_dist_partition
@@ -247,12 +245,11 @@ CREATE TABLE replicate_reference_table_reference_two(column1 int);
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
 FROM
-    pg_dist_shard_placement
+    pg_dist_shard_placement_view
 WHERE
     nodeport = :worker_2_port;
 
-SELECT *
-FROM pg_dist_colocation
+SELECT shardcount, replicationfactor, distributioncolumntype, distributioncolumncollation FROM pg_dist_colocation
 WHERE colocationid IN
     (SELECT colocationid
      FROM pg_dist_partition
@@ -278,13 +275,12 @@ RESET client_min_messages;
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
 FROM
-    pg_dist_shard_placement
+    pg_dist_shard_placement_view
 WHERE
     nodeport = :worker_2_port
 ORDER BY shardid, nodeport;
 
-SELECT *
-FROM pg_dist_colocation
+SELECT shardcount, replicationfactor, distributioncolumntype, distributioncolumncollation FROM pg_dist_colocation
 WHERE colocationid IN
     (SELECT colocationid
      FROM pg_dist_partition
@@ -359,13 +355,12 @@ SELECT create_reference_table('replicate_reference_table_drop');
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
 FROM
-    pg_dist_shard_placement
+    pg_dist_shard_placement_view
 WHERE
     nodeport = :worker_2_port
 ORDER BY shardid, nodeport;
 
-SELECT *
-FROM pg_dist_colocation
+SELECT shardcount, replicationfactor, distributioncolumntype, distributioncolumncollation FROM pg_dist_colocation
 WHERE colocationid IN
     (SELECT colocationid
      FROM pg_dist_partition
@@ -380,7 +375,7 @@ COMMIT;
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
 FROM
-    pg_dist_shard_placement
+    pg_dist_shard_placement_view
 WHERE
     nodeport = :worker_2_port
 ORDER BY shardid, nodeport;
@@ -398,13 +393,12 @@ SELECT create_reference_table('replicate_reference_table_schema.table1');
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
 FROM
-    pg_dist_shard_placement
+    pg_dist_shard_placement_view
 WHERE
     nodeport = :worker_2_port
 ORDER BY shardid, nodeport;
 
-SELECT *
-FROM pg_dist_colocation
+SELECT shardcount, replicationfactor, distributioncolumntype, distributioncolumncollation FROM pg_dist_colocation
 WHERE colocationid IN
     (SELECT colocationid
      FROM pg_dist_partition
@@ -416,13 +410,12 @@ SELECT 1 FROM master_add_node('localhost', :worker_2_port);
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
 FROM
-    pg_dist_shard_placement
+    pg_dist_shard_placement_view
 WHERE
     nodeport = :worker_2_port
 ORDER BY shardid, nodeport;
 
-SELECT *
-FROM pg_dist_colocation
+SELECT shardcount, replicationfactor, distributioncolumntype, distributioncolumncollation FROM pg_dist_colocation
 WHERE colocationid IN
     (SELECT colocationid
      FROM pg_dist_partition
@@ -447,7 +440,7 @@ SELECT create_reference_table('ref_table_1'),
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
 FROM
-    pg_dist_shard_placement
+    pg_dist_shard_placement_view
 WHERE
     nodeport = :worker_2_port
 ORDER BY shardid, nodeport;
@@ -458,7 +451,7 @@ SELECT 1 FROM master_add_node('localhost', :worker_2_port);
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
 FROM
-    pg_dist_shard_placement
+    pg_dist_shard_placement_view
 WHERE
     nodeport = :worker_2_port
 ORDER BY shardid, nodeport;
@@ -480,7 +473,7 @@ SELECT 1 FROM master_add_inactive_node('localhost', :worker_2_port);
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
 FROM
-    pg_dist_shard_placement
+    pg_dist_shard_placement_view
 WHERE
     shardid IN (SELECT
                     shardid
@@ -497,7 +490,7 @@ SELECT 1 FROM master_activate_node('localhost', :worker_2_port);
 SELECT
     shardid, shardstate, shardlength, nodename, nodeport
 FROM
-    pg_dist_shard_placement
+    pg_dist_shard_placement_view
 WHERE
     shardid IN (SELECT
                     shardid
@@ -519,9 +512,6 @@ SET client_min_messages to ERROR;
 
 SELECT master_add_node('invalid-node-name', 9999);
 
-SET client_min_messages to DEFAULT;
-\set VERBOSITY default
-
 -- drop unnecassary tables
 DROP TABLE initially_not_replicated_reference_table;
 
@@ -529,5 +519,4 @@ DROP TABLE initially_not_replicated_reference_table;
 INSERT INTO pg_dist_shard_placement (SELECT * FROM tmp_shard_placement);
 DROP TABLE tmp_shard_placement;
 
-RESET citus.shard_replication_factor;
-RESET citus.replication_model;
+DROP SCHEMA replicate_reference_table CASCADE;

From c168a53ebc2fcf479a1452d5ff9f5b2b0740073f Mon Sep 17 00:00:00 2001
From: Hadi Moshayedi <hadi@moshayedi.net>
Date: Fri, 3 Apr 2020 16:54:50 -0700
Subject: [PATCH 4/5] Tests for replicate_reference_tables

---
 .../multi_replicate_reference_table.out       | 81 ++++++++++++++++++-
 .../sql/multi_replicate_reference_table.sql   | 45 ++++++++++-
 2 files changed, 124 insertions(+), 2 deletions(-)

diff --git a/src/test/regress/expected/multi_replicate_reference_table.out b/src/test/regress/expected/multi_replicate_reference_table.out
index e17901906..cabf02679 100644
--- a/src/test/regress/expected/multi_replicate_reference_table.out
+++ b/src/test/regress/expected/multi_replicate_reference_table.out
@@ -760,13 +760,92 @@ ORDER BY 1,4,5;
  1370015 |          1 |           0 | localhost |    57637
 (1 row)
 
--- this should have no effect
+SELECT 1 FROM master_remove_node('localhost', :worker_2_port);
+ ?column?
+---------------------------------------------------------------------
+        1
+(1 row)
+
+CREATE TABLE ref_table(a int);
+SELECT create_reference_table('ref_table');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+INSERT INTO ref_table SELECT * FROM generate_series(1, 10);
 SELECT 1 FROM master_add_node('localhost', :worker_2_port);
  ?column?
 ---------------------------------------------------------------------
         1
 (1 row)
 
+-- verify we cannot replicate reference tables in a transaction modifying pg_dist_node
+BEGIN;
+SELECT 1 FROM master_add_inactive_node('invalid-node-name', 9999);
+ ?column?
+---------------------------------------------------------------------
+        1
+(1 row)
+
+SELECT replicate_reference_tables();
+ERROR:  cannot replicate reference tables in a transaction that modified node metadata
+ROLLBACK;
+-- verify we cannot replicate reference tables in a transaction which
+-- modified reference tables
+BEGIN;
+DELETE FROM ref_table;
+SELECT replicate_reference_tables();
+ERROR:  cannot replicate reference tables in a transaction that modified a reference table
+ROLLBACK;
+BEGIN;
+ALTER TABLE ref_table ADD COLUMN b int;
+SELECT replicate_reference_tables();
+ERROR:  cannot replicate reference tables in a transaction that modified a reference table
+ROLLBACK;
+BEGIN;
+CREATE INDEX ref_idx ON ref_table(a);
+SELECT replicate_reference_tables();
+ERROR:  cannot replicate reference tables in a transaction that modified a reference table
+ROLLBACK;
+--
+-- read from reference table, then replicate, then write. verify
+-- placements are consistent.
+--
+BEGIN;
+SELECT count(*) FROM ref_table;
+ count
+---------------------------------------------------------------------
+    10
+(1 row)
+
+SELECT replicate_reference_tables();
+ replicate_reference_tables
+---------------------------------------------------------------------
+
+(1 row)
+
+INSERT INTO ref_table VALUES (11);
+SELECT count(*), sum(a) FROM ref_table;
+ count | sum
+---------------------------------------------------------------------
+    11 |  66
+(1 row)
+
+UPDATE ref_table SET a = a + 1;
+SELECT sum(a) FROM ref_table;
+ sum
+---------------------------------------------------------------------
+  77
+(1 row)
+
+COMMIT;
+SELECT min(result) = max(result) AS consistent FROM run_command_on_placements('ref_table', 'SELECT sum(a) FROM %s');
+ consistent
+---------------------------------------------------------------------
+ t
+(1 row)
+
 -- test adding an invalid node while we have reference tables to replicate
 -- set client message level to ERROR and verbosity to terse to supporess
 -- OS-dependent host name resolution warnings
diff --git a/src/test/regress/sql/multi_replicate_reference_table.sql b/src/test/regress/sql/multi_replicate_reference_table.sql
index 2cc1634d7..56d458296 100644
--- a/src/test/regress/sql/multi_replicate_reference_table.sql
+++ b/src/test/regress/sql/multi_replicate_reference_table.sql
@@ -501,9 +501,52 @@ WHERE
     AND nodeport != :master_port
 ORDER BY 1,4,5;
 
--- this should have no effect
+SELECT 1 FROM master_remove_node('localhost', :worker_2_port);
+
+CREATE TABLE ref_table(a int);
+SELECT create_reference_table('ref_table');
+INSERT INTO ref_table SELECT * FROM generate_series(1, 10);
+
 SELECT 1 FROM master_add_node('localhost', :worker_2_port);
 
+-- verify we cannot replicate reference tables in a transaction modifying pg_dist_node
+BEGIN;
+SELECT 1 FROM master_add_inactive_node('invalid-node-name', 9999);
+SELECT replicate_reference_tables();
+ROLLBACK;
+
+-- verify we cannot replicate reference tables in a transaction which
+-- modified reference tables
+BEGIN;
+DELETE FROM ref_table;
+SELECT replicate_reference_tables();
+ROLLBACK;
+
+BEGIN;
+ALTER TABLE ref_table ADD COLUMN b int;
+SELECT replicate_reference_tables();
+ROLLBACK;
+
+BEGIN;
+CREATE INDEX ref_idx ON ref_table(a);
+SELECT replicate_reference_tables();
+ROLLBACK;
+
+--
+-- read from reference table, then replicate, then write. verify
+-- placements are consistent.
+--
+BEGIN;
+SELECT count(*) FROM ref_table;
+SELECT replicate_reference_tables();
+INSERT INTO ref_table VALUES (11);
+SELECT count(*), sum(a) FROM ref_table;
+UPDATE ref_table SET a = a + 1;
+SELECT sum(a) FROM ref_table;
+COMMIT;
+
+SELECT min(result) = max(result) AS consistent FROM run_command_on_placements('ref_table', 'SELECT sum(a) FROM %s');
+
 -- test adding an invalid node while we have reference tables to replicate
 -- set client message level to ERROR and verbosity to terse to supporess
 -- OS-dependent host name resolution warnings

From dda53a0bbad7dac6daba034240a60182c4f1ca4e Mon Sep 17 00:00:00 2001
From: Hadi Moshayedi <hadi@moshayedi.net>
Date: Mon, 6 Apr 2020 14:50:04 -0700
Subject: [PATCH 5/5] GUC for replicate reference tables on activate.

---
 .../distributed/master/master_repair_shards.c |   6 -
 .../distributed/metadata/node_metadata.c      |  12 +
 src/backend/distributed/shared_library_init.c |  11 +
 .../distributed/sql/citus--9.2-4--9.3-2.sql   |   2 +-
 .../{9.3-1.sql => 9.3-2.sql}                  |   0
 .../distributed/utils/reference_table_utils.c | 167 +++++--
 src/include/distributed/master_protocol.h     |   5 +
 .../distributed/reference_table_utils.h       |   2 +-
 src/include/distributed/worker_manager.h      |   1 +
 src/test/regress/bin/normalize.sed            |   3 +
 .../expected/failure_add_disable_node.out     |   1 +
 ...add_node_vs_reference_table_operations.out | 411 +++++++++++++++++-
 .../isolation_dump_global_wait_edges.out      |  18 +-
 .../regress/expected/local_shard_copy.out     |   1 +
 .../regress/expected/multi_metadata_sync.out  |   8 +-
 .../expected/multi_mx_node_metadata.out       |   1 +
 .../multi_remove_node_reference_table.out     |   7 +
 .../multi_replicate_reference_table.out       |  68 +++
 .../multi_transactional_drop_shards.out       |   4 +-
 .../expected/propagate_extension_commands.out |  24 +-
 ...dd_node_vs_reference_table_operations.spec |  54 ++-
 .../regress/sql/failure_add_disable_node.sql  |   1 +
 src/test/regress/sql/local_shard_copy.sql     |   1 +
 src/test/regress/sql/multi_metadata_sync.sql  |   1 +
 .../regress/sql/multi_mx_node_metadata.sql    |   1 +
 .../sql/multi_remove_node_reference_table.sql |   7 +
 .../sql/multi_replicate_reference_table.sql   |  30 ++
 .../sql/multi_transactional_drop_shards.sql   |   2 +
 .../sql/propagate_extension_commands.sql      |  24 +-
 29 files changed, 744 insertions(+), 129 deletions(-)
 rename src/backend/distributed/sql/udfs/replicate_reference_tables/{9.3-1.sql => 9.3-2.sql} (100%)

diff --git a/src/backend/distributed/master/master_repair_shards.c b/src/backend/distributed/master/master_repair_shards.c
index c3b24f50f..de0c20d4e 100644
--- a/src/backend/distributed/master/master_repair_shards.c
+++ b/src/backend/distributed/master/master_repair_shards.c
@@ -43,12 +43,6 @@
 #include "utils/lsyscache.h"
 #include "utils/palloc.h"
 
-
-#define TRANSFER_MODE_AUTOMATIC 'a'
-#define TRANSFER_MODE_FORCE_LOGICAL 'l'
-#define TRANSFER_MODE_BLOCK_WRITES 'b'
-
-
 /* local function forward declarations */
 static char LookupShardTransferMode(Oid shardReplicationModeOid);
 static void ErrorIfTableCannotBeReplicated(Oid relationId);
diff --git a/src/backend/distributed/metadata/node_metadata.c b/src/backend/distributed/metadata/node_metadata.c
index 4cc9f5d17..768a9e1df 100644
--- a/src/backend/distributed/metadata/node_metadata.c
+++ b/src/backend/distributed/metadata/node_metadata.c
@@ -60,6 +60,12 @@ int GroupSize = 1;
 /* config variable managed via guc.c */
 char *CurrentCluster = "default";
 
+/*
+ * Config variable to control whether we should replicate reference tables on
+ * node activation or we should defer it to shard creation.
+ */
+bool ReplicateReferenceTablesOnActivate = true;
+
 /* did current transaction modify pg_dist_node? */
 bool TransactionModifiedNodeMetadata = false;
 
@@ -381,6 +387,12 @@ SetUpDistributedTableDependencies(WorkerNode *newWorkerNode)
 		ReplicateAllDependenciesToNode(newWorkerNode->workerName,
 									   newWorkerNode->workerPort);
 
+		if (ReplicateReferenceTablesOnActivate)
+		{
+			ReplicateAllReferenceTablesToNode(newWorkerNode->workerName,
+											  newWorkerNode->workerPort);
+		}
+
 		/*
 		 * Let the maintenance daemon do the hard work of syncing the metadata.
 		 * We prefer this because otherwise node activation might fail within
diff --git a/src/backend/distributed/shared_library_init.c b/src/backend/distributed/shared_library_init.c
index 1df2b8eaf..b3590663c 100644
--- a/src/backend/distributed/shared_library_init.c
+++ b/src/backend/distributed/shared_library_init.c
@@ -53,6 +53,7 @@
 #include "distributed/multi_server_executor.h"
 #include "distributed/pg_dist_partition.h"
 #include "distributed/placement_connection.h"
+#include "distributed/reference_table_utils.h"
 #include "distributed/relation_access_tracking.h"
 #include "distributed/run_from_same_connection.h"
 #include "distributed/query_pushdown_planning.h"
@@ -1370,6 +1371,16 @@ RegisterCitusConfigVariables(void)
 		GUC_NO_SHOW_ALL,
 		NULL, NULL, NULL);
 
+	DefineCustomBoolVariable(
+		"citus.replicate_reference_tables_on_activate",
+		NULL,
+		NULL,
+		&ReplicateReferenceTablesOnActivate,
+		true,
+		PGC_USERSET,
+		GUC_NO_SHOW_ALL,
+		NULL, NULL, NULL);
+
 	/* warn about config items in the citus namespace that are not registered above */
 	EmitWarningsOnPlaceholders("citus");
 }
diff --git a/src/backend/distributed/sql/citus--9.2-4--9.3-2.sql b/src/backend/distributed/sql/citus--9.2-4--9.3-2.sql
index 25ba9cba0..f8c3b14f9 100644
--- a/src/backend/distributed/sql/citus--9.2-4--9.3-2.sql
+++ b/src/backend/distributed/sql/citus--9.2-4--9.3-2.sql
@@ -4,4 +4,4 @@
 
 #include "udfs/citus_extradata_container/9.3-2.sql"
 #include "udfs/update_distributed_table_colocation/9.3-2.sql"
-#include "udfs/replicate_reference_tables/9.3-1.sql"
+#include "udfs/replicate_reference_tables/9.3-2.sql"
diff --git a/src/backend/distributed/sql/udfs/replicate_reference_tables/9.3-1.sql b/src/backend/distributed/sql/udfs/replicate_reference_tables/9.3-2.sql
similarity index 100%
rename from src/backend/distributed/sql/udfs/replicate_reference_tables/9.3-1.sql
rename to src/backend/distributed/sql/udfs/replicate_reference_tables/9.3-2.sql
diff --git a/src/backend/distributed/utils/reference_table_utils.c b/src/backend/distributed/utils/reference_table_utils.c
index 923a008ee..ecf75ee23 100644
--- a/src/backend/distributed/utils/reference_table_utils.c
+++ b/src/backend/distributed/utils/reference_table_utils.c
@@ -22,6 +22,7 @@
 #include "distributed/master_metadata_utility.h"
 #include "distributed/metadata_cache.h"
 #include "distributed/metadata_sync.h"
+#include "distributed/multi_executor.h"
 #include "distributed/multi_logical_planner.h"
 #include "distributed/reference_table_utils.h"
 #include "distributed/relation_access_tracking.h"
@@ -38,11 +39,12 @@
 #include "utils/lsyscache.h"
 #include "utils/rel.h"
 
-
 /* local function forward declarations */
-static List * WorkersWithoutReferenceTablePlacement(uint64 shardId);
-static void CopyShardPlacementToWorkerNode(ShardPlacement *sourceShardPlacement,
-										   WorkerNode *workerNode, const char *userName);
+static List * WorkersWithoutReferenceTablePlacement(uint64 shardId, LOCKMODE lockMode);
+static StringInfo CopyShardPlacementToWorkerNodeQuery(
+	ShardPlacement *sourceShardPlacement,
+	WorkerNode *workerNode,
+	char transferMode);
 static void ReplicateSingleShardTableToAllNodes(Oid relationId);
 static void ReplicateShardToAllNodes(ShardInterval *shardInterval);
 static void ReplicateShardToNode(ShardInterval *shardInterval, char *nodeName,
@@ -110,7 +112,20 @@ EnsureReferenceTablesExistOnAllNodes(void)
 		return;
 	}
 
-	/* prevent this function from running concurrently with itself */
+	/*
+	 * Prevent this function from running concurrently with itself.
+	 *
+	 * It also prevents concurrent DROP TABLE or DROP SCHEMA. We need this
+	 * because through-out this function we assume values in referenceTableIdList
+	 * are still valid.
+	 *
+	 * We don't need to handle other kinds of reference table DML/DDL here, since
+	 * master_copy_shard_placement gets enough locks for that.
+	 *
+	 * We also don't need special handling for concurrent create_refernece_table.
+	 * Since that will trigger a call to this function from another backend,
+	 * which will block until our call is finished.
+	 */
 	int colocationId = CreateReferenceTableColocationId();
 	LockColocationId(colocationId, ExclusiveLock);
 
@@ -123,18 +138,26 @@ EnsureReferenceTablesExistOnAllNodes(void)
 	}
 
 	Oid referenceTableId = linitial_oid(referenceTableIdList);
+	const char *referenceTableName = get_rel_name(referenceTableId);
 	List *shardIntervalList = LoadShardIntervalList(referenceTableId);
 	if (list_length(shardIntervalList) == 0)
 	{
 		/* check for corrupt metadata */
 		ereport(ERROR, (errmsg("reference table \"%s\" does not have a shard",
-							   get_rel_name(referenceTableId))));
+							   referenceTableName)));
 	}
 
 	ShardInterval *shardInterval = (ShardInterval *) linitial(shardIntervalList);
 	uint64 shardId = shardInterval->shardId;
 
-	List *newWorkersList = WorkersWithoutReferenceTablePlacement(shardId);
+	/*
+	 * We only take an access share lock, otherwise we'll hold up master_add_node.
+	 * In case of create_reference_table() and upgrade_to_reference_table(), where
+	 * we don't want concurrent writes to pg_dist_node, we have already acquired
+	 * ShareLock on pg_dist_node.
+	 */
+	List *newWorkersList = WorkersWithoutReferenceTablePlacement(shardId,
+																 AccessShareLock);
 	if (list_length(newWorkersList) == 0)
 	{
 		/* nothing to do, no need for lock */
@@ -171,21 +194,48 @@ EnsureReferenceTablesExistOnAllNodes(void)
 	if (sourceShardPlacement == NULL)
 	{
 		/* check for corrupt metadata */
-		ereport(ERROR, (errmsg("reference table shard " UINT64_FORMAT " does not "
-																	  "have an active shard placement",
+		ereport(ERROR, (errmsg("reference table shard "
+							   UINT64_FORMAT
+							   " does not have an active shard placement",
 							   shardId)));
 	}
 
 	WorkerNode *newWorkerNode = NULL;
 	foreach_ptr(newWorkerNode, newWorkersList)
 	{
+		ereport(NOTICE, (errmsg("replicating reference table '%s' to %s:%d ...",
+								referenceTableName, newWorkerNode->workerName,
+								newWorkerNode->workerPort)));
+
 		/*
 		 * Call master_copy_shard_placement using citus extension owner. Current
 		 * user might not have permissions to do the copy.
 		 */
 		const char *userName = CitusExtensionOwnerName();
-		CopyShardPlacementToWorkerNode(sourceShardPlacement, newWorkerNode,
-									   userName);
+		int connectionFlags = OUTSIDE_TRANSACTION;
+
+		MultiConnection *connection = GetNodeUserDatabaseConnection(
+			connectionFlags, "localhost", PostPortNumber,
+			userName, NULL);
+
+		if (PQstatus(connection->pgConn) == CONNECTION_OK)
+		{
+			StringInfo placementCopyCommand =
+				CopyShardPlacementToWorkerNodeQuery(sourceShardPlacement,
+													newWorkerNode,
+													TRANSFER_MODE_AUTOMATIC);
+			ExecuteCriticalRemoteCommand(connection, placementCopyCommand->data);
+		}
+		else
+		{
+			ereport(ERROR, (errmsg("could not open a connection to localhost "
+								   "when replicating reference tables"),
+							errdetail(
+								"citus.replicate_reference_tables_on_activate = false "
+								"requires localhost connectivity.")));
+		}
+
+		CloseConnection(connection);
 	}
 
 	/*
@@ -224,14 +274,13 @@ AnyRelationsModifiedInTransaction(List *relationIdList)
  * supposed to.
  */
 static List *
-WorkersWithoutReferenceTablePlacement(uint64 shardId)
+WorkersWithoutReferenceTablePlacement(uint64 shardId, LOCKMODE lockMode)
 {
 	List *workersWithoutPlacements = NIL;
 
 	List *shardPlacementList = ActiveShardPlacementList(shardId);
 
-	/* we only take an access share lock, otherwise we'll hold up master_add_node */
-	List *workerNodeList = ReferenceTablePlacementNodeList(AccessShareLock);
+	List *workerNodeList = ReferenceTablePlacementNodeList(lockMode);
 	workerNodeList = SortList(workerNodeList, CompareWorkerNodes);
 
 	WorkerNode *workerNode = NULL;
@@ -252,33 +301,33 @@ WorkersWithoutReferenceTablePlacement(uint64 shardId)
 
 
 /*
- * CopyShardPlacementToWorkerNode runs master_copy_shard_placement
- * using the given username by connecting to localhost.
+ * CopyShardPlacementToWorkerNodeQuery returns the master_copy_shard_placement
+ * command to copy the given shard placement to given node.
  */
-static void
-CopyShardPlacementToWorkerNode(ShardPlacement *sourceShardPlacement,
-							   WorkerNode *workerNode,
-							   const char *userName)
+static StringInfo
+CopyShardPlacementToWorkerNodeQuery(ShardPlacement *sourceShardPlacement,
+									WorkerNode *workerNode,
+									char transferMode)
 {
-	int connectionFlags = OUTSIDE_TRANSACTION;
 	StringInfo queryString = makeStringInfo();
 
-	MultiConnection *connection = GetNodeUserDatabaseConnection(
-		connectionFlags, "localhost", PostPortNumber,
-		userName, NULL);
+	const char *transferModeString =
+		transferMode == TRANSFER_MODE_BLOCK_WRITES ? "block_writes" :
+		transferMode == TRANSFER_MODE_FORCE_LOGICAL ? "force_logical" :
+		"auto";
 
 	appendStringInfo(queryString,
 					 "SELECT master_copy_shard_placement("
-					 UINT64_FORMAT ", %s, %d, %s, %d, do_repair := false)",
+					 UINT64_FORMAT ", %s, %d, %s, %d, do_repair := false, "
+								   "transfer_mode := %s)",
 					 sourceShardPlacement->shardId,
 					 quote_literal_cstr(sourceShardPlacement->nodeName),
 					 sourceShardPlacement->nodePort,
 					 quote_literal_cstr(workerNode->workerName),
-					 workerNode->workerPort);
+					 workerNode->workerPort,
+					 quote_literal_cstr(transferModeString));
 
-	elog(DEBUG3, "%s", queryString->data);
-
-	ExecuteCriticalRemoteCommand(connection, queryString->data);
+	return queryString;
 }
 
 
@@ -657,3 +706,63 @@ ReferenceTableReplicationFactor(void)
 	int replicationFactor = list_length(nodeList);
 	return replicationFactor;
 }
+
+
+/*
+ * ReplicateAllReferenceTablesToNode function finds all reference tables and
+ * replicates them to the given worker node. It also modifies pg_dist_colocation
+ * table to update the replication factor column when necessary. This function
+ * skips reference tables if that node already has healthy placement of that
+ * reference table to prevent unnecessary data transfer.
+ */
+void
+ReplicateAllReferenceTablesToNode(char *nodeName, int nodePort)
+{
+	List *referenceTableList = ReferenceTableOidList();
+
+	/* if there is no reference table, we do not need to replicate anything */
+	if (list_length(referenceTableList) > 0)
+	{
+		List *referenceShardIntervalList = NIL;
+
+		/*
+		 * We sort the reference table list to prevent deadlocks in concurrent
+		 * ReplicateAllReferenceTablesToAllNodes calls.
+		 */
+		referenceTableList = SortList(referenceTableList, CompareOids);
+		Oid referenceTableId = InvalidOid;
+		foreach_oid(referenceTableId, referenceTableList)
+		{
+			List *shardIntervalList = LoadShardIntervalList(referenceTableId);
+			ShardInterval *shardInterval = (ShardInterval *) linitial(shardIntervalList);
+
+			referenceShardIntervalList = lappend(referenceShardIntervalList,
+												 shardInterval);
+		}
+
+		if (ClusterHasKnownMetadataWorkers())
+		{
+			BlockWritesToShardList(referenceShardIntervalList);
+		}
+
+		ShardInterval *shardInterval = NULL;
+		foreach_ptr(shardInterval, referenceShardIntervalList)
+		{
+			uint64 shardId = shardInterval->shardId;
+
+			LockShardDistributionMetadata(shardId, ExclusiveLock);
+
+			ReplicateShardToNode(shardInterval, nodeName, nodePort);
+		}
+
+		/* create foreign constraints between reference tables */
+		foreach_ptr(shardInterval, referenceShardIntervalList)
+		{
+			char *tableOwner = TableOwner(shardInterval->relationId);
+			List *commandList = CopyShardForeignConstraintCommandList(shardInterval);
+
+			SendCommandListToWorkerInSingleTransaction(nodeName, nodePort, tableOwner,
+													   commandList);
+		}
+	}
+}
diff --git a/src/include/distributed/master_protocol.h b/src/include/distributed/master_protocol.h
index b9f5b2671..1755a101d 100644
--- a/src/include/distributed/master_protocol.h
+++ b/src/include/distributed/master_protocol.h
@@ -48,6 +48,11 @@
 #define CANDIDATE_NODE_FIELDS 2
 #define WORKER_NODE_FIELDS 2
 
+/* transfer mode for master_copy_shard_placement */
+#define TRANSFER_MODE_AUTOMATIC 'a'
+#define TRANSFER_MODE_FORCE_LOGICAL 'l'
+#define TRANSFER_MODE_BLOCK_WRITES 'b'
+
 /* Name of columnar foreign data wrapper */
 #define CSTORE_FDW_NAME "cstore_fdw"
 
diff --git a/src/include/distributed/reference_table_utils.h b/src/include/distributed/reference_table_utils.h
index 4484e4411..0fac3c41d 100644
--- a/src/include/distributed/reference_table_utils.h
+++ b/src/include/distributed/reference_table_utils.h
@@ -22,6 +22,6 @@ extern uint32 CreateReferenceTableColocationId(void);
 extern void DeleteAllReferenceTablePlacementsFromNodeGroup(int32 groupId);
 extern int CompareOids(const void *leftElement, const void *rightElement);
 extern int ReferenceTableReplicationFactor(void);
-
+extern void ReplicateAllReferenceTablesToNode(char *nodeName, int nodePort);
 
 #endif /* REFERENCE_TABLE_UTILS_H_ */
diff --git a/src/include/distributed/worker_manager.h b/src/include/distributed/worker_manager.h
index f5bb00ba9..336e1eedf 100644
--- a/src/include/distributed/worker_manager.h
+++ b/src/include/distributed/worker_manager.h
@@ -61,6 +61,7 @@ typedef struct WorkerNode
 extern int MaxWorkerNodesTracked;
 extern char *WorkerListFileName;
 extern char *CurrentCluster;
+extern bool ReplicateReferenceTablesOnActivate;
 
 
 /* Function declarations for finding worker nodes to place shards on */
diff --git a/src/test/regress/bin/normalize.sed b/src/test/regress/bin/normalize.sed
index 872f090ca..343d05057 100644
--- a/src/test/regress/bin/normalize.sed
+++ b/src/test/regress/bin/normalize.sed
@@ -111,3 +111,6 @@ s/worker_hash_partition_table  \([0-9]+/worker_hash_partition_table  \(xxxxxxx/g
 # ignore first parameter for citus_extradata_container due to differences between pg11 and pg12
 # can be removed when we remove PG_VERSION_NUM >= 120000
 s/pg_catalog.citus_extradata_container\([0-9]+/pg_catalog.citus_extradata_container\(XXX/g
+
+# ignore referene table replication messages
+/replicating reference table.*$/d
diff --git a/src/test/regress/expected/failure_add_disable_node.out b/src/test/regress/expected/failure_add_disable_node.out
index c37f9a0ef..aec4d756e 100644
--- a/src/test/regress/expected/failure_add_disable_node.out
+++ b/src/test/regress/expected/failure_add_disable_node.out
@@ -11,6 +11,7 @@ SELECT citus.mitmproxy('conn.allow()');
 (1 row)
 
 SET citus.next_shard_id TO 200000;
+SET citus.replicate_reference_tables_on_activate TO off;
 -- verify we have all worker nodes present
 SELECT * FROM master_get_active_worker_nodes()
 ORDER BY 1, 2;
diff --git a/src/test/regress/expected/isolation_add_node_vs_reference_table_operations.out b/src/test/regress/expected/isolation_add_node_vs_reference_table_operations.out
index a7a3eb3a2..72e4df0ce 100644
--- a/src/test/regress/expected/isolation_add_node_vs_reference_table_operations.out
+++ b/src/test/regress/expected/isolation_add_node_vs_reference_table_operations.out
@@ -1,9 +1,12 @@
 Parsed test spec with 2 sessions
 
-starting permutation: s2-load-metadata-cache s1-begin s1-add-second-worker s2-copy-to-reference-table s1-commit s2-print-content
+starting permutation: s1-do-not-replicate-on-activate s2-load-metadata-cache s1-begin s1-add-second-worker s2-copy-to-reference-table s1-commit s2-print-content
 create_distributed_table
 
 
+step s1-do-not-replicate-on-activate:
+ SET citus.replicate_reference_tables_on_activate TO off;
+
 step s2-load-metadata-cache:
  COPY test_reference_table FROM PROGRAM 'echo 1 && echo 2 && echo 3 && echo 4 && echo 5';
 
@@ -38,10 +41,13 @@ master_remove_node
 
 
 
-starting permutation: s2-load-metadata-cache s2-begin s2-copy-to-reference-table s1-add-second-worker s2-commit s2-print-content
+starting permutation: s1-do-not-replicate-on-activate s2-load-metadata-cache s2-begin s2-copy-to-reference-table s1-add-second-worker s2-commit s2-print-content
 create_distributed_table
 
 
+step s1-do-not-replicate-on-activate:
+ SET citus.replicate_reference_tables_on_activate TO off;
+
 step s2-load-metadata-cache:
  COPY test_reference_table FROM PROGRAM 'echo 1 && echo 2 && echo 3 && echo 4 && echo 5';
 
@@ -76,10 +82,13 @@ master_remove_node
 
 
 
-starting permutation: s2-load-metadata-cache s1-begin s1-add-second-worker s2-insert-to-reference-table s1-commit s2-print-content
+starting permutation: s1-do-not-replicate-on-activate s2-load-metadata-cache s1-begin s1-add-second-worker s2-insert-to-reference-table s1-commit s2-print-content
 create_distributed_table
 
 
+step s1-do-not-replicate-on-activate:
+ SET citus.replicate_reference_tables_on_activate TO off;
+
 step s2-load-metadata-cache:
  COPY test_reference_table FROM PROGRAM 'echo 1 && echo 2 && echo 3 && echo 4 && echo 5';
 
@@ -114,10 +123,13 @@ master_remove_node
 
 
 
-starting permutation: s2-load-metadata-cache s2-begin s2-insert-to-reference-table s1-add-second-worker s2-commit s2-print-content
+starting permutation: s1-do-not-replicate-on-activate s2-load-metadata-cache s2-begin s2-insert-to-reference-table s1-add-second-worker s2-commit s2-print-content
 create_distributed_table
 
 
+step s1-do-not-replicate-on-activate:
+ SET citus.replicate_reference_tables_on_activate TO off;
+
 step s2-load-metadata-cache:
  COPY test_reference_table FROM PROGRAM 'echo 1 && echo 2 && echo 3 && echo 4 && echo 5';
 
@@ -152,10 +164,13 @@ master_remove_node
 
 
 
-starting permutation: s2-load-metadata-cache s1-begin s1-add-second-worker s2-ddl-on-reference-table s1-commit s2-print-index-count
+starting permutation: s1-do-not-replicate-on-activate s2-load-metadata-cache s1-begin s1-add-second-worker s2-ddl-on-reference-table s1-commit s2-print-index-count
 create_distributed_table
 
 
+step s1-do-not-replicate-on-activate:
+ SET citus.replicate_reference_tables_on_activate TO off;
+
 step s2-load-metadata-cache:
  COPY test_reference_table FROM PROGRAM 'echo 1 && echo 2 && echo 3 && echo 4 && echo 5';
 
@@ -191,10 +206,13 @@ master_remove_node
 
 
 
-starting permutation: s2-load-metadata-cache s2-begin s2-ddl-on-reference-table s1-add-second-worker s2-commit s2-print-index-count
+starting permutation: s1-do-not-replicate-on-activate s2-load-metadata-cache s2-begin s2-ddl-on-reference-table s1-add-second-worker s2-commit s2-print-index-count
 create_distributed_table
 
 
+step s1-do-not-replicate-on-activate:
+ SET citus.replicate_reference_tables_on_activate TO off;
+
 step s2-load-metadata-cache:
  COPY test_reference_table FROM PROGRAM 'echo 1 && echo 2 && echo 3 && echo 4 && echo 5';
 
@@ -230,10 +248,13 @@ master_remove_node
 
 
 
-starting permutation: s2-load-metadata-cache s1-begin s1-add-second-worker s2-create-reference-table-2 s1-commit s2-print-content-2
+starting permutation: s1-do-not-replicate-on-activate s2-load-metadata-cache s1-begin s1-add-second-worker s2-create-reference-table-2 s1-commit s2-print-content-2
 create_distributed_table
 
 
+step s1-do-not-replicate-on-activate:
+ SET citus.replicate_reference_tables_on_activate TO off;
+
 step s2-load-metadata-cache:
  COPY test_reference_table FROM PROGRAM 'echo 1 && echo 2 && echo 3 && echo 4 && echo 5';
 
@@ -273,10 +294,13 @@ master_remove_node
 
 
 
-starting permutation: s2-load-metadata-cache s2-begin s2-create-reference-table-2 s1-add-second-worker s2-commit s2-print-content-2
+starting permutation: s1-do-not-replicate-on-activate s2-load-metadata-cache s2-begin s2-create-reference-table-2 s1-add-second-worker s2-commit s2-print-content-2
 create_distributed_table
 
 
+step s1-do-not-replicate-on-activate:
+ SET citus.replicate_reference_tables_on_activate TO off;
+
 step s2-load-metadata-cache:
  COPY test_reference_table FROM PROGRAM 'echo 1 && echo 2 && echo 3 && echo 4 && echo 5';
 
@@ -315,10 +339,13 @@ master_remove_node
 
 
 
-starting permutation: s1-begin s1-add-second-worker s2-copy-to-reference-table s1-commit s2-print-content
+starting permutation: s1-do-not-replicate-on-activate s1-begin s1-add-second-worker s2-copy-to-reference-table s1-commit s2-print-content
 create_distributed_table
 
 
+step s1-do-not-replicate-on-activate:
+ SET citus.replicate_reference_tables_on_activate TO off;
+
 step s1-begin:
     BEGIN;
 
@@ -350,10 +377,13 @@ master_remove_node
 
 
 
-starting permutation: s2-begin s2-copy-to-reference-table s1-add-second-worker s2-commit s2-print-content
+starting permutation: s1-do-not-replicate-on-activate s2-begin s2-copy-to-reference-table s1-add-second-worker s2-commit s2-print-content
 create_distributed_table
 
 
+step s1-do-not-replicate-on-activate:
+ SET citus.replicate_reference_tables_on_activate TO off;
+
 step s2-begin:
  BEGIN;
 
@@ -385,10 +415,13 @@ master_remove_node
 
 
 
-starting permutation: s1-begin s1-add-second-worker s2-insert-to-reference-table s1-commit s2-print-content
+starting permutation: s1-do-not-replicate-on-activate s1-begin s1-add-second-worker s2-insert-to-reference-table s1-commit s2-print-content
 create_distributed_table
 
 
+step s1-do-not-replicate-on-activate:
+ SET citus.replicate_reference_tables_on_activate TO off;
+
 step s1-begin:
     BEGIN;
 
@@ -420,10 +453,13 @@ master_remove_node
 
 
 
-starting permutation: s2-begin s2-insert-to-reference-table s1-add-second-worker s2-commit s2-print-content
+starting permutation: s1-do-not-replicate-on-activate s2-begin s2-insert-to-reference-table s1-add-second-worker s2-commit s2-print-content
 create_distributed_table
 
 
+step s1-do-not-replicate-on-activate:
+ SET citus.replicate_reference_tables_on_activate TO off;
+
 step s2-begin:
  BEGIN;
 
@@ -455,10 +491,13 @@ master_remove_node
 
 
 
-starting permutation: s1-begin s1-add-second-worker s2-ddl-on-reference-table s1-commit s2-print-index-count
+starting permutation: s1-do-not-replicate-on-activate s1-begin s1-add-second-worker s2-ddl-on-reference-table s1-commit s2-print-index-count
 create_distributed_table
 
 
+step s1-do-not-replicate-on-activate:
+ SET citus.replicate_reference_tables_on_activate TO off;
+
 step s1-begin:
     BEGIN;
 
@@ -491,10 +530,13 @@ master_remove_node
 
 
 
-starting permutation: s2-begin s2-ddl-on-reference-table s1-add-second-worker s2-commit s2-print-index-count
+starting permutation: s1-do-not-replicate-on-activate s2-begin s2-ddl-on-reference-table s1-add-second-worker s2-commit s2-print-index-count
 create_distributed_table
 
 
+step s1-do-not-replicate-on-activate:
+ SET citus.replicate_reference_tables_on_activate TO off;
+
 step s2-begin:
  BEGIN;
 
@@ -527,10 +569,13 @@ master_remove_node
 
 
 
-starting permutation: s1-begin s1-add-second-worker s2-create-reference-table-2 s1-commit s2-print-content-2
+starting permutation: s1-do-not-replicate-on-activate s1-begin s1-add-second-worker s2-create-reference-table-2 s1-commit s2-print-content-2
 create_distributed_table
 
 
+step s1-do-not-replicate-on-activate:
+ SET citus.replicate_reference_tables_on_activate TO off;
+
 step s1-begin:
     BEGIN;
 
@@ -567,10 +612,13 @@ master_remove_node
 
 
 
-starting permutation: s2-begin s2-create-reference-table-2 s1-add-second-worker s2-commit s2-print-content-2
+starting permutation: s1-do-not-replicate-on-activate s2-begin s2-create-reference-table-2 s1-add-second-worker s2-commit s2-print-content-2
 create_distributed_table
 
 
+step s1-do-not-replicate-on-activate:
+ SET citus.replicate_reference_tables_on_activate TO off;
+
 step s2-begin:
  BEGIN;
 
@@ -606,10 +654,339 @@ master_remove_node
 
 
 
-starting permutation: s1-add-second-worker s2-begin s1-begin s1-drop-reference-table s2-replicate-reference-tables s1-commit s2-commit
+starting permutation: s1-replicate-on-activate s1-begin s1-add-second-worker s2-copy-to-reference-table s1-commit s2-print-content
 create_distributed_table
 
 
+step s1-replicate-on-activate:
+ SET citus.replicate_reference_tables_on_activate TO on;
+
+step s1-begin:
+    BEGIN;
+
+step s1-add-second-worker:
+ SELECT 1 FROM master_add_node('localhost', 57638);
+
+?column?
+
+1
+step s2-copy-to-reference-table:
+ COPY test_reference_table FROM PROGRAM 'echo 1 && echo 2 && echo 3 && echo 4 && echo 5';
+ <waiting ...>
+step s1-commit:
+    COMMIT;
+
+step s2-copy-to-reference-table: <... completed>
+step s2-print-content:
+ SELECT
+  nodeport, success, result
+ FROM
+  run_command_on_placements('test_reference_table', 'select count(*) from %s')
+ ORDER BY
+  nodeport;
+
+nodeport       success        result
+
+57637          t              5
+57638          t              5
+master_remove_node
+
+
+
+
+starting permutation: s1-replicate-on-activate s2-begin s2-copy-to-reference-table s1-add-second-worker s2-commit s2-print-content
+create_distributed_table
+
+
+step s1-replicate-on-activate:
+ SET citus.replicate_reference_tables_on_activate TO on;
+
+step s2-begin:
+ BEGIN;
+
+step s2-copy-to-reference-table:
+ COPY test_reference_table FROM PROGRAM 'echo 1 && echo 2 && echo 3 && echo 4 && echo 5';
+
+step s1-add-second-worker:
+ SELECT 1 FROM master_add_node('localhost', 57638);
+ <waiting ...>
+step s2-commit:
+ COMMIT;
+
+step s1-add-second-worker: <... completed>
+?column?
+
+1
+step s2-print-content:
+ SELECT
+  nodeport, success, result
+ FROM
+  run_command_on_placements('test_reference_table', 'select count(*) from %s')
+ ORDER BY
+  nodeport;
+
+nodeport       success        result
+
+57637          t              5
+57638          t              5
+master_remove_node
+
+
+
+
+starting permutation: s1-replicate-on-activate s1-begin s1-add-second-worker s2-insert-to-reference-table s1-commit s2-print-content
+create_distributed_table
+
+
+step s1-replicate-on-activate:
+ SET citus.replicate_reference_tables_on_activate TO on;
+
+step s1-begin:
+    BEGIN;
+
+step s1-add-second-worker:
+ SELECT 1 FROM master_add_node('localhost', 57638);
+
+?column?
+
+1
+step s2-insert-to-reference-table:
+ INSERT INTO test_reference_table VALUES (6);
+ <waiting ...>
+step s1-commit:
+    COMMIT;
+
+step s2-insert-to-reference-table: <... completed>
+step s2-print-content:
+ SELECT
+  nodeport, success, result
+ FROM
+  run_command_on_placements('test_reference_table', 'select count(*) from %s')
+ ORDER BY
+  nodeport;
+
+nodeport       success        result
+
+57637          t              1
+57638          t              1
+master_remove_node
+
+
+
+
+starting permutation: s1-replicate-on-activate s2-begin s2-insert-to-reference-table s1-add-second-worker s2-commit s2-print-content
+create_distributed_table
+
+
+step s1-replicate-on-activate:
+ SET citus.replicate_reference_tables_on_activate TO on;
+
+step s2-begin:
+ BEGIN;
+
+step s2-insert-to-reference-table:
+ INSERT INTO test_reference_table VALUES (6);
+
+step s1-add-second-worker:
+ SELECT 1 FROM master_add_node('localhost', 57638);
+ <waiting ...>
+step s2-commit:
+ COMMIT;
+
+step s1-add-second-worker: <... completed>
+?column?
+
+1
+step s2-print-content:
+ SELECT
+  nodeport, success, result
+ FROM
+  run_command_on_placements('test_reference_table', 'select count(*) from %s')
+ ORDER BY
+  nodeport;
+
+nodeport       success        result
+
+57637          t              1
+57638          t              1
+master_remove_node
+
+
+
+
+starting permutation: s1-replicate-on-activate s1-begin s1-add-second-worker s2-ddl-on-reference-table s1-commit s2-print-index-count
+create_distributed_table
+
+
+step s1-replicate-on-activate:
+ SET citus.replicate_reference_tables_on_activate TO on;
+
+step s1-begin:
+    BEGIN;
+
+step s1-add-second-worker:
+ SELECT 1 FROM master_add_node('localhost', 57638);
+
+?column?
+
+1
+step s2-ddl-on-reference-table:
+ CREATE INDEX reference_index ON test_reference_table(test_id);
+ <waiting ...>
+step s1-commit:
+    COMMIT;
+
+step s2-ddl-on-reference-table: <... completed>
+step s2-print-index-count:
+ SELECT
+  nodeport, success, result
+ FROM
+  run_command_on_placements('test_reference_table', 'select count(*) from pg_indexes WHERE tablename = ''%s''')
+ ORDER BY
+  nodeport;
+
+nodeport       success        result
+
+57637          t              1
+57638          t              1
+master_remove_node
+
+
+
+
+starting permutation: s1-replicate-on-activate s2-begin s2-ddl-on-reference-table s1-add-second-worker s2-commit s2-print-index-count
+create_distributed_table
+
+
+step s1-replicate-on-activate:
+ SET citus.replicate_reference_tables_on_activate TO on;
+
+step s2-begin:
+ BEGIN;
+
+step s2-ddl-on-reference-table:
+ CREATE INDEX reference_index ON test_reference_table(test_id);
+
+step s1-add-second-worker:
+ SELECT 1 FROM master_add_node('localhost', 57638);
+ <waiting ...>
+step s2-commit:
+ COMMIT;
+
+step s1-add-second-worker: <... completed>
+?column?
+
+1
+step s2-print-index-count:
+ SELECT
+  nodeport, success, result
+ FROM
+  run_command_on_placements('test_reference_table', 'select count(*) from pg_indexes WHERE tablename = ''%s''')
+ ORDER BY
+  nodeport;
+
+nodeport       success        result
+
+57637          t              1
+57638          t              1
+master_remove_node
+
+
+
+
+starting permutation: s1-replicate-on-activate s1-begin s1-add-second-worker s2-create-reference-table-2 s1-commit s2-print-content-2
+create_distributed_table
+
+
+step s1-replicate-on-activate:
+ SET citus.replicate_reference_tables_on_activate TO on;
+
+step s1-begin:
+    BEGIN;
+
+step s1-add-second-worker:
+ SELECT 1 FROM master_add_node('localhost', 57638);
+
+?column?
+
+1
+step s2-create-reference-table-2:
+ SELECT create_reference_table('test_reference_table_2');
+ <waiting ...>
+step s1-commit:
+    COMMIT;
+
+step s2-create-reference-table-2: <... completed>
+create_reference_table
+
+
+step s2-print-content-2:
+ SELECT
+  nodeport, success, result
+ FROM
+  run_command_on_placements('test_reference_table_2', 'select count(*) from %s')
+ ORDER BY
+  nodeport;
+
+nodeport       success        result
+
+57637          t              1
+57638          t              1
+master_remove_node
+
+
+
+
+starting permutation: s1-replicate-on-activate s2-begin s2-create-reference-table-2 s1-add-second-worker s2-commit s2-print-content-2
+create_distributed_table
+
+
+step s1-replicate-on-activate:
+ SET citus.replicate_reference_tables_on_activate TO on;
+
+step s2-begin:
+ BEGIN;
+
+step s2-create-reference-table-2:
+ SELECT create_reference_table('test_reference_table_2');
+
+create_reference_table
+
+
+step s1-add-second-worker:
+ SELECT 1 FROM master_add_node('localhost', 57638);
+ <waiting ...>
+step s2-commit:
+ COMMIT;
+
+step s1-add-second-worker: <... completed>
+?column?
+
+1
+step s2-print-content-2:
+ SELECT
+  nodeport, success, result
+ FROM
+  run_command_on_placements('test_reference_table_2', 'select count(*) from %s')
+ ORDER BY
+  nodeport;
+
+nodeport       success        result
+
+57637          t              1
+57638          t              1
+master_remove_node
+
+
+
+
+starting permutation: s1-do-not-replicate-on-activate s1-add-second-worker s2-begin s1-begin s1-drop-reference-table s2-replicate-reference-tables s1-commit s2-commit
+create_distributed_table
+
+
+step s1-do-not-replicate-on-activate:
+ SET citus.replicate_reference_tables_on_activate TO off;
+
 step s1-add-second-worker:
  SELECT 1 FROM master_add_node('localhost', 57638);
 
diff --git a/src/test/regress/expected/isolation_dump_global_wait_edges.out b/src/test/regress/expected/isolation_dump_global_wait_edges.out
index 6e26a07a7..9384bd287 100644
--- a/src/test/regress/expected/isolation_dump_global_wait_edges.out
+++ b/src/test/regress/expected/isolation_dump_global_wait_edges.out
@@ -28,11 +28,11 @@ step detector-dump-wait-edges:
 
 waiting_transaction_numblocking_transaction_numblocking_transaction_waiting
 
-360            359            f
+392            391            f
 transactionnumberwaitingtransactionnumbers
 
-359
-360            359
+391
+392            391
 step s1-abort:
     ABORT;
 
@@ -75,14 +75,14 @@ step detector-dump-wait-edges:
 
 waiting_transaction_numblocking_transaction_numblocking_transaction_waiting
 
-364            363            f
-365            363            f
-365            364            t
+396            395            f
+397            395            f
+397            396            t
 transactionnumberwaitingtransactionnumbers
 
-363
-364            363
-365            363,364
+395
+396            395
+397            395,396
 step s1-abort:
     ABORT;
 
diff --git a/src/test/regress/expected/local_shard_copy.out b/src/test/regress/expected/local_shard_copy.out
index e07f595a8..ef9e5e615 100644
--- a/src/test/regress/expected/local_shard_copy.out
+++ b/src/test/regress/expected/local_shard_copy.out
@@ -2,6 +2,7 @@ CREATE SCHEMA local_shard_copy;
 SET search_path TO local_shard_copy;
 SET client_min_messages TO DEBUG;
 SET citus.next_shard_id TO 1570000;
+SET citus.replicate_reference_tables_on_activate TO off;
 SELECT * FROM master_add_node('localhost', :master_port, groupid := 0);
 DEBUG:  schema "public" already exists, skipping
 DETAIL:  NOTICE from localhost:xxxxx
diff --git a/src/test/regress/expected/multi_metadata_sync.out b/src/test/regress/expected/multi_metadata_sync.out
index 99557f0b6..3cbd72a5b 100644
--- a/src/test/regress/expected/multi_metadata_sync.out
+++ b/src/test/regress/expected/multi_metadata_sync.out
@@ -4,6 +4,7 @@
 -- Tests for metadata snapshot functions, metadata syncing functions and propagation of
 -- metadata changes to MX tables.
 ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 1310000;
+SET citus.replicate_reference_tables_on_activate TO off;
 SELECT nextval('pg_catalog.pg_dist_placement_placementid_seq') AS last_placement_id
 \gset
 ALTER SEQUENCE pg_catalog.pg_dist_placement_placementid_seq RESTART 100000;
@@ -1394,6 +1395,7 @@ WHERE logicalrelid='mx_ref'::regclass;
 
 \c - - - :master_port
 SELECT master_add_node('localhost', :worker_2_port);
+NOTICE:  Replicating reference table "mx_ref" to the node localhost:xxxxx
  master_add_node
 ---------------------------------------------------------------------
                7
@@ -1406,7 +1408,8 @@ ORDER BY shardid, nodeport;
  shardid | nodename  | nodeport
 ---------------------------------------------------------------------
  1310073 | localhost |    57637
-(1 row)
+ 1310073 | localhost |    57638
+(2 rows)
 
 \c - - - :worker_1_port
 SELECT shardid, nodename, nodeport
@@ -1416,7 +1419,8 @@ ORDER BY shardid, nodeport;
  shardid | nodename  | nodeport
 ---------------------------------------------------------------------
  1310073 | localhost |    57637
-(1 row)
+ 1310073 | localhost |    57638
+(2 rows)
 
 -- Get the metadata back into a consistent state
 \c - - - :master_port
diff --git a/src/test/regress/expected/multi_mx_node_metadata.out b/src/test/regress/expected/multi_mx_node_metadata.out
index 8f6720887..c28eba404 100644
--- a/src/test/regress/expected/multi_mx_node_metadata.out
+++ b/src/test/regress/expected/multi_mx_node_metadata.out
@@ -8,6 +8,7 @@ SELECT nextval('pg_catalog.pg_dist_shardid_seq') AS last_shard_id \gset
 SET citus.replication_model TO streaming;
 SET citus.shard_count TO 8;
 SET citus.shard_replication_factor TO 1;
+SET citus.replicate_reference_tables_on_activate TO off;
 -- Simulates a readonly node by setting default_transaction_read_only.
 CREATE FUNCTION mark_node_readonly(hostname TEXT, port INTEGER, isreadonly BOOLEAN)
     RETURNS TEXT
diff --git a/src/test/regress/expected/multi_remove_node_reference_table.out b/src/test/regress/expected/multi_remove_node_reference_table.out
index 97b0d17d6..f30915788 100644
--- a/src/test/regress/expected/multi_remove_node_reference_table.out
+++ b/src/test/regress/expected/multi_remove_node_reference_table.out
@@ -3,6 +3,7 @@
 --
 -- Tests that check the metadata after master_remove_node.
 SET citus.next_shard_id TO 1380000;
+SET citus.replicate_reference_tables_on_activate TO off;
 ALTER SEQUENCE pg_catalog.pg_dist_colocationid_seq RESTART 1380000;
 ALTER SEQUENCE pg_catalog.pg_dist_groupid_seq RESTART 1380000;
 ALTER SEQUENCE pg_catalog.pg_dist_node_nodeid_seq RESTART 1380000;
@@ -216,6 +217,7 @@ WHERE
 (0 rows)
 
 \c - - - :master_port
+SET citus.replicate_reference_tables_on_activate TO off;
 -- remove same node twice
 SELECT master_remove_node('localhost', :worker_2_port);
 ERROR:  node at "localhost:xxxxx" does not exist
@@ -448,6 +450,7 @@ WHERE
 (0 rows)
 
 \c - - - :master_port
+SET citus.replicate_reference_tables_on_activate TO off;
 -- re-add the node for next tests
 SELECT 1 FROM master_add_node('localhost', :worker_2_port);
  ?column?
@@ -570,6 +573,7 @@ SELECT * FROM remove_node_reference_table;
 (1 row)
 
 \c - - - :master_port
+SET citus.replicate_reference_tables_on_activate TO off;
 -- re-add the node for next tests
 SELECT 1 FROM master_add_node('localhost', :worker_2_port);
  ?column?
@@ -679,6 +683,7 @@ WHERE
 (0 rows)
 
 \c - - - :master_port
+SET citus.replicate_reference_tables_on_activate TO off;
 SET citus.next_shard_id TO 1380001;
 -- verify table structure is changed
 SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.remove_node_reference_table'::regclass;
@@ -885,6 +890,7 @@ WHERE
 (0 rows)
 
 \c - - - :master_port
+SET citus.replicate_reference_tables_on_activate TO off;
 -- re-add the node for next tests
 SELECT 1 FROM master_add_node('localhost', :worker_2_port);
  ?column?
@@ -994,6 +1000,7 @@ WHERE
 (0 rows)
 
 \c - - - :master_port
+SET citus.replicate_reference_tables_on_activate TO off;
 -- re-add the node for next tests
 SELECT 1 FROM master_activate_node('localhost', :worker_2_port);
  ?column?
diff --git a/src/test/regress/expected/multi_replicate_reference_table.out b/src/test/regress/expected/multi_replicate_reference_table.out
index cabf02679..bf7c9377c 100644
--- a/src/test/regress/expected/multi_replicate_reference_table.out
+++ b/src/test/regress/expected/multi_replicate_reference_table.out
@@ -8,6 +8,7 @@ SET citus.next_shard_id TO 1370000;
 ALTER SEQUENCE pg_catalog.pg_dist_colocationid_seq RESTART 1370000;
 ALTER SEQUENCE pg_catalog.pg_dist_groupid_seq RESTART 1370000;
 ALTER SEQUENCE pg_catalog.pg_dist_node_nodeid_seq RESTART 1370000;
+SET citus.replicate_reference_tables_on_activate TO off;
 -- only query shards created in this test
 CREATE VIEW pg_dist_shard_placement_view AS
 SELECT * FROM pg_dist_shard_placement WHERE shardid BETWEEN 1370000 AND 1380000;
@@ -846,6 +847,73 @@ SELECT min(result) = max(result) AS consistent FROM run_command_on_placements('r
  t
 (1 row)
 
+SET client_min_messages TO WARNING;
+SELECT count(*) AS ref_table_placements FROM pg_dist_shard NATURAL JOIN pg_dist_shard_placement WHERE logicalrelid = 'ref_table'::regclass \gset
+-- remove reference table replica from worker 2
+SELECT 1 FROM master_remove_node('localhost', :worker_2_port);
+ ?column?
+---------------------------------------------------------------------
+        1
+(1 row)
+
+SELECT count(*) - :ref_table_placements FROM pg_dist_shard NATURAL JOIN pg_dist_shard_placement WHERE logicalrelid = 'ref_table'::regclass;
+ ?column?
+---------------------------------------------------------------------
+       -1
+(1 row)
+
+-- test setting citus.replicate_reference_tables_on_activate to on
+-- master_add_node
+SET citus.replicate_reference_tables_on_activate TO on;
+SELECT 1 FROM master_add_node('localhost', :worker_2_port);
+ ?column?
+---------------------------------------------------------------------
+        1
+(1 row)
+
+SELECT count(*) - :ref_table_placements FROM pg_dist_shard NATURAL JOIN pg_dist_shard_placement WHERE logicalrelid = 'ref_table'::regclass;
+ ?column?
+---------------------------------------------------------------------
+        0
+(1 row)
+
+-- master_activate_node
+SELECT 1 FROM master_remove_node('localhost', :worker_2_port);
+ ?column?
+---------------------------------------------------------------------
+        1
+(1 row)
+
+SELECT 1 FROM master_add_inactive_node('localhost', :worker_2_port);
+ ?column?
+---------------------------------------------------------------------
+        1
+(1 row)
+
+SELECT count(*) - :ref_table_placements FROM pg_dist_shard NATURAL JOIN pg_dist_shard_placement WHERE logicalrelid = 'ref_table'::regclass;
+ ?column?
+---------------------------------------------------------------------
+       -1
+(1 row)
+
+SELECT 1 FROM master_activate_node('localhost', :worker_2_port);
+ ?column?
+---------------------------------------------------------------------
+        1
+(1 row)
+
+SELECT count(*) - :ref_table_placements FROM pg_dist_shard NATURAL JOIN pg_dist_shard_placement WHERE logicalrelid = 'ref_table'::regclass;
+ ?column?
+---------------------------------------------------------------------
+        0
+(1 row)
+
+SELECT min(result) = max(result) AS consistent FROM run_command_on_placements('ref_table', 'SELECT sum(a) FROM %s');
+ consistent
+---------------------------------------------------------------------
+ t
+(1 row)
+
 -- test adding an invalid node while we have reference tables to replicate
 -- set client message level to ERROR and verbosity to terse to supporess
 -- OS-dependent host name resolution warnings
diff --git a/src/test/regress/expected/multi_transactional_drop_shards.out b/src/test/regress/expected/multi_transactional_drop_shards.out
index 93043aa92..d11038810 100644
--- a/src/test/regress/expected/multi_transactional_drop_shards.out
+++ b/src/test/regress/expected/multi_transactional_drop_shards.out
@@ -4,6 +4,7 @@
 -- Tests that check the metadata returned by the master node.
 ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 1410000;
 SET citus.shard_count TO 4;
+SET client_min_messages TO WARNING;
 -- test DROP TABLE(ergo master_drop_all_shards) in transaction, then ROLLBACK
 CREATE TABLE transactional_drop_shards(column1 int);
 SELECT create_distributed_table('transactional_drop_shards', 'column1');
@@ -653,6 +654,7 @@ ORDER BY
 (0 rows)
 
 \c - - - :master_port
+SET client_min_messages TO WARNING;
 -- try using the coordinator as a worker and then dropping the table
 SELECT 1 FROM master_add_node('localhost', :master_port);
  ?column?
@@ -685,8 +687,6 @@ SELECT stop_metadata_sync_to_node('localhost', :worker_1_port);
 
 -- test DROP TABLE as a non-superuser in a transaction block
 CREATE USER try_drop_table WITH LOGIN;
-NOTICE:  not propagating CREATE ROLE/USER commands to worker nodes
-HINT:  Connect to worker nodes directly to manually create all necessary users and roles.
 SELECT run_command_on_workers('CREATE USER try_drop_table WITH LOGIN');
       run_command_on_workers
 ---------------------------------------------------------------------
diff --git a/src/test/regress/expected/propagate_extension_commands.out b/src/test/regress/expected/propagate_extension_commands.out
index 8138a6e32..36f7254c9 100644
--- a/src/test/regress/expected/propagate_extension_commands.out
+++ b/src/test/regress/expected/propagate_extension_commands.out
@@ -1,6 +1,7 @@
 CREATE SCHEMA "extension'test";
 -- use  a schema name with escape character
 SET search_path TO "extension'test";
+SET client_min_messages TO WARNING;
 -- create an extension on the given search_path
 -- the extension is on contrib, so should be avaliable for the regression tests
 CREATE EXTENSION seg;
@@ -88,10 +89,8 @@ SELECT create_reference_table('ref_table');
 (1 row)
 
 -- now,  drop the extension, recreate it with an older version and update it to latest version
-SET client_min_messages TO WARNING;
 DROP EXTENSION isn CASCADE;
 CREATE EXTENSION isn WITH VERSION "1.1";
-RESET client_min_messages;
 -- before updating the version, ensure the current version
 SELECT run_command_on_workers($$SELECT extversion FROM pg_extension WHERE extname = 'isn'$$);
  run_command_on_workers
@@ -137,22 +136,16 @@ SELECT run_command_on_workers($$SELECT nspname from pg_namespace where oid=(SELE
  (localhost,57638,t,public)
 (2 rows)
 
--- SET client_min_messages TO WARNING before executing a DROP EXTENSION statement
-SET client_min_messages TO WARNING;
 -- drop the extension finally
 DROP EXTENSION isn CASCADE;
--- restore client_min_messages after DROP EXTENSION
-RESET client_min_messages;
 -- now make sure that the reference tables depending on an extension can be succesfully created.
 -- we should also ensure that we replicate this reference table (and hence the extension)
 -- to new nodes after calling master_activate_node.
 -- now, first drop seg and existing objects before next test
-SET client_min_messages TO WARNING;
 DROP EXTENSION seg CASCADE;
 -- but as we have only 2 ports in postgresql tests, let's remove one of the nodes first
 -- before remove, first remove the existing relations (due to the other tests)
 DROP SCHEMA "extension'test" CASCADE;
-RESET client_min_messages;
 SELECT 1 from master_remove_node('localhost', :worker_2_port);
  ?column?
 ---------------------------------------------------------------------
@@ -321,11 +314,7 @@ SELECT run_command_on_workers($$SELECT count(*) FROM pg_extension WHERE extname
 CREATE TABLE t1 (A int);
 CREATE VIEW v1 AS select * from t1;
 ALTER EXTENSION seg ADD VIEW v1;
-NOTICE:  Citus does not propagate adding/dropping member objects
-HINT:  You can add/drop the member objects on the workers as well.
 ALTER EXTENSION seg DROP VIEW v1;
-NOTICE:  Citus does not propagate adding/dropping member objects
-HINT:  You can add/drop the member objects on the workers as well.
 DROP VIEW v1;
 DROP TABLE t1;
 -- drop multiple extensions at the same time
@@ -334,8 +323,6 @@ CREATE EXTENSION isn WITH VERSION '1.1' SCHEMA public;
 set citus.enable_ddl_propagation to 'off';
 CREATE EXTENSION pg_buffercache;
 set citus.enable_ddl_propagation to 'on';
--- SET client_min_messages TO WARNING before executing a DROP EXTENSION statement
-SET client_min_messages TO WARNING;
 DROP EXTENSION pg_buffercache, isn CASCADE;
 SELECT count(*) FROM pg_extension WHERE extname IN ('pg_buffercache', 'isn');
  count
@@ -343,10 +330,6 @@ SELECT count(*) FROM pg_extension WHERE extname IN ('pg_buffercache', 'isn');
      0
 (1 row)
 
--- restore client_min_messages after DROP EXTENSION
-RESET client_min_messages;
--- SET client_min_messages TO WARNING before executing a DROP EXTENSION statement
-SET client_min_messages TO WARNING;
 -- drop extension should just work
 DROP EXTENSION seg CASCADE;
 SELECT count(*) FROM citus.pg_dist_object WHERE objid = (SELECT oid FROM pg_extension WHERE extname = 'seg');
@@ -362,8 +345,6 @@ SELECT run_command_on_workers($$SELECT count(*) FROM pg_extension WHERE extname
  (localhost,57638,t,0)
 (2 rows)
 
--- restore client_min_messages after DROP EXTENSION
-RESET client_min_messages;
 -- make sure that the extension is not avaliable anymore as a distributed object
 SELECT count(*) FROM citus.pg_dist_object WHERE objid = (SELECT oid FROM pg_extension WHERE extname IN ('seg', 'isn'));
  count
@@ -402,13 +383,11 @@ SELECT run_command_on_workers($$SELECT extversion FROM pg_extension WHERE extnam
 (2 rows)
 
 -- drop the schema and all the objects
-SET client_min_messages TO WARNING;
 DROP SCHEMA "extension'test" CASCADE;
 -- recreate for the next tests
 CREATE SCHEMA "extension'test";
 -- use  a schema name with escape character
 SET search_path TO "extension'test";
-RESET client_min_messages;
 -- remove the node, we'll add back again
 SELECT 1 from master_remove_node('localhost', :worker_2_port);
  ?column?
@@ -462,5 +441,4 @@ SELECT run_command_on_workers($$SELECT count(*) FROM pg_extension WHERE extname
 (2 rows)
 
 -- drop the schema and all the objects
-SET client_min_messages TO WARNING;
 DROP SCHEMA "extension'test" CASCADE;
diff --git a/src/test/regress/spec/isolation_add_node_vs_reference_table_operations.spec b/src/test/regress/spec/isolation_add_node_vs_reference_table_operations.spec
index 9ff681f63..72517669a 100644
--- a/src/test/regress/spec/isolation_add_node_vs_reference_table_operations.spec
+++ b/src/test/regress/spec/isolation_add_node_vs_reference_table_operations.spec
@@ -29,6 +29,16 @@ step "s1-begin"
     BEGIN;
 }
 
+step "s1-replicate-on-activate"
+{
+	SET citus.replicate_reference_tables_on_activate TO on;
+}
+
+step "s1-do-not-replicate-on-activate"
+{
+	SET citus.replicate_reference_tables_on_activate TO off;
+}
+
 step "s1-add-second-worker"
 {
 	SELECT 1 FROM master_add_node('localhost', 57638);
@@ -129,25 +139,35 @@ step "s2-print-index-count"
 // note that we need to run "s1-load-metadata-cache" and "s2-load-metadata-cache"
 // to ensure that metadata is cached otherwise the test would be useless since
 // the cache would be empty and the metadata data is gathered from the tables directly
-permutation "s2-load-metadata-cache" "s1-begin" "s1-add-second-worker" "s2-copy-to-reference-table" "s1-commit" "s2-print-content"
-permutation "s2-load-metadata-cache" "s2-begin" "s2-copy-to-reference-table" "s1-add-second-worker" "s2-commit" "s2-print-content"
-permutation "s2-load-metadata-cache" "s1-begin" "s1-add-second-worker" "s2-insert-to-reference-table" "s1-commit" "s2-print-content"
-permutation "s2-load-metadata-cache" "s2-begin" "s2-insert-to-reference-table" "s1-add-second-worker" "s2-commit" "s2-print-content"
-permutation "s2-load-metadata-cache" "s1-begin" "s1-add-second-worker" "s2-ddl-on-reference-table" "s1-commit" "s2-print-index-count"
-permutation "s2-load-metadata-cache" "s2-begin" "s2-ddl-on-reference-table" "s1-add-second-worker" "s2-commit" "s2-print-index-count"
-permutation "s2-load-metadata-cache" "s1-begin" "s1-add-second-worker" "s2-create-reference-table-2" "s1-commit" "s2-print-content-2"
-permutation "s2-load-metadata-cache" "s2-begin" "s2-create-reference-table-2" "s1-add-second-worker" "s2-commit" "s2-print-content-2"
+permutation "s1-do-not-replicate-on-activate" "s2-load-metadata-cache" "s1-begin" "s1-add-second-worker" "s2-copy-to-reference-table" "s1-commit" "s2-print-content"
+permutation "s1-do-not-replicate-on-activate" "s2-load-metadata-cache" "s2-begin" "s2-copy-to-reference-table" "s1-add-second-worker" "s2-commit" "s2-print-content"
+permutation "s1-do-not-replicate-on-activate" "s2-load-metadata-cache" "s1-begin" "s1-add-second-worker" "s2-insert-to-reference-table" "s1-commit" "s2-print-content"
+permutation "s1-do-not-replicate-on-activate" "s2-load-metadata-cache" "s2-begin" "s2-insert-to-reference-table" "s1-add-second-worker" "s2-commit" "s2-print-content"
+permutation "s1-do-not-replicate-on-activate" "s2-load-metadata-cache" "s1-begin" "s1-add-second-worker" "s2-ddl-on-reference-table" "s1-commit" "s2-print-index-count"
+permutation "s1-do-not-replicate-on-activate" "s2-load-metadata-cache" "s2-begin" "s2-ddl-on-reference-table" "s1-add-second-worker" "s2-commit" "s2-print-index-count"
+permutation "s1-do-not-replicate-on-activate" "s2-load-metadata-cache" "s1-begin" "s1-add-second-worker" "s2-create-reference-table-2" "s1-commit" "s2-print-content-2"
+permutation "s1-do-not-replicate-on-activate" "s2-load-metadata-cache" "s2-begin" "s2-create-reference-table-2" "s1-add-second-worker" "s2-commit" "s2-print-content-2"
 
 
 // same tests without loading the cache
-permutation "s1-begin" "s1-add-second-worker" "s2-copy-to-reference-table" "s1-commit" "s2-print-content"
-permutation "s2-begin" "s2-copy-to-reference-table" "s1-add-second-worker" "s2-commit" "s2-print-content"
-permutation "s1-begin" "s1-add-second-worker" "s2-insert-to-reference-table" "s1-commit" "s2-print-content"
-permutation "s2-begin" "s2-insert-to-reference-table" "s1-add-second-worker" "s2-commit" "s2-print-content"
-permutation "s1-begin" "s1-add-second-worker" "s2-ddl-on-reference-table" "s1-commit" "s2-print-index-count"
-permutation "s2-begin" "s2-ddl-on-reference-table" "s1-add-second-worker" "s2-commit" "s2-print-index-count"
-permutation "s1-begin" "s1-add-second-worker" "s2-create-reference-table-2" "s1-commit" "s2-print-content-2"
-permutation "s2-begin" "s2-create-reference-table-2" "s1-add-second-worker" "s2-commit" "s2-print-content-2"
+permutation "s1-do-not-replicate-on-activate" "s1-begin" "s1-add-second-worker" "s2-copy-to-reference-table" "s1-commit" "s2-print-content"
+permutation "s1-do-not-replicate-on-activate" "s2-begin" "s2-copy-to-reference-table" "s1-add-second-worker" "s2-commit" "s2-print-content"
+permutation "s1-do-not-replicate-on-activate" "s1-begin" "s1-add-second-worker" "s2-insert-to-reference-table" "s1-commit" "s2-print-content"
+permutation "s1-do-not-replicate-on-activate" "s2-begin" "s2-insert-to-reference-table" "s1-add-second-worker" "s2-commit" "s2-print-content"
+permutation "s1-do-not-replicate-on-activate" "s1-begin" "s1-add-second-worker" "s2-ddl-on-reference-table" "s1-commit" "s2-print-index-count"
+permutation "s1-do-not-replicate-on-activate" "s2-begin" "s2-ddl-on-reference-table" "s1-add-second-worker" "s2-commit" "s2-print-index-count"
+permutation "s1-do-not-replicate-on-activate" "s1-begin" "s1-add-second-worker" "s2-create-reference-table-2" "s1-commit" "s2-print-content-2"
+permutation "s1-do-not-replicate-on-activate" "s2-begin" "s2-create-reference-table-2" "s1-add-second-worker" "s2-commit" "s2-print-content-2"
+
+// same tests with replicate on activate
+permutation "s1-replicate-on-activate" "s1-begin" "s1-add-second-worker" "s2-copy-to-reference-table" "s1-commit" "s2-print-content"
+permutation "s1-replicate-on-activate" "s2-begin" "s2-copy-to-reference-table" "s1-add-second-worker" "s2-commit" "s2-print-content"
+permutation "s1-replicate-on-activate" "s1-begin" "s1-add-second-worker" "s2-insert-to-reference-table" "s1-commit" "s2-print-content"
+permutation "s1-replicate-on-activate" "s2-begin" "s2-insert-to-reference-table" "s1-add-second-worker" "s2-commit" "s2-print-content"
+permutation "s1-replicate-on-activate" "s1-begin" "s1-add-second-worker" "s2-ddl-on-reference-table" "s1-commit" "s2-print-index-count"
+permutation "s1-replicate-on-activate" "s2-begin" "s2-ddl-on-reference-table" "s1-add-second-worker" "s2-commit" "s2-print-index-count"
+permutation "s1-replicate-on-activate" "s1-begin" "s1-add-second-worker" "s2-create-reference-table-2" "s1-commit" "s2-print-content-2"
+permutation "s1-replicate-on-activate" "s2-begin" "s2-create-reference-table-2" "s1-add-second-worker" "s2-commit" "s2-print-content-2"
 
 // verify drop table blocks replicate reference tables
-permutation "s1-add-second-worker" "s2-begin" "s1-begin" "s1-drop-reference-table" "s2-replicate-reference-tables" "s1-commit" "s2-commit"
+permutation "s1-do-not-replicate-on-activate" "s1-add-second-worker" "s2-begin" "s1-begin" "s1-drop-reference-table" "s2-replicate-reference-tables" "s1-commit" "s2-commit"
diff --git a/src/test/regress/sql/failure_add_disable_node.sql b/src/test/regress/sql/failure_add_disable_node.sql
index 4cb8cd860..eff908ae2 100644
--- a/src/test/regress/sql/failure_add_disable_node.sql
+++ b/src/test/regress/sql/failure_add_disable_node.sql
@@ -8,6 +8,7 @@
 SELECT citus.mitmproxy('conn.allow()');
 
 SET citus.next_shard_id TO 200000;
+SET citus.replicate_reference_tables_on_activate TO off;
 
 -- verify we have all worker nodes present
 SELECT * FROM master_get_active_worker_nodes()
diff --git a/src/test/regress/sql/local_shard_copy.sql b/src/test/regress/sql/local_shard_copy.sql
index 7240c62a0..f0b94e21a 100644
--- a/src/test/regress/sql/local_shard_copy.sql
+++ b/src/test/regress/sql/local_shard_copy.sql
@@ -3,6 +3,7 @@ SET search_path TO local_shard_copy;
 
 SET client_min_messages TO DEBUG;
 SET citus.next_shard_id TO 1570000;
+SET citus.replicate_reference_tables_on_activate TO off;
 
 SELECT * FROM master_add_node('localhost', :master_port, groupid := 0);
 
diff --git a/src/test/regress/sql/multi_metadata_sync.sql b/src/test/regress/sql/multi_metadata_sync.sql
index c0f7652b1..ce3669500 100644
--- a/src/test/regress/sql/multi_metadata_sync.sql
+++ b/src/test/regress/sql/multi_metadata_sync.sql
@@ -7,6 +7,7 @@
 
 
 ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 1310000;
+SET citus.replicate_reference_tables_on_activate TO off;
 
 SELECT nextval('pg_catalog.pg_dist_placement_placementid_seq') AS last_placement_id
 \gset
diff --git a/src/test/regress/sql/multi_mx_node_metadata.sql b/src/test/regress/sql/multi_mx_node_metadata.sql
index e31a34038..ebde6af4f 100644
--- a/src/test/regress/sql/multi_mx_node_metadata.sql
+++ b/src/test/regress/sql/multi_mx_node_metadata.sql
@@ -11,6 +11,7 @@ SELECT nextval('pg_catalog.pg_dist_shardid_seq') AS last_shard_id \gset
 SET citus.replication_model TO streaming;
 SET citus.shard_count TO 8;
 SET citus.shard_replication_factor TO 1;
+SET citus.replicate_reference_tables_on_activate TO off;
 
 -- Simulates a readonly node by setting default_transaction_read_only.
 CREATE FUNCTION mark_node_readonly(hostname TEXT, port INTEGER, isreadonly BOOLEAN)
diff --git a/src/test/regress/sql/multi_remove_node_reference_table.sql b/src/test/regress/sql/multi_remove_node_reference_table.sql
index dca5618d2..376171d20 100644
--- a/src/test/regress/sql/multi_remove_node_reference_table.sql
+++ b/src/test/regress/sql/multi_remove_node_reference_table.sql
@@ -5,6 +5,7 @@
 
 
 SET citus.next_shard_id TO 1380000;
+SET citus.replicate_reference_tables_on_activate TO off;
 ALTER SEQUENCE pg_catalog.pg_dist_colocationid_seq RESTART 1380000;
 ALTER SEQUENCE pg_catalog.pg_dist_groupid_seq RESTART 1380000;
 ALTER SEQUENCE pg_catalog.pg_dist_node_nodeid_seq RESTART 1380000;
@@ -115,6 +116,7 @@ WHERE
     nodeport = :worker_2_port;
 
 \c - - - :master_port
+SET citus.replicate_reference_tables_on_activate TO off;
 
 -- remove same node twice
 SELECT master_remove_node('localhost', :worker_2_port);
@@ -260,6 +262,7 @@ WHERE
     nodeport = :worker_2_port;
 
 \c - - - :master_port
+SET citus.replicate_reference_tables_on_activate TO off;
 
 -- re-add the node for next tests
 SELECT 1 FROM master_add_node('localhost', :worker_2_port);
@@ -335,6 +338,7 @@ WHERE
 SELECT * FROM remove_node_reference_table;
 
 \c - - - :master_port
+SET citus.replicate_reference_tables_on_activate TO off;
 
 -- re-add the node for next tests
 SELECT 1 FROM master_add_node('localhost', :worker_2_port);
@@ -406,6 +410,7 @@ WHERE
     nodeport = :worker_2_port;
 
 \c - - - :master_port
+SET citus.replicate_reference_tables_on_activate TO off;
 
 SET citus.next_shard_id TO 1380001;
 
@@ -528,6 +533,7 @@ WHERE
     nodeport = :worker_2_port;
 
 \c - - - :master_port
+SET citus.replicate_reference_tables_on_activate TO off;
 
 -- re-add the node for next tests
 SELECT 1 FROM master_add_node('localhost', :worker_2_port);
@@ -599,6 +605,7 @@ WHERE
     nodeport = :worker_2_port;
 
 \c - - - :master_port
+SET citus.replicate_reference_tables_on_activate TO off;
 
 -- re-add the node for next tests
 SELECT 1 FROM master_activate_node('localhost', :worker_2_port);
diff --git a/src/test/regress/sql/multi_replicate_reference_table.sql b/src/test/regress/sql/multi_replicate_reference_table.sql
index 56d458296..7b1951bfb 100644
--- a/src/test/regress/sql/multi_replicate_reference_table.sql
+++ b/src/test/regress/sql/multi_replicate_reference_table.sql
@@ -11,6 +11,8 @@ ALTER SEQUENCE pg_catalog.pg_dist_colocationid_seq RESTART 1370000;
 ALTER SEQUENCE pg_catalog.pg_dist_groupid_seq RESTART 1370000;
 ALTER SEQUENCE pg_catalog.pg_dist_node_nodeid_seq RESTART 1370000;
 
+SET citus.replicate_reference_tables_on_activate TO off;
+
 -- only query shards created in this test
 CREATE VIEW pg_dist_shard_placement_view AS
 SELECT * FROM pg_dist_shard_placement WHERE shardid BETWEEN 1370000 AND 1380000;
@@ -547,6 +549,34 @@ COMMIT;
 
 SELECT min(result) = max(result) AS consistent FROM run_command_on_placements('ref_table', 'SELECT sum(a) FROM %s');
 
+SET client_min_messages TO WARNING;
+
+SELECT count(*) AS ref_table_placements FROM pg_dist_shard NATURAL JOIN pg_dist_shard_placement WHERE logicalrelid = 'ref_table'::regclass \gset
+
+-- remove reference table replica from worker 2
+SELECT 1 FROM master_remove_node('localhost', :worker_2_port);
+
+SELECT count(*) - :ref_table_placements FROM pg_dist_shard NATURAL JOIN pg_dist_shard_placement WHERE logicalrelid = 'ref_table'::regclass;
+
+-- test setting citus.replicate_reference_tables_on_activate to on
+-- master_add_node
+SET citus.replicate_reference_tables_on_activate TO on;
+SELECT 1 FROM master_add_node('localhost', :worker_2_port);
+
+SELECT count(*) - :ref_table_placements FROM pg_dist_shard NATURAL JOIN pg_dist_shard_placement WHERE logicalrelid = 'ref_table'::regclass;
+
+-- master_activate_node
+SELECT 1 FROM master_remove_node('localhost', :worker_2_port);
+SELECT 1 FROM master_add_inactive_node('localhost', :worker_2_port);
+
+SELECT count(*) - :ref_table_placements FROM pg_dist_shard NATURAL JOIN pg_dist_shard_placement WHERE logicalrelid = 'ref_table'::regclass;
+
+SELECT 1 FROM master_activate_node('localhost', :worker_2_port);
+
+SELECT count(*) - :ref_table_placements FROM pg_dist_shard NATURAL JOIN pg_dist_shard_placement WHERE logicalrelid = 'ref_table'::regclass;
+
+SELECT min(result) = max(result) AS consistent FROM run_command_on_placements('ref_table', 'SELECT sum(a) FROM %s');
+
 -- test adding an invalid node while we have reference tables to replicate
 -- set client message level to ERROR and verbosity to terse to supporess
 -- OS-dependent host name resolution warnings
diff --git a/src/test/regress/sql/multi_transactional_drop_shards.sql b/src/test/regress/sql/multi_transactional_drop_shards.sql
index f574dedcf..f4b6d83b9 100644
--- a/src/test/regress/sql/multi_transactional_drop_shards.sql
+++ b/src/test/regress/sql/multi_transactional_drop_shards.sql
@@ -7,6 +7,7 @@
 ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 1410000;
 
 SET citus.shard_count TO 4;
+SET client_min_messages TO WARNING;
 
 -- test DROP TABLE(ergo master_drop_all_shards) in transaction, then ROLLBACK
 CREATE TABLE transactional_drop_shards(column1 int);
@@ -362,6 +363,7 @@ ORDER BY
     shardid, nodename, nodeport;
 
 \c - - - :master_port
+SET client_min_messages TO WARNING;
 
 -- try using the coordinator as a worker and then dropping the table
 SELECT 1 FROM master_add_node('localhost', :master_port);
diff --git a/src/test/regress/sql/propagate_extension_commands.sql b/src/test/regress/sql/propagate_extension_commands.sql
index f52233a60..ad7ba749b 100644
--- a/src/test/regress/sql/propagate_extension_commands.sql
+++ b/src/test/regress/sql/propagate_extension_commands.sql
@@ -3,6 +3,8 @@ CREATE SCHEMA "extension'test";
 -- use  a schema name with escape character
 SET search_path TO "extension'test";
 
+SET client_min_messages TO WARNING;
+
 -- create an extension on the given search_path
 -- the extension is on contrib, so should be avaliable for the regression tests
 CREATE EXTENSION seg;
@@ -48,10 +50,8 @@ CREATE TABLE ref_table (a public.issn);
 SELECT create_reference_table('ref_table');
 
 -- now,  drop the extension, recreate it with an older version and update it to latest version
-SET client_min_messages TO WARNING;
 DROP EXTENSION isn CASCADE;
 CREATE EXTENSION isn WITH VERSION "1.1";
-RESET client_min_messages;
 
 -- before updating the version, ensure the current version
 SELECT run_command_on_workers($$SELECT extversion FROM pg_extension WHERE extname = 'isn'$$);
@@ -77,26 +77,20 @@ SELECT count(*) FROM citus.pg_dist_object WHERE objid = (SELECT oid FROM pg_exte
 -- show that the ALTER EXTENSION command is propagated
 SELECT run_command_on_workers($$SELECT nspname from pg_namespace where oid=(SELECT extnamespace FROM pg_extension WHERE extname = 'isn')$$);
 
--- SET client_min_messages TO WARNING before executing a DROP EXTENSION statement
-SET client_min_messages TO WARNING;
 -- drop the extension finally
 DROP EXTENSION isn CASCADE;
--- restore client_min_messages after DROP EXTENSION
-RESET client_min_messages;
 
 -- now make sure that the reference tables depending on an extension can be succesfully created.
 -- we should also ensure that we replicate this reference table (and hence the extension)
 -- to new nodes after calling master_activate_node.
 
 -- now, first drop seg and existing objects before next test
-SET client_min_messages TO WARNING;
 DROP EXTENSION seg CASCADE;
 
 -- but as we have only 2 ports in postgresql tests, let's remove one of the nodes first
 -- before remove, first remove the existing relations (due to the other tests)
 
 DROP SCHEMA "extension'test" CASCADE;
-RESET client_min_messages;
 SELECT 1 from master_remove_node('localhost', :worker_2_port);
 
 -- then create the extension
@@ -189,15 +183,8 @@ set citus.enable_ddl_propagation to 'off';
 CREATE EXTENSION pg_buffercache;
 set citus.enable_ddl_propagation to 'on';
 
--- SET client_min_messages TO WARNING before executing a DROP EXTENSION statement
-SET client_min_messages TO WARNING;
 DROP EXTENSION pg_buffercache, isn CASCADE;
 SELECT count(*) FROM pg_extension WHERE extname IN ('pg_buffercache', 'isn');
--- restore client_min_messages after DROP EXTENSION
-RESET client_min_messages;
-
--- SET client_min_messages TO WARNING before executing a DROP EXTENSION statement
-SET client_min_messages TO WARNING;
 
 -- drop extension should just work
 DROP EXTENSION seg CASCADE;
@@ -205,9 +192,6 @@ DROP EXTENSION seg CASCADE;
 SELECT count(*) FROM citus.pg_dist_object WHERE objid = (SELECT oid FROM pg_extension WHERE extname = 'seg');
 SELECT run_command_on_workers($$SELECT count(*) FROM pg_extension WHERE extname = 'seg'$$);
 
--- restore client_min_messages after DROP EXTENSION
-RESET client_min_messages;
-
 -- make sure that the extension is not avaliable anymore as a distributed object
 SELECT count(*) FROM citus.pg_dist_object WHERE objid = (SELECT oid FROM pg_extension WHERE extname IN ('seg', 'isn'));
 
@@ -229,7 +213,6 @@ ROLLBACK;
 SELECT run_command_on_workers($$SELECT extversion FROM pg_extension WHERE extname = 'seg'$$);
 
 -- drop the schema and all the objects
-SET client_min_messages TO WARNING;
 DROP SCHEMA "extension'test" CASCADE;
 
 -- recreate for the next tests
@@ -238,8 +221,6 @@ CREATE SCHEMA "extension'test";
 -- use  a schema name with escape character
 SET search_path TO "extension'test";
 
-RESET client_min_messages;
-
 -- remove the node, we'll add back again
 SELECT 1 from master_remove_node('localhost', :worker_2_port);
 
@@ -269,5 +250,4 @@ SELECT count(*) FROM citus.pg_dist_object WHERE objid IN (SELECT oid FROM pg_ext
 SELECT run_command_on_workers($$SELECT count(*) FROM pg_extension WHERE extname IN ('seg', 'isn')$$);
 
 -- drop the schema and all the objects
-SET client_min_messages TO WARNING;
 DROP SCHEMA "extension'test" CASCADE;