Avoid multiple pg_dist_colocation records being created for reference tables

master_deactivate_node is updated to decrement the replication factor
Otherwise deactivation could have create_reference_table produce a second record

UpdateColocationGroupReplicationFactor is renamed UpdateColocationGroupReplicationFactorForReferenceTables
& the implementation looks up the record based on distributioncolumntype == InvalidOid, rather than by id
Otherwise the record's replication factor fails to be maintained when there are no reference tables

src/backend/distributed/master/master_metadata_utility.c

@@ -1253,62 +1253,64 @@ UpdateShardPlacementState(uint64 placementId, char shardState)
 
 
 /*
- * UpdateColocationGroupReplicationFactor finds colocation group record for given
+ * UpdateColocationGroupReplicationFactorForReferenceTables updates the
- * colocationId and updates its replication factor to given replicationFactor value.
+ * replicationFactor for the pg_dist_colocation record for reference tables.
  * Since we do not cache pg_dist_colocation table, we do not need to invalidate the
  * cache after updating replication factor.
  */
 void
-UpdateColocationGroupReplicationFactor(uint32 colocationId, int replicationFactor)
+UpdateColocationGroupReplicationFactorForReferenceTables(int replicationFactor)
 {
 	Relation pgDistColocation = NULL;
 	SysScanDesc scanDescriptor = NULL;
 	ScanKeyData scanKey[1];
 	int scanKeyCount = 1;
-	bool indexOK = true;
+	bool indexOK = false;
 	HeapTuple heapTuple = NULL;
-	HeapTuple newHeapTuple = NULL;
 	TupleDesc tupleDescriptor = NULL;
 
-	Datum values[Natts_pg_dist_colocation];
+	/*
-	bool isnull[Natts_pg_dist_colocation];
+	 * All reference tables share a colocation entry with:
-	bool replace[Natts_pg_dist_colocation];
+	 * shardCount = 1, replicationFactor = activeWorkerCount, distributiontype = InvalidOid
-
+	 * Find the record based on distributiontype = InvalidOid, as this uniquely identifies the group.
-	/* we first search for colocation group by its colocation id */
+	 */
 	pgDistColocation = heap_open(DistColocationRelationId(), RowExclusiveLock);
 	tupleDescriptor = RelationGetDescr(pgDistColocation);
-	ScanKeyInit(&scanKey[0], Anum_pg_dist_colocation_colocationid, BTEqualStrategyNumber,
+	ScanKeyInit(&scanKey[0], Anum_pg_dist_colocation_distributioncolumntype,
-				F_OIDEQ, ObjectIdGetDatum(colocationId));
+				BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(InvalidOid));
 
 	scanDescriptor = systable_beginscan(pgDistColocation,
-										DistColocationColocationidIndexId(), indexOK,
+										InvalidOid, indexOK,
 										NULL, scanKeyCount, scanKey);
 
 	heapTuple = systable_getnext(scanDescriptor);
-	if (!HeapTupleIsValid(heapTuple))
+	if (HeapTupleIsValid(heapTuple))
 	{
-		ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT),
+		/* after finding the group, update its replication factor */
-						errmsg("could not find valid entry for colocation group "
+		/* if it doesn't exist, no worries, it'll be created when needed */
-							   "%d", colocationId)));
+		HeapTuple newHeapTuple = NULL;
+		Datum values[Natts_pg_dist_colocation];
+		bool isnull[Natts_pg_dist_colocation];
+		bool replace[Natts_pg_dist_colocation];
+
+		memset(replace, false, sizeof(replace));
+		memset(isnull, false, sizeof(isnull));
+		memset(values, 0, sizeof(values));
+
+		values[Anum_pg_dist_colocation_replicationfactor - 1] = Int32GetDatum(
+			replicationFactor);
+		replace[Anum_pg_dist_colocation_replicationfactor - 1] = true;
+
+		newHeapTuple = heap_modify_tuple(heapTuple, tupleDescriptor, values, isnull,
+										 replace);
+
+		CatalogTupleUpdate(pgDistColocation, &newHeapTuple->t_self, newHeapTuple);
+
+		CommandCounterIncrement();
+
+		heap_freetuple(newHeapTuple);
 	}
 
-	/* after we find colocation group, we update it with new values */
-	memset(replace, false, sizeof(replace));
-	memset(isnull, false, sizeof(isnull));
-	memset(values, 0, sizeof(values));
-
-	values[Anum_pg_dist_colocation_replicationfactor - 1] = Int32GetDatum(
-		replicationFactor);
-	replace[Anum_pg_dist_colocation_replicationfactor - 1] = true;
-
-	newHeapTuple = heap_modify_tuple(heapTuple, tupleDescriptor, values, isnull, replace);
-
-	CatalogTupleUpdate(pgDistColocation, &newHeapTuple->t_self, newHeapTuple);
-
-	CommandCounterIncrement();
-
-	heap_freetuple(newHeapTuple);
-
 	systable_endscan(scanDescriptor);
 	heap_close(pgDistColocation, NoLock);
 }

src/backend/distributed/utils/node_metadata.c

@@ -284,6 +284,12 @@ master_disable_node(PG_FUNCTION_ARGS)
 
 	SetNodeState(nodeName, nodePort, isActive);
 
+	if (WorkerNodeIsPrimary(workerNode))
+	{
+		UpdateColocationGroupReplicationFactorForReferenceTables(
+			ActivePrimaryNodeCount());
+	}
+
 	PG_RETURN_VOID();
 }
 
@@ -875,7 +881,6 @@ RemoveNodeFromCluster(char *nodeName, int32 nodePort)
 	const bool onlyConsiderActivePlacements = false;
 	char *nodeDeleteCommand = NULL;
 	WorkerNode *workerNode = NULL;
-	List *referenceTableList = NIL;
 	uint32 deletedNodeId = INVALID_PLACEMENT_ID;
 
 	EnsureCoordinator();
@@ -908,25 +913,10 @@ RemoveNodeFromCluster(char *nodeName, int32 nodePort)
 
 	DeleteNodeRow(nodeName, nodePort);
 
-	/*
-	 * After deleting reference tables placements, we will update replication factor
-	 * column for colocation group of reference tables so that replication factor will
-	 * be equal to worker count.
-	 */
 	if (WorkerNodeIsPrimary(workerNode))
 	{
-		referenceTableList = ReferenceTableOidList();
+		UpdateColocationGroupReplicationFactorForReferenceTables(
-		if (list_length(referenceTableList) != 0)
+			ActivePrimaryNodeCount());
-		{
-			Oid firstReferenceTableId = linitial_oid(referenceTableList);
-			uint32 referenceTableColocationId = TableColocationId(firstReferenceTableId);
-
-			List *workerNodeList = ActivePrimaryNodeList();
-			int workerCount = list_length(workerNodeList);
-
-			UpdateColocationGroupReplicationFactor(referenceTableColocationId,
-												   workerCount);
-		}
 	}
 
 	nodeDeleteCommand = NodeDeleteCommand(deletedNodeId);

src/backend/distributed/utils/reference_table_utils.c

@@ -129,60 +129,52 @@ void
 ReplicateAllReferenceTablesToNode(char *nodeName, int nodePort)
 {
 	List *referenceTableList = ReferenceTableOidList();
-	List *referenceShardIntervalList = NIL;
 	ListCell *referenceTableCell = NULL;
-	ListCell *referenceShardIntervalCell = NULL;
+	uint32 workerCount = ActivePrimaryNodeCount();
-	List *workerNodeList = ActivePrimaryNodeList();
-	uint32 workerCount = 0;
-	Oid firstReferenceTableId = InvalidOid;
-	uint32 referenceTableColocationId = INVALID_COLOCATION_ID;
 
-	/* if there is no reference table, we do not need to do anything */
+	/* if there is no reference table, we do not need to replicate anything */
-	if (list_length(referenceTableList) == 0)
+	if (list_length(referenceTableList) > 0)
 	{
-		return;
+		List *referenceShardIntervalList = NIL;
+		ListCell *referenceShardIntervalCell = NULL;
+
+		/*
+		 * We sort the reference table list to prevent deadlocks in concurrent
+		 * ReplicateAllReferenceTablesToAllNodes calls.
+		 */
+		referenceTableList = SortList(referenceTableList, CompareOids);
+		foreach(referenceTableCell, referenceTableList)
+		{
+			Oid referenceTableId = lfirst_oid(referenceTableCell);
+			List *shardIntervalList = LoadShardIntervalList(referenceTableId);
+			ShardInterval *shardInterval = (ShardInterval *) linitial(shardIntervalList);
+
+			referenceShardIntervalList = lappend(referenceShardIntervalList,
+												 shardInterval);
+		}
+
+		if (ClusterHasKnownMetadataWorkers())
+		{
+			BlockWritesToShardList(referenceShardIntervalList);
+		}
+
+		foreach(referenceShardIntervalCell, referenceShardIntervalList)
+		{
+			ShardInterval *shardInterval = (ShardInterval *) lfirst(
+				referenceShardIntervalCell);
+			uint64 shardId = shardInterval->shardId;
+
+			LockShardDistributionMetadata(shardId, ExclusiveLock);
+
+			ReplicateShardToNode(shardInterval, nodeName, nodePort);
+		}
 	}
 
 	/*
-	 * We sort the reference table list to prevent deadlocks in concurrent
+	 * Update replication factor column for colocation group of reference tables
-	 * ReplicateAllReferenceTablesToAllNodes calls.
+	 * so that worker count will be equal to replication factor again.
 	 */
-	referenceTableList = SortList(referenceTableList, CompareOids);
+	UpdateColocationGroupReplicationFactorForReferenceTables(workerCount);
-	foreach(referenceTableCell, referenceTableList)
-	{
-		Oid referenceTableId = lfirst_oid(referenceTableCell);
-		List *shardIntervalList = LoadShardIntervalList(referenceTableId);
-		ShardInterval *shardInterval = (ShardInterval *) linitial(shardIntervalList);
-
-		referenceShardIntervalList = lappend(referenceShardIntervalList,
-											 shardInterval);
-	}
-
-	if (ClusterHasKnownMetadataWorkers())
-	{
-		BlockWritesToShardList(referenceShardIntervalList);
-	}
-
-	foreach(referenceShardIntervalCell, referenceShardIntervalList)
-	{
-		ShardInterval *shardInterval = (ShardInterval *) lfirst(
-			referenceShardIntervalCell);
-		uint64 shardId = shardInterval->shardId;
-
-		LockShardDistributionMetadata(shardId, ExclusiveLock);
-
-		ReplicateShardToNode(shardInterval, nodeName, nodePort);
-	}
-
-	/*
-	 * After replicating reference tables, we will update replication factor column for
-	 * colocation group of reference tables so that worker count will be equal to
-	 * replication factor again.
-	 */
-	workerCount = list_length(workerNodeList);
-	firstReferenceTableId = linitial_oid(referenceTableList);
-	referenceTableColocationId = TableColocationId(firstReferenceTableId);
-	UpdateColocationGroupReplicationFactor(referenceTableColocationId, workerCount);
 }
 
 

src/include/distributed/master_metadata_utility.h

@@ -145,8 +145,8 @@ extern void DeletePartitionRow(Oid distributedRelationId);
 extern void DeleteShardRow(uint64 shardId);
 extern void UpdateShardPlacementState(uint64 placementId, char shardState);
 extern void DeleteShardPlacementRow(uint64 placementId);
-extern void UpdateColocationGroupReplicationFactor(uint32 colocationId,
+extern void UpdateColocationGroupReplicationFactorForReferenceTables(int
-												   int replicationFactor);
+																	 replicationFactor);
 extern void CreateDistributedTable(Oid relationId, Var *distributionColumn,
 								   char distributionMethod, char *colocateWithTableName,
 								   bool viaDeprecatedAPI);

src/test/regress/expected/multi_metadata_sync.out

@@ -1251,6 +1251,14 @@ SELECT create_reference_table('mx_ref');
  
 (1 row)
 
+-- make sure that adding/removing nodes doesn't cause
+-- multiple colocation entries for reference tables
+SELECT count(*) FROM pg_dist_colocation WHERE distributioncolumntype = 0;
+ count 
+-------
+     1
+(1 row)
+
 \dt mx_ref
          List of relations
  Schema |  Name  | Type  |  Owner   

src/test/regress/expected/multi_remove_node_reference_table.out

@@ -141,7 +141,7 @@ WHERE colocationid IN
      WHERE logicalrelid = 'remove_node_reference_table'::regclass);
  colocationid | shardcount | replicationfactor | distributioncolumntype 
 --------------+------------+-------------------+------------------------
-      1370005 |          1 |                 2 |                      0
+        10004 |          1 |                 2 |                      0
 (1 row)
 
      
@@ -196,7 +196,7 @@ WHERE colocationid IN
      WHERE logicalrelid = 'remove_node_reference_table'::regclass);
  colocationid | shardcount | replicationfactor | distributioncolumntype 
 --------------+------------+-------------------+------------------------
-      1370005 |          1 |                 1 |                      0
+        10004 |          1 |                 1 |                      0
 (1 row)
 
 \c - - - :worker_1_port
@@ -277,7 +277,7 @@ WHERE colocationid IN
      WHERE logicalrelid = 'remove_node_reference_table'::regclass);
  colocationid | shardcount | replicationfactor | distributioncolumntype 
 --------------+------------+-------------------+------------------------
-      1370005 |          1 |                 2 |                      0
+        10004 |          1 |                 2 |                      0
 (1 row)
 
      
@@ -335,7 +335,7 @@ WHERE colocationid IN
      WHERE logicalrelid = 'remove_node_reference_table'::regclass);
  colocationid | shardcount | replicationfactor | distributioncolumntype 
 --------------+------------+-------------------+------------------------
-      1370005 |          1 |                 2 |                      0
+        10004 |          1 |                 2 |                      0
 (1 row)
 
 \c - - - :worker_1_port
@@ -385,7 +385,7 @@ WHERE colocationid IN
      WHERE logicalrelid = 'remove_node_reference_table'::regclass);
  colocationid | shardcount | replicationfactor | distributioncolumntype 
 --------------+------------+-------------------+------------------------
-      1370005 |          1 |                 2 |                      0
+        10004 |          1 |                 2 |                      0
 (1 row)
 
      
@@ -442,7 +442,7 @@ WHERE colocationid IN
      WHERE logicalrelid = 'remove_node_reference_table'::regclass);
  colocationid | shardcount | replicationfactor | distributioncolumntype 
 --------------+------------+-------------------+------------------------
-      1370005 |          1 |                 1 |                      0
+        10004 |          1 |                 1 |                      0
 (1 row)
 
      
@@ -500,7 +500,7 @@ WHERE colocationid IN
      WHERE logicalrelid = 'remove_node_reference_table'::regclass);
  colocationid | shardcount | replicationfactor | distributioncolumntype 
 --------------+------------+-------------------+------------------------
-      1370005 |          1 |                 2 |                      0
+        10004 |          1 |                 2 |                      0
 (1 row)
 
 \c - - - :worker_1_port
@@ -558,7 +558,7 @@ WHERE colocationid IN
      WHERE logicalrelid = 'remove_node_reference_table'::regclass);
  colocationid | shardcount | replicationfactor | distributioncolumntype 
 --------------+------------+-------------------+------------------------
-      1370005 |          1 |                 1 |                      0
+        10004 |          1 |                 1 |                      0
 (1 row)
 
 --verify the data is inserted
@@ -629,7 +629,7 @@ WHERE colocationid IN
      WHERE logicalrelid = 'remove_node_reference_table'::regclass);
  colocationid | shardcount | replicationfactor | distributioncolumntype 
 --------------+------------+-------------------+------------------------
-      1370005 |          1 |                 2 |                      0
+        10004 |          1 |                 2 |                      0
 (1 row)
 
 \c - - - :worker_1_port
@@ -686,7 +686,7 @@ WHERE colocationid IN
      WHERE logicalrelid = 'remove_node_reference_table'::regclass);
  colocationid | shardcount | replicationfactor | distributioncolumntype 
 --------------+------------+-------------------+------------------------
-      1370005 |          1 |                 1 |                      0
+        10004 |          1 |                 1 |                      0
 (1 row)
 
      
@@ -753,7 +753,7 @@ WHERE colocationid IN
      WHERE logicalrelid = 'remove_node_reference_table'::regclass);
  colocationid | shardcount | replicationfactor | distributioncolumntype 
 --------------+------------+-------------------+------------------------
-      1370005 |          1 |                 2 |                      0
+        10004 |          1 |                 2 |                      0
 (1 row)
 
 BEGIN;
@@ -840,7 +840,7 @@ WHERE colocationid IN
      WHERE logicalrelid = 'remove_node_reference_table_schema.table1'::regclass);
  colocationid | shardcount | replicationfactor | distributioncolumntype 
 --------------+------------+-------------------+------------------------
-      1370004 |          1 |                 2 |                      0
+        10004 |          1 |                 2 |                      0
 (1 row)
 
      
@@ -897,7 +897,7 @@ WHERE colocationid IN
      WHERE logicalrelid = 'remove_node_reference_table_schema.table1'::regclass);
  colocationid | shardcount | replicationfactor | distributioncolumntype 
 --------------+------------+-------------------+------------------------
-      1370004 |          1 |                 1 |                      0
+        10004 |          1 |                 1 |                      0
 (1 row)
 
 \c - - - :worker_1_port
@@ -959,7 +959,7 @@ WHERE colocationid IN
      WHERE logicalrelid = 'remove_node_reference_table'::regclass);
  colocationid | shardcount | replicationfactor | distributioncolumntype 
 --------------+------------+-------------------+------------------------
-      1370004 |          1 |                 2 |                      0
+        10004 |          1 |                 2 |                      0
 (1 row)
 
 \c - - - :worker_1_port
@@ -1016,7 +1016,7 @@ WHERE colocationid IN
      WHERE logicalrelid = 'remove_node_reference_table'::regclass);
  colocationid | shardcount | replicationfactor | distributioncolumntype 
 --------------+------------+-------------------+------------------------
-      1370004 |          1 |                 2 |                      0
+        10004 |          1 |                 1 |                      0
 (1 row)
 
 \c - - - :worker_1_port

src/test/regress/expected/multi_replicate_reference_table.out

@@ -117,7 +117,7 @@ WHERE colocationid IN
      WHERE logicalrelid = 'replicate_reference_table_valid'::regclass);
  colocationid | shardcount | replicationfactor | distributioncolumntype 
 --------------+------------+-------------------+------------------------
-      1370000 |          1 |                 1 |                      0
+        10004 |          1 |                 1 |                      0
 (1 row)
 
 SELECT 1 FROM master_add_node('localhost', :worker_2_port);
@@ -147,7 +147,7 @@ WHERE colocationid IN
      WHERE logicalrelid = 'replicate_reference_table_valid'::regclass);
  colocationid | shardcount | replicationfactor | distributioncolumntype 
 --------------+------------+-------------------+------------------------
-      1370000 |          1 |                 2 |                      0
+        10004 |          1 |                 2 |                      0
 (1 row)
 
 -- test add same node twice
@@ -171,7 +171,7 @@ WHERE colocationid IN
      WHERE logicalrelid = 'replicate_reference_table_valid'::regclass);
  colocationid | shardcount | replicationfactor | distributioncolumntype 
 --------------+------------+-------------------+------------------------
-      1370000 |          1 |                 2 |                      0
+        10004 |          1 |                 2 |                      0
 (1 row)
 
 SELECT 1 FROM master_add_node('localhost', :worker_2_port);
@@ -200,7 +200,7 @@ WHERE colocationid IN
      WHERE logicalrelid = 'replicate_reference_table_valid'::regclass);
  colocationid | shardcount | replicationfactor | distributioncolumntype 
 --------------+------------+-------------------+------------------------
-      1370000 |          1 |                 2 |                      0
+        10004 |          1 |                 2 |                      0
 (1 row)
 
 DROP TABLE replicate_reference_table_valid;
@@ -237,7 +237,7 @@ WHERE colocationid IN
      WHERE logicalrelid = 'replicate_reference_table_rollback'::regclass);
  colocationid | shardcount | replicationfactor | distributioncolumntype 
 --------------+------------+-------------------+------------------------
-      1370001 |          1 |                 1 |                      0
+        10004 |          1 |                 1 |                      0
 (1 row)
 
 BEGIN;
@@ -268,7 +268,7 @@ WHERE colocationid IN
      WHERE logicalrelid = 'replicate_reference_table_rollback'::regclass);
  colocationid | shardcount | replicationfactor | distributioncolumntype 
 --------------+------------+-------------------+------------------------
-      1370001 |          1 |                 1 |                      0
+        10004 |          1 |                 1 |                      0
 (1 row)
 
 DROP TABLE replicate_reference_table_rollback;
@@ -299,7 +299,7 @@ WHERE colocationid IN
      WHERE logicalrelid = 'replicate_reference_table_commit'::regclass);
  colocationid | shardcount | replicationfactor | distributioncolumntype 
 --------------+------------+-------------------+------------------------
-      1370001 |          1 |                 1 |                      0
+        10004 |          1 |                 1 |                      0
 (1 row)
 
 BEGIN;
@@ -331,7 +331,7 @@ WHERE colocationid IN
      WHERE logicalrelid = 'replicate_reference_table_commit'::regclass);
  colocationid | shardcount | replicationfactor | distributioncolumntype 
 --------------+------------+-------------------+------------------------
-      1370001 |          1 |                 2 |                      0
+        10004 |          1 |                 2 |                      0
 (1 row)
 
 DROP TABLE replicate_reference_table_commit;
@@ -381,7 +381,7 @@ WHERE colocationid IN
      WHERE logicalrelid = 'replicate_reference_table_reference_one'::regclass);
  colocationid | shardcount | replicationfactor | distributioncolumntype 
 --------------+------------+-------------------+------------------------
-      1370002 |          1 |                 1 |                      0
+        10004 |          1 |                 1 |                      0
 (1 row)
 
 SELECT
@@ -393,7 +393,7 @@ WHERE
 ORDER BY logicalrelid;
               logicalrelid               | partmethod | colocationid | repmodel 
 -----------------------------------------+------------+--------------+----------
- replicate_reference_table_reference_one | n          |      1370002 | t
+ replicate_reference_table_reference_one | n          |        10004 | t
  replicate_reference_table_hash          | h          |      1360004 | c
 (2 rows)
 
@@ -443,7 +443,7 @@ WHERE colocationid IN
      WHERE logicalrelid = 'replicate_reference_table_reference_one'::regclass);
  colocationid | shardcount | replicationfactor | distributioncolumntype 
 --------------+------------+-------------------+------------------------
-      1370002 |          1 |                 2 |                      0
+        10004 |          1 |                 2 |                      0
 (1 row)
 
 SELECT
@@ -456,9 +456,9 @@ ORDER BY
 	logicalrelid;
               logicalrelid               | partmethod | colocationid | repmodel 
 -----------------------------------------+------------+--------------+----------
- replicate_reference_table_reference_one | n          |      1370002 | t
+ replicate_reference_table_reference_one | n          |        10004 | t
- replicate_reference_table_hash          | n          |      1370002 | t
+ replicate_reference_table_hash          | n          |        10004 | t
- replicate_reference_table_reference_two | n          |      1370002 | t
+ replicate_reference_table_reference_two | n          |        10004 | t
 (3 rows)
 
 DROP TABLE replicate_reference_table_reference_one;
@@ -542,7 +542,7 @@ WHERE colocationid IN
      WHERE logicalrelid = 'replicate_reference_table_drop'::regclass);
  colocationid | shardcount | replicationfactor | distributioncolumntype 
 --------------+------------+-------------------+------------------------
-      1370003 |          1 |                 1 |                      0
+        10004 |          1 |                 1 |                      0
 (1 row)
 
 BEGIN;
@@ -605,7 +605,7 @@ WHERE colocationid IN
      WHERE logicalrelid = 'replicate_reference_table_schema.table1'::regclass);
  colocationid | shardcount | replicationfactor | distributioncolumntype 
 --------------+------------+-------------------+------------------------
-      1370004 |          1 |                 1 |                      0
+        10004 |          1 |                 1 |                      0
 (1 row)
 
 SELECT 1 FROM master_add_node('localhost', :worker_2_port);
@@ -635,7 +635,7 @@ WHERE colocationid IN
      WHERE logicalrelid = 'replicate_reference_table_schema.table1'::regclass);
  colocationid | shardcount | replicationfactor | distributioncolumntype 
 --------------+------------+-------------------+------------------------
-      1370004 |          1 |                 2 |                      0
+        10004 |          1 |                 2 |                      0
 (1 row)
 
 DROP TABLE replicate_reference_table_schema.table1;

src/test/regress/expected/multi_upgrade_reference_table.out

@@ -180,7 +180,7 @@ WHERE
     logicalrelid = 'upgrade_reference_table_append'::regclass;
  partmethod | partkeyisnull | colocationid | repmodel 
 ------------+---------------+--------------+----------
- n          | t             |        10005 | t
+ n          | t             |        10004 | t
 (1 row)
 
 SELECT
@@ -202,7 +202,7 @@ WHERE colocationid IN
      WHERE logicalrelid = 'upgrade_reference_table_append'::regclass);
  colocationid | shardcount | replicationfactor | distributioncolumntype 
 --------------+------------+-------------------+------------------------
-        10005 |          1 |                 2 |                      0
+        10004 |          1 |                 2 |                      0
 (1 row)
 
 SELECT
@@ -293,7 +293,7 @@ WHERE
     logicalrelid = 'upgrade_reference_table_one_worker'::regclass;
  partmethod | partkeyisnull | colocationid | repmodel 
 ------------+---------------+--------------+----------
- n          | t             |        10005 | t
+ n          | t             |        10004 | t
 (1 row)
 
 SELECT
@@ -315,7 +315,7 @@ WHERE colocationid IN
      WHERE logicalrelid = 'upgrade_reference_table_one_worker'::regclass);
  colocationid | shardcount | replicationfactor | distributioncolumntype 
 --------------+------------+-------------------+------------------------
-        10005 |          1 |                 2 |                      0
+        10004 |          1 |                 2 |                      0
 (1 row)
 
 SELECT
@@ -409,7 +409,7 @@ WHERE
     logicalrelid = 'upgrade_reference_table_one_unhealthy'::regclass;
  partmethod | partkeyisnull | colocationid | repmodel 
 ------------+---------------+--------------+----------
- n          | t             |        10005 | t
+ n          | t             |        10004 | t
 (1 row)
 
 SELECT
@@ -431,7 +431,7 @@ WHERE colocationid IN
      WHERE logicalrelid = 'upgrade_reference_table_one_unhealthy'::regclass);
  colocationid | shardcount | replicationfactor | distributioncolumntype 
 --------------+------------+-------------------+------------------------
-        10005 |          1 |                 2 |                      0
+        10004 |          1 |                 2 |                      0
 (1 row)
 
 SELECT
@@ -523,7 +523,7 @@ WHERE
     logicalrelid = 'upgrade_reference_table_both_healthy'::regclass;
  partmethod | partkeyisnull | colocationid | repmodel 
 ------------+---------------+--------------+----------
- n          | t             |        10005 | t
+ n          | t             |        10004 | t
 (1 row)
 
 SELECT
@@ -545,7 +545,7 @@ WHERE colocationid IN
      WHERE logicalrelid = 'upgrade_reference_table_both_healthy'::regclass);
  colocationid | shardcount | replicationfactor | distributioncolumntype 
 --------------+------------+-------------------+------------------------
-        10005 |          1 |                 2 |                      0
+        10004 |          1 |                 2 |                      0
 (1 row)
 
 SELECT
@@ -752,7 +752,7 @@ WHERE
     logicalrelid = 'upgrade_reference_table_transaction_commit'::regclass;
  partmethod | partkeyisnull | colocationid | repmodel 
 ------------+---------------+--------------+----------
- n          | t             |        10005 | t
+ n          | t             |        10004 | t
 (1 row)
 
 SELECT
@@ -774,7 +774,7 @@ WHERE colocationid IN
      WHERE logicalrelid = 'upgrade_reference_table_transaction_commit'::regclass);
  colocationid | shardcount | replicationfactor | distributioncolumntype 
 --------------+------------+-------------------+------------------------
-        10005 |          1 |                 2 |                      0
+        10004 |          1 |                 2 |                      0
 (1 row)
 
 SELECT
@@ -1001,7 +1001,7 @@ WHERE
     logicalrelid = 'upgrade_reference_table_mx'::regclass;
  partmethod | partkeyisnull | colocationid | repmodel 
 ------------+---------------+--------------+----------
- n          | t             |        10005 | t
+ n          | t             |        10004 | t
 (1 row)
 
 SELECT
@@ -1023,7 +1023,7 @@ WHERE colocationid IN
      WHERE logicalrelid = 'upgrade_reference_table_mx'::regclass);
  colocationid | shardcount | replicationfactor | distributioncolumntype 
 --------------+------------+-------------------+------------------------
-        10005 |          1 |                 2 |                      0
+        10004 |          1 |                 2 |                      0
 (1 row)
 
 SELECT
@@ -1051,7 +1051,7 @@ WHERE
     logicalrelid = 'upgrade_reference_table_mx'::regclass;
  partmethod | partkeyisnull | colocationid | repmodel 
 ------------+---------------+--------------+----------
- n          | t             |        10005 | t
+ n          | t             |        10004 | t
 (1 row)
 
 SELECT

src/test/regress/sql/multi_metadata_sync.sql

@@ -569,6 +569,11 @@ DROP USER mx_user;
 \c - - - :master_port
 CREATE TABLE mx_ref (col_1 int, col_2 text);
 SELECT create_reference_table('mx_ref');
+
+-- make sure that adding/removing nodes doesn't cause
+-- multiple colocation entries for reference tables
+SELECT count(*) FROM pg_dist_colocation WHERE distributioncolumntype = 0;
+
 \dt mx_ref
 
 \c - - - :worker_1_port