prevent master_remove_node(coordinator) when coordinator has reference table replicas involved in a fkey const. with a local table

src/backend/distributed/commands/foreign_constraint.c

@@ -40,6 +40,7 @@ static void ForeignConstraintFindDistKeys(HeapTuple pgConstraintTuple,
 										  Var *referencedDistColumn,
 										  int *referencingAttrIndex,
 										  int *referencedAttrIndex);
+static Oid GetCoordinatorLocalTableHavingFKeyWithReferenceTable(Oid referenceTableOid);
 
 /*
  * ConstraintIsAForeignKeyToReferenceTable checks if the given constraint is a
@@ -597,6 +598,116 @@ ErrorIfUnsupportedAlterAddDropFKeyBetweenReferecenceAndLocalTable(Oid referencin
 }
 
 
+/*
+ * ErrorIfCoordinatorHasLocalTableReferencingReferenceTable errors out if we
+ * if coordinator has reference table replica for given reference table and if
+ * it is involved in a foreign key constraint with a coordinator local table.
+ */
+void
+ErrorIfCoordinatorHasLocalTableReferencingReferenceTable(Oid referenceTableOid)
+{
+	Assert(OidIsValid(referenceTableOid));
+
+	Oid localTableOid = GetCoordinatorLocalTableHavingFKeyWithReferenceTable(
+		referenceTableOid);
+
+	/*
+	 * There is a local table involved in a foreign key constraint with
+	 * reference table with referenceTableOid
+	 */
+	if (OidIsValid(localTableOid))
+	{
+		const char *localTableName = get_rel_name(localTableOid);
+		const char *referenceTableName = get_rel_name(referenceTableOid);
+
+		Assert(localTableName != NULL && referenceTableName != NULL);
+
+		ereport(ERROR, (errcode(ERRCODE_DEPENDENT_OBJECTS_STILL_EXIST),
+						errmsg(
+							"cannot remove reference table placements from coordinator"),
+						errdetail(
+							"Local table \"%s\" is involved in a foreign key constraint with "
+							"reference table \"%s\", which has replica(s) in coordinator",
+							localTableName, referenceTableName),
+						errhint(
+							"DROP foreign key constraint between them or drop referenced "
+							"table with DROP ... CASCADE.")));
+	}
+}
+
+
+/*
+ * GetCoordinatorLocalTableHavingFKeyWithReferenceTable returns OID of the
+ * local table that is involved in a foreign key constraint with the reference
+ * table with referenceTableOid.
+ * It does that by scanning pg_constraint for foreign key constraints.
+ *
+ * If there does not exist such a foreign key constraint, returns InvalidOid.
+ * If there exists more than one such a foreign key constraint, we return the
+ * first local table we encounter while scanning pg_constraint
+ */
+static Oid
+GetCoordinatorLocalTableHavingFKeyWithReferenceTable(Oid referenceTableOid)
+{
+	Oid referenceTableShardOid = GetOnlyShardOidOfReferenceTable(referenceTableOid);
+
+	/* early exit if we could not hit to a healthy shard relation */
+	if (!OidIsValid(referenceTableShardOid))
+	{
+		return InvalidOid;
+	}
+
+	int scanKeyCount = 1;
+	ScanKeyData scanKey[1];
+
+	Relation pgConstraint = heap_open(ConstraintRelationId, AccessShareLock);
+	ScanKeyInit(&scanKey[0], Anum_pg_constraint_contype, BTEqualStrategyNumber, F_CHAREQ,
+				CharGetDatum(CONSTRAINT_FOREIGN));
+	SysScanDesc scanDescriptor =
+		systable_beginscan(pgConstraint, InvalidOid, false, NULL, scanKeyCount, scanKey);
+
+	HeapTuple heapTuple = systable_getnext(scanDescriptor);
+
+	while (HeapTupleIsValid(heapTuple))
+	{
+		Form_pg_constraint constraintForm = (Form_pg_constraint) GETSTRUCT(heapTuple);
+
+		if (constraintForm->confrelid == referenceTableShardOid)
+		{
+			Oid referencingTableId = constraintForm->conrelid;
+
+			if (!IsCitusTable(referencingTableId))
+			{
+				return referencingTableId;
+			}
+		}
+		else if (constraintForm->conrelid == referenceTableShardOid)
+		{
+			Oid referencedTableId = constraintForm->confrelid;
+
+			if (!IsCitusTable(referencedTableId))
+			{
+				return referencedTableId;
+			}
+		}
+
+		/*
+		 * If this is not such a relation from/to the given relation, we should
+		 * simply skip.
+		 */
+		heapTuple = systable_getnext(scanDescriptor);
+		continue;
+	}
+
+	/* clean up scan and close system catalog */
+
+	systable_endscan(scanDescriptor);
+	heap_close(pgConstraint, AccessShareLock);
+
+	return InvalidOid;
+}
+
+
 /*
  * ColumnAppearsInForeignKeyToReferenceTable checks if there is a foreign key
  * constraint from/to a reference table on the given column. We iterate

src/backend/distributed/master/master_metadata_utility.c

@@ -511,7 +511,7 @@ LoadShardIntervalList(Oid relationId)
 Oid
 GetOnlyShardOidOfReferenceTable(Oid referenceTableOid)
 {
-	DistTableCacheEntry *cacheEntry = DistributedTableCacheEntry(referenceTableOid);
+	const CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(referenceTableOid);
 
 	/* assert that it is a "valid" "reference table" */
 	Assert(cacheEntry != NULL && cacheEntry->shardIntervalArrayLength == 1);

src/backend/distributed/utils/reference_table_utils.c

@@ -15,6 +15,7 @@
 #include "access/heapam.h"
 #include "access/htup_details.h"
 #include "access/genam.h"
+#include "catalog/pg_constraint.h"
 #include "distributed/colocation_utils.h"
 #include "distributed/commands.h"
 #include "distributed/listutils.h"
@@ -391,15 +392,18 @@ CreateReferenceTableColocationId()
 
 
 /*
- * DeleteAllReferenceTablePlacementsFromNodeGroup function iterates over list of reference
- * tables and deletes all reference table placements from pg_dist_placement table
- * for given group.
+ * DeleteAllReferenceTablePlacementsFromNodeGroup function iterates over list of
+ * reference tables and deletes all reference table placements from pg_dist_placement
+ * table for given group.
+ *
+ * Error out if we are to remove coordinator if it has reference table replicas and
+ * if any of those replicas is involved in a foreign constraint with a coordinator
+ * local table.
  */
 void
 DeleteAllReferenceTablePlacementsFromNodeGroup(int32 groupId)
 {
 	List *referenceTableList = ReferenceTableOidList();
-	List *referenceShardIntervalList = NIL;
 
 	/* if there are no reference tables, we do not need to do anything */
 	if (list_length(referenceTableList) == 0)
@@ -414,7 +418,8 @@ DeleteAllReferenceTablePlacementsFromNodeGroup(int32 groupId)
 	referenceTableList = SortList(referenceTableList, CompareOids);
 	if (ClusterHasKnownMetadataWorkers())
 	{
-		referenceShardIntervalList = GetSortedReferenceShardIntervals(referenceTableList);
+		List *referenceShardIntervalList = GetSortedReferenceShardIntervals(
+			referenceTableList);
 
 		BlockWritesToShardList(referenceShardIntervalList);
 	}
@@ -432,12 +437,24 @@ DeleteAllReferenceTablePlacementsFromNodeGroup(int32 groupId)
 		}
 
 		GroupShardPlacement *placement = (GroupShardPlacement *) linitial(placements);
+		uint64 referenceTableShardId = placement->shardId;
 
-		LockShardDistributionMetadata(placement->shardId, ExclusiveLock);
+		LockShardDistributionMetadata(referenceTableShardId, ExclusiveLock);
+
+		/*
+		 * Error out if we are to remove coordinator and if reference table with
+		 * referenceTableId is involved in a foreign constraint with a coordinator
+		 * local table.
+		 */
+		if (groupId == COORDINATOR_GROUP_ID)
+		{
+			ErrorIfCoordinatorHasLocalTableReferencingReferenceTable(referenceTableId);
+		}
 
 		DeleteShardPlacementRow(placement->placementId);
 
 		resetStringInfo(deletePlacementCommand);
+
 		appendStringInfo(deletePlacementCommand,
 						 "DELETE FROM pg_dist_placement WHERE placementid = "
 						 UINT64_FORMAT,

src/include/distributed/commands.h

@@ -107,6 +107,8 @@ extern void ErrorIfUnsupportedAlterAddDropFKeyBetweenReferecenceAndLocalTable(Oi
 																			  alterTableType,
 																			  Constraint *
 																			  constraint);
+extern void ErrorIfCoordinatorHasLocalTableReferencingReferenceTable(Oid
+																	 referenceTableOid);
 extern bool ColumnAppearsInForeignKeyToReferenceTable(char *columnName, Oid
 													  relationId);
 extern List * GetTableForeignConstraintCommands(Oid relationId);