From 35eac2318d680841ed503bf89720b02b1bb48a0e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?mehmet=20furkan=20=C5=9Fahin?= <sahinffurkan@gmail.com>
Date: Fri, 29 Jun 2018 17:55:07 +0300
Subject: [PATCH] lock referenced reference table metadata is added

For certain operations in enterprise, we need to lock the
referenced reference table shard distribution metadata
---
 .../distributed/master/master_repair_shards.c | 11 +++-
 src/backend/distributed/utils/resource_lock.c | 56 +++++++++++++++++++
 src/include/distributed/resource_lock.h       |  4 ++
 3 files changed, 70 insertions(+), 1 deletion(-)

diff --git a/src/backend/distributed/master/master_repair_shards.c b/src/backend/distributed/master/master_repair_shards.c
index c14cfc523..1f84a0889 100644
--- a/src/backend/distributed/master/master_repair_shards.c
+++ b/src/backend/distributed/master/master_repair_shards.c
@@ -188,6 +188,16 @@ RepairShardPlacement(int64 shardId, char *sourceNodeName, int32 sourceNodePort,
 								  "not supported.", relationName)));
 	}
 
+	/*
+	 * We take a lock on the referenced table if there is a foreign constraint
+	 * during the copy procedure. If we do not block DMLs on the referenced
+	 * table, we cannot avoid the inconsistency between the two copies of the
+	 * data. Currently, we do not support replication factor > 1 on the tables
+	 * with foreign constraints, so this command will fail for this case anyway.
+	 * However, it is taken as a precaution in case we support it one day.
+	 */
+	LockReferencedReferenceShardDistributionMetadata(shardId, ExclusiveLock);
+
 	/*
 	 * We plan to move the placement to the healthy state, so we need to grab a shard
 	 * metadata lock (in exclusive mode).
@@ -394,7 +404,6 @@ CopyShardForeignConstraintCommandList(ShardInterval *shardInterval)
 	return copyShardForeignConstraintCommandList;
 }
 
-
 /*
  * ConstuctQualifiedShardName creates the fully qualified name string of the
  * given shard in <schema>.<table_name>_<shard_id> format.
diff --git a/src/backend/distributed/utils/resource_lock.c b/src/backend/distributed/utils/resource_lock.c
index 555a89261..06fd61a99 100644
--- a/src/backend/distributed/utils/resource_lock.c
+++ b/src/backend/distributed/utils/resource_lock.c
@@ -25,6 +25,7 @@
 #include "distributed/distributed_planner.h"
 #include "distributed/multi_router_executor.h"
 #include "distributed/relay_utility.h"
+#include "distributed/reference_table_utils.h"
 #include "distributed/resource_lock.h"
 #include "distributed/shardinterval_utils.h"
 #include "distributed/worker_protocol.h"
@@ -33,6 +34,7 @@
 
 /* local function forward declarations */
 static LOCKMODE IntToLockMode(int mode);
+static List * GetSortedReferenceShardIntervals(List *relationList);
 
 
 /* exports for SQL callable functions */
@@ -164,6 +166,60 @@ LockShardDistributionMetadata(int64 shardId, LOCKMODE lockMode)
 }
 
 
+/*
+ * LockReferencedReferenceShardDistributionMetadata acquires the given lock
+ * on the reference tables which has a foreign key from the given relation.
+ */
+void
+LockReferencedReferenceShardDistributionMetadata(uint64 shardId, LOCKMODE lock)
+{
+	ListCell *shardIntervalCell = NULL;
+	Oid relationId = RelationIdForShard(shardId);
+
+	DistTableCacheEntry *cacheEntry = DistributedTableCacheEntry(relationId);
+	List *referencedRelationList = cacheEntry->referencedRelationsViaForeignKey;
+
+	List *shardIntervalList = GetSortedReferenceShardIntervals(referencedRelationList);
+	foreach(shardIntervalCell, shardIntervalList)
+	{
+		ShardInterval *shardInterval = (ShardInterval *) lfirst(shardIntervalCell);
+
+		LockShardDistributionMetadata(shardInterval->shardId, lock);
+	}
+}
+
+
+/*
+ * GetSortedReferenceShards iterates through the given relation list.
+ * Lists the shards of reference tables and returns the list after sorting.
+ */
+static List *
+GetSortedReferenceShardIntervals(List *relationList)
+{
+	List *shardIntervalList = NIL;
+	ListCell *relationCell = NULL;
+
+	foreach(relationCell, relationList)
+	{
+		Oid relationId = lfirst_oid(relationCell);
+		List *currentShardIntervalList = NIL;
+
+		if (PartitionMethod(relationId) != DISTRIBUTE_BY_NONE)
+		{
+			continue;
+		}
+
+		currentShardIntervalList = LoadShardIntervalList(relationId);
+		shardIntervalList = lappend(shardIntervalList, linitial(
+										currentShardIntervalList));
+	}
+
+	shardIntervalList = SortList(shardIntervalList, CompareShardIntervalsById);
+
+	return shardIntervalList;
+}
+
+
 /*
  * TryLockShardDistributionMetadata tries to grab a lock for distribution
  * metadata related to the specified shard, returning false if the lock
diff --git a/src/include/distributed/resource_lock.h b/src/include/distributed/resource_lock.h
index 014b68e61..a83519088 100644
--- a/src/include/distributed/resource_lock.h
+++ b/src/include/distributed/resource_lock.h
@@ -67,6 +67,10 @@ typedef enum AdvisoryLocktagClass
 extern void LockShardDistributionMetadata(int64 shardId, LOCKMODE lockMode);
 extern bool TryLockShardDistributionMetadata(int64 shardId, LOCKMODE lockMode);
 
+/* Lock shard/relation metadata of the referenced reference table if exists */
+extern void LockReferencedReferenceShardDistributionMetadata(uint64 shardId, LOCKMODE
+															 lock);
+
 /* Lock shard data, for DML commands or remote fetches */
 extern void LockShardResource(uint64 shardId, LOCKMODE lockmode);
 extern void UnlockShardResource(uint64 shardId, LOCKMODE lockmode);