From 277bdf5bb12b1f4e208d551c5fc3ff098145b905 Mon Sep 17 00:00:00 2001
From: Burak Velioglu <buvelio@microsoft.com>
Date: Thu, 27 Jan 2022 12:01:56 +0300
Subject: [PATCH] Update start_metadata and stop_metadata command definitions

---
 .../distributed/metadata/metadata_sync.c      | 46 +++++++++++++++++--
 .../distributed/metadata/node_metadata.c      |  4 +-
 src/include/distributed/worker_manager.h      |  1 +
 3 files changed, 43 insertions(+), 8 deletions(-)

diff --git a/src/backend/distributed/metadata/metadata_sync.c b/src/backend/distributed/metadata/metadata_sync.c
index 29d7aa962..29316d26b 100644
--- a/src/backend/distributed/metadata/metadata_sync.c
+++ b/src/backend/distributed/metadata/metadata_sync.c
@@ -106,6 +106,7 @@ static RoleSpec * GetRoleSpecObjectForGrantStmt(Oid roleOid);
 static List * GenerateGrantOnSchemaQueriesFromAclItem(Oid schemaOid,
 													  AclItem *aclItem);
 static void SetLocalEnableDependencyCreation(bool state);
+static void SetLocalReplicateReferenceTablesOnActivate(bool state);
 static char * GenerateSetRoleQuery(Oid roleOid);
 static void MetadataSyncSigTermHandler(SIGNAL_ARGS);
 static void MetadataSyncSigAlrmHandler(SIGNAL_ARGS);
@@ -162,7 +163,12 @@ start_metadata_sync_to_node(PG_FUNCTION_ARGS)
 
 	char *nodeNameString = text_to_cstring(nodeName);
 
-	SyncNodeMetadataToNode(nodeNameString, nodePort);
+	bool prevReplicateRefTablesOnActivate = ReplicateReferenceTablesOnActivate;
+	SetLocalReplicateReferenceTablesOnActivate(false);
+
+	ActivateNode(nodeNameString, nodePort);
+
+	SetLocalReplicateReferenceTablesOnActivate(prevReplicateRefTablesOnActivate);
 
 	PG_RETURN_VOID();
 }
@@ -516,12 +522,28 @@ DropMetadataSnapshotOnNode(WorkerNode *workerNode)
 
 	char *userName = CurrentUserName();
 
-	/* generate the queries which drop the metadata */
-	List *dropMetadataCommandList = NodeMetadataDropCommands();
-
+	/*
+	 * Detach partitions, break dependencies between sequences and table then
+	 * remove shell tables first.
+	 */
+	List *dropMetadataCommandList = DetachPartitionCommandList();
 	dropMetadataCommandList = lappend(dropMetadataCommandList,
-									  LocalGroupIdUpdateCommand(0));
+									  BREAK_CITUS_TABLE_SEQUENCE_DEPENDENCY_COMMAND);
+	dropMetadataCommandList = lappend(dropMetadataCommandList,
+									  REMOVE_ALL_SHELL_TABLES_COMMAND);
+	dropMetadataCommandList = list_concat(dropMetadataCommandList,
+										  NodeMetadataDropCommands());
+	dropMetadataCommandList = lappend(dropMetadataCommandList, LocalGroupIdUpdateCommand(
+										  0));
 
+	/* remove all dist table and object/table related metadata afterwards */
+	dropMetadataCommandList = lappend(dropMetadataCommandList, DELETE_ALL_PARTITIONS);
+	dropMetadataCommandList = lappend(dropMetadataCommandList, DELETE_ALL_SHARDS);
+	dropMetadataCommandList = lappend(dropMetadataCommandList, DELETE_ALL_PLACEMENTS);
+	dropMetadataCommandList = lappend(dropMetadataCommandList,
+									  DELETE_ALL_DISTRIBUTED_OBJECTS);
+
+	EnsureSequentialModeMetadataOperations();
 	SendOptionalMetadataCommandListToWorkerInCoordinatedTransaction(
 		workerNode->workerName,
 		workerNode->workerPort,
@@ -1854,6 +1876,20 @@ SetLocalEnableDependencyCreation(bool state)
 }
 
 
+/*
+ * SetLocalReplicateReferenceTablesOnActivate sets the
+ * replicate_reference_tables_on_activate locally
+ */
+void
+SetLocalReplicateReferenceTablesOnActivate(bool state)
+{
+	set_config_option("citus.replicate_reference_tables_on_activate",
+					  state == true ? "on" : "off",
+					  (superuser() ? PGC_SUSET : PGC_USERSET), PGC_S_SESSION,
+					  GUC_ACTION_LOCAL, true, 0, false);
+}
+
+
 static char *
 GenerateSetRoleQuery(Oid roleOid)
 {
diff --git a/src/backend/distributed/metadata/node_metadata.c b/src/backend/distributed/metadata/node_metadata.c
index 8f4b586f5..127755411 100644
--- a/src/backend/distributed/metadata/node_metadata.c
+++ b/src/backend/distributed/metadata/node_metadata.c
@@ -92,7 +92,6 @@ typedef struct NodeMetadata
 } NodeMetadata;
 
 /* local function forward declarations */
-static int ActivateNode(char *nodeName, int nodePort);
 static void RemoveNodeFromCluster(char *nodeName, int32 nodePort);
 static void ErrorIfNodeContainsNonRemovablePlacements(WorkerNode *workerNode);
 static bool PlacementHasActivePlacementOnAnotherGroup(GroupShardPlacement
@@ -799,7 +798,6 @@ SyncObjectDependenciesToNode(WorkerNode *workerNode)
 		return;
 	}
 
-	EnsureNoModificationsHaveBeenDone();
 	Assert(ShouldPropagate());
 
 	List *commandList = SyncObjectDependenciesCommandList(workerNode);
@@ -1054,7 +1052,7 @@ PrimaryNodeForGroup(int32 groupId, bool *groupContainsNodes)
  * includes only replicating the reference tables and setting isactive column of the
  * given node.
  */
-static int
+int
 ActivateNode(char *nodeName, int nodePort)
 {
 	bool isActive = true;
diff --git a/src/include/distributed/worker_manager.h b/src/include/distributed/worker_manager.h
index 2fbfd4659..7e8aecb61 100644
--- a/src/include/distributed/worker_manager.h
+++ b/src/include/distributed/worker_manager.h
@@ -63,6 +63,7 @@ extern char *WorkerListFileName;
 extern char *CurrentCluster;
 extern bool ReplicateReferenceTablesOnActivate;
 
+extern int ActivateNode(char *nodeName, int nodePort);
 
 /* Function declarations for finding worker nodes to place shards on */
 extern WorkerNode * WorkerGetRandomCandidateNode(List *currentNodeList);