From 363127d4ae62e76482f11a93572a9a08addfe51b Mon Sep 17 00:00:00 2001
From: Onur Tirtir <onurcantirtir@gmail.com>
Date: Wed, 28 Feb 2024 17:59:53 +0300
Subject: [PATCH 01/78] phase1: move all the logic into a new file

---
 .../non_main_db_distribute_object_ops.c       | 396 ++++++++++++++++++
 .../distributed/commands/utility_hook.c       | 381 -----------------
 src/include/distributed/commands.h            |   5 +
 3 files changed, 401 insertions(+), 381 deletions(-)
 create mode 100644 src/backend/distributed/commands/non_main_db_distribute_object_ops.c

diff --git a/src/backend/distributed/commands/non_main_db_distribute_object_ops.c b/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
new file mode 100644
index 000000000..fa759affc
--- /dev/null
+++ b/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
@@ -0,0 +1,396 @@
+/*-------------------------------------------------------------------------
+ *
+ * non_main_db_distribute_object_ops.c
+ *
+ *    Routines to support node-wide object management commands from non-main
+ *    databases.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "access/xact.h"
+#include "nodes/nodes.h"
+#include "nodes/parsenodes.h"
+#include "utils/builtins.h"
+
+#include "distributed/commands.h"
+#include "distributed/deparser.h"
+#include "distributed/listutils.h"
+#include "distributed/metadata_cache.h"
+#include "distributed/remote_transaction.h"
+
+
+#define EXECUTE_COMMAND_ON_REMOTE_NODES_AS_USER \
+	"SELECT citus_internal.execute_command_on_remote_nodes_as_user(%s, %s)"
+#define START_MANAGEMENT_TRANSACTION \
+	"SELECT citus_internal.start_management_transaction('%lu')"
+#define MARK_OBJECT_DISTRIBUTED \
+	"SELECT citus_internal.mark_object_distributed(%d, %s, %d, %s)"
+#define UNMARK_OBJECT_DISTRIBUTED \
+	"SELECT pg_catalog.citus_unmark_object_distributed(%d, %d, %d,%s)"
+
+
+/* see NonMainDbDistributedStatementInfo for the explanation of these flags */
+typedef enum DistObjectOperation
+{
+	NO_DIST_OBJECT_OPERATION,
+	MARK_DISTRIBUTED_GLOBALLY,
+	UNMARK_DISTRIBUTED_LOCALLY
+} DistObjectOperation;
+
+/*
+ * NonMainDbDistributedStatementInfo is used to determine whether a statement is
+ * supported from non-main databases and whether it should be marked or unmarked
+ * as distributed.
+ *
+ * When creating a distributed object, we always have to mark such objects as
+ * "distributed" but while for some object types we can delegate this to main
+ * database, for some others we have to explicitly send a command to all nodes
+ * in this code-path to achieve this. Callers need to provide
+ * MARK_DISTRIBUTED_GLOBALLY when that is the case.
+ *
+ * Similarly when dropping a distributed object, we always have to unmark such
+ * objects as "distributed" and our utility hook on remote nodes achieve this
+ * via UnmarkNodeWideObjectsDistributed() because the commands that we send to
+ * workers are executed via main db. However for the local node, this is not the
+ * case as we're not in the main db. For this reason, callers need to provide
+ * UNMARK_DISTRIBUTED_LOCALLY to unmark an object for local node as well.
+ */
+typedef struct NonMainDbDistributedStatementInfo
+{
+	int statementType;
+	DistObjectOperation DistObjectOperation;
+
+	/*
+	 * checkSupportedObjectTypes is a callback function that checks whether
+	 * type of the object referred to by given statement is supported.
+	 *
+	 * Can be NULL if not applicable for the statement type.
+	 */
+	bool (*checkSupportedObjectTypes)(Node *node);
+} NonMainDbDistributedStatementInfo;
+
+/*
+ * DistObjectOperationParams is used to pass parameters to the
+ * MarkObjectDistributedGloballyFromNonMainDb function and
+ * UnMarkObjectDistributedLocallyFromNonMainDb functions.
+ */
+typedef struct DistObjectOperationParams
+{
+	char *name;
+	Oid id;
+	uint16 catalogRelId;
+} DistObjectOperationParams;
+
+
+/*
+ * checkSupportedObjectTypes callbacks for
+ * NonMainDbDistributedStatementInfo objects.
+ */
+static bool NonMainDbCheckSupportedObjectTypeForGrant(Node *node);
+static bool NonMainDbCheckSupportedObjectTypeForSecLabel(Node *node);
+
+/*
+ * NonMainDbSupportedStatements is an array of statements that are supported
+ * from non-main databases.
+ */
+ObjectType supportedObjectTypesForGrantStmt[] = { OBJECT_DATABASE };
+static const NonMainDbDistributedStatementInfo NonMainDbSupportedStatements[] = {
+	{ T_GrantRoleStmt, NO_DIST_OBJECT_OPERATION, NULL },
+	{ T_CreateRoleStmt, MARK_DISTRIBUTED_GLOBALLY, NULL },
+	{ T_DropRoleStmt, UNMARK_DISTRIBUTED_LOCALLY, NULL },
+	{ T_AlterRoleStmt, NO_DIST_OBJECT_OPERATION, NULL },
+	{ T_GrantStmt, NO_DIST_OBJECT_OPERATION, NonMainDbCheckSupportedObjectTypeForGrant },
+	{ T_CreatedbStmt, NO_DIST_OBJECT_OPERATION, NULL },
+	{ T_DropdbStmt, NO_DIST_OBJECT_OPERATION, NULL },
+	{ T_SecLabelStmt, NO_DIST_OBJECT_OPERATION,
+	  NonMainDbCheckSupportedObjectTypeForSecLabel },
+};
+
+
+static bool IsStatementSupportedFromNonMainDb(Node *parsetree);
+static bool StatementRequiresMarkDistributedGloballyFromNonMainDb(Node *parsetree);
+static bool StatementRequiresUnmarkDistributedLocallyFromNonMainDb(Node *parsetree);
+static void MarkObjectDistributedGloballyFromNonMainDb(Node *parsetree);
+static void UnMarkObjectDistributedLocallyFromNonMainDb(List *unmarkDistributedList);
+static List * GetDistObjectOperationParams(Node *parsetree);
+
+
+/*
+ * IsCommandToCreateOrDropMainDB checks if this query creates or drops the
+ * main database, so we can make an exception and not send this query to
+ * the main database.
+ */
+bool
+IsCommandToCreateOrDropMainDB(Node *parsetree)
+{
+	if (IsA(parsetree, CreatedbStmt))
+	{
+		CreatedbStmt *createdbStmt = castNode(CreatedbStmt, parsetree);
+		return strcmp(createdbStmt->dbname, MainDb) == 0;
+	}
+	else if (IsA(parsetree, DropdbStmt))
+	{
+		DropdbStmt *dropdbStmt = castNode(DropdbStmt, parsetree);
+		return strcmp(dropdbStmt->dbname, MainDb) == 0;
+	}
+
+	return false;
+}
+
+
+/*
+ * RunPreprocessMainDBCommand runs the necessary commands for a query, in main
+ * database before query is run on the local node with PrevProcessUtility
+ */
+void
+RunPreprocessMainDBCommand(Node *parsetree)
+{
+	if (!IsStatementSupportedFromNonMainDb(parsetree))
+	{
+		return;
+	}
+
+	char *queryString = DeparseTreeNode(parsetree);
+
+	if (IsA(parsetree, CreatedbStmt) ||
+		IsA(parsetree, DropdbStmt))
+	{
+		IsMainDBCommandInXact = false;
+		RunCitusMainDBQuery((char *) queryString);
+		return;
+	}
+
+	IsMainDBCommandInXact = true;
+
+	StringInfo mainDBQuery = makeStringInfo();
+	appendStringInfo(mainDBQuery,
+					 START_MANAGEMENT_TRANSACTION,
+					 GetCurrentFullTransactionId().value);
+	RunCitusMainDBQuery(mainDBQuery->data);
+
+	mainDBQuery = makeStringInfo();
+	appendStringInfo(mainDBQuery,
+					 EXECUTE_COMMAND_ON_REMOTE_NODES_AS_USER,
+					 quote_literal_cstr(queryString),
+					 quote_literal_cstr(CurrentUserName()));
+	RunCitusMainDBQuery(mainDBQuery->data);
+
+	if (StatementRequiresUnmarkDistributedLocallyFromNonMainDb(parsetree))
+	{
+		List *unmarkParams = GetDistObjectOperationParams(parsetree);
+		UnMarkObjectDistributedLocallyFromNonMainDb(unmarkParams);
+	}
+}
+
+
+/*
+ * RunPostprocessMainDBCommand runs the necessary commands for a query, in main
+ * database after query is run on the local node with PrevProcessUtility
+ */
+void
+RunPostprocessMainDBCommand(Node *parsetree)
+{
+	if (IsStatementSupportedFromNonMainDb(parsetree) &&
+		StatementRequiresMarkDistributedGloballyFromNonMainDb(parsetree))
+	{
+		MarkObjectDistributedGloballyFromNonMainDb(parsetree);
+	}
+}
+
+
+/*
+ * IsStatementSupportedFromNonMainDb returns true if the statement is supported from a
+ * non-main database.
+ */
+static bool
+IsStatementSupportedFromNonMainDb(Node *parsetree)
+{
+	NodeTag type = nodeTag(parsetree);
+
+	for (int i = 0; i < sizeof(NonMainDbSupportedStatements) /
+		 sizeof(NonMainDbSupportedStatements[0]); i++)
+	{
+		if (type != NonMainDbSupportedStatements[i].statementType)
+		{
+			continue;
+		}
+
+		return !NonMainDbSupportedStatements[i].checkSupportedObjectTypes ||
+			   NonMainDbSupportedStatements[i].checkSupportedObjectTypes(parsetree);
+	}
+
+	return false;
+}
+
+
+/*
+ * StatementRequiresMarkDistributedGloballyFromNonMainDb returns true if the statement should be marked
+ * as distributed when executed from a non-main database.
+ */
+static bool
+StatementRequiresMarkDistributedGloballyFromNonMainDb(Node *parsetree)
+{
+	NodeTag type = nodeTag(parsetree);
+
+	for (int i = 0; i < sizeof(NonMainDbSupportedStatements) /
+		 sizeof(NonMainDbSupportedStatements[0]); i++)
+	{
+		if (type == NonMainDbSupportedStatements[i].statementType)
+		{
+			return NonMainDbSupportedStatements[i].DistObjectOperation ==
+				   MARK_DISTRIBUTED_GLOBALLY;
+		}
+	}
+
+	return false;
+}
+
+
+/*
+ * StatementRequiresUnmarkDistributedLocallyFromNonMainDb returns true if the statement should be unmarked
+ * as distributed when executed from a non-main database.
+ */
+static bool
+StatementRequiresUnmarkDistributedLocallyFromNonMainDb(Node *parsetree)
+{
+	NodeTag type = nodeTag(parsetree);
+
+	for (int i = 0; i < sizeof(NonMainDbSupportedStatements) /
+		 sizeof(NonMainDbSupportedStatements[0]); i++)
+	{
+		if (type == NonMainDbSupportedStatements[i].statementType)
+		{
+			return NonMainDbSupportedStatements[i].DistObjectOperation ==
+				   UNMARK_DISTRIBUTED_LOCALLY;
+		}
+	}
+
+	return false;
+}
+
+
+/*
+ * MarkObjectDistributedGloballyFromNonMainDb marks the given object as distributed on the
+ * non-main database.
+ */
+static void
+MarkObjectDistributedGloballyFromNonMainDb(Node *parsetree)
+{
+	List *distObjectOperationParams =
+		GetDistObjectOperationParams(parsetree);
+
+	DistObjectOperationParams *distObjectOperationParam = NULL;
+
+	foreach_ptr(distObjectOperationParam, distObjectOperationParams)
+	{
+		StringInfo mainDBQuery = makeStringInfo();
+		appendStringInfo(mainDBQuery,
+						 MARK_OBJECT_DISTRIBUTED,
+						 distObjectOperationParam->catalogRelId,
+						 quote_literal_cstr(distObjectOperationParam->name),
+						 distObjectOperationParam->id,
+						 quote_literal_cstr(CurrentUserName()));
+		RunCitusMainDBQuery(mainDBQuery->data);
+	}
+}
+
+
+/*
+ * UnMarkObjectDistributedLocallyFromNonMainDb unmarks the given object as distributed on the
+ * non-main database.
+ */
+static void
+UnMarkObjectDistributedLocallyFromNonMainDb(List *markObjectDistributedParamList)
+{
+	DistObjectOperationParams *markObjectDistributedParam = NULL;
+	int subObjectId = 0;
+	char *checkObjectExistence = "false";
+	foreach_ptr(markObjectDistributedParam, markObjectDistributedParamList)
+	{
+		StringInfo query = makeStringInfo();
+		appendStringInfo(query,
+						 UNMARK_OBJECT_DISTRIBUTED,
+						 AuthIdRelationId,
+						 markObjectDistributedParam->id,
+						 subObjectId, checkObjectExistence);
+		RunCitusMainDBQuery(query->data);
+	}
+}
+
+
+/*
+ * GetDistObjectOperationParams returns DistObjectOperationParams for the target
+ * object of given parsetree.
+ */
+List *
+GetDistObjectOperationParams(Node *parsetree)
+{
+	List *paramsList = NIL;
+	if (IsA(parsetree, CreateRoleStmt))
+	{
+		CreateRoleStmt *stmt = castNode(CreateRoleStmt, parsetree);
+		DistObjectOperationParams *params =
+			(DistObjectOperationParams *) palloc(sizeof(DistObjectOperationParams));
+		params->name = stmt->role;
+		params->catalogRelId = AuthIdRelationId;
+		params->id = get_role_oid(stmt->role, false);
+
+		paramsList = lappend(paramsList, params);
+	}
+	else if (IsA(parsetree, DropRoleStmt))
+	{
+		DropRoleStmt *stmt = castNode(DropRoleStmt, parsetree);
+		RoleSpec *roleSpec;
+		foreach_ptr(roleSpec, stmt->roles)
+		{
+			DistObjectOperationParams *params = (DistObjectOperationParams *) palloc(
+				sizeof(DistObjectOperationParams));
+
+			Oid roleOid = get_role_oid(roleSpec->rolename, true);
+
+			if (roleOid == InvalidOid)
+			{
+				continue;
+			}
+
+			params->id = roleOid;
+			params->name = roleSpec->rolename;
+			params->catalogRelId = AuthIdRelationId;
+
+			paramsList = lappend(paramsList, params);
+		}
+	}
+	else
+	{
+		elog(ERROR, "unsupported statement type");
+	}
+
+	return paramsList;
+}
+
+
+/*
+ * NonMainDbCheckSupportedObjectTypeForGrant implements checkSupportedObjectTypes
+ * callback for GrantStmt.
+ */
+static bool
+NonMainDbCheckSupportedObjectTypeForGrant(Node *node)
+{
+	GrantStmt *stmt = castNode(GrantStmt, node);
+	return stmt->objtype == OBJECT_DATABASE;
+}
+
+
+/*
+ * NonMainDbCheckSupportedObjectTypeForSecLabel implements checkSupportedObjectTypes
+ * callback for SecLabel.
+ */
+static bool
+NonMainDbCheckSupportedObjectTypeForSecLabel(Node *node)
+{
+	SecLabelStmt *stmt = castNode(SecLabelStmt, node);
+	return stmt->objtype == OBJECT_ROLE;
+}
diff --git a/src/backend/distributed/commands/utility_hook.c b/src/backend/distributed/commands/utility_hook.c
index e264713dd..96c515fe8 100644
--- a/src/backend/distributed/commands/utility_hook.c
+++ b/src/backend/distributed/commands/utility_hook.c
@@ -87,69 +87,6 @@
 #include "distributed/worker_shard_visibility.h"
 #include "distributed/worker_transaction.h"
 
-#define EXECUTE_COMMAND_ON_REMOTE_NODES_AS_USER \
-	"SELECT citus_internal.execute_command_on_remote_nodes_as_user(%s, %s)"
-#define START_MANAGEMENT_TRANSACTION \
-	"SELECT citus_internal.start_management_transaction('%lu')"
-#define MARK_OBJECT_DISTRIBUTED \
-	"SELECT citus_internal.mark_object_distributed(%d, %s, %d, %s)"
-#define UNMARK_OBJECT_DISTRIBUTED \
-	"SELECT pg_catalog.citus_unmark_object_distributed(%d, %d, %d,%s)"
-
-/* see NonMainDbDistributedStatementInfo for the explanation of these flags */
-typedef enum DistObjectOperation
-{
-	NO_DIST_OBJECT_OPERATION,
-	MARK_DISTRIBUTED_GLOBALLY,
-	UNMARK_DISTRIBUTED_LOCALLY
-} DistObjectOperation;
-
-
-/*
- * NonMainDbDistributedStatementInfo is used to determine whether a statement is
- * supported from non-main databases and whether it should be marked or unmarked
- * as distributed.
- *
- * When creating a distributed object, we always have to mark such objects as
- * "distributed" but while for some object types we can delegate this to main
- * database, for some others we have to explicitly send a command to all nodes
- * in this code-path to achieve this. Callers need to provide
- * MARK_DISTRIBUTED_GLOBALLY when that is the case.
- *
- * Similarly when dropping a distributed object, we always have to unmark such
- * objects as "distributed" and our utility hook on remote nodes achieve this
- * via UnmarkNodeWideObjectsDistributed() because the commands that we send to
- * workers are executed via main db. However for the local node, this is not the
- * case as we're not in the main db. For this reason, callers need to provide
- * UNMARK_DISTRIBUTED_LOCALLY to unmark an object for local node as well.
- */
-typedef struct NonMainDbDistributedStatementInfo
-{
-	int statementType;
-	DistObjectOperation DistObjectOperation;
-
-	/*
-	 * checkSupportedObjectTypes is a callback function that checks whether
-	 * type of the object referred to by given statement is supported.
-	 *
-	 * Can be NULL if not applicable for the statement type.
-	 */
-	bool (*checkSupportedObjectTypes)(Node *node);
-} NonMainDbDistributedStatementInfo;
-
-/*
- * DistObjectOperationParams is used to pass parameters to the
- * MarkObjectDistributedGloballyFromNonMainDb function and
- * UnMarkObjectDistributedLocallyFromNonMainDb functions.
- */
-typedef struct DistObjectOperationParams
-{
-	char *name;
-	Oid id;
-	uint16 catalogRelId;
-} DistObjectOperationParams;
-
-
 bool EnableDDLPropagation = true; /* ddl propagation is enabled */
 int CreateObjectPropagationMode = CREATE_OBJECT_PROPAGATION_IMMEDIATE;
 PropSetCmdBehavior PropagateSetCommands = PROPSETCMD_NONE; /* SET prop off */
@@ -179,46 +116,6 @@ static bool IsDropSchemaOrDB(Node *parsetree);
 static bool ShouldCheckUndistributeCitusLocalTables(void);
 
 
-/*
- * Functions to support commands used to manage node-wide objects from non-main
- * databases.
- */
-static bool IsCommandToCreateOrDropMainDB(Node *parsetree);
-static void RunPreprocessMainDBCommand(Node *parsetree);
-static void RunPostprocessMainDBCommand(Node *parsetree);
-static bool IsStatementSupportedFromNonMainDb(Node *parsetree);
-static bool StatementRequiresMarkDistributedGloballyFromNonMainDb(Node *parsetree);
-static bool StatementRequiresUnmarkDistributedLocallyFromNonMainDb(Node *parsetree);
-static void MarkObjectDistributedGloballyFromNonMainDb(Node *parsetree);
-static void UnMarkObjectDistributedLocallyFromNonMainDb(List *unmarkDistributedList);
-static List * GetDistObjectOperationParams(Node *parsetree);
-
-/*
- * checkSupportedObjectTypes callbacks for
- * NonMainDbDistributedStatementInfo objects.
- */
-static bool NonMainDbCheckSupportedObjectTypeForGrant(Node *node);
-static bool NonMainDbCheckSupportedObjectTypeForSecLabel(Node *node);
-
-
-/*
- * NonMainDbSupportedStatements is an array of statements that are supported
- * from non-main databases.
- */
-ObjectType supportedObjectTypesForGrantStmt[] = { OBJECT_DATABASE };
-static const NonMainDbDistributedStatementInfo NonMainDbSupportedStatements[] = {
-	{ T_GrantRoleStmt, NO_DIST_OBJECT_OPERATION, NULL },
-	{ T_CreateRoleStmt, MARK_DISTRIBUTED_GLOBALLY, NULL },
-	{ T_DropRoleStmt, UNMARK_DISTRIBUTED_LOCALLY, NULL },
-	{ T_AlterRoleStmt, NO_DIST_OBJECT_OPERATION, NULL },
-	{ T_GrantStmt, NO_DIST_OBJECT_OPERATION, NonMainDbCheckSupportedObjectTypeForGrant },
-	{ T_CreatedbStmt, NO_DIST_OBJECT_OPERATION, NULL },
-	{ T_DropdbStmt, NO_DIST_OBJECT_OPERATION, NULL },
-	{ T_SecLabelStmt, NO_DIST_OBJECT_OPERATION,
-	  NonMainDbCheckSupportedObjectTypeForSecLabel },
-};
-
-
 /*
  * ProcessUtilityParseTree is a convenience method to create a PlannedStmt out of
  * pieces of a utility statement before invoking ProcessUtility.
@@ -1715,281 +1612,3 @@ DropSchemaOrDBInProgress(void)
 {
 	return activeDropSchemaOrDBs > 0;
 }
-
-
-/*
- * IsCommandToCreateOrDropMainDB checks if this query creates or drops the
- * main database, so we can make an exception and not send this query to
- * the main database.
- */
-static bool
-IsCommandToCreateOrDropMainDB(Node *parsetree)
-{
-	if (IsA(parsetree, CreatedbStmt))
-	{
-		CreatedbStmt *createdbStmt = castNode(CreatedbStmt, parsetree);
-		return strcmp(createdbStmt->dbname, MainDb) == 0;
-	}
-	else if (IsA(parsetree, DropdbStmt))
-	{
-		DropdbStmt *dropdbStmt = castNode(DropdbStmt, parsetree);
-		return strcmp(dropdbStmt->dbname, MainDb) == 0;
-	}
-
-	return false;
-}
-
-
-/*
- * RunPreprocessMainDBCommand runs the necessary commands for a query, in main
- * database before query is run on the local node with PrevProcessUtility
- */
-static void
-RunPreprocessMainDBCommand(Node *parsetree)
-{
-	if (!IsStatementSupportedFromNonMainDb(parsetree))
-	{
-		return;
-	}
-
-	char *queryString = DeparseTreeNode(parsetree);
-
-	if (IsA(parsetree, CreatedbStmt) ||
-		IsA(parsetree, DropdbStmt))
-	{
-		IsMainDBCommandInXact = false;
-		RunCitusMainDBQuery((char *) queryString);
-		return;
-	}
-
-	IsMainDBCommandInXact = true;
-
-	StringInfo mainDBQuery = makeStringInfo();
-	appendStringInfo(mainDBQuery,
-					 START_MANAGEMENT_TRANSACTION,
-					 GetCurrentFullTransactionId().value);
-	RunCitusMainDBQuery(mainDBQuery->data);
-
-	mainDBQuery = makeStringInfo();
-	appendStringInfo(mainDBQuery,
-					 EXECUTE_COMMAND_ON_REMOTE_NODES_AS_USER,
-					 quote_literal_cstr(queryString),
-					 quote_literal_cstr(CurrentUserName()));
-	RunCitusMainDBQuery(mainDBQuery->data);
-
-	if (StatementRequiresUnmarkDistributedLocallyFromNonMainDb(parsetree))
-	{
-		List *unmarkParams = GetDistObjectOperationParams(parsetree);
-		UnMarkObjectDistributedLocallyFromNonMainDb(unmarkParams);
-	}
-}
-
-
-/*
- * RunPostprocessMainDBCommand runs the necessary commands for a query, in main
- * database after query is run on the local node with PrevProcessUtility
- */
-static void
-RunPostprocessMainDBCommand(Node *parsetree)
-{
-	if (IsStatementSupportedFromNonMainDb(parsetree) &&
-		StatementRequiresMarkDistributedGloballyFromNonMainDb(parsetree))
-	{
-		MarkObjectDistributedGloballyFromNonMainDb(parsetree);
-	}
-}
-
-
-/*
- * IsStatementSupportedFromNonMainDb returns true if the statement is supported from a
- * non-main database.
- */
-static bool
-IsStatementSupportedFromNonMainDb(Node *parsetree)
-{
-	NodeTag type = nodeTag(parsetree);
-
-	for (int i = 0; i < sizeof(NonMainDbSupportedStatements) /
-		 sizeof(NonMainDbSupportedStatements[0]); i++)
-	{
-		if (type != NonMainDbSupportedStatements[i].statementType)
-		{
-			continue;
-		}
-
-		return !NonMainDbSupportedStatements[i].checkSupportedObjectTypes ||
-			   NonMainDbSupportedStatements[i].checkSupportedObjectTypes(parsetree);
-	}
-
-	return false;
-}
-
-
-/*
- * StatementRequiresMarkDistributedGloballyFromNonMainDb returns true if the statement should be marked
- * as distributed when executed from a non-main database.
- */
-static bool
-StatementRequiresMarkDistributedGloballyFromNonMainDb(Node *parsetree)
-{
-	NodeTag type = nodeTag(parsetree);
-
-	for (int i = 0; i < sizeof(NonMainDbSupportedStatements) /
-		 sizeof(NonMainDbSupportedStatements[0]); i++)
-	{
-		if (type == NonMainDbSupportedStatements[i].statementType)
-		{
-			return NonMainDbSupportedStatements[i].DistObjectOperation ==
-				   MARK_DISTRIBUTED_GLOBALLY;
-		}
-	}
-
-	return false;
-}
-
-
-/*
- * StatementRequiresUnmarkDistributedLocallyFromNonMainDb returns true if the statement should be unmarked
- * as distributed when executed from a non-main database.
- */
-static bool
-StatementRequiresUnmarkDistributedLocallyFromNonMainDb(Node *parsetree)
-{
-	NodeTag type = nodeTag(parsetree);
-
-	for (int i = 0; i < sizeof(NonMainDbSupportedStatements) /
-		 sizeof(NonMainDbSupportedStatements[0]); i++)
-	{
-		if (type == NonMainDbSupportedStatements[i].statementType)
-		{
-			return NonMainDbSupportedStatements[i].DistObjectOperation ==
-				   UNMARK_DISTRIBUTED_LOCALLY;
-		}
-	}
-
-	return false;
-}
-
-
-/*
- * MarkObjectDistributedGloballyFromNonMainDb marks the given object as distributed on the
- * non-main database.
- */
-static void
-MarkObjectDistributedGloballyFromNonMainDb(Node *parsetree)
-{
-	List *distObjectOperationParams =
-		GetDistObjectOperationParams(parsetree);
-
-	DistObjectOperationParams *distObjectOperationParam = NULL;
-
-	foreach_ptr(distObjectOperationParam, distObjectOperationParams)
-	{
-		StringInfo mainDBQuery = makeStringInfo();
-		appendStringInfo(mainDBQuery,
-						 MARK_OBJECT_DISTRIBUTED,
-						 distObjectOperationParam->catalogRelId,
-						 quote_literal_cstr(distObjectOperationParam->name),
-						 distObjectOperationParam->id,
-						 quote_literal_cstr(CurrentUserName()));
-		RunCitusMainDBQuery(mainDBQuery->data);
-	}
-}
-
-
-/*
- * UnMarkObjectDistributedLocallyFromNonMainDb unmarks the given object as distributed on the
- * non-main database.
- */
-static void
-UnMarkObjectDistributedLocallyFromNonMainDb(List *markObjectDistributedParamList)
-{
-	DistObjectOperationParams *markObjectDistributedParam = NULL;
-	int subObjectId = 0;
-	char *checkObjectExistence = "false";
-	foreach_ptr(markObjectDistributedParam, markObjectDistributedParamList)
-	{
-		StringInfo query = makeStringInfo();
-		appendStringInfo(query,
-						 UNMARK_OBJECT_DISTRIBUTED,
-						 AuthIdRelationId,
-						 markObjectDistributedParam->id,
-						 subObjectId, checkObjectExistence);
-		RunCitusMainDBQuery(query->data);
-	}
-}
-
-
-/*
- * GetDistObjectOperationParams returns DistObjectOperationParams for the target
- * object of given parsetree.
- */
-List *
-GetDistObjectOperationParams(Node *parsetree)
-{
-	List *paramsList = NIL;
-	if (IsA(parsetree, CreateRoleStmt))
-	{
-		CreateRoleStmt *stmt = castNode(CreateRoleStmt, parsetree);
-		DistObjectOperationParams *params =
-			(DistObjectOperationParams *) palloc(sizeof(DistObjectOperationParams));
-		params->name = stmt->role;
-		params->catalogRelId = AuthIdRelationId;
-		params->id = get_role_oid(stmt->role, false);
-
-		paramsList = lappend(paramsList, params);
-	}
-	else if (IsA(parsetree, DropRoleStmt))
-	{
-		DropRoleStmt *stmt = castNode(DropRoleStmt, parsetree);
-		RoleSpec *roleSpec;
-		foreach_ptr(roleSpec, stmt->roles)
-		{
-			DistObjectOperationParams *params = (DistObjectOperationParams *) palloc(
-				sizeof(DistObjectOperationParams));
-
-			Oid roleOid = get_role_oid(roleSpec->rolename, true);
-
-			if (roleOid == InvalidOid)
-			{
-				continue;
-			}
-
-			params->id = roleOid;
-			params->name = roleSpec->rolename;
-			params->catalogRelId = AuthIdRelationId;
-
-			paramsList = lappend(paramsList, params);
-		}
-	}
-	else
-	{
-		elog(ERROR, "unsupported statement type");
-	}
-
-	return paramsList;
-}
-
-
-/*
- * NonMainDbCheckSupportedObjectTypeForGrant implements checkSupportedObjectTypes
- * callback for GrantStmt.
- */
-static bool
-NonMainDbCheckSupportedObjectTypeForGrant(Node *node)
-{
-	GrantStmt *stmt = castNode(GrantStmt, node);
-	return stmt->objtype == OBJECT_DATABASE;
-}
-
-
-/*
- * NonMainDbCheckSupportedObjectTypeForSecLabel implements checkSupportedObjectTypes
- * callback for SecLabel.
- */
-static bool
-NonMainDbCheckSupportedObjectTypeForSecLabel(Node *node)
-{
-	SecLabelStmt *stmt = castNode(SecLabelStmt, node);
-	return stmt->objtype == OBJECT_ROLE;
-}
diff --git a/src/include/distributed/commands.h b/src/include/distributed/commands.h
index de15553e7..37043a91e 100644
--- a/src/include/distributed/commands.h
+++ b/src/include/distributed/commands.h
@@ -104,6 +104,11 @@ typedef struct DistributeObjectOps
 
 const DistributeObjectOps * GetDistributeObjectOps(Node *node);
 
+/* functions to support node-wide object management commands from non-main dbs */
+extern bool IsCommandToCreateOrDropMainDB(Node *parsetree);
+extern void RunPreprocessMainDBCommand(Node *parsetree);
+extern void RunPostprocessMainDBCommand(Node *parsetree);
+
 /*
  * Flags that can be passed to GetForeignKeyOids to indicate
  * which foreign key constraint OIDs are to be extracted

From 0ba3a9bb5b47840f56e8108c5d96ed418139d2ae Mon Sep 17 00:00:00 2001
From: Onur Tirtir <onurcantirtir@gmail.com>
Date: Wed, 28 Feb 2024 18:06:19 +0300
Subject: [PATCH 02/78] fix builds

---
 .../distributed/commands/non_main_db_distribute_object_ops.c     | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/backend/distributed/commands/non_main_db_distribute_object_ops.c b/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
index fa759affc..2cbf3486a 100644
--- a/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
+++ b/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
@@ -11,6 +11,7 @@
 #include "postgres.h"
 
 #include "access/xact.h"
+#include "catalog/pg_authid_d.h"
 #include "nodes/nodes.h"
 #include "nodes/parsenodes.h"
 #include "utils/builtins.h"

From fadee4e49409d0efbd3c9bb6cad6dc86cfab0191 Mon Sep 17 00:00:00 2001
From: Onur Tirtir <onurcantirtir@gmail.com>
Date: Wed, 28 Feb 2024 18:25:33 +0300
Subject: [PATCH 03/78] phase 2: remove the clutter in utiity_hook.c and some
 funct renaming

---
 .../non_main_db_distribute_object_ops.c       | 51 ++++++++++++++-----
 .../distributed/commands/utility_hook.c       | 41 ++++++---------
 src/include/distributed/commands.h            |  4 +-
 3 files changed, 54 insertions(+), 42 deletions(-)

diff --git a/src/backend/distributed/commands/non_main_db_distribute_object_ops.c b/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
index 2cbf3486a..c2fc3891b 100644
--- a/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
+++ b/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
@@ -143,25 +143,42 @@ IsCommandToCreateOrDropMainDB(Node *parsetree)
 
 
 /*
- * RunPreprocessMainDBCommand runs the necessary commands for a query, in main
- * database before query is run on the local node with PrevProcessUtility
+ * RunPreprocessNonMainDBCommand runs the necessary commands for a query, in main
+ * database before query is run on the local node with PrevProcessUtility.
+ *
+ * Returns true if previous utility hook needs to be skipped after completing
+ * preprocess phase.
  */
-void
-RunPreprocessMainDBCommand(Node *parsetree)
+bool
+RunPreprocessNonMainDBCommand(Node *parsetree)
 {
-	if (!IsStatementSupportedFromNonMainDb(parsetree))
+	if (IsMainDB || !IsStatementSupportedFromNonMainDb(parsetree))
 	{
-		return;
+		return false;
+	}
+
+	if (IsCommandToCreateOrDropMainDB(parsetree))
+	{
+		/*
+		 * We don't try to send the query to the main database if the CREATE/DROP DATABASE
+		 * command is for the main database itself, this is a very rare case but it's
+		 * exercised by our test suite.
+		 */
+		return false;
 	}
 
 	char *queryString = DeparseTreeNode(parsetree);
 
-	if (IsA(parsetree, CreatedbStmt) ||
-		IsA(parsetree, DropdbStmt))
+	if (IsA(parsetree, CreatedbStmt) || IsA(parsetree, DropdbStmt))
 	{
+		/*
+		 * We always execute CREATE/DROP DATABASE from the main database. There are no
+		 * transactional visibility issues, since these commands are non-transactional.
+		 * And this way we only have to consider one codepath when creating databases.
+		 */
 		IsMainDBCommandInXact = false;
 		RunCitusMainDBQuery((char *) queryString);
-		return;
+		return true;
 	}
 
 	IsMainDBCommandInXact = true;
@@ -184,18 +201,24 @@ RunPreprocessMainDBCommand(Node *parsetree)
 		List *unmarkParams = GetDistObjectOperationParams(parsetree);
 		UnMarkObjectDistributedLocallyFromNonMainDb(unmarkParams);
 	}
+
+	return false;
 }
 
 
 /*
- * RunPostprocessMainDBCommand runs the necessary commands for a query, in main
- * database after query is run on the local node with PrevProcessUtility
+ * RunPostprocessNonMainDBCommand runs the necessary commands for a query, in main
+ * database after query is run on the local node with PrevProcessUtility.
  */
 void
-RunPostprocessMainDBCommand(Node *parsetree)
+RunPostprocessNonMainDBCommand(Node *parsetree)
 {
-	if (IsStatementSupportedFromNonMainDb(parsetree) &&
-		StatementRequiresMarkDistributedGloballyFromNonMainDb(parsetree))
+	if (IsMainDB || !IsStatementSupportedFromNonMainDb(parsetree))
+	{
+		return;
+	}
+
+	if (StatementRequiresMarkDistributedGloballyFromNonMainDb(parsetree))
 	{
 		MarkObjectDistributedGloballyFromNonMainDb(parsetree);
 	}
diff --git a/src/backend/distributed/commands/utility_hook.c b/src/backend/distributed/commands/utility_hook.c
index 96c515fe8..9426e13c0 100644
--- a/src/backend/distributed/commands/utility_hook.c
+++ b/src/backend/distributed/commands/utility_hook.c
@@ -247,36 +247,25 @@ citus_ProcessUtility(PlannedStmt *pstmt,
 	if (!CitusHasBeenLoaded())
 	{
 		/*
-		 * We always execute CREATE/DROP DATABASE from the main database. There are no
-		 * transactional visibility issues, since these commands are non-transactional.
-		 * And this way we only have to consider one codepath when creating databases.
-		 * We don't try to send the query to the main database if the CREATE/DROP DATABASE
-		 * command is for the main database itself, this is a very rare case but it's
-		 * exercised by our test suite.
+		 * Process the command via RunPreprocessNonMainDBCommand and
+		 * RunPostprocessNonMainDBCommand hooks if we're in a non-main database
+		 * and if the command is a node-wide object management command that we
+		 * support from non-main databases.
 		 */
-		if (!IsMainDB &&
-			!IsCommandToCreateOrDropMainDB(parsetree))
-		{
-			RunPreprocessMainDBCommand(parsetree);
 
-			if (IsA(parsetree, CreatedbStmt) ||
-				IsA(parsetree, DropdbStmt))
-			{
-				return;
-			}
+		bool shouldSkipPrevUtilityHook = RunPreprocessNonMainDBCommand(parsetree);
+
+		if (!shouldSkipPrevUtilityHook)
+		{
+			/*
+			 * Ensure that utility commands do not behave any differently until CREATE
+			 * EXTENSION is invoked.
+			 */
+			PrevProcessUtility(pstmt, queryString, false, context,
+							   params, queryEnv, dest, completionTag);
 		}
 
-		/*
-		 * Ensure that utility commands do not behave any differently until CREATE
-		 * EXTENSION is invoked.
-		 */
-		PrevProcessUtility(pstmt, queryString, false, context,
-						   params, queryEnv, dest, completionTag);
-
-		if (!IsMainDB)
-		{
-			RunPostprocessMainDBCommand(parsetree);
-		}
+		RunPostprocessNonMainDBCommand(parsetree);
 
 		return;
 	}
diff --git a/src/include/distributed/commands.h b/src/include/distributed/commands.h
index 37043a91e..e9468521f 100644
--- a/src/include/distributed/commands.h
+++ b/src/include/distributed/commands.h
@@ -106,8 +106,8 @@ const DistributeObjectOps * GetDistributeObjectOps(Node *node);
 
 /* functions to support node-wide object management commands from non-main dbs */
 extern bool IsCommandToCreateOrDropMainDB(Node *parsetree);
-extern void RunPreprocessMainDBCommand(Node *parsetree);
-extern void RunPostprocessMainDBCommand(Node *parsetree);
+extern bool RunPreprocessNonMainDBCommand(Node *parsetree);
+extern void RunPostprocessNonMainDBCommand(Node *parsetree);
 
 /*
  * Flags that can be passed to GetForeignKeyOids to indicate

From 3bdecb10a0aa29b0db6fb202c15c2fc416936800 Mon Sep 17 00:00:00 2001
From: Onur Tirtir <onurcantirtir@gmail.com>
Date: Wed, 28 Feb 2024 18:44:49 +0300
Subject: [PATCH 04/78] phase3: generalize the fact that some commands cannot
 be exec in a xact

---
 .../non_main_db_distribute_object_ops.c       | 130 +++++++++++-------
 src/include/distributed/commands.h            |   1 -
 2 files changed, 80 insertions(+), 51 deletions(-)

diff --git a/src/backend/distributed/commands/non_main_db_distribute_object_ops.c b/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
index c2fc3891b..cb0d7ddfc 100644
--- a/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
+++ b/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
@@ -62,7 +62,7 @@ typedef enum DistObjectOperation
 typedef struct NonMainDbDistributedStatementInfo
 {
 	int statementType;
-	DistObjectOperation DistObjectOperation;
+	DistObjectOperation distObjectOperation;
 
 	/*
 	 * checkSupportedObjectTypes is a callback function that checks whether
@@ -71,6 +71,9 @@ typedef struct NonMainDbDistributedStatementInfo
 	 * Can be NULL if not applicable for the statement type.
 	 */
 	bool (*checkSupportedObjectTypes)(Node *node);
+
+	/* indicates whether the statement cannot be executed in a transaction */
+	bool cannotBeExecutedInTransaction;
 } NonMainDbDistributedStatementInfo;
 
 /*
@@ -90,6 +93,8 @@ typedef struct DistObjectOperationParams
  * checkSupportedObjectTypes callbacks for
  * NonMainDbDistributedStatementInfo objects.
  */
+static bool NonMainDbCheckSupportedObjectTypeForCreateDatabase(Node *node);
+static bool NonMainDbCheckSupportedObjectTypeForDropDatabase(Node *node);
 static bool NonMainDbCheckSupportedObjectTypeForGrant(Node *node);
 static bool NonMainDbCheckSupportedObjectTypeForSecLabel(Node *node);
 
@@ -99,49 +104,30 @@ static bool NonMainDbCheckSupportedObjectTypeForSecLabel(Node *node);
  */
 ObjectType supportedObjectTypesForGrantStmt[] = { OBJECT_DATABASE };
 static const NonMainDbDistributedStatementInfo NonMainDbSupportedStatements[] = {
-	{ T_GrantRoleStmt, NO_DIST_OBJECT_OPERATION, NULL },
-	{ T_CreateRoleStmt, MARK_DISTRIBUTED_GLOBALLY, NULL },
-	{ T_DropRoleStmt, UNMARK_DISTRIBUTED_LOCALLY, NULL },
-	{ T_AlterRoleStmt, NO_DIST_OBJECT_OPERATION, NULL },
-	{ T_GrantStmt, NO_DIST_OBJECT_OPERATION, NonMainDbCheckSupportedObjectTypeForGrant },
-	{ T_CreatedbStmt, NO_DIST_OBJECT_OPERATION, NULL },
-	{ T_DropdbStmt, NO_DIST_OBJECT_OPERATION, NULL },
+	{ T_GrantRoleStmt, NO_DIST_OBJECT_OPERATION, NULL, false },
+	{ T_CreateRoleStmt, MARK_DISTRIBUTED_GLOBALLY, NULL, false },
+	{ T_DropRoleStmt, UNMARK_DISTRIBUTED_LOCALLY, NULL, false },
+	{ T_AlterRoleStmt, NO_DIST_OBJECT_OPERATION, NULL, false },
+	{ T_GrantStmt, NO_DIST_OBJECT_OPERATION,
+	  NonMainDbCheckSupportedObjectTypeForGrant, false },
+	{ T_CreatedbStmt, NO_DIST_OBJECT_OPERATION,
+	  NonMainDbCheckSupportedObjectTypeForCreateDatabase, true },
+	{ T_DropdbStmt, NO_DIST_OBJECT_OPERATION,
+	  NonMainDbCheckSupportedObjectTypeForDropDatabase, true },
 	{ T_SecLabelStmt, NO_DIST_OBJECT_OPERATION,
-	  NonMainDbCheckSupportedObjectTypeForSecLabel },
+	  NonMainDbCheckSupportedObjectTypeForSecLabel, false },
 };
 
 
 static bool IsStatementSupportedFromNonMainDb(Node *parsetree);
 static bool StatementRequiresMarkDistributedGloballyFromNonMainDb(Node *parsetree);
 static bool StatementRequiresUnmarkDistributedLocallyFromNonMainDb(Node *parsetree);
+static bool StatementCannotBeExecutedInTransaction(Node *parsetree);
 static void MarkObjectDistributedGloballyFromNonMainDb(Node *parsetree);
 static void UnMarkObjectDistributedLocallyFromNonMainDb(List *unmarkDistributedList);
 static List * GetDistObjectOperationParams(Node *parsetree);
 
 
-/*
- * IsCommandToCreateOrDropMainDB checks if this query creates or drops the
- * main database, so we can make an exception and not send this query to
- * the main database.
- */
-bool
-IsCommandToCreateOrDropMainDB(Node *parsetree)
-{
-	if (IsA(parsetree, CreatedbStmt))
-	{
-		CreatedbStmt *createdbStmt = castNode(CreatedbStmt, parsetree);
-		return strcmp(createdbStmt->dbname, MainDb) == 0;
-	}
-	else if (IsA(parsetree, DropdbStmt))
-	{
-		DropdbStmt *dropdbStmt = castNode(DropdbStmt, parsetree);
-		return strcmp(dropdbStmt->dbname, MainDb) == 0;
-	}
-
-	return false;
-}
-
-
 /*
  * RunPreprocessNonMainDBCommand runs the necessary commands for a query, in main
  * database before query is run on the local node with PrevProcessUtility.
@@ -157,25 +143,15 @@ RunPreprocessNonMainDBCommand(Node *parsetree)
 		return false;
 	}
 
-	if (IsCommandToCreateOrDropMainDB(parsetree))
-	{
-		/*
-		 * We don't try to send the query to the main database if the CREATE/DROP DATABASE
-		 * command is for the main database itself, this is a very rare case but it's
-		 * exercised by our test suite.
-		 */
-		return false;
-	}
-
 	char *queryString = DeparseTreeNode(parsetree);
 
-	if (IsA(parsetree, CreatedbStmt) || IsA(parsetree, DropdbStmt))
+	/*
+	 * For the commands that cannot be executed in a transaction, there are no
+	 * transactional visibility issues. We directly route them to main database
+	 * so that we only have to consider one code-path when creating databases.
+	 */
+	if (StatementCannotBeExecutedInTransaction(parsetree))
 	{
-		/*
-		 * We always execute CREATE/DROP DATABASE from the main database. There are no
-		 * transactional visibility issues, since these commands are non-transactional.
-		 * And this way we only have to consider one codepath when creating databases.
-		 */
 		IsMainDBCommandInXact = false;
 		RunCitusMainDBQuery((char *) queryString);
 		return true;
@@ -264,7 +240,7 @@ StatementRequiresMarkDistributedGloballyFromNonMainDb(Node *parsetree)
 	{
 		if (type == NonMainDbSupportedStatements[i].statementType)
 		{
-			return NonMainDbSupportedStatements[i].DistObjectOperation ==
+			return NonMainDbSupportedStatements[i].distObjectOperation ==
 				   MARK_DISTRIBUTED_GLOBALLY;
 		}
 	}
@@ -287,7 +263,7 @@ StatementRequiresUnmarkDistributedLocallyFromNonMainDb(Node *parsetree)
 	{
 		if (type == NonMainDbSupportedStatements[i].statementType)
 		{
-			return NonMainDbSupportedStatements[i].DistObjectOperation ==
+			return NonMainDbSupportedStatements[i].distObjectOperation ==
 				   UNMARK_DISTRIBUTED_LOCALLY;
 		}
 	}
@@ -296,6 +272,28 @@ StatementRequiresUnmarkDistributedLocallyFromNonMainDb(Node *parsetree)
 }
 
 
+/*
+ * StatementCannotBeExecutedInTransaction returns true if the statement cannot be executed in a
+ * transaction.
+ */
+static bool
+StatementCannotBeExecutedInTransaction(Node *parsetree)
+{
+	NodeTag type = nodeTag(parsetree);
+
+	for (int i = 0; i < sizeof(NonMainDbSupportedStatements) /
+		 sizeof(NonMainDbSupportedStatements[0]); i++)
+	{
+		if (type == NonMainDbSupportedStatements[i].statementType)
+		{
+			return NonMainDbSupportedStatements[i].cannotBeExecutedInTransaction;
+		}
+	}
+
+	return false;
+}
+
+
 /*
  * MarkObjectDistributedGloballyFromNonMainDb marks the given object as distributed on the
  * non-main database.
@@ -396,6 +394,38 @@ GetDistObjectOperationParams(Node *parsetree)
 }
 
 
+/*
+ * NonMainDbCheckSupportedObjectTypeForCreateDatabase implements checkSupportedObjectTypes
+ * callback for CreatedbStmt.
+ *
+ * We don't try to send the query to the main database if the CREATE DATABASE
+ * command is for the main database itself, this is a very rare case but it's
+ * exercised by our test suite.
+ */
+static bool
+NonMainDbCheckSupportedObjectTypeForCreateDatabase(Node *node)
+{
+	CreatedbStmt *stmt = castNode(CreatedbStmt, node);
+	return strcmp(stmt->dbname, MainDb) != 0;
+}
+
+
+/*
+ * NonMainDbCheckSupportedObjectTypeForDropDatabase implements checkSupportedObjectTypes
+ * callback for DropdbStmt.
+ *
+ * We don't try to send the query to the main database if the DROP DATABASE
+ * command is for the main database itself, this is a very rare case but it's
+ * exercised by our test suite.
+ */
+static bool
+NonMainDbCheckSupportedObjectTypeForDropDatabase(Node *node)
+{
+	DropdbStmt *stmt = castNode(DropdbStmt, node);
+	return strcmp(stmt->dbname, MainDb) != 0;
+}
+
+
 /*
  * NonMainDbCheckSupportedObjectTypeForGrant implements checkSupportedObjectTypes
  * callback for GrantStmt.
diff --git a/src/include/distributed/commands.h b/src/include/distributed/commands.h
index e9468521f..084308a8f 100644
--- a/src/include/distributed/commands.h
+++ b/src/include/distributed/commands.h
@@ -105,7 +105,6 @@ typedef struct DistributeObjectOps
 const DistributeObjectOps * GetDistributeObjectOps(Node *node);
 
 /* functions to support node-wide object management commands from non-main dbs */
-extern bool IsCommandToCreateOrDropMainDB(Node *parsetree);
 extern bool RunPreprocessNonMainDBCommand(Node *parsetree);
 extern void RunPostprocessNonMainDBCommand(Node *parsetree);
 

From cd20ab76c0d7ac51fa54a98819a9536296339c5c Mon Sep 17 00:00:00 2001
From: Onur Tirtir <onurcantirtir@gmail.com>
Date: Thu, 29 Feb 2024 00:50:42 +0300
Subject: [PATCH 05/78] phase4: improve the infra and add nice comments

---
 .../non_main_db_distribute_object_ops.c       | 563 +++++++++---------
 1 file changed, 282 insertions(+), 281 deletions(-)

diff --git a/src/backend/distributed/commands/non_main_db_distribute_object_ops.c b/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
index cb0d7ddfc..c144b7216 100644
--- a/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
+++ b/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
@@ -5,6 +5,15 @@
  *    Routines to support node-wide object management commands from non-main
  *    databases.
  *
+ *    RunPreprocessNonMainDBCommand and RunPostprocessNonMainDBCommand are
+ *    the entrypoints for this module. These functions are called from
+ *    utility_hook.c to support some of the node-wide object management
+ *    commands from non-main databases.
+ *
+ *    To add support for a new command type, one needs to define a new
+ *    NonMainDbDistributeObjectOps object and add it to
+ *    GetNonMainDbDistributeObjectOps.
+ *
  *-------------------------------------------------------------------------
  */
 
@@ -30,102 +39,133 @@
 #define MARK_OBJECT_DISTRIBUTED \
 	"SELECT citus_internal.mark_object_distributed(%d, %s, %d, %s)"
 #define UNMARK_OBJECT_DISTRIBUTED \
-	"SELECT pg_catalog.citus_unmark_object_distributed(%d, %d, %d,%s)"
+	"SELECT pg_catalog.citus_unmark_object_distributed(%d, %d, %d, %s)"
 
 
-/* see NonMainDbDistributedStatementInfo for the explanation of these flags */
-typedef enum DistObjectOperation
-{
-	NO_DIST_OBJECT_OPERATION,
-	MARK_DISTRIBUTED_GLOBALLY,
-	UNMARK_DISTRIBUTED_LOCALLY
-} DistObjectOperation;
-
 /*
- * NonMainDbDistributedStatementInfo is used to determine whether a statement is
- * supported from non-main databases and whether it should be marked or unmarked
- * as distributed.
+ * Structs used to implement getMarkDistributedParams and
+ * getUnmarkDistributedParams callbacks for NonMainDbDistributeObjectOps.
  *
- * When creating a distributed object, we always have to mark such objects as
- * "distributed" but while for some object types we can delegate this to main
- * database, for some others we have to explicitly send a command to all nodes
- * in this code-path to achieve this. Callers need to provide
- * MARK_DISTRIBUTED_GLOBALLY when that is the case.
- *
- * Similarly when dropping a distributed object, we always have to unmark such
- * objects as "distributed" and our utility hook on remote nodes achieve this
- * via UnmarkNodeWideObjectsDistributed() because the commands that we send to
- * workers are executed via main db. However for the local node, this is not the
- * case as we're not in the main db. For this reason, callers need to provide
- * UNMARK_DISTRIBUTED_LOCALLY to unmark an object for local node as well.
+ * Main difference between these two is that while
+ * MarkDistributedGloballyParams contains the name of the object, the other
+ * doesn't. This is because, when marking an object -that is created from a
+ * non-main db- as distributed, citus_internal.mark_object_distributed()
+ * cannot find its name since the object is not visible to outer transaction
+ * (or, read as "the transaction in non-main db").
  */
-typedef struct NonMainDbDistributedStatementInfo
-{
-	int statementType;
-	DistObjectOperation distObjectOperation;
-
-	/*
-	 * checkSupportedObjectTypes is a callback function that checks whether
-	 * type of the object referred to by given statement is supported.
-	 *
-	 * Can be NULL if not applicable for the statement type.
-	 */
-	bool (*checkSupportedObjectTypes)(Node *node);
-
-	/* indicates whether the statement cannot be executed in a transaction */
-	bool cannotBeExecutedInTransaction;
-} NonMainDbDistributedStatementInfo;
-
-/*
- * DistObjectOperationParams is used to pass parameters to the
- * MarkObjectDistributedGloballyFromNonMainDb function and
- * UnMarkObjectDistributedLocallyFromNonMainDb functions.
- */
-typedef struct DistObjectOperationParams
+typedef struct MarkDistributedGloballyParams
 {
 	char *name;
 	Oid id;
 	uint16 catalogRelId;
-} DistObjectOperationParams;
+} MarkDistributedGloballyParams;
+
+typedef struct UnmarkDistributedLocallyParams
+{
+	Oid id;
+	uint16 catalogRelId;
+} UnmarkDistributedLocallyParams;
+
+/*
+ * NonMainDbDistributeObjectOps contains the necessary callbacks / flags to
+ * support node-wide object management commands from non-main databases.
+ *
+ * getMarkDistributedParams / getUnmarkDistributedParams:
+ *   When creating a distributed object, we always have to mark such objects
+ *   as "distributed" but while for some object types we can delegate this to
+ *   main database, for some others we have to explicitly send a command to
+ *   all nodes in this code-path to achieve this. Callers need to implement
+ *   getMarkDistributedParams when that is the case.
+ *
+ *   Similarly when dropping a distributed object, we always have to unmark
+ *   the target object as "distributed" and our utility hook on remote nodes
+ *   achieve this via UnmarkNodeWideObjectsDistributed() because the commands
+ *   that we send to workers are executed via main db. However for the local
+ *   node, this is not the case as we're not in the main db. For this reason,
+ *   callers need to implement getUnmarkDistributedParams to unmark an object
+ *   for local node as well.
+ *
+ *   We don't expect both of these to be NULL at the same time. However, it's
+ *   okay if both of these are NULL.
+ *
+ *   Finally, while getMarkDistributedParams is expected to return "a list of
+ *   objects", this is not the case for getUnmarkDistributedParams. This is
+ *   because, while we expect that a drop command might drop multiple objects
+ *   at once, we don't expect a create command to create multiple objects at
+ *   once.
+ *
+ *  cannotBeExecutedInTransaction:
+ *   Indicates whether the statement cannot be executed in a transaction. If
+ *   this is set to true, the statement will be executed directly on the main
+ *   database because there are no transactional visibility issues for such
+ *   commands.
+ *
+ *   And when this is true, we don't expect getMarkDistributedParams and
+ *   getUnmarkDistributedParams to be implemented.
+ */
+typedef struct NonMainDbDistributeObjectOps
+{
+	MarkDistributedGloballyParams * (*getMarkDistributedParams)(Node * parsetree);
+	List * (*getUnmarkDistributedParams)(Node * parsetree);
+	bool cannotBeExecutedInTransaction;
+} NonMainDbDistributeObjectOps;
 
 
 /*
- * checkSupportedObjectTypes callbacks for
- * NonMainDbDistributedStatementInfo objects.
+ * getMarkDistributedParams and getUnmarkDistributedParams callbacks for
+ * NonMainDbDistributeObjectOps.
  */
-static bool NonMainDbCheckSupportedObjectTypeForCreateDatabase(Node *node);
-static bool NonMainDbCheckSupportedObjectTypeForDropDatabase(Node *node);
-static bool NonMainDbCheckSupportedObjectTypeForGrant(Node *node);
-static bool NonMainDbCheckSupportedObjectTypeForSecLabel(Node *node);
+static MarkDistributedGloballyParams * CreateRoleStmtGetMarkDistributedParams(
+	Node *parsetree);
+static List * DropRoleStmtGetUnmarkDistributedParams(Node *parsetree);
 
-/*
- * NonMainDbSupportedStatements is an array of statements that are supported
- * from non-main databases.
- */
-ObjectType supportedObjectTypesForGrantStmt[] = { OBJECT_DATABASE };
-static const NonMainDbDistributedStatementInfo NonMainDbSupportedStatements[] = {
-	{ T_GrantRoleStmt, NO_DIST_OBJECT_OPERATION, NULL, false },
-	{ T_CreateRoleStmt, MARK_DISTRIBUTED_GLOBALLY, NULL, false },
-	{ T_DropRoleStmt, UNMARK_DISTRIBUTED_LOCALLY, NULL, false },
-	{ T_AlterRoleStmt, NO_DIST_OBJECT_OPERATION, NULL, false },
-	{ T_GrantStmt, NO_DIST_OBJECT_OPERATION,
-	  NonMainDbCheckSupportedObjectTypeForGrant, false },
-	{ T_CreatedbStmt, NO_DIST_OBJECT_OPERATION,
-	  NonMainDbCheckSupportedObjectTypeForCreateDatabase, true },
-	{ T_DropdbStmt, NO_DIST_OBJECT_OPERATION,
-	  NonMainDbCheckSupportedObjectTypeForDropDatabase, true },
-	{ T_SecLabelStmt, NO_DIST_OBJECT_OPERATION,
-	  NonMainDbCheckSupportedObjectTypeForSecLabel, false },
+
+/* NonMainDbDistributeObjectOps for different command types */
+static NonMainDbDistributeObjectOps Any_CreateRole = {
+	.getMarkDistributedParams = CreateRoleStmtGetMarkDistributedParams,
+	.getUnmarkDistributedParams = NULL,
+	.cannotBeExecutedInTransaction = false
+};
+static NonMainDbDistributeObjectOps Any_DropRole = {
+	.getMarkDistributedParams = NULL,
+	.getUnmarkDistributedParams = DropRoleStmtGetUnmarkDistributedParams,
+	.cannotBeExecutedInTransaction = false
+};
+static NonMainDbDistributeObjectOps Any_AlterRole = {
+	.getMarkDistributedParams = NULL,
+	.getUnmarkDistributedParams = NULL,
+	.cannotBeExecutedInTransaction = false
+};
+static NonMainDbDistributeObjectOps Any_GrantRole = {
+	.cannotBeExecutedInTransaction = false
+};
+static NonMainDbDistributeObjectOps Database_Create = {
+	.getMarkDistributedParams = NULL,
+	.getUnmarkDistributedParams = NULL,
+	.cannotBeExecutedInTransaction = true
+};
+static NonMainDbDistributeObjectOps Database_Drop = {
+	.getMarkDistributedParams = NULL,
+	.getUnmarkDistributedParams = NULL,
+	.cannotBeExecutedInTransaction = true
+};
+static NonMainDbDistributeObjectOps Database_Grant = {
+	.getMarkDistributedParams = NULL,
+	.getUnmarkDistributedParams = NULL,
+	.cannotBeExecutedInTransaction = false
+};
+static NonMainDbDistributeObjectOps Role_SecLabel = {
+	.getMarkDistributedParams = NULL,
+	.getUnmarkDistributedParams = NULL,
+	.cannotBeExecutedInTransaction = false
 };
 
 
-static bool IsStatementSupportedFromNonMainDb(Node *parsetree);
-static bool StatementRequiresMarkDistributedGloballyFromNonMainDb(Node *parsetree);
-static bool StatementRequiresUnmarkDistributedLocallyFromNonMainDb(Node *parsetree);
-static bool StatementCannotBeExecutedInTransaction(Node *parsetree);
-static void MarkObjectDistributedGloballyFromNonMainDb(Node *parsetree);
-static void UnMarkObjectDistributedLocallyFromNonMainDb(List *unmarkDistributedList);
-static List * GetDistObjectOperationParams(Node *parsetree);
+/* other static function declarations */
+const NonMainDbDistributeObjectOps * GetNonMainDbDistributeObjectOps(Node *parsetree);
+static void MarkObjectDistributedGloballyOnMainDbs(
+	MarkDistributedGloballyParams *markDistributedParams);
+static void UnmarkObjectDistributedOnLocalMainDb(List *unmarkDistributedList);
 
 
 /*
@@ -138,7 +178,13 @@ static List * GetDistObjectOperationParams(Node *parsetree);
 bool
 RunPreprocessNonMainDBCommand(Node *parsetree)
 {
-	if (IsMainDB || !IsStatementSupportedFromNonMainDb(parsetree))
+	if (IsMainDB)
+	{
+		return false;
+	}
+
+	const NonMainDbDistributeObjectOps *ops = GetNonMainDbDistributeObjectOps(parsetree);
+	if (!ops)
 	{
 		return false;
 	}
@@ -148,9 +194,9 @@ RunPreprocessNonMainDBCommand(Node *parsetree)
 	/*
 	 * For the commands that cannot be executed in a transaction, there are no
 	 * transactional visibility issues. We directly route them to main database
-	 * so that we only have to consider one code-path when creating databases.
+	 * so that we only have to consider one code-path for such commands.
 	 */
-	if (StatementCannotBeExecutedInTransaction(parsetree))
+	if (ops->cannotBeExecutedInTransaction)
 	{
 		IsMainDBCommandInXact = false;
 		RunCitusMainDBQuery((char *) queryString);
@@ -172,10 +218,10 @@ RunPreprocessNonMainDBCommand(Node *parsetree)
 					 quote_literal_cstr(CurrentUserName()));
 	RunCitusMainDBQuery(mainDBQuery->data);
 
-	if (StatementRequiresUnmarkDistributedLocallyFromNonMainDb(parsetree))
+	if (ops->getUnmarkDistributedParams)
 	{
-		List *unmarkParams = GetDistObjectOperationParams(parsetree);
-		UnMarkObjectDistributedLocallyFromNonMainDb(unmarkParams);
+		List *unmarkDistributedParamsList = ops->getUnmarkDistributedParams(parsetree);
+		UnmarkObjectDistributedOnLocalMainDb(unmarkDistributedParamsList);
 	}
 
 	return false;
@@ -189,154 +235,165 @@ RunPreprocessNonMainDBCommand(Node *parsetree)
 void
 RunPostprocessNonMainDBCommand(Node *parsetree)
 {
-	if (IsMainDB || !IsStatementSupportedFromNonMainDb(parsetree))
+	if (IsMainDB)
 	{
 		return;
 	}
 
-	if (StatementRequiresMarkDistributedGloballyFromNonMainDb(parsetree))
+	const NonMainDbDistributeObjectOps *ops = GetNonMainDbDistributeObjectOps(parsetree);
+	if (!ops)
 	{
-		MarkObjectDistributedGloballyFromNonMainDb(parsetree);
+		return;
+	}
+
+	if (ops->getMarkDistributedParams)
+	{
+		MarkDistributedGloballyParams *markDistributedParams =
+			ops->getMarkDistributedParams(parsetree);
+		MarkObjectDistributedGloballyOnMainDbs(markDistributedParams);
 	}
 }
 
 
 /*
- * IsStatementSupportedFromNonMainDb returns true if the statement is supported from a
- * non-main database.
+ * GetNonMainDbDistributeObjectOps returns the NonMainDbDistributeObjectOps for given
+ * command if it's node-wide object management command that's supported from non-main
+ * databases.
  */
-static bool
-IsStatementSupportedFromNonMainDb(Node *parsetree)
+const NonMainDbDistributeObjectOps *
+GetNonMainDbDistributeObjectOps(Node *parsetree)
 {
-	NodeTag type = nodeTag(parsetree);
-
-	for (int i = 0; i < sizeof(NonMainDbSupportedStatements) /
-		 sizeof(NonMainDbSupportedStatements[0]); i++)
+	switch (nodeTag(parsetree))
 	{
-		if (type != NonMainDbSupportedStatements[i].statementType)
+		case T_CreateRoleStmt:
 		{
-			continue;
+			return &Any_CreateRole;
 		}
 
-		return !NonMainDbSupportedStatements[i].checkSupportedObjectTypes ||
-			   NonMainDbSupportedStatements[i].checkSupportedObjectTypes(parsetree);
-	}
-
-	return false;
-}
-
-
-/*
- * StatementRequiresMarkDistributedGloballyFromNonMainDb returns true if the statement should be marked
- * as distributed when executed from a non-main database.
- */
-static bool
-StatementRequiresMarkDistributedGloballyFromNonMainDb(Node *parsetree)
-{
-	NodeTag type = nodeTag(parsetree);
-
-	for (int i = 0; i < sizeof(NonMainDbSupportedStatements) /
-		 sizeof(NonMainDbSupportedStatements[0]); i++)
-	{
-		if (type == NonMainDbSupportedStatements[i].statementType)
+		case T_DropRoleStmt:
 		{
-			return NonMainDbSupportedStatements[i].distObjectOperation ==
-				   MARK_DISTRIBUTED_GLOBALLY;
+			return &Any_DropRole;
 		}
-	}
 
-	return false;
-}
-
-
-/*
- * StatementRequiresUnmarkDistributedLocallyFromNonMainDb returns true if the statement should be unmarked
- * as distributed when executed from a non-main database.
- */
-static bool
-StatementRequiresUnmarkDistributedLocallyFromNonMainDb(Node *parsetree)
-{
-	NodeTag type = nodeTag(parsetree);
-
-	for (int i = 0; i < sizeof(NonMainDbSupportedStatements) /
-		 sizeof(NonMainDbSupportedStatements[0]); i++)
-	{
-		if (type == NonMainDbSupportedStatements[i].statementType)
+		case T_AlterRoleStmt:
 		{
-			return NonMainDbSupportedStatements[i].distObjectOperation ==
-				   UNMARK_DISTRIBUTED_LOCALLY;
+			return &Any_AlterRole;
 		}
-	}
 
-	return false;
-}
-
-
-/*
- * StatementCannotBeExecutedInTransaction returns true if the statement cannot be executed in a
- * transaction.
- */
-static bool
-StatementCannotBeExecutedInTransaction(Node *parsetree)
-{
-	NodeTag type = nodeTag(parsetree);
-
-	for (int i = 0; i < sizeof(NonMainDbSupportedStatements) /
-		 sizeof(NonMainDbSupportedStatements[0]); i++)
-	{
-		if (type == NonMainDbSupportedStatements[i].statementType)
+		case T_GrantRoleStmt:
 		{
-			return NonMainDbSupportedStatements[i].cannotBeExecutedInTransaction;
+			return &Any_GrantRole;
 		}
-	}
 
-	return false;
+		case T_CreatedbStmt:
+		{
+			CreatedbStmt *stmt = castNode(CreatedbStmt, parsetree);
+
+			/*
+			 * We don't try to send the query to the main database if the CREATE
+			 * DATABASE command is for the main database itself, this is a very
+			 * rare case but it's exercised by our test suite.
+			 */
+			if (strcmp(stmt->dbname, MainDb) != 0)
+			{
+				return &Database_Create;
+			}
+
+			return NULL;
+		}
+
+		case T_DropdbStmt:
+		{
+			DropdbStmt *stmt = castNode(DropdbStmt, parsetree);
+
+			/*
+			 * We don't try to send the query to the main database if the DROP
+			 * DATABASE command is for the main database itself, this is a very
+			 * rare case but it's exercised by our test suite.
+			 */
+			if (strcmp(stmt->dbname, MainDb) != 0)
+			{
+				return &Database_Drop;
+			}
+
+			return NULL;
+		}
+
+		case T_GrantStmt:
+		{
+			GrantStmt *stmt = castNode(GrantStmt, parsetree);
+
+			switch (stmt->objtype)
+			{
+				case OBJECT_DATABASE:
+				{
+					return &Database_Grant;
+				}
+
+				default:
+					return NULL;
+			}
+		}
+
+		case T_SecLabelStmt:
+		{
+			SecLabelStmt *stmt = castNode(SecLabelStmt, parsetree);
+
+			switch (stmt->objtype)
+			{
+				case OBJECT_ROLE:
+				{
+					return &Role_SecLabel;
+				}
+
+				default:
+					return NULL;
+			}
+		}
+
+		default:
+			return NULL;
+	}
 }
 
 
 /*
- * MarkObjectDistributedGloballyFromNonMainDb marks the given object as distributed on the
- * non-main database.
+ * MarkObjectDistributedGloballyOnMainDbs marks an object as
+ * distributed on all main databases globally.
  */
 static void
-MarkObjectDistributedGloballyFromNonMainDb(Node *parsetree)
+MarkObjectDistributedGloballyOnMainDbs(
+	MarkDistributedGloballyParams *markDistributedParams)
 {
-	List *distObjectOperationParams =
-		GetDistObjectOperationParams(parsetree);
-
-	DistObjectOperationParams *distObjectOperationParam = NULL;
-
-	foreach_ptr(distObjectOperationParam, distObjectOperationParams)
-	{
-		StringInfo mainDBQuery = makeStringInfo();
-		appendStringInfo(mainDBQuery,
-						 MARK_OBJECT_DISTRIBUTED,
-						 distObjectOperationParam->catalogRelId,
-						 quote_literal_cstr(distObjectOperationParam->name),
-						 distObjectOperationParam->id,
-						 quote_literal_cstr(CurrentUserName()));
-		RunCitusMainDBQuery(mainDBQuery->data);
-	}
+	StringInfo mainDBQuery = makeStringInfo();
+	appendStringInfo(mainDBQuery,
+					 MARK_OBJECT_DISTRIBUTED,
+					 markDistributedParams->catalogRelId,
+					 quote_literal_cstr(markDistributedParams->name),
+					 markDistributedParams->id,
+					 quote_literal_cstr(CurrentUserName()));
+	RunCitusMainDBQuery(mainDBQuery->data);
 }
 
 
 /*
- * UnMarkObjectDistributedLocallyFromNonMainDb unmarks the given object as distributed on the
- * non-main database.
+ * UnmarkObjectDistributedOnLocalMainDb unmarks a list of objects as
+ * distributed on the local main database.
  */
 static void
-UnMarkObjectDistributedLocallyFromNonMainDb(List *markObjectDistributedParamList)
+UnmarkObjectDistributedOnLocalMainDb(List *unmarkDistributedParamsList)
 {
-	DistObjectOperationParams *markObjectDistributedParam = NULL;
 	int subObjectId = 0;
 	char *checkObjectExistence = "false";
-	foreach_ptr(markObjectDistributedParam, markObjectDistributedParamList)
+
+	UnmarkDistributedLocallyParams *unmarkDistributedParams = NULL;
+	foreach_ptr(unmarkDistributedParams, unmarkDistributedParamsList)
 	{
 		StringInfo query = makeStringInfo();
 		appendStringInfo(query,
 						 UNMARK_OBJECT_DISTRIBUTED,
-						 AuthIdRelationId,
-						 markObjectDistributedParam->id,
+						 unmarkDistributedParams->catalogRelId,
+						 unmarkDistributedParams->id,
 						 subObjectId, checkObjectExistence);
 		RunCitusMainDBQuery(query->data);
 	}
@@ -344,107 +401,51 @@ UnMarkObjectDistributedLocallyFromNonMainDb(List *markObjectDistributedParamList
 
 
 /*
- * GetDistObjectOperationParams returns DistObjectOperationParams for the target
- * object of given parsetree.
+ * getMarkDistributedParams and getUnmarkDistributedParams callback implementations
+ * for NonMainDbDistributeObjectOps start here.
  */
-List *
-GetDistObjectOperationParams(Node *parsetree)
+static MarkDistributedGloballyParams *
+CreateRoleStmtGetMarkDistributedParams(Node *parsetree)
 {
+	CreateRoleStmt *stmt = castNode(CreateRoleStmt, parsetree);
+	MarkDistributedGloballyParams *params =
+		(MarkDistributedGloballyParams *) palloc(sizeof(MarkDistributedGloballyParams));
+	params->name = stmt->role;
+	params->catalogRelId = AuthIdRelationId;
+
+	/* object must exist as we've just created it */
+	bool missingOk = false;
+	params->id = get_role_oid(stmt->role, missingOk);
+
+	return params;
+}
+
+
+static List *
+DropRoleStmtGetUnmarkDistributedParams(Node *parsetree)
+{
+	DropRoleStmt *stmt = castNode(DropRoleStmt, parsetree);
+
 	List *paramsList = NIL;
-	if (IsA(parsetree, CreateRoleStmt))
+
+	RoleSpec *roleSpec = NULL;
+	foreach_ptr(roleSpec, stmt->roles)
 	{
-		CreateRoleStmt *stmt = castNode(CreateRoleStmt, parsetree);
-		DistObjectOperationParams *params =
-			(DistObjectOperationParams *) palloc(sizeof(DistObjectOperationParams));
-		params->name = stmt->role;
+		UnmarkDistributedLocallyParams *params =
+			(UnmarkDistributedLocallyParams *) palloc(
+				sizeof(UnmarkDistributedLocallyParams));
+
+		Oid roleOid = get_role_oid(roleSpec->rolename, stmt->missing_ok);
+		if (roleOid == InvalidOid)
+		{
+			continue;
+		}
+
+		params->id = roleOid;
 		params->catalogRelId = AuthIdRelationId;
-		params->id = get_role_oid(stmt->role, false);
 
 		paramsList = lappend(paramsList, params);
 	}
-	else if (IsA(parsetree, DropRoleStmt))
-	{
-		DropRoleStmt *stmt = castNode(DropRoleStmt, parsetree);
-		RoleSpec *roleSpec;
-		foreach_ptr(roleSpec, stmt->roles)
-		{
-			DistObjectOperationParams *params = (DistObjectOperationParams *) palloc(
-				sizeof(DistObjectOperationParams));
-
-			Oid roleOid = get_role_oid(roleSpec->rolename, true);
-
-			if (roleOid == InvalidOid)
-			{
-				continue;
-			}
-
-			params->id = roleOid;
-			params->name = roleSpec->rolename;
-			params->catalogRelId = AuthIdRelationId;
-
-			paramsList = lappend(paramsList, params);
-		}
-	}
-	else
-	{
-		elog(ERROR, "unsupported statement type");
-	}
 
 	return paramsList;
 }
-
-
-/*
- * NonMainDbCheckSupportedObjectTypeForCreateDatabase implements checkSupportedObjectTypes
- * callback for CreatedbStmt.
- *
- * We don't try to send the query to the main database if the CREATE DATABASE
- * command is for the main database itself, this is a very rare case but it's
- * exercised by our test suite.
- */
-static bool
-NonMainDbCheckSupportedObjectTypeForCreateDatabase(Node *node)
-{
-	CreatedbStmt *stmt = castNode(CreatedbStmt, node);
-	return strcmp(stmt->dbname, MainDb) != 0;
-}
-
-
-/*
- * NonMainDbCheckSupportedObjectTypeForDropDatabase implements checkSupportedObjectTypes
- * callback for DropdbStmt.
- *
- * We don't try to send the query to the main database if the DROP DATABASE
- * command is for the main database itself, this is a very rare case but it's
- * exercised by our test suite.
- */
-static bool
-NonMainDbCheckSupportedObjectTypeForDropDatabase(Node *node)
-{
-	DropdbStmt *stmt = castNode(DropdbStmt, node);
-	return strcmp(stmt->dbname, MainDb) != 0;
-}
-
-
-/*
- * NonMainDbCheckSupportedObjectTypeForGrant implements checkSupportedObjectTypes
- * callback for GrantStmt.
- */
-static bool
-NonMainDbCheckSupportedObjectTypeForGrant(Node *node)
-{
-	GrantStmt *stmt = castNode(GrantStmt, node);
-	return stmt->objtype == OBJECT_DATABASE;
-}
-
-
-/*
- * NonMainDbCheckSupportedObjectTypeForSecLabel implements checkSupportedObjectTypes
- * callback for SecLabel.
- */
-static bool
-NonMainDbCheckSupportedObjectTypeForSecLabel(Node *node)
-{
-	SecLabelStmt *stmt = castNode(SecLabelStmt, node);
-	return stmt->objtype == OBJECT_ROLE;
-}

From d9d9519300e6d120107c74f4772d20e02d9597e1 Mon Sep 17 00:00:00 2001
From: Onur Tirtir <onurcantirtir@gmail.com>
Date: Thu, 29 Feb 2024 01:13:56 +0300
Subject: [PATCH 06/78] rename

---
 .../commands/non_main_db_distribute_object_ops.c          | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/backend/distributed/commands/non_main_db_distribute_object_ops.c b/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
index c144b7216..a64fde0bc 100644
--- a/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
+++ b/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
@@ -165,7 +165,7 @@ static NonMainDbDistributeObjectOps Role_SecLabel = {
 const NonMainDbDistributeObjectOps * GetNonMainDbDistributeObjectOps(Node *parsetree);
 static void MarkObjectDistributedGloballyOnMainDbs(
 	MarkDistributedGloballyParams *markDistributedParams);
-static void UnmarkObjectDistributedOnLocalMainDb(List *unmarkDistributedList);
+static void UnmarkObjectsDistributedOnLocalMainDb(List *unmarkDistributedList);
 
 
 /*
@@ -221,7 +221,7 @@ RunPreprocessNonMainDBCommand(Node *parsetree)
 	if (ops->getUnmarkDistributedParams)
 	{
 		List *unmarkDistributedParamsList = ops->getUnmarkDistributedParams(parsetree);
-		UnmarkObjectDistributedOnLocalMainDb(unmarkDistributedParamsList);
+		UnmarkObjectsDistributedOnLocalMainDb(unmarkDistributedParamsList);
 	}
 
 	return false;
@@ -377,11 +377,11 @@ MarkObjectDistributedGloballyOnMainDbs(
 
 
 /*
- * UnmarkObjectDistributedOnLocalMainDb unmarks a list of objects as
+ * UnmarkObjectsDistributedOnLocalMainDb unmarks a list of objects as
  * distributed on the local main database.
  */
 static void
-UnmarkObjectDistributedOnLocalMainDb(List *unmarkDistributedParamsList)
+UnmarkObjectsDistributedOnLocalMainDb(List *unmarkDistributedParamsList)
 {
 	int subObjectId = 0;
 	char *checkObjectExistence = "false";

From 80360e464baafcaab922a8dceb8a472d393c43ab Mon Sep 17 00:00:00 2001
From: Onur Tirtir <onurcantirtir@gmail.com>
Date: Thu, 29 Feb 2024 01:17:25 +0300
Subject: [PATCH 07/78] improve

---
 .../commands/non_main_db_distribute_object_ops.c | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/src/backend/distributed/commands/non_main_db_distribute_object_ops.c b/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
index a64fde0bc..c71052c3d 100644
--- a/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
+++ b/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
@@ -120,7 +120,11 @@ static MarkDistributedGloballyParams * CreateRoleStmtGetMarkDistributedParams(
 static List * DropRoleStmtGetUnmarkDistributedParams(Node *parsetree);
 
 
-/* NonMainDbDistributeObjectOps for different command types */
+/*
+ * NonMainDbDistributeObjectOps for different command types.
+ *
+ * Naming of these structs are stolen from distribute_object_ops.c.
+ */
 static NonMainDbDistributeObjectOps Any_CreateRole = {
 	.getMarkDistributedParams = CreateRoleStmtGetMarkDistributedParams,
 	.getUnmarkDistributedParams = NULL,
@@ -137,14 +141,16 @@ static NonMainDbDistributeObjectOps Any_AlterRole = {
 	.cannotBeExecutedInTransaction = false
 };
 static NonMainDbDistributeObjectOps Any_GrantRole = {
+	.getMarkDistributedParams = NULL,
+	.getUnmarkDistributedParams = NULL,
 	.cannotBeExecutedInTransaction = false
 };
-static NonMainDbDistributeObjectOps Database_Create = {
+static NonMainDbDistributeObjectOps Any_CreateDatabase = {
 	.getMarkDistributedParams = NULL,
 	.getUnmarkDistributedParams = NULL,
 	.cannotBeExecutedInTransaction = true
 };
-static NonMainDbDistributeObjectOps Database_Drop = {
+static NonMainDbDistributeObjectOps Any_DropDatabase = {
 	.getMarkDistributedParams = NULL,
 	.getUnmarkDistributedParams = NULL,
 	.cannotBeExecutedInTransaction = true
@@ -296,7 +302,7 @@ GetNonMainDbDistributeObjectOps(Node *parsetree)
 			 */
 			if (strcmp(stmt->dbname, MainDb) != 0)
 			{
-				return &Database_Create;
+				return &Any_CreateDatabase;
 			}
 
 			return NULL;
@@ -313,7 +319,7 @@ GetNonMainDbDistributeObjectOps(Node *parsetree)
 			 */
 			if (strcmp(stmt->dbname, MainDb) != 0)
 			{
-				return &Database_Drop;
+				return &Any_DropDatabase;
 			}
 
 			return NULL;

From b13d6763e773b03b2f82bd36e005d92d23b31317 Mon Sep 17 00:00:00 2001
From: Onur Tirtir <onurcantirtir@gmail.com>
Date: Thu, 29 Feb 2024 01:20:27 +0300
Subject: [PATCH 08/78] improve

---
 .../commands/non_main_db_distribute_object_ops.c            | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/backend/distributed/commands/non_main_db_distribute_object_ops.c b/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
index c71052c3d..021fcf478 100644
--- a/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
+++ b/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
@@ -123,7 +123,11 @@ static List * DropRoleStmtGetUnmarkDistributedParams(Node *parsetree);
 /*
  * NonMainDbDistributeObjectOps for different command types.
  *
- * Naming of these structs are stolen from distribute_object_ops.c.
+ * Naming of these structs are stolen from distribute_object_ops.c. We use
+ * "Any" if the command doesn't have any other variations or if the structs
+ * implements it for all variations. For example, while we name the struct
+ * for CreateRoleStmt as Any_CreateRole, we name the struct that implements
+ * SecLabelStmt for role objects as Role_SecLabel.
  */
 static NonMainDbDistributeObjectOps Any_CreateRole = {
 	.getMarkDistributedParams = CreateRoleStmtGetMarkDistributedParams,

From c4d4b42d8b17e8e319bc3dec0c792265e66f8193 Mon Sep 17 00:00:00 2001
From: Onur Tirtir <onurcantirtir@gmail.com>
Date: Thu, 29 Feb 2024 01:32:59 +0300
Subject: [PATCH 09/78] improve

---
 .../non_main_db_distribute_object_ops.c       | 22 +++++++++----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/src/backend/distributed/commands/non_main_db_distribute_object_ops.c b/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
index 021fcf478..3c21c7b4e 100644
--- a/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
+++ b/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
@@ -124,47 +124,47 @@ static List * DropRoleStmtGetUnmarkDistributedParams(Node *parsetree);
  * NonMainDbDistributeObjectOps for different command types.
  *
  * Naming of these structs are stolen from distribute_object_ops.c. We use
- * "Any" if the command doesn't have any other variations or if the structs
- * implements it for all variations. For example, while we name the struct
+ * "Any" if the command doesn't have any other variations or if the struct
+ * implements it for all its variations. For example, while we name the struct
  * for CreateRoleStmt as Any_CreateRole, we name the struct that implements
  * SecLabelStmt for role objects as Role_SecLabel.
  */
-static NonMainDbDistributeObjectOps Any_CreateRole = {
+static const NonMainDbDistributeObjectOps Any_CreateRole = {
 	.getMarkDistributedParams = CreateRoleStmtGetMarkDistributedParams,
 	.getUnmarkDistributedParams = NULL,
 	.cannotBeExecutedInTransaction = false
 };
-static NonMainDbDistributeObjectOps Any_DropRole = {
+static const NonMainDbDistributeObjectOps Any_DropRole = {
 	.getMarkDistributedParams = NULL,
 	.getUnmarkDistributedParams = DropRoleStmtGetUnmarkDistributedParams,
 	.cannotBeExecutedInTransaction = false
 };
-static NonMainDbDistributeObjectOps Any_AlterRole = {
+static const NonMainDbDistributeObjectOps Any_AlterRole = {
 	.getMarkDistributedParams = NULL,
 	.getUnmarkDistributedParams = NULL,
 	.cannotBeExecutedInTransaction = false
 };
-static NonMainDbDistributeObjectOps Any_GrantRole = {
+static const NonMainDbDistributeObjectOps Any_GrantRole = {
 	.getMarkDistributedParams = NULL,
 	.getUnmarkDistributedParams = NULL,
 	.cannotBeExecutedInTransaction = false
 };
-static NonMainDbDistributeObjectOps Any_CreateDatabase = {
+static const NonMainDbDistributeObjectOps Any_CreateDatabase = {
 	.getMarkDistributedParams = NULL,
 	.getUnmarkDistributedParams = NULL,
 	.cannotBeExecutedInTransaction = true
 };
-static NonMainDbDistributeObjectOps Any_DropDatabase = {
+static const NonMainDbDistributeObjectOps Any_DropDatabase = {
 	.getMarkDistributedParams = NULL,
 	.getUnmarkDistributedParams = NULL,
 	.cannotBeExecutedInTransaction = true
 };
-static NonMainDbDistributeObjectOps Database_Grant = {
+static const NonMainDbDistributeObjectOps Database_Grant = {
 	.getMarkDistributedParams = NULL,
 	.getUnmarkDistributedParams = NULL,
 	.cannotBeExecutedInTransaction = false
 };
-static NonMainDbDistributeObjectOps Role_SecLabel = {
+static const NonMainDbDistributeObjectOps Role_SecLabel = {
 	.getMarkDistributedParams = NULL,
 	.getUnmarkDistributedParams = NULL,
 	.cannotBeExecutedInTransaction = false
@@ -175,7 +175,7 @@ static NonMainDbDistributeObjectOps Role_SecLabel = {
 const NonMainDbDistributeObjectOps * GetNonMainDbDistributeObjectOps(Node *parsetree);
 static void MarkObjectDistributedGloballyOnMainDbs(
 	MarkDistributedGloballyParams *markDistributedParams);
-static void UnmarkObjectsDistributedOnLocalMainDb(List *unmarkDistributedList);
+static void UnmarkObjectsDistributedOnLocalMainDb(List *unmarkDistributedParamsList);
 
 
 /*

From 7ad5a2479b4429329188ce8c2da07a0fcfd8ddf6 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Mon, 18 Mar 2024 16:26:08 +0300
Subject: [PATCH 10/78] Moves code from the branch

---
 .../non_main_db_distribute_object_ops.c       |  48 ++++++++
 .../alter_database_from_nonmain_db.out        | 111 ++++++++++++++++++
 src/test/regress/expected/pg15.out            |  16 +++
 src/test/regress/multi_1_schedule             |   1 +
 .../sql/alter_database_from_nonmain_db.sql    |  64 ++++++++++
 src/test/regress/sql/pg15.sql                 |  18 +++
 6 files changed, 258 insertions(+)
 create mode 100644 src/test/regress/expected/alter_database_from_nonmain_db.out
 create mode 100644 src/test/regress/sql/alter_database_from_nonmain_db.sql

diff --git a/src/backend/distributed/commands/non_main_db_distribute_object_ops.c b/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
index 3c21c7b4e..2254d0d68 100644
--- a/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
+++ b/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
@@ -159,6 +159,25 @@ static const NonMainDbDistributeObjectOps Any_DropDatabase = {
 	.getUnmarkDistributedParams = NULL,
 	.cannotBeExecutedInTransaction = true
 };
+
+static const NonMainDbDistributeObjectOps Any_AlterDatabase = {
+	.getMarkDistributedParams = NULL,
+	.getUnmarkDistributedParams = NULL,
+	.cannotBeExecutedInTransaction = true
+};
+
+static const NonMainDbDistributeObjectOps Any_AlterDatabaseSet = {
+	.getMarkDistributedParams = NULL,
+	.getUnmarkDistributedParams = NULL,
+	.cannotBeExecutedInTransaction = true
+};
+
+static const NonMainDbDistributeObjectOps Any_AlterDatabaseRefreshColl = {
+	.getMarkDistributedParams = NULL,
+	.getUnmarkDistributedParams = NULL,
+	.cannotBeExecutedInTransaction = false
+};
+
 static const NonMainDbDistributeObjectOps Database_Grant = {
 	.getMarkDistributedParams = NULL,
 	.getUnmarkDistributedParams = NULL,
@@ -329,6 +348,35 @@ GetNonMainDbDistributeObjectOps(Node *parsetree)
 			return NULL;
 		}
 
+		case T_AlterDatabaseStmt:
+		{
+			AlterDatabaseStmt *stmt = castNode(AlterDatabaseStmt, parsetree);
+
+			/*
+			 * We don't try to send the query to the main database if the ALTER
+			 * DATABASE command is for the main database itself, this is a very
+			 * rare case but it's exercised by our test suite.
+			 */
+			if (strcmp(stmt->dbname, MainDb) != 0)
+			{
+				return &Any_AlterDatabase;
+			}
+
+			return NULL;
+		}
+
+		case T_AlterDatabaseSetStmt:
+		{
+			return &Any_AlterDatabaseSet;
+
+		}
+#if PG_VERSION_NUM >= PG_VERSION_15
+		case T_AlterDatabaseRefreshCollStmt:
+		{
+			return &Any_AlterDatabaseRefreshColl;
+		}
+#endif
+
 		case T_GrantStmt:
 		{
 			GrantStmt *stmt = castNode(GrantStmt, parsetree);
diff --git a/src/test/regress/expected/alter_database_from_nonmain_db.out b/src/test/regress/expected/alter_database_from_nonmain_db.out
new file mode 100644
index 000000000..763db4396
--- /dev/null
+++ b/src/test/regress/expected/alter_database_from_nonmain_db.out
@@ -0,0 +1,111 @@
+SET citus.superuser TO 'postgres';
+set citus.enable_create_database_propagation=on;
+create database test_alter_db_from_nonmain_db;
+create database altered_database;
+reset citus.enable_create_database_propagation;
+\c regression;
+set citus.enable_create_database_propagation=on;
+\c test_alter_db_from_nonmain_db
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
+alter database altered_database rename to altered_database_renamed;
+alter database altered_database_renamed rename to altered_database;
+alter database altered_database with
+    ALLOW_CONNECTIONS false
+    CONNECTION LIMIT 1
+    IS_TEMPLATE true;
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database WITH ALLOW_CONNECTIONS false CONNECTION LIMIT 1 IS_TEMPLATE true;', 'postgres')
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database altered_database with
+    ALLOW_CONNECTIONS true
+    CONNECTION LIMIT 0
+    IS_TEMPLATE false;
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database WITH ALLOW_CONNECTIONS true CONNECTION LIMIT 0 IS_TEMPLATE false;', 'postgres')
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+\c regression
+create role test_owner_non_main_db;
+\c test_alter_db_from_nonmain_db
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
+set citus.enable_create_database_propagation=on;
+alter database altered_database owner to test_owner_non_main_db;
+alter database altered_database owner to CURRENT_USER;
+alter database altered_database set default_transaction_read_only = true;
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database SET default_transaction_read_only = ''true''', 'postgres')
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+set default_transaction_read_only = false;
+alter database altered_database set default_transaction_read_only from current;
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database SET default_transaction_read_only FROM CURRENT', 'postgres')
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database altered_database set default_transaction_read_only to DEFAULT;
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database SET default_transaction_read_only TO DEFAULT', 'postgres')
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database altered_database RESET default_transaction_read_only;
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database RESET default_transaction_read_only', 'postgres')
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database altered_database SET TIME ZONE '-7';
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database SET timezone = ''-7''', 'postgres')
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database altered_database set TIME ZONE LOCAL;
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database SET timezone TO DEFAULT', 'postgres')
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database altered_database set TIME ZONE DEFAULT;
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database SET timezone TO DEFAULT', 'postgres')
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database altered_database RESET TIME ZONE;
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database RESET timezone', 'postgres')
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database altered_database SET TIME ZONE INTERVAL '-08:00' HOUR TO MINUTE;
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database SET TIME ZONE INTERVAL ''@ 8 hours ago''', 'postgres')
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database altered_database RESET TIME ZONE;
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database RESET timezone', 'postgres')
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database altered_database set default_transaction_isolation = 'serializable';
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database SET default_transaction_isolation = ''serializable''', 'postgres')
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+set default_transaction_isolation = 'read committed';
+alter database altered_database set default_transaction_isolation from current;
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database SET default_transaction_isolation FROM CURRENT', 'postgres')
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database altered_database set default_transaction_isolation to DEFAULT;
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database SET default_transaction_isolation TO DEFAULT', 'postgres')
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database altered_database RESET default_transaction_isolation;
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database RESET default_transaction_isolation', 'postgres')
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database altered_database set statement_timeout = 1000;
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database SET statement_timeout = 1000', 'postgres')
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+set statement_timeout = 2000;
+alter database altered_database set statement_timeout from current;
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database SET statement_timeout FROM CURRENT', 'postgres')
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database altered_database set statement_timeout to DEFAULT;
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database SET statement_timeout TO DEFAULT', 'postgres')
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database altered_database RESET statement_timeout;
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database RESET statement_timeout', 'postgres')
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database altered_database set lock_timeout = 1201.5;
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database SET lock_timeout = 1201.5', 'postgres')
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+set lock_timeout = 1202.5;
+alter database altered_database set lock_timeout from current;
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database SET lock_timeout FROM CURRENT', 'postgres')
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database altered_database set lock_timeout to DEFAULT;
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database SET lock_timeout TO DEFAULT', 'postgres')
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database altered_database RESET lock_timeout;
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database RESET lock_timeout', 'postgres')
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+ALTER DATABASE altered_database RESET ALL;
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database RESET ALL', 'postgres')
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+\c regression
+set citus.enable_create_database_propagation=on;
+drop database altered_database;
+drop database test_alter_db_from_nonmain_db;
+reset citus.enable_create_database_propagation;
+drop role test_owner_non_main_db;
diff --git a/src/test/regress/expected/pg15.out b/src/test/regress/expected/pg15.out
index eff8b0ce6..a5c69b704 100644
--- a/src/test/regress/expected/pg15.out
+++ b/src/test/regress/expected/pg15.out
@@ -1546,3 +1546,19 @@ SET client_min_messages TO ERROR;
 DROP SCHEMA pg15 CASCADE;
 DROP ROLE rls_tenant_1;
 DROP ROLE rls_tenant_2;
+-- test refresh collation version on non-main databases
+SET citus.enable_create_database_propagation TO on;
+create database alter_db_from_nonmain_db_pg15;
+\c alter_db_from_nonmain_db_pg15
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = '%ALTER DATABASE%';
+ALTER DATABASE alter_db_from_nonmain_db_pg15 REFRESH COLLATION VERSION;
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE alter_db_from_nonmain_db_pg15 REFRESH COLLATION VERSION;', 'postgres')
+NOTICE:  version has not changed
+reset citus.log_remote_commands;
+reset citus.grep_remote_commands;
+\c regression
+SET citus.enable_create_database_propagation TO on;
+drop database alter_db_from_nonmain_db_pg15;
+reset citus.enable_create_database_propagation;
+SET citus.enable_create_database_propagation TO OFF;
diff --git a/src/test/regress/multi_1_schedule b/src/test/regress/multi_1_schedule
index 015f74973..868d78cdf 100644
--- a/src/test/regress/multi_1_schedule
+++ b/src/test/regress/multi_1_schedule
@@ -63,6 +63,7 @@ test: alter_database_propagation
 
 test: citus_shards
 test: reassign_owned
+test: alter_database_from_nonmain_db
 
 # ----------
 # multi_citus_tools tests utility functions written for citus tools
diff --git a/src/test/regress/sql/alter_database_from_nonmain_db.sql b/src/test/regress/sql/alter_database_from_nonmain_db.sql
new file mode 100644
index 000000000..aa9a5d191
--- /dev/null
+++ b/src/test/regress/sql/alter_database_from_nonmain_db.sql
@@ -0,0 +1,64 @@
+SET citus.superuser TO 'postgres';
+set citus.enable_create_database_propagation=on;
+create database test_alter_db_from_nonmain_db;
+create database altered_database;
+reset citus.enable_create_database_propagation;
+\c regression;
+set citus.enable_create_database_propagation=on;
+\c test_alter_db_from_nonmain_db
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
+alter database altered_database rename to altered_database_renamed;
+alter database altered_database_renamed rename to altered_database;
+
+alter database altered_database with
+    ALLOW_CONNECTIONS false
+    CONNECTION LIMIT 1
+    IS_TEMPLATE true;
+alter database altered_database with
+    ALLOW_CONNECTIONS true
+    CONNECTION LIMIT 0
+    IS_TEMPLATE false;
+
+\c regression
+create role test_owner_non_main_db;
+\c test_alter_db_from_nonmain_db
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
+set citus.enable_create_database_propagation=on;
+alter database altered_database owner to test_owner_non_main_db;
+alter database altered_database owner to CURRENT_USER;
+alter database altered_database set default_transaction_read_only = true;
+set default_transaction_read_only = false;
+alter database altered_database set default_transaction_read_only from current;
+alter database altered_database set default_transaction_read_only to DEFAULT;
+alter database altered_database RESET default_transaction_read_only;
+alter database altered_database SET TIME ZONE '-7';
+alter database altered_database set TIME ZONE LOCAL;
+alter database altered_database set TIME ZONE DEFAULT;
+alter database altered_database RESET TIME ZONE;
+alter database altered_database SET TIME ZONE INTERVAL '-08:00' HOUR TO MINUTE;
+alter database altered_database RESET TIME ZONE;
+alter database altered_database set default_transaction_isolation = 'serializable';
+set default_transaction_isolation = 'read committed';
+alter database altered_database set default_transaction_isolation from current;
+alter database altered_database set default_transaction_isolation to DEFAULT;
+alter database altered_database RESET default_transaction_isolation;
+alter database altered_database set statement_timeout = 1000;
+set statement_timeout = 2000;
+alter database altered_database set statement_timeout from current;
+alter database altered_database set statement_timeout to DEFAULT;
+alter database altered_database RESET statement_timeout;
+alter database altered_database set lock_timeout = 1201.5;
+set lock_timeout = 1202.5;
+alter database altered_database set lock_timeout from current;
+alter database altered_database set lock_timeout to DEFAULT;
+alter database altered_database RESET lock_timeout;
+ALTER DATABASE altered_database RESET ALL;
+\c regression
+set citus.enable_create_database_propagation=on;
+drop database altered_database;
+drop database test_alter_db_from_nonmain_db;
+reset citus.enable_create_database_propagation;
+
+drop role test_owner_non_main_db;
diff --git a/src/test/regress/sql/pg15.sql b/src/test/regress/sql/pg15.sql
index cd9dab58c..4ecd1866e 100644
--- a/src/test/regress/sql/pg15.sql
+++ b/src/test/regress/sql/pg15.sql
@@ -989,3 +989,21 @@ SET client_min_messages TO ERROR;
 DROP SCHEMA pg15 CASCADE;
 DROP ROLE rls_tenant_1;
 DROP ROLE rls_tenant_2;
+
+-- test refresh collation version on non-main databases
+SET citus.enable_create_database_propagation TO on;
+create database alter_db_from_nonmain_db_pg15;
+
+\c alter_db_from_nonmain_db_pg15
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = '%ALTER DATABASE%';
+ALTER DATABASE alter_db_from_nonmain_db_pg15 REFRESH COLLATION VERSION;
+
+reset citus.log_remote_commands;
+reset citus.grep_remote_commands;
+
+\c regression
+SET citus.enable_create_database_propagation TO on;
+drop database alter_db_from_nonmain_db_pg15;
+reset citus.enable_create_database_propagation;
+SET citus.enable_create_database_propagation TO OFF;

From ff194002388ab9a7a0f51221360d1bb7e35a6859 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Mon, 18 Mar 2024 17:46:29 +0300
Subject: [PATCH 11/78] Fixes test errors

---
 .../non_main_db_distribute_object_ops.c       |  14 +-
 .../alter_database_from_nonmain_db.out        | 151 +++++++++++++++---
 .../sql/alter_database_from_nonmain_db.sql    |  20 +++
 3 files changed, 159 insertions(+), 26 deletions(-)

diff --git a/src/backend/distributed/commands/non_main_db_distribute_object_ops.c b/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
index 2254d0d68..54e58aa71 100644
--- a/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
+++ b/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
@@ -367,9 +367,21 @@ GetNonMainDbDistributeObjectOps(Node *parsetree)
 
 		case T_AlterDatabaseSetStmt:
 		{
-			return &Any_AlterDatabaseSet;
+			AlterDatabaseSetStmt *stmt = castNode(AlterDatabaseSetStmt, parsetree);
 
+			/*
+			 * We don't try to send the query to the main database if the ALTER
+			 * DATABASE command is for the main database itself, this is a very
+			 * rare case but it's exercised by our test suite.
+			 */
+			if (strcmp(stmt->dbname, MainDb) != 0)
+			{
+				return &Any_AlterDatabaseSet;
+			}
+
+			return NULL;
 		}
+
 #if PG_VERSION_NUM >= PG_VERSION_15
 		case T_AlterDatabaseRefreshCollStmt:
 		{
diff --git a/src/test/regress/expected/alter_database_from_nonmain_db.out b/src/test/regress/expected/alter_database_from_nonmain_db.out
index 763db4396..2743fffca 100644
--- a/src/test/regress/expected/alter_database_from_nonmain_db.out
+++ b/src/test/regress/expected/alter_database_from_nonmain_db.out
@@ -5,23 +5,43 @@ create database altered_database;
 reset citus.enable_create_database_propagation;
 \c regression;
 set citus.enable_create_database_propagation=on;
+\set alter_db_tablespace :abs_srcdir '/tmp_check/ts3'
+CREATE TABLESPACE alter_db_tablespace LOCATION :'alter_db_tablespace';
+\c - - - :worker_1_port
+\set alter_db_tablespace :abs_srcdir '/tmp_check/ts4'
+CREATE TABLESPACE alter_db_tablespace LOCATION :'alter_db_tablespace';
+\c - - - :worker_2_port
+\set alter_db_tablespace :abs_srcdir '/tmp_check/ts5'
+CREATE TABLESPACE alter_db_tablespace LOCATION :'alter_db_tablespace';
 \c test_alter_db_from_nonmain_db
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
+alter database altered_database set tablespace alter_db_tablespace;
+NOTICE:  issuing ALTER DATABASE altered_database SET TABLESPACE alter_db_tablespace
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+ERROR:  operation is not allowed on this node
+HINT:  Connect to the coordinator and run it again.
+CONTEXT:  while executing command on localhost:xxxxx
 alter database altered_database rename to altered_database_renamed;
 alter database altered_database_renamed rename to altered_database;
 alter database altered_database with
     ALLOW_CONNECTIONS false
     CONNECTION LIMIT 1
     IS_TEMPLATE true;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database WITH ALLOW_CONNECTIONS false CONNECTION LIMIT 1 IS_TEMPLATE true;', 'postgres')
+NOTICE:  issuing ALTER DATABASE altered_database WITH ALLOW_CONNECTIONS false CONNECTION LIMIT 1 IS_TEMPLATE true;
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+ERROR:  operation is not allowed on this node
+HINT:  Connect to the coordinator and run it again.
+CONTEXT:  while executing command on localhost:xxxxx
 alter database altered_database with
     ALLOW_CONNECTIONS true
     CONNECTION LIMIT 0
     IS_TEMPLATE false;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database WITH ALLOW_CONNECTIONS true CONNECTION LIMIT 0 IS_TEMPLATE false;', 'postgres')
+NOTICE:  issuing ALTER DATABASE altered_database WITH ALLOW_CONNECTIONS true CONNECTION LIMIT 0 IS_TEMPLATE false;
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+ERROR:  operation is not allowed on this node
+HINT:  Connect to the coordinator and run it again.
+CONTEXT:  while executing command on localhost:xxxxx
 \c regression
 create role test_owner_non_main_db;
 \c test_alter_db_from_nonmain_db
@@ -31,81 +51,162 @@ set citus.enable_create_database_propagation=on;
 alter database altered_database owner to test_owner_non_main_db;
 alter database altered_database owner to CURRENT_USER;
 alter database altered_database set default_transaction_read_only = true;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database SET default_transaction_read_only = ''true''', 'postgres')
+NOTICE:  issuing ALTER DATABASE altered_database SET default_transaction_read_only = 'true'
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+ERROR:  operation is not allowed on this node
+HINT:  Connect to the coordinator and run it again.
+CONTEXT:  while executing command on localhost:xxxxx
 set default_transaction_read_only = false;
 alter database altered_database set default_transaction_read_only from current;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database SET default_transaction_read_only FROM CURRENT', 'postgres')
+NOTICE:  issuing ALTER DATABASE altered_database SET default_transaction_read_only FROM CURRENT
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+ERROR:  operation is not allowed on this node
+HINT:  Connect to the coordinator and run it again.
+CONTEXT:  while executing command on localhost:xxxxx
 alter database altered_database set default_transaction_read_only to DEFAULT;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database SET default_transaction_read_only TO DEFAULT', 'postgres')
+NOTICE:  issuing ALTER DATABASE altered_database SET default_transaction_read_only TO DEFAULT
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+ERROR:  operation is not allowed on this node
+HINT:  Connect to the coordinator and run it again.
+CONTEXT:  while executing command on localhost:xxxxx
 alter database altered_database RESET default_transaction_read_only;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database RESET default_transaction_read_only', 'postgres')
+NOTICE:  issuing ALTER DATABASE altered_database RESET default_transaction_read_only
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+ERROR:  operation is not allowed on this node
+HINT:  Connect to the coordinator and run it again.
+CONTEXT:  while executing command on localhost:xxxxx
 alter database altered_database SET TIME ZONE '-7';
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database SET timezone = ''-7''', 'postgres')
+NOTICE:  issuing ALTER DATABASE altered_database SET timezone = '-7'
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+ERROR:  operation is not allowed on this node
+HINT:  Connect to the coordinator and run it again.
+CONTEXT:  while executing command on localhost:xxxxx
 alter database altered_database set TIME ZONE LOCAL;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database SET timezone TO DEFAULT', 'postgres')
+NOTICE:  issuing ALTER DATABASE altered_database SET timezone TO DEFAULT
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+ERROR:  operation is not allowed on this node
+HINT:  Connect to the coordinator and run it again.
+CONTEXT:  while executing command on localhost:xxxxx
 alter database altered_database set TIME ZONE DEFAULT;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database SET timezone TO DEFAULT', 'postgres')
+NOTICE:  issuing ALTER DATABASE altered_database SET timezone TO DEFAULT
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+ERROR:  operation is not allowed on this node
+HINT:  Connect to the coordinator and run it again.
+CONTEXT:  while executing command on localhost:xxxxx
 alter database altered_database RESET TIME ZONE;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database RESET timezone', 'postgres')
+NOTICE:  issuing ALTER DATABASE altered_database RESET timezone
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+ERROR:  operation is not allowed on this node
+HINT:  Connect to the coordinator and run it again.
+CONTEXT:  while executing command on localhost:xxxxx
 alter database altered_database SET TIME ZONE INTERVAL '-08:00' HOUR TO MINUTE;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database SET TIME ZONE INTERVAL ''@ 8 hours ago''', 'postgres')
+NOTICE:  issuing ALTER DATABASE altered_database SET TIME ZONE INTERVAL '@ 8 hours ago'
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+ERROR:  operation is not allowed on this node
+HINT:  Connect to the coordinator and run it again.
+CONTEXT:  while executing command on localhost:xxxxx
 alter database altered_database RESET TIME ZONE;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database RESET timezone', 'postgres')
+NOTICE:  issuing ALTER DATABASE altered_database RESET timezone
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+ERROR:  operation is not allowed on this node
+HINT:  Connect to the coordinator and run it again.
+CONTEXT:  while executing command on localhost:xxxxx
 alter database altered_database set default_transaction_isolation = 'serializable';
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database SET default_transaction_isolation = ''serializable''', 'postgres')
+NOTICE:  issuing ALTER DATABASE altered_database SET default_transaction_isolation = 'serializable'
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+ERROR:  operation is not allowed on this node
+HINT:  Connect to the coordinator and run it again.
+CONTEXT:  while executing command on localhost:xxxxx
 set default_transaction_isolation = 'read committed';
 alter database altered_database set default_transaction_isolation from current;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database SET default_transaction_isolation FROM CURRENT', 'postgres')
+NOTICE:  issuing ALTER DATABASE altered_database SET default_transaction_isolation FROM CURRENT
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+ERROR:  operation is not allowed on this node
+HINT:  Connect to the coordinator and run it again.
+CONTEXT:  while executing command on localhost:xxxxx
 alter database altered_database set default_transaction_isolation to DEFAULT;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database SET default_transaction_isolation TO DEFAULT', 'postgres')
+NOTICE:  issuing ALTER DATABASE altered_database SET default_transaction_isolation TO DEFAULT
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+ERROR:  operation is not allowed on this node
+HINT:  Connect to the coordinator and run it again.
+CONTEXT:  while executing command on localhost:xxxxx
 alter database altered_database RESET default_transaction_isolation;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database RESET default_transaction_isolation', 'postgres')
+NOTICE:  issuing ALTER DATABASE altered_database RESET default_transaction_isolation
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+ERROR:  operation is not allowed on this node
+HINT:  Connect to the coordinator and run it again.
+CONTEXT:  while executing command on localhost:xxxxx
 alter database altered_database set statement_timeout = 1000;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database SET statement_timeout = 1000', 'postgres')
+NOTICE:  issuing ALTER DATABASE altered_database SET statement_timeout = 1000
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+ERROR:  operation is not allowed on this node
+HINT:  Connect to the coordinator and run it again.
+CONTEXT:  while executing command on localhost:xxxxx
 set statement_timeout = 2000;
 alter database altered_database set statement_timeout from current;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database SET statement_timeout FROM CURRENT', 'postgres')
+NOTICE:  issuing ALTER DATABASE altered_database SET statement_timeout FROM CURRENT
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+ERROR:  operation is not allowed on this node
+HINT:  Connect to the coordinator and run it again.
+CONTEXT:  while executing command on localhost:xxxxx
 alter database altered_database set statement_timeout to DEFAULT;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database SET statement_timeout TO DEFAULT', 'postgres')
+NOTICE:  issuing ALTER DATABASE altered_database SET statement_timeout TO DEFAULT
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+ERROR:  operation is not allowed on this node
+HINT:  Connect to the coordinator and run it again.
+CONTEXT:  while executing command on localhost:xxxxx
 alter database altered_database RESET statement_timeout;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database RESET statement_timeout', 'postgres')
+NOTICE:  issuing ALTER DATABASE altered_database RESET statement_timeout
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+ERROR:  operation is not allowed on this node
+HINT:  Connect to the coordinator and run it again.
+CONTEXT:  while executing command on localhost:xxxxx
 alter database altered_database set lock_timeout = 1201.5;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database SET lock_timeout = 1201.5', 'postgres')
+NOTICE:  issuing ALTER DATABASE altered_database SET lock_timeout = 1201.5
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+ERROR:  operation is not allowed on this node
+HINT:  Connect to the coordinator and run it again.
+CONTEXT:  while executing command on localhost:xxxxx
 set lock_timeout = 1202.5;
 alter database altered_database set lock_timeout from current;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database SET lock_timeout FROM CURRENT', 'postgres')
+NOTICE:  issuing ALTER DATABASE altered_database SET lock_timeout FROM CURRENT
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+ERROR:  operation is not allowed on this node
+HINT:  Connect to the coordinator and run it again.
+CONTEXT:  while executing command on localhost:xxxxx
 alter database altered_database set lock_timeout to DEFAULT;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database SET lock_timeout TO DEFAULT', 'postgres')
+NOTICE:  issuing ALTER DATABASE altered_database SET lock_timeout TO DEFAULT
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+ERROR:  operation is not allowed on this node
+HINT:  Connect to the coordinator and run it again.
+CONTEXT:  while executing command on localhost:xxxxx
 alter database altered_database RESET lock_timeout;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database RESET lock_timeout', 'postgres')
+NOTICE:  issuing ALTER DATABASE altered_database RESET lock_timeout
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+ERROR:  operation is not allowed on this node
+HINT:  Connect to the coordinator and run it again.
+CONTEXT:  while executing command on localhost:xxxxx
 ALTER DATABASE altered_database RESET ALL;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database RESET ALL', 'postgres')
+NOTICE:  issuing ALTER DATABASE altered_database RESET ALL
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+ERROR:  operation is not allowed on this node
+HINT:  Connect to the coordinator and run it again.
+CONTEXT:  while executing command on localhost:xxxxx
 \c regression
 set citus.enable_create_database_propagation=on;
 drop database altered_database;
 drop database test_alter_db_from_nonmain_db;
 reset citus.enable_create_database_propagation;
 drop role test_owner_non_main_db;
+SELECT result FROM run_command_on_all_nodes(
+  $$
+  drop tablespace alter_db_tablespace
+  $$
+);
+     result
+---------------------------------------------------------------------
+ DROP TABLESPACE
+ DROP TABLESPACE
+ DROP TABLESPACE
+(3 rows)
+
diff --git a/src/test/regress/sql/alter_database_from_nonmain_db.sql b/src/test/regress/sql/alter_database_from_nonmain_db.sql
index aa9a5d191..04ab1e7ef 100644
--- a/src/test/regress/sql/alter_database_from_nonmain_db.sql
+++ b/src/test/regress/sql/alter_database_from_nonmain_db.sql
@@ -5,9 +5,23 @@ create database altered_database;
 reset citus.enable_create_database_propagation;
 \c regression;
 set citus.enable_create_database_propagation=on;
+
+\set alter_db_tablespace :abs_srcdir '/tmp_check/ts3'
+CREATE TABLESPACE alter_db_tablespace LOCATION :'alter_db_tablespace';
+
+\c - - - :worker_1_port
+\set alter_db_tablespace :abs_srcdir '/tmp_check/ts4'
+CREATE TABLESPACE alter_db_tablespace LOCATION :'alter_db_tablespace';
+
+\c - - - :worker_2_port
+\set alter_db_tablespace :abs_srcdir '/tmp_check/ts5'
+CREATE TABLESPACE alter_db_tablespace LOCATION :'alter_db_tablespace';
+
 \c test_alter_db_from_nonmain_db
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
+
+alter database altered_database set tablespace alter_db_tablespace;
 alter database altered_database rename to altered_database_renamed;
 alter database altered_database_renamed rename to altered_database;
 
@@ -62,3 +76,9 @@ drop database test_alter_db_from_nonmain_db;
 reset citus.enable_create_database_propagation;
 
 drop role test_owner_non_main_db;
+
+SELECT result FROM run_command_on_all_nodes(
+  $$
+  drop tablespace alter_db_tablespace
+  $$
+);

From 1842c2e3b6d66538d2e50ca2491efc0b0e64c8d1 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Mon, 18 Mar 2024 18:04:23 +0300
Subject: [PATCH 12/78] Fixes pg14 compile error

---
 .../distributed/commands/non_main_db_distribute_object_ops.c    | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/backend/distributed/commands/non_main_db_distribute_object_ops.c b/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
index 54e58aa71..393fd023b 100644
--- a/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
+++ b/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
@@ -172,11 +172,13 @@ static const NonMainDbDistributeObjectOps Any_AlterDatabaseSet = {
 	.cannotBeExecutedInTransaction = true
 };
 
+#if PG_VERSION_NUM >= PG_VERSION_15
 static const NonMainDbDistributeObjectOps Any_AlterDatabaseRefreshColl = {
 	.getMarkDistributedParams = NULL,
 	.getUnmarkDistributedParams = NULL,
 	.cannotBeExecutedInTransaction = false
 };
+#endif
 
 static const NonMainDbDistributeObjectOps Database_Grant = {
 	.getMarkDistributedParams = NULL,

From 2fade943594fee1df321f273848dfd961616493c Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Mon, 18 Mar 2024 18:14:07 +0300
Subject: [PATCH 13/78] Fixes test errors

---
 .../alter_database_from_nonmain_db.out        | 124 +++++++++---------
 .../sql/alter_database_from_nonmain_db.sql    |  72 +++++-----
 2 files changed, 98 insertions(+), 98 deletions(-)

diff --git a/src/test/regress/expected/alter_database_from_nonmain_db.out b/src/test/regress/expected/alter_database_from_nonmain_db.out
index 2743fffca..6716b90a2 100644
--- a/src/test/regress/expected/alter_database_from_nonmain_db.out
+++ b/src/test/regress/expected/alter_database_from_nonmain_db.out
@@ -1,43 +1,43 @@
 SET citus.superuser TO 'postgres';
 set citus.enable_create_database_propagation=on;
 create database test_alter_db_from_nonmain_db;
-create database altered_database;
+create database "altered_database!'2";
 reset citus.enable_create_database_propagation;
 \c regression;
 set citus.enable_create_database_propagation=on;
 \set alter_db_tablespace :abs_srcdir '/tmp_check/ts3'
-CREATE TABLESPACE alter_db_tablespace LOCATION :'alter_db_tablespace';
+CREATE TABLESPACE "ts-needs\!escape2" LOCATION :'alter_db_tablespace';
 \c - - - :worker_1_port
 \set alter_db_tablespace :abs_srcdir '/tmp_check/ts4'
-CREATE TABLESPACE alter_db_tablespace LOCATION :'alter_db_tablespace';
+CREATE TABLESPACE "ts-needs\!escape2" LOCATION :'alter_db_tablespace';
 \c - - - :worker_2_port
 \set alter_db_tablespace :abs_srcdir '/tmp_check/ts5'
-CREATE TABLESPACE alter_db_tablespace LOCATION :'alter_db_tablespace';
+CREATE TABLESPACE "ts-needs\!escape2" LOCATION :'alter_db_tablespace';
 \c test_alter_db_from_nonmain_db
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
-alter database altered_database set tablespace alter_db_tablespace;
-NOTICE:  issuing ALTER DATABASE altered_database SET TABLESPACE alter_db_tablespace
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 ERROR:  operation is not allowed on this node
 HINT:  Connect to the coordinator and run it again.
 CONTEXT:  while executing command on localhost:xxxxx
-alter database altered_database rename to altered_database_renamed;
-alter database altered_database_renamed rename to altered_database;
-alter database altered_database with
+alter database "altered_database!'2" rename to altered_database_renamed;
+alter database altered_database_renamed rename to "altered_database!'2";
+alter database "altered_database!'2" with
     ALLOW_CONNECTIONS false
     CONNECTION LIMIT 1
     IS_TEMPLATE true;
-NOTICE:  issuing ALTER DATABASE altered_database WITH ALLOW_CONNECTIONS false CONNECTION LIMIT 1 IS_TEMPLATE true;
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" WITH ALLOW_CONNECTIONS false CONNECTION LIMIT 1 IS_TEMPLATE true;
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 ERROR:  operation is not allowed on this node
 HINT:  Connect to the coordinator and run it again.
 CONTEXT:  while executing command on localhost:xxxxx
-alter database altered_database with
+alter database "altered_database!'2" with
     ALLOW_CONNECTIONS true
     CONNECTION LIMIT 0
     IS_TEMPLATE false;
-NOTICE:  issuing ALTER DATABASE altered_database WITH ALLOW_CONNECTIONS true CONNECTION LIMIT 0 IS_TEMPLATE false;
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" WITH ALLOW_CONNECTIONS true CONNECTION LIMIT 0 IS_TEMPLATE false;
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 ERROR:  operation is not allowed on this node
 HINT:  Connect to the coordinator and run it again.
@@ -48,159 +48,159 @@ create role test_owner_non_main_db;
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
 set citus.enable_create_database_propagation=on;
-alter database altered_database owner to test_owner_non_main_db;
-alter database altered_database owner to CURRENT_USER;
-alter database altered_database set default_transaction_read_only = true;
-NOTICE:  issuing ALTER DATABASE altered_database SET default_transaction_read_only = 'true'
+alter database "altered_database!'2" owner to test_owner_non_main_db;
+alter database "altered_database!'2" owner to CURRENT_USER;
+alter database "altered_database!'2" set default_transaction_read_only = true;
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET default_transaction_read_only = 'true'
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 ERROR:  operation is not allowed on this node
 HINT:  Connect to the coordinator and run it again.
 CONTEXT:  while executing command on localhost:xxxxx
 set default_transaction_read_only = false;
-alter database altered_database set default_transaction_read_only from current;
-NOTICE:  issuing ALTER DATABASE altered_database SET default_transaction_read_only FROM CURRENT
+alter database "altered_database!'2" set default_transaction_read_only from current;
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET default_transaction_read_only FROM CURRENT
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 ERROR:  operation is not allowed on this node
 HINT:  Connect to the coordinator and run it again.
 CONTEXT:  while executing command on localhost:xxxxx
-alter database altered_database set default_transaction_read_only to DEFAULT;
-NOTICE:  issuing ALTER DATABASE altered_database SET default_transaction_read_only TO DEFAULT
+alter database "altered_database!'2" set default_transaction_read_only to DEFAULT;
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET default_transaction_read_only TO DEFAULT
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 ERROR:  operation is not allowed on this node
 HINT:  Connect to the coordinator and run it again.
 CONTEXT:  while executing command on localhost:xxxxx
-alter database altered_database RESET default_transaction_read_only;
-NOTICE:  issuing ALTER DATABASE altered_database RESET default_transaction_read_only
+alter database "altered_database!'2" RESET default_transaction_read_only;
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" RESET default_transaction_read_only
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 ERROR:  operation is not allowed on this node
 HINT:  Connect to the coordinator and run it again.
 CONTEXT:  while executing command on localhost:xxxxx
-alter database altered_database SET TIME ZONE '-7';
-NOTICE:  issuing ALTER DATABASE altered_database SET timezone = '-7'
+alter database "altered_database!'2" SET TIME ZONE '-7';
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET timezone = '-7'
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 ERROR:  operation is not allowed on this node
 HINT:  Connect to the coordinator and run it again.
 CONTEXT:  while executing command on localhost:xxxxx
-alter database altered_database set TIME ZONE LOCAL;
-NOTICE:  issuing ALTER DATABASE altered_database SET timezone TO DEFAULT
+alter database "altered_database!'2" set TIME ZONE LOCAL;
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET timezone TO DEFAULT
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 ERROR:  operation is not allowed on this node
 HINT:  Connect to the coordinator and run it again.
 CONTEXT:  while executing command on localhost:xxxxx
-alter database altered_database set TIME ZONE DEFAULT;
-NOTICE:  issuing ALTER DATABASE altered_database SET timezone TO DEFAULT
+alter database "altered_database!'2" set TIME ZONE DEFAULT;
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET timezone TO DEFAULT
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 ERROR:  operation is not allowed on this node
 HINT:  Connect to the coordinator and run it again.
 CONTEXT:  while executing command on localhost:xxxxx
-alter database altered_database RESET TIME ZONE;
-NOTICE:  issuing ALTER DATABASE altered_database RESET timezone
+alter database "altered_database!'2" RESET TIME ZONE;
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" RESET timezone
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 ERROR:  operation is not allowed on this node
 HINT:  Connect to the coordinator and run it again.
 CONTEXT:  while executing command on localhost:xxxxx
-alter database altered_database SET TIME ZONE INTERVAL '-08:00' HOUR TO MINUTE;
-NOTICE:  issuing ALTER DATABASE altered_database SET TIME ZONE INTERVAL '@ 8 hours ago'
+alter database "altered_database!'2" SET TIME ZONE INTERVAL '-08:00' HOUR TO MINUTE;
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TIME ZONE INTERVAL '@ 8 hours ago'
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 ERROR:  operation is not allowed on this node
 HINT:  Connect to the coordinator and run it again.
 CONTEXT:  while executing command on localhost:xxxxx
-alter database altered_database RESET TIME ZONE;
-NOTICE:  issuing ALTER DATABASE altered_database RESET timezone
+alter database "altered_database!'2" RESET TIME ZONE;
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" RESET timezone
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 ERROR:  operation is not allowed on this node
 HINT:  Connect to the coordinator and run it again.
 CONTEXT:  while executing command on localhost:xxxxx
-alter database altered_database set default_transaction_isolation = 'serializable';
-NOTICE:  issuing ALTER DATABASE altered_database SET default_transaction_isolation = 'serializable'
+alter database "altered_database!'2" set default_transaction_isolation = 'serializable';
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET default_transaction_isolation = 'serializable'
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 ERROR:  operation is not allowed on this node
 HINT:  Connect to the coordinator and run it again.
 CONTEXT:  while executing command on localhost:xxxxx
 set default_transaction_isolation = 'read committed';
-alter database altered_database set default_transaction_isolation from current;
-NOTICE:  issuing ALTER DATABASE altered_database SET default_transaction_isolation FROM CURRENT
+alter database "altered_database!'2" set default_transaction_isolation from current;
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET default_transaction_isolation FROM CURRENT
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 ERROR:  operation is not allowed on this node
 HINT:  Connect to the coordinator and run it again.
 CONTEXT:  while executing command on localhost:xxxxx
-alter database altered_database set default_transaction_isolation to DEFAULT;
-NOTICE:  issuing ALTER DATABASE altered_database SET default_transaction_isolation TO DEFAULT
+alter database "altered_database!'2" set default_transaction_isolation to DEFAULT;
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET default_transaction_isolation TO DEFAULT
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 ERROR:  operation is not allowed on this node
 HINT:  Connect to the coordinator and run it again.
 CONTEXT:  while executing command on localhost:xxxxx
-alter database altered_database RESET default_transaction_isolation;
-NOTICE:  issuing ALTER DATABASE altered_database RESET default_transaction_isolation
+alter database "altered_database!'2" RESET default_transaction_isolation;
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" RESET default_transaction_isolation
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 ERROR:  operation is not allowed on this node
 HINT:  Connect to the coordinator and run it again.
 CONTEXT:  while executing command on localhost:xxxxx
-alter database altered_database set statement_timeout = 1000;
-NOTICE:  issuing ALTER DATABASE altered_database SET statement_timeout = 1000
+alter database "altered_database!'2" set statement_timeout = 1000;
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET statement_timeout = 1000
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 ERROR:  operation is not allowed on this node
 HINT:  Connect to the coordinator and run it again.
 CONTEXT:  while executing command on localhost:xxxxx
 set statement_timeout = 2000;
-alter database altered_database set statement_timeout from current;
-NOTICE:  issuing ALTER DATABASE altered_database SET statement_timeout FROM CURRENT
+alter database "altered_database!'2" set statement_timeout from current;
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET statement_timeout FROM CURRENT
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 ERROR:  operation is not allowed on this node
 HINT:  Connect to the coordinator and run it again.
 CONTEXT:  while executing command on localhost:xxxxx
-alter database altered_database set statement_timeout to DEFAULT;
-NOTICE:  issuing ALTER DATABASE altered_database SET statement_timeout TO DEFAULT
+alter database "altered_database!'2" set statement_timeout to DEFAULT;
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET statement_timeout TO DEFAULT
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 ERROR:  operation is not allowed on this node
 HINT:  Connect to the coordinator and run it again.
 CONTEXT:  while executing command on localhost:xxxxx
-alter database altered_database RESET statement_timeout;
-NOTICE:  issuing ALTER DATABASE altered_database RESET statement_timeout
+alter database "altered_database!'2" RESET statement_timeout;
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" RESET statement_timeout
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 ERROR:  operation is not allowed on this node
 HINT:  Connect to the coordinator and run it again.
 CONTEXT:  while executing command on localhost:xxxxx
-alter database altered_database set lock_timeout = 1201.5;
-NOTICE:  issuing ALTER DATABASE altered_database SET lock_timeout = 1201.5
+alter database "altered_database!'2" set lock_timeout = 1201.5;
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET lock_timeout = 1201.5
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 ERROR:  operation is not allowed on this node
 HINT:  Connect to the coordinator and run it again.
 CONTEXT:  while executing command on localhost:xxxxx
 set lock_timeout = 1202.5;
-alter database altered_database set lock_timeout from current;
-NOTICE:  issuing ALTER DATABASE altered_database SET lock_timeout FROM CURRENT
+alter database "altered_database!'2" set lock_timeout from current;
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET lock_timeout FROM CURRENT
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 ERROR:  operation is not allowed on this node
 HINT:  Connect to the coordinator and run it again.
 CONTEXT:  while executing command on localhost:xxxxx
-alter database altered_database set lock_timeout to DEFAULT;
-NOTICE:  issuing ALTER DATABASE altered_database SET lock_timeout TO DEFAULT
+alter database "altered_database!'2" set lock_timeout to DEFAULT;
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET lock_timeout TO DEFAULT
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 ERROR:  operation is not allowed on this node
 HINT:  Connect to the coordinator and run it again.
 CONTEXT:  while executing command on localhost:xxxxx
-alter database altered_database RESET lock_timeout;
-NOTICE:  issuing ALTER DATABASE altered_database RESET lock_timeout
+alter database "altered_database!'2" RESET lock_timeout;
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" RESET lock_timeout
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 ERROR:  operation is not allowed on this node
 HINT:  Connect to the coordinator and run it again.
 CONTEXT:  while executing command on localhost:xxxxx
-ALTER DATABASE altered_database RESET ALL;
-NOTICE:  issuing ALTER DATABASE altered_database RESET ALL
+ALTER DATABASE "altered_database!'2" RESET ALL;
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" RESET ALL
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 ERROR:  operation is not allowed on this node
 HINT:  Connect to the coordinator and run it again.
 CONTEXT:  while executing command on localhost:xxxxx
 \c regression
 set citus.enable_create_database_propagation=on;
-drop database altered_database;
+drop database "altered_database!'2";
 drop database test_alter_db_from_nonmain_db;
 reset citus.enable_create_database_propagation;
 drop role test_owner_non_main_db;
 SELECT result FROM run_command_on_all_nodes(
   $$
-  drop tablespace alter_db_tablespace
+  drop tablespace "ts-needs\!escape2"
   $$
 );
      result
diff --git a/src/test/regress/sql/alter_database_from_nonmain_db.sql b/src/test/regress/sql/alter_database_from_nonmain_db.sql
index 04ab1e7ef..7528832b0 100644
--- a/src/test/regress/sql/alter_database_from_nonmain_db.sql
+++ b/src/test/regress/sql/alter_database_from_nonmain_db.sql
@@ -1,35 +1,35 @@
 SET citus.superuser TO 'postgres';
 set citus.enable_create_database_propagation=on;
 create database test_alter_db_from_nonmain_db;
-create database altered_database;
+create database "altered_database!'2";
 reset citus.enable_create_database_propagation;
 \c regression;
 set citus.enable_create_database_propagation=on;
 
 \set alter_db_tablespace :abs_srcdir '/tmp_check/ts3'
-CREATE TABLESPACE alter_db_tablespace LOCATION :'alter_db_tablespace';
+CREATE TABLESPACE "ts-needs\!escape2" LOCATION :'alter_db_tablespace';
 
 \c - - - :worker_1_port
 \set alter_db_tablespace :abs_srcdir '/tmp_check/ts4'
-CREATE TABLESPACE alter_db_tablespace LOCATION :'alter_db_tablespace';
+CREATE TABLESPACE "ts-needs\!escape2" LOCATION :'alter_db_tablespace';
 
 \c - - - :worker_2_port
 \set alter_db_tablespace :abs_srcdir '/tmp_check/ts5'
-CREATE TABLESPACE alter_db_tablespace LOCATION :'alter_db_tablespace';
+CREATE TABLESPACE "ts-needs\!escape2" LOCATION :'alter_db_tablespace';
 
 \c test_alter_db_from_nonmain_db
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
 
-alter database altered_database set tablespace alter_db_tablespace;
-alter database altered_database rename to altered_database_renamed;
-alter database altered_database_renamed rename to altered_database;
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+alter database "altered_database!'2" rename to altered_database_renamed;
+alter database altered_database_renamed rename to "altered_database!'2";
 
-alter database altered_database with
+alter database "altered_database!'2" with
     ALLOW_CONNECTIONS false
     CONNECTION LIMIT 1
     IS_TEMPLATE true;
-alter database altered_database with
+alter database "altered_database!'2" with
     ALLOW_CONNECTIONS true
     CONNECTION LIMIT 0
     IS_TEMPLATE false;
@@ -40,38 +40,38 @@ create role test_owner_non_main_db;
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
 set citus.enable_create_database_propagation=on;
-alter database altered_database owner to test_owner_non_main_db;
-alter database altered_database owner to CURRENT_USER;
-alter database altered_database set default_transaction_read_only = true;
+alter database "altered_database!'2" owner to test_owner_non_main_db;
+alter database "altered_database!'2" owner to CURRENT_USER;
+alter database "altered_database!'2" set default_transaction_read_only = true;
 set default_transaction_read_only = false;
-alter database altered_database set default_transaction_read_only from current;
-alter database altered_database set default_transaction_read_only to DEFAULT;
-alter database altered_database RESET default_transaction_read_only;
-alter database altered_database SET TIME ZONE '-7';
-alter database altered_database set TIME ZONE LOCAL;
-alter database altered_database set TIME ZONE DEFAULT;
-alter database altered_database RESET TIME ZONE;
-alter database altered_database SET TIME ZONE INTERVAL '-08:00' HOUR TO MINUTE;
-alter database altered_database RESET TIME ZONE;
-alter database altered_database set default_transaction_isolation = 'serializable';
+alter database "altered_database!'2" set default_transaction_read_only from current;
+alter database "altered_database!'2" set default_transaction_read_only to DEFAULT;
+alter database "altered_database!'2" RESET default_transaction_read_only;
+alter database "altered_database!'2" SET TIME ZONE '-7';
+alter database "altered_database!'2" set TIME ZONE LOCAL;
+alter database "altered_database!'2" set TIME ZONE DEFAULT;
+alter database "altered_database!'2" RESET TIME ZONE;
+alter database "altered_database!'2" SET TIME ZONE INTERVAL '-08:00' HOUR TO MINUTE;
+alter database "altered_database!'2" RESET TIME ZONE;
+alter database "altered_database!'2" set default_transaction_isolation = 'serializable';
 set default_transaction_isolation = 'read committed';
-alter database altered_database set default_transaction_isolation from current;
-alter database altered_database set default_transaction_isolation to DEFAULT;
-alter database altered_database RESET default_transaction_isolation;
-alter database altered_database set statement_timeout = 1000;
+alter database "altered_database!'2" set default_transaction_isolation from current;
+alter database "altered_database!'2" set default_transaction_isolation to DEFAULT;
+alter database "altered_database!'2" RESET default_transaction_isolation;
+alter database "altered_database!'2" set statement_timeout = 1000;
 set statement_timeout = 2000;
-alter database altered_database set statement_timeout from current;
-alter database altered_database set statement_timeout to DEFAULT;
-alter database altered_database RESET statement_timeout;
-alter database altered_database set lock_timeout = 1201.5;
+alter database "altered_database!'2" set statement_timeout from current;
+alter database "altered_database!'2" set statement_timeout to DEFAULT;
+alter database "altered_database!'2" RESET statement_timeout;
+alter database "altered_database!'2" set lock_timeout = 1201.5;
 set lock_timeout = 1202.5;
-alter database altered_database set lock_timeout from current;
-alter database altered_database set lock_timeout to DEFAULT;
-alter database altered_database RESET lock_timeout;
-ALTER DATABASE altered_database RESET ALL;
+alter database "altered_database!'2" set lock_timeout from current;
+alter database "altered_database!'2" set lock_timeout to DEFAULT;
+alter database "altered_database!'2" RESET lock_timeout;
+ALTER DATABASE "altered_database!'2" RESET ALL;
 \c regression
 set citus.enable_create_database_propagation=on;
-drop database altered_database;
+drop database "altered_database!'2";
 drop database test_alter_db_from_nonmain_db;
 reset citus.enable_create_database_propagation;
 
@@ -79,6 +79,6 @@ drop role test_owner_non_main_db;
 
 SELECT result FROM run_command_on_all_nodes(
   $$
-  drop tablespace alter_db_tablespace
+  drop tablespace "ts-needs\!escape2"
   $$
 );

From 9815a7262d0a0365bc8954d1a187d4daeaaa4363 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Fri, 22 Mar 2024 11:37:21 +0300
Subject: [PATCH 14/78] Adds db rename and refreshcoll

---
 src/backend/distributed/commands/database.c   |   8 +-
 .../non_main_db_distribute_object_ops.c       | 115 ++++++++++++++++--
 .../distributed/commands/utility_hook.c       |  12 +-
 src/include/distributed/commands.h            |   1 +
 .../expected/minimal_cluster_management.out   |  41 +++++++
 .../expected/multi_cluster_management.out     |  24 ++++
 src/test/regress/pg_regress_multi.pl          |   1 +
 .../sql/alter_database_from_nonmain_db.sql    |  11 ++
 src/test/regress/sql/failure_setup.sql        |  12 ++
 .../sql/minimal_cluster_management.sql        |  21 ++++
 .../regress/sql/multi_cluster_management.sql  |  13 ++
 11 files changed, 244 insertions(+), 15 deletions(-)

diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c
index 5479a59ed..be444e23b 100644
--- a/src/backend/distributed/commands/database.c
+++ b/src/backend/distributed/commands/database.c
@@ -254,7 +254,7 @@ FilterDistributedDatabases(List *databases)
  * IsSetTablespaceStatement returns true if the statement is a SET TABLESPACE statement,
  * false otherwise.
  */
-static bool
+bool
 IsSetTablespaceStatement(AlterDatabaseStmt *stmt)
 {
 	DefElem *def = NULL;
@@ -293,7 +293,11 @@ PreprocessAlterDatabaseStmt(Node *node, const char *queryString,
 		return NIL;
 	}
 
-	EnsureCoordinator();
+	if (!IsSetTablespaceStatement(stmt))
+	{
+		EnsureCoordinator();
+	}
+
 	SerializeDistributedDDLsOnObjectClassObject(OCLASS_DATABASE, stmt->dbname);
 
 	char *sql = DeparseTreeNode((Node *) stmt);
diff --git a/src/backend/distributed/commands/non_main_db_distribute_object_ops.c b/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
index b777936d3..52d025f70 100644
--- a/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
+++ b/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
@@ -61,7 +61,7 @@
  */
 typedef struct NonMainDbDistributeObjectOps
 {
-	bool cannotBeExecutedInTransaction;
+	bool (*cannotBeExecutedInTransaction)(Node *parsetree);
 	bool (*checkSupportedObjectType)(Node *parsetree);
 } NonMainDbDistributeObjectOps;
 
@@ -73,41 +73,69 @@ static bool CreateDbStmtCheckSupportedObjectType(Node *node);
 static bool DropDbStmtCheckSupportedObjectType(Node *node);
 static bool GrantStmtCheckSupportedObjectType(Node *node);
 static bool SecLabelStmtCheckSupportedObjectType(Node *node);
+static bool AlterDbStmtCheckSupportedObjectType(Node *node);
+static bool AlterDbCanNotBeExecutedInTransaction(Node *node);
+static bool CanNotBeExecutedInTransaction_True(Node *node);
+static bool CanNotBeExecutedInTransaction_False(Node *node);
+static bool AlterDbRenameCheckSupportedObjectType(Node *node);
+static bool AlterDbOwnerCheckSupportedObjectType(Node *node);
 
 /*
  * OperationArray that holds NonMainDbDistributeObjectOps for different command types.
  */
 static const NonMainDbDistributeObjectOps *const OperationArray[] = {
 	[T_CreateRoleStmt] = &(NonMainDbDistributeObjectOps) {
-		.cannotBeExecutedInTransaction = false,
+		.cannotBeExecutedInTransaction = CanNotBeExecutedInTransaction_False,
 		.checkSupportedObjectType = NULL
 	},
 	[T_DropRoleStmt] = &(NonMainDbDistributeObjectOps) {
-		.cannotBeExecutedInTransaction = false,
+		.cannotBeExecutedInTransaction = CanNotBeExecutedInTransaction_False,
 		.checkSupportedObjectType = NULL
 	},
 	[T_AlterRoleStmt] = &(NonMainDbDistributeObjectOps) {
-		.cannotBeExecutedInTransaction = false,
+		.cannotBeExecutedInTransaction = CanNotBeExecutedInTransaction_False,
 		.checkSupportedObjectType = NULL
 	},
 	[T_GrantRoleStmt] = &(NonMainDbDistributeObjectOps) {
-		.cannotBeExecutedInTransaction = false,
+		.cannotBeExecutedInTransaction = CanNotBeExecutedInTransaction_False,
 		.checkSupportedObjectType = NULL
 	},
 	[T_CreatedbStmt] = &(NonMainDbDistributeObjectOps) {
-		.cannotBeExecutedInTransaction = true,
+		.cannotBeExecutedInTransaction = CanNotBeExecutedInTransaction_True,
 		.checkSupportedObjectType = CreateDbStmtCheckSupportedObjectType
 	},
 	[T_DropdbStmt] = &(NonMainDbDistributeObjectOps) {
-		.cannotBeExecutedInTransaction = true,
+		.cannotBeExecutedInTransaction = CanNotBeExecutedInTransaction_True,
 		.checkSupportedObjectType = DropDbStmtCheckSupportedObjectType
 	},
+	[T_AlterDatabaseSetStmt] = &(NonMainDbDistributeObjectOps) {
+		.cannotBeExecutedInTransaction = CanNotBeExecutedInTransaction_False,
+		.checkSupportedObjectType = NULL
+	},
+	[T_AlterDatabaseStmt] = &(NonMainDbDistributeObjectOps) {
+		.cannotBeExecutedInTransaction = AlterDbCanNotBeExecutedInTransaction,
+		.checkSupportedObjectType = AlterDbStmtCheckSupportedObjectType
+	},
+#if PG_VERSION_NUM >= PG_VERSION_15
+	[T_AlterDatabaseRefreshCollStmt] = &(NonMainDbDistributeObjectOps) {
+		.cannotBeExecutedInTransaction = CanNotBeExecutedInTransaction_False,
+		.checkSupportedObjectType = NULL
+	},
+#endif
+	[T_RenameStmt] = &(NonMainDbDistributeObjectOps) {
+		.cannotBeExecutedInTransaction = CanNotBeExecutedInTransaction_False,
+		.checkSupportedObjectType = AlterDbRenameCheckSupportedObjectType
+	},
+	[T_AlterOwnerStmt] = &(NonMainDbDistributeObjectOps) {
+		.cannotBeExecutedInTransaction = CanNotBeExecutedInTransaction_False,
+		.checkSupportedObjectType = AlterDbOwnerCheckSupportedObjectType
+	},
 	[T_GrantStmt] = &(NonMainDbDistributeObjectOps) {
-		.cannotBeExecutedInTransaction = false,
+		.cannotBeExecutedInTransaction = CanNotBeExecutedInTransaction_False,
 		.checkSupportedObjectType = GrantStmtCheckSupportedObjectType
 	},
 	[T_SecLabelStmt] = &(NonMainDbDistributeObjectOps) {
-		.cannotBeExecutedInTransaction = false,
+		.cannotBeExecutedInTransaction = CanNotBeExecutedInTransaction_False,
 		.checkSupportedObjectType = SecLabelStmtCheckSupportedObjectType
 	},
 };
@@ -132,6 +160,15 @@ static void UnmarkObjectDistributedOnLocalMainDb(uint16 catalogRelId, Oid object
 bool
 RunPreprocessNonMainDBCommand(Node *parsetree)
 {
+	/* NodeTag tag = nodeTag(parsetree); */
+	/* if ( tag == T_AlterDatabaseSetStmt){ */
+	/* 	AlterDatabaseSetStmt *stmt = castNode(AlterDatabaseSetStmt, parsetree); */
+	/* 	if (strcmp(stmt->dbname, MainDb) == 0){ */
+	/* 		return false; */
+	/* 	} */
+	/* } */
+
+
 	if (IsMainDB)
 	{
 		return false;
@@ -150,7 +187,7 @@ RunPreprocessNonMainDBCommand(Node *parsetree)
 	 * transactional visibility issues. We directly route them to main database
 	 * so that we only have to consider one code-path for such commands.
 	 */
-	if (ops->cannotBeExecutedInTransaction)
+	if (ops->cannotBeExecutedInTransaction(parsetree))
 	{
 		IsMainDBCommandInXact = false;
 		RunCitusMainDBQuery((char *) queryString);
@@ -335,6 +372,50 @@ DropDbStmtCheckSupportedObjectType(Node *node)
 }
 
 
+static bool
+AlterDbStmtCheckSupportedObjectType(Node *node)
+{
+	/*
+	 * We don't try to send the query to the main database if the ALTER
+	 * DATABASE command is for the main database itself, this is a very
+	 * rare case but it's exercised by our test suite.
+	 */
+	AlterDatabaseStmt *stmt = castNode(AlterDatabaseStmt, node);
+	if (!IsSetTablespaceStatement(stmt))
+	{
+		return true;
+	}
+	else
+	{
+		return IsSetTablespaceStatement(stmt) && strcmp(stmt->dbname, MainDb) != 0;
+	}
+}
+
+
+static bool
+AlterDbCanNotBeExecutedInTransaction(Node *node)
+{
+	AlterDatabaseStmt *stmt = castNode(AlterDatabaseStmt, node);
+	return IsSetTablespaceStatement(stmt);
+}
+
+
+static bool
+AlterDbRenameCheckSupportedObjectType(Node *node)
+{
+	RenameStmt *stmt = castNode(RenameStmt, node);
+	return stmt->renameType == OBJECT_DATABASE;
+}
+
+
+static bool
+AlterDbOwnerCheckSupportedObjectType(Node *node)
+{
+	AlterOwnerStmt *stmt = castNode(AlterOwnerStmt, node);
+	return stmt->objectType == OBJECT_DATABASE;
+}
+
+
 static bool
 GrantStmtCheckSupportedObjectType(Node *node)
 {
@@ -349,3 +430,17 @@ SecLabelStmtCheckSupportedObjectType(Node *node)
 	SecLabelStmt *stmt = castNode(SecLabelStmt, node);
 	return stmt->objtype == OBJECT_ROLE;
 }
+
+
+static bool
+CanNotBeExecutedInTransaction_True(Node *node)
+{
+	return true;
+}
+
+
+static bool
+CanNotBeExecutedInTransaction_False(Node *node)
+{
+	return false;
+}
diff --git a/src/backend/distributed/commands/utility_hook.c b/src/backend/distributed/commands/utility_hook.c
index 9426e13c0..00f0362d5 100644
--- a/src/backend/distributed/commands/utility_hook.c
+++ b/src/backend/distributed/commands/utility_hook.c
@@ -252,8 +252,12 @@ citus_ProcessUtility(PlannedStmt *pstmt,
 		 * and if the command is a node-wide object management command that we
 		 * support from non-main databases.
 		 */
+		bool shouldSkipPrevUtilityHook = false;
 
-		bool shouldSkipPrevUtilityHook = RunPreprocessNonMainDBCommand(parsetree);
+		if (EnableDDLPropagation)
+		{
+			shouldSkipPrevUtilityHook = RunPreprocessNonMainDBCommand(parsetree);
+		}
 
 		if (!shouldSkipPrevUtilityHook)
 		{
@@ -264,8 +268,10 @@ citus_ProcessUtility(PlannedStmt *pstmt,
 			PrevProcessUtility(pstmt, queryString, false, context,
 							   params, queryEnv, dest, completionTag);
 		}
-
-		RunPostprocessNonMainDBCommand(parsetree);
+		if (EnableDDLPropagation)
+		{
+			RunPostprocessNonMainDBCommand(parsetree);
+		}
 
 		return;
 	}
diff --git a/src/include/distributed/commands.h b/src/include/distributed/commands.h
index 084308a8f..8ce5c72ac 100644
--- a/src/include/distributed/commands.h
+++ b/src/include/distributed/commands.h
@@ -256,6 +256,7 @@ extern List * PreprocessAlterDatabaseRenameStmt(Node *node, const char *queryStr
 extern List * PostprocessAlterDatabaseRenameStmt(Node *node, const char *queryString);
 extern void EnsureSupportedCreateDatabaseCommand(CreatedbStmt *stmt);
 extern char * CreateDatabaseDDLCommand(Oid dbId);
+extern bool IsSetTablespaceStatement(AlterDatabaseStmt *stmt);
 
 
 /* domain.c - forward declarations */
diff --git a/src/test/regress/expected/minimal_cluster_management.out b/src/test/regress/expected/minimal_cluster_management.out
index d05e83ed5..87f8be882 100644
--- a/src/test/regress/expected/minimal_cluster_management.out
+++ b/src/test/regress/expected/minimal_cluster_management.out
@@ -1,3 +1,44 @@
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+ pg_reload_conf
+---------------------------------------------------------------------
+ t
+(1 row)
+
+\c - - - :worker_1_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+ pg_reload_conf
+---------------------------------------------------------------------
+ t
+(1 row)
+
+\c - - - :worker_2_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+ pg_reload_conf
+---------------------------------------------------------------------
+ t
+(1 row)
+
+\c - - - :master_port
+\c - - - :worker_1_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+ pg_reload_conf
+---------------------------------------------------------------------
+ t
+(1 row)
+
+\c - - - :worker_2_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+ pg_reload_conf
+---------------------------------------------------------------------
+ t
+(1 row)
+
+\c - - - :master_port
 SET citus.next_shard_id TO 1220000;
 ALTER SEQUENCE pg_catalog.pg_dist_colocationid_seq RESTART 1390000;
 ALTER SEQUENCE pg_catalog.pg_dist_groupid_seq RESTART 1;
diff --git a/src/test/regress/expected/multi_cluster_management.out b/src/test/regress/expected/multi_cluster_management.out
index 3eb549ab5..bfc3a1111 100644
--- a/src/test/regress/expected/multi_cluster_management.out
+++ b/src/test/regress/expected/multi_cluster_management.out
@@ -1,3 +1,27 @@
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+ pg_reload_conf
+---------------------------------------------------------------------
+ t
+(1 row)
+
+\c - - - :worker_1_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+ pg_reload_conf
+---------------------------------------------------------------------
+ t
+(1 row)
+
+\c - - - :worker_2_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+ pg_reload_conf
+---------------------------------------------------------------------
+ t
+(1 row)
+
+\c - - - :master_port
 SET citus.next_shard_id TO 1220000;
 ALTER SEQUENCE pg_catalog.pg_dist_colocationid_seq RESTART 1390000;
 ALTER SEQUENCE pg_catalog.pg_dist_groupid_seq RESTART 1;
diff --git a/src/test/regress/pg_regress_multi.pl b/src/test/regress/pg_regress_multi.pl
index c9a85d523..32bdf5b16 100755
--- a/src/test/regress/pg_regress_multi.pl
+++ b/src/test/regress/pg_regress_multi.pl
@@ -492,6 +492,7 @@ push(@pgOptions, "citus.stat_tenants_limit = 2");
 push(@pgOptions, "citus.stat_tenants_track = 'ALL'");
 push(@pgOptions, "citus.main_db = 'regression'");
 push(@pgOptions, "citus.superuser = 'postgres'");
+push(@pgOptions, "citus.enable_ddl_propagation=false");
 
 # Some tests look at shards in pg_class, make sure we can usually see them:
 push(@pgOptions, "citus.show_shards_for_app_name_prefixes='pg_regress'");
diff --git a/src/test/regress/sql/alter_database_from_nonmain_db.sql b/src/test/regress/sql/alter_database_from_nonmain_db.sql
index 7528832b0..95d061b40 100644
--- a/src/test/regress/sql/alter_database_from_nonmain_db.sql
+++ b/src/test/regress/sql/alter_database_from_nonmain_db.sql
@@ -1,5 +1,7 @@
 SET citus.superuser TO 'postgres';
 set citus.enable_create_database_propagation=on;
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%CREATE DATABASE%";
 create database test_alter_db_from_nonmain_db;
 create database "altered_database!'2";
 reset citus.enable_create_database_propagation;
@@ -18,7 +20,12 @@ CREATE TABLESPACE "ts-needs\!escape2" LOCATION :'alter_db_tablespace';
 CREATE TABLESPACE "ts-needs\!escape2" LOCATION :'alter_db_tablespace';
 
 \c test_alter_db_from_nonmain_db
+set citus.enable_ddl_propagation=true;
+
 set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%CREATE DATABASE%";
+create database test1;
+
 set citus.grep_remote_commands = "%ALTER DATABASE%";
 
 alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
@@ -35,8 +42,10 @@ alter database "altered_database!'2" with
     IS_TEMPLATE false;
 
 \c regression
+set citus.enable_ddl_propagation=true;
 create role test_owner_non_main_db;
 \c test_alter_db_from_nonmain_db
+set citus.enable_ddl_propagation=true;
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
 set citus.enable_create_database_propagation=on;
@@ -70,6 +79,8 @@ alter database "altered_database!'2" set lock_timeout to DEFAULT;
 alter database "altered_database!'2" RESET lock_timeout;
 ALTER DATABASE "altered_database!'2" RESET ALL;
 \c regression
+show citus.enable_ddl_propagation;
+set citus.enable_ddl_propagation=true;
 set citus.enable_create_database_propagation=on;
 drop database "altered_database!'2";
 drop database test_alter_db_from_nonmain_db;
diff --git a/src/test/regress/sql/failure_setup.sql b/src/test/regress/sql/failure_setup.sql
index 4c209f14d..eb8f836cf 100644
--- a/src/test/regress/sql/failure_setup.sql
+++ b/src/test/regress/sql/failure_setup.sql
@@ -1,3 +1,15 @@
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+
+\c - - - :worker_1_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+
+\c - - - :worker_2_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+
+\c - - - :master_port
 SELECT citus.mitmproxy('conn.allow()');
 
 -- add the workers
diff --git a/src/test/regress/sql/minimal_cluster_management.sql b/src/test/regress/sql/minimal_cluster_management.sql
index 30f69d43d..1bb17c753 100644
--- a/src/test/regress/sql/minimal_cluster_management.sql
+++ b/src/test/regress/sql/minimal_cluster_management.sql
@@ -1,3 +1,24 @@
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+
+\c - - - :worker_1_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+
+\c - - - :worker_2_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+
+\c - - - :master_port
+\c - - - :worker_1_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+
+\c - - - :worker_2_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+
+\c - - - :master_port
 SET citus.next_shard_id TO 1220000;
 ALTER SEQUENCE pg_catalog.pg_dist_colocationid_seq RESTART 1390000;
 ALTER SEQUENCE pg_catalog.pg_dist_groupid_seq RESTART 1;
diff --git a/src/test/regress/sql/multi_cluster_management.sql b/src/test/regress/sql/multi_cluster_management.sql
index 86fbd15b6..ed3d574b6 100644
--- a/src/test/regress/sql/multi_cluster_management.sql
+++ b/src/test/regress/sql/multi_cluster_management.sql
@@ -1,3 +1,16 @@
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+
+\c - - - :worker_1_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+
+\c - - - :worker_2_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+
+\c - - - :master_port
+
 SET citus.next_shard_id TO 1220000;
 ALTER SEQUENCE pg_catalog.pg_dist_colocationid_seq RESTART 1390000;
 ALTER SEQUENCE pg_catalog.pg_dist_groupid_seq RESTART 1;

From d3962381e9b01b3d84cc9aefc126281b7c3ea246 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Fri, 22 Mar 2024 11:53:10 +0300
Subject: [PATCH 15/78] Fixes test errors

---
 .../alter_database_from_nonmain_db.out        | 140 +++++-------------
 .../sql/alter_database_from_nonmain_db.sql    |   9 --
 2 files changed, 35 insertions(+), 114 deletions(-)

diff --git a/src/test/regress/expected/alter_database_from_nonmain_db.out b/src/test/regress/expected/alter_database_from_nonmain_db.out
index 6716b90a2..9a1312c67 100644
--- a/src/test/regress/expected/alter_database_from_nonmain_db.out
+++ b/src/test/regress/expected/alter_database_from_nonmain_db.out
@@ -19,29 +19,24 @@ set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
 NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-ERROR:  operation is not allowed on this node
-HINT:  Connect to the coordinator and run it again.
-CONTEXT:  while executing command on localhost:xxxxx
 alter database "altered_database!'2" rename to altered_database_renamed;
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" RENAME TO altered_database_renamed', 'postgres')
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 alter database altered_database_renamed rename to "altered_database!'2";
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database_renamed RENAME TO "altered_database!''2"', 'postgres')
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 alter database "altered_database!'2" with
     ALLOW_CONNECTIONS false
     CONNECTION LIMIT 1
     IS_TEMPLATE true;
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" WITH ALLOW_CONNECTIONS false CONNECTION LIMIT 1 IS_TEMPLATE true;
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" WITH ALLOW_CONNECTIONS false CONNECTION LIMIT 1 IS_TEMPLATE true;', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-ERROR:  operation is not allowed on this node
-HINT:  Connect to the coordinator and run it again.
-CONTEXT:  while executing command on localhost:xxxxx
 alter database "altered_database!'2" with
     ALLOW_CONNECTIONS true
     CONNECTION LIMIT 0
     IS_TEMPLATE false;
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" WITH ALLOW_CONNECTIONS true CONNECTION LIMIT 0 IS_TEMPLATE false;
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" WITH ALLOW_CONNECTIONS true CONNECTION LIMIT 0 IS_TEMPLATE false;', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-ERROR:  operation is not allowed on this node
-HINT:  Connect to the coordinator and run it again.
-CONTEXT:  while executing command on localhost:xxxxx
 \c regression
 create role test_owner_non_main_db;
 \c test_alter_db_from_nonmain_db
@@ -49,149 +44,84 @@ set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
 set citus.enable_create_database_propagation=on;
 alter database "altered_database!'2" owner to test_owner_non_main_db;
-alter database "altered_database!'2" owner to CURRENT_USER;
-alter database "altered_database!'2" set default_transaction_read_only = true;
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET default_transaction_read_only = 'true'
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" OWNER TO test_owner_non_main_db;', 'postgres')
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database "altered_database!'2" owner to CURRENT_USER;
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" OWNER TO postgres;', 'postgres')
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database "altered_database!'2" set default_transaction_read_only = true;
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET default_transaction_read_only = ''true''', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-ERROR:  operation is not allowed on this node
-HINT:  Connect to the coordinator and run it again.
-CONTEXT:  while executing command on localhost:xxxxx
 set default_transaction_read_only = false;
 alter database "altered_database!'2" set default_transaction_read_only from current;
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET default_transaction_read_only FROM CURRENT
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET default_transaction_read_only FROM CURRENT', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-ERROR:  operation is not allowed on this node
-HINT:  Connect to the coordinator and run it again.
-CONTEXT:  while executing command on localhost:xxxxx
 alter database "altered_database!'2" set default_transaction_read_only to DEFAULT;
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET default_transaction_read_only TO DEFAULT
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET default_transaction_read_only TO DEFAULT', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-ERROR:  operation is not allowed on this node
-HINT:  Connect to the coordinator and run it again.
-CONTEXT:  while executing command on localhost:xxxxx
 alter database "altered_database!'2" RESET default_transaction_read_only;
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" RESET default_transaction_read_only
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" RESET default_transaction_read_only', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-ERROR:  operation is not allowed on this node
-HINT:  Connect to the coordinator and run it again.
-CONTEXT:  while executing command on localhost:xxxxx
 alter database "altered_database!'2" SET TIME ZONE '-7';
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET timezone = '-7'
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET timezone = ''-7''', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-ERROR:  operation is not allowed on this node
-HINT:  Connect to the coordinator and run it again.
-CONTEXT:  while executing command on localhost:xxxxx
 alter database "altered_database!'2" set TIME ZONE LOCAL;
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET timezone TO DEFAULT
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET timezone TO DEFAULT', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-ERROR:  operation is not allowed on this node
-HINT:  Connect to the coordinator and run it again.
-CONTEXT:  while executing command on localhost:xxxxx
 alter database "altered_database!'2" set TIME ZONE DEFAULT;
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET timezone TO DEFAULT
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET timezone TO DEFAULT', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-ERROR:  operation is not allowed on this node
-HINT:  Connect to the coordinator and run it again.
-CONTEXT:  while executing command on localhost:xxxxx
 alter database "altered_database!'2" RESET TIME ZONE;
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" RESET timezone
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" RESET timezone', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-ERROR:  operation is not allowed on this node
-HINT:  Connect to the coordinator and run it again.
-CONTEXT:  while executing command on localhost:xxxxx
 alter database "altered_database!'2" SET TIME ZONE INTERVAL '-08:00' HOUR TO MINUTE;
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TIME ZONE INTERVAL '@ 8 hours ago'
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET TIME ZONE INTERVAL ''@ 8 hours ago''', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-ERROR:  operation is not allowed on this node
-HINT:  Connect to the coordinator and run it again.
-CONTEXT:  while executing command on localhost:xxxxx
 alter database "altered_database!'2" RESET TIME ZONE;
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" RESET timezone
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" RESET timezone', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-ERROR:  operation is not allowed on this node
-HINT:  Connect to the coordinator and run it again.
-CONTEXT:  while executing command on localhost:xxxxx
 alter database "altered_database!'2" set default_transaction_isolation = 'serializable';
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET default_transaction_isolation = 'serializable'
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET default_transaction_isolation = ''serializable''', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-ERROR:  operation is not allowed on this node
-HINT:  Connect to the coordinator and run it again.
-CONTEXT:  while executing command on localhost:xxxxx
 set default_transaction_isolation = 'read committed';
 alter database "altered_database!'2" set default_transaction_isolation from current;
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET default_transaction_isolation FROM CURRENT
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET default_transaction_isolation FROM CURRENT', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-ERROR:  operation is not allowed on this node
-HINT:  Connect to the coordinator and run it again.
-CONTEXT:  while executing command on localhost:xxxxx
 alter database "altered_database!'2" set default_transaction_isolation to DEFAULT;
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET default_transaction_isolation TO DEFAULT
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET default_transaction_isolation TO DEFAULT', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-ERROR:  operation is not allowed on this node
-HINT:  Connect to the coordinator and run it again.
-CONTEXT:  while executing command on localhost:xxxxx
 alter database "altered_database!'2" RESET default_transaction_isolation;
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" RESET default_transaction_isolation
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" RESET default_transaction_isolation', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-ERROR:  operation is not allowed on this node
-HINT:  Connect to the coordinator and run it again.
-CONTEXT:  while executing command on localhost:xxxxx
 alter database "altered_database!'2" set statement_timeout = 1000;
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET statement_timeout = 1000
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET statement_timeout = 1000', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-ERROR:  operation is not allowed on this node
-HINT:  Connect to the coordinator and run it again.
-CONTEXT:  while executing command on localhost:xxxxx
 set statement_timeout = 2000;
 alter database "altered_database!'2" set statement_timeout from current;
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET statement_timeout FROM CURRENT
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET statement_timeout FROM CURRENT', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-ERROR:  operation is not allowed on this node
-HINT:  Connect to the coordinator and run it again.
-CONTEXT:  while executing command on localhost:xxxxx
 alter database "altered_database!'2" set statement_timeout to DEFAULT;
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET statement_timeout TO DEFAULT
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET statement_timeout TO DEFAULT', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-ERROR:  operation is not allowed on this node
-HINT:  Connect to the coordinator and run it again.
-CONTEXT:  while executing command on localhost:xxxxx
 alter database "altered_database!'2" RESET statement_timeout;
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" RESET statement_timeout
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" RESET statement_timeout', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-ERROR:  operation is not allowed on this node
-HINT:  Connect to the coordinator and run it again.
-CONTEXT:  while executing command on localhost:xxxxx
 alter database "altered_database!'2" set lock_timeout = 1201.5;
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET lock_timeout = 1201.5
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET lock_timeout = 1201.5', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-ERROR:  operation is not allowed on this node
-HINT:  Connect to the coordinator and run it again.
-CONTEXT:  while executing command on localhost:xxxxx
 set lock_timeout = 1202.5;
 alter database "altered_database!'2" set lock_timeout from current;
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET lock_timeout FROM CURRENT
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET lock_timeout FROM CURRENT', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-ERROR:  operation is not allowed on this node
-HINT:  Connect to the coordinator and run it again.
-CONTEXT:  while executing command on localhost:xxxxx
 alter database "altered_database!'2" set lock_timeout to DEFAULT;
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET lock_timeout TO DEFAULT
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET lock_timeout TO DEFAULT', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-ERROR:  operation is not allowed on this node
-HINT:  Connect to the coordinator and run it again.
-CONTEXT:  while executing command on localhost:xxxxx
 alter database "altered_database!'2" RESET lock_timeout;
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" RESET lock_timeout
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" RESET lock_timeout', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-ERROR:  operation is not allowed on this node
-HINT:  Connect to the coordinator and run it again.
-CONTEXT:  while executing command on localhost:xxxxx
 ALTER DATABASE "altered_database!'2" RESET ALL;
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" RESET ALL
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" RESET ALL', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-ERROR:  operation is not allowed on this node
-HINT:  Connect to the coordinator and run it again.
-CONTEXT:  while executing command on localhost:xxxxx
 \c regression
 set citus.enable_create_database_propagation=on;
 drop database "altered_database!'2";
diff --git a/src/test/regress/sql/alter_database_from_nonmain_db.sql b/src/test/regress/sql/alter_database_from_nonmain_db.sql
index 95d061b40..28cb3888b 100644
--- a/src/test/regress/sql/alter_database_from_nonmain_db.sql
+++ b/src/test/regress/sql/alter_database_from_nonmain_db.sql
@@ -1,7 +1,5 @@
 SET citus.superuser TO 'postgres';
 set citus.enable_create_database_propagation=on;
-set citus.log_remote_commands = true;
-set citus.grep_remote_commands = "%CREATE DATABASE%";
 create database test_alter_db_from_nonmain_db;
 create database "altered_database!'2";
 reset citus.enable_create_database_propagation;
@@ -20,11 +18,8 @@ CREATE TABLESPACE "ts-needs\!escape2" LOCATION :'alter_db_tablespace';
 CREATE TABLESPACE "ts-needs\!escape2" LOCATION :'alter_db_tablespace';
 
 \c test_alter_db_from_nonmain_db
-set citus.enable_ddl_propagation=true;
 
 set citus.log_remote_commands = true;
-set citus.grep_remote_commands = "%CREATE DATABASE%";
-create database test1;
 
 set citus.grep_remote_commands = "%ALTER DATABASE%";
 
@@ -42,10 +37,8 @@ alter database "altered_database!'2" with
     IS_TEMPLATE false;
 
 \c regression
-set citus.enable_ddl_propagation=true;
 create role test_owner_non_main_db;
 \c test_alter_db_from_nonmain_db
-set citus.enable_ddl_propagation=true;
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
 set citus.enable_create_database_propagation=on;
@@ -79,8 +72,6 @@ alter database "altered_database!'2" set lock_timeout to DEFAULT;
 alter database "altered_database!'2" RESET lock_timeout;
 ALTER DATABASE "altered_database!'2" RESET ALL;
 \c regression
-show citus.enable_ddl_propagation;
-set citus.enable_ddl_propagation=true;
 set citus.enable_create_database_propagation=on;
 drop database "altered_database!'2";
 drop database test_alter_db_from_nonmain_db;

From 426397fa774586a3bae6cceaa987ad7e59f4b012 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Fri, 22 Mar 2024 12:58:58 +0300
Subject: [PATCH 16/78] Adds check enabled for some sqls

---
 .../expected/minimal_cluster_management.out   | 17 -------------
 .../expected/multi_follower_sanity_check.out  | 23 +++++++++++++++++
 .../regress/expected/multi_test_helpers.out   | 24 ++++++++++++++++++
 .../expected/single_node_enterprise.out       | 25 +++++++++++++++++++
 .../sql/minimal_cluster_management.sql        |  8 ------
 .../sql/multi_follower_sanity_check.sql       | 10 ++++++++
 src/test/regress/sql/multi_test_helpers.sql   | 10 ++++++++
 .../regress/sql/single_node_enterprise.sql    | 13 ++++++++++
 8 files changed, 105 insertions(+), 25 deletions(-)

diff --git a/src/test/regress/expected/minimal_cluster_management.out b/src/test/regress/expected/minimal_cluster_management.out
index 87f8be882..09fcf7b4a 100644
--- a/src/test/regress/expected/minimal_cluster_management.out
+++ b/src/test/regress/expected/minimal_cluster_management.out
@@ -21,23 +21,6 @@ SELECT pg_reload_conf();
  t
 (1 row)
 
-\c - - - :master_port
-\c - - - :worker_1_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
- pg_reload_conf
----------------------------------------------------------------------
- t
-(1 row)
-
-\c - - - :worker_2_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
- pg_reload_conf
----------------------------------------------------------------------
- t
-(1 row)
-
 \c - - - :master_port
 SET citus.next_shard_id TO 1220000;
 ALTER SEQUENCE pg_catalog.pg_dist_colocationid_seq RESTART 1390000;
diff --git a/src/test/regress/expected/multi_follower_sanity_check.out b/src/test/regress/expected/multi_follower_sanity_check.out
index 6d5a0e1f6..b24ae4e30 100644
--- a/src/test/regress/expected/multi_follower_sanity_check.out
+++ b/src/test/regress/expected/multi_follower_sanity_check.out
@@ -1,4 +1,27 @@
 -- check that the nodes are all in read-only mode and rejecting write queries
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+ pg_reload_conf
+---------------------------------------------------------------------
+ t
+(1 row)
+
+\c - - - :worker_1_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+ pg_reload_conf
+---------------------------------------------------------------------
+ t
+(1 row)
+
+\c - - - :worker_2_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+ pg_reload_conf
+---------------------------------------------------------------------
+ t
+(1 row)
+
 \c - - - :follower_master_port
 CREATE TABLE tab (a int);
 ERROR:  cannot execute CREATE TABLE in a read-only transaction
diff --git a/src/test/regress/expected/multi_test_helpers.out b/src/test/regress/expected/multi_test_helpers.out
index 0f31f2354..36b1be250 100644
--- a/src/test/regress/expected/multi_test_helpers.out
+++ b/src/test/regress/expected/multi_test_helpers.out
@@ -1,4 +1,28 @@
 -- File to create functions and helpers needed for subsequent tests
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+ pg_reload_conf
+---------------------------------------------------------------------
+ t
+(1 row)
+
+\c - - - :worker_1_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+ pg_reload_conf
+---------------------------------------------------------------------
+ t
+(1 row)
+
+\c - - - :worker_2_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+ pg_reload_conf
+---------------------------------------------------------------------
+ t
+(1 row)
+
+\c - - - :master_port
 -- create a helper function to create objects on each node
 CREATE OR REPLACE FUNCTION run_command_on_master_and_workers(p_sql text)
 RETURNS void LANGUAGE plpgsql AS $$
diff --git a/src/test/regress/expected/single_node_enterprise.out b/src/test/regress/expected/single_node_enterprise.out
index 79f231864..9e3ffad76 100644
--- a/src/test/regress/expected/single_node_enterprise.out
+++ b/src/test/regress/expected/single_node_enterprise.out
@@ -1,3 +1,28 @@
+
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+ pg_reload_conf
+---------------------------------------------------------------------
+ t
+(1 row)
+
+\c - - - :worker_1_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+ pg_reload_conf
+---------------------------------------------------------------------
+ t
+(1 row)
+
+\c - - - :worker_2_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+ pg_reload_conf
+---------------------------------------------------------------------
+ t
+(1 row)
+
+\c - - - :master_port
 -- we already have lots of tests targeting
 -- single node citus clusters in sql/single_node.sql
 -- in this file, we are testing enterprise features
diff --git a/src/test/regress/sql/minimal_cluster_management.sql b/src/test/regress/sql/minimal_cluster_management.sql
index 1bb17c753..fffcb18fa 100644
--- a/src/test/regress/sql/minimal_cluster_management.sql
+++ b/src/test/regress/sql/minimal_cluster_management.sql
@@ -9,14 +9,6 @@ SELECT pg_reload_conf();
 ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
 SELECT pg_reload_conf();
 
-\c - - - :master_port
-\c - - - :worker_1_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
-
-\c - - - :worker_2_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
 
 \c - - - :master_port
 SET citus.next_shard_id TO 1220000;
diff --git a/src/test/regress/sql/multi_follower_sanity_check.sql b/src/test/regress/sql/multi_follower_sanity_check.sql
index 3059b9f9f..47e1d724e 100644
--- a/src/test/regress/sql/multi_follower_sanity_check.sql
+++ b/src/test/regress/sql/multi_follower_sanity_check.sql
@@ -1,4 +1,14 @@
 -- check that the nodes are all in read-only mode and rejecting write queries
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+
+\c - - - :worker_1_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+
+\c - - - :worker_2_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
 
 \c - - - :follower_master_port
 CREATE TABLE tab (a int);
diff --git a/src/test/regress/sql/multi_test_helpers.sql b/src/test/regress/sql/multi_test_helpers.sql
index 7d218361c..350240494 100644
--- a/src/test/regress/sql/multi_test_helpers.sql
+++ b/src/test/regress/sql/multi_test_helpers.sql
@@ -1,5 +1,15 @@
 -- File to create functions and helpers needed for subsequent tests
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
 
+\c - - - :worker_1_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+
+\c - - - :worker_2_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+\c - - - :master_port
 -- create a helper function to create objects on each node
 CREATE OR REPLACE FUNCTION run_command_on_master_and_workers(p_sql text)
 RETURNS void LANGUAGE plpgsql AS $$
diff --git a/src/test/regress/sql/single_node_enterprise.sql b/src/test/regress/sql/single_node_enterprise.sql
index 19393ba24..6cfbbadc8 100644
--- a/src/test/regress/sql/single_node_enterprise.sql
+++ b/src/test/regress/sql/single_node_enterprise.sql
@@ -1,6 +1,19 @@
 -- we already have lots of tests targeting
 -- single node citus clusters in sql/single_node.sql
 -- in this file, we are testing enterprise features
+
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+
+\c - - - :worker_1_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+
+\c - - - :worker_2_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+
+\c - - - :master_port
 CREATE SCHEMA single_node_ent;
 SET search_path TO single_node_ent;
 SET citus.shard_count TO 4;

From 2df875c92f5fc55e14d69e7574c5df5b0a4a650b Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Fri, 22 Mar 2024 13:38:26 +0300
Subject: [PATCH 17/78] Fixes multiextension

---
 src/test/regress/expected/multi_extension.out | 24 +++++++++++++++++++
 src/test/regress/sql/multi_extension.sql      | 11 +++++++++
 2 files changed, 35 insertions(+)

diff --git a/src/test/regress/expected/multi_extension.out b/src/test/regress/expected/multi_extension.out
index aaafce715..e24f02a8a 100644
--- a/src/test/regress/expected/multi_extension.out
+++ b/src/test/regress/expected/multi_extension.out
@@ -7,6 +7,30 @@
 -- not done yet.
 --
 -- Upgrade tests verify the objects are added in citus_finish_pg_upgrade()
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+ pg_reload_conf
+---------------------------------------------------------------------
+ t
+(1 row)
+
+\c - - - :worker_1_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+ pg_reload_conf
+---------------------------------------------------------------------
+ t
+(1 row)
+
+\c - - - :worker_2_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+ pg_reload_conf
+---------------------------------------------------------------------
+ t
+(1 row)
+
+\c - - - :master_port
 SET citus.next_shard_id TO 580000;
 CREATE SCHEMA multi_extension;
 SELECT $definition$
diff --git a/src/test/regress/sql/multi_extension.sql b/src/test/regress/sql/multi_extension.sql
index 8cbbbc3ed..f8168b07f 100644
--- a/src/test/regress/sql/multi_extension.sql
+++ b/src/test/regress/sql/multi_extension.sql
@@ -7,7 +7,18 @@
 -- not done yet.
 --
 -- Upgrade tests verify the objects are added in citus_finish_pg_upgrade()
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
 
+\c - - - :worker_1_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+
+\c - - - :worker_2_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+
+\c - - - :master_port
 SET citus.next_shard_id TO 580000;
 CREATE SCHEMA multi_extension;
 

From a4563cab4547888080285b5ecea097d874788ee6 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Fri, 22 Mar 2024 14:31:54 +0300
Subject: [PATCH 18/78] Adds additional test file for multiextension

---
 .../regress/expected/multi_extension_0.out    | 1990 +++++++++++++++++
 1 file changed, 1990 insertions(+)
 create mode 100644 src/test/regress/expected/multi_extension_0.out

diff --git a/src/test/regress/expected/multi_extension_0.out b/src/test/regress/expected/multi_extension_0.out
new file mode 100644
index 000000000..fb798c8d4
--- /dev/null
+++ b/src/test/regress/expected/multi_extension_0.out
@@ -0,0 +1,1990 @@
+--
+-- MULTI_EXTENSION
+--
+-- Tests around extension creation / upgrades
+--
+-- It'd be nice to script generation of this file, but alas, that's
+-- not done yet.
+--
+-- Upgrade tests verify the objects are added in citus_finish_pg_upgrade()
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+ pg_reload_conf
+---------------------------------------------------------------------
+ t
+(1 row)
+
+\c - - - :worker_1_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+ pg_reload_conf
+---------------------------------------------------------------------
+ t
+(1 row)
+
+\c - - - :worker_2_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+ pg_reload_conf
+---------------------------------------------------------------------
+ t
+(1 row)
+
+\c - - - :master_port
+SET citus.next_shard_id TO 580000;
+CREATE SCHEMA multi_extension;
+SELECT $definition$
+CREATE OR REPLACE FUNCTION test.maintenance_worker()
+    RETURNS pg_stat_activity
+    LANGUAGE plpgsql
+AS $$
+DECLARE
+   activity record;
+BEGIN
+    DO 'BEGIN END'; -- Force maintenance daemon to start
+    -- we don't want to wait forever; loop will exit after 20 seconds
+    FOR i IN 1 .. 200 LOOP
+        PERFORM pg_stat_clear_snapshot();
+        SELECT * INTO activity FROM pg_stat_activity
+        WHERE application_name = 'Citus Maintenance Daemon' AND datname = current_database();
+        IF activity.pid IS NOT NULL THEN
+            RETURN activity;
+        ELSE
+            PERFORM pg_sleep(0.1);
+        END IF ;
+    END LOOP;
+    -- fail if we reach the end of this loop
+    raise 'Waited too long for maintenance daemon to start';
+END;
+$$;
+$definition$ create_function_test_maintenance_worker
+\gset
+CREATE TABLE multi_extension.prev_objects(description text);
+CREATE TABLE multi_extension.extension_diff(previous_object text COLLATE "C",
+                            current_object text COLLATE "C");
+CREATE FUNCTION multi_extension.print_extension_changes()
+RETURNS TABLE(previous_object text, current_object text)
+AS $func$
+BEGIN
+    SET LOCAL search_path TO multi_extension;
+	TRUNCATE TABLE extension_diff;
+
+	CREATE TABLE current_objects AS
+	SELECT pg_catalog.pg_describe_object(classid, objid, 0)
+           || ' ' ||
+           coalesce(pg_catalog.pg_get_function_result(objid), '') AS description
+	FROM pg_catalog.pg_depend, pg_catalog.pg_extension e
+	WHERE refclassid = 'pg_catalog.pg_extension'::pg_catalog.regclass
+		AND refobjid = e.oid
+		AND deptype = 'e'
+		AND e.extname='citus';
+
+	INSERT INTO extension_diff
+	SELECT p.description previous_object, c.description current_object
+	FROM current_objects c FULL JOIN prev_objects p
+	ON p.description = c.description
+	WHERE (p.description is null OR c.description is null)
+		  AND c.description IS DISTINCT FROM 'function any_value(anyelement) anyelement'
+          AND c.description IS DISTINCT FROM 'function any_value_agg(anyelement,anyelement) anyelement';
+
+	DROP TABLE prev_objects;
+	ALTER TABLE current_objects RENAME TO prev_objects;
+
+	RETURN QUERY SELECT * FROM extension_diff ORDER BY 1, 2;
+END
+$func$ LANGUAGE plpgsql;
+CREATE SCHEMA test;
+:create_function_test_maintenance_worker
+-- check maintenance daemon is started
+SELECT datname, current_database(),
+    usename, (SELECT extowner::regrole::text FROM pg_extension WHERE extname = 'citus')
+FROM test.maintenance_worker();
+  datname   | current_database | usename  | extowner
+---------------------------------------------------------------------
+ regression | regression       | postgres | postgres
+(1 row)
+
+-- ensure no unexpected objects were created outside pg_catalog
+SELECT pgio.type, pgio.identity
+FROM pg_depend AS pgd,
+	 pg_extension AS pge,
+	 LATERAL pg_identify_object(pgd.classid, pgd.objid, pgd.objsubid) AS pgio
+WHERE pgd.refclassid = 'pg_extension'::regclass AND
+	  pgd.refobjid   = pge.oid AND
+	  pge.extname    = 'citus' AND
+	  pgio.schema    NOT IN ('pg_catalog', 'citus', 'citus_internal', 'test', 'columnar', 'columnar_internal')
+ORDER BY 1, 2;
+ type |       identity
+---------------------------------------------------------------------
+ view | public.citus_schemas
+ view | public.citus_tables
+(2 rows)
+
+-- DROP EXTENSION pre-created by the regression suite
+DROP EXTENSION citus;
+DROP EXTENSION citus_columnar;
+\c
+-- these tests switch between citus versions and call ddl's that require pg_dist_object to be created
+SET citus.enable_metadata_sync TO 'false';
+SET citus.enable_version_checks TO 'false';
+SET columnar.enable_version_checks TO 'false';
+CREATE EXTENSION citus VERSION '8.0-1';
+ALTER EXTENSION citus UPDATE TO '8.0-2';
+ALTER EXTENSION citus UPDATE TO '8.0-3';
+ALTER EXTENSION citus UPDATE TO '8.0-4';
+ALTER EXTENSION citus UPDATE TO '8.0-5';
+ALTER EXTENSION citus UPDATE TO '8.0-6';
+ALTER EXTENSION citus UPDATE TO '8.0-7';
+ALTER EXTENSION citus UPDATE TO '8.0-8';
+ALTER EXTENSION citus UPDATE TO '8.0-9';
+ALTER EXTENSION citus UPDATE TO '8.0-10';
+ALTER EXTENSION citus UPDATE TO '8.0-11';
+ALTER EXTENSION citus UPDATE TO '8.0-12';
+ALTER EXTENSION citus UPDATE TO '8.0-13';
+ALTER EXTENSION citus UPDATE TO '8.1-1';
+ALTER EXTENSION citus UPDATE TO '8.2-1';
+ALTER EXTENSION citus UPDATE TO '8.2-2';
+ALTER EXTENSION citus UPDATE TO '8.2-3';
+ALTER EXTENSION citus UPDATE TO '8.2-4';
+ALTER EXTENSION citus UPDATE TO '8.3-1';
+ALTER EXTENSION citus UPDATE TO '9.0-1';
+ALTER EXTENSION citus UPDATE TO '9.0-2';
+ALTER EXTENSION citus UPDATE TO '9.1-1';
+ALTER EXTENSION citus UPDATE TO '9.2-1';
+ALTER EXTENSION citus UPDATE TO '9.2-2';
+-- Snapshot of state at 9.2-2
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object |                                                                                                        current_object
+---------------------------------------------------------------------
+                 | event trigger citus_cascade_to_partition
+                 | function alter_role_if_exists(text,text) boolean
+                 | function array_cat_agg(anycompatiblearray) anycompatiblearray
+                 | function assign_distributed_transaction_id(integer,bigint,timestamp with time zone) void
+                 | function authinfo_valid(text) boolean
+                 | function broadcast_intermediate_result(text,text) bigint
+                 | function check_distributed_deadlocks() boolean
+                 | function citus_add_rebalance_strategy(name,regproc,regproc,regproc,real,real) void
+                 | function citus_blocking_pids(integer) integer[]
+                 | function citus_create_restore_point(text) pg_lsn
+                 | function citus_dist_stat_activity() SETOF record
+                 | function citus_drop_trigger() event_trigger
+                 | function citus_executor_name(integer) text
+                 | function citus_extradata_container(internal) void
+                 | function citus_finish_pg_upgrade() void
+                 | function citus_internal.find_groupid_for_node(text,integer) integer
+                 | function citus_internal.pg_dist_node_trigger_func() trigger
+                 | function citus_internal.pg_dist_rebalance_strategy_enterprise_check() trigger
+                 | function citus_internal.pg_dist_rebalance_strategy_trigger_func() trigger
+                 | function citus_internal.pg_dist_shard_placement_trigger_func() trigger
+                 | function citus_internal.refresh_isolation_tester_prepared_statement() void
+                 | function citus_internal.replace_isolation_tester_func() void
+                 | function citus_internal.restore_isolation_tester_func() void
+                 | function citus_isolation_test_session_is_blocked(integer,integer[]) boolean
+                 | function citus_json_concatenate(json,json) json
+                 | function citus_json_concatenate_final(json) json
+                 | function citus_jsonb_concatenate(jsonb,jsonb) jsonb
+                 | function citus_jsonb_concatenate_final(jsonb) jsonb
+                 | function citus_node_capacity_1(integer) real
+                 | function citus_prepare_pg_upgrade() void
+                 | function citus_query_stats() SETOF record
+                 | function citus_relation_size(regclass) bigint
+                 | function citus_server_id() uuid
+                 | function citus_set_default_rebalance_strategy(text) void
+                 | function citus_shard_allowed_on_node_true(bigint,integer) boolean
+                 | function citus_shard_cost_1(bigint) real
+                 | function citus_shard_cost_by_disk_size(bigint) real
+                 | function citus_stat_statements() SETOF record
+                 | function citus_stat_statements_reset() void
+                 | function citus_table_is_visible(oid) boolean
+                 | function citus_table_size(regclass) bigint
+                 | function citus_text_send_as_jsonb(text) bytea
+                 | function citus_total_relation_size(regclass) bigint
+                 | function citus_truncate_trigger() trigger
+                 | function citus_validate_rebalance_strategy_functions(regproc,regproc,regproc) void
+                 | function citus_version() text
+                 | function citus_worker_stat_activity() SETOF record
+                 | function column_name_to_column(regclass,text) text
+                 | function column_to_column_name(regclass,text) text
+                 | function coord_combine_agg(oid,cstring,anyelement) anyelement
+                 | function coord_combine_agg_ffunc(internal,oid,cstring,anyelement) anyelement
+                 | function coord_combine_agg_sfunc(internal,oid,cstring,anyelement) internal
+                 | function create_distributed_function(regprocedure,text,text) void
+                 | function create_distributed_table(regclass,text,citus.distribution_type,text) void
+                 | function create_intermediate_result(text,text) bigint
+                 | function create_reference_table(regclass) void
+                 | function distributed_tables_colocated(regclass,regclass) boolean
+                 | function dump_global_wait_edges() SETOF record
+                 | function dump_local_wait_edges() SETOF record
+                 | function fetch_intermediate_results(text[],text,integer) bigint
+                 | function get_all_active_transactions() SETOF record
+                 | function get_colocated_shard_array(bigint) bigint[]
+                 | function get_colocated_table_array(regclass) regclass[]
+                 | function get_current_transaction_id() record
+                 | function get_global_active_transactions() SETOF record
+                 | function get_rebalance_progress() TABLE(sessionid integer, table_name regclass, shardid bigint, shard_size bigint, sourcename text, sourceport integer, targetname text, targetport integer, progress bigint)
+                 | function get_rebalance_table_shards_plan(regclass,real,integer,bigint[],boolean,name) TABLE(table_name regclass, shardid bigint, shard_size bigint, sourcename text, sourceport integer, targetname text, targetport integer)
+                 | function get_shard_id_for_distribution_column(regclass,"any") bigint
+                 | function isolate_tenant_to_new_shard(regclass,"any",text) bigint
+                 | function json_cat_agg(json) json
+                 | function jsonb_cat_agg(jsonb) jsonb
+                 | function lock_relation_if_exists(text,text) boolean
+                 | function lock_shard_metadata(integer,bigint[]) void
+                 | function lock_shard_resources(integer,bigint[]) void
+                 | function mark_tables_colocated(regclass,regclass[]) void
+                 | function master_activate_node(text,integer) integer
+                 | function master_add_inactive_node(text,integer,integer,noderole,name) integer
+                 | function master_add_node(text,integer,integer,noderole,name) integer
+                 | function master_add_secondary_node(text,integer,text,integer,name) integer
+                 | function master_append_table_to_shard(bigint,text,text,integer) real
+                 | function master_apply_delete_command(text) integer
+                 | function master_conninfo_cache_invalidate() trigger
+                 | function master_copy_shard_placement(bigint,text,integer,text,integer,boolean,citus.shard_transfer_mode) void
+                 | function master_create_distributed_table(regclass,text,citus.distribution_type) void
+                 | function master_create_empty_shard(text) bigint
+                 | function master_create_worker_shards(text,integer,integer) void
+                 | function master_disable_node(text,integer) void
+                 | function master_dist_local_group_cache_invalidate() trigger
+                 | function master_dist_node_cache_invalidate() trigger
+                 | function master_dist_object_cache_invalidate() trigger
+                 | function master_dist_partition_cache_invalidate() trigger
+                 | function master_dist_placement_cache_invalidate() trigger
+                 | function master_dist_shard_cache_invalidate() trigger
+                 | function master_drain_node(text,integer,citus.shard_transfer_mode,name) void
+                 | function master_drop_all_shards(regclass,text,text) integer
+                 | function master_drop_sequences(text[]) void
+                 | function master_get_active_worker_nodes() SETOF record
+                 | function master_get_new_placementid() bigint
+                 | function master_get_new_shardid() bigint
+                 | function master_get_table_ddl_events(text) SETOF text
+                 | function master_get_table_metadata(text) record
+                 | function master_modify_multiple_shards(text) integer
+                 | function master_move_shard_placement(bigint,text,integer,text,integer,citus.shard_transfer_mode) void
+                 | function master_remove_distributed_table_metadata_from_workers(regclass,text,text) void
+                 | function master_remove_node(text,integer) void
+                 | function master_remove_partition_metadata(regclass,text,text) void
+                 | function master_run_on_worker(text[],integer[],text[],boolean) SETOF record
+                 | function master_set_node_property(text,integer,text,boolean) void
+                 | function master_unmark_object_distributed(oid,oid,integer) void
+                 | function master_update_node(integer,text,integer,boolean,integer) void
+                 | function master_update_shard_statistics(bigint) bigint
+                 | function master_update_table_statistics(regclass) void
+                 | function poolinfo_valid(text) boolean
+                 | function read_intermediate_result(text,citus_copy_format) SETOF record
+                 | function read_intermediate_results(text[],citus_copy_format) SETOF record
+                 | function rebalance_table_shards(regclass,real,integer,bigint[],citus.shard_transfer_mode,boolean,name) void
+                 | function recover_prepared_transactions() integer
+                 | function relation_is_a_known_shard(regclass) boolean
+                 | function replicate_table_shards(regclass,integer,integer,bigint[],citus.shard_transfer_mode) void
+                 | function role_exists(name) boolean
+                 | function run_command_on_colocated_placements(regclass,regclass,text,boolean) SETOF record
+                 | function run_command_on_placements(regclass,text,boolean) SETOF record
+                 | function run_command_on_shards(regclass,text,boolean) SETOF record
+                 | function run_command_on_workers(text,boolean) SETOF record
+                 | function shard_name(regclass,bigint) text
+                 | function start_metadata_sync_to_node(text,integer) void
+                 | function stop_metadata_sync_to_node(text,integer) void
+                 | function task_tracker_assign_task(bigint,integer,text) void
+                 | function task_tracker_cleanup_job(bigint) void
+                 | function task_tracker_conninfo_cache_invalidate() trigger
+                 | function task_tracker_task_status(bigint,integer) integer
+                 | function upgrade_to_reference_table(regclass) void
+                 | function worker_append_table_to_shard(text,text,text,integer) void
+                 | function worker_apply_inter_shard_ddl_command(bigint,text,bigint,text,text) void
+                 | function worker_apply_sequence_command(text) void
+                 | function worker_apply_sequence_command(text,regtype) void
+                 | function worker_apply_shard_ddl_command(bigint,text) void
+                 | function worker_apply_shard_ddl_command(bigint,text,text) void
+                 | function worker_cleanup_job_schema_cache() void
+                 | function worker_create_or_replace_object(text) boolean
+                 | function worker_create_schema(bigint,text) void
+                 | function worker_create_truncate_trigger(regclass) void
+                 | function worker_drop_distributed_table(text) void
+                 | function worker_execute_sql_task(bigint,integer,text,boolean) bigint
+                 | function worker_fetch_foreign_file(text,text,bigint,text[],integer[]) void
+                 | function worker_fetch_partition_file(bigint,integer,integer,integer,text,integer) void
+                 | function worker_hash("any") integer
+                 | function worker_hash_partition_table(bigint,integer,text,text,oid,anyarray) void
+                 | function worker_merge_files_and_run_query(bigint,integer,text,text) void
+                 | function worker_merge_files_into_table(bigint,integer,text[],text[]) void
+                 | function worker_partial_agg(oid,anyelement) cstring
+                 | function worker_partial_agg_ffunc(internal) cstring
+                 | function worker_partial_agg_sfunc(internal,oid,anyelement) internal
+                 | function worker_partition_query_result(text,text,integer,citus.distribution_type,text[],text[],boolean) SETOF record
+                 | function worker_range_partition_table(bigint,integer,text,text,oid,anyarray) void
+                 | function worker_repartition_cleanup(bigint) void
+                 | schema citus
+                 | schema citus_internal
+                 | sequence pg_dist_colocationid_seq
+                 | sequence pg_dist_groupid_seq
+                 | sequence pg_dist_node_nodeid_seq
+                 | sequence pg_dist_placement_placementid_seq
+                 | sequence pg_dist_shardid_seq
+                 | table citus.pg_dist_object
+                 | table pg_dist_authinfo
+                 | table pg_dist_colocation
+                 | table pg_dist_local_group
+                 | table pg_dist_node
+                 | table pg_dist_node_metadata
+                 | table pg_dist_partition
+                 | table pg_dist_placement
+                 | table pg_dist_poolinfo
+                 | table pg_dist_rebalance_strategy
+                 | table pg_dist_shard
+                 | table pg_dist_transaction
+                 | type citus.distribution_type
+                 | type citus.shard_transfer_mode
+                 | type citus_copy_format
+                 | type noderole
+                 | view citus_dist_stat_activity
+                 | view citus_lock_waits
+                 | view citus_shard_indexes_on_worker
+                 | view citus_shards_on_worker
+                 | view citus_stat_statements
+                 | view citus_worker_stat_activity
+                 | view pg_dist_shard_placement
+(186 rows)
+
+-- Test downgrade to 9.2-2 from 9.2-4
+ALTER EXTENSION citus UPDATE TO '9.2-4';
+ALTER EXTENSION citus UPDATE TO '9.2-2';
+-- Should be empty result since upgrade+downgrade should be a no-op
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+/*
+ * As we mistakenly bumped schema version to 9.3-1 in a bad release, we support
+ * updating citus schema from 9.3-1 to 9.2-4, but we do not support updates to 9.3-1.
+ *
+ * Hence the query below should fail.
+ */
+ALTER EXTENSION citus UPDATE TO '9.3-1';
+ERROR:  extension "citus" has no update path from version "9.2-2" to version "9.3-1"
+ALTER EXTENSION citus UPDATE TO '9.2-4';
+-- Snapshot of state at 9.2-4
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Test downgrade to 9.2-4 from 9.3-2
+ALTER EXTENSION citus UPDATE TO '9.3-2';
+ALTER EXTENSION citus UPDATE TO '9.2-4';
+-- Should be empty result since upgrade+downgrade should be a no-op
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Snapshot of state at 9.3-2
+ALTER EXTENSION citus UPDATE TO '9.3-2';
+SELECT * FROM multi_extension.print_extension_changes();
+                  previous_object                  |                            current_object
+---------------------------------------------------------------------
+ function citus_extradata_container(internal) void |
+                                                   | function citus_extradata_container(internal) SETOF record
+                                                   | function citus_remote_connection_stats() SETOF record
+                                                   | function replicate_reference_tables() void
+                                                   | function truncate_local_data_after_distributing_table(regclass) void
+                                                   | function update_distributed_table_colocation(regclass,text) void
+                                                   | function worker_create_or_alter_role(text,text,text) boolean
+(7 rows)
+
+-- Test downgrade to 9.3-2 from 9.4-1
+ALTER EXTENSION citus UPDATE TO '9.4-1';
+ALTER EXTENSION citus UPDATE TO '9.3-2';
+-- Should be empty result since upgrade+downgrade should be a no-op
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Snapshot of state at 9.4-1
+ALTER EXTENSION citus UPDATE TO '9.4-1';
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object |                                                    current_object
+---------------------------------------------------------------------
+                 | function worker_last_saved_explain_analyze() TABLE(explain_analyze_output text, execution_duration double precision)
+                 | function worker_save_query_explain_analyze(text,jsonb) SETOF record
+(2 rows)
+
+-- Test upgrade paths for backported citus_pg_upgrade functions
+ALTER EXTENSION citus UPDATE TO '9.4-2';
+ALTER EXTENSION citus UPDATE TO '9.4-1';
+-- Should be empty result, even though the downgrade doesn't undo the upgrade, the
+-- function signature doesn't change, which is reflected here.
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+ALTER EXTENSION citus UPDATE TO '9.4-2';
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Snapshot of state at 9.4-1
+ALTER EXTENSION citus UPDATE TO '9.4-1';
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Test upgrade paths for backported improvement of master_update_table_statistics function
+ALTER EXTENSION citus UPDATE TO '9.4-3';
+-- should see the new source code with internal function citus_update_table_statistics
+SELECT prosrc FROM pg_proc WHERE proname = 'master_update_table_statistics' ORDER BY 1;
+            prosrc
+---------------------------------------------------------------------
+ citus_update_table_statistics
+(1 row)
+
+ALTER EXTENSION citus UPDATE TO '9.4-2';
+-- should see the old source code
+SELECT prosrc FROM pg_proc WHERE proname = 'master_update_table_statistics' ORDER BY 1;
+                               prosrc
+---------------------------------------------------------------------
+                                                                   +
+ DECLARE                                                           +
+  colocated_tables regclass[];                                     +
+ BEGIN                                                             +
+  SELECT get_colocated_table_array(relation) INTO colocated_tables;+
+  PERFORM                                                          +
+   master_update_shard_statistics(shardid)                         +
+  FROM                                                             +
+   pg_dist_shard                                                   +
+  WHERE                                                            +
+   logicalrelid = ANY (colocated_tables);                          +
+ END;                                                              +
+
+(1 row)
+
+-- Should be empty result
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+ALTER EXTENSION citus UPDATE TO '9.4-3';
+-- should see the new source code with internal function citus_update_table_statistics
+SELECT prosrc FROM pg_proc WHERE proname = 'master_update_table_statistics' ORDER BY 1;
+            prosrc
+---------------------------------------------------------------------
+ citus_update_table_statistics
+(1 row)
+
+-- Should be empty result
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Snapshot of state at 9.4-1
+ALTER EXTENSION citus UPDATE TO '9.4-1';
+-- should see the old source code
+SELECT prosrc FROM pg_proc WHERE proname = 'master_update_table_statistics' ORDER BY 1;
+                               prosrc
+---------------------------------------------------------------------
+                                                                   +
+ DECLARE                                                           +
+  colocated_tables regclass[];                                     +
+ BEGIN                                                             +
+  SELECT get_colocated_table_array(relation) INTO colocated_tables;+
+  PERFORM                                                          +
+   master_update_shard_statistics(shardid)                         +
+  FROM                                                             +
+   pg_dist_shard                                                   +
+  WHERE                                                            +
+   logicalrelid = ANY (colocated_tables);                          +
+ END;                                                              +
+
+(1 row)
+
+-- Should be empty result
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Test downgrade to 9.4-1 from 9.5-1
+ALTER EXTENSION citus UPDATE TO '9.5-1';
+-- TODO: This test should be moved to a valid downgrade testing suite where the downgrade is done, both on the schema and the binaries. Later changes in Citus made a C vs Schema discrepancy error here
+-- BEGIN;
+--   SET citus.enable_metadata_sync TO on;
+--   SELECT master_add_node('localhost', :master_port, groupId=>0);
+--   CREATE TABLE citus_local_table (a int);
+--   SELECT create_citus_local_table('citus_local_table');
+--   RESET citus.enable_metadata_sync;
+--
+--   -- downgrade from 9.5-1 to 9.4-1 should fail as we have a citus local table
+--   ALTER EXTENSION citus UPDATE TO '9.4-1';
+-- ROLLBACK;
+-- now we can downgrade as there is no citus local table
+ALTER EXTENSION citus UPDATE TO '9.4-1';
+-- Should be empty result since upgrade+downgrade should be a no-op
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Snapshot of state at 9.5-1
+ALTER EXTENSION citus UPDATE TO '9.5-1';
+SELECT * FROM multi_extension.print_extension_changes();
+                             previous_object                              |                             current_object
+---------------------------------------------------------------------
+ function master_drop_sequences(text[]) void                              |
+ function task_tracker_assign_task(bigint,integer,text) void              |
+ function task_tracker_cleanup_job(bigint) void                           |
+ function task_tracker_conninfo_cache_invalidate() trigger                |
+ function task_tracker_task_status(bigint,integer) integer                |
+ function worker_execute_sql_task(bigint,integer,text,boolean) bigint     |
+ function worker_merge_files_and_run_query(bigint,integer,text,text) void |
+                                                                          | function create_citus_local_table(regclass) void
+                                                                          | function undistribute_table(regclass) void
+                                                                          | function worker_record_sequence_dependency(regclass,regclass,name) void
+(10 rows)
+
+-- Test upgrade paths for backported citus_pg_upgrade functions
+ALTER EXTENSION citus UPDATE TO '9.5-2';
+ALTER EXTENSION citus UPDATE TO '9.5-1';
+-- Should be empty result, even though the downgrade doesn't undo the upgrade, the
+-- function signature doesn't change, which is reflected here.
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+ALTER EXTENSION citus UPDATE TO '9.5-2';
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Snapshot of state at 9.5-1
+ALTER EXTENSION citus UPDATE TO '9.5-1';
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Test upgrade paths for backported improvement of master_update_table_statistics function
+ALTER EXTENSION citus UPDATE TO '9.5-3';
+-- should see the new source code with internal function citus_update_table_statistics
+SELECT prosrc FROM pg_proc WHERE proname = 'master_update_table_statistics' ORDER BY 1;
+            prosrc
+---------------------------------------------------------------------
+ citus_update_table_statistics
+(1 row)
+
+ALTER EXTENSION citus UPDATE TO '9.5-2';
+-- should see the old source code
+SELECT prosrc FROM pg_proc WHERE proname = 'master_update_table_statistics' ORDER BY 1;
+                               prosrc
+---------------------------------------------------------------------
+                                                                   +
+ DECLARE                                                           +
+  colocated_tables regclass[];                                     +
+ BEGIN                                                             +
+  SELECT get_colocated_table_array(relation) INTO colocated_tables;+
+  PERFORM                                                          +
+   master_update_shard_statistics(shardid)                         +
+  FROM                                                             +
+   pg_dist_shard                                                   +
+  WHERE                                                            +
+   logicalrelid = ANY (colocated_tables);                          +
+ END;                                                              +
+
+(1 row)
+
+-- Should be empty result
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+ALTER EXTENSION citus UPDATE TO '9.5-3';
+-- should see the new source code with internal function citus_update_table_statistics
+SELECT prosrc FROM pg_proc WHERE proname = 'master_update_table_statistics' ORDER BY 1;
+            prosrc
+---------------------------------------------------------------------
+ citus_update_table_statistics
+(1 row)
+
+-- Should be empty result
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Snapshot of state at 9.5-1
+ALTER EXTENSION citus UPDATE TO '9.5-1';
+-- should see the old source code
+SELECT prosrc FROM pg_proc WHERE proname = 'master_update_table_statistics' ORDER BY 1;
+                               prosrc
+---------------------------------------------------------------------
+                                                                   +
+ DECLARE                                                           +
+  colocated_tables regclass[];                                     +
+ BEGIN                                                             +
+  SELECT get_colocated_table_array(relation) INTO colocated_tables;+
+  PERFORM                                                          +
+   master_update_shard_statistics(shardid)                         +
+  FROM                                                             +
+   pg_dist_shard                                                   +
+  WHERE                                                            +
+   logicalrelid = ANY (colocated_tables);                          +
+ END;                                                              +
+
+(1 row)
+
+-- Should be empty result
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- We removed the upgrade paths to 10.0-1, 10.0-2 and 10.0-3 due to a bug that blocked
+-- upgrades to 10.0, Therefore we test upgrades to 10.0-4 instead
+-- Test downgrade to 9.5-1 from 10.0-4
+ALTER EXTENSION citus UPDATE TO '10.0-4';
+ALTER EXTENSION citus UPDATE TO '9.5-1';
+-- Should be empty result since upgrade+downgrade should be a no-op
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Snapshot of state at 10.0-4
+ALTER EXTENSION citus UPDATE TO '10.0-4';
+SELECT * FROM multi_extension.print_extension_changes();
+                                   previous_object                                    |                                                current_object
+---------------------------------------------------------------------
+ function citus_total_relation_size(regclass) bigint                                  |
+ function create_citus_local_table(regclass) void                                     |
+ function mark_tables_colocated(regclass,regclass[]) void                             |
+ function master_conninfo_cache_invalidate() trigger                                  |
+ function master_create_distributed_table(regclass,text,citus.distribution_type) void |
+ function master_create_worker_shards(text,integer,integer) void                      |
+ function master_dist_local_group_cache_invalidate() trigger                          |
+ function master_dist_node_cache_invalidate() trigger                                 |
+ function master_dist_object_cache_invalidate() trigger                               |
+ function master_dist_partition_cache_invalidate() trigger                            |
+ function master_dist_placement_cache_invalidate() trigger                            |
+ function master_dist_shard_cache_invalidate() trigger                                |
+ function master_drop_all_shards(regclass,text,text) integer                          |
+ function master_modify_multiple_shards(text) integer                                 |
+ function undistribute_table(regclass) void                                           |
+ function upgrade_to_reference_table(regclass) void                                   |
+                                                                                      | access method columnar
+                                                                                      | function alter_columnar_table_reset(regclass,boolean,boolean,boolean,boolean) void
+                                                                                      | function alter_columnar_table_set(regclass,integer,integer,name,integer) void
+                                                                                      | function alter_distributed_table(regclass,text,integer,text,boolean) void
+                                                                                      | function alter_old_partitions_set_access_method(regclass,timestamp with time zone,name)
+                                                                                      | function alter_table_set_access_method(regclass,text) void
+                                                                                      | function citus_activate_node(text,integer) integer
+                                                                                      | function citus_add_inactive_node(text,integer,integer,noderole,name) integer
+                                                                                      | function citus_add_local_table_to_metadata(regclass,boolean) void
+                                                                                      | function citus_add_node(text,integer,integer,noderole,name) integer
+                                                                                      | function citus_add_secondary_node(text,integer,text,integer,name) integer
+                                                                                      | function citus_conninfo_cache_invalidate() trigger
+                                                                                      | function citus_copy_shard_placement(bigint,text,integer,text,integer,boolean,citus.shard_transfer_mode) void
+                                                                                      | function citus_disable_node(text,integer) void
+                                                                                      | function citus_dist_local_group_cache_invalidate() trigger
+                                                                                      | function citus_dist_node_cache_invalidate() trigger
+                                                                                      | function citus_dist_object_cache_invalidate() trigger
+                                                                                      | function citus_dist_partition_cache_invalidate() trigger
+                                                                                      | function citus_dist_placement_cache_invalidate() trigger
+                                                                                      | function citus_dist_shard_cache_invalidate() trigger
+                                                                                      | function citus_drain_node(text,integer,citus.shard_transfer_mode,name) void
+                                                                                      | function citus_drop_all_shards(regclass,text,text) integer
+                                                                                      | function citus_get_active_worker_nodes() SETOF record
+                                                                                      | function citus_internal.columnar_ensure_objects_exist() void
+                                                                                      | function citus_move_shard_placement(bigint,text,integer,text,integer,citus.shard_transfer_mode) void
+                                                                                      | function citus_remove_node(text,integer) void
+                                                                                      | function citus_set_coordinator_host(text,integer,noderole,name) void
+                                                                                      | function citus_set_node_property(text,integer,text,boolean) void
+                                                                                      | function citus_shard_sizes() SETOF record
+                                                                                      | function citus_total_relation_size(regclass,boolean) bigint
+                                                                                      | function citus_unmark_object_distributed(oid,oid,integer) void
+                                                                                      | function citus_update_node(integer,text,integer,boolean,integer) void
+                                                                                      | function citus_update_shard_statistics(bigint) bigint
+                                                                                      | function citus_update_table_statistics(regclass) void
+                                                                                      | function columnar.columnar_handler(internal) table_am_handler
+                                                                                      | function fix_pre_citus10_partitioned_table_constraint_names() SETOF regclass
+                                                                                      | function fix_pre_citus10_partitioned_table_constraint_names(regclass) void
+                                                                                      | function notify_constraint_dropped() void
+                                                                                      | function remove_local_tables_from_metadata() void
+                                                                                      | function time_partition_range(regclass) record
+                                                                                      | function undistribute_table(regclass,boolean) void
+                                                                                      | function worker_change_sequence_dependency(regclass,regclass,regclass) void
+                                                                                      | function worker_fix_pre_citus10_partitioned_table_constraint_names(regclass,bigint,text) void
+                                                                                      | schema columnar
+                                                                                      | sequence columnar.storageid_seq
+                                                                                      | table columnar.chunk
+                                                                                      | table columnar.chunk_group
+                                                                                      | table columnar.options
+                                                                                      | table columnar.stripe
+                                                                                      | view citus_shards
+                                                                                      | view public.citus_tables
+                                                                                      | view time_partitions
+(68 rows)
+
+-- check that we depend on the existence of public schema, and we can not drop it now
+DROP SCHEMA public;
+ERROR:  cannot drop schema public because other objects depend on it
+DETAIL:  extension citus depends on schema public
+HINT:  Use DROP ... CASCADE to drop the dependent objects too.
+-- verify that citus_tables view is on pg_catalog if public schema is absent.
+ALTER EXTENSION citus UPDATE TO '9.5-1';
+DROP SCHEMA public;
+ALTER EXTENSION citus UPDATE TO '10.0-4';
+SELECT * FROM multi_extension.print_extension_changes();
+      previous_object      |   current_object
+---------------------------------------------------------------------
+ view public.citus_tables  |
+                           | view citus_tables
+(2 rows)
+
+-- recreate public schema, and recreate citus_tables in the public schema by default
+CREATE SCHEMA public;
+-- In PG15, public schema is owned by pg_database_owner role
+-- Relevant PG commit: b073c3ccd06e4cb845e121387a43faa8c68a7b62
+SHOW server_version \gset
+SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15
+\gset
+\if :server_version_ge_15
+ALTER SCHEMA public OWNER TO pg_database_owner;
+\endif
+GRANT ALL ON SCHEMA public TO public;
+ALTER EXTENSION citus UPDATE TO '9.5-1';
+ALTER EXTENSION citus UPDATE TO '10.0-4';
+SELECT * FROM multi_extension.print_extension_changes();
+  previous_object   |      current_object
+---------------------------------------------------------------------
+ view citus_tables  |
+                    | view public.citus_tables
+(2 rows)
+
+-- not print "HINT: " to hide current lib version
+\set VERBOSITY terse
+CREATE TABLE columnar_table(a INT, b INT) USING columnar;
+SET citus.enable_version_checks TO ON;
+SET columnar.enable_version_checks TO ON;
+-- all should throw an error due to version mismatch
+VACUUM FULL columnar_table;
+ERROR:  loaded Citus library version differs from installed extension version
+INSERT INTO columnar_table SELECT i FROM generate_series(1, 10) i;
+ERROR:  loaded Citus library version differs from installed extension version
+VACUUM columnar_table;
+WARNING:  loaded Citus library version differs from installed extension version
+TRUNCATE columnar_table;
+ERROR:  loaded Citus library version differs from installed extension version
+DROP TABLE columnar_table;
+ERROR:  loaded Citus library version differs from installed extension version
+CREATE INDEX ON columnar_table (a);
+ERROR:  loaded Citus library version differs from installed extension version
+ALTER TABLE columnar_table SET(columnar.compression = pglz);
+ERROR:  loaded Citus library version differs from installed extension version
+ALTER TABLE columnar_table RESET (columnar.compression);
+ERROR:  loaded Citus library version differs from installed extension version
+INSERT INTO columnar_table SELECT * FROM columnar_table;
+ERROR:  loaded Citus library version differs from installed extension version
+SELECT 1 FROM columnar_table; -- columnar custom scan
+ERROR:  loaded Citus library version differs from installed extension version
+SET columnar.enable_custom_scan TO OFF;
+SELECT 1 FROM columnar_table; -- seq scan
+ERROR:  loaded Citus library version differs from installed extension version
+CREATE TABLE new_columnar_table (a int) USING columnar;
+ERROR:  loaded Citus library version differs from installed extension version
+-- disable version checks for other sessions too
+ALTER SYSTEM SET citus.enable_version_checks TO OFF;
+SELECT pg_reload_conf();
+ pg_reload_conf
+---------------------------------------------------------------------
+ t
+(1 row)
+
+-- do cleanup for the rest of the tests
+SET citus.enable_version_checks TO OFF;
+SET columnar.enable_version_checks TO OFF;
+DROP TABLE columnar_table;
+RESET columnar.enable_custom_scan;
+\set VERBOSITY default
+-- Test downgrade to 10.0-4 from 10.1-1
+ALTER EXTENSION citus UPDATE TO '10.1-1';
+ALTER EXTENSION citus UPDATE TO '10.0-4';
+-- Should be empty result since upgrade+downgrade should be a no-op
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Snapshot of state at 10.1-1
+ALTER EXTENSION citus UPDATE TO '10.1-1';
+SELECT * FROM multi_extension.print_extension_changes();
+                                                                                                        previous_object                                                                                                        |                                                                                                                          current_object
+---------------------------------------------------------------------
+ function citus_add_rebalance_strategy(name,regproc,regproc,regproc,real,real) void                                                                                                                                            |
+ function citus_internal.columnar_ensure_objects_exist() void                                                                                                                                                                  |
+ function citus_internal.pg_dist_rebalance_strategy_enterprise_check() trigger                                                                                                                                                 |
+ function create_distributed_table(regclass,text,citus.distribution_type,text) void                                                                                                                                            |
+ function get_rebalance_progress() TABLE(sessionid integer, table_name regclass, shardid bigint, shard_size bigint, sourcename text, sourceport integer, targetname text, targetport integer, progress bigint)                 |
+ function get_rebalance_table_shards_plan(regclass,real,integer,bigint[],boolean,name) TABLE(table_name regclass, shardid bigint, shard_size bigint, sourcename text, sourceport integer, targetname text, targetport integer) |
+                                                                                                                                                                                                                               | function citus_add_rebalance_strategy(name,regproc,regproc,regproc,real,real,real) void
+                                                                                                                                                                                                                               | function citus_cleanup_orphaned_shards()
+                                                                                                                                                                                                                               | function citus_local_disk_space_stats() record
+                                                                                                                                                                                                                               | function create_distributed_table(regclass,text,citus.distribution_type,text,integer) void
+                                                                                                                                                                                                                               | function get_rebalance_progress() TABLE(sessionid integer, table_name regclass, shardid bigint, shard_size bigint, sourcename text, sourceport integer, targetname text, targetport integer, progress bigint, source_shard_size bigint, target_shard_size bigint)
+                                                                                                                                                                                                                               | function get_rebalance_table_shards_plan(regclass,real,integer,bigint[],boolean,name,real) TABLE(table_name regclass, shardid bigint, shard_size bigint, sourcename text, sourceport integer, targetname text, targetport integer)
+                                                                                                                                                                                                                               | function worker_partitioned_relation_size(regclass) bigint
+                                                                                                                                                                                                                               | function worker_partitioned_relation_total_size(regclass) bigint
+                                                                                                                                                                                                                               | function worker_partitioned_table_size(regclass) bigint
+(15 rows)
+
+-- Test downgrade to 10.1-1 from 10.2-1
+ALTER EXTENSION citus UPDATE TO '10.2-1';
+ALTER EXTENSION citus UPDATE TO '10.1-1';
+-- Should be empty result since upgrade+downgrade should be a no-op
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Snapshot of state at 10.2-1
+ALTER EXTENSION citus UPDATE TO '10.2-1';
+SELECT * FROM multi_extension.print_extension_changes();
+                      previous_object                       |                                                                                     current_object
+---------------------------------------------------------------------
+ function citus_drop_all_shards(regclass,text,text) integer |
+ function stop_metadata_sync_to_node(text,integer) void     |
+                                                            | function citus_drop_all_shards(regclass,text,text,boolean) integer
+                                                            | function citus_internal.downgrade_columnar_storage(regclass) void
+                                                            | function citus_internal.upgrade_columnar_storage(regclass) void
+                                                            | function citus_internal_add_partition_metadata(regclass,"char",text,integer,"char") void
+                                                            | function citus_internal_add_placement_metadata(bigint,integer,bigint,integer,bigint) void
+                                                            | function citus_internal_add_shard_metadata(regclass,bigint,"char",text,text) void
+                                                            | function citus_internal_delete_shard_metadata(bigint) void
+                                                            | function citus_internal_update_placement_metadata(bigint,integer,integer) void
+                                                            | function citus_internal_update_relation_colocation(oid,integer) void
+                                                            | function create_time_partitions(regclass,interval,timestamp with time zone,timestamp with time zone) boolean
+                                                            | function drop_old_time_partitions(regclass,timestamp with time zone)
+                                                            | function get_missing_time_partition_ranges(regclass,interval,timestamp with time zone,timestamp with time zone) TABLE(partition_name text, range_from_value text, range_to_value text)
+                                                            | function stop_metadata_sync_to_node(text,integer,boolean) void
+                                                            | function worker_nextval(regclass) integer
+(16 rows)
+
+-- Test downgrade to 10.2-1 from 10.2-2
+ALTER EXTENSION citus UPDATE TO '10.2-2';
+ALTER EXTENSION citus UPDATE TO '10.2-1';
+-- Should be empty result since upgrade+downgrade should be a no-op
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Snapshot of state at 10.2-2
+ALTER EXTENSION citus UPDATE TO '10.2-2';
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Test downgrade to 10.2-2 from 10.2-3
+ALTER EXTENSION citus UPDATE TO '10.2-3';
+ALTER EXTENSION citus UPDATE TO '10.2-2';
+-- Should be empty result since upgrade+downgrade should be a no-op
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Snapshot of state at 10.2-3
+ALTER EXTENSION citus UPDATE TO '10.2-3';
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Test downgrade to 10.2-3 from 10.2-4
+ALTER EXTENSION citus UPDATE TO '10.2-4';
+ALTER EXTENSION citus UPDATE TO '10.2-3';
+-- Make sure that we don't delete pg_depend entries added in
+-- columnar--10.2-3--10.2-4.sql when downgrading to 10.2-3.
+SELECT COUNT(*)=10
+FROM pg_depend
+WHERE classid = 'pg_am'::regclass::oid AND
+      objid = (select oid from pg_am where amname = 'columnar') AND
+      objsubid = 0 AND
+      refclassid = 'pg_class'::regclass::oid AND
+      refobjsubid = 0 AND
+      deptype = 'n';
+ ?column?
+---------------------------------------------------------------------
+ t
+(1 row)
+
+-- Should be empty result since upgrade+downgrade should be a no-op
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Snapshot of state at 10.2-4
+ALTER EXTENSION citus UPDATE TO '10.2-4';
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object |                              current_object
+---------------------------------------------------------------------
+                 | function citus_internal.columnar_ensure_am_depends_catalog() void
+                 | function fix_all_partition_shard_index_names() SETOF regclass
+                 | function fix_partition_shard_index_names(regclass) void
+                 | function worker_fix_partition_shard_index_names(regclass,text,text) void
+(4 rows)
+
+-- There was a bug when downgrading to 10.2-2 from 10.2-4
+-- Test that we do not have any issues with this particular downgrade
+ALTER EXTENSION citus UPDATE TO '10.2-2';
+ALTER EXTENSION citus UPDATE TO '10.2-4';
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Test downgrade to 10.2-4 from 10.2-5
+ALTER EXTENSION citus UPDATE TO '10.2-5';
+ALTER EXTENSION citus UPDATE TO '10.2-4';
+-- Should be empty result since upgrade+downgrade should be a no-op
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Snapshot of state at 10.2-5
+ALTER EXTENSION citus UPDATE TO '10.2-5';
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Make sure that we defined dependencies from all rel objects (tables,
+-- indexes, sequences ..) to columnar table access method ...
+SELECT pg_class.oid INTO columnar_schema_members
+FROM pg_class, pg_namespace
+WHERE pg_namespace.oid=pg_class.relnamespace AND
+      pg_namespace.nspname='columnar';
+SELECT refobjid INTO columnar_schema_members_pg_depend
+FROM pg_depend
+WHERE classid = 'pg_am'::regclass::oid AND
+      objid = (select oid from pg_am where amname = 'columnar') AND
+      objsubid = 0 AND
+      refclassid = 'pg_class'::regclass::oid AND
+      refobjsubid = 0 AND
+      deptype = 'n';
+-- ... , so this should be empty,
+(TABLE columnar_schema_members EXCEPT TABLE columnar_schema_members_pg_depend)
+UNION
+(TABLE columnar_schema_members_pg_depend EXCEPT TABLE columnar_schema_members);
+ oid
+---------------------------------------------------------------------
+(0 rows)
+
+-- ... , and both columnar_schema_members_pg_depend & columnar_schema_members
+-- should have 10 entries.
+SELECT COUNT(*)=10 FROM columnar_schema_members_pg_depend;
+ ?column?
+---------------------------------------------------------------------
+ t
+(1 row)
+
+DROP TABLE columnar_schema_members, columnar_schema_members_pg_depend;
+-- Use a synthetic pg_dist_shard record to show that upgrade fails
+-- when there are cstore_fdw tables
+INSERT INTO pg_dist_shard (logicalrelid, shardid, shardstorage) VALUES ('pg_dist_shard', 1, 'c');
+ALTER EXTENSION citus UPDATE TO '11.0-1';
+ERROR:  cstore_fdw tables are deprecated as of Citus 11.0
+HINT:  Install Citus 10.2 and convert your cstore_fdw tables to the columnar access method before upgrading further
+CONTEXT:  PL/pgSQL function inline_code_block line XX at RAISE
+DELETE FROM pg_dist_shard WHERE shardid = 1;
+-- partitioned table count is tracked on Citus 11 upgrade
+CREATE TABLE e_transactions(order_id varchar(255) NULL, transaction_id int) PARTITION BY LIST(transaction_id);
+CREATE TABLE orders_2020_07_01
+PARTITION OF e_transactions FOR VALUES IN (1,2,3);
+INSERT INTO pg_dist_partition VALUES ('e_transactions'::regclass,'h', '{VAR :varno 1 :varattno 1 :vartype 1043 :vartypmod 259 :varcollid 100 :varnullingrels (b) :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location -1}', 7, 's');
+SELECT
+	(metadata->>'partitioned_citus_table_exists_pre_11')::boolean as partitioned_citus_table_exists_pre_11,
+	(metadata->>'partitioned_citus_table_exists_pre_11') IS NULL as is_null
+FROM
+	pg_dist_node_metadata;
+ partitioned_citus_table_exists_pre_11 | is_null
+---------------------------------------------------------------------
+                                       | t
+(1 row)
+
+-- Test downgrade to 10.2-5 from 11.0-1
+ALTER EXTENSION citus UPDATE TO '11.0-1';
+SELECT
+	(metadata->>'partitioned_citus_table_exists_pre_11')::boolean as partitioned_citus_table_exists_pre_11,
+	(metadata->>'partitioned_citus_table_exists_pre_11') IS NULL as is_null
+FROM
+	pg_dist_node_metadata;
+ partitioned_citus_table_exists_pre_11 | is_null
+---------------------------------------------------------------------
+ t                                     | f
+(1 row)
+
+DELETE FROM pg_dist_partition WHERE logicalrelid = 'e_transactions'::regclass;
+DROP TABLE e_transactions;
+ALTER EXTENSION citus UPDATE TO '10.2-5';
+-- Should be empty result since upgrade+downgrade should be a no-op
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Snapshot of state at 11.0-1
+ALTER EXTENSION citus UPDATE TO '11.0-1';
+SELECT * FROM multi_extension.print_extension_changes();
+                                                   previous_object                                                    |                                                            current_object
+---------------------------------------------------------------------
+ function citus_disable_node(text,integer) void                                                                       |
+ function citus_dist_stat_activity() SETOF record                                                                     |
+ function citus_worker_stat_activity() SETOF record                                                                   |
+ function create_distributed_function(regprocedure,text,text) void                                                    |
+ function master_append_table_to_shard(bigint,text,text,integer) real                                                 |
+ function master_apply_delete_command(text) integer                                                                   |
+ function master_get_table_metadata(text) record                                                                      |
+ function worker_partition_query_result(text,text,integer,citus.distribution_type,text[],text[],boolean) SETOF record |
+ table citus.pg_dist_object                                                                                           |
+ view citus_worker_stat_activity                                                                                      |
+                                                                                                                      | function citus_backend_gpid() bigint
+                                                                                                                      | function citus_calculate_gpid(integer,integer) bigint
+                                                                                                                      | function citus_check_cluster_node_health() SETOF record
+                                                                                                                      | function citus_check_connection_to_node(text,integer) boolean
+                                                                                                                      | function citus_coordinator_nodeid() integer
+                                                                                                                      | function citus_disable_node(text,integer,boolean) void
+                                                                                                                      | function citus_finalize_upgrade_to_citus11(boolean) boolean
+                                                                                                                      | function citus_internal_add_colocation_metadata(integer,integer,integer,regtype,oid) void
+                                                                                                                      | function citus_internal_add_object_metadata(text,text[],text[],integer,integer,boolean) void
+                                                                                                                      | function citus_internal_delete_colocation_metadata(integer) void
+                                                                                                                      | function citus_internal_global_blocked_processes() SETOF record
+                                                                                                                      | function citus_internal_local_blocked_processes() SETOF record
+                                                                                                                      | function citus_nodeid_for_gpid(bigint) integer
+                                                                                                                      | function citus_nodename_for_nodeid(integer) text
+                                                                                                                      | function citus_nodeport_for_nodeid(integer) integer
+                                                                                                                      | function citus_pid_for_gpid(bigint) integer
+                                                                                                                      | function citus_run_local_command(text) void
+                                                                                                                      | function citus_shard_indexes_on_worker() SETOF record
+                                                                                                                      | function citus_shards_on_worker() SETOF record
+                                                                                                                      | function citus_stat_activity() SETOF record
+                                                                                                                      | function create_distributed_function(regprocedure,text,text,boolean) void
+                                                                                                                      | function get_nodeid_for_groupid(integer) integer
+                                                                                                                      | function pg_cancel_backend(bigint) boolean
+                                                                                                                      | function pg_terminate_backend(bigint,bigint) boolean
+                                                                                                                      | function run_command_on_all_nodes(text,boolean,boolean) SETOF record
+                                                                                                                      | function worker_create_or_replace_object(text[]) boolean
+                                                                                                                      | function worker_drop_sequence_dependency(text) void
+                                                                                                                      | function worker_drop_shell_table(text) void
+                                                                                                                      | function worker_partition_query_result(text,text,integer,citus.distribution_type,text[],text[],boolean,boolean,boolean) SETOF record
+                                                                                                                      | table pg_dist_object
+                                                                                                                      | view citus_stat_activity
+(41 rows)
+
+-- Test downgrade to 11.0-1 from 11.0-2
+ALTER EXTENSION citus UPDATE TO '11.0-2';
+ALTER EXTENSION citus UPDATE TO '11.0-1';
+-- Should be empty result since upgrade+downgrade should be a no-op
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Snapshot of state at 11.0-2
+ALTER EXTENSION citus UPDATE TO '11.0-2';
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object |                         current_object
+---------------------------------------------------------------------
+                 | function citus_finish_citus_upgrade()
+                 | function citus_is_coordinator() boolean
+                 | function run_command_on_coordinator(text,boolean) SETOF record
+                 | function start_metadata_sync_to_all_nodes() boolean
+(4 rows)
+
+-- Test downgrade to 11.0-2 from 11.0-3
+ALTER EXTENSION citus UPDATE TO '11.0-3';
+ALTER EXTENSION citus UPDATE TO '11.0-2';
+-- Should be empty result since upgrade+downgrade should be a no-op
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Snapshot of state at 11.0-3
+ALTER EXTENSION citus UPDATE TO '11.0-3';
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Test downgrade to 11.0-3 from 11.0-4
+ALTER EXTENSION citus UPDATE TO '11.0-4';
+ALTER EXTENSION citus UPDATE TO '11.0-3';
+-- Should be empty result since upgrade+downgrade should be a no-op
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Snapshot of state at 11.0-4
+ALTER EXTENSION citus UPDATE TO '11.0-4';
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Test downgrade to 11.0-4 from 11.1-1
+ALTER EXTENSION citus UPDATE TO '11.1-1';
+ALTER EXTENSION citus UPDATE TO '11.0-4';
+-- Should be empty result since upgrade+downgrade should be a no-op
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Test CREATE EXTENSION when Citus already exists but Citus_Columnar does not. Should skip
+CREATE EXTENSION IF NOT EXISTS citus;
+NOTICE:  extension "citus" already exists, skipping
+CREATE EXTENSION citus;
+ERROR:  extension "citus" already exists
+-- Snapshot of state at 11.1-1
+ALTER EXTENSION citus UPDATE TO '11.1-1';
+SELECT * FROM multi_extension.print_extension_changes();
+                                                                                                                          previous_object                                                                                                                          |                                                                                                                                     current_object
+---------------------------------------------------------------------
+ access method columnar                                                                                                                                                                                                                                            |
+ function alter_columnar_table_reset(regclass,boolean,boolean,boolean,boolean) void                                                                                                                                                                                |
+ function alter_columnar_table_set(regclass,integer,integer,name,integer) void                                                                                                                                                                                     |
+ function citus_copy_shard_placement(bigint,text,integer,text,integer,boolean,citus.shard_transfer_mode) void                                                                                                                                                      |
+ function citus_internal.columnar_ensure_am_depends_catalog() void                                                                                                                                                                                                 |
+ function citus_internal.downgrade_columnar_storage(regclass) void                                                                                                                                                                                                 |
+ function citus_internal.upgrade_columnar_storage(regclass) void                                                                                                                                                                                                   |
+ function columnar.columnar_handler(internal) table_am_handler                                                                                                                                                                                                     |
+ function get_rebalance_progress() TABLE(sessionid integer, table_name regclass, shardid bigint, shard_size bigint, sourcename text, sourceport integer, targetname text, targetport integer, progress bigint, source_shard_size bigint, target_shard_size bigint) |
+ function isolate_tenant_to_new_shard(regclass,"any",text) bigint                                                                                                                                                                                                  |
+ function replicate_reference_tables() void                                                                                                                                                                                                                        |
+ function worker_cleanup_job_schema_cache() void                                                                                                                                                                                                                   |
+ function worker_create_schema(bigint,text) void                                                                                                                                                                                                                   |
+ function worker_fetch_foreign_file(text,text,bigint,text[],integer[]) void                                                                                                                                                                                        |
+ function worker_fetch_partition_file(bigint,integer,integer,integer,text,integer) void                                                                                                                                                                            |
+ function worker_hash_partition_table(bigint,integer,text,text,oid,anyarray) void                                                                                                                                                                                  |
+ function worker_merge_files_into_table(bigint,integer,text[],text[]) void                                                                                                                                                                                         |
+ function worker_range_partition_table(bigint,integer,text,text,oid,anyarray) void                                                                                                                                                                                 |
+ function worker_repartition_cleanup(bigint) void                                                                                                                                                                                                                  |
+ schema columnar                                                                                                                                                                                                                                                   |
+ sequence columnar.storageid_seq                                                                                                                                                                                                                                   |
+ table columnar.chunk                                                                                                                                                                                                                                              |
+ table columnar.chunk_group                                                                                                                                                                                                                                        |
+ table columnar.options                                                                                                                                                                                                                                            |
+ table columnar.stripe                                                                                                                                                                                                                                             |
+                                                                                                                                                                                                                                                                   | function citus_cleanup_orphaned_resources()
+                                                                                                                                                                                                                                                                   | function citus_copy_shard_placement(bigint,text,integer,text,integer,citus.shard_transfer_mode) void
+                                                                                                                                                                                                                                                                   | function citus_internal_delete_partition_metadata(regclass) void
+                                                                                                                                                                                                                                                                   | function citus_job_cancel(bigint) void
+                                                                                                                                                                                                                                                                   | function citus_job_wait(bigint,citus_job_status) void
+                                                                                                                                                                                                                                                                   | function citus_locks() SETOF record
+                                                                                                                                                                                                                                                                   | function citus_rebalance_start(name,boolean,citus.shard_transfer_mode) bigint
+                                                                                                                                                                                                                                                                   | function citus_rebalance_stop() void
+                                                                                                                                                                                                                                                                   | function citus_rebalance_wait() void
+                                                                                                                                                                                                                                                                   | function citus_split_shard_by_split_points(bigint,text[],integer[],citus.shard_transfer_mode) void
+                                                                                                                                                                                                                                                                   | function create_distributed_table_concurrently(regclass,text,citus.distribution_type,text,integer) void
+                                                                                                                                                                                                                                                                   | function get_rebalance_progress() TABLE(sessionid integer, table_name regclass, shardid bigint, shard_size bigint, sourcename text, sourceport integer, targetname text, targetport integer, progress bigint, source_shard_size bigint, target_shard_size bigint, operation_type text)
+                                                                                                                                                                                                                                                                   | function isolate_tenant_to_new_shard(regclass,"any",text,citus.shard_transfer_mode) bigint
+                                                                                                                                                                                                                                                                   | function replicate_reference_tables(citus.shard_transfer_mode) void
+                                                                                                                                                                                                                                                                   | function worker_copy_table_to_node(regclass,integer) void
+                                                                                                                                                                                                                                                                   | function worker_split_copy(bigint,text,split_copy_info[]) void
+                                                                                                                                                                                                                                                                   | function worker_split_shard_release_dsm() void
+                                                                                                                                                                                                                                                                   | function worker_split_shard_replication_setup(split_shard_info[]) SETOF replication_slot_info
+                                                                                                                                                                                                                                                                   | sequence pg_dist_background_job_job_id_seq
+                                                                                                                                                                                                                                                                   | sequence pg_dist_background_task_task_id_seq
+                                                                                                                                                                                                                                                                   | sequence pg_dist_cleanup_recordid_seq
+                                                                                                                                                                                                                                                                   | sequence pg_dist_operationid_seq
+                                                                                                                                                                                                                                                                   | table pg_dist_background_job
+                                                                                                                                                                                                                                                                   | table pg_dist_background_task
+                                                                                                                                                                                                                                                                   | table pg_dist_background_task_depend
+                                                                                                                                                                                                                                                                   | table pg_dist_cleanup
+                                                                                                                                                                                                                                                                   | type citus_job_status
+                                                                                                                                                                                                                                                                   | type citus_task_status
+                                                                                                                                                                                                                                                                   | type replication_slot_info
+                                                                                                                                                                                                                                                                   | type split_copy_info
+                                                                                                                                                                                                                                                                   | type split_shard_info
+                                                                                                                                                                                                                                                                   | view citus_locks
+(57 rows)
+
+-- Test downgrade to 11.1-1 from 11.2-1
+ALTER EXTENSION citus UPDATE TO '11.2-1';
+ALTER EXTENSION citus UPDATE TO '11.1-1';
+-- Should be empty result since upgrade+downgrade should be a no-op
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Snapshot of state at 11.2-1
+ALTER EXTENSION citus UPDATE TO '11.2-1';
+ALTER EXTENSION citus_columnar UPDATE TO '11.2-1';
+-- Make sure that we defined dependencies from all rel objects (tables,
+-- indexes, sequences ..) to columnar table access method ...
+SELECT pg_class.oid INTO columnar_schema_members
+FROM pg_class, pg_namespace
+WHERE pg_namespace.oid=pg_class.relnamespace AND
+      pg_namespace.nspname='columnar_internal' AND
+      pg_class.relname NOT IN ('chunk_group_pkey',
+                               'chunk_pkey',
+                               'options_pkey',
+                               'stripe_first_row_number_idx',
+                               'stripe_pkey');
+SELECT refobjid INTO columnar_schema_members_pg_depend
+FROM pg_depend
+WHERE classid = 'pg_am'::regclass::oid AND
+      objid = (select oid from pg_am where amname = 'columnar') AND
+      objsubid = 0 AND
+      refclassid = 'pg_class'::regclass::oid AND
+      refobjsubid = 0 AND
+      deptype = 'n';
+-- ... , so this should be empty,
+(TABLE columnar_schema_members EXCEPT TABLE columnar_schema_members_pg_depend)
+UNION
+(TABLE columnar_schema_members_pg_depend EXCEPT TABLE columnar_schema_members);
+ oid
+---------------------------------------------------------------------
+(0 rows)
+
+-- ... , and both columnar_schema_members_pg_depend & columnar_schema_members
+-- should have 5 entries.
+SELECT COUNT(*)=5 FROM columnar_schema_members_pg_depend;
+ ?column?
+---------------------------------------------------------------------
+ t
+(1 row)
+
+DROP TABLE columnar_schema_members, columnar_schema_members_pg_depend;
+SELECT * FROM multi_extension.print_extension_changes();
+                                                                                                                                    previous_object                                                                                                                                     |                                                                                                                                                              current_object
+---------------------------------------------------------------------
+ function get_rebalance_progress() TABLE(sessionid integer, table_name regclass, shardid bigint, shard_size bigint, sourcename text, sourceport integer, targetname text, targetport integer, progress bigint, source_shard_size bigint, target_shard_size bigint, operation_type text) |
+ function worker_append_table_to_shard(text,text,text,integer) void                                                                                                                                                                                                                     |
+ function worker_split_shard_replication_setup(split_shard_info[]) SETOF replication_slot_info                                                                                                                                                                                          |
+                                                                                                                                                                                                                                                                                        | function citus_copy_shard_placement(bigint,integer,integer,citus.shard_transfer_mode) void
+                                                                                                                                                                                                                                                                                        | function citus_get_node_clock() cluster_clock
+                                                                                                                                                                                                                                                                                        | function citus_get_transaction_clock() cluster_clock
+                                                                                                                                                                                                                                                                                        | function citus_internal_add_placement_metadata(bigint,bigint,integer,bigint) void
+                                                                                                                                                                                                                                                                                        | function citus_internal_adjust_local_clock_to_remote(cluster_clock) void
+                                                                                                                                                                                                                                                                                        | function citus_is_clock_after(cluster_clock,cluster_clock) boolean
+                                                                                                                                                                                                                                                                                        | function citus_job_list() TABLE(job_id bigint, state citus_job_status, job_type name, description text, started_at timestamp with time zone, finished_at timestamp with time zone)
+                                                                                                                                                                                                                                                                                        | function citus_job_status(bigint,boolean) TABLE(job_id bigint, state citus_job_status, job_type name, description text, started_at timestamp with time zone, finished_at timestamp with time zone, details jsonb)
+                                                                                                                                                                                                                                                                                        | function citus_move_shard_placement(bigint,integer,integer,citus.shard_transfer_mode) void
+                                                                                                                                                                                                                                                                                        | function citus_rebalance_status(boolean) TABLE(job_id bigint, state citus_job_status, job_type name, description text, started_at timestamp with time zone, finished_at timestamp with time zone, details jsonb)
+                                                                                                                                                                                                                                                                                        | function citus_task_wait(bigint,citus_task_status) void
+                                                                                                                                                                                                                                                                                        | function cluster_clock_cmp(cluster_clock,cluster_clock) integer
+                                                                                                                                                                                                                                                                                        | function cluster_clock_eq(cluster_clock,cluster_clock) boolean
+                                                                                                                                                                                                                                                                                        | function cluster_clock_ge(cluster_clock,cluster_clock) boolean
+                                                                                                                                                                                                                                                                                        | function cluster_clock_gt(cluster_clock,cluster_clock) boolean
+                                                                                                                                                                                                                                                                                        | function cluster_clock_in(cstring) cluster_clock
+                                                                                                                                                                                                                                                                                        | function cluster_clock_le(cluster_clock,cluster_clock) boolean
+                                                                                                                                                                                                                                                                                        | function cluster_clock_logical(cluster_clock) bigint
+                                                                                                                                                                                                                                                                                        | function cluster_clock_lt(cluster_clock,cluster_clock) boolean
+                                                                                                                                                                                                                                                                                        | function cluster_clock_ne(cluster_clock,cluster_clock) boolean
+                                                                                                                                                                                                                                                                                        | function cluster_clock_out(cluster_clock) cstring
+                                                                                                                                                                                                                                                                                        | function cluster_clock_recv(internal) cluster_clock
+                                                                                                                                                                                                                                                                                        | function cluster_clock_send(cluster_clock) bytea
+                                                                                                                                                                                                                                                                                        | function get_rebalance_progress() TABLE(sessionid integer, table_name regclass, shardid bigint, shard_size bigint, sourcename text, sourceport integer, targetname text, targetport integer, progress bigint, source_shard_size bigint, target_shard_size bigint, operation_type text, source_lsn pg_lsn, target_lsn pg_lsn, status text)
+                                                                                                                                                                                                                                                                                        | function worker_split_shard_replication_setup(split_shard_info[],bigint) SETOF replication_slot_info
+                                                                                                                                                                                                                                                                                        | operator <(cluster_clock,cluster_clock)
+                                                                                                                                                                                                                                                                                        | operator <=(cluster_clock,cluster_clock)
+                                                                                                                                                                                                                                                                                        | operator <>(cluster_clock,cluster_clock)
+                                                                                                                                                                                                                                                                                        | operator =(cluster_clock,cluster_clock)
+                                                                                                                                                                                                                                                                                        | operator >(cluster_clock,cluster_clock)
+                                                                                                                                                                                                                                                                                        | operator >=(cluster_clock,cluster_clock)
+                                                                                                                                                                                                                                                                                        | operator class cluster_clock_ops for access method btree
+                                                                                                                                                                                                                                                                                        | operator family cluster_clock_ops for access method btree
+                                                                                                                                                                                                                                                                                        | sequence pg_dist_clock_logical_seq
+                                                                                                                                                                                                                                                                                        | type cluster_clock
+(38 rows)
+
+-- Test downgrade to 11.2-1 from 11.2-2
+ALTER EXTENSION citus UPDATE TO '11.2-2';
+ALTER EXTENSION citus UPDATE TO '11.2-1';
+-- Should be empty result since upgrade+downgrade should be a no-op
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Snapshot of state at 11.2-2
+ALTER EXTENSION citus UPDATE TO '11.2-2';
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object |                          current_object
+---------------------------------------------------------------------
+                 | function worker_adjust_identity_column_seq_ranges(regclass) void
+(1 row)
+
+-- Test downgrade to 11.2-2 from 11.3-1
+ALTER EXTENSION citus UPDATE TO '11.3-1';
+ALTER EXTENSION citus UPDATE TO '11.2-2';
+-- Should be empty result since upgrade+downgrade should be a no-op
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Snapshot of state at 11.3-1
+ALTER EXTENSION citus UPDATE TO '11.3-1';
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object |                             current_object
+---------------------------------------------------------------------
+                 | function citus_internal_is_replication_origin_tracking_active() boolean
+                 | function citus_internal_mark_node_not_synced(integer,integer) void
+                 | function citus_internal_start_replication_origin_tracking() void
+                 | function citus_internal_stop_replication_origin_tracking() void
+                 | function citus_stat_tenants(boolean) SETOF record
+                 | function citus_stat_tenants_local(boolean) SETOF record
+                 | function citus_stat_tenants_local_reset() void
+                 | function citus_stat_tenants_reset() void
+                 | function worker_drop_all_shell_tables(boolean)
+                 | view citus_stat_tenants
+                 | view citus_stat_tenants_local
+(11 rows)
+
+-- Test downgrade to 11.3-1 from 11.3-2
+ALTER EXTENSION citus UPDATE TO '11.3-2';
+ALTER EXTENSION citus UPDATE TO '11.3-1';
+-- Should be empty result since upgrade+downgrade should be a no-op
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Snapshot of state at 11.3-2
+ALTER EXTENSION citus UPDATE TO '11.3-2';
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Test downgrade to 11.3-2 from 12.0-1
+ALTER EXTENSION citus UPDATE TO '12.0-1';
+CREATE TABLE null_shard_key (x int, y int);
+SET citus.shard_replication_factor TO 1;
+SELECT create_distributed_table('null_shard_key', null);
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+-- Show that we cannot downgrade to 11.3-2 becuase the cluster has a
+-- distributed table with single-shard.
+ALTER EXTENSION citus UPDATE TO '11.3-2';
+ERROR:  cannot downgrade Citus because there are distributed tables without a shard key.
+DETAIL:  To downgrade Citus to an older version, you should first convert those tables to Postgres tables by executing SELECT undistribute_table("%s").
+HINT:  You can find the distributed tables without a shard key in the cluster by using the following query: "SELECT * FROM citus_tables WHERE distribution_column = '<none>' AND colocation_id > 0".
+CONTEXT:  PL/pgSQL function inline_code_block line XX at RAISE
+DROP TABLE null_shard_key;
+ALTER EXTENSION citus UPDATE TO '11.3-2';
+-- Should be empty result since upgrade+downgrade should be a no-op
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Snapshot of state at 12.0-1
+ALTER EXTENSION citus UPDATE TO '12.0-1';
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object |                              current_object
+---------------------------------------------------------------------
+                 | function citus_internal_add_tenant_schema(oid,integer) void
+                 | function citus_internal_delete_tenant_schema(oid) void
+                 | function citus_internal_unregister_tenant_schema_globally(oid,text) void
+                 | function citus_schema_distribute(regnamespace) void
+                 | function citus_schema_undistribute(regnamespace) void
+                 | function citus_stat_tenants_local_internal(boolean) SETOF record
+                 | table pg_dist_schema
+                 | view public.citus_schemas
+(8 rows)
+
+-- Test downgrade to 12.0-1 from 12.1-1
+ALTER EXTENSION citus UPDATE TO '12.1-1';
+ALTER EXTENSION citus UPDATE TO '12.0-1';
+-- Should be empty result since upgrade+downgrade should be a no-op
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Snapshot of state at 12.1-1
+ALTER EXTENSION citus UPDATE TO '12.1-1';
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object |                                     current_object
+---------------------------------------------------------------------
+                 | function citus_internal_delete_placement_metadata(bigint) void
+                 | function citus_internal_update_none_dist_table_metadata(oid,"char",bigint,boolean) void
+                 | function citus_pause_node_within_txn(integer,boolean,integer) void
+                 | function citus_schema_move(regnamespace,integer,citus.shard_transfer_mode) void
+                 | function citus_schema_move(regnamespace,text,integer,citus.shard_transfer_mode) void
+(5 rows)
+
+-- Test downgrade to 12.1-1 from 12.2-1
+ALTER EXTENSION citus UPDATE TO '12.2-1';
+ALTER EXTENSION citus UPDATE TO '12.1-1';
+-- Should be empty result since upgrade+downgrade should be a no-op
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Snapshot of state at 12.2-1
+ALTER EXTENSION citus UPDATE TO '12.2-1';
+SELECT * FROM multi_extension.print_extension_changes();
+                        previous_object                         |                                        current_object
+---------------------------------------------------------------------
+ function citus_unmark_object_distributed(oid,oid,integer) void |
+                                                                | function citus_internal.acquire_citus_advisory_object_class_lock(integer,cstring) void
+                                                                | function citus_internal.add_colocation_metadata(integer,integer,integer,regtype,oid) void
+                                                                | function citus_internal.add_object_metadata(text,text[],text[],integer,integer,boolean) void
+                                                                | function citus_internal.add_partition_metadata(regclass,"char",text,integer,"char") void
+                                                                | function citus_internal.add_placement_metadata(bigint,bigint,integer,bigint) void
+                                                                | function citus_internal.add_shard_metadata(regclass,bigint,"char",text,text) void
+                                                                | function citus_internal.add_tenant_schema(oid,integer) void
+                                                                | function citus_internal.adjust_local_clock_to_remote(cluster_clock) void
+                                                                | function citus_internal.commit_management_command_2pc() void
+                                                                | function citus_internal.database_command(text) void
+                                                                | function citus_internal.delete_colocation_metadata(integer) void
+                                                                | function citus_internal.delete_partition_metadata(regclass) void
+                                                                | function citus_internal.delete_placement_metadata(bigint) void
+                                                                | function citus_internal.delete_shard_metadata(bigint) void
+                                                                | function citus_internal.delete_tenant_schema(oid) void
+                                                                | function citus_internal.execute_command_on_remote_nodes_as_user(text,text) void
+                                                                | function citus_internal.global_blocked_processes() SETOF record
+                                                                | function citus_internal.is_replication_origin_tracking_active() boolean
+                                                                | function citus_internal.local_blocked_processes() SETOF record
+                                                                | function citus_internal.mark_node_not_synced(integer,integer) void
+                                                                | function citus_internal.mark_object_distributed(oid,text,oid,text) void
+                                                                | function citus_internal.start_management_transaction(xid8) void
+                                                                | function citus_internal.start_replication_origin_tracking() void
+                                                                | function citus_internal.stop_replication_origin_tracking() void
+                                                                | function citus_internal.unregister_tenant_schema_globally(oid,text) void
+                                                                | function citus_internal.update_none_dist_table_metadata(oid,"char",bigint,boolean) void
+                                                                | function citus_internal.update_placement_metadata(bigint,integer,integer) void
+                                                                | function citus_internal.update_relation_colocation(oid,integer) void
+                                                                | function citus_unmark_object_distributed(oid,oid,integer,boolean) void
+(30 rows)
+
+DROP TABLE multi_extension.prev_objects, multi_extension.extension_diff;
+-- show running version
+SHOW citus.version;
+ citus.version
+---------------------------------------------------------------------
+ 12.2devel
+(1 row)
+
+-- ensure no unexpected objects were created outside pg_catalog
+SELECT pgio.type, pgio.identity
+FROM pg_depend AS pgd,
+	 pg_extension AS pge,
+	 LATERAL pg_identify_object(pgd.classid, pgd.objid, pgd.objsubid) AS pgio
+WHERE pgd.refclassid = 'pg_extension'::regclass AND
+	  pgd.refobjid   = pge.oid AND
+	  pge.extname    = 'citus' AND
+	  pgio.schema    NOT IN ('pg_catalog', 'citus', 'citus_internal', 'test', 'columnar', 'columnar_internal')
+ORDER BY 1, 2;
+ type |       identity
+---------------------------------------------------------------------
+ view | public.citus_schemas
+ view | public.citus_tables
+(2 rows)
+
+-- see incompatible version errors out
+RESET citus.enable_version_checks;
+RESET columnar.enable_version_checks;
+-- reset version check config for other sessions too
+ALTER SYSTEM RESET citus.enable_version_checks;
+SELECT pg_reload_conf();
+ pg_reload_conf
+---------------------------------------------------------------------
+ t
+(1 row)
+
+DROP EXTENSION citus;
+DROP EXTENSION citus_columnar;
+CREATE EXTENSION citus VERSION '8.0-1';
+ERROR:  specified version incompatible with loaded Citus library
+DETAIL:  Loaded library requires 12.2, but 8.0-1 was specified.
+HINT:  If a newer library is present, restart the database and try the command again.
+-- Test non-distributed queries work even in version mismatch
+SET citus.enable_version_checks TO 'false';
+SET columnar.enable_version_checks TO 'false';
+CREATE EXTENSION citus VERSION '8.1-1';
+SET citus.enable_version_checks TO 'true';
+SET columnar.enable_version_checks TO 'true';
+-- Test CREATE TABLE
+CREATE TABLE version_mismatch_table(column1 int);
+-- Test COPY
+\copy version_mismatch_table FROM STDIN;
+-- Test INSERT
+INSERT INTO version_mismatch_table(column1) VALUES(5);
+-- Test SELECT
+SELECT * FROM version_mismatch_table ORDER BY column1;
+ column1
+---------------------------------------------------------------------
+       0
+       1
+       2
+       3
+       4
+       5
+(6 rows)
+
+-- Test SELECT from pg_catalog
+SELECT d.datname as "Name",
+       pg_catalog.pg_get_userbyid(d.datdba) as "Owner",
+       pg_catalog.array_to_string(d.datacl, E'\n') AS "Access privileges"
+FROM pg_catalog.pg_database d
+ORDER BY 1;
+    Name    |  Owner   |   Access privileges
+---------------------------------------------------------------------
+ postgres   | postgres |
+ regression | postgres |
+ template0  | postgres | =c/postgres          +
+            |          | postgres=CTc/postgres
+ template1  | postgres | =c/postgres          +
+            |          | postgres=CTc/postgres
+(4 rows)
+
+-- We should not distribute table in version mistmatch
+SELECT create_distributed_table('version_mismatch_table', 'column1');
+ERROR:  loaded Citus library version differs from installed extension version
+DETAIL:  Loaded library requires 12.2, but the installed extension version is 8.1-1.
+HINT:  Run ALTER EXTENSION citus UPDATE and try again.
+-- This function will cause fail in next ALTER EXTENSION
+CREATE OR REPLACE FUNCTION pg_catalog.relation_is_a_known_shard(regclass)
+RETURNS void LANGUAGE plpgsql
+AS $function$
+BEGIN
+END;
+$function$;
+ERROR:  cannot change return type of existing function
+HINT:  Use DROP FUNCTION relation_is_a_known_shard(regclass) first.
+SET citus.enable_version_checks TO 'false';
+SET columnar.enable_version_checks TO 'false';
+-- This will fail because of previous function declaration
+ALTER EXTENSION citus UPDATE TO '9.1-1';
+-- We can DROP problematic function and continue ALTER EXTENSION even when version checks are on
+SET citus.enable_version_checks TO 'true';
+SET columnar.enable_version_checks TO 'true';
+DROP FUNCTION pg_catalog.relation_is_a_known_shard(regclass);
+ERROR:  cannot drop function relation_is_a_known_shard(regclass) because extension citus requires it
+HINT:  You can drop extension citus instead.
+SET citus.enable_version_checks TO 'false';
+SET columnar.enable_version_checks TO 'false';
+ALTER EXTENSION citus UPDATE TO '9.1-1';
+NOTICE:  version "9.1-1" of extension "citus" is already installed
+-- Test updating to the latest version without specifying the version number
+ALTER EXTENSION citus UPDATE;
+-- re-create in newest version
+DROP EXTENSION citus;
+DROP EXTENSION citus_columnar;
+\c
+CREATE EXTENSION citus;
+-- test cache invalidation in workers
+\c - - - :worker_1_port
+DROP EXTENSION citus;
+DROP EXTENSION citus_columnar;
+SET citus.enable_version_checks TO 'false';
+SET columnar.enable_version_checks TO 'false';
+CREATE EXTENSION citus VERSION '8.0-1';
+SET citus.enable_version_checks TO 'true';
+SET columnar.enable_version_checks TO 'true';
+-- during ALTER EXTENSION, we should invalidate the cache
+ALTER EXTENSION citus UPDATE;
+-- if cache is invalidated succesfull, this \d should work without any problem
+\d
+            List of relations
+ Schema |     Name      | Type |  Owner
+---------------------------------------------------------------------
+ public | citus_schemas | view | postgres
+ public | citus_tables  | view | postgres
+(2 rows)
+
+\c - - - :master_port
+-- test https://github.com/citusdata/citus/issues/3409
+CREATE USER testuser2 SUPERUSER;
+SET ROLE testuser2;
+DROP EXTENSION Citus;
+-- Loop until we see there's no maintenance daemon running
+DO $$begin
+    for i in 0 .. 100 loop
+        if i = 100 then raise 'Waited too long'; end if;
+        PERFORM pg_stat_clear_snapshot();
+        perform * from pg_stat_activity where application_name = 'Citus Maintenance Daemon';
+        if not found then exit; end if;
+        perform pg_sleep(0.1);
+    end loop;
+end$$;
+SELECT datid, datname, usename FROM pg_stat_activity WHERE application_name = 'Citus Maintenance Daemon';
+ datid | datname | usename
+---------------------------------------------------------------------
+(0 rows)
+
+CREATE EXTENSION Citus;
+-- Loop until we there's a maintenance daemon running
+DO $$begin
+    for i in 0 .. 100 loop
+        if i = 100 then raise 'Waited too long'; end if;
+        PERFORM pg_stat_clear_snapshot();
+        perform * from pg_stat_activity where application_name = 'Citus Maintenance Daemon';
+        if found then exit; end if;
+        perform pg_sleep(0.1);
+    end loop;
+end$$;
+SELECT datid, datname, usename FROM pg_stat_activity WHERE application_name = 'Citus Maintenance Daemon';
+ datid |  datname   |  usename
+---------------------------------------------------------------------
+ 16384 | regression | testuser2
+(1 row)
+
+RESET ROLE;
+-- check that maintenance daemon gets (re-)started for the right user
+DROP EXTENSION citus;
+CREATE USER testuser SUPERUSER;
+SET ROLE testuser;
+CREATE EXTENSION citus;
+SELECT datname, current_database(),
+    usename, (SELECT extowner::regrole::text FROM pg_extension WHERE extname = 'citus')
+FROM test.maintenance_worker();
+  datname   | current_database | usename  | extowner
+---------------------------------------------------------------------
+ regression | regression       | postgres | testuser
+(1 row)
+
+-- and recreate as the right owner
+RESET ROLE;
+DROP EXTENSION citus;
+CREATE EXTENSION citus;
+-- Check that maintenance daemon can also be started in another database
+CREATE DATABASE another;
+NOTICE:  Citus partially supports CREATE DATABASE for distributed databases
+DETAIL:  Citus does not propagate CREATE DATABASE command to other nodes
+HINT:  You can manually create a database and its extensions on other nodes.
+\c another
+CREATE EXTENSION citus;
+CREATE SCHEMA test;
+:create_function_test_maintenance_worker
+-- see that the daemon started
+SELECT datname, current_database(),
+    usename, (SELECT extowner::regrole::text FROM pg_extension WHERE extname = 'citus')
+FROM test.maintenance_worker();
+ datname | current_database | usename  | extowner
+---------------------------------------------------------------------
+ another | another          | postgres | postgres
+(1 row)
+
+-- Test that database with active worker can be dropped.
+\c regression
+CREATE SCHEMA test_daemon;
+-- we create a similar function on the regression database
+-- note that this function checks for the existence of the daemon
+-- when not found, returns true else tries for 5 times and
+-- returns false
+CREATE OR REPLACE FUNCTION test_daemon.maintenance_daemon_died(p_dbname text)
+    RETURNS boolean
+    LANGUAGE plpgsql
+AS $$
+DECLARE
+   activity record;
+BEGIN
+    PERFORM pg_stat_clear_snapshot();
+    SELECT * INTO activity FROM pg_stat_activity
+    WHERE application_name = 'Citus Maintenance Daemon' AND datname = p_dbname;
+    IF activity.pid IS NULL THEN
+        RETURN true;
+    ELSE
+        RETURN false;
+    END IF;
+END;
+$$;
+-- drop the database and see that the daemon is dead
+DROP DATABASE another;
+SELECT
+    *
+FROM
+    test_daemon.maintenance_daemon_died('another');
+ maintenance_daemon_died
+---------------------------------------------------------------------
+ t
+(1 row)
+
+-- we don't need the schema and the function anymore
+DROP SCHEMA test_daemon CASCADE;
+NOTICE:  drop cascades to function test_daemon.maintenance_daemon_died(text)
+-- verify citus does not crash while creating a table when run against an older worker
+-- create_distributed_table piggybacks multiple commands into single one, if one worker
+-- did not have the required UDF it should fail instead of crash.
+-- create a test database, configure citus with single node
+CREATE DATABASE another;
+NOTICE:  Citus partially supports CREATE DATABASE for distributed databases
+DETAIL:  Citus does not propagate CREATE DATABASE command to other nodes
+HINT:  You can manually create a database and its extensions on other nodes.
+\c - - - :worker_1_port
+CREATE DATABASE another;
+NOTICE:  Citus partially supports CREATE DATABASE for distributed databases
+DETAIL:  Citus does not propagate CREATE DATABASE command to other nodes
+HINT:  You can manually create a database and its extensions on other nodes.
+\c - - - :master_port
+\c another
+CREATE EXTENSION citus;
+\c - - - :worker_1_port
+CREATE EXTENSION citus;
+\c - - - :master_port
+SELECT FROM master_add_node('localhost', :worker_1_port);
+--
+(1 row)
+
+\c - - - :worker_1_port
+ALTER FUNCTION assign_distributed_transaction_id(initiator_node_identifier integer, transaction_number bigint, transaction_stamp timestamp with time zone)
+RENAME TO dummy_assign_function;
+\c - - - :master_port
+SET citus.shard_replication_factor to 1;
+-- create_distributed_table command should fail
+CREATE TABLE t1(a int, b int);
+SET client_min_messages TO ERROR;
+DO $$
+BEGIN
+        BEGIN
+                SELECT create_distributed_table('t1', 'a');
+        EXCEPTION WHEN OTHERS THEN
+                RAISE 'create distributed table failed';
+        END;
+END;
+$$;
+ERROR:  create distributed table failed
+CONTEXT:  PL/pgSQL function inline_code_block line XX at RAISE
+\c regression
+\c - - - :master_port
+DROP DATABASE another;
+\c - - - :worker_1_port
+DROP DATABASE another;
+\c - - - :master_port
+-- only the regression database should have a maintenance daemon
+SELECT count(*) FROM pg_stat_activity WHERE application_name = 'Citus Maintenance Daemon';
+ count
+---------------------------------------------------------------------
+     1
+(1 row)
+
+-- recreate the extension immediately after the maintenancae daemon errors
+SELECT pg_cancel_backend(pid) FROM pg_stat_activity WHERE application_name = 'Citus Maintenance Daemon';
+ pg_cancel_backend
+---------------------------------------------------------------------
+ t
+(1 row)
+
+DROP EXTENSION citus;
+CREATE EXTENSION citus;
+-- wait for maintenance daemon restart
+SELECT datname, current_database(),
+    usename, (SELECT extowner::regrole::text FROM pg_extension WHERE extname = 'citus')
+FROM test.maintenance_worker();
+  datname   | current_database | usename  | extowner
+---------------------------------------------------------------------
+ regression | regression       | postgres | postgres
+(1 row)
+
+-- confirm that there is only one maintenance daemon
+SELECT count(*) FROM pg_stat_activity WHERE application_name = 'Citus Maintenance Daemon';
+ count
+---------------------------------------------------------------------
+     1
+(1 row)
+
+-- kill the maintenance daemon
+SELECT pg_cancel_backend(pid) FROM pg_stat_activity WHERE application_name = 'Citus Maintenance Daemon';
+ pg_cancel_backend
+---------------------------------------------------------------------
+ t
+(1 row)
+
+-- reconnect
+\c - - - :master_port
+-- run something that goes through planner hook and therefore kicks of maintenance daemon
+SELECT 1;
+ ?column?
+---------------------------------------------------------------------
+        1
+(1 row)
+
+-- wait for maintenance daemon restart
+SELECT datname, current_database(),
+    usename, (SELECT extowner::regrole::text FROM pg_extension WHERE extname = 'citus')
+FROM test.maintenance_worker();
+  datname   | current_database | usename  | extowner
+---------------------------------------------------------------------
+ regression | regression       | postgres | postgres
+(1 row)
+
+-- confirm that there is only one maintenance daemon
+SELECT count(*) FROM pg_stat_activity WHERE application_name = 'Citus Maintenance Daemon';
+ count
+---------------------------------------------------------------------
+     1
+(1 row)
+
+-- confirm that we can create a distributed table concurrently on an empty node
+DROP EXTENSION citus;
+CREATE EXTENSION citus;
+CREATE TABLE test (x int, y int);
+INSERT INTO test VALUES (1,2);
+SET citus.shard_replication_factor TO 1;
+SELECT create_distributed_table_concurrently('test','x');
+NOTICE:  relation test does not have a REPLICA IDENTITY or PRIMARY KEY
+DETAIL:  UPDATE and DELETE commands on the relation will error out during create_distributed_table_concurrently unless there is a REPLICA IDENTITY or PRIMARY KEY. INSERT commands will still work.
+ create_distributed_table_concurrently
+---------------------------------------------------------------------
+
+(1 row)
+
+DROP TABLE test;
+TRUNCATE pg_dist_node;
+-- confirm that we can create a distributed table on an empty node
+CREATE TABLE test (x int, y int);
+INSERT INTO test VALUES (1,2);
+SET citus.shard_replication_factor TO 1;
+SELECT create_distributed_table('test','x');
+NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is no longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.test$$)
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+DROP TABLE test;
+TRUNCATE pg_dist_node;
+-- confirm that we can create a single-shard table on an empty node
+CREATE TABLE test (x int, y int);
+INSERT INTO test VALUES (1,2);
+SET citus.shard_replication_factor TO 1;
+SELECT create_distributed_table('test', null, colocate_with=>'none', distribution_type=>null);
+NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is no longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.test$$)
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+-- and make sure that we can't remove the coordinator due to "test"
+SELECT citus_remove_node('localhost', :master_port);
+ERROR:  cannot remove or disable the node localhost:xxxxx because because it contains the only shard placement for shard xxxxx
+DETAIL:  One of the table(s) that prevents the operation complete successfully is public.test
+HINT:  To proceed, either drop the tables or use undistribute_table() function to convert them to local tables
+DROP TABLE test;
+-- and now we should be able to remove the coordinator
+SELECT citus_remove_node('localhost', :master_port);
+ citus_remove_node
+---------------------------------------------------------------------
+
+(1 row)
+
+-- confirm that we can create a tenant schema / table on an empty node
+SET citus.enable_schema_based_sharding TO ON;
+CREATE SCHEMA tenant_schema;
+CREATE TABLE tenant_schema.test(x int, y int);
+SELECT colocationid = (
+    SELECT colocationid FROM pg_dist_partition WHERE logicalrelid = 'tenant_schema.test'::regclass
+)
+FROM pg_dist_schema
+WHERE schemaid::regnamespace::text = 'tenant_schema';
+ ?column?
+---------------------------------------------------------------------
+ t
+(1 row)
+
+-- and make sure that we can't remove the coordinator due to "test"
+SELECT citus_remove_node('localhost', :master_port);
+ERROR:  cannot remove or disable the node localhost:xxxxx because because it contains the only shard placement for shard xxxxx
+DETAIL:  One of the table(s) that prevents the operation complete successfully is tenant_schema.test
+HINT:  To proceed, either drop the tables or use undistribute_table() function to convert them to local tables
+BEGIN;
+  SET LOCAL client_min_messages TO WARNING;
+  DROP SCHEMA tenant_schema CASCADE;
+COMMIT;
+-- and now we should be able to remove the coordinator
+SELECT citus_remove_node('localhost', :master_port);
+ citus_remove_node
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE SCHEMA tenant_schema;
+-- Make sure that we can sync metadata for empty tenant schemas
+-- when adding the first node to the cluster.
+SELECT 1 FROM citus_add_node('localhost', :worker_1_port);
+ ?column?
+---------------------------------------------------------------------
+        1
+(1 row)
+
+DROP SCHEMA tenant_schema;
+SELECT citus_remove_node('localhost', :worker_1_port);
+ citus_remove_node
+---------------------------------------------------------------------
+
+(1 row)
+
+RESET citus.enable_schema_based_sharding;
+-- confirm that we can create a reference table on an empty node
+CREATE TABLE test (x int, y int);
+INSERT INTO test VALUES (1,2);
+SELECT create_reference_table('test');
+NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is no longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.test$$)
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+DROP TABLE test;
+TRUNCATE pg_dist_node;
+-- confirm that we can create a local table on an empty node
+CREATE TABLE test (x int, y int);
+INSERT INTO test VALUES (1,2);
+SELECT citus_add_local_table_to_metadata('test');
+ citus_add_local_table_to_metadata
+---------------------------------------------------------------------
+
+(1 row)
+
+DROP TABLE test;
+-- Verify that we don't consider the schemas created by extensions as tenant schemas.
+-- Easiest way of verifying this is to drop and re-create columnar extension.
+DROP EXTENSION citus_columnar;
+SET citus.enable_schema_based_sharding TO ON;
+CREATE EXTENSION citus_columnar;
+SELECT COUNT(*)=0 FROM pg_dist_schema
+WHERE schemaid IN ('columnar'::regnamespace, 'columnar_internal'::regnamespace);
+ ?column?
+---------------------------------------------------------------------
+ t
+(1 row)
+
+RESET citus.enable_schema_based_sharding;
+DROP EXTENSION citus;
+CREATE EXTENSION citus;
+DROP TABLE version_mismatch_table;
+DROP SCHEMA multi_extension;
+ERROR:  cannot drop schema multi_extension because other objects depend on it
+DETAIL:  function multi_extension.print_extension_changes() depends on schema multi_extension
+HINT:  Use DROP ... CASCADE to drop the dependent objects too.

From c558caf4bb16f20139321daa12fab1811c02b746 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Fri, 22 Mar 2024 14:48:06 +0300
Subject: [PATCH 19/78] Fixes some test errors

---
 src/test/regress/expected/failure_setup.out   | 24 +++++++++++++++++++
 .../expected/single_node_enterprise.out       |  8 +++----
 2 files changed, 28 insertions(+), 4 deletions(-)

diff --git a/src/test/regress/expected/failure_setup.out b/src/test/regress/expected/failure_setup.out
index 4cbb4b0a4..7a26f8097 100644
--- a/src/test/regress/expected/failure_setup.out
+++ b/src/test/regress/expected/failure_setup.out
@@ -1,3 +1,27 @@
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+ pg_reload_conf
+---------------------------------------------------------------------
+ t
+(1 row)
+
+\c - - - :worker_1_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+ pg_reload_conf
+---------------------------------------------------------------------
+ t
+(1 row)
+
+\c - - - :worker_2_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+ pg_reload_conf
+---------------------------------------------------------------------
+ t
+(1 row)
+
+\c - - - :master_port
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy
 ---------------------------------------------------------------------
diff --git a/src/test/regress/expected/single_node_enterprise.out b/src/test/regress/expected/single_node_enterprise.out
index 9e3ffad76..b93bf752f 100644
--- a/src/test/regress/expected/single_node_enterprise.out
+++ b/src/test/regress/expected/single_node_enterprise.out
@@ -1,4 +1,6 @@
-
+-- we already have lots of tests targeting
+-- single node citus clusters in sql/single_node.sql
+-- in this file, we are testing enterprise features
 ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
 SELECT pg_reload_conf();
  pg_reload_conf
@@ -23,9 +25,7 @@ SELECT pg_reload_conf();
 (1 row)
 
 \c - - - :master_port
--- we already have lots of tests targeting
--- single node citus clusters in sql/single_node.sql
--- in this file, we are testing enterprise features
+
 CREATE SCHEMA single_node_ent;
 SET search_path TO single_node_ent;
 SET citus.shard_count TO 4;

From 48e959522bb705fc84b2e1d7d5d27e382560c9fb Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Fri, 22 Mar 2024 14:51:05 +0300
Subject: [PATCH 20/78] Fixes indentation errors

---
 .../commands/non_main_db_distribute_object_ops.c         | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/src/backend/distributed/commands/non_main_db_distribute_object_ops.c b/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
index 52d025f70..ce09a6fc8 100644
--- a/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
+++ b/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
@@ -160,15 +160,6 @@ static void UnmarkObjectDistributedOnLocalMainDb(uint16 catalogRelId, Oid object
 bool
 RunPreprocessNonMainDBCommand(Node *parsetree)
 {
-	/* NodeTag tag = nodeTag(parsetree); */
-	/* if ( tag == T_AlterDatabaseSetStmt){ */
-	/* 	AlterDatabaseSetStmt *stmt = castNode(AlterDatabaseSetStmt, parsetree); */
-	/* 	if (strcmp(stmt->dbname, MainDb) == 0){ */
-	/* 		return false; */
-	/* 	} */
-	/* } */
-
-
 	if (IsMainDB)
 	{
 		return false;

From 1b24b3c5ff38c1cac9f7b9014e6074d810e77744 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Fri, 22 Mar 2024 15:04:17 +0300
Subject: [PATCH 21/78] Fixes test error

---
 src/test/regress/expected/single_node_enterprise.out | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/test/regress/expected/single_node_enterprise.out b/src/test/regress/expected/single_node_enterprise.out
index b93bf752f..94741a755 100644
--- a/src/test/regress/expected/single_node_enterprise.out
+++ b/src/test/regress/expected/single_node_enterprise.out
@@ -25,7 +25,6 @@ SELECT pg_reload_conf();
 (1 row)
 
 \c - - - :master_port
-
 CREATE SCHEMA single_node_ent;
 SET search_path TO single_node_ent;
 SET citus.shard_count TO 4;

From 2bb01f9e76957b68b399c038aeee94ac2ee772cb Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Fri, 22 Mar 2024 16:22:25 +0300
Subject: [PATCH 22/78] Adds config as seperate file

---
 .../regress/enterprise_isolation_schedule     |  1 +
 .../expected/enable_ddl_propagation.out       | 19 +++++++++++++++
 .../regress/expected/multi_test_helpers.out   | 24 -------------------
 src/test/regress/failure_schedule             |  1 +
 src/test/regress/multi_1_schedule             |  1 +
 .../regress/sql/enable_ddl_propagation.sql    | 11 +++++++++
 src/test/regress/sql/multi_test_helpers.sql   | 10 --------
 7 files changed, 33 insertions(+), 34 deletions(-)
 create mode 100644 src/test/regress/expected/enable_ddl_propagation.out
 create mode 100644 src/test/regress/sql/enable_ddl_propagation.sql

diff --git a/src/test/regress/enterprise_isolation_schedule b/src/test/regress/enterprise_isolation_schedule
index 689a7db75..ae22dac84 100644
--- a/src/test/regress/enterprise_isolation_schedule
+++ b/src/test/regress/enterprise_isolation_schedule
@@ -3,6 +3,7 @@ test: isolation_setup
 # tests that change node metadata should precede
 # isolation_cluster_management such that tests
 # that come later can be parallelized
+test: enable_ddl_propagation
 test: isolation_cluster_management
 
 test: isolation_move_placement_vs_move_placement
diff --git a/src/test/regress/expected/enable_ddl_propagation.out b/src/test/regress/expected/enable_ddl_propagation.out
new file mode 100644
index 000000000..eb990e11d
--- /dev/null
+++ b/src/test/regress/expected/enable_ddl_propagation.out
@@ -0,0 +1,19 @@
+---------------------------------------------------------------------
+ t
+(1 row)
+
+\c - - - :worker_1_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+ pg_reload_conf
+---------------------------------------------------------------------
+ t
+(1 row)
+
+\c - - - :worker_2_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+ pg_reload_conf
+---------------------------------------------------------------------
+ t
+(1 row)
diff --git a/src/test/regress/expected/multi_test_helpers.out b/src/test/regress/expected/multi_test_helpers.out
index 36b1be250..0f31f2354 100644
--- a/src/test/regress/expected/multi_test_helpers.out
+++ b/src/test/regress/expected/multi_test_helpers.out
@@ -1,28 +1,4 @@
 -- File to create functions and helpers needed for subsequent tests
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
- pg_reload_conf
----------------------------------------------------------------------
- t
-(1 row)
-
-\c - - - :worker_1_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
- pg_reload_conf
----------------------------------------------------------------------
- t
-(1 row)
-
-\c - - - :worker_2_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
- pg_reload_conf
----------------------------------------------------------------------
- t
-(1 row)
-
-\c - - - :master_port
 -- create a helper function to create objects on each node
 CREATE OR REPLACE FUNCTION run_command_on_master_and_workers(p_sql text)
 RETURNS void LANGUAGE plpgsql AS $$
diff --git a/src/test/regress/failure_schedule b/src/test/regress/failure_schedule
index 8b992422e..f308fd7d2 100644
--- a/src/test/regress/failure_schedule
+++ b/src/test/regress/failure_schedule
@@ -1,4 +1,5 @@
 # import this file (from psql you can use \i) to use mitmproxy manually
+test: enable_ddl_propagation
 test: failure_test_helpers
 
 # this should only be run by pg_regress_multi, you don't need it
diff --git a/src/test/regress/multi_1_schedule b/src/test/regress/multi_1_schedule
index 868d78cdf..aa742096d 100644
--- a/src/test/regress/multi_1_schedule
+++ b/src/test/regress/multi_1_schedule
@@ -64,6 +64,7 @@ test: alter_database_propagation
 test: citus_shards
 test: reassign_owned
 test: alter_database_from_nonmain_db
+test: enable_ddl_propagation
 
 # ----------
 # multi_citus_tools tests utility functions written for citus tools
diff --git a/src/test/regress/sql/enable_ddl_propagation.sql b/src/test/regress/sql/enable_ddl_propagation.sql
new file mode 100644
index 000000000..b859d8a28
--- /dev/null
+++ b/src/test/regress/sql/enable_ddl_propagation.sql
@@ -0,0 +1,11 @@
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+
+\c - - - :worker_1_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+
+\c - - - :worker_2_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+
diff --git a/src/test/regress/sql/multi_test_helpers.sql b/src/test/regress/sql/multi_test_helpers.sql
index 350240494..7d218361c 100644
--- a/src/test/regress/sql/multi_test_helpers.sql
+++ b/src/test/regress/sql/multi_test_helpers.sql
@@ -1,15 +1,5 @@
 -- File to create functions and helpers needed for subsequent tests
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
 
-\c - - - :worker_1_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
-
-\c - - - :worker_2_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
-\c - - - :master_port
 -- create a helper function to create objects on each node
 CREATE OR REPLACE FUNCTION run_command_on_master_and_workers(p_sql text)
 RETURNS void LANGUAGE plpgsql AS $$

From 816540860b3b65861a2d992356dee8c2481c3d2f Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Mon, 25 Mar 2024 11:43:23 +0300
Subject: [PATCH 23/78] Adds ddl_propagation to multi_test_helpers

---
 .../regress/expected/multi_test_helpers.out   | 24 +++++++++++++++++++
 src/test/regress/sql/multi_test_helpers.sql   | 12 ++++++++++
 2 files changed, 36 insertions(+)

diff --git a/src/test/regress/expected/multi_test_helpers.out b/src/test/regress/expected/multi_test_helpers.out
index 0f31f2354..36b1be250 100644
--- a/src/test/regress/expected/multi_test_helpers.out
+++ b/src/test/regress/expected/multi_test_helpers.out
@@ -1,4 +1,28 @@
 -- File to create functions and helpers needed for subsequent tests
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+ pg_reload_conf
+---------------------------------------------------------------------
+ t
+(1 row)
+
+\c - - - :worker_1_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+ pg_reload_conf
+---------------------------------------------------------------------
+ t
+(1 row)
+
+\c - - - :worker_2_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+ pg_reload_conf
+---------------------------------------------------------------------
+ t
+(1 row)
+
+\c - - - :master_port
 -- create a helper function to create objects on each node
 CREATE OR REPLACE FUNCTION run_command_on_master_and_workers(p_sql text)
 RETURNS void LANGUAGE plpgsql AS $$
diff --git a/src/test/regress/sql/multi_test_helpers.sql b/src/test/regress/sql/multi_test_helpers.sql
index 7d218361c..5fec8f8ae 100644
--- a/src/test/regress/sql/multi_test_helpers.sql
+++ b/src/test/regress/sql/multi_test_helpers.sql
@@ -1,5 +1,17 @@
 -- File to create functions and helpers needed for subsequent tests
 
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+
+\c - - - :worker_1_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+
+\c - - - :worker_2_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+
+\c - - - :master_port
 -- create a helper function to create objects on each node
 CREATE OR REPLACE FUNCTION run_command_on_master_and_workers(p_sql text)
 RETURNS void LANGUAGE plpgsql AS $$

From 7ab7ffe6b9f7331a89d57ba356ceaffb08fb465e Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Mon, 25 Mar 2024 11:56:19 +0300
Subject: [PATCH 24/78] Fixes enable_ddl propagation

---
 src/test/regress/expected/enable_ddl_propagation.out | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/test/regress/expected/enable_ddl_propagation.out b/src/test/regress/expected/enable_ddl_propagation.out
index eb990e11d..d8322ba8c 100644
--- a/src/test/regress/expected/enable_ddl_propagation.out
+++ b/src/test/regress/expected/enable_ddl_propagation.out
@@ -1,3 +1,6 @@
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+ pg_reload_conf
 ---------------------------------------------------------------------
  t
 (1 row)

From 18a8108df20b4fabb2a8784cc3e93716fe0e0a57 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Mon, 25 Mar 2024 12:10:17 +0300
Subject: [PATCH 25/78] Fixes test errors

---
 src/test/regress/enterprise_isolation_schedule       | 1 -
 src/test/regress/expected/enable_ddl_propagation.out | 1 +
 src/test/regress/expected/failure_test_helpers.out   | 1 +
 src/test/regress/failure_schedule                    | 1 -
 src/test/regress/multi_1_schedule                    | 2 +-
 src/test/regress/sql/failure_test_helpers.sql        | 1 +
 6 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/test/regress/enterprise_isolation_schedule b/src/test/regress/enterprise_isolation_schedule
index ae22dac84..689a7db75 100644
--- a/src/test/regress/enterprise_isolation_schedule
+++ b/src/test/regress/enterprise_isolation_schedule
@@ -3,7 +3,6 @@ test: isolation_setup
 # tests that change node metadata should precede
 # isolation_cluster_management such that tests
 # that come later can be parallelized
-test: enable_ddl_propagation
 test: isolation_cluster_management
 
 test: isolation_move_placement_vs_move_placement
diff --git a/src/test/regress/expected/enable_ddl_propagation.out b/src/test/regress/expected/enable_ddl_propagation.out
index d8322ba8c..dc5cc36b2 100644
--- a/src/test/regress/expected/enable_ddl_propagation.out
+++ b/src/test/regress/expected/enable_ddl_propagation.out
@@ -20,3 +20,4 @@ SELECT pg_reload_conf();
 ---------------------------------------------------------------------
  t
 (1 row)
+
diff --git a/src/test/regress/expected/failure_test_helpers.out b/src/test/regress/expected/failure_test_helpers.out
index da63a985f..7689e986c 100644
--- a/src/test/regress/expected/failure_test_helpers.out
+++ b/src/test/regress/expected/failure_test_helpers.out
@@ -4,6 +4,7 @@
 ALTER SYSTEM SET citus.distributed_deadlock_detection_factor TO -1;
 ALTER SYSTEM SET citus.recover_2pc_interval TO -1;
 ALTER SYSTEM set citus.enable_statistics_collection TO false;
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
 SELECT pg_reload_conf();
  pg_reload_conf
 ---------------------------------------------------------------------
diff --git a/src/test/regress/failure_schedule b/src/test/regress/failure_schedule
index f308fd7d2..8b992422e 100644
--- a/src/test/regress/failure_schedule
+++ b/src/test/regress/failure_schedule
@@ -1,5 +1,4 @@
 # import this file (from psql you can use \i) to use mitmproxy manually
-test: enable_ddl_propagation
 test: failure_test_helpers
 
 # this should only be run by pg_regress_multi, you don't need it
diff --git a/src/test/regress/multi_1_schedule b/src/test/regress/multi_1_schedule
index aa742096d..6fc2ca36c 100644
--- a/src/test/regress/multi_1_schedule
+++ b/src/test/regress/multi_1_schedule
@@ -15,6 +15,7 @@
 # ---
 # Tests around schema changes, these are run first, so there's no preexisting objects.
 # ---
+test: enable_ddl_propagation
 test: multi_extension
 test: multi_test_helpers multi_test_helpers_superuser multi_create_fdw
 test: single_node
@@ -64,7 +65,6 @@ test: alter_database_propagation
 test: citus_shards
 test: reassign_owned
 test: alter_database_from_nonmain_db
-test: enable_ddl_propagation
 
 # ----------
 # multi_citus_tools tests utility functions written for citus tools
diff --git a/src/test/regress/sql/failure_test_helpers.sql b/src/test/regress/sql/failure_test_helpers.sql
index b7f9eae3a..56938783d 100644
--- a/src/test/regress/sql/failure_test_helpers.sql
+++ b/src/test/regress/sql/failure_test_helpers.sql
@@ -4,6 +4,7 @@
 ALTER SYSTEM SET citus.distributed_deadlock_detection_factor TO -1;
 ALTER SYSTEM SET citus.recover_2pc_interval TO -1;
 ALTER SYSTEM set citus.enable_statistics_collection TO false;
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
 SELECT pg_reload_conf();
 
 -- Add some helper functions for sending commands to mitmproxy

From f88d66cfa89967149ccf66651e28dc8f269b1717 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Mon, 25 Mar 2024 12:47:14 +0300
Subject: [PATCH 26/78] Fixes enterprise isolation

---
 .../isolation_enable_ddl_propagation.out       | 15 +++++++++++++++
 src/test/regress/isolation_schedule            |  4 +++-
 .../spec/isolation_enable_ddl_propagation.spec | 18 ++++++++++++++++++
 3 files changed, 36 insertions(+), 1 deletion(-)
 create mode 100644 src/test/regress/expected/isolation_enable_ddl_propagation.out
 create mode 100644 src/test/regress/spec/isolation_enable_ddl_propagation.spec

diff --git a/src/test/regress/expected/isolation_enable_ddl_propagation.out b/src/test/regress/expected/isolation_enable_ddl_propagation.out
new file mode 100644
index 000000000..88f38b5bc
--- /dev/null
+++ b/src/test/regress/expected/isolation_enable_ddl_propagation.out
@@ -0,0 +1,15 @@
+Parsed test spec with 1 sessions
+
+starting permutation: s1-begin s1-commit
+run_command_on_workers
+---------------------------------------------------------------------
+(localhost,57637,t,t)
+(localhost,57638,t,t)
+(2 rows)
+
+step s1-begin:
+ BEGIN;
+
+step s1-commit:
+ COMMIT;
+
diff --git a/src/test/regress/isolation_schedule b/src/test/regress/isolation_schedule
index 1b0f1427a..398fffa9b 100644
--- a/src/test/regress/isolation_schedule
+++ b/src/test/regress/isolation_schedule
@@ -1,4 +1,5 @@
-test: isolation_setup
+test: isolation_enable_ddl_propagation
+
 test: isolation_add_remove_node
 test: isolation_update_node
 test: isolation_update_node_lock_writes
@@ -7,6 +8,7 @@ test: isolation_add_node_vs_reference_table_operations
 test: isolation_create_table_vs_add_remove_node
 test: isolation_master_update_node
 test: isolation_shouldhaveshards
+test: isolation_setup
 
 test: isolation_extension_commands
 
diff --git a/src/test/regress/spec/isolation_enable_ddl_propagation.spec b/src/test/regress/spec/isolation_enable_ddl_propagation.spec
new file mode 100644
index 000000000..7240653d7
--- /dev/null
+++ b/src/test/regress/spec/isolation_enable_ddl_propagation.spec
@@ -0,0 +1,18 @@
+setup
+{
+	SELECT run_command_on_workers('ALTER SYSTEM SET citus.enable_ddl_propagation TO ON');
+	SELECT run_command_on_workers('SELECT pg_reload_conf()');
+}
+
+session "s1"
+step "s1-begin"
+{
+	BEGIN;
+}
+
+step "s1-commit"
+{
+	COMMIT;
+}
+
+permutation "s1-begin" "s1-commit"

From ac2ca9376849767a33a254ed4ad8d8a1995c7f42 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Mon, 25 Mar 2024 13:01:17 +0300
Subject: [PATCH 27/78] Adds logs to show flag value

---
 .../regress/spec/isolation_move_placement_vs_move_placement.spec | 1 +
 src/test/regress/sql/multi_follower_dml.sql                      | 1 +
 2 files changed, 2 insertions(+)

diff --git a/src/test/regress/spec/isolation_move_placement_vs_move_placement.spec b/src/test/regress/spec/isolation_move_placement_vs_move_placement.spec
index b55ab6e54..7c2930b90 100644
--- a/src/test/regress/spec/isolation_move_placement_vs_move_placement.spec
+++ b/src/test/regress/spec/isolation_move_placement_vs_move_placement.spec
@@ -2,6 +2,7 @@
 // so setting the corresponding shard here is useful
 setup
 {
+	show citus.enable_ddl_propagation;
 	SET citus.shard_count TO 2;
 	SET citus.shard_replication_factor TO 1;
 	ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 102011;
diff --git a/src/test/regress/sql/multi_follower_dml.sql b/src/test/regress/sql/multi_follower_dml.sql
index a3d548b12..805097319 100644
--- a/src/test/regress/sql/multi_follower_dml.sql
+++ b/src/test/regress/sql/multi_follower_dml.sql
@@ -1,5 +1,6 @@
 \c - - - :master_port
 
+show citus.enable_ddl_propagation;
 SET citus.shard_replication_factor TO 2;
 CREATE TABLE the_replicated_table (a int, b int, z bigserial);
 SELECT create_distributed_table('the_replicated_table', 'a');

From c049b5159f5fa9926ba4be444c484f714598a19d Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Mon, 25 Mar 2024 13:03:54 +0300
Subject: [PATCH 28/78] Changes the order for isolation setup

---
 src/test/regress/isolation_schedule | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/test/regress/isolation_schedule b/src/test/regress/isolation_schedule
index 398fffa9b..5c5f9ccbb 100644
--- a/src/test/regress/isolation_schedule
+++ b/src/test/regress/isolation_schedule
@@ -1,5 +1,5 @@
 test: isolation_enable_ddl_propagation
-
+test: isolation_setup
 test: isolation_add_remove_node
 test: isolation_update_node
 test: isolation_update_node_lock_writes
@@ -8,7 +8,6 @@ test: isolation_add_node_vs_reference_table_operations
 test: isolation_create_table_vs_add_remove_node
 test: isolation_master_update_node
 test: isolation_shouldhaveshards
-test: isolation_setup
 
 test: isolation_extension_commands
 

From 35c808e1200ad7a8e6c5b67fd1f1f6a59c7f03fb Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Mon, 25 Mar 2024 13:16:41 +0300
Subject: [PATCH 29/78] Adds follower ddl propagation

---
 .../expected/multi_follower_sanity_check.out  | 24 +++++++++++++++++++
 .../sql/multi_follower_sanity_check.sql       | 13 ++++++++++
 2 files changed, 37 insertions(+)

diff --git a/src/test/regress/expected/multi_follower_sanity_check.out b/src/test/regress/expected/multi_follower_sanity_check.out
index b24ae4e30..4fee237fc 100644
--- a/src/test/regress/expected/multi_follower_sanity_check.out
+++ b/src/test/regress/expected/multi_follower_sanity_check.out
@@ -22,6 +22,30 @@ SELECT pg_reload_conf();
  t
 (1 row)
 
+\c - - - :follower_master_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+ pg_reload_conf
+---------------------------------------------------------------------
+ t
+(1 row)
+
+\c - - - :follower_worker_1_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+ pg_reload_conf
+---------------------------------------------------------------------
+ t
+(1 row)
+
+\c - - - :follower_worker_2_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+ pg_reload_conf
+---------------------------------------------------------------------
+ t
+(1 row)
+
 \c - - - :follower_master_port
 CREATE TABLE tab (a int);
 ERROR:  cannot execute CREATE TABLE in a read-only transaction
diff --git a/src/test/regress/sql/multi_follower_sanity_check.sql b/src/test/regress/sql/multi_follower_sanity_check.sql
index 47e1d724e..eb8c2ee33 100644
--- a/src/test/regress/sql/multi_follower_sanity_check.sql
+++ b/src/test/regress/sql/multi_follower_sanity_check.sql
@@ -10,6 +10,19 @@ SELECT pg_reload_conf();
 ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
 SELECT pg_reload_conf();
 
+\c - - - :follower_master_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+
+\c - - - :follower_worker_1_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+
+\c - - - :follower_worker_2_port
+ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
+SELECT pg_reload_conf();
+
+
 \c - - - :follower_master_port
 CREATE TABLE tab (a int);
 \c - - - :follower_worker_1_port

From 1f0ec0fc90ca8dd1f32f7179f520728c86dd8dd9 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Mon, 25 Mar 2024 15:00:33 +0300
Subject: [PATCH 30/78] Uses enable_ddl_propagation file

---
 src/test/regress/base_schedule                |  1 +
 src/test/regress/columnar_schedule            |  1 +
 ...enterprise_isolation_logicalrep_1_schedule |  1 +
 ...enterprise_isolation_logicalrep_2_schedule |  1 +
 ...enterprise_isolation_logicalrep_3_schedule |  1 +
 .../regress/enterprise_isolation_schedule     |  1 +
 src/test/regress/enterprise_schedule          |  1 +
 .../expected/enable_ddl_propagation.out       | 29 +++++++------------
 .../regress/expected/multi_test_helpers.out   | 25 ----------------
 src/test/regress/failure_base_schedule        |  1 +
 src/test/regress/isolation_schedule           |  2 +-
 src/test/regress/minimal_schedule             |  1 +
 src/test/regress/multi_schedule               |  1 +
 .../regress/sql/enable_ddl_propagation.sql    | 13 ++-------
 .../sql/minimal_cluster_management.sql        | 13 ---------
 src/test/regress/sql/multi_test_helpers.sql   | 12 --------
 .../regress/sql/single_node_enterprise.sql    | 13 ---------
 17 files changed, 23 insertions(+), 94 deletions(-)

diff --git a/src/test/regress/base_schedule b/src/test/regress/base_schedule
index 65f439acc..f56adb5a3 100644
--- a/src/test/regress/base_schedule
+++ b/src/test/regress/base_schedule
@@ -1,6 +1,7 @@
 # ----------
 # Only run few basic tests to set up a testing environment
 # ----------
+test: enable_ddl_propagation
 test: multi_test_helpers multi_test_helpers_superuser multi_create_fdw columnar_test_helpers failure_test_helpers
 test: multi_cluster_management
 test: multi_test_catalog_views
diff --git a/src/test/regress/columnar_schedule b/src/test/regress/columnar_schedule
index 602af0fc7..ef8b2b288 100644
--- a/src/test/regress/columnar_schedule
+++ b/src/test/regress/columnar_schedule
@@ -1,3 +1,4 @@
+test: enable_ddl_propagation
 test: multi_test_helpers multi_test_helpers_superuser columnar_test_helpers
 test: multi_cluster_management
 test: multi_test_catalog_views
diff --git a/src/test/regress/enterprise_isolation_logicalrep_1_schedule b/src/test/regress/enterprise_isolation_logicalrep_1_schedule
index 2656e96d5..509548db1 100644
--- a/src/test/regress/enterprise_isolation_logicalrep_1_schedule
+++ b/src/test/regress/enterprise_isolation_logicalrep_1_schedule
@@ -4,6 +4,7 @@ test: isolation_setup
 # isolation_cluster_management such that tests
 # that come later can be parallelized
 test: isolation_cluster_management
+test: isolation_enable_ddl_propagation
 
 test: isolation_logical_replication_single_shard_commands
 test: isolation_logical_replication_nonsu_nonbypassrls
diff --git a/src/test/regress/enterprise_isolation_logicalrep_2_schedule b/src/test/regress/enterprise_isolation_logicalrep_2_schedule
index e8915cb27..8aae14af6 100644
--- a/src/test/regress/enterprise_isolation_logicalrep_2_schedule
+++ b/src/test/regress/enterprise_isolation_logicalrep_2_schedule
@@ -4,6 +4,7 @@ test: isolation_setup
 # isolation_cluster_management such that tests
 # that come later can be parallelized
 test: isolation_cluster_management
+test: isolation_enable_ddl_propagation
 
 test: isolation_logical_replication_single_shard_commands_on_mx
 test: isolation_logical_replication_multi_shard_commands_on_mx
diff --git a/src/test/regress/enterprise_isolation_logicalrep_3_schedule b/src/test/regress/enterprise_isolation_logicalrep_3_schedule
index 105dcc049..06b3aacdd 100644
--- a/src/test/regress/enterprise_isolation_logicalrep_3_schedule
+++ b/src/test/regress/enterprise_isolation_logicalrep_3_schedule
@@ -4,6 +4,7 @@ test: isolation_setup
 # isolation_cluster_management such that tests
 # that come later can be parallelized
 test: isolation_cluster_management
+test: isolation_enable_ddl_propagation
 
 test: isolation_logical_replication_with_partitioning
 test: isolation_logical_replication_binaryless
diff --git a/src/test/regress/enterprise_isolation_schedule b/src/test/regress/enterprise_isolation_schedule
index 689a7db75..bde8f225d 100644
--- a/src/test/regress/enterprise_isolation_schedule
+++ b/src/test/regress/enterprise_isolation_schedule
@@ -4,6 +4,7 @@ test: isolation_setup
 # isolation_cluster_management such that tests
 # that come later can be parallelized
 test: isolation_cluster_management
+test: isolation_enable_ddl_propagation
 
 test: isolation_move_placement_vs_move_placement
 test: isolation_move_placement_vs_modification
diff --git a/src/test/regress/enterprise_schedule b/src/test/regress/enterprise_schedule
index 9a832c4d6..59156def7 100644
--- a/src/test/regress/enterprise_schedule
+++ b/src/test/regress/enterprise_schedule
@@ -1,5 +1,6 @@
 # should come before multi_cluster_management
 # as it touches node metadata
+test: enable_ddl_propagation
 test: single_node_enterprise
 
 test: multi_test_helpers multi_test_helpers_superuser
diff --git a/src/test/regress/expected/enable_ddl_propagation.out b/src/test/regress/expected/enable_ddl_propagation.out
index dc5cc36b2..6ad3f9bcd 100644
--- a/src/test/regress/expected/enable_ddl_propagation.out
+++ b/src/test/regress/expected/enable_ddl_propagation.out
@@ -1,23 +1,14 @@
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
- pg_reload_conf
+SELECT run_command_on_workers('ALTER SYSTEM SET citus.enable_ddl_propagation TO ON');
+       run_command_on_workers
 ---------------------------------------------------------------------
- t
-(1 row)
+ (localhost,57637,t,"ALTER SYSTEM")
+ (localhost,57638,t,"ALTER SYSTEM")
+(2 rows)
 
-\c - - - :worker_1_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
- pg_reload_conf
+SELECT run_command_on_workers('SELECT pg_reload_conf()');
+ run_command_on_workers
 ---------------------------------------------------------------------
- t
-(1 row)
-
-\c - - - :worker_2_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
- pg_reload_conf
----------------------------------------------------------------------
- t
-(1 row)
+ (localhost,57637,t,t)
+ (localhost,57638,t,t)
+(2 rows)
 
diff --git a/src/test/regress/expected/multi_test_helpers.out b/src/test/regress/expected/multi_test_helpers.out
index 36b1be250..cfe6ff204 100644
--- a/src/test/regress/expected/multi_test_helpers.out
+++ b/src/test/regress/expected/multi_test_helpers.out
@@ -1,28 +1,3 @@
--- File to create functions and helpers needed for subsequent tests
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
- pg_reload_conf
----------------------------------------------------------------------
- t
-(1 row)
-
-\c - - - :worker_1_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
- pg_reload_conf
----------------------------------------------------------------------
- t
-(1 row)
-
-\c - - - :worker_2_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
- pg_reload_conf
----------------------------------------------------------------------
- t
-(1 row)
-
-\c - - - :master_port
 -- create a helper function to create objects on each node
 CREATE OR REPLACE FUNCTION run_command_on_master_and_workers(p_sql text)
 RETURNS void LANGUAGE plpgsql AS $$
diff --git a/src/test/regress/failure_base_schedule b/src/test/regress/failure_base_schedule
index 03ee96ff7..491b3b94a 100644
--- a/src/test/regress/failure_base_schedule
+++ b/src/test/regress/failure_base_schedule
@@ -1,4 +1,5 @@
 # import this file (from psql you can use \i) to use mitmproxy manually
+test: enable_ddl_propagation
 test: failure_test_helpers
 
 # this should only be run by pg_regress_multi, you don't need it
diff --git a/src/test/regress/isolation_schedule b/src/test/regress/isolation_schedule
index 5c5f9ccbb..52aa49f6c 100644
--- a/src/test/regress/isolation_schedule
+++ b/src/test/regress/isolation_schedule
@@ -1,4 +1,3 @@
-test: isolation_enable_ddl_propagation
 test: isolation_setup
 test: isolation_add_remove_node
 test: isolation_update_node
@@ -15,6 +14,7 @@ test: isolation_extension_commands
 # isolation_cluster_management such that tests
 # that come later can be parallelized
 test: isolation_cluster_management
+test: isolation_enable_ddl_propagation
 
 # the following tests depend on the distributed
 # transactionId, so should not be parallelized
diff --git a/src/test/regress/minimal_schedule b/src/test/regress/minimal_schedule
index 8b0cfff70..072039a25 100644
--- a/src/test/regress/minimal_schedule
+++ b/src/test/regress/minimal_schedule
@@ -1,2 +1,3 @@
 test: minimal_cluster_management
+test: enable_ddl_propagation
 test: multi_test_helpers multi_test_helpers_superuser multi_create_fdw columnar_test_helpers multi_test_catalog_views tablespace
diff --git a/src/test/regress/multi_schedule b/src/test/regress/multi_schedule
index af5921e60..f24ebdeae 100644
--- a/src/test/regress/multi_schedule
+++ b/src/test/regress/multi_schedule
@@ -1,3 +1,4 @@
+test: enable_ddl_propagation
 test: multi_test_helpers multi_test_helpers_superuser
 test: multi_cluster_management
 test: create_role_propagation
diff --git a/src/test/regress/sql/enable_ddl_propagation.sql b/src/test/regress/sql/enable_ddl_propagation.sql
index b859d8a28..ff7143359 100644
--- a/src/test/regress/sql/enable_ddl_propagation.sql
+++ b/src/test/regress/sql/enable_ddl_propagation.sql
@@ -1,11 +1,2 @@
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
-
-\c - - - :worker_1_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
-
-\c - - - :worker_2_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
-
+SELECT run_command_on_workers('ALTER SYSTEM SET citus.enable_ddl_propagation TO ON');
+SELECT run_command_on_workers('SELECT pg_reload_conf()');
diff --git a/src/test/regress/sql/minimal_cluster_management.sql b/src/test/regress/sql/minimal_cluster_management.sql
index fffcb18fa..30f69d43d 100644
--- a/src/test/regress/sql/minimal_cluster_management.sql
+++ b/src/test/regress/sql/minimal_cluster_management.sql
@@ -1,16 +1,3 @@
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
-
-\c - - - :worker_1_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
-
-\c - - - :worker_2_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
-
-
-\c - - - :master_port
 SET citus.next_shard_id TO 1220000;
 ALTER SEQUENCE pg_catalog.pg_dist_colocationid_seq RESTART 1390000;
 ALTER SEQUENCE pg_catalog.pg_dist_groupid_seq RESTART 1;
diff --git a/src/test/regress/sql/multi_test_helpers.sql b/src/test/regress/sql/multi_test_helpers.sql
index 5fec8f8ae..7d218361c 100644
--- a/src/test/regress/sql/multi_test_helpers.sql
+++ b/src/test/regress/sql/multi_test_helpers.sql
@@ -1,17 +1,5 @@
 -- File to create functions and helpers needed for subsequent tests
 
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
-
-\c - - - :worker_1_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
-
-\c - - - :worker_2_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
-
-\c - - - :master_port
 -- create a helper function to create objects on each node
 CREATE OR REPLACE FUNCTION run_command_on_master_and_workers(p_sql text)
 RETURNS void LANGUAGE plpgsql AS $$
diff --git a/src/test/regress/sql/single_node_enterprise.sql b/src/test/regress/sql/single_node_enterprise.sql
index 6cfbbadc8..19393ba24 100644
--- a/src/test/regress/sql/single_node_enterprise.sql
+++ b/src/test/regress/sql/single_node_enterprise.sql
@@ -1,19 +1,6 @@
 -- we already have lots of tests targeting
 -- single node citus clusters in sql/single_node.sql
 -- in this file, we are testing enterprise features
-
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
-
-\c - - - :worker_1_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
-
-\c - - - :worker_2_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
-
-\c - - - :master_port
 CREATE SCHEMA single_node_ent;
 SET search_path TO single_node_ent;
 SET citus.shard_count TO 4;

From 2cbf8708cf7c001460a418f5f7a72f4a68534883 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Mon, 25 Mar 2024 15:30:59 +0300
Subject: [PATCH 31/78] Fixes test errors

---
 src/test/regress/expected/multi_test_helpers.out | 1 +
 src/test/regress/multi_schedule                  | 1 +
 src/test/regress/sql/multi_follower_dml.sql      | 1 -
 src/test/regress/sql/multi_test_helpers.sql      | 1 -
 4 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/test/regress/expected/multi_test_helpers.out b/src/test/regress/expected/multi_test_helpers.out
index cfe6ff204..0f31f2354 100644
--- a/src/test/regress/expected/multi_test_helpers.out
+++ b/src/test/regress/expected/multi_test_helpers.out
@@ -1,3 +1,4 @@
+-- File to create functions and helpers needed for subsequent tests
 -- create a helper function to create objects on each node
 CREATE OR REPLACE FUNCTION run_command_on_master_and_workers(p_sql text)
 RETURNS void LANGUAGE plpgsql AS $$
diff --git a/src/test/regress/multi_schedule b/src/test/regress/multi_schedule
index f24ebdeae..2fd1db9f9 100644
--- a/src/test/regress/multi_schedule
+++ b/src/test/regress/multi_schedule
@@ -1,6 +1,7 @@
 test: enable_ddl_propagation
 test: multi_test_helpers multi_test_helpers_superuser
 test: multi_cluster_management
+test: enable_ddl_propagation
 test: create_role_propagation
 test: pg16
 test: multi_create_fdw
diff --git a/src/test/regress/sql/multi_follower_dml.sql b/src/test/regress/sql/multi_follower_dml.sql
index 805097319..a3d548b12 100644
--- a/src/test/regress/sql/multi_follower_dml.sql
+++ b/src/test/regress/sql/multi_follower_dml.sql
@@ -1,6 +1,5 @@
 \c - - - :master_port
 
-show citus.enable_ddl_propagation;
 SET citus.shard_replication_factor TO 2;
 CREATE TABLE the_replicated_table (a int, b int, z bigserial);
 SELECT create_distributed_table('the_replicated_table', 'a');
diff --git a/src/test/regress/sql/multi_test_helpers.sql b/src/test/regress/sql/multi_test_helpers.sql
index 7d218361c..0a67ac64e 100644
--- a/src/test/regress/sql/multi_test_helpers.sql
+++ b/src/test/regress/sql/multi_test_helpers.sql
@@ -1,5 +1,4 @@
 -- File to create functions and helpers needed for subsequent tests
-
 -- create a helper function to create objects on each node
 CREATE OR REPLACE FUNCTION run_command_on_master_and_workers(p_sql text)
 RETURNS void LANGUAGE plpgsql AS $$

From d70304af860ae9f8f59bcbd08dae32539b9bf382 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Mon, 25 Mar 2024 15:45:10 +0300
Subject: [PATCH 32/78] Fixes test errors

---
 src/test/regress/columnar_schedule            |  1 +
 .../expected/enable_ddl_propagation.out       | 22 ++++++++++---------
 .../expected/enable_ddl_propagation_0.out     | 12 ++++++++++
 .../regress/sql/enable_ddl_propagation.sql    |  4 ++--
 4 files changed, 27 insertions(+), 12 deletions(-)
 create mode 100644 src/test/regress/expected/enable_ddl_propagation_0.out

diff --git a/src/test/regress/columnar_schedule b/src/test/regress/columnar_schedule
index ef8b2b288..1a6c2fce5 100644
--- a/src/test/regress/columnar_schedule
+++ b/src/test/regress/columnar_schedule
@@ -1,6 +1,7 @@
 test: enable_ddl_propagation
 test: multi_test_helpers multi_test_helpers_superuser columnar_test_helpers
 test: multi_cluster_management
+test: enable_ddl_propagation
 test: multi_test_catalog_views
 
 test: remove_coordinator_from_metadata
diff --git a/src/test/regress/expected/enable_ddl_propagation.out b/src/test/regress/expected/enable_ddl_propagation.out
index 6ad3f9bcd..bc079d1d1 100644
--- a/src/test/regress/expected/enable_ddl_propagation.out
+++ b/src/test/regress/expected/enable_ddl_propagation.out
@@ -1,14 +1,16 @@
-SELECT run_command_on_workers('ALTER SYSTEM SET citus.enable_ddl_propagation TO ON');
-       run_command_on_workers
+SELECT run_command_on_all_nodes('ALTER SYSTEM SET citus.enable_ddl_propagation TO ON');
+ run_command_on_all_nodes
 ---------------------------------------------------------------------
- (localhost,57637,t,"ALTER SYSTEM")
- (localhost,57638,t,"ALTER SYSTEM")
-(2 rows)
+ (17,t,"ALTER SYSTEM")
+ (16,t,"ALTER SYSTEM")
+ (18,t,"ALTER SYSTEM")
+(3 rows)
 
-SELECT run_command_on_workers('SELECT pg_reload_conf()');
- run_command_on_workers
+SELECT run_command_on_all_nodes('SELECT pg_reload_conf()');
+ run_command_on_all_nodes
 ---------------------------------------------------------------------
- (localhost,57637,t,t)
- (localhost,57638,t,t)
-(2 rows)
+ (17,t,t)
+ (16,t,t)
+ (18,t,t)
+(3 rows)
 
diff --git a/src/test/regress/expected/enable_ddl_propagation_0.out b/src/test/regress/expected/enable_ddl_propagation_0.out
new file mode 100644
index 000000000..6bf5cc091
--- /dev/null
+++ b/src/test/regress/expected/enable_ddl_propagation_0.out
@@ -0,0 +1,12 @@
+SELECT run_command_on_all_nodes('ALTER SYSTEM SET citus.enable_ddl_propagation TO ON');
+ run_command_on_all_nodes
+---------------------------------------------------------------------
+ (17,t,"ALTER SYSTEM")
+(1 rows)
+
+SELECT run_command_on_all_nodes('SELECT pg_reload_conf()');
+ run_command_on_all_nodes
+---------------------------------------------------------------------
+ (17,t,t)
+(1 rows)
+
diff --git a/src/test/regress/sql/enable_ddl_propagation.sql b/src/test/regress/sql/enable_ddl_propagation.sql
index ff7143359..778791fef 100644
--- a/src/test/regress/sql/enable_ddl_propagation.sql
+++ b/src/test/regress/sql/enable_ddl_propagation.sql
@@ -1,2 +1,2 @@
-SELECT run_command_on_workers('ALTER SYSTEM SET citus.enable_ddl_propagation TO ON');
-SELECT run_command_on_workers('SELECT pg_reload_conf()');
+SELECT run_command_on_all_nodes('ALTER SYSTEM SET citus.enable_ddl_propagation TO ON');
+SELECT run_command_on_all_nodes('SELECT pg_reload_conf()');

From 68985c21d621bfce8ac4dca5c40a1bd20e0dbe58 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Mon, 25 Mar 2024 16:03:39 +0300
Subject: [PATCH 33/78] Fixes enable_ddl propagation

---
 .../expected/enable_ddl_propagation.out       | 20 ++++++++-----------
 .../regress/sql/enable_ddl_propagation.sql    |  4 ++--
 2 files changed, 10 insertions(+), 14 deletions(-)

diff --git a/src/test/regress/expected/enable_ddl_propagation.out b/src/test/regress/expected/enable_ddl_propagation.out
index bc079d1d1..fe67dbc05 100644
--- a/src/test/regress/expected/enable_ddl_propagation.out
+++ b/src/test/regress/expected/enable_ddl_propagation.out
@@ -1,16 +1,12 @@
-SELECT run_command_on_all_nodes('ALTER SYSTEM SET citus.enable_ddl_propagation TO ON');
- run_command_on_all_nodes
+SELECT result from  run_command_on_all_nodes('ALTER SYSTEM SET citus.enable_ddl_propagation TO ON');
+    result
 ---------------------------------------------------------------------
- (17,t,"ALTER SYSTEM")
- (16,t,"ALTER SYSTEM")
- (18,t,"ALTER SYSTEM")
-(3 rows)
+ ALTER SYSTEM
+(1 rows)
 
-SELECT run_command_on_all_nodes('SELECT pg_reload_conf()');
- run_command_on_all_nodes
+SELECT result from  run_command_on_all_nodes('SELECT pg_reload_conf()');
+ result
 ---------------------------------------------------------------------
- (17,t,t)
- (16,t,t)
- (18,t,t)
-(3 rows)
+ t
+(1 rows)
 
diff --git a/src/test/regress/sql/enable_ddl_propagation.sql b/src/test/regress/sql/enable_ddl_propagation.sql
index 778791fef..4f1640744 100644
--- a/src/test/regress/sql/enable_ddl_propagation.sql
+++ b/src/test/regress/sql/enable_ddl_propagation.sql
@@ -1,2 +1,2 @@
-SELECT run_command_on_all_nodes('ALTER SYSTEM SET citus.enable_ddl_propagation TO ON');
-SELECT run_command_on_all_nodes('SELECT pg_reload_conf()');
+SELECT result from  run_command_on_all_nodes('ALTER SYSTEM SET citus.enable_ddl_propagation TO ON');
+SELECT result from  run_command_on_all_nodes('SELECT pg_reload_conf()');

From a6a76c7a36fb57ea64f55112cb8d638400e23172 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Mon, 25 Mar 2024 16:33:37 +0300
Subject: [PATCH 34/78] Fixes enable ddl prop

---
 .../regress/expected/enable_ddl_propagation.out  |  8 ++++++--
 .../expected/enable_ddl_propagation_0.out        | 16 ++++++++--------
 2 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/src/test/regress/expected/enable_ddl_propagation.out b/src/test/regress/expected/enable_ddl_propagation.out
index fe67dbc05..11cc7b82f 100644
--- a/src/test/regress/expected/enable_ddl_propagation.out
+++ b/src/test/regress/expected/enable_ddl_propagation.out
@@ -2,11 +2,15 @@ SELECT result from  run_command_on_all_nodes('ALTER SYSTEM SET citus.enable_ddl_
     result
 ---------------------------------------------------------------------
  ALTER SYSTEM
-(1 rows)
+ ALTER SYSTEM
+ ALTER SYSTEM
+(3 rows)
 
 SELECT result from  run_command_on_all_nodes('SELECT pg_reload_conf()');
  result
 ---------------------------------------------------------------------
  t
-(1 rows)
+ t
+ t
+(3 rows)
 
diff --git a/src/test/regress/expected/enable_ddl_propagation_0.out b/src/test/regress/expected/enable_ddl_propagation_0.out
index 6bf5cc091..efbedf854 100644
--- a/src/test/regress/expected/enable_ddl_propagation_0.out
+++ b/src/test/regress/expected/enable_ddl_propagation_0.out
@@ -1,12 +1,12 @@
-SELECT run_command_on_all_nodes('ALTER SYSTEM SET citus.enable_ddl_propagation TO ON');
- run_command_on_all_nodes
+SELECT result from  run_command_on_all_nodes('ALTER SYSTEM SET citus.enable_ddl_propagation TO ON');
+    result
 ---------------------------------------------------------------------
- (17,t,"ALTER SYSTEM")
-(1 rows)
+ ALTER SYSTEM
+(1 row)
 
-SELECT run_command_on_all_nodes('SELECT pg_reload_conf()');
- run_command_on_all_nodes
+SELECT result from  run_command_on_all_nodes('SELECT pg_reload_conf()');
+ result
 ---------------------------------------------------------------------
- (17,t,t)
-(1 rows)
+ t
+(1 row)
 

From 48b84b177a89c4326a76a4dd3b4ef9143888f8a4 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Mon, 25 Mar 2024 16:34:57 +0300
Subject: [PATCH 35/78] Fixes single node ent

---
 .../expected/single_node_enterprise.out       | 24 -------------------
 1 file changed, 24 deletions(-)

diff --git a/src/test/regress/expected/single_node_enterprise.out b/src/test/regress/expected/single_node_enterprise.out
index 94741a755..79f231864 100644
--- a/src/test/regress/expected/single_node_enterprise.out
+++ b/src/test/regress/expected/single_node_enterprise.out
@@ -1,30 +1,6 @@
 -- we already have lots of tests targeting
 -- single node citus clusters in sql/single_node.sql
 -- in this file, we are testing enterprise features
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
- pg_reload_conf
----------------------------------------------------------------------
- t
-(1 row)
-
-\c - - - :worker_1_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
- pg_reload_conf
----------------------------------------------------------------------
- t
-(1 row)
-
-\c - - - :worker_2_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
- pg_reload_conf
----------------------------------------------------------------------
- t
-(1 row)
-
-\c - - - :master_port
 CREATE SCHEMA single_node_ent;
 SET search_path TO single_node_ent;
 SET citus.shard_count TO 4;

From b74ae1ea18883e4e8e93774b8e5a40916fd5f3ba Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Mon, 25 Mar 2024 16:54:56 +0300
Subject: [PATCH 36/78] Fixes failure tests

---
 src/test/regress/failure_schedule | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/test/regress/failure_schedule b/src/test/regress/failure_schedule
index 8b992422e..f308fd7d2 100644
--- a/src/test/regress/failure_schedule
+++ b/src/test/regress/failure_schedule
@@ -1,4 +1,5 @@
 # import this file (from psql you can use \i) to use mitmproxy manually
+test: enable_ddl_propagation
 test: failure_test_helpers
 
 # this should only be run by pg_regress_multi, you don't need it

From 3564492c9020cddccd13fa74abdf37ad4139d3fc Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Mon, 25 Mar 2024 16:56:00 +0300
Subject: [PATCH 37/78] Rollbacks failure setup

---
 src/test/regress/expected/failure_setup.out | 24 ---------------------
 src/test/regress/sql/failure_setup.sql      | 11 ----------
 2 files changed, 35 deletions(-)

diff --git a/src/test/regress/expected/failure_setup.out b/src/test/regress/expected/failure_setup.out
index 7a26f8097..4cbb4b0a4 100644
--- a/src/test/regress/expected/failure_setup.out
+++ b/src/test/regress/expected/failure_setup.out
@@ -1,27 +1,3 @@
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
- pg_reload_conf
----------------------------------------------------------------------
- t
-(1 row)
-
-\c - - - :worker_1_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
- pg_reload_conf
----------------------------------------------------------------------
- t
-(1 row)
-
-\c - - - :worker_2_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
- pg_reload_conf
----------------------------------------------------------------------
- t
-(1 row)
-
-\c - - - :master_port
 SELECT citus.mitmproxy('conn.allow()');
  mitmproxy
 ---------------------------------------------------------------------
diff --git a/src/test/regress/sql/failure_setup.sql b/src/test/regress/sql/failure_setup.sql
index eb8f836cf..1dcc0e962 100644
--- a/src/test/regress/sql/failure_setup.sql
+++ b/src/test/regress/sql/failure_setup.sql
@@ -1,14 +1,3 @@
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
-
-\c - - - :worker_1_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
-
-\c - - - :worker_2_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
-
 \c - - - :master_port
 SELECT citus.mitmproxy('conn.allow()');
 

From a29c8125aa88b6e240cda55c567c439dadbc116a Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Mon, 25 Mar 2024 17:04:03 +0300
Subject: [PATCH 38/78] Fixes isolation tests

---
 .../expected/isolation_enable_ddl_propagation.out  |  9 +++++----
 .../isolation_enable_ddl_propagation_0.out         | 14 ++++++++++++++
 src/test/regress/isolation_schedule                |  1 +
 .../spec/isolation_enable_ddl_propagation.spec     |  4 ++--
 4 files changed, 22 insertions(+), 6 deletions(-)
 create mode 100644 src/test/regress/expected/isolation_enable_ddl_propagation_0.out

diff --git a/src/test/regress/expected/isolation_enable_ddl_propagation.out b/src/test/regress/expected/isolation_enable_ddl_propagation.out
index 88f38b5bc..d04e68f61 100644
--- a/src/test/regress/expected/isolation_enable_ddl_propagation.out
+++ b/src/test/regress/expected/isolation_enable_ddl_propagation.out
@@ -1,11 +1,12 @@
 Parsed test spec with 1 sessions
 
 starting permutation: s1-begin s1-commit
-run_command_on_workers
+result
 ---------------------------------------------------------------------
-(localhost,57637,t,t)
-(localhost,57638,t,t)
-(2 rows)
+t
+t
+t
+(3 rows)
 
 step s1-begin:
  BEGIN;
diff --git a/src/test/regress/expected/isolation_enable_ddl_propagation_0.out b/src/test/regress/expected/isolation_enable_ddl_propagation_0.out
new file mode 100644
index 000000000..8bfe44546
--- /dev/null
+++ b/src/test/regress/expected/isolation_enable_ddl_propagation_0.out
@@ -0,0 +1,14 @@
+Parsed test spec with 1 sessions
+
+starting permutation: s1-begin s1-commit
+result
+---------------------------------------------------------------------
+t
+(1 row)
+
+step s1-begin:
+ BEGIN;
+
+step s1-commit:
+ COMMIT;
+
diff --git a/src/test/regress/isolation_schedule b/src/test/regress/isolation_schedule
index 52aa49f6c..826c45792 100644
--- a/src/test/regress/isolation_schedule
+++ b/src/test/regress/isolation_schedule
@@ -1,3 +1,4 @@
+test: isolation_enable_ddl_propagation
 test: isolation_setup
 test: isolation_add_remove_node
 test: isolation_update_node
diff --git a/src/test/regress/spec/isolation_enable_ddl_propagation.spec b/src/test/regress/spec/isolation_enable_ddl_propagation.spec
index 7240653d7..e94578088 100644
--- a/src/test/regress/spec/isolation_enable_ddl_propagation.spec
+++ b/src/test/regress/spec/isolation_enable_ddl_propagation.spec
@@ -1,7 +1,7 @@
 setup
 {
-	SELECT run_command_on_workers('ALTER SYSTEM SET citus.enable_ddl_propagation TO ON');
-	SELECT run_command_on_workers('SELECT pg_reload_conf()');
+	SELECT result from run_command_on_all_nodes('ALTER SYSTEM SET citus.enable_ddl_propagation TO ON');
+	SELECT result from run_command_on_all_nodes('SELECT pg_reload_conf()');
 }
 
 session "s1"

From 3df18fe1745b0bb52b16653f54e98a24d79d5046 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Mon, 25 Mar 2024 17:38:16 +0300
Subject: [PATCH 39/78] Adds into base_isolation

---
 src/test/regress/base_isolation_schedule | 1 +
 src/test/regress/minimal_schedule        | 1 +
 2 files changed, 2 insertions(+)

diff --git a/src/test/regress/base_isolation_schedule b/src/test/regress/base_isolation_schedule
index 1fc2f7de5..fcc9c85fd 100644
--- a/src/test/regress/base_isolation_schedule
+++ b/src/test/regress/base_isolation_schedule
@@ -3,3 +3,4 @@
 # ----------
 test: isolation_setup
 test: isolation_cluster_management
+test: isolation_enable_ddl_propagation
diff --git a/src/test/regress/minimal_schedule b/src/test/regress/minimal_schedule
index 072039a25..1f524797d 100644
--- a/src/test/regress/minimal_schedule
+++ b/src/test/regress/minimal_schedule
@@ -1,3 +1,4 @@
+test: enable_ddl_propagation
 test: minimal_cluster_management
 test: enable_ddl_propagation
 test: multi_test_helpers multi_test_helpers_superuser multi_create_fdw columnar_test_helpers multi_test_catalog_views tablespace

From 8952e060022e5d3569a80cf40c087b26007eb440 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Mon, 25 Mar 2024 17:41:09 +0300
Subject: [PATCH 40/78] Fixes failure setup

---
 src/test/regress/sql/failure_setup.sql | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/test/regress/sql/failure_setup.sql b/src/test/regress/sql/failure_setup.sql
index 1dcc0e962..4c209f14d 100644
--- a/src/test/regress/sql/failure_setup.sql
+++ b/src/test/regress/sql/failure_setup.sql
@@ -1,4 +1,3 @@
-\c - - - :master_port
 SELECT citus.mitmproxy('conn.allow()');
 
 -- add the workers

From 24f9d599d02ce0f0c475b228cb1925b96c21f955 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Mon, 25 Mar 2024 18:18:16 +0300
Subject: [PATCH 41/78] Fixes minimal_cluster_management issue

---
 .../expected/minimal_cluster_management.out   | 24 -------------------
 .../regress/sql/multi_cluster_management.sql  | 13 ----------
 2 files changed, 37 deletions(-)

diff --git a/src/test/regress/expected/minimal_cluster_management.out b/src/test/regress/expected/minimal_cluster_management.out
index 09fcf7b4a..d05e83ed5 100644
--- a/src/test/regress/expected/minimal_cluster_management.out
+++ b/src/test/regress/expected/minimal_cluster_management.out
@@ -1,27 +1,3 @@
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
- pg_reload_conf
----------------------------------------------------------------------
- t
-(1 row)
-
-\c - - - :worker_1_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
- pg_reload_conf
----------------------------------------------------------------------
- t
-(1 row)
-
-\c - - - :worker_2_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
- pg_reload_conf
----------------------------------------------------------------------
- t
-(1 row)
-
-\c - - - :master_port
 SET citus.next_shard_id TO 1220000;
 ALTER SEQUENCE pg_catalog.pg_dist_colocationid_seq RESTART 1390000;
 ALTER SEQUENCE pg_catalog.pg_dist_groupid_seq RESTART 1;
diff --git a/src/test/regress/sql/multi_cluster_management.sql b/src/test/regress/sql/multi_cluster_management.sql
index ed3d574b6..86fbd15b6 100644
--- a/src/test/regress/sql/multi_cluster_management.sql
+++ b/src/test/regress/sql/multi_cluster_management.sql
@@ -1,16 +1,3 @@
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
-
-\c - - - :worker_1_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
-
-\c - - - :worker_2_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
-
-\c - - - :master_port
-
 SET citus.next_shard_id TO 1220000;
 ALTER SEQUENCE pg_catalog.pg_dist_colocationid_seq RESTART 1390000;
 ALTER SEQUENCE pg_catalog.pg_dist_groupid_seq RESTART 1;

From 625c8d4af091db202cb3217e19c05516763465a5 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Mon, 25 Mar 2024 18:20:20 +0300
Subject: [PATCH 42/78] Fixes test issues

---
 .../regress/spec/isolation_move_placement_vs_move_placement.spec | 1 -
 src/test/regress/sql/multi_test_helpers.sql                      | 1 +
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/regress/spec/isolation_move_placement_vs_move_placement.spec b/src/test/regress/spec/isolation_move_placement_vs_move_placement.spec
index 7c2930b90..b55ab6e54 100644
--- a/src/test/regress/spec/isolation_move_placement_vs_move_placement.spec
+++ b/src/test/regress/spec/isolation_move_placement_vs_move_placement.spec
@@ -2,7 +2,6 @@
 // so setting the corresponding shard here is useful
 setup
 {
-	show citus.enable_ddl_propagation;
 	SET citus.shard_count TO 2;
 	SET citus.shard_replication_factor TO 1;
 	ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 102011;
diff --git a/src/test/regress/sql/multi_test_helpers.sql b/src/test/regress/sql/multi_test_helpers.sql
index 0a67ac64e..7d218361c 100644
--- a/src/test/regress/sql/multi_test_helpers.sql
+++ b/src/test/regress/sql/multi_test_helpers.sql
@@ -1,4 +1,5 @@
 -- File to create functions and helpers needed for subsequent tests
+
 -- create a helper function to create objects on each node
 CREATE OR REPLACE FUNCTION run_command_on_master_and_workers(p_sql text)
 RETURNS void LANGUAGE plpgsql AS $$

From 602c8c1206ad1e98cde4d2c7d4431a85c5c081f4 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Mon, 25 Mar 2024 18:23:22 +0300
Subject: [PATCH 43/78] Removes unnecessary changes

---
 .../expected/multi_follower_sanity_check.out  | 23 -------------------
 src/test/regress/multi_follower_schedule      |  1 +
 src/test/regress/multi_mx_schedule            |  1 +
 src/test/regress/sql/multi_extension.sql      | 11 ---------
 .../sql/multi_follower_sanity_check.sql       | 11 ---------
 5 files changed, 2 insertions(+), 45 deletions(-)

diff --git a/src/test/regress/expected/multi_follower_sanity_check.out b/src/test/regress/expected/multi_follower_sanity_check.out
index 4fee237fc..6124f26c9 100644
--- a/src/test/regress/expected/multi_follower_sanity_check.out
+++ b/src/test/regress/expected/multi_follower_sanity_check.out
@@ -1,27 +1,4 @@
 -- check that the nodes are all in read-only mode and rejecting write queries
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
- pg_reload_conf
----------------------------------------------------------------------
- t
-(1 row)
-
-\c - - - :worker_1_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
- pg_reload_conf
----------------------------------------------------------------------
- t
-(1 row)
-
-\c - - - :worker_2_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
- pg_reload_conf
----------------------------------------------------------------------
- t
-(1 row)
-
 \c - - - :follower_master_port
 ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
 SELECT pg_reload_conf();
diff --git a/src/test/regress/multi_follower_schedule b/src/test/regress/multi_follower_schedule
index c1f0ac6cb..686d6a38c 100644
--- a/src/test/regress/multi_follower_schedule
+++ b/src/test/regress/multi_follower_schedule
@@ -1,3 +1,4 @@
+test: enable_ddl_propagation
 test: multi_follower_sanity_check
 test: follower_single_node
 test: multi_follower_select_statements
diff --git a/src/test/regress/multi_mx_schedule b/src/test/regress/multi_mx_schedule
index 6654b4ab0..6e8b652e4 100644
--- a/src/test/regress/multi_mx_schedule
+++ b/src/test/regress/multi_mx_schedule
@@ -13,6 +13,7 @@
 # ---
 # Tests around schema changes, these are run first, so there's no preexisting objects.
 # ---
+test: enable_ddl_propagation
 test: multi_extension
 test: multi_test_helpers multi_test_helpers_superuser
 test: multi_mx_node_metadata
diff --git a/src/test/regress/sql/multi_extension.sql b/src/test/regress/sql/multi_extension.sql
index f8168b07f..8cbbbc3ed 100644
--- a/src/test/regress/sql/multi_extension.sql
+++ b/src/test/regress/sql/multi_extension.sql
@@ -7,18 +7,7 @@
 -- not done yet.
 --
 -- Upgrade tests verify the objects are added in citus_finish_pg_upgrade()
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
 
-\c - - - :worker_1_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
-
-\c - - - :worker_2_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
-
-\c - - - :master_port
 SET citus.next_shard_id TO 580000;
 CREATE SCHEMA multi_extension;
 
diff --git a/src/test/regress/sql/multi_follower_sanity_check.sql b/src/test/regress/sql/multi_follower_sanity_check.sql
index eb8c2ee33..bdd0f5b55 100644
--- a/src/test/regress/sql/multi_follower_sanity_check.sql
+++ b/src/test/regress/sql/multi_follower_sanity_check.sql
@@ -1,15 +1,4 @@
 -- check that the nodes are all in read-only mode and rejecting write queries
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
-
-\c - - - :worker_1_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
-
-\c - - - :worker_2_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
-
 \c - - - :follower_master_port
 ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
 SELECT pg_reload_conf();

From 20135540509d309387bf589fcaa7bc9df30ef7aa Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Mon, 25 Mar 2024 18:27:47 +0300
Subject: [PATCH 44/78] Removes unnecessary enable_ddl_propagation entries

---
 src/test/regress/base_isolation_schedule                    | 2 +-
 src/test/regress/columnar_schedule                          | 1 -
 src/test/regress/enterprise_isolation_logicalrep_1_schedule | 2 +-
 src/test/regress/enterprise_isolation_logicalrep_2_schedule | 2 +-
 src/test/regress/enterprise_isolation_logicalrep_3_schedule | 2 +-
 src/test/regress/enterprise_isolation_schedule              | 2 +-
 src/test/regress/isolation_schedule                         | 1 -
 src/test/regress/minimal_schedule                           | 1 -
 src/test/regress/multi_schedule                             | 1 -
 9 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/src/test/regress/base_isolation_schedule b/src/test/regress/base_isolation_schedule
index fcc9c85fd..d19a0463b 100644
--- a/src/test/regress/base_isolation_schedule
+++ b/src/test/regress/base_isolation_schedule
@@ -1,6 +1,6 @@
 # ----------
 # isolation setup steps
 # ----------
+test: isolation_enable_ddl_propagation
 test: isolation_setup
 test: isolation_cluster_management
-test: isolation_enable_ddl_propagation
diff --git a/src/test/regress/columnar_schedule b/src/test/regress/columnar_schedule
index 1a6c2fce5..ef8b2b288 100644
--- a/src/test/regress/columnar_schedule
+++ b/src/test/regress/columnar_schedule
@@ -1,7 +1,6 @@
 test: enable_ddl_propagation
 test: multi_test_helpers multi_test_helpers_superuser columnar_test_helpers
 test: multi_cluster_management
-test: enable_ddl_propagation
 test: multi_test_catalog_views
 
 test: remove_coordinator_from_metadata
diff --git a/src/test/regress/enterprise_isolation_logicalrep_1_schedule b/src/test/regress/enterprise_isolation_logicalrep_1_schedule
index 509548db1..10a77aa18 100644
--- a/src/test/regress/enterprise_isolation_logicalrep_1_schedule
+++ b/src/test/regress/enterprise_isolation_logicalrep_1_schedule
@@ -1,10 +1,10 @@
+test: isolation_enable_ddl_propagation
 test: isolation_setup
 
 # tests that change node metadata should precede
 # isolation_cluster_management such that tests
 # that come later can be parallelized
 test: isolation_cluster_management
-test: isolation_enable_ddl_propagation
 
 test: isolation_logical_replication_single_shard_commands
 test: isolation_logical_replication_nonsu_nonbypassrls
diff --git a/src/test/regress/enterprise_isolation_logicalrep_2_schedule b/src/test/regress/enterprise_isolation_logicalrep_2_schedule
index 8aae14af6..167190391 100644
--- a/src/test/regress/enterprise_isolation_logicalrep_2_schedule
+++ b/src/test/regress/enterprise_isolation_logicalrep_2_schedule
@@ -1,10 +1,10 @@
+test: isolation_enable_ddl_propagation
 test: isolation_setup
 
 # tests that change node metadata should precede
 # isolation_cluster_management such that tests
 # that come later can be parallelized
 test: isolation_cluster_management
-test: isolation_enable_ddl_propagation
 
 test: isolation_logical_replication_single_shard_commands_on_mx
 test: isolation_logical_replication_multi_shard_commands_on_mx
diff --git a/src/test/regress/enterprise_isolation_logicalrep_3_schedule b/src/test/regress/enterprise_isolation_logicalrep_3_schedule
index 06b3aacdd..9b252005e 100644
--- a/src/test/regress/enterprise_isolation_logicalrep_3_schedule
+++ b/src/test/regress/enterprise_isolation_logicalrep_3_schedule
@@ -1,10 +1,10 @@
+test: isolation_enable_ddl_propagation
 test: isolation_setup
 
 # tests that change node metadata should precede
 # isolation_cluster_management such that tests
 # that come later can be parallelized
 test: isolation_cluster_management
-test: isolation_enable_ddl_propagation
 
 test: isolation_logical_replication_with_partitioning
 test: isolation_logical_replication_binaryless
diff --git a/src/test/regress/enterprise_isolation_schedule b/src/test/regress/enterprise_isolation_schedule
index bde8f225d..e0fc691d8 100644
--- a/src/test/regress/enterprise_isolation_schedule
+++ b/src/test/regress/enterprise_isolation_schedule
@@ -1,10 +1,10 @@
+test: isolation_enable_ddl_propagation
 test: isolation_setup
 
 # tests that change node metadata should precede
 # isolation_cluster_management such that tests
 # that come later can be parallelized
 test: isolation_cluster_management
-test: isolation_enable_ddl_propagation
 
 test: isolation_move_placement_vs_move_placement
 test: isolation_move_placement_vs_modification
diff --git a/src/test/regress/isolation_schedule b/src/test/regress/isolation_schedule
index 826c45792..5c5f9ccbb 100644
--- a/src/test/regress/isolation_schedule
+++ b/src/test/regress/isolation_schedule
@@ -15,7 +15,6 @@ test: isolation_extension_commands
 # isolation_cluster_management such that tests
 # that come later can be parallelized
 test: isolation_cluster_management
-test: isolation_enable_ddl_propagation
 
 # the following tests depend on the distributed
 # transactionId, so should not be parallelized
diff --git a/src/test/regress/minimal_schedule b/src/test/regress/minimal_schedule
index 1f524797d..fe2d9ce94 100644
--- a/src/test/regress/minimal_schedule
+++ b/src/test/regress/minimal_schedule
@@ -1,4 +1,3 @@
 test: enable_ddl_propagation
 test: minimal_cluster_management
-test: enable_ddl_propagation
 test: multi_test_helpers multi_test_helpers_superuser multi_create_fdw columnar_test_helpers multi_test_catalog_views tablespace
diff --git a/src/test/regress/multi_schedule b/src/test/regress/multi_schedule
index 2fd1db9f9..f24ebdeae 100644
--- a/src/test/regress/multi_schedule
+++ b/src/test/regress/multi_schedule
@@ -1,7 +1,6 @@
 test: enable_ddl_propagation
 test: multi_test_helpers multi_test_helpers_superuser
 test: multi_cluster_management
-test: enable_ddl_propagation
 test: create_role_propagation
 test: pg16
 test: multi_create_fdw

From fbb38dd7ea8f080c7c9e855e73b1aa851c41500a Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Mon, 25 Mar 2024 18:43:17 +0300
Subject: [PATCH 45/78] Fixes additional tests

---
 .../expected/multi_cluster_management.out     | 24 -------------------
 src/test/regress/operations_schedule          |  1 +
 src/test/regress/split_schedule               |  1 +
 3 files changed, 2 insertions(+), 24 deletions(-)

diff --git a/src/test/regress/expected/multi_cluster_management.out b/src/test/regress/expected/multi_cluster_management.out
index bfc3a1111..3eb549ab5 100644
--- a/src/test/regress/expected/multi_cluster_management.out
+++ b/src/test/regress/expected/multi_cluster_management.out
@@ -1,27 +1,3 @@
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
- pg_reload_conf
----------------------------------------------------------------------
- t
-(1 row)
-
-\c - - - :worker_1_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
- pg_reload_conf
----------------------------------------------------------------------
- t
-(1 row)
-
-\c - - - :worker_2_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
- pg_reload_conf
----------------------------------------------------------------------
- t
-(1 row)
-
-\c - - - :master_port
 SET citus.next_shard_id TO 1220000;
 ALTER SEQUENCE pg_catalog.pg_dist_colocationid_seq RESTART 1390000;
 ALTER SEQUENCE pg_catalog.pg_dist_groupid_seq RESTART 1;
diff --git a/src/test/regress/operations_schedule b/src/test/regress/operations_schedule
index 6dbc303c2..0f8b5b816 100644
--- a/src/test/regress/operations_schedule
+++ b/src/test/regress/operations_schedule
@@ -1,3 +1,4 @@
+test: enable_ddl_propagation
 test: multi_test_helpers multi_test_helpers_superuser
 test: multi_cluster_management
 test: multi_test_catalog_views
diff --git a/src/test/regress/split_schedule b/src/test/regress/split_schedule
index 53c422eab..fa445f151 100644
--- a/src/test/regress/split_schedule
+++ b/src/test/regress/split_schedule
@@ -1,5 +1,6 @@
 # Split Shard tests.
 # Include tests from 'minimal_schedule' for setup.
+test: enable_ddl_propagation
 test: multi_test_helpers multi_test_helpers_superuser columnar_test_helpers
 test: multi_cluster_management
 test: remove_coordinator_from_metadata

From b0282e6219095216151e00d46dc8ddf95d718d0a Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Mon, 25 Mar 2024 19:06:51 +0300
Subject: [PATCH 46/78] Adds a comment

---
 src/backend/distributed/commands/database.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c
index be444e23b..bc5908629 100644
--- a/src/backend/distributed/commands/database.c
+++ b/src/backend/distributed/commands/database.c
@@ -292,7 +292,9 @@ PreprocessAlterDatabaseStmt(Node *node, const char *queryString,
 	{
 		return NIL;
 	}
-
+	/* Since ALTER TABLE SET TABLESPACE statement is not supported */
+	/* inside a transaction block, we need to send the command to the */
+	/* main database directly to make it work */
 	if (!IsSetTablespaceStatement(stmt))
 	{
 		EnsureCoordinator();

From 7bde99c7d764f80e353caf4c8dbdce23e929f8a1 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Mon, 25 Mar 2024 19:08:41 +0300
Subject: [PATCH 47/78] Fixes multi extension test

---
 src/test/regress/expected/multi_extension.out | 24 -------------------
 1 file changed, 24 deletions(-)

diff --git a/src/test/regress/expected/multi_extension.out b/src/test/regress/expected/multi_extension.out
index e24f02a8a..aaafce715 100644
--- a/src/test/regress/expected/multi_extension.out
+++ b/src/test/regress/expected/multi_extension.out
@@ -7,30 +7,6 @@
 -- not done yet.
 --
 -- Upgrade tests verify the objects are added in citus_finish_pg_upgrade()
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
- pg_reload_conf
----------------------------------------------------------------------
- t
-(1 row)
-
-\c - - - :worker_1_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
- pg_reload_conf
----------------------------------------------------------------------
- t
-(1 row)
-
-\c - - - :worker_2_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
- pg_reload_conf
----------------------------------------------------------------------
- t
-(1 row)
-
-\c - - - :master_port
 SET citus.next_shard_id TO 580000;
 CREATE SCHEMA multi_extension;
 SELECT $definition$

From 603cec47dfad3e693fb5001cff4adac337c3644f Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Tue, 26 Mar 2024 10:34:00 +0300
Subject: [PATCH 48/78] Adds enable ddl prop after multi_cluster_management

---
 src/test/regress/base_schedule       | 1 +
 src/test/regress/columnar_schedule   | 1 +
 src/test/regress/enterprise_schedule | 1 +
 src/test/regress/multi_1_schedule    | 1 +
 src/test/regress/multi_mx_schedule   | 2 ++
 src/test/regress/multi_schedule      | 1 +
 src/test/regress/mx_base_schedule    | 1 +
 src/test/regress/operations_schedule | 1 +
 src/test/regress/split_schedule      | 1 +
 9 files changed, 10 insertions(+)

diff --git a/src/test/regress/base_schedule b/src/test/regress/base_schedule
index f56adb5a3..0ce3ed1c0 100644
--- a/src/test/regress/base_schedule
+++ b/src/test/regress/base_schedule
@@ -4,6 +4,7 @@
 test: enable_ddl_propagation
 test: multi_test_helpers multi_test_helpers_superuser multi_create_fdw columnar_test_helpers failure_test_helpers
 test: multi_cluster_management
+test: enable_ddl_propagation
 test: multi_test_catalog_views
 test: multi_create_table
 test: multi_behavioral_analytics_create_table
diff --git a/src/test/regress/columnar_schedule b/src/test/regress/columnar_schedule
index ef8b2b288..1a6c2fce5 100644
--- a/src/test/regress/columnar_schedule
+++ b/src/test/regress/columnar_schedule
@@ -1,6 +1,7 @@
 test: enable_ddl_propagation
 test: multi_test_helpers multi_test_helpers_superuser columnar_test_helpers
 test: multi_cluster_management
+test: enable_ddl_propagation
 test: multi_test_catalog_views
 
 test: remove_coordinator_from_metadata
diff --git a/src/test/regress/enterprise_schedule b/src/test/regress/enterprise_schedule
index 59156def7..4e0802d20 100644
--- a/src/test/regress/enterprise_schedule
+++ b/src/test/regress/enterprise_schedule
@@ -5,6 +5,7 @@ test: single_node_enterprise
 
 test: multi_test_helpers multi_test_helpers_superuser
 test: multi_cluster_management
+test: enable_ddl_propagation
 test: multi_test_catalog_views
 test: multi_data_types
 test: multi_tenant_isolation
diff --git a/src/test/regress/multi_1_schedule b/src/test/regress/multi_1_schedule
index 6fc2ca36c..394e3fea4 100644
--- a/src/test/regress/multi_1_schedule
+++ b/src/test/regress/multi_1_schedule
@@ -22,6 +22,7 @@ test: single_node
 test: relation_access_tracking_single_node
 test: single_node_truncate
 test: multi_cluster_management
+test: enable_ddl_propagation
 
 # below tests are placed right after multi_cluster_management as we do
 # remove/add node operations and we do not want any preexisting objects
diff --git a/src/test/regress/multi_mx_schedule b/src/test/regress/multi_mx_schedule
index 6e8b652e4..e332f4897 100644
--- a/src/test/regress/multi_mx_schedule
+++ b/src/test/regress/multi_mx_schedule
@@ -18,6 +18,8 @@ test: multi_extension
 test: multi_test_helpers multi_test_helpers_superuser
 test: multi_mx_node_metadata
 test: multi_cluster_management
+test: enable_ddl_propagation
+
 test: remove_coordinator_from_metadata
 test: multi_mx_function_table_reference
 test: multi_test_catalog_views
diff --git a/src/test/regress/multi_schedule b/src/test/regress/multi_schedule
index f24ebdeae..2fd1db9f9 100644
--- a/src/test/regress/multi_schedule
+++ b/src/test/regress/multi_schedule
@@ -1,6 +1,7 @@
 test: enable_ddl_propagation
 test: multi_test_helpers multi_test_helpers_superuser
 test: multi_cluster_management
+test: enable_ddl_propagation
 test: create_role_propagation
 test: pg16
 test: multi_create_fdw
diff --git a/src/test/regress/mx_base_schedule b/src/test/regress/mx_base_schedule
index 598e03d13..88dca6019 100644
--- a/src/test/regress/mx_base_schedule
+++ b/src/test/regress/mx_base_schedule
@@ -3,6 +3,7 @@
 # ----------
 test: multi_test_helpers multi_test_helpers_superuser
 test: multi_cluster_management
+test: enable_ddl_propagation
 test: multi_mx_function_table_reference
 test: multi_test_catalog_views
 
diff --git a/src/test/regress/operations_schedule b/src/test/regress/operations_schedule
index 0f8b5b816..86b46c7e5 100644
--- a/src/test/regress/operations_schedule
+++ b/src/test/regress/operations_schedule
@@ -1,6 +1,7 @@
 test: enable_ddl_propagation
 test: multi_test_helpers multi_test_helpers_superuser
 test: multi_cluster_management
+test: enable_ddl_propagation
 test: multi_test_catalog_views
 test: worker_copy_table_to_node
 test: shard_rebalancer_unit
diff --git a/src/test/regress/split_schedule b/src/test/regress/split_schedule
index fa445f151..f70b19d1a 100644
--- a/src/test/regress/split_schedule
+++ b/src/test/regress/split_schedule
@@ -3,6 +3,7 @@
 test: enable_ddl_propagation
 test: multi_test_helpers multi_test_helpers_superuser columnar_test_helpers
 test: multi_cluster_management
+test: enable_ddl_propagation
 test: remove_coordinator_from_metadata
 test: multi_test_catalog_views
 test: tablespace

From c47ac5e1da74eb464f70b6d0a4037f1b797c3d18 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Tue, 26 Mar 2024 10:37:40 +0300
Subject: [PATCH 49/78] Fixes indentation

---
 src/backend/distributed/commands/database.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c
index bc5908629..0447f98b5 100644
--- a/src/backend/distributed/commands/database.c
+++ b/src/backend/distributed/commands/database.c
@@ -292,6 +292,7 @@ PreprocessAlterDatabaseStmt(Node *node, const char *queryString,
 	{
 		return NIL;
 	}
+
 	/* Since ALTER TABLE SET TABLESPACE statement is not supported */
 	/* inside a transaction block, we need to send the command to the */
 	/* main database directly to make it work */

From ce6d76fbb1d3254ccb1765cd89d48f52f9ed5725 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Tue, 26 Mar 2024 10:56:31 +0300
Subject: [PATCH 50/78] Adds log

---
 src/test/regress/sql/minimal_cluster_management.sql | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/test/regress/sql/minimal_cluster_management.sql b/src/test/regress/sql/minimal_cluster_management.sql
index 30f69d43d..a393cf7d2 100644
--- a/src/test/regress/sql/minimal_cluster_management.sql
+++ b/src/test/regress/sql/minimal_cluster_management.sql
@@ -41,3 +41,4 @@ DROP TABLE test_dist;
 ALTER SEQUENCE pg_catalog.pg_dist_node_nodeid_seq RESTART 30;
 ALTER SEQUENCE pg_catalog.pg_dist_groupid_seq RESTART 18;
 ALTER SEQUENCE pg_catalog.pg_dist_placement_placementid_seq RESTART 83;
+show citus.enable_ddl_propagation

From 21bbe5fdf796392d6f49a2be555400273731e044 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Tue, 26 Mar 2024 11:03:40 +0300
Subject: [PATCH 51/78] Fixes Flaky tests

---
 src/test/regress/minimal_schedule | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/test/regress/minimal_schedule b/src/test/regress/minimal_schedule
index fe2d9ce94..1f524797d 100644
--- a/src/test/regress/minimal_schedule
+++ b/src/test/regress/minimal_schedule
@@ -1,3 +1,4 @@
 test: enable_ddl_propagation
 test: minimal_cluster_management
+test: enable_ddl_propagation
 test: multi_test_helpers multi_test_helpers_superuser multi_create_fdw columnar_test_helpers multi_test_catalog_views tablespace

From 64722925936787fa1fcd7f7e589c421b37cb4e39 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Tue, 26 Mar 2024 11:18:13 +0300
Subject: [PATCH 52/78] Adds logs

---
 src/test/regress/mx_base_schedule                   | 1 +
 src/test/regress/sql/minimal_cluster_management.sql | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/test/regress/mx_base_schedule b/src/test/regress/mx_base_schedule
index 88dca6019..4c7cdfca4 100644
--- a/src/test/regress/mx_base_schedule
+++ b/src/test/regress/mx_base_schedule
@@ -1,6 +1,7 @@
 # ----------
 # Only run few basic tests to set up a testing environment
 # ----------
+test: enable_ddl_propagation
 test: multi_test_helpers multi_test_helpers_superuser
 test: multi_cluster_management
 test: enable_ddl_propagation
diff --git a/src/test/regress/sql/minimal_cluster_management.sql b/src/test/regress/sql/minimal_cluster_management.sql
index a393cf7d2..c17c74fdb 100644
--- a/src/test/regress/sql/minimal_cluster_management.sql
+++ b/src/test/regress/sql/minimal_cluster_management.sql
@@ -41,4 +41,4 @@ DROP TABLE test_dist;
 ALTER SEQUENCE pg_catalog.pg_dist_node_nodeid_seq RESTART 30;
 ALTER SEQUENCE pg_catalog.pg_dist_groupid_seq RESTART 18;
 ALTER SEQUENCE pg_catalog.pg_dist_placement_placementid_seq RESTART 83;
-show citus.enable_ddl_propagation
+select result from run_command_on_all_nodes($$show citus.enable_ddl_propagation$$);

From 56171a4a17dcb35b1585278bcb013ce46c9e66e0 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Tue, 26 Mar 2024 13:02:43 +0300
Subject: [PATCH 53/78] Enriches tests

---
 .../alter_database_from_nonmain_db.out        | 18 +++++++++++++
 .../sql/alter_database_from_nonmain_db.sql    | 25 +++++++++++++++++--
 .../sql/minimal_cluster_management.sql        |  1 -
 3 files changed, 41 insertions(+), 3 deletions(-)

diff --git a/src/test/regress/expected/alter_database_from_nonmain_db.out b/src/test/regress/expected/alter_database_from_nonmain_db.out
index 9a1312c67..bb1f97612 100644
--- a/src/test/regress/expected/alter_database_from_nonmain_db.out
+++ b/src/test/regress/expected/alter_database_from_nonmain_db.out
@@ -19,9 +19,18 @@ set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
 NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+\c test_alter_db_from_nonmain_db - - :worker_1_port
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
+alter database "altered_database!'2" set tablespace "pg_default";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 alter database "altered_database!'2" rename to altered_database_renamed;
 NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" RENAME TO altered_database_renamed', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+\c test_alter_db_from_nonmain_db - - :worker_2_port
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database altered_database_renamed rename to "altered_database!'2";
 NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database_renamed RENAME TO "altered_database!''2"', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
@@ -46,6 +55,9 @@ set citus.enable_create_database_propagation=on;
 alter database "altered_database!'2" owner to test_owner_non_main_db;
 NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" OWNER TO test_owner_non_main_db;', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+\c test_alter_db_from_nonmain_db - - :worker_1_port
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" owner to CURRENT_USER;
 NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" OWNER TO postgres;', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
@@ -53,6 +65,9 @@ alter database "altered_database!'2" set default_transaction_read_only = true;
 NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET default_transaction_read_only = ''true''', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 set default_transaction_read_only = false;
+\c test_alter_db_from_nonmain_db - - :worker_2_port
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" set default_transaction_read_only from current;
 NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET default_transaction_read_only FROM CURRENT', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
@@ -68,6 +83,9 @@ DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 alter database "altered_database!'2" set TIME ZONE LOCAL;
 NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET timezone TO DEFAULT', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+\c test_alter_db_from_nonmain_db
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" set TIME ZONE DEFAULT;
 NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET timezone TO DEFAULT', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
diff --git a/src/test/regress/sql/alter_database_from_nonmain_db.sql b/src/test/regress/sql/alter_database_from_nonmain_db.sql
index 28cb3888b..b8a99a495 100644
--- a/src/test/regress/sql/alter_database_from_nonmain_db.sql
+++ b/src/test/regress/sql/alter_database_from_nonmain_db.sql
@@ -18,13 +18,20 @@ CREATE TABLESPACE "ts-needs\!escape2" LOCATION :'alter_db_tablespace';
 CREATE TABLESPACE "ts-needs\!escape2" LOCATION :'alter_db_tablespace';
 
 \c test_alter_db_from_nonmain_db
-
 set citus.log_remote_commands = true;
-
 set citus.grep_remote_commands = "%ALTER DATABASE%";
 
 alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+
+\c test_alter_db_from_nonmain_db - - :worker_1_port
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
+alter database "altered_database!'2" set tablespace "pg_default";
 alter database "altered_database!'2" rename to altered_database_renamed;
+
+\c test_alter_db_from_nonmain_db - - :worker_2_port
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database altered_database_renamed rename to "altered_database!'2";
 
 alter database "altered_database!'2" with
@@ -38,19 +45,32 @@ alter database "altered_database!'2" with
 
 \c regression
 create role test_owner_non_main_db;
+
 \c test_alter_db_from_nonmain_db
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
 set citus.enable_create_database_propagation=on;
 alter database "altered_database!'2" owner to test_owner_non_main_db;
+
+\c test_alter_db_from_nonmain_db - - :worker_1_port
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" owner to CURRENT_USER;
 alter database "altered_database!'2" set default_transaction_read_only = true;
 set default_transaction_read_only = false;
+
+\c test_alter_db_from_nonmain_db - - :worker_2_port
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" set default_transaction_read_only from current;
 alter database "altered_database!'2" set default_transaction_read_only to DEFAULT;
 alter database "altered_database!'2" RESET default_transaction_read_only;
 alter database "altered_database!'2" SET TIME ZONE '-7';
 alter database "altered_database!'2" set TIME ZONE LOCAL;
+
+\c test_alter_db_from_nonmain_db
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" set TIME ZONE DEFAULT;
 alter database "altered_database!'2" RESET TIME ZONE;
 alter database "altered_database!'2" SET TIME ZONE INTERVAL '-08:00' HOUR TO MINUTE;
@@ -71,6 +91,7 @@ alter database "altered_database!'2" set lock_timeout from current;
 alter database "altered_database!'2" set lock_timeout to DEFAULT;
 alter database "altered_database!'2" RESET lock_timeout;
 ALTER DATABASE "altered_database!'2" RESET ALL;
+
 \c regression
 set citus.enable_create_database_propagation=on;
 drop database "altered_database!'2";
diff --git a/src/test/regress/sql/minimal_cluster_management.sql b/src/test/regress/sql/minimal_cluster_management.sql
index c17c74fdb..30f69d43d 100644
--- a/src/test/regress/sql/minimal_cluster_management.sql
+++ b/src/test/regress/sql/minimal_cluster_management.sql
@@ -41,4 +41,3 @@ DROP TABLE test_dist;
 ALTER SEQUENCE pg_catalog.pg_dist_node_nodeid_seq RESTART 30;
 ALTER SEQUENCE pg_catalog.pg_dist_groupid_seq RESTART 18;
 ALTER SEQUENCE pg_catalog.pg_dist_placement_placementid_seq RESTART 83;
-select result from run_command_on_all_nodes($$show citus.enable_ddl_propagation$$);

From aaa103015738a993c071d997bc4663850f5da891 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Tue, 26 Mar 2024 14:16:03 +0300
Subject: [PATCH 54/78] Fixes multi_follower_schedule execution

---
 src/test/regress/citus_tests/run_test.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/test/regress/citus_tests/run_test.py b/src/test/regress/citus_tests/run_test.py
index 9a648c0ab..cbaea065d 100755
--- a/src/test/regress/citus_tests/run_test.py
+++ b/src/test/regress/citus_tests/run_test.py
@@ -224,6 +224,7 @@ DEPS = {
         ],
         repeatable=False,
     ),
+    "multi_follower_sanity_check": TestDeps("multi_follower_schedule"),
 }
 
 
@@ -324,6 +325,8 @@ def run_schedule_with_multiregress(test_name, schedule, dependencies, args):
         "failure"
     ):
         make_recipe = "check-failure-custom-schedule"
+    elif dependencies.schedule == "multi_follower_schedule":
+        make_recipe = "check-follower-cluster"
     else:
         make_recipe = "check-custom-schedule"
 
@@ -360,6 +363,9 @@ def default_base_schedule(test_schedule, args):
     if "operations" in test_schedule:
         return "minimal_schedule"
 
+    if "follower" in test_schedule:
+        return "multi_follower_schedule"
+
     if "pg_upgrade" in test_schedule:
         return "minimal_pg_upgrade_schedule"
 

From 3b6236bd1c04398199f2e41ba6b7f25150656251 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Tue, 26 Mar 2024 16:30:55 +0300
Subject: [PATCH 55/78] Fixes flaky tests

---
 src/test/regress/base_schedule                  |  1 -
 src/test/regress/citus_tests/run_test.py        | 12 +++++++++++-
 src/test/regress/columnar_schedule              |  1 -
 src/test/regress/enterprise_minimal_schedule    |  1 +
 src/test/regress/enterprise_schedule            |  1 -
 .../expected/minimal_cluster_management.out     | 17 +++++++++++++++++
 .../expected/multi_cluster_management.out       | 17 +++++++++++++++++
 src/test/regress/minimal_schedule               |  1 -
 src/test/regress/multi_1_schedule               |  1 -
 src/test/regress/multi_mx_schedule              |  2 --
 src/test/regress/multi_schedule                 |  1 -
 src/test/regress/mx_base_schedule               |  1 -
 src/test/regress/operations_schedule            |  1 -
 src/test/regress/split_schedule                 |  1 -
 .../regress/sql/minimal_cluster_management.sql  |  3 +++
 .../regress/sql/multi_cluster_management.sql    |  3 +++
 16 files changed, 52 insertions(+), 12 deletions(-)

diff --git a/src/test/regress/base_schedule b/src/test/regress/base_schedule
index 0ce3ed1c0..f56adb5a3 100644
--- a/src/test/regress/base_schedule
+++ b/src/test/regress/base_schedule
@@ -4,7 +4,6 @@
 test: enable_ddl_propagation
 test: multi_test_helpers multi_test_helpers_superuser multi_create_fdw columnar_test_helpers failure_test_helpers
 test: multi_cluster_management
-test: enable_ddl_propagation
 test: multi_test_catalog_views
 test: multi_create_table
 test: multi_behavioral_analytics_create_table
diff --git a/src/test/regress/citus_tests/run_test.py b/src/test/regress/citus_tests/run_test.py
index cbaea065d..6fd9193ec 100755
--- a/src/test/regress/citus_tests/run_test.py
+++ b/src/test/regress/citus_tests/run_test.py
@@ -422,7 +422,17 @@ def find_test_schedule_and_line(test_name, args):
 
 def test_dependencies(test_name, test_schedule, schedule_line, args):
     if test_name in DEPS:
-        return DEPS[test_name]
+        # since enable_ddl_propagation is a must to execute tests
+        # below block adds enable_ddl_propagation as a dependency
+        # as the first element of extra tests if schedule is not configured
+        test_deps = DEPS[test_name]
+        ddl_propagation_test = "isolation_enable_ddl_propagation" if test_name.startswith("isolation") else "enable_ddl_propagation"
+
+        if test_deps is not None and test_deps.schedule is None:
+            test_deps.direct_extra_tests = [ddl_propagation_test] + test_deps.direct_extra_tests
+            return test_deps
+        else:
+            return DEPS[test_name]
 
     if "citus_upgrade" in test_schedule:
         return TestDeps(None, citus_upgrade_infra=True)
diff --git a/src/test/regress/columnar_schedule b/src/test/regress/columnar_schedule
index 1a6c2fce5..ef8b2b288 100644
--- a/src/test/regress/columnar_schedule
+++ b/src/test/regress/columnar_schedule
@@ -1,7 +1,6 @@
 test: enable_ddl_propagation
 test: multi_test_helpers multi_test_helpers_superuser columnar_test_helpers
 test: multi_cluster_management
-test: enable_ddl_propagation
 test: multi_test_catalog_views
 
 test: remove_coordinator_from_metadata
diff --git a/src/test/regress/enterprise_minimal_schedule b/src/test/regress/enterprise_minimal_schedule
index 32a671caf..10c4f623b 100644
--- a/src/test/regress/enterprise_minimal_schedule
+++ b/src/test/regress/enterprise_minimal_schedule
@@ -1,3 +1,4 @@
+test: enable_ddl_propagation
 test: single_node_enterprise
 test: multi_test_helpers multi_test_helpers_superuser
 test: multi_cluster_management
diff --git a/src/test/regress/enterprise_schedule b/src/test/regress/enterprise_schedule
index 4e0802d20..59156def7 100644
--- a/src/test/regress/enterprise_schedule
+++ b/src/test/regress/enterprise_schedule
@@ -5,7 +5,6 @@ test: single_node_enterprise
 
 test: multi_test_helpers multi_test_helpers_superuser
 test: multi_cluster_management
-test: enable_ddl_propagation
 test: multi_test_catalog_views
 test: multi_data_types
 test: multi_tenant_isolation
diff --git a/src/test/regress/expected/minimal_cluster_management.out b/src/test/regress/expected/minimal_cluster_management.out
index d05e83ed5..d9db708f5 100644
--- a/src/test/regress/expected/minimal_cluster_management.out
+++ b/src/test/regress/expected/minimal_cluster_management.out
@@ -69,3 +69,20 @@ DROP TABLE test_dist;
 ALTER SEQUENCE pg_catalog.pg_dist_node_nodeid_seq RESTART 30;
 ALTER SEQUENCE pg_catalog.pg_dist_groupid_seq RESTART 18;
 ALTER SEQUENCE pg_catalog.pg_dist_placement_placementid_seq RESTART 83;
+-- Enable ddl sync in all nodes
+SELECT result from  run_command_on_all_nodes('ALTER SYSTEM SET citus.enable_ddl_propagation TO ON');
+    result
+---------------------------------------------------------------------
+ ALTER SYSTEM
+ ALTER SYSTEM
+ ALTER SYSTEM
+(3 rows)
+
+SELECT result from  run_command_on_all_nodes('SELECT pg_reload_conf()');
+ result
+---------------------------------------------------------------------
+ t
+ t
+ t
+(3 rows)
+
diff --git a/src/test/regress/expected/multi_cluster_management.out b/src/test/regress/expected/multi_cluster_management.out
index 3eb549ab5..cb341f223 100644
--- a/src/test/regress/expected/multi_cluster_management.out
+++ b/src/test/regress/expected/multi_cluster_management.out
@@ -1264,3 +1264,20 @@ SELECT bool_and(hasmetadata) AND bool_and(metadatasynced) FROM pg_dist_node WHER
 -- keep permissions compatible accross versions, in regression
 -- tests.
 GRANT ALL ON SCHEMA public TO PUBLIC;
+-- Enable ddl sync in all nodes
+SELECT result from  run_command_on_all_nodes('ALTER SYSTEM SET citus.enable_ddl_propagation TO ON');
+    result
+---------------------------------------------------------------------
+ ALTER SYSTEM
+ ALTER SYSTEM
+ ALTER SYSTEM
+(3 rows)
+
+SELECT result from  run_command_on_all_nodes('SELECT pg_reload_conf()');
+ result
+---------------------------------------------------------------------
+ t
+ t
+ t
+(3 rows)
+
diff --git a/src/test/regress/minimal_schedule b/src/test/regress/minimal_schedule
index 1f524797d..fe2d9ce94 100644
--- a/src/test/regress/minimal_schedule
+++ b/src/test/regress/minimal_schedule
@@ -1,4 +1,3 @@
 test: enable_ddl_propagation
 test: minimal_cluster_management
-test: enable_ddl_propagation
 test: multi_test_helpers multi_test_helpers_superuser multi_create_fdw columnar_test_helpers multi_test_catalog_views tablespace
diff --git a/src/test/regress/multi_1_schedule b/src/test/regress/multi_1_schedule
index 394e3fea4..6fc2ca36c 100644
--- a/src/test/regress/multi_1_schedule
+++ b/src/test/regress/multi_1_schedule
@@ -22,7 +22,6 @@ test: single_node
 test: relation_access_tracking_single_node
 test: single_node_truncate
 test: multi_cluster_management
-test: enable_ddl_propagation
 
 # below tests are placed right after multi_cluster_management as we do
 # remove/add node operations and we do not want any preexisting objects
diff --git a/src/test/regress/multi_mx_schedule b/src/test/regress/multi_mx_schedule
index e332f4897..6e8b652e4 100644
--- a/src/test/regress/multi_mx_schedule
+++ b/src/test/regress/multi_mx_schedule
@@ -18,8 +18,6 @@ test: multi_extension
 test: multi_test_helpers multi_test_helpers_superuser
 test: multi_mx_node_metadata
 test: multi_cluster_management
-test: enable_ddl_propagation
-
 test: remove_coordinator_from_metadata
 test: multi_mx_function_table_reference
 test: multi_test_catalog_views
diff --git a/src/test/regress/multi_schedule b/src/test/regress/multi_schedule
index 2fd1db9f9..f24ebdeae 100644
--- a/src/test/regress/multi_schedule
+++ b/src/test/regress/multi_schedule
@@ -1,7 +1,6 @@
 test: enable_ddl_propagation
 test: multi_test_helpers multi_test_helpers_superuser
 test: multi_cluster_management
-test: enable_ddl_propagation
 test: create_role_propagation
 test: pg16
 test: multi_create_fdw
diff --git a/src/test/regress/mx_base_schedule b/src/test/regress/mx_base_schedule
index 4c7cdfca4..947986640 100644
--- a/src/test/regress/mx_base_schedule
+++ b/src/test/regress/mx_base_schedule
@@ -4,7 +4,6 @@
 test: enable_ddl_propagation
 test: multi_test_helpers multi_test_helpers_superuser
 test: multi_cluster_management
-test: enable_ddl_propagation
 test: multi_mx_function_table_reference
 test: multi_test_catalog_views
 
diff --git a/src/test/regress/operations_schedule b/src/test/regress/operations_schedule
index 86b46c7e5..0f8b5b816 100644
--- a/src/test/regress/operations_schedule
+++ b/src/test/regress/operations_schedule
@@ -1,7 +1,6 @@
 test: enable_ddl_propagation
 test: multi_test_helpers multi_test_helpers_superuser
 test: multi_cluster_management
-test: enable_ddl_propagation
 test: multi_test_catalog_views
 test: worker_copy_table_to_node
 test: shard_rebalancer_unit
diff --git a/src/test/regress/split_schedule b/src/test/regress/split_schedule
index f70b19d1a..fa445f151 100644
--- a/src/test/regress/split_schedule
+++ b/src/test/regress/split_schedule
@@ -3,7 +3,6 @@
 test: enable_ddl_propagation
 test: multi_test_helpers multi_test_helpers_superuser columnar_test_helpers
 test: multi_cluster_management
-test: enable_ddl_propagation
 test: remove_coordinator_from_metadata
 test: multi_test_catalog_views
 test: tablespace
diff --git a/src/test/regress/sql/minimal_cluster_management.sql b/src/test/regress/sql/minimal_cluster_management.sql
index 30f69d43d..80c1059a7 100644
--- a/src/test/regress/sql/minimal_cluster_management.sql
+++ b/src/test/regress/sql/minimal_cluster_management.sql
@@ -41,3 +41,6 @@ DROP TABLE test_dist;
 ALTER SEQUENCE pg_catalog.pg_dist_node_nodeid_seq RESTART 30;
 ALTER SEQUENCE pg_catalog.pg_dist_groupid_seq RESTART 18;
 ALTER SEQUENCE pg_catalog.pg_dist_placement_placementid_seq RESTART 83;
+-- Enable ddl sync in all nodes
+SELECT result from  run_command_on_all_nodes('ALTER SYSTEM SET citus.enable_ddl_propagation TO ON');
+SELECT result from  run_command_on_all_nodes('SELECT pg_reload_conf()');
diff --git a/src/test/regress/sql/multi_cluster_management.sql b/src/test/regress/sql/multi_cluster_management.sql
index 86fbd15b6..41f962f35 100644
--- a/src/test/regress/sql/multi_cluster_management.sql
+++ b/src/test/regress/sql/multi_cluster_management.sql
@@ -537,3 +537,6 @@ SELECT bool_and(hasmetadata) AND bool_and(metadatasynced) FROM pg_dist_node WHER
 -- keep permissions compatible accross versions, in regression
 -- tests.
 GRANT ALL ON SCHEMA public TO PUBLIC;
+-- Enable ddl sync in all nodes
+SELECT result from  run_command_on_all_nodes('ALTER SYSTEM SET citus.enable_ddl_propagation TO ON');
+SELECT result from  run_command_on_all_nodes('SELECT pg_reload_conf()');

From a78a77edca18b9b897f9f70bad3aaab002d1b41f Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Tue, 26 Mar 2024 16:53:08 +0300
Subject: [PATCH 56/78] Fixes indentation

---
 src/test/regress/citus_tests/run_test.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/test/regress/citus_tests/run_test.py b/src/test/regress/citus_tests/run_test.py
index 6fd9193ec..f22a097b0 100755
--- a/src/test/regress/citus_tests/run_test.py
+++ b/src/test/regress/citus_tests/run_test.py
@@ -426,10 +426,16 @@ def test_dependencies(test_name, test_schedule, schedule_line, args):
         # below block adds enable_ddl_propagation as a dependency
         # as the first element of extra tests if schedule is not configured
         test_deps = DEPS[test_name]
-        ddl_propagation_test = "isolation_enable_ddl_propagation" if test_name.startswith("isolation") else "enable_ddl_propagation"
+        ddl_propagation_test = (
+            "isolation_enable_ddl_propagation"
+            if test_name.startswith("isolation")
+            else "enable_ddl_propagation"
+        )
 
         if test_deps is not None and test_deps.schedule is None:
-            test_deps.direct_extra_tests = [ddl_propagation_test] + test_deps.direct_extra_tests
+            test_deps.direct_extra_tests = [
+                ddl_propagation_test
+            ] + test_deps.direct_extra_tests
             return test_deps
         else:
             return DEPS[test_name]

From c458ae13e8746ec9d979ae20b211bb84a7c6d88f Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Wed, 27 Mar 2024 10:28:10 +0300
Subject: [PATCH 57/78] Fixes isolation and failure lock issues

---
 src/test/regress/expected/failure_setup.out      | 16 ++++++++++++++++
 .../expected/isolation_cluster_management.out    | 16 ++++++++++++++++
 .../spec/isolation_cluster_management.spec       |  2 ++
 src/test/regress/sql/failure_setup.sql           |  3 +++
 4 files changed, 37 insertions(+)

diff --git a/src/test/regress/expected/failure_setup.out b/src/test/regress/expected/failure_setup.out
index 4cbb4b0a4..d799718ca 100644
--- a/src/test/regress/expected/failure_setup.out
+++ b/src/test/regress/expected/failure_setup.out
@@ -17,3 +17,19 @@ SELECT master_add_node('localhost', :worker_2_proxy_port);  -- an mitmproxy whic
                2
 (1 row)
 
+SELECT result from  run_command_on_all_nodes('ALTER SYSTEM SET citus.enable_ddl_propagation TO ON');
+    result
+---------------------------------------------------------------------
+ ALTER SYSTEM
+ ALTER SYSTEM
+ ALTER SYSTEM
+(3 rows)
+
+SELECT result from  run_command_on_all_nodes('SELECT pg_reload_conf()');
+ result
+---------------------------------------------------------------------
+ t
+ t
+ t
+(3 rows)
+
diff --git a/src/test/regress/expected/isolation_cluster_management.out b/src/test/regress/expected/isolation_cluster_management.out
index 63488f743..c978d10e4 100644
--- a/src/test/regress/expected/isolation_cluster_management.out
+++ b/src/test/regress/expected/isolation_cluster_management.out
@@ -4,6 +4,8 @@ starting permutation: s1a
 step s1a:
     SELECT 1 FROM master_add_node('localhost', 57637);
     SELECT 1 FROM master_add_node('localhost', 57638);
+ SELECT result from run_command_on_all_nodes('ALTER SYSTEM SET citus.enable_ddl_propagation TO ON');
+ SELECT result from run_command_on_all_nodes('SELECT pg_reload_conf()');
 
 ?column?
 ---------------------------------------------------------------------
@@ -15,3 +17,17 @@ step s1a:
        1
 (1 row)
 
+result
+---------------------------------------------------------------------
+ALTER SYSTEM
+ALTER SYSTEM
+ALTER SYSTEM
+(3 rows)
+
+result
+---------------------------------------------------------------------
+t
+t
+t
+(3 rows)
+
diff --git a/src/test/regress/spec/isolation_cluster_management.spec b/src/test/regress/spec/isolation_cluster_management.spec
index 4715a7509..29d954467 100644
--- a/src/test/regress/spec/isolation_cluster_management.spec
+++ b/src/test/regress/spec/isolation_cluster_management.spec
@@ -3,6 +3,8 @@ step "s1a"
 {
     SELECT 1 FROM master_add_node('localhost', 57637);
     SELECT 1 FROM master_add_node('localhost', 57638);
+	SELECT result from run_command_on_all_nodes('ALTER SYSTEM SET citus.enable_ddl_propagation TO ON');
+	SELECT result from run_command_on_all_nodes('SELECT pg_reload_conf()');
 }
 
 permutation "s1a"
diff --git a/src/test/regress/sql/failure_setup.sql b/src/test/regress/sql/failure_setup.sql
index 4c209f14d..9a4c21e53 100644
--- a/src/test/regress/sql/failure_setup.sql
+++ b/src/test/regress/sql/failure_setup.sql
@@ -3,3 +3,6 @@ SELECT citus.mitmproxy('conn.allow()');
 -- add the workers
 SELECT master_add_node('localhost', :worker_1_port);
 SELECT master_add_node('localhost', :worker_2_proxy_port);  -- an mitmproxy which forwards to the second worker
+
+SELECT result from  run_command_on_all_nodes('ALTER SYSTEM SET citus.enable_ddl_propagation TO ON');
+SELECT result from  run_command_on_all_nodes('SELECT pg_reload_conf()');

From 082e35687cc34095db1ed5d8bb5ede50a7730160 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Wed, 27 Mar 2024 11:03:21 +0300
Subject: [PATCH 58/78] Fixes review comments

---
 src/backend/distributed/commands/database.c                     | 2 +-
 .../distributed/commands/non_main_db_distribute_object_ops.c    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c
index 0447f98b5..c9279b661 100644
--- a/src/backend/distributed/commands/database.c
+++ b/src/backend/distributed/commands/database.c
@@ -293,7 +293,7 @@ PreprocessAlterDatabaseStmt(Node *node, const char *queryString,
 		return NIL;
 	}
 
-	/* Since ALTER TABLE SET TABLESPACE statement is not supported */
+	/* Since ALTER DATABASE SET TABLESPACE statement is not supported */
 	/* inside a transaction block, we need to send the command to the */
 	/* main database directly to make it work */
 	if (!IsSetTablespaceStatement(stmt))
diff --git a/src/backend/distributed/commands/non_main_db_distribute_object_ops.c b/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
index ce09a6fc8..d9e744069 100644
--- a/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
+++ b/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
@@ -378,7 +378,7 @@ AlterDbStmtCheckSupportedObjectType(Node *node)
 	}
 	else
 	{
-		return IsSetTablespaceStatement(stmt) && strcmp(stmt->dbname, MainDb) != 0;
+		return strcmp(stmt->dbname, MainDb) != 0;
 	}
 }
 

From 24e7389d3ec65e91a172c6a71f270c9ecec34b76 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Wed, 27 Mar 2024 13:51:23 +0300
Subject: [PATCH 59/78] Fixes review comments

---
 .../non_main_db_distribute_object_ops.c       | 55 ++++++++++---------
 1 file changed, 29 insertions(+), 26 deletions(-)

diff --git a/src/backend/distributed/commands/non_main_db_distribute_object_ops.c b/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
index d9e744069..23e8ea3ac 100644
--- a/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
+++ b/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
@@ -74,68 +74,68 @@ static bool DropDbStmtCheckSupportedObjectType(Node *node);
 static bool GrantStmtCheckSupportedObjectType(Node *node);
 static bool SecLabelStmtCheckSupportedObjectType(Node *node);
 static bool AlterDbStmtCheckSupportedObjectType(Node *node);
-static bool AlterDbCanNotBeExecutedInTransaction(Node *node);
-static bool CanNotBeExecutedInTransaction_True(Node *node);
-static bool CanNotBeExecutedInTransaction_False(Node *node);
 static bool AlterDbRenameCheckSupportedObjectType(Node *node);
 static bool AlterDbOwnerCheckSupportedObjectType(Node *node);
+static bool CannotBeExecutedInTransaction_True(Node *node);
+static bool CannotBeExecutedInTransaction_False(Node *node);
+static bool AlterDbCannotBeExecutedInTransaction(Node *node);
 
 /*
  * OperationArray that holds NonMainDbDistributeObjectOps for different command types.
  */
 static const NonMainDbDistributeObjectOps *const OperationArray[] = {
 	[T_CreateRoleStmt] = &(NonMainDbDistributeObjectOps) {
-		.cannotBeExecutedInTransaction = CanNotBeExecutedInTransaction_False,
+		.cannotBeExecutedInTransaction = CannotBeExecutedInTransaction_False,
 		.checkSupportedObjectType = NULL
 	},
 	[T_DropRoleStmt] = &(NonMainDbDistributeObjectOps) {
-		.cannotBeExecutedInTransaction = CanNotBeExecutedInTransaction_False,
+		.cannotBeExecutedInTransaction = CannotBeExecutedInTransaction_False,
 		.checkSupportedObjectType = NULL
 	},
 	[T_AlterRoleStmt] = &(NonMainDbDistributeObjectOps) {
-		.cannotBeExecutedInTransaction = CanNotBeExecutedInTransaction_False,
+		.cannotBeExecutedInTransaction = CannotBeExecutedInTransaction_False,
 		.checkSupportedObjectType = NULL
 	},
 	[T_GrantRoleStmt] = &(NonMainDbDistributeObjectOps) {
-		.cannotBeExecutedInTransaction = CanNotBeExecutedInTransaction_False,
+		.cannotBeExecutedInTransaction = CannotBeExecutedInTransaction_False,
 		.checkSupportedObjectType = NULL
 	},
 	[T_CreatedbStmt] = &(NonMainDbDistributeObjectOps) {
-		.cannotBeExecutedInTransaction = CanNotBeExecutedInTransaction_True,
+		.cannotBeExecutedInTransaction = CannotBeExecutedInTransaction_True,
 		.checkSupportedObjectType = CreateDbStmtCheckSupportedObjectType
 	},
 	[T_DropdbStmt] = &(NonMainDbDistributeObjectOps) {
-		.cannotBeExecutedInTransaction = CanNotBeExecutedInTransaction_True,
+		.cannotBeExecutedInTransaction = CannotBeExecutedInTransaction_True,
 		.checkSupportedObjectType = DropDbStmtCheckSupportedObjectType
 	},
 	[T_AlterDatabaseSetStmt] = &(NonMainDbDistributeObjectOps) {
-		.cannotBeExecutedInTransaction = CanNotBeExecutedInTransaction_False,
+		.cannotBeExecutedInTransaction = CannotBeExecutedInTransaction_False,
 		.checkSupportedObjectType = NULL
 	},
 	[T_AlterDatabaseStmt] = &(NonMainDbDistributeObjectOps) {
-		.cannotBeExecutedInTransaction = AlterDbCanNotBeExecutedInTransaction,
+		.cannotBeExecutedInTransaction = AlterDbCannotBeExecutedInTransaction,
 		.checkSupportedObjectType = AlterDbStmtCheckSupportedObjectType
 	},
 #if PG_VERSION_NUM >= PG_VERSION_15
 	[T_AlterDatabaseRefreshCollStmt] = &(NonMainDbDistributeObjectOps) {
-		.cannotBeExecutedInTransaction = CanNotBeExecutedInTransaction_False,
+		.cannotBeExecutedInTransaction = CannotBeExecutedInTransaction_False,
 		.checkSupportedObjectType = NULL
 	},
 #endif
 	[T_RenameStmt] = &(NonMainDbDistributeObjectOps) {
-		.cannotBeExecutedInTransaction = CanNotBeExecutedInTransaction_False,
+		.cannotBeExecutedInTransaction = CannotBeExecutedInTransaction_False,
 		.checkSupportedObjectType = AlterDbRenameCheckSupportedObjectType
 	},
 	[T_AlterOwnerStmt] = &(NonMainDbDistributeObjectOps) {
-		.cannotBeExecutedInTransaction = CanNotBeExecutedInTransaction_False,
+		.cannotBeExecutedInTransaction = CannotBeExecutedInTransaction_False,
 		.checkSupportedObjectType = AlterDbOwnerCheckSupportedObjectType
 	},
 	[T_GrantStmt] = &(NonMainDbDistributeObjectOps) {
-		.cannotBeExecutedInTransaction = CanNotBeExecutedInTransaction_False,
+		.cannotBeExecutedInTransaction = CannotBeExecutedInTransaction_False,
 		.checkSupportedObjectType = GrantStmtCheckSupportedObjectType
 	},
 	[T_SecLabelStmt] = &(NonMainDbDistributeObjectOps) {
-		.cannotBeExecutedInTransaction = CanNotBeExecutedInTransaction_False,
+		.cannotBeExecutedInTransaction = CannotBeExecutedInTransaction_False,
 		.checkSupportedObjectType = SecLabelStmtCheckSupportedObjectType
 	},
 };
@@ -383,14 +383,6 @@ AlterDbStmtCheckSupportedObjectType(Node *node)
 }
 
 
-static bool
-AlterDbCanNotBeExecutedInTransaction(Node *node)
-{
-	AlterDatabaseStmt *stmt = castNode(AlterDatabaseStmt, node);
-	return IsSetTablespaceStatement(stmt);
-}
-
-
 static bool
 AlterDbRenameCheckSupportedObjectType(Node *node)
 {
@@ -423,15 +415,26 @@ SecLabelStmtCheckSupportedObjectType(Node *node)
 }
 
 
+/*
+ * cannotBeExecutedInTransaction callbacks for OperationArray lie below.
+ */
 static bool
-CanNotBeExecutedInTransaction_True(Node *node)
+CannotBeExecutedInTransaction_True(Node *node)
 {
 	return true;
 }
 
 
 static bool
-CanNotBeExecutedInTransaction_False(Node *node)
+CannotBeExecutedInTransaction_False(Node *node)
 {
 	return false;
 }
+
+
+static bool
+AlterDbCannotBeExecutedInTransaction(Node *node)
+{
+	AlterDatabaseStmt *stmt = castNode(AlterDatabaseStmt, node);
+	return IsSetTablespaceStatement(stmt);
+}

From 18e3924530d442b246a00fc855a3b8048b764fc1 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Thu, 28 Mar 2024 12:49:08 +0300
Subject: [PATCH 60/78] Fixes review comments

---
 .../non_main_db_distribute_object_ops.c       | 23 +------------------
 .../isolation_enable_ddl_propagation.out      | 18 +++++++--------
 .../isolation_enable_ddl_propagation.spec     | 18 +++++++--------
 3 files changed, 18 insertions(+), 41 deletions(-)

diff --git a/src/backend/distributed/commands/non_main_db_distribute_object_ops.c b/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
index 23e8ea3ac..0d22d12fa 100644
--- a/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
+++ b/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
@@ -73,7 +73,6 @@ static bool CreateDbStmtCheckSupportedObjectType(Node *node);
 static bool DropDbStmtCheckSupportedObjectType(Node *node);
 static bool GrantStmtCheckSupportedObjectType(Node *node);
 static bool SecLabelStmtCheckSupportedObjectType(Node *node);
-static bool AlterDbStmtCheckSupportedObjectType(Node *node);
 static bool AlterDbRenameCheckSupportedObjectType(Node *node);
 static bool AlterDbOwnerCheckSupportedObjectType(Node *node);
 static bool CannotBeExecutedInTransaction_True(Node *node);
@@ -114,7 +113,7 @@ static const NonMainDbDistributeObjectOps *const OperationArray[] = {
 	},
 	[T_AlterDatabaseStmt] = &(NonMainDbDistributeObjectOps) {
 		.cannotBeExecutedInTransaction = AlterDbCannotBeExecutedInTransaction,
-		.checkSupportedObjectType = AlterDbStmtCheckSupportedObjectType
+		.checkSupportedObjectType = NULL
 	},
 #if PG_VERSION_NUM >= PG_VERSION_15
 	[T_AlterDatabaseRefreshCollStmt] = &(NonMainDbDistributeObjectOps) {
@@ -363,26 +362,6 @@ DropDbStmtCheckSupportedObjectType(Node *node)
 }
 
 
-static bool
-AlterDbStmtCheckSupportedObjectType(Node *node)
-{
-	/*
-	 * We don't try to send the query to the main database if the ALTER
-	 * DATABASE command is for the main database itself, this is a very
-	 * rare case but it's exercised by our test suite.
-	 */
-	AlterDatabaseStmt *stmt = castNode(AlterDatabaseStmt, node);
-	if (!IsSetTablespaceStatement(stmt))
-	{
-		return true;
-	}
-	else
-	{
-		return strcmp(stmt->dbname, MainDb) != 0;
-	}
-}
-
-
 static bool
 AlterDbRenameCheckSupportedObjectType(Node *node)
 {
diff --git a/src/test/regress/expected/isolation_enable_ddl_propagation.out b/src/test/regress/expected/isolation_enable_ddl_propagation.out
index d04e68f61..d5cea0b6f 100644
--- a/src/test/regress/expected/isolation_enable_ddl_propagation.out
+++ b/src/test/regress/expected/isolation_enable_ddl_propagation.out
@@ -1,16 +1,16 @@
 Parsed test spec with 1 sessions
 
-starting permutation: s1-begin s1-commit
-result
+starting permutation: s1a
+all_ok
 ---------------------------------------------------------------------
 t
-t
-t
-(3 rows)
+(1 row)
 
-step s1-begin:
- BEGIN;
+step s1a:
+ SELECT 1;
 
-step s1-commit:
- COMMIT;
+?column?
+---------------------------------------------------------------------
+       1
+(1 row)
 
diff --git a/src/test/regress/spec/isolation_enable_ddl_propagation.spec b/src/test/regress/spec/isolation_enable_ddl_propagation.spec
index e94578088..cd2b0bf12 100644
--- a/src/test/regress/spec/isolation_enable_ddl_propagation.spec
+++ b/src/test/regress/spec/isolation_enable_ddl_propagation.spec
@@ -1,18 +1,16 @@
+// The code we care about is in the setup stage, because it needs to be executed outside of a transaction.
+// For the setup stage to be executed we need at least one permutation though.
+// Therefore, we added s1a as a non-functional step, just make the setup step to work.
 setup
 {
-	SELECT result from run_command_on_all_nodes('ALTER SYSTEM SET citus.enable_ddl_propagation TO ON');
-	SELECT result from run_command_on_all_nodes('SELECT pg_reload_conf()');
+	SELECT bool_and(result='ALTER SYSTEM') AS all_ok FROM  run_command_on_all_nodes('ALTER SYSTEM SET citus.enable_ddl_propagation TO ON');
+	SELECT bool_and(result='t') AS all_ok FROM run_command_on_all_nodes('SELECT pg_reload_conf()');
 }
 
 session "s1"
-step "s1-begin"
+step "s1a"
 {
-	BEGIN;
+	SELECT 1;
 }
 
-step "s1-commit"
-{
-	COMMIT;
-}
-
-permutation "s1-begin" "s1-commit"
+permutation "s1a"

From 98111eed96790f9f2f45e1c2339a4d1977c20a21 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Thu, 28 Mar 2024 13:12:04 +0300
Subject: [PATCH 61/78] Fixes review comments

---
 .../commands/non_main_db_distribute_object_ops.c     |  5 +++--
 src/backend/distributed/commands/utility_hook.c      | 12 +++---------
 2 files changed, 6 insertions(+), 11 deletions(-)

diff --git a/src/backend/distributed/commands/non_main_db_distribute_object_ops.c b/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
index 0d22d12fa..9cbe43162 100644
--- a/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
+++ b/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
@@ -28,6 +28,7 @@
 #include "utils/builtins.h"
 
 #include "distributed/commands.h"
+#include "distributed/commands/utility_hook.h"
 #include "distributed/deparser.h"
 #include "distributed/listutils.h"
 #include "distributed/metadata_cache.h"
@@ -159,7 +160,7 @@ static void UnmarkObjectDistributedOnLocalMainDb(uint16 catalogRelId, Oid object
 bool
 RunPreprocessNonMainDBCommand(Node *parsetree)
 {
-	if (IsMainDB)
+	if (IsMainDB || !EnableDDLPropagation)
 	{
 		return false;
 	}
@@ -215,7 +216,7 @@ RunPreprocessNonMainDBCommand(Node *parsetree)
 void
 RunPostprocessNonMainDBCommand(Node *parsetree)
 {
-	if (IsMainDB || !GetNonMainDbDistributeObjectOps(parsetree))
+	if (IsMainDB || !EnableDDLPropagation || !GetNonMainDbDistributeObjectOps(parsetree))
 	{
 		return;
 	}
diff --git a/src/backend/distributed/commands/utility_hook.c b/src/backend/distributed/commands/utility_hook.c
index 00f0362d5..9426e13c0 100644
--- a/src/backend/distributed/commands/utility_hook.c
+++ b/src/backend/distributed/commands/utility_hook.c
@@ -252,12 +252,8 @@ citus_ProcessUtility(PlannedStmt *pstmt,
 		 * and if the command is a node-wide object management command that we
 		 * support from non-main databases.
 		 */
-		bool shouldSkipPrevUtilityHook = false;
 
-		if (EnableDDLPropagation)
-		{
-			shouldSkipPrevUtilityHook = RunPreprocessNonMainDBCommand(parsetree);
-		}
+		bool shouldSkipPrevUtilityHook = RunPreprocessNonMainDBCommand(parsetree);
 
 		if (!shouldSkipPrevUtilityHook)
 		{
@@ -268,10 +264,8 @@ citus_ProcessUtility(PlannedStmt *pstmt,
 			PrevProcessUtility(pstmt, queryString, false, context,
 							   params, queryEnv, dest, completionTag);
 		}
-		if (EnableDDLPropagation)
-		{
-			RunPostprocessNonMainDBCommand(parsetree);
-		}
+
+		RunPostprocessNonMainDBCommand(parsetree);
 
 		return;
 	}

From 98b600058bd1e482cd81c12b00900c2c33151169 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=BCrkan=20=C4=B0ndibay?= <gindibay@microsoft.com>
Date: Thu, 28 Mar 2024 13:14:58 +0300
Subject: [PATCH 62/78] Update
 src/test/regress/expected/minimal_cluster_management.out

Co-authored-by: Onur Tirtir <onurcantirtir@gmail.com>
---
 src/test/regress/expected/minimal_cluster_management.out | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/regress/expected/minimal_cluster_management.out b/src/test/regress/expected/minimal_cluster_management.out
index d9db708f5..2d752e12f 100644
--- a/src/test/regress/expected/minimal_cluster_management.out
+++ b/src/test/regress/expected/minimal_cluster_management.out
@@ -69,7 +69,7 @@ DROP TABLE test_dist;
 ALTER SEQUENCE pg_catalog.pg_dist_node_nodeid_seq RESTART 30;
 ALTER SEQUENCE pg_catalog.pg_dist_groupid_seq RESTART 18;
 ALTER SEQUENCE pg_catalog.pg_dist_placement_placementid_seq RESTART 83;
--- Enable ddl sync in all nodes
+-- enable ddl propagation in all nodes
 SELECT result from  run_command_on_all_nodes('ALTER SYSTEM SET citus.enable_ddl_propagation TO ON');
     result
 ---------------------------------------------------------------------

From 7a59910ea20f7c24aff7d7a079119fcbd2c79421 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=BCrkan=20=C4=B0ndibay?= <gindibay@microsoft.com>
Date: Thu, 28 Mar 2024 13:15:10 +0300
Subject: [PATCH 63/78] Update
 src/test/regress/expected/multi_cluster_management.out

Co-authored-by: Onur Tirtir <onurcantirtir@gmail.com>
---
 src/test/regress/expected/multi_cluster_management.out | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/regress/expected/multi_cluster_management.out b/src/test/regress/expected/multi_cluster_management.out
index cb341f223..10a159075 100644
--- a/src/test/regress/expected/multi_cluster_management.out
+++ b/src/test/regress/expected/multi_cluster_management.out
@@ -1264,7 +1264,7 @@ SELECT bool_and(hasmetadata) AND bool_and(metadatasynced) FROM pg_dist_node WHER
 -- keep permissions compatible accross versions, in regression
 -- tests.
 GRANT ALL ON SCHEMA public TO PUBLIC;
--- Enable ddl sync in all nodes
+-- enable ddl propagation in all nodes
 SELECT result from  run_command_on_all_nodes('ALTER SYSTEM SET citus.enable_ddl_propagation TO ON');
     result
 ---------------------------------------------------------------------

From c790b568fac431ccffd8b61ff1f89c0dd9c3e423 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Thu, 28 Mar 2024 13:16:28 +0300
Subject: [PATCH 64/78] Removes unnecessary output files

---
 .../regress/expected/multi_extension_0.out    | 1990 -----------------
 1 file changed, 1990 deletions(-)
 delete mode 100644 src/test/regress/expected/multi_extension_0.out

diff --git a/src/test/regress/expected/multi_extension_0.out b/src/test/regress/expected/multi_extension_0.out
deleted file mode 100644
index fb798c8d4..000000000
--- a/src/test/regress/expected/multi_extension_0.out
+++ /dev/null
@@ -1,1990 +0,0 @@
---
--- MULTI_EXTENSION
---
--- Tests around extension creation / upgrades
---
--- It'd be nice to script generation of this file, but alas, that's
--- not done yet.
---
--- Upgrade tests verify the objects are added in citus_finish_pg_upgrade()
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
- pg_reload_conf
----------------------------------------------------------------------
- t
-(1 row)
-
-\c - - - :worker_1_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
- pg_reload_conf
----------------------------------------------------------------------
- t
-(1 row)
-
-\c - - - :worker_2_port
-ALTER SYSTEM SET citus.enable_ddl_propagation = 'true';
-SELECT pg_reload_conf();
- pg_reload_conf
----------------------------------------------------------------------
- t
-(1 row)
-
-\c - - - :master_port
-SET citus.next_shard_id TO 580000;
-CREATE SCHEMA multi_extension;
-SELECT $definition$
-CREATE OR REPLACE FUNCTION test.maintenance_worker()
-    RETURNS pg_stat_activity
-    LANGUAGE plpgsql
-AS $$
-DECLARE
-   activity record;
-BEGIN
-    DO 'BEGIN END'; -- Force maintenance daemon to start
-    -- we don't want to wait forever; loop will exit after 20 seconds
-    FOR i IN 1 .. 200 LOOP
-        PERFORM pg_stat_clear_snapshot();
-        SELECT * INTO activity FROM pg_stat_activity
-        WHERE application_name = 'Citus Maintenance Daemon' AND datname = current_database();
-        IF activity.pid IS NOT NULL THEN
-            RETURN activity;
-        ELSE
-            PERFORM pg_sleep(0.1);
-        END IF ;
-    END LOOP;
-    -- fail if we reach the end of this loop
-    raise 'Waited too long for maintenance daemon to start';
-END;
-$$;
-$definition$ create_function_test_maintenance_worker
-\gset
-CREATE TABLE multi_extension.prev_objects(description text);
-CREATE TABLE multi_extension.extension_diff(previous_object text COLLATE "C",
-                            current_object text COLLATE "C");
-CREATE FUNCTION multi_extension.print_extension_changes()
-RETURNS TABLE(previous_object text, current_object text)
-AS $func$
-BEGIN
-    SET LOCAL search_path TO multi_extension;
-	TRUNCATE TABLE extension_diff;
-
-	CREATE TABLE current_objects AS
-	SELECT pg_catalog.pg_describe_object(classid, objid, 0)
-           || ' ' ||
-           coalesce(pg_catalog.pg_get_function_result(objid), '') AS description
-	FROM pg_catalog.pg_depend, pg_catalog.pg_extension e
-	WHERE refclassid = 'pg_catalog.pg_extension'::pg_catalog.regclass
-		AND refobjid = e.oid
-		AND deptype = 'e'
-		AND e.extname='citus';
-
-	INSERT INTO extension_diff
-	SELECT p.description previous_object, c.description current_object
-	FROM current_objects c FULL JOIN prev_objects p
-	ON p.description = c.description
-	WHERE (p.description is null OR c.description is null)
-		  AND c.description IS DISTINCT FROM 'function any_value(anyelement) anyelement'
-          AND c.description IS DISTINCT FROM 'function any_value_agg(anyelement,anyelement) anyelement';
-
-	DROP TABLE prev_objects;
-	ALTER TABLE current_objects RENAME TO prev_objects;
-
-	RETURN QUERY SELECT * FROM extension_diff ORDER BY 1, 2;
-END
-$func$ LANGUAGE plpgsql;
-CREATE SCHEMA test;
-:create_function_test_maintenance_worker
--- check maintenance daemon is started
-SELECT datname, current_database(),
-    usename, (SELECT extowner::regrole::text FROM pg_extension WHERE extname = 'citus')
-FROM test.maintenance_worker();
-  datname   | current_database | usename  | extowner
----------------------------------------------------------------------
- regression | regression       | postgres | postgres
-(1 row)
-
--- ensure no unexpected objects were created outside pg_catalog
-SELECT pgio.type, pgio.identity
-FROM pg_depend AS pgd,
-	 pg_extension AS pge,
-	 LATERAL pg_identify_object(pgd.classid, pgd.objid, pgd.objsubid) AS pgio
-WHERE pgd.refclassid = 'pg_extension'::regclass AND
-	  pgd.refobjid   = pge.oid AND
-	  pge.extname    = 'citus' AND
-	  pgio.schema    NOT IN ('pg_catalog', 'citus', 'citus_internal', 'test', 'columnar', 'columnar_internal')
-ORDER BY 1, 2;
- type |       identity
----------------------------------------------------------------------
- view | public.citus_schemas
- view | public.citus_tables
-(2 rows)
-
--- DROP EXTENSION pre-created by the regression suite
-DROP EXTENSION citus;
-DROP EXTENSION citus_columnar;
-\c
--- these tests switch between citus versions and call ddl's that require pg_dist_object to be created
-SET citus.enable_metadata_sync TO 'false';
-SET citus.enable_version_checks TO 'false';
-SET columnar.enable_version_checks TO 'false';
-CREATE EXTENSION citus VERSION '8.0-1';
-ALTER EXTENSION citus UPDATE TO '8.0-2';
-ALTER EXTENSION citus UPDATE TO '8.0-3';
-ALTER EXTENSION citus UPDATE TO '8.0-4';
-ALTER EXTENSION citus UPDATE TO '8.0-5';
-ALTER EXTENSION citus UPDATE TO '8.0-6';
-ALTER EXTENSION citus UPDATE TO '8.0-7';
-ALTER EXTENSION citus UPDATE TO '8.0-8';
-ALTER EXTENSION citus UPDATE TO '8.0-9';
-ALTER EXTENSION citus UPDATE TO '8.0-10';
-ALTER EXTENSION citus UPDATE TO '8.0-11';
-ALTER EXTENSION citus UPDATE TO '8.0-12';
-ALTER EXTENSION citus UPDATE TO '8.0-13';
-ALTER EXTENSION citus UPDATE TO '8.1-1';
-ALTER EXTENSION citus UPDATE TO '8.2-1';
-ALTER EXTENSION citus UPDATE TO '8.2-2';
-ALTER EXTENSION citus UPDATE TO '8.2-3';
-ALTER EXTENSION citus UPDATE TO '8.2-4';
-ALTER EXTENSION citus UPDATE TO '8.3-1';
-ALTER EXTENSION citus UPDATE TO '9.0-1';
-ALTER EXTENSION citus UPDATE TO '9.0-2';
-ALTER EXTENSION citus UPDATE TO '9.1-1';
-ALTER EXTENSION citus UPDATE TO '9.2-1';
-ALTER EXTENSION citus UPDATE TO '9.2-2';
--- Snapshot of state at 9.2-2
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object |                                                                                                        current_object
----------------------------------------------------------------------
-                 | event trigger citus_cascade_to_partition
-                 | function alter_role_if_exists(text,text) boolean
-                 | function array_cat_agg(anycompatiblearray) anycompatiblearray
-                 | function assign_distributed_transaction_id(integer,bigint,timestamp with time zone) void
-                 | function authinfo_valid(text) boolean
-                 | function broadcast_intermediate_result(text,text) bigint
-                 | function check_distributed_deadlocks() boolean
-                 | function citus_add_rebalance_strategy(name,regproc,regproc,regproc,real,real) void
-                 | function citus_blocking_pids(integer) integer[]
-                 | function citus_create_restore_point(text) pg_lsn
-                 | function citus_dist_stat_activity() SETOF record
-                 | function citus_drop_trigger() event_trigger
-                 | function citus_executor_name(integer) text
-                 | function citus_extradata_container(internal) void
-                 | function citus_finish_pg_upgrade() void
-                 | function citus_internal.find_groupid_for_node(text,integer) integer
-                 | function citus_internal.pg_dist_node_trigger_func() trigger
-                 | function citus_internal.pg_dist_rebalance_strategy_enterprise_check() trigger
-                 | function citus_internal.pg_dist_rebalance_strategy_trigger_func() trigger
-                 | function citus_internal.pg_dist_shard_placement_trigger_func() trigger
-                 | function citus_internal.refresh_isolation_tester_prepared_statement() void
-                 | function citus_internal.replace_isolation_tester_func() void
-                 | function citus_internal.restore_isolation_tester_func() void
-                 | function citus_isolation_test_session_is_blocked(integer,integer[]) boolean
-                 | function citus_json_concatenate(json,json) json
-                 | function citus_json_concatenate_final(json) json
-                 | function citus_jsonb_concatenate(jsonb,jsonb) jsonb
-                 | function citus_jsonb_concatenate_final(jsonb) jsonb
-                 | function citus_node_capacity_1(integer) real
-                 | function citus_prepare_pg_upgrade() void
-                 | function citus_query_stats() SETOF record
-                 | function citus_relation_size(regclass) bigint
-                 | function citus_server_id() uuid
-                 | function citus_set_default_rebalance_strategy(text) void
-                 | function citus_shard_allowed_on_node_true(bigint,integer) boolean
-                 | function citus_shard_cost_1(bigint) real
-                 | function citus_shard_cost_by_disk_size(bigint) real
-                 | function citus_stat_statements() SETOF record
-                 | function citus_stat_statements_reset() void
-                 | function citus_table_is_visible(oid) boolean
-                 | function citus_table_size(regclass) bigint
-                 | function citus_text_send_as_jsonb(text) bytea
-                 | function citus_total_relation_size(regclass) bigint
-                 | function citus_truncate_trigger() trigger
-                 | function citus_validate_rebalance_strategy_functions(regproc,regproc,regproc) void
-                 | function citus_version() text
-                 | function citus_worker_stat_activity() SETOF record
-                 | function column_name_to_column(regclass,text) text
-                 | function column_to_column_name(regclass,text) text
-                 | function coord_combine_agg(oid,cstring,anyelement) anyelement
-                 | function coord_combine_agg_ffunc(internal,oid,cstring,anyelement) anyelement
-                 | function coord_combine_agg_sfunc(internal,oid,cstring,anyelement) internal
-                 | function create_distributed_function(regprocedure,text,text) void
-                 | function create_distributed_table(regclass,text,citus.distribution_type,text) void
-                 | function create_intermediate_result(text,text) bigint
-                 | function create_reference_table(regclass) void
-                 | function distributed_tables_colocated(regclass,regclass) boolean
-                 | function dump_global_wait_edges() SETOF record
-                 | function dump_local_wait_edges() SETOF record
-                 | function fetch_intermediate_results(text[],text,integer) bigint
-                 | function get_all_active_transactions() SETOF record
-                 | function get_colocated_shard_array(bigint) bigint[]
-                 | function get_colocated_table_array(regclass) regclass[]
-                 | function get_current_transaction_id() record
-                 | function get_global_active_transactions() SETOF record
-                 | function get_rebalance_progress() TABLE(sessionid integer, table_name regclass, shardid bigint, shard_size bigint, sourcename text, sourceport integer, targetname text, targetport integer, progress bigint)
-                 | function get_rebalance_table_shards_plan(regclass,real,integer,bigint[],boolean,name) TABLE(table_name regclass, shardid bigint, shard_size bigint, sourcename text, sourceport integer, targetname text, targetport integer)
-                 | function get_shard_id_for_distribution_column(regclass,"any") bigint
-                 | function isolate_tenant_to_new_shard(regclass,"any",text) bigint
-                 | function json_cat_agg(json) json
-                 | function jsonb_cat_agg(jsonb) jsonb
-                 | function lock_relation_if_exists(text,text) boolean
-                 | function lock_shard_metadata(integer,bigint[]) void
-                 | function lock_shard_resources(integer,bigint[]) void
-                 | function mark_tables_colocated(regclass,regclass[]) void
-                 | function master_activate_node(text,integer) integer
-                 | function master_add_inactive_node(text,integer,integer,noderole,name) integer
-                 | function master_add_node(text,integer,integer,noderole,name) integer
-                 | function master_add_secondary_node(text,integer,text,integer,name) integer
-                 | function master_append_table_to_shard(bigint,text,text,integer) real
-                 | function master_apply_delete_command(text) integer
-                 | function master_conninfo_cache_invalidate() trigger
-                 | function master_copy_shard_placement(bigint,text,integer,text,integer,boolean,citus.shard_transfer_mode) void
-                 | function master_create_distributed_table(regclass,text,citus.distribution_type) void
-                 | function master_create_empty_shard(text) bigint
-                 | function master_create_worker_shards(text,integer,integer) void
-                 | function master_disable_node(text,integer) void
-                 | function master_dist_local_group_cache_invalidate() trigger
-                 | function master_dist_node_cache_invalidate() trigger
-                 | function master_dist_object_cache_invalidate() trigger
-                 | function master_dist_partition_cache_invalidate() trigger
-                 | function master_dist_placement_cache_invalidate() trigger
-                 | function master_dist_shard_cache_invalidate() trigger
-                 | function master_drain_node(text,integer,citus.shard_transfer_mode,name) void
-                 | function master_drop_all_shards(regclass,text,text) integer
-                 | function master_drop_sequences(text[]) void
-                 | function master_get_active_worker_nodes() SETOF record
-                 | function master_get_new_placementid() bigint
-                 | function master_get_new_shardid() bigint
-                 | function master_get_table_ddl_events(text) SETOF text
-                 | function master_get_table_metadata(text) record
-                 | function master_modify_multiple_shards(text) integer
-                 | function master_move_shard_placement(bigint,text,integer,text,integer,citus.shard_transfer_mode) void
-                 | function master_remove_distributed_table_metadata_from_workers(regclass,text,text) void
-                 | function master_remove_node(text,integer) void
-                 | function master_remove_partition_metadata(regclass,text,text) void
-                 | function master_run_on_worker(text[],integer[],text[],boolean) SETOF record
-                 | function master_set_node_property(text,integer,text,boolean) void
-                 | function master_unmark_object_distributed(oid,oid,integer) void
-                 | function master_update_node(integer,text,integer,boolean,integer) void
-                 | function master_update_shard_statistics(bigint) bigint
-                 | function master_update_table_statistics(regclass) void
-                 | function poolinfo_valid(text) boolean
-                 | function read_intermediate_result(text,citus_copy_format) SETOF record
-                 | function read_intermediate_results(text[],citus_copy_format) SETOF record
-                 | function rebalance_table_shards(regclass,real,integer,bigint[],citus.shard_transfer_mode,boolean,name) void
-                 | function recover_prepared_transactions() integer
-                 | function relation_is_a_known_shard(regclass) boolean
-                 | function replicate_table_shards(regclass,integer,integer,bigint[],citus.shard_transfer_mode) void
-                 | function role_exists(name) boolean
-                 | function run_command_on_colocated_placements(regclass,regclass,text,boolean) SETOF record
-                 | function run_command_on_placements(regclass,text,boolean) SETOF record
-                 | function run_command_on_shards(regclass,text,boolean) SETOF record
-                 | function run_command_on_workers(text,boolean) SETOF record
-                 | function shard_name(regclass,bigint) text
-                 | function start_metadata_sync_to_node(text,integer) void
-                 | function stop_metadata_sync_to_node(text,integer) void
-                 | function task_tracker_assign_task(bigint,integer,text) void
-                 | function task_tracker_cleanup_job(bigint) void
-                 | function task_tracker_conninfo_cache_invalidate() trigger
-                 | function task_tracker_task_status(bigint,integer) integer
-                 | function upgrade_to_reference_table(regclass) void
-                 | function worker_append_table_to_shard(text,text,text,integer) void
-                 | function worker_apply_inter_shard_ddl_command(bigint,text,bigint,text,text) void
-                 | function worker_apply_sequence_command(text) void
-                 | function worker_apply_sequence_command(text,regtype) void
-                 | function worker_apply_shard_ddl_command(bigint,text) void
-                 | function worker_apply_shard_ddl_command(bigint,text,text) void
-                 | function worker_cleanup_job_schema_cache() void
-                 | function worker_create_or_replace_object(text) boolean
-                 | function worker_create_schema(bigint,text) void
-                 | function worker_create_truncate_trigger(regclass) void
-                 | function worker_drop_distributed_table(text) void
-                 | function worker_execute_sql_task(bigint,integer,text,boolean) bigint
-                 | function worker_fetch_foreign_file(text,text,bigint,text[],integer[]) void
-                 | function worker_fetch_partition_file(bigint,integer,integer,integer,text,integer) void
-                 | function worker_hash("any") integer
-                 | function worker_hash_partition_table(bigint,integer,text,text,oid,anyarray) void
-                 | function worker_merge_files_and_run_query(bigint,integer,text,text) void
-                 | function worker_merge_files_into_table(bigint,integer,text[],text[]) void
-                 | function worker_partial_agg(oid,anyelement) cstring
-                 | function worker_partial_agg_ffunc(internal) cstring
-                 | function worker_partial_agg_sfunc(internal,oid,anyelement) internal
-                 | function worker_partition_query_result(text,text,integer,citus.distribution_type,text[],text[],boolean) SETOF record
-                 | function worker_range_partition_table(bigint,integer,text,text,oid,anyarray) void
-                 | function worker_repartition_cleanup(bigint) void
-                 | schema citus
-                 | schema citus_internal
-                 | sequence pg_dist_colocationid_seq
-                 | sequence pg_dist_groupid_seq
-                 | sequence pg_dist_node_nodeid_seq
-                 | sequence pg_dist_placement_placementid_seq
-                 | sequence pg_dist_shardid_seq
-                 | table citus.pg_dist_object
-                 | table pg_dist_authinfo
-                 | table pg_dist_colocation
-                 | table pg_dist_local_group
-                 | table pg_dist_node
-                 | table pg_dist_node_metadata
-                 | table pg_dist_partition
-                 | table pg_dist_placement
-                 | table pg_dist_poolinfo
-                 | table pg_dist_rebalance_strategy
-                 | table pg_dist_shard
-                 | table pg_dist_transaction
-                 | type citus.distribution_type
-                 | type citus.shard_transfer_mode
-                 | type citus_copy_format
-                 | type noderole
-                 | view citus_dist_stat_activity
-                 | view citus_lock_waits
-                 | view citus_shard_indexes_on_worker
-                 | view citus_shards_on_worker
-                 | view citus_stat_statements
-                 | view citus_worker_stat_activity
-                 | view pg_dist_shard_placement
-(186 rows)
-
--- Test downgrade to 9.2-2 from 9.2-4
-ALTER EXTENSION citus UPDATE TO '9.2-4';
-ALTER EXTENSION citus UPDATE TO '9.2-2';
--- Should be empty result since upgrade+downgrade should be a no-op
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
-/*
- * As we mistakenly bumped schema version to 9.3-1 in a bad release, we support
- * updating citus schema from 9.3-1 to 9.2-4, but we do not support updates to 9.3-1.
- *
- * Hence the query below should fail.
- */
-ALTER EXTENSION citus UPDATE TO '9.3-1';
-ERROR:  extension "citus" has no update path from version "9.2-2" to version "9.3-1"
-ALTER EXTENSION citus UPDATE TO '9.2-4';
--- Snapshot of state at 9.2-4
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
--- Test downgrade to 9.2-4 from 9.3-2
-ALTER EXTENSION citus UPDATE TO '9.3-2';
-ALTER EXTENSION citus UPDATE TO '9.2-4';
--- Should be empty result since upgrade+downgrade should be a no-op
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
--- Snapshot of state at 9.3-2
-ALTER EXTENSION citus UPDATE TO '9.3-2';
-SELECT * FROM multi_extension.print_extension_changes();
-                  previous_object                  |                            current_object
----------------------------------------------------------------------
- function citus_extradata_container(internal) void |
-                                                   | function citus_extradata_container(internal) SETOF record
-                                                   | function citus_remote_connection_stats() SETOF record
-                                                   | function replicate_reference_tables() void
-                                                   | function truncate_local_data_after_distributing_table(regclass) void
-                                                   | function update_distributed_table_colocation(regclass,text) void
-                                                   | function worker_create_or_alter_role(text,text,text) boolean
-(7 rows)
-
--- Test downgrade to 9.3-2 from 9.4-1
-ALTER EXTENSION citus UPDATE TO '9.4-1';
-ALTER EXTENSION citus UPDATE TO '9.3-2';
--- Should be empty result since upgrade+downgrade should be a no-op
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
--- Snapshot of state at 9.4-1
-ALTER EXTENSION citus UPDATE TO '9.4-1';
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object |                                                    current_object
----------------------------------------------------------------------
-                 | function worker_last_saved_explain_analyze() TABLE(explain_analyze_output text, execution_duration double precision)
-                 | function worker_save_query_explain_analyze(text,jsonb) SETOF record
-(2 rows)
-
--- Test upgrade paths for backported citus_pg_upgrade functions
-ALTER EXTENSION citus UPDATE TO '9.4-2';
-ALTER EXTENSION citus UPDATE TO '9.4-1';
--- Should be empty result, even though the downgrade doesn't undo the upgrade, the
--- function signature doesn't change, which is reflected here.
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
-ALTER EXTENSION citus UPDATE TO '9.4-2';
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
--- Snapshot of state at 9.4-1
-ALTER EXTENSION citus UPDATE TO '9.4-1';
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
--- Test upgrade paths for backported improvement of master_update_table_statistics function
-ALTER EXTENSION citus UPDATE TO '9.4-3';
--- should see the new source code with internal function citus_update_table_statistics
-SELECT prosrc FROM pg_proc WHERE proname = 'master_update_table_statistics' ORDER BY 1;
-            prosrc
----------------------------------------------------------------------
- citus_update_table_statistics
-(1 row)
-
-ALTER EXTENSION citus UPDATE TO '9.4-2';
--- should see the old source code
-SELECT prosrc FROM pg_proc WHERE proname = 'master_update_table_statistics' ORDER BY 1;
-                               prosrc
----------------------------------------------------------------------
-                                                                   +
- DECLARE                                                           +
-  colocated_tables regclass[];                                     +
- BEGIN                                                             +
-  SELECT get_colocated_table_array(relation) INTO colocated_tables;+
-  PERFORM                                                          +
-   master_update_shard_statistics(shardid)                         +
-  FROM                                                             +
-   pg_dist_shard                                                   +
-  WHERE                                                            +
-   logicalrelid = ANY (colocated_tables);                          +
- END;                                                              +
-
-(1 row)
-
--- Should be empty result
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
-ALTER EXTENSION citus UPDATE TO '9.4-3';
--- should see the new source code with internal function citus_update_table_statistics
-SELECT prosrc FROM pg_proc WHERE proname = 'master_update_table_statistics' ORDER BY 1;
-            prosrc
----------------------------------------------------------------------
- citus_update_table_statistics
-(1 row)
-
--- Should be empty result
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
--- Snapshot of state at 9.4-1
-ALTER EXTENSION citus UPDATE TO '9.4-1';
--- should see the old source code
-SELECT prosrc FROM pg_proc WHERE proname = 'master_update_table_statistics' ORDER BY 1;
-                               prosrc
----------------------------------------------------------------------
-                                                                   +
- DECLARE                                                           +
-  colocated_tables regclass[];                                     +
- BEGIN                                                             +
-  SELECT get_colocated_table_array(relation) INTO colocated_tables;+
-  PERFORM                                                          +
-   master_update_shard_statistics(shardid)                         +
-  FROM                                                             +
-   pg_dist_shard                                                   +
-  WHERE                                                            +
-   logicalrelid = ANY (colocated_tables);                          +
- END;                                                              +
-
-(1 row)
-
--- Should be empty result
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
--- Test downgrade to 9.4-1 from 9.5-1
-ALTER EXTENSION citus UPDATE TO '9.5-1';
--- TODO: This test should be moved to a valid downgrade testing suite where the downgrade is done, both on the schema and the binaries. Later changes in Citus made a C vs Schema discrepancy error here
--- BEGIN;
---   SET citus.enable_metadata_sync TO on;
---   SELECT master_add_node('localhost', :master_port, groupId=>0);
---   CREATE TABLE citus_local_table (a int);
---   SELECT create_citus_local_table('citus_local_table');
---   RESET citus.enable_metadata_sync;
---
---   -- downgrade from 9.5-1 to 9.4-1 should fail as we have a citus local table
---   ALTER EXTENSION citus UPDATE TO '9.4-1';
--- ROLLBACK;
--- now we can downgrade as there is no citus local table
-ALTER EXTENSION citus UPDATE TO '9.4-1';
--- Should be empty result since upgrade+downgrade should be a no-op
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
--- Snapshot of state at 9.5-1
-ALTER EXTENSION citus UPDATE TO '9.5-1';
-SELECT * FROM multi_extension.print_extension_changes();
-                             previous_object                              |                             current_object
----------------------------------------------------------------------
- function master_drop_sequences(text[]) void                              |
- function task_tracker_assign_task(bigint,integer,text) void              |
- function task_tracker_cleanup_job(bigint) void                           |
- function task_tracker_conninfo_cache_invalidate() trigger                |
- function task_tracker_task_status(bigint,integer) integer                |
- function worker_execute_sql_task(bigint,integer,text,boolean) bigint     |
- function worker_merge_files_and_run_query(bigint,integer,text,text) void |
-                                                                          | function create_citus_local_table(regclass) void
-                                                                          | function undistribute_table(regclass) void
-                                                                          | function worker_record_sequence_dependency(regclass,regclass,name) void
-(10 rows)
-
--- Test upgrade paths for backported citus_pg_upgrade functions
-ALTER EXTENSION citus UPDATE TO '9.5-2';
-ALTER EXTENSION citus UPDATE TO '9.5-1';
--- Should be empty result, even though the downgrade doesn't undo the upgrade, the
--- function signature doesn't change, which is reflected here.
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
-ALTER EXTENSION citus UPDATE TO '9.5-2';
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
--- Snapshot of state at 9.5-1
-ALTER EXTENSION citus UPDATE TO '9.5-1';
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
--- Test upgrade paths for backported improvement of master_update_table_statistics function
-ALTER EXTENSION citus UPDATE TO '9.5-3';
--- should see the new source code with internal function citus_update_table_statistics
-SELECT prosrc FROM pg_proc WHERE proname = 'master_update_table_statistics' ORDER BY 1;
-            prosrc
----------------------------------------------------------------------
- citus_update_table_statistics
-(1 row)
-
-ALTER EXTENSION citus UPDATE TO '9.5-2';
--- should see the old source code
-SELECT prosrc FROM pg_proc WHERE proname = 'master_update_table_statistics' ORDER BY 1;
-                               prosrc
----------------------------------------------------------------------
-                                                                   +
- DECLARE                                                           +
-  colocated_tables regclass[];                                     +
- BEGIN                                                             +
-  SELECT get_colocated_table_array(relation) INTO colocated_tables;+
-  PERFORM                                                          +
-   master_update_shard_statistics(shardid)                         +
-  FROM                                                             +
-   pg_dist_shard                                                   +
-  WHERE                                                            +
-   logicalrelid = ANY (colocated_tables);                          +
- END;                                                              +
-
-(1 row)
-
--- Should be empty result
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
-ALTER EXTENSION citus UPDATE TO '9.5-3';
--- should see the new source code with internal function citus_update_table_statistics
-SELECT prosrc FROM pg_proc WHERE proname = 'master_update_table_statistics' ORDER BY 1;
-            prosrc
----------------------------------------------------------------------
- citus_update_table_statistics
-(1 row)
-
--- Should be empty result
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
--- Snapshot of state at 9.5-1
-ALTER EXTENSION citus UPDATE TO '9.5-1';
--- should see the old source code
-SELECT prosrc FROM pg_proc WHERE proname = 'master_update_table_statistics' ORDER BY 1;
-                               prosrc
----------------------------------------------------------------------
-                                                                   +
- DECLARE                                                           +
-  colocated_tables regclass[];                                     +
- BEGIN                                                             +
-  SELECT get_colocated_table_array(relation) INTO colocated_tables;+
-  PERFORM                                                          +
-   master_update_shard_statistics(shardid)                         +
-  FROM                                                             +
-   pg_dist_shard                                                   +
-  WHERE                                                            +
-   logicalrelid = ANY (colocated_tables);                          +
- END;                                                              +
-
-(1 row)
-
--- Should be empty result
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
--- We removed the upgrade paths to 10.0-1, 10.0-2 and 10.0-3 due to a bug that blocked
--- upgrades to 10.0, Therefore we test upgrades to 10.0-4 instead
--- Test downgrade to 9.5-1 from 10.0-4
-ALTER EXTENSION citus UPDATE TO '10.0-4';
-ALTER EXTENSION citus UPDATE TO '9.5-1';
--- Should be empty result since upgrade+downgrade should be a no-op
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
--- Snapshot of state at 10.0-4
-ALTER EXTENSION citus UPDATE TO '10.0-4';
-SELECT * FROM multi_extension.print_extension_changes();
-                                   previous_object                                    |                                                current_object
----------------------------------------------------------------------
- function citus_total_relation_size(regclass) bigint                                  |
- function create_citus_local_table(regclass) void                                     |
- function mark_tables_colocated(regclass,regclass[]) void                             |
- function master_conninfo_cache_invalidate() trigger                                  |
- function master_create_distributed_table(regclass,text,citus.distribution_type) void |
- function master_create_worker_shards(text,integer,integer) void                      |
- function master_dist_local_group_cache_invalidate() trigger                          |
- function master_dist_node_cache_invalidate() trigger                                 |
- function master_dist_object_cache_invalidate() trigger                               |
- function master_dist_partition_cache_invalidate() trigger                            |
- function master_dist_placement_cache_invalidate() trigger                            |
- function master_dist_shard_cache_invalidate() trigger                                |
- function master_drop_all_shards(regclass,text,text) integer                          |
- function master_modify_multiple_shards(text) integer                                 |
- function undistribute_table(regclass) void                                           |
- function upgrade_to_reference_table(regclass) void                                   |
-                                                                                      | access method columnar
-                                                                                      | function alter_columnar_table_reset(regclass,boolean,boolean,boolean,boolean) void
-                                                                                      | function alter_columnar_table_set(regclass,integer,integer,name,integer) void
-                                                                                      | function alter_distributed_table(regclass,text,integer,text,boolean) void
-                                                                                      | function alter_old_partitions_set_access_method(regclass,timestamp with time zone,name)
-                                                                                      | function alter_table_set_access_method(regclass,text) void
-                                                                                      | function citus_activate_node(text,integer) integer
-                                                                                      | function citus_add_inactive_node(text,integer,integer,noderole,name) integer
-                                                                                      | function citus_add_local_table_to_metadata(regclass,boolean) void
-                                                                                      | function citus_add_node(text,integer,integer,noderole,name) integer
-                                                                                      | function citus_add_secondary_node(text,integer,text,integer,name) integer
-                                                                                      | function citus_conninfo_cache_invalidate() trigger
-                                                                                      | function citus_copy_shard_placement(bigint,text,integer,text,integer,boolean,citus.shard_transfer_mode) void
-                                                                                      | function citus_disable_node(text,integer) void
-                                                                                      | function citus_dist_local_group_cache_invalidate() trigger
-                                                                                      | function citus_dist_node_cache_invalidate() trigger
-                                                                                      | function citus_dist_object_cache_invalidate() trigger
-                                                                                      | function citus_dist_partition_cache_invalidate() trigger
-                                                                                      | function citus_dist_placement_cache_invalidate() trigger
-                                                                                      | function citus_dist_shard_cache_invalidate() trigger
-                                                                                      | function citus_drain_node(text,integer,citus.shard_transfer_mode,name) void
-                                                                                      | function citus_drop_all_shards(regclass,text,text) integer
-                                                                                      | function citus_get_active_worker_nodes() SETOF record
-                                                                                      | function citus_internal.columnar_ensure_objects_exist() void
-                                                                                      | function citus_move_shard_placement(bigint,text,integer,text,integer,citus.shard_transfer_mode) void
-                                                                                      | function citus_remove_node(text,integer) void
-                                                                                      | function citus_set_coordinator_host(text,integer,noderole,name) void
-                                                                                      | function citus_set_node_property(text,integer,text,boolean) void
-                                                                                      | function citus_shard_sizes() SETOF record
-                                                                                      | function citus_total_relation_size(regclass,boolean) bigint
-                                                                                      | function citus_unmark_object_distributed(oid,oid,integer) void
-                                                                                      | function citus_update_node(integer,text,integer,boolean,integer) void
-                                                                                      | function citus_update_shard_statistics(bigint) bigint
-                                                                                      | function citus_update_table_statistics(regclass) void
-                                                                                      | function columnar.columnar_handler(internal) table_am_handler
-                                                                                      | function fix_pre_citus10_partitioned_table_constraint_names() SETOF regclass
-                                                                                      | function fix_pre_citus10_partitioned_table_constraint_names(regclass) void
-                                                                                      | function notify_constraint_dropped() void
-                                                                                      | function remove_local_tables_from_metadata() void
-                                                                                      | function time_partition_range(regclass) record
-                                                                                      | function undistribute_table(regclass,boolean) void
-                                                                                      | function worker_change_sequence_dependency(regclass,regclass,regclass) void
-                                                                                      | function worker_fix_pre_citus10_partitioned_table_constraint_names(regclass,bigint,text) void
-                                                                                      | schema columnar
-                                                                                      | sequence columnar.storageid_seq
-                                                                                      | table columnar.chunk
-                                                                                      | table columnar.chunk_group
-                                                                                      | table columnar.options
-                                                                                      | table columnar.stripe
-                                                                                      | view citus_shards
-                                                                                      | view public.citus_tables
-                                                                                      | view time_partitions
-(68 rows)
-
--- check that we depend on the existence of public schema, and we can not drop it now
-DROP SCHEMA public;
-ERROR:  cannot drop schema public because other objects depend on it
-DETAIL:  extension citus depends on schema public
-HINT:  Use DROP ... CASCADE to drop the dependent objects too.
--- verify that citus_tables view is on pg_catalog if public schema is absent.
-ALTER EXTENSION citus UPDATE TO '9.5-1';
-DROP SCHEMA public;
-ALTER EXTENSION citus UPDATE TO '10.0-4';
-SELECT * FROM multi_extension.print_extension_changes();
-      previous_object      |   current_object
----------------------------------------------------------------------
- view public.citus_tables  |
-                           | view citus_tables
-(2 rows)
-
--- recreate public schema, and recreate citus_tables in the public schema by default
-CREATE SCHEMA public;
--- In PG15, public schema is owned by pg_database_owner role
--- Relevant PG commit: b073c3ccd06e4cb845e121387a43faa8c68a7b62
-SHOW server_version \gset
-SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15
-\gset
-\if :server_version_ge_15
-ALTER SCHEMA public OWNER TO pg_database_owner;
-\endif
-GRANT ALL ON SCHEMA public TO public;
-ALTER EXTENSION citus UPDATE TO '9.5-1';
-ALTER EXTENSION citus UPDATE TO '10.0-4';
-SELECT * FROM multi_extension.print_extension_changes();
-  previous_object   |      current_object
----------------------------------------------------------------------
- view citus_tables  |
-                    | view public.citus_tables
-(2 rows)
-
--- not print "HINT: " to hide current lib version
-\set VERBOSITY terse
-CREATE TABLE columnar_table(a INT, b INT) USING columnar;
-SET citus.enable_version_checks TO ON;
-SET columnar.enable_version_checks TO ON;
--- all should throw an error due to version mismatch
-VACUUM FULL columnar_table;
-ERROR:  loaded Citus library version differs from installed extension version
-INSERT INTO columnar_table SELECT i FROM generate_series(1, 10) i;
-ERROR:  loaded Citus library version differs from installed extension version
-VACUUM columnar_table;
-WARNING:  loaded Citus library version differs from installed extension version
-TRUNCATE columnar_table;
-ERROR:  loaded Citus library version differs from installed extension version
-DROP TABLE columnar_table;
-ERROR:  loaded Citus library version differs from installed extension version
-CREATE INDEX ON columnar_table (a);
-ERROR:  loaded Citus library version differs from installed extension version
-ALTER TABLE columnar_table SET(columnar.compression = pglz);
-ERROR:  loaded Citus library version differs from installed extension version
-ALTER TABLE columnar_table RESET (columnar.compression);
-ERROR:  loaded Citus library version differs from installed extension version
-INSERT INTO columnar_table SELECT * FROM columnar_table;
-ERROR:  loaded Citus library version differs from installed extension version
-SELECT 1 FROM columnar_table; -- columnar custom scan
-ERROR:  loaded Citus library version differs from installed extension version
-SET columnar.enable_custom_scan TO OFF;
-SELECT 1 FROM columnar_table; -- seq scan
-ERROR:  loaded Citus library version differs from installed extension version
-CREATE TABLE new_columnar_table (a int) USING columnar;
-ERROR:  loaded Citus library version differs from installed extension version
--- disable version checks for other sessions too
-ALTER SYSTEM SET citus.enable_version_checks TO OFF;
-SELECT pg_reload_conf();
- pg_reload_conf
----------------------------------------------------------------------
- t
-(1 row)
-
--- do cleanup for the rest of the tests
-SET citus.enable_version_checks TO OFF;
-SET columnar.enable_version_checks TO OFF;
-DROP TABLE columnar_table;
-RESET columnar.enable_custom_scan;
-\set VERBOSITY default
--- Test downgrade to 10.0-4 from 10.1-1
-ALTER EXTENSION citus UPDATE TO '10.1-1';
-ALTER EXTENSION citus UPDATE TO '10.0-4';
--- Should be empty result since upgrade+downgrade should be a no-op
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
--- Snapshot of state at 10.1-1
-ALTER EXTENSION citus UPDATE TO '10.1-1';
-SELECT * FROM multi_extension.print_extension_changes();
-                                                                                                        previous_object                                                                                                        |                                                                                                                          current_object
----------------------------------------------------------------------
- function citus_add_rebalance_strategy(name,regproc,regproc,regproc,real,real) void                                                                                                                                            |
- function citus_internal.columnar_ensure_objects_exist() void                                                                                                                                                                  |
- function citus_internal.pg_dist_rebalance_strategy_enterprise_check() trigger                                                                                                                                                 |
- function create_distributed_table(regclass,text,citus.distribution_type,text) void                                                                                                                                            |
- function get_rebalance_progress() TABLE(sessionid integer, table_name regclass, shardid bigint, shard_size bigint, sourcename text, sourceport integer, targetname text, targetport integer, progress bigint)                 |
- function get_rebalance_table_shards_plan(regclass,real,integer,bigint[],boolean,name) TABLE(table_name regclass, shardid bigint, shard_size bigint, sourcename text, sourceport integer, targetname text, targetport integer) |
-                                                                                                                                                                                                                               | function citus_add_rebalance_strategy(name,regproc,regproc,regproc,real,real,real) void
-                                                                                                                                                                                                                               | function citus_cleanup_orphaned_shards()
-                                                                                                                                                                                                                               | function citus_local_disk_space_stats() record
-                                                                                                                                                                                                                               | function create_distributed_table(regclass,text,citus.distribution_type,text,integer) void
-                                                                                                                                                                                                                               | function get_rebalance_progress() TABLE(sessionid integer, table_name regclass, shardid bigint, shard_size bigint, sourcename text, sourceport integer, targetname text, targetport integer, progress bigint, source_shard_size bigint, target_shard_size bigint)
-                                                                                                                                                                                                                               | function get_rebalance_table_shards_plan(regclass,real,integer,bigint[],boolean,name,real) TABLE(table_name regclass, shardid bigint, shard_size bigint, sourcename text, sourceport integer, targetname text, targetport integer)
-                                                                                                                                                                                                                               | function worker_partitioned_relation_size(regclass) bigint
-                                                                                                                                                                                                                               | function worker_partitioned_relation_total_size(regclass) bigint
-                                                                                                                                                                                                                               | function worker_partitioned_table_size(regclass) bigint
-(15 rows)
-
--- Test downgrade to 10.1-1 from 10.2-1
-ALTER EXTENSION citus UPDATE TO '10.2-1';
-ALTER EXTENSION citus UPDATE TO '10.1-1';
--- Should be empty result since upgrade+downgrade should be a no-op
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
--- Snapshot of state at 10.2-1
-ALTER EXTENSION citus UPDATE TO '10.2-1';
-SELECT * FROM multi_extension.print_extension_changes();
-                      previous_object                       |                                                                                     current_object
----------------------------------------------------------------------
- function citus_drop_all_shards(regclass,text,text) integer |
- function stop_metadata_sync_to_node(text,integer) void     |
-                                                            | function citus_drop_all_shards(regclass,text,text,boolean) integer
-                                                            | function citus_internal.downgrade_columnar_storage(regclass) void
-                                                            | function citus_internal.upgrade_columnar_storage(regclass) void
-                                                            | function citus_internal_add_partition_metadata(regclass,"char",text,integer,"char") void
-                                                            | function citus_internal_add_placement_metadata(bigint,integer,bigint,integer,bigint) void
-                                                            | function citus_internal_add_shard_metadata(regclass,bigint,"char",text,text) void
-                                                            | function citus_internal_delete_shard_metadata(bigint) void
-                                                            | function citus_internal_update_placement_metadata(bigint,integer,integer) void
-                                                            | function citus_internal_update_relation_colocation(oid,integer) void
-                                                            | function create_time_partitions(regclass,interval,timestamp with time zone,timestamp with time zone) boolean
-                                                            | function drop_old_time_partitions(regclass,timestamp with time zone)
-                                                            | function get_missing_time_partition_ranges(regclass,interval,timestamp with time zone,timestamp with time zone) TABLE(partition_name text, range_from_value text, range_to_value text)
-                                                            | function stop_metadata_sync_to_node(text,integer,boolean) void
-                                                            | function worker_nextval(regclass) integer
-(16 rows)
-
--- Test downgrade to 10.2-1 from 10.2-2
-ALTER EXTENSION citus UPDATE TO '10.2-2';
-ALTER EXTENSION citus UPDATE TO '10.2-1';
--- Should be empty result since upgrade+downgrade should be a no-op
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
--- Snapshot of state at 10.2-2
-ALTER EXTENSION citus UPDATE TO '10.2-2';
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
--- Test downgrade to 10.2-2 from 10.2-3
-ALTER EXTENSION citus UPDATE TO '10.2-3';
-ALTER EXTENSION citus UPDATE TO '10.2-2';
--- Should be empty result since upgrade+downgrade should be a no-op
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
--- Snapshot of state at 10.2-3
-ALTER EXTENSION citus UPDATE TO '10.2-3';
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
--- Test downgrade to 10.2-3 from 10.2-4
-ALTER EXTENSION citus UPDATE TO '10.2-4';
-ALTER EXTENSION citus UPDATE TO '10.2-3';
--- Make sure that we don't delete pg_depend entries added in
--- columnar--10.2-3--10.2-4.sql when downgrading to 10.2-3.
-SELECT COUNT(*)=10
-FROM pg_depend
-WHERE classid = 'pg_am'::regclass::oid AND
-      objid = (select oid from pg_am where amname = 'columnar') AND
-      objsubid = 0 AND
-      refclassid = 'pg_class'::regclass::oid AND
-      refobjsubid = 0 AND
-      deptype = 'n';
- ?column?
----------------------------------------------------------------------
- t
-(1 row)
-
--- Should be empty result since upgrade+downgrade should be a no-op
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
--- Snapshot of state at 10.2-4
-ALTER EXTENSION citus UPDATE TO '10.2-4';
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object |                              current_object
----------------------------------------------------------------------
-                 | function citus_internal.columnar_ensure_am_depends_catalog() void
-                 | function fix_all_partition_shard_index_names() SETOF regclass
-                 | function fix_partition_shard_index_names(regclass) void
-                 | function worker_fix_partition_shard_index_names(regclass,text,text) void
-(4 rows)
-
--- There was a bug when downgrading to 10.2-2 from 10.2-4
--- Test that we do not have any issues with this particular downgrade
-ALTER EXTENSION citus UPDATE TO '10.2-2';
-ALTER EXTENSION citus UPDATE TO '10.2-4';
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
--- Test downgrade to 10.2-4 from 10.2-5
-ALTER EXTENSION citus UPDATE TO '10.2-5';
-ALTER EXTENSION citus UPDATE TO '10.2-4';
--- Should be empty result since upgrade+downgrade should be a no-op
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
--- Snapshot of state at 10.2-5
-ALTER EXTENSION citus UPDATE TO '10.2-5';
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
--- Make sure that we defined dependencies from all rel objects (tables,
--- indexes, sequences ..) to columnar table access method ...
-SELECT pg_class.oid INTO columnar_schema_members
-FROM pg_class, pg_namespace
-WHERE pg_namespace.oid=pg_class.relnamespace AND
-      pg_namespace.nspname='columnar';
-SELECT refobjid INTO columnar_schema_members_pg_depend
-FROM pg_depend
-WHERE classid = 'pg_am'::regclass::oid AND
-      objid = (select oid from pg_am where amname = 'columnar') AND
-      objsubid = 0 AND
-      refclassid = 'pg_class'::regclass::oid AND
-      refobjsubid = 0 AND
-      deptype = 'n';
--- ... , so this should be empty,
-(TABLE columnar_schema_members EXCEPT TABLE columnar_schema_members_pg_depend)
-UNION
-(TABLE columnar_schema_members_pg_depend EXCEPT TABLE columnar_schema_members);
- oid
----------------------------------------------------------------------
-(0 rows)
-
--- ... , and both columnar_schema_members_pg_depend & columnar_schema_members
--- should have 10 entries.
-SELECT COUNT(*)=10 FROM columnar_schema_members_pg_depend;
- ?column?
----------------------------------------------------------------------
- t
-(1 row)
-
-DROP TABLE columnar_schema_members, columnar_schema_members_pg_depend;
--- Use a synthetic pg_dist_shard record to show that upgrade fails
--- when there are cstore_fdw tables
-INSERT INTO pg_dist_shard (logicalrelid, shardid, shardstorage) VALUES ('pg_dist_shard', 1, 'c');
-ALTER EXTENSION citus UPDATE TO '11.0-1';
-ERROR:  cstore_fdw tables are deprecated as of Citus 11.0
-HINT:  Install Citus 10.2 and convert your cstore_fdw tables to the columnar access method before upgrading further
-CONTEXT:  PL/pgSQL function inline_code_block line XX at RAISE
-DELETE FROM pg_dist_shard WHERE shardid = 1;
--- partitioned table count is tracked on Citus 11 upgrade
-CREATE TABLE e_transactions(order_id varchar(255) NULL, transaction_id int) PARTITION BY LIST(transaction_id);
-CREATE TABLE orders_2020_07_01
-PARTITION OF e_transactions FOR VALUES IN (1,2,3);
-INSERT INTO pg_dist_partition VALUES ('e_transactions'::regclass,'h', '{VAR :varno 1 :varattno 1 :vartype 1043 :vartypmod 259 :varcollid 100 :varnullingrels (b) :varlevelsup 0 :varnosyn 1 :varattnosyn 1 :location -1}', 7, 's');
-SELECT
-	(metadata->>'partitioned_citus_table_exists_pre_11')::boolean as partitioned_citus_table_exists_pre_11,
-	(metadata->>'partitioned_citus_table_exists_pre_11') IS NULL as is_null
-FROM
-	pg_dist_node_metadata;
- partitioned_citus_table_exists_pre_11 | is_null
----------------------------------------------------------------------
-                                       | t
-(1 row)
-
--- Test downgrade to 10.2-5 from 11.0-1
-ALTER EXTENSION citus UPDATE TO '11.0-1';
-SELECT
-	(metadata->>'partitioned_citus_table_exists_pre_11')::boolean as partitioned_citus_table_exists_pre_11,
-	(metadata->>'partitioned_citus_table_exists_pre_11') IS NULL as is_null
-FROM
-	pg_dist_node_metadata;
- partitioned_citus_table_exists_pre_11 | is_null
----------------------------------------------------------------------
- t                                     | f
-(1 row)
-
-DELETE FROM pg_dist_partition WHERE logicalrelid = 'e_transactions'::regclass;
-DROP TABLE e_transactions;
-ALTER EXTENSION citus UPDATE TO '10.2-5';
--- Should be empty result since upgrade+downgrade should be a no-op
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
--- Snapshot of state at 11.0-1
-ALTER EXTENSION citus UPDATE TO '11.0-1';
-SELECT * FROM multi_extension.print_extension_changes();
-                                                   previous_object                                                    |                                                            current_object
----------------------------------------------------------------------
- function citus_disable_node(text,integer) void                                                                       |
- function citus_dist_stat_activity() SETOF record                                                                     |
- function citus_worker_stat_activity() SETOF record                                                                   |
- function create_distributed_function(regprocedure,text,text) void                                                    |
- function master_append_table_to_shard(bigint,text,text,integer) real                                                 |
- function master_apply_delete_command(text) integer                                                                   |
- function master_get_table_metadata(text) record                                                                      |
- function worker_partition_query_result(text,text,integer,citus.distribution_type,text[],text[],boolean) SETOF record |
- table citus.pg_dist_object                                                                                           |
- view citus_worker_stat_activity                                                                                      |
-                                                                                                                      | function citus_backend_gpid() bigint
-                                                                                                                      | function citus_calculate_gpid(integer,integer) bigint
-                                                                                                                      | function citus_check_cluster_node_health() SETOF record
-                                                                                                                      | function citus_check_connection_to_node(text,integer) boolean
-                                                                                                                      | function citus_coordinator_nodeid() integer
-                                                                                                                      | function citus_disable_node(text,integer,boolean) void
-                                                                                                                      | function citus_finalize_upgrade_to_citus11(boolean) boolean
-                                                                                                                      | function citus_internal_add_colocation_metadata(integer,integer,integer,regtype,oid) void
-                                                                                                                      | function citus_internal_add_object_metadata(text,text[],text[],integer,integer,boolean) void
-                                                                                                                      | function citus_internal_delete_colocation_metadata(integer) void
-                                                                                                                      | function citus_internal_global_blocked_processes() SETOF record
-                                                                                                                      | function citus_internal_local_blocked_processes() SETOF record
-                                                                                                                      | function citus_nodeid_for_gpid(bigint) integer
-                                                                                                                      | function citus_nodename_for_nodeid(integer) text
-                                                                                                                      | function citus_nodeport_for_nodeid(integer) integer
-                                                                                                                      | function citus_pid_for_gpid(bigint) integer
-                                                                                                                      | function citus_run_local_command(text) void
-                                                                                                                      | function citus_shard_indexes_on_worker() SETOF record
-                                                                                                                      | function citus_shards_on_worker() SETOF record
-                                                                                                                      | function citus_stat_activity() SETOF record
-                                                                                                                      | function create_distributed_function(regprocedure,text,text,boolean) void
-                                                                                                                      | function get_nodeid_for_groupid(integer) integer
-                                                                                                                      | function pg_cancel_backend(bigint) boolean
-                                                                                                                      | function pg_terminate_backend(bigint,bigint) boolean
-                                                                                                                      | function run_command_on_all_nodes(text,boolean,boolean) SETOF record
-                                                                                                                      | function worker_create_or_replace_object(text[]) boolean
-                                                                                                                      | function worker_drop_sequence_dependency(text) void
-                                                                                                                      | function worker_drop_shell_table(text) void
-                                                                                                                      | function worker_partition_query_result(text,text,integer,citus.distribution_type,text[],text[],boolean,boolean,boolean) SETOF record
-                                                                                                                      | table pg_dist_object
-                                                                                                                      | view citus_stat_activity
-(41 rows)
-
--- Test downgrade to 11.0-1 from 11.0-2
-ALTER EXTENSION citus UPDATE TO '11.0-2';
-ALTER EXTENSION citus UPDATE TO '11.0-1';
--- Should be empty result since upgrade+downgrade should be a no-op
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
--- Snapshot of state at 11.0-2
-ALTER EXTENSION citus UPDATE TO '11.0-2';
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object |                         current_object
----------------------------------------------------------------------
-                 | function citus_finish_citus_upgrade()
-                 | function citus_is_coordinator() boolean
-                 | function run_command_on_coordinator(text,boolean) SETOF record
-                 | function start_metadata_sync_to_all_nodes() boolean
-(4 rows)
-
--- Test downgrade to 11.0-2 from 11.0-3
-ALTER EXTENSION citus UPDATE TO '11.0-3';
-ALTER EXTENSION citus UPDATE TO '11.0-2';
--- Should be empty result since upgrade+downgrade should be a no-op
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
--- Snapshot of state at 11.0-3
-ALTER EXTENSION citus UPDATE TO '11.0-3';
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
--- Test downgrade to 11.0-3 from 11.0-4
-ALTER EXTENSION citus UPDATE TO '11.0-4';
-ALTER EXTENSION citus UPDATE TO '11.0-3';
--- Should be empty result since upgrade+downgrade should be a no-op
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
--- Snapshot of state at 11.0-4
-ALTER EXTENSION citus UPDATE TO '11.0-4';
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
--- Test downgrade to 11.0-4 from 11.1-1
-ALTER EXTENSION citus UPDATE TO '11.1-1';
-ALTER EXTENSION citus UPDATE TO '11.0-4';
--- Should be empty result since upgrade+downgrade should be a no-op
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
--- Test CREATE EXTENSION when Citus already exists but Citus_Columnar does not. Should skip
-CREATE EXTENSION IF NOT EXISTS citus;
-NOTICE:  extension "citus" already exists, skipping
-CREATE EXTENSION citus;
-ERROR:  extension "citus" already exists
--- Snapshot of state at 11.1-1
-ALTER EXTENSION citus UPDATE TO '11.1-1';
-SELECT * FROM multi_extension.print_extension_changes();
-                                                                                                                          previous_object                                                                                                                          |                                                                                                                                     current_object
----------------------------------------------------------------------
- access method columnar                                                                                                                                                                                                                                            |
- function alter_columnar_table_reset(regclass,boolean,boolean,boolean,boolean) void                                                                                                                                                                                |
- function alter_columnar_table_set(regclass,integer,integer,name,integer) void                                                                                                                                                                                     |
- function citus_copy_shard_placement(bigint,text,integer,text,integer,boolean,citus.shard_transfer_mode) void                                                                                                                                                      |
- function citus_internal.columnar_ensure_am_depends_catalog() void                                                                                                                                                                                                 |
- function citus_internal.downgrade_columnar_storage(regclass) void                                                                                                                                                                                                 |
- function citus_internal.upgrade_columnar_storage(regclass) void                                                                                                                                                                                                   |
- function columnar.columnar_handler(internal) table_am_handler                                                                                                                                                                                                     |
- function get_rebalance_progress() TABLE(sessionid integer, table_name regclass, shardid bigint, shard_size bigint, sourcename text, sourceport integer, targetname text, targetport integer, progress bigint, source_shard_size bigint, target_shard_size bigint) |
- function isolate_tenant_to_new_shard(regclass,"any",text) bigint                                                                                                                                                                                                  |
- function replicate_reference_tables() void                                                                                                                                                                                                                        |
- function worker_cleanup_job_schema_cache() void                                                                                                                                                                                                                   |
- function worker_create_schema(bigint,text) void                                                                                                                                                                                                                   |
- function worker_fetch_foreign_file(text,text,bigint,text[],integer[]) void                                                                                                                                                                                        |
- function worker_fetch_partition_file(bigint,integer,integer,integer,text,integer) void                                                                                                                                                                            |
- function worker_hash_partition_table(bigint,integer,text,text,oid,anyarray) void                                                                                                                                                                                  |
- function worker_merge_files_into_table(bigint,integer,text[],text[]) void                                                                                                                                                                                         |
- function worker_range_partition_table(bigint,integer,text,text,oid,anyarray) void                                                                                                                                                                                 |
- function worker_repartition_cleanup(bigint) void                                                                                                                                                                                                                  |
- schema columnar                                                                                                                                                                                                                                                   |
- sequence columnar.storageid_seq                                                                                                                                                                                                                                   |
- table columnar.chunk                                                                                                                                                                                                                                              |
- table columnar.chunk_group                                                                                                                                                                                                                                        |
- table columnar.options                                                                                                                                                                                                                                            |
- table columnar.stripe                                                                                                                                                                                                                                             |
-                                                                                                                                                                                                                                                                   | function citus_cleanup_orphaned_resources()
-                                                                                                                                                                                                                                                                   | function citus_copy_shard_placement(bigint,text,integer,text,integer,citus.shard_transfer_mode) void
-                                                                                                                                                                                                                                                                   | function citus_internal_delete_partition_metadata(regclass) void
-                                                                                                                                                                                                                                                                   | function citus_job_cancel(bigint) void
-                                                                                                                                                                                                                                                                   | function citus_job_wait(bigint,citus_job_status) void
-                                                                                                                                                                                                                                                                   | function citus_locks() SETOF record
-                                                                                                                                                                                                                                                                   | function citus_rebalance_start(name,boolean,citus.shard_transfer_mode) bigint
-                                                                                                                                                                                                                                                                   | function citus_rebalance_stop() void
-                                                                                                                                                                                                                                                                   | function citus_rebalance_wait() void
-                                                                                                                                                                                                                                                                   | function citus_split_shard_by_split_points(bigint,text[],integer[],citus.shard_transfer_mode) void
-                                                                                                                                                                                                                                                                   | function create_distributed_table_concurrently(regclass,text,citus.distribution_type,text,integer) void
-                                                                                                                                                                                                                                                                   | function get_rebalance_progress() TABLE(sessionid integer, table_name regclass, shardid bigint, shard_size bigint, sourcename text, sourceport integer, targetname text, targetport integer, progress bigint, source_shard_size bigint, target_shard_size bigint, operation_type text)
-                                                                                                                                                                                                                                                                   | function isolate_tenant_to_new_shard(regclass,"any",text,citus.shard_transfer_mode) bigint
-                                                                                                                                                                                                                                                                   | function replicate_reference_tables(citus.shard_transfer_mode) void
-                                                                                                                                                                                                                                                                   | function worker_copy_table_to_node(regclass,integer) void
-                                                                                                                                                                                                                                                                   | function worker_split_copy(bigint,text,split_copy_info[]) void
-                                                                                                                                                                                                                                                                   | function worker_split_shard_release_dsm() void
-                                                                                                                                                                                                                                                                   | function worker_split_shard_replication_setup(split_shard_info[]) SETOF replication_slot_info
-                                                                                                                                                                                                                                                                   | sequence pg_dist_background_job_job_id_seq
-                                                                                                                                                                                                                                                                   | sequence pg_dist_background_task_task_id_seq
-                                                                                                                                                                                                                                                                   | sequence pg_dist_cleanup_recordid_seq
-                                                                                                                                                                                                                                                                   | sequence pg_dist_operationid_seq
-                                                                                                                                                                                                                                                                   | table pg_dist_background_job
-                                                                                                                                                                                                                                                                   | table pg_dist_background_task
-                                                                                                                                                                                                                                                                   | table pg_dist_background_task_depend
-                                                                                                                                                                                                                                                                   | table pg_dist_cleanup
-                                                                                                                                                                                                                                                                   | type citus_job_status
-                                                                                                                                                                                                                                                                   | type citus_task_status
-                                                                                                                                                                                                                                                                   | type replication_slot_info
-                                                                                                                                                                                                                                                                   | type split_copy_info
-                                                                                                                                                                                                                                                                   | type split_shard_info
-                                                                                                                                                                                                                                                                   | view citus_locks
-(57 rows)
-
--- Test downgrade to 11.1-1 from 11.2-1
-ALTER EXTENSION citus UPDATE TO '11.2-1';
-ALTER EXTENSION citus UPDATE TO '11.1-1';
--- Should be empty result since upgrade+downgrade should be a no-op
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
--- Snapshot of state at 11.2-1
-ALTER EXTENSION citus UPDATE TO '11.2-1';
-ALTER EXTENSION citus_columnar UPDATE TO '11.2-1';
--- Make sure that we defined dependencies from all rel objects (tables,
--- indexes, sequences ..) to columnar table access method ...
-SELECT pg_class.oid INTO columnar_schema_members
-FROM pg_class, pg_namespace
-WHERE pg_namespace.oid=pg_class.relnamespace AND
-      pg_namespace.nspname='columnar_internal' AND
-      pg_class.relname NOT IN ('chunk_group_pkey',
-                               'chunk_pkey',
-                               'options_pkey',
-                               'stripe_first_row_number_idx',
-                               'stripe_pkey');
-SELECT refobjid INTO columnar_schema_members_pg_depend
-FROM pg_depend
-WHERE classid = 'pg_am'::regclass::oid AND
-      objid = (select oid from pg_am where amname = 'columnar') AND
-      objsubid = 0 AND
-      refclassid = 'pg_class'::regclass::oid AND
-      refobjsubid = 0 AND
-      deptype = 'n';
--- ... , so this should be empty,
-(TABLE columnar_schema_members EXCEPT TABLE columnar_schema_members_pg_depend)
-UNION
-(TABLE columnar_schema_members_pg_depend EXCEPT TABLE columnar_schema_members);
- oid
----------------------------------------------------------------------
-(0 rows)
-
--- ... , and both columnar_schema_members_pg_depend & columnar_schema_members
--- should have 5 entries.
-SELECT COUNT(*)=5 FROM columnar_schema_members_pg_depend;
- ?column?
----------------------------------------------------------------------
- t
-(1 row)
-
-DROP TABLE columnar_schema_members, columnar_schema_members_pg_depend;
-SELECT * FROM multi_extension.print_extension_changes();
-                                                                                                                                    previous_object                                                                                                                                     |                                                                                                                                                              current_object
----------------------------------------------------------------------
- function get_rebalance_progress() TABLE(sessionid integer, table_name regclass, shardid bigint, shard_size bigint, sourcename text, sourceport integer, targetname text, targetport integer, progress bigint, source_shard_size bigint, target_shard_size bigint, operation_type text) |
- function worker_append_table_to_shard(text,text,text,integer) void                                                                                                                                                                                                                     |
- function worker_split_shard_replication_setup(split_shard_info[]) SETOF replication_slot_info                                                                                                                                                                                          |
-                                                                                                                                                                                                                                                                                        | function citus_copy_shard_placement(bigint,integer,integer,citus.shard_transfer_mode) void
-                                                                                                                                                                                                                                                                                        | function citus_get_node_clock() cluster_clock
-                                                                                                                                                                                                                                                                                        | function citus_get_transaction_clock() cluster_clock
-                                                                                                                                                                                                                                                                                        | function citus_internal_add_placement_metadata(bigint,bigint,integer,bigint) void
-                                                                                                                                                                                                                                                                                        | function citus_internal_adjust_local_clock_to_remote(cluster_clock) void
-                                                                                                                                                                                                                                                                                        | function citus_is_clock_after(cluster_clock,cluster_clock) boolean
-                                                                                                                                                                                                                                                                                        | function citus_job_list() TABLE(job_id bigint, state citus_job_status, job_type name, description text, started_at timestamp with time zone, finished_at timestamp with time zone)
-                                                                                                                                                                                                                                                                                        | function citus_job_status(bigint,boolean) TABLE(job_id bigint, state citus_job_status, job_type name, description text, started_at timestamp with time zone, finished_at timestamp with time zone, details jsonb)
-                                                                                                                                                                                                                                                                                        | function citus_move_shard_placement(bigint,integer,integer,citus.shard_transfer_mode) void
-                                                                                                                                                                                                                                                                                        | function citus_rebalance_status(boolean) TABLE(job_id bigint, state citus_job_status, job_type name, description text, started_at timestamp with time zone, finished_at timestamp with time zone, details jsonb)
-                                                                                                                                                                                                                                                                                        | function citus_task_wait(bigint,citus_task_status) void
-                                                                                                                                                                                                                                                                                        | function cluster_clock_cmp(cluster_clock,cluster_clock) integer
-                                                                                                                                                                                                                                                                                        | function cluster_clock_eq(cluster_clock,cluster_clock) boolean
-                                                                                                                                                                                                                                                                                        | function cluster_clock_ge(cluster_clock,cluster_clock) boolean
-                                                                                                                                                                                                                                                                                        | function cluster_clock_gt(cluster_clock,cluster_clock) boolean
-                                                                                                                                                                                                                                                                                        | function cluster_clock_in(cstring) cluster_clock
-                                                                                                                                                                                                                                                                                        | function cluster_clock_le(cluster_clock,cluster_clock) boolean
-                                                                                                                                                                                                                                                                                        | function cluster_clock_logical(cluster_clock) bigint
-                                                                                                                                                                                                                                                                                        | function cluster_clock_lt(cluster_clock,cluster_clock) boolean
-                                                                                                                                                                                                                                                                                        | function cluster_clock_ne(cluster_clock,cluster_clock) boolean
-                                                                                                                                                                                                                                                                                        | function cluster_clock_out(cluster_clock) cstring
-                                                                                                                                                                                                                                                                                        | function cluster_clock_recv(internal) cluster_clock
-                                                                                                                                                                                                                                                                                        | function cluster_clock_send(cluster_clock) bytea
-                                                                                                                                                                                                                                                                                        | function get_rebalance_progress() TABLE(sessionid integer, table_name regclass, shardid bigint, shard_size bigint, sourcename text, sourceport integer, targetname text, targetport integer, progress bigint, source_shard_size bigint, target_shard_size bigint, operation_type text, source_lsn pg_lsn, target_lsn pg_lsn, status text)
-                                                                                                                                                                                                                                                                                        | function worker_split_shard_replication_setup(split_shard_info[],bigint) SETOF replication_slot_info
-                                                                                                                                                                                                                                                                                        | operator <(cluster_clock,cluster_clock)
-                                                                                                                                                                                                                                                                                        | operator <=(cluster_clock,cluster_clock)
-                                                                                                                                                                                                                                                                                        | operator <>(cluster_clock,cluster_clock)
-                                                                                                                                                                                                                                                                                        | operator =(cluster_clock,cluster_clock)
-                                                                                                                                                                                                                                                                                        | operator >(cluster_clock,cluster_clock)
-                                                                                                                                                                                                                                                                                        | operator >=(cluster_clock,cluster_clock)
-                                                                                                                                                                                                                                                                                        | operator class cluster_clock_ops for access method btree
-                                                                                                                                                                                                                                                                                        | operator family cluster_clock_ops for access method btree
-                                                                                                                                                                                                                                                                                        | sequence pg_dist_clock_logical_seq
-                                                                                                                                                                                                                                                                                        | type cluster_clock
-(38 rows)
-
--- Test downgrade to 11.2-1 from 11.2-2
-ALTER EXTENSION citus UPDATE TO '11.2-2';
-ALTER EXTENSION citus UPDATE TO '11.2-1';
--- Should be empty result since upgrade+downgrade should be a no-op
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
--- Snapshot of state at 11.2-2
-ALTER EXTENSION citus UPDATE TO '11.2-2';
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object |                          current_object
----------------------------------------------------------------------
-                 | function worker_adjust_identity_column_seq_ranges(regclass) void
-(1 row)
-
--- Test downgrade to 11.2-2 from 11.3-1
-ALTER EXTENSION citus UPDATE TO '11.3-1';
-ALTER EXTENSION citus UPDATE TO '11.2-2';
--- Should be empty result since upgrade+downgrade should be a no-op
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
--- Snapshot of state at 11.3-1
-ALTER EXTENSION citus UPDATE TO '11.3-1';
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object |                             current_object
----------------------------------------------------------------------
-                 | function citus_internal_is_replication_origin_tracking_active() boolean
-                 | function citus_internal_mark_node_not_synced(integer,integer) void
-                 | function citus_internal_start_replication_origin_tracking() void
-                 | function citus_internal_stop_replication_origin_tracking() void
-                 | function citus_stat_tenants(boolean) SETOF record
-                 | function citus_stat_tenants_local(boolean) SETOF record
-                 | function citus_stat_tenants_local_reset() void
-                 | function citus_stat_tenants_reset() void
-                 | function worker_drop_all_shell_tables(boolean)
-                 | view citus_stat_tenants
-                 | view citus_stat_tenants_local
-(11 rows)
-
--- Test downgrade to 11.3-1 from 11.3-2
-ALTER EXTENSION citus UPDATE TO '11.3-2';
-ALTER EXTENSION citus UPDATE TO '11.3-1';
--- Should be empty result since upgrade+downgrade should be a no-op
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
--- Snapshot of state at 11.3-2
-ALTER EXTENSION citus UPDATE TO '11.3-2';
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
--- Test downgrade to 11.3-2 from 12.0-1
-ALTER EXTENSION citus UPDATE TO '12.0-1';
-CREATE TABLE null_shard_key (x int, y int);
-SET citus.shard_replication_factor TO 1;
-SELECT create_distributed_table('null_shard_key', null);
- create_distributed_table
----------------------------------------------------------------------
-
-(1 row)
-
--- Show that we cannot downgrade to 11.3-2 becuase the cluster has a
--- distributed table with single-shard.
-ALTER EXTENSION citus UPDATE TO '11.3-2';
-ERROR:  cannot downgrade Citus because there are distributed tables without a shard key.
-DETAIL:  To downgrade Citus to an older version, you should first convert those tables to Postgres tables by executing SELECT undistribute_table("%s").
-HINT:  You can find the distributed tables without a shard key in the cluster by using the following query: "SELECT * FROM citus_tables WHERE distribution_column = '<none>' AND colocation_id > 0".
-CONTEXT:  PL/pgSQL function inline_code_block line XX at RAISE
-DROP TABLE null_shard_key;
-ALTER EXTENSION citus UPDATE TO '11.3-2';
--- Should be empty result since upgrade+downgrade should be a no-op
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
--- Snapshot of state at 12.0-1
-ALTER EXTENSION citus UPDATE TO '12.0-1';
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object |                              current_object
----------------------------------------------------------------------
-                 | function citus_internal_add_tenant_schema(oid,integer) void
-                 | function citus_internal_delete_tenant_schema(oid) void
-                 | function citus_internal_unregister_tenant_schema_globally(oid,text) void
-                 | function citus_schema_distribute(regnamespace) void
-                 | function citus_schema_undistribute(regnamespace) void
-                 | function citus_stat_tenants_local_internal(boolean) SETOF record
-                 | table pg_dist_schema
-                 | view public.citus_schemas
-(8 rows)
-
--- Test downgrade to 12.0-1 from 12.1-1
-ALTER EXTENSION citus UPDATE TO '12.1-1';
-ALTER EXTENSION citus UPDATE TO '12.0-1';
--- Should be empty result since upgrade+downgrade should be a no-op
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
--- Snapshot of state at 12.1-1
-ALTER EXTENSION citus UPDATE TO '12.1-1';
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object |                                     current_object
----------------------------------------------------------------------
-                 | function citus_internal_delete_placement_metadata(bigint) void
-                 | function citus_internal_update_none_dist_table_metadata(oid,"char",bigint,boolean) void
-                 | function citus_pause_node_within_txn(integer,boolean,integer) void
-                 | function citus_schema_move(regnamespace,integer,citus.shard_transfer_mode) void
-                 | function citus_schema_move(regnamespace,text,integer,citus.shard_transfer_mode) void
-(5 rows)
-
--- Test downgrade to 12.1-1 from 12.2-1
-ALTER EXTENSION citus UPDATE TO '12.2-1';
-ALTER EXTENSION citus UPDATE TO '12.1-1';
--- Should be empty result since upgrade+downgrade should be a no-op
-SELECT * FROM multi_extension.print_extension_changes();
- previous_object | current_object
----------------------------------------------------------------------
-(0 rows)
-
--- Snapshot of state at 12.2-1
-ALTER EXTENSION citus UPDATE TO '12.2-1';
-SELECT * FROM multi_extension.print_extension_changes();
-                        previous_object                         |                                        current_object
----------------------------------------------------------------------
- function citus_unmark_object_distributed(oid,oid,integer) void |
-                                                                | function citus_internal.acquire_citus_advisory_object_class_lock(integer,cstring) void
-                                                                | function citus_internal.add_colocation_metadata(integer,integer,integer,regtype,oid) void
-                                                                | function citus_internal.add_object_metadata(text,text[],text[],integer,integer,boolean) void
-                                                                | function citus_internal.add_partition_metadata(regclass,"char",text,integer,"char") void
-                                                                | function citus_internal.add_placement_metadata(bigint,bigint,integer,bigint) void
-                                                                | function citus_internal.add_shard_metadata(regclass,bigint,"char",text,text) void
-                                                                | function citus_internal.add_tenant_schema(oid,integer) void
-                                                                | function citus_internal.adjust_local_clock_to_remote(cluster_clock) void
-                                                                | function citus_internal.commit_management_command_2pc() void
-                                                                | function citus_internal.database_command(text) void
-                                                                | function citus_internal.delete_colocation_metadata(integer) void
-                                                                | function citus_internal.delete_partition_metadata(regclass) void
-                                                                | function citus_internal.delete_placement_metadata(bigint) void
-                                                                | function citus_internal.delete_shard_metadata(bigint) void
-                                                                | function citus_internal.delete_tenant_schema(oid) void
-                                                                | function citus_internal.execute_command_on_remote_nodes_as_user(text,text) void
-                                                                | function citus_internal.global_blocked_processes() SETOF record
-                                                                | function citus_internal.is_replication_origin_tracking_active() boolean
-                                                                | function citus_internal.local_blocked_processes() SETOF record
-                                                                | function citus_internal.mark_node_not_synced(integer,integer) void
-                                                                | function citus_internal.mark_object_distributed(oid,text,oid,text) void
-                                                                | function citus_internal.start_management_transaction(xid8) void
-                                                                | function citus_internal.start_replication_origin_tracking() void
-                                                                | function citus_internal.stop_replication_origin_tracking() void
-                                                                | function citus_internal.unregister_tenant_schema_globally(oid,text) void
-                                                                | function citus_internal.update_none_dist_table_metadata(oid,"char",bigint,boolean) void
-                                                                | function citus_internal.update_placement_metadata(bigint,integer,integer) void
-                                                                | function citus_internal.update_relation_colocation(oid,integer) void
-                                                                | function citus_unmark_object_distributed(oid,oid,integer,boolean) void
-(30 rows)
-
-DROP TABLE multi_extension.prev_objects, multi_extension.extension_diff;
--- show running version
-SHOW citus.version;
- citus.version
----------------------------------------------------------------------
- 12.2devel
-(1 row)
-
--- ensure no unexpected objects were created outside pg_catalog
-SELECT pgio.type, pgio.identity
-FROM pg_depend AS pgd,
-	 pg_extension AS pge,
-	 LATERAL pg_identify_object(pgd.classid, pgd.objid, pgd.objsubid) AS pgio
-WHERE pgd.refclassid = 'pg_extension'::regclass AND
-	  pgd.refobjid   = pge.oid AND
-	  pge.extname    = 'citus' AND
-	  pgio.schema    NOT IN ('pg_catalog', 'citus', 'citus_internal', 'test', 'columnar', 'columnar_internal')
-ORDER BY 1, 2;
- type |       identity
----------------------------------------------------------------------
- view | public.citus_schemas
- view | public.citus_tables
-(2 rows)
-
--- see incompatible version errors out
-RESET citus.enable_version_checks;
-RESET columnar.enable_version_checks;
--- reset version check config for other sessions too
-ALTER SYSTEM RESET citus.enable_version_checks;
-SELECT pg_reload_conf();
- pg_reload_conf
----------------------------------------------------------------------
- t
-(1 row)
-
-DROP EXTENSION citus;
-DROP EXTENSION citus_columnar;
-CREATE EXTENSION citus VERSION '8.0-1';
-ERROR:  specified version incompatible with loaded Citus library
-DETAIL:  Loaded library requires 12.2, but 8.0-1 was specified.
-HINT:  If a newer library is present, restart the database and try the command again.
--- Test non-distributed queries work even in version mismatch
-SET citus.enable_version_checks TO 'false';
-SET columnar.enable_version_checks TO 'false';
-CREATE EXTENSION citus VERSION '8.1-1';
-SET citus.enable_version_checks TO 'true';
-SET columnar.enable_version_checks TO 'true';
--- Test CREATE TABLE
-CREATE TABLE version_mismatch_table(column1 int);
--- Test COPY
-\copy version_mismatch_table FROM STDIN;
--- Test INSERT
-INSERT INTO version_mismatch_table(column1) VALUES(5);
--- Test SELECT
-SELECT * FROM version_mismatch_table ORDER BY column1;
- column1
----------------------------------------------------------------------
-       0
-       1
-       2
-       3
-       4
-       5
-(6 rows)
-
--- Test SELECT from pg_catalog
-SELECT d.datname as "Name",
-       pg_catalog.pg_get_userbyid(d.datdba) as "Owner",
-       pg_catalog.array_to_string(d.datacl, E'\n') AS "Access privileges"
-FROM pg_catalog.pg_database d
-ORDER BY 1;
-    Name    |  Owner   |   Access privileges
----------------------------------------------------------------------
- postgres   | postgres |
- regression | postgres |
- template0  | postgres | =c/postgres          +
-            |          | postgres=CTc/postgres
- template1  | postgres | =c/postgres          +
-            |          | postgres=CTc/postgres
-(4 rows)
-
--- We should not distribute table in version mistmatch
-SELECT create_distributed_table('version_mismatch_table', 'column1');
-ERROR:  loaded Citus library version differs from installed extension version
-DETAIL:  Loaded library requires 12.2, but the installed extension version is 8.1-1.
-HINT:  Run ALTER EXTENSION citus UPDATE and try again.
--- This function will cause fail in next ALTER EXTENSION
-CREATE OR REPLACE FUNCTION pg_catalog.relation_is_a_known_shard(regclass)
-RETURNS void LANGUAGE plpgsql
-AS $function$
-BEGIN
-END;
-$function$;
-ERROR:  cannot change return type of existing function
-HINT:  Use DROP FUNCTION relation_is_a_known_shard(regclass) first.
-SET citus.enable_version_checks TO 'false';
-SET columnar.enable_version_checks TO 'false';
--- This will fail because of previous function declaration
-ALTER EXTENSION citus UPDATE TO '9.1-1';
--- We can DROP problematic function and continue ALTER EXTENSION even when version checks are on
-SET citus.enable_version_checks TO 'true';
-SET columnar.enable_version_checks TO 'true';
-DROP FUNCTION pg_catalog.relation_is_a_known_shard(regclass);
-ERROR:  cannot drop function relation_is_a_known_shard(regclass) because extension citus requires it
-HINT:  You can drop extension citus instead.
-SET citus.enable_version_checks TO 'false';
-SET columnar.enable_version_checks TO 'false';
-ALTER EXTENSION citus UPDATE TO '9.1-1';
-NOTICE:  version "9.1-1" of extension "citus" is already installed
--- Test updating to the latest version without specifying the version number
-ALTER EXTENSION citus UPDATE;
--- re-create in newest version
-DROP EXTENSION citus;
-DROP EXTENSION citus_columnar;
-\c
-CREATE EXTENSION citus;
--- test cache invalidation in workers
-\c - - - :worker_1_port
-DROP EXTENSION citus;
-DROP EXTENSION citus_columnar;
-SET citus.enable_version_checks TO 'false';
-SET columnar.enable_version_checks TO 'false';
-CREATE EXTENSION citus VERSION '8.0-1';
-SET citus.enable_version_checks TO 'true';
-SET columnar.enable_version_checks TO 'true';
--- during ALTER EXTENSION, we should invalidate the cache
-ALTER EXTENSION citus UPDATE;
--- if cache is invalidated succesfull, this \d should work without any problem
-\d
-            List of relations
- Schema |     Name      | Type |  Owner
----------------------------------------------------------------------
- public | citus_schemas | view | postgres
- public | citus_tables  | view | postgres
-(2 rows)
-
-\c - - - :master_port
--- test https://github.com/citusdata/citus/issues/3409
-CREATE USER testuser2 SUPERUSER;
-SET ROLE testuser2;
-DROP EXTENSION Citus;
--- Loop until we see there's no maintenance daemon running
-DO $$begin
-    for i in 0 .. 100 loop
-        if i = 100 then raise 'Waited too long'; end if;
-        PERFORM pg_stat_clear_snapshot();
-        perform * from pg_stat_activity where application_name = 'Citus Maintenance Daemon';
-        if not found then exit; end if;
-        perform pg_sleep(0.1);
-    end loop;
-end$$;
-SELECT datid, datname, usename FROM pg_stat_activity WHERE application_name = 'Citus Maintenance Daemon';
- datid | datname | usename
----------------------------------------------------------------------
-(0 rows)
-
-CREATE EXTENSION Citus;
--- Loop until we there's a maintenance daemon running
-DO $$begin
-    for i in 0 .. 100 loop
-        if i = 100 then raise 'Waited too long'; end if;
-        PERFORM pg_stat_clear_snapshot();
-        perform * from pg_stat_activity where application_name = 'Citus Maintenance Daemon';
-        if found then exit; end if;
-        perform pg_sleep(0.1);
-    end loop;
-end$$;
-SELECT datid, datname, usename FROM pg_stat_activity WHERE application_name = 'Citus Maintenance Daemon';
- datid |  datname   |  usename
----------------------------------------------------------------------
- 16384 | regression | testuser2
-(1 row)
-
-RESET ROLE;
--- check that maintenance daemon gets (re-)started for the right user
-DROP EXTENSION citus;
-CREATE USER testuser SUPERUSER;
-SET ROLE testuser;
-CREATE EXTENSION citus;
-SELECT datname, current_database(),
-    usename, (SELECT extowner::regrole::text FROM pg_extension WHERE extname = 'citus')
-FROM test.maintenance_worker();
-  datname   | current_database | usename  | extowner
----------------------------------------------------------------------
- regression | regression       | postgres | testuser
-(1 row)
-
--- and recreate as the right owner
-RESET ROLE;
-DROP EXTENSION citus;
-CREATE EXTENSION citus;
--- Check that maintenance daemon can also be started in another database
-CREATE DATABASE another;
-NOTICE:  Citus partially supports CREATE DATABASE for distributed databases
-DETAIL:  Citus does not propagate CREATE DATABASE command to other nodes
-HINT:  You can manually create a database and its extensions on other nodes.
-\c another
-CREATE EXTENSION citus;
-CREATE SCHEMA test;
-:create_function_test_maintenance_worker
--- see that the daemon started
-SELECT datname, current_database(),
-    usename, (SELECT extowner::regrole::text FROM pg_extension WHERE extname = 'citus')
-FROM test.maintenance_worker();
- datname | current_database | usename  | extowner
----------------------------------------------------------------------
- another | another          | postgres | postgres
-(1 row)
-
--- Test that database with active worker can be dropped.
-\c regression
-CREATE SCHEMA test_daemon;
--- we create a similar function on the regression database
--- note that this function checks for the existence of the daemon
--- when not found, returns true else tries for 5 times and
--- returns false
-CREATE OR REPLACE FUNCTION test_daemon.maintenance_daemon_died(p_dbname text)
-    RETURNS boolean
-    LANGUAGE plpgsql
-AS $$
-DECLARE
-   activity record;
-BEGIN
-    PERFORM pg_stat_clear_snapshot();
-    SELECT * INTO activity FROM pg_stat_activity
-    WHERE application_name = 'Citus Maintenance Daemon' AND datname = p_dbname;
-    IF activity.pid IS NULL THEN
-        RETURN true;
-    ELSE
-        RETURN false;
-    END IF;
-END;
-$$;
--- drop the database and see that the daemon is dead
-DROP DATABASE another;
-SELECT
-    *
-FROM
-    test_daemon.maintenance_daemon_died('another');
- maintenance_daemon_died
----------------------------------------------------------------------
- t
-(1 row)
-
--- we don't need the schema and the function anymore
-DROP SCHEMA test_daemon CASCADE;
-NOTICE:  drop cascades to function test_daemon.maintenance_daemon_died(text)
--- verify citus does not crash while creating a table when run against an older worker
--- create_distributed_table piggybacks multiple commands into single one, if one worker
--- did not have the required UDF it should fail instead of crash.
--- create a test database, configure citus with single node
-CREATE DATABASE another;
-NOTICE:  Citus partially supports CREATE DATABASE for distributed databases
-DETAIL:  Citus does not propagate CREATE DATABASE command to other nodes
-HINT:  You can manually create a database and its extensions on other nodes.
-\c - - - :worker_1_port
-CREATE DATABASE another;
-NOTICE:  Citus partially supports CREATE DATABASE for distributed databases
-DETAIL:  Citus does not propagate CREATE DATABASE command to other nodes
-HINT:  You can manually create a database and its extensions on other nodes.
-\c - - - :master_port
-\c another
-CREATE EXTENSION citus;
-\c - - - :worker_1_port
-CREATE EXTENSION citus;
-\c - - - :master_port
-SELECT FROM master_add_node('localhost', :worker_1_port);
---
-(1 row)
-
-\c - - - :worker_1_port
-ALTER FUNCTION assign_distributed_transaction_id(initiator_node_identifier integer, transaction_number bigint, transaction_stamp timestamp with time zone)
-RENAME TO dummy_assign_function;
-\c - - - :master_port
-SET citus.shard_replication_factor to 1;
--- create_distributed_table command should fail
-CREATE TABLE t1(a int, b int);
-SET client_min_messages TO ERROR;
-DO $$
-BEGIN
-        BEGIN
-                SELECT create_distributed_table('t1', 'a');
-        EXCEPTION WHEN OTHERS THEN
-                RAISE 'create distributed table failed';
-        END;
-END;
-$$;
-ERROR:  create distributed table failed
-CONTEXT:  PL/pgSQL function inline_code_block line XX at RAISE
-\c regression
-\c - - - :master_port
-DROP DATABASE another;
-\c - - - :worker_1_port
-DROP DATABASE another;
-\c - - - :master_port
--- only the regression database should have a maintenance daemon
-SELECT count(*) FROM pg_stat_activity WHERE application_name = 'Citus Maintenance Daemon';
- count
----------------------------------------------------------------------
-     1
-(1 row)
-
--- recreate the extension immediately after the maintenancae daemon errors
-SELECT pg_cancel_backend(pid) FROM pg_stat_activity WHERE application_name = 'Citus Maintenance Daemon';
- pg_cancel_backend
----------------------------------------------------------------------
- t
-(1 row)
-
-DROP EXTENSION citus;
-CREATE EXTENSION citus;
--- wait for maintenance daemon restart
-SELECT datname, current_database(),
-    usename, (SELECT extowner::regrole::text FROM pg_extension WHERE extname = 'citus')
-FROM test.maintenance_worker();
-  datname   | current_database | usename  | extowner
----------------------------------------------------------------------
- regression | regression       | postgres | postgres
-(1 row)
-
--- confirm that there is only one maintenance daemon
-SELECT count(*) FROM pg_stat_activity WHERE application_name = 'Citus Maintenance Daemon';
- count
----------------------------------------------------------------------
-     1
-(1 row)
-
--- kill the maintenance daemon
-SELECT pg_cancel_backend(pid) FROM pg_stat_activity WHERE application_name = 'Citus Maintenance Daemon';
- pg_cancel_backend
----------------------------------------------------------------------
- t
-(1 row)
-
--- reconnect
-\c - - - :master_port
--- run something that goes through planner hook and therefore kicks of maintenance daemon
-SELECT 1;
- ?column?
----------------------------------------------------------------------
-        1
-(1 row)
-
--- wait for maintenance daemon restart
-SELECT datname, current_database(),
-    usename, (SELECT extowner::regrole::text FROM pg_extension WHERE extname = 'citus')
-FROM test.maintenance_worker();
-  datname   | current_database | usename  | extowner
----------------------------------------------------------------------
- regression | regression       | postgres | postgres
-(1 row)
-
--- confirm that there is only one maintenance daemon
-SELECT count(*) FROM pg_stat_activity WHERE application_name = 'Citus Maintenance Daemon';
- count
----------------------------------------------------------------------
-     1
-(1 row)
-
--- confirm that we can create a distributed table concurrently on an empty node
-DROP EXTENSION citus;
-CREATE EXTENSION citus;
-CREATE TABLE test (x int, y int);
-INSERT INTO test VALUES (1,2);
-SET citus.shard_replication_factor TO 1;
-SELECT create_distributed_table_concurrently('test','x');
-NOTICE:  relation test does not have a REPLICA IDENTITY or PRIMARY KEY
-DETAIL:  UPDATE and DELETE commands on the relation will error out during create_distributed_table_concurrently unless there is a REPLICA IDENTITY or PRIMARY KEY. INSERT commands will still work.
- create_distributed_table_concurrently
----------------------------------------------------------------------
-
-(1 row)
-
-DROP TABLE test;
-TRUNCATE pg_dist_node;
--- confirm that we can create a distributed table on an empty node
-CREATE TABLE test (x int, y int);
-INSERT INTO test VALUES (1,2);
-SET citus.shard_replication_factor TO 1;
-SELECT create_distributed_table('test','x');
-NOTICE:  Copying data from local table...
-NOTICE:  copying the data has completed
-DETAIL:  The local data in the table is no longer visible, but is still on disk.
-HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.test$$)
- create_distributed_table
----------------------------------------------------------------------
-
-(1 row)
-
-DROP TABLE test;
-TRUNCATE pg_dist_node;
--- confirm that we can create a single-shard table on an empty node
-CREATE TABLE test (x int, y int);
-INSERT INTO test VALUES (1,2);
-SET citus.shard_replication_factor TO 1;
-SELECT create_distributed_table('test', null, colocate_with=>'none', distribution_type=>null);
-NOTICE:  Copying data from local table...
-NOTICE:  copying the data has completed
-DETAIL:  The local data in the table is no longer visible, but is still on disk.
-HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.test$$)
- create_distributed_table
----------------------------------------------------------------------
-
-(1 row)
-
--- and make sure that we can't remove the coordinator due to "test"
-SELECT citus_remove_node('localhost', :master_port);
-ERROR:  cannot remove or disable the node localhost:xxxxx because because it contains the only shard placement for shard xxxxx
-DETAIL:  One of the table(s) that prevents the operation complete successfully is public.test
-HINT:  To proceed, either drop the tables or use undistribute_table() function to convert them to local tables
-DROP TABLE test;
--- and now we should be able to remove the coordinator
-SELECT citus_remove_node('localhost', :master_port);
- citus_remove_node
----------------------------------------------------------------------
-
-(1 row)
-
--- confirm that we can create a tenant schema / table on an empty node
-SET citus.enable_schema_based_sharding TO ON;
-CREATE SCHEMA tenant_schema;
-CREATE TABLE tenant_schema.test(x int, y int);
-SELECT colocationid = (
-    SELECT colocationid FROM pg_dist_partition WHERE logicalrelid = 'tenant_schema.test'::regclass
-)
-FROM pg_dist_schema
-WHERE schemaid::regnamespace::text = 'tenant_schema';
- ?column?
----------------------------------------------------------------------
- t
-(1 row)
-
--- and make sure that we can't remove the coordinator due to "test"
-SELECT citus_remove_node('localhost', :master_port);
-ERROR:  cannot remove or disable the node localhost:xxxxx because because it contains the only shard placement for shard xxxxx
-DETAIL:  One of the table(s) that prevents the operation complete successfully is tenant_schema.test
-HINT:  To proceed, either drop the tables or use undistribute_table() function to convert them to local tables
-BEGIN;
-  SET LOCAL client_min_messages TO WARNING;
-  DROP SCHEMA tenant_schema CASCADE;
-COMMIT;
--- and now we should be able to remove the coordinator
-SELECT citus_remove_node('localhost', :master_port);
- citus_remove_node
----------------------------------------------------------------------
-
-(1 row)
-
-CREATE SCHEMA tenant_schema;
--- Make sure that we can sync metadata for empty tenant schemas
--- when adding the first node to the cluster.
-SELECT 1 FROM citus_add_node('localhost', :worker_1_port);
- ?column?
----------------------------------------------------------------------
-        1
-(1 row)
-
-DROP SCHEMA tenant_schema;
-SELECT citus_remove_node('localhost', :worker_1_port);
- citus_remove_node
----------------------------------------------------------------------
-
-(1 row)
-
-RESET citus.enable_schema_based_sharding;
--- confirm that we can create a reference table on an empty node
-CREATE TABLE test (x int, y int);
-INSERT INTO test VALUES (1,2);
-SELECT create_reference_table('test');
-NOTICE:  Copying data from local table...
-NOTICE:  copying the data has completed
-DETAIL:  The local data in the table is no longer visible, but is still on disk.
-HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.test$$)
- create_reference_table
----------------------------------------------------------------------
-
-(1 row)
-
-DROP TABLE test;
-TRUNCATE pg_dist_node;
--- confirm that we can create a local table on an empty node
-CREATE TABLE test (x int, y int);
-INSERT INTO test VALUES (1,2);
-SELECT citus_add_local_table_to_metadata('test');
- citus_add_local_table_to_metadata
----------------------------------------------------------------------
-
-(1 row)
-
-DROP TABLE test;
--- Verify that we don't consider the schemas created by extensions as tenant schemas.
--- Easiest way of verifying this is to drop and re-create columnar extension.
-DROP EXTENSION citus_columnar;
-SET citus.enable_schema_based_sharding TO ON;
-CREATE EXTENSION citus_columnar;
-SELECT COUNT(*)=0 FROM pg_dist_schema
-WHERE schemaid IN ('columnar'::regnamespace, 'columnar_internal'::regnamespace);
- ?column?
----------------------------------------------------------------------
- t
-(1 row)
-
-RESET citus.enable_schema_based_sharding;
-DROP EXTENSION citus;
-CREATE EXTENSION citus;
-DROP TABLE version_mismatch_table;
-DROP SCHEMA multi_extension;
-ERROR:  cannot drop schema multi_extension because other objects depend on it
-DETAIL:  function multi_extension.print_extension_changes() depends on schema multi_extension
-HINT:  Use DROP ... CASCADE to drop the dependent objects too.

From 48fdee016a94b40f552209472226a94ae8ac2907 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Thu, 28 Mar 2024 13:31:47 +0300
Subject: [PATCH 65/78] Fixes test errors

---
 src/test/regress/sql/minimal_cluster_management.sql | 2 +-
 src/test/regress/sql/multi_cluster_management.sql   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/test/regress/sql/minimal_cluster_management.sql b/src/test/regress/sql/minimal_cluster_management.sql
index 80c1059a7..360931eba 100644
--- a/src/test/regress/sql/minimal_cluster_management.sql
+++ b/src/test/regress/sql/minimal_cluster_management.sql
@@ -41,6 +41,6 @@ DROP TABLE test_dist;
 ALTER SEQUENCE pg_catalog.pg_dist_node_nodeid_seq RESTART 30;
 ALTER SEQUENCE pg_catalog.pg_dist_groupid_seq RESTART 18;
 ALTER SEQUENCE pg_catalog.pg_dist_placement_placementid_seq RESTART 83;
--- Enable ddl sync in all nodes
+-- enable ddl propagation in all nodes
 SELECT result from  run_command_on_all_nodes('ALTER SYSTEM SET citus.enable_ddl_propagation TO ON');
 SELECT result from  run_command_on_all_nodes('SELECT pg_reload_conf()');
diff --git a/src/test/regress/sql/multi_cluster_management.sql b/src/test/regress/sql/multi_cluster_management.sql
index 41f962f35..c10c1e36e 100644
--- a/src/test/regress/sql/multi_cluster_management.sql
+++ b/src/test/regress/sql/multi_cluster_management.sql
@@ -537,6 +537,6 @@ SELECT bool_and(hasmetadata) AND bool_and(metadatasynced) FROM pg_dist_node WHER
 -- keep permissions compatible accross versions, in regression
 -- tests.
 GRANT ALL ON SCHEMA public TO PUBLIC;
--- Enable ddl sync in all nodes
+-- enable ddl propagation in all nodes
 SELECT result from  run_command_on_all_nodes('ALTER SYSTEM SET citus.enable_ddl_propagation TO ON');
 SELECT result from  run_command_on_all_nodes('SELECT pg_reload_conf()');

From ccac41b711d37f4d9525cdd7ad6e6bec1b5dae93 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Fri, 29 Mar 2024 13:02:30 +0300
Subject: [PATCH 66/78] Fixes from any node issues

---
 src/backend/distributed/commands/database.c   | 57 +++++++++++++------
 .../commands/distribute_object_ops.c          |  2 +-
 src/include/distributed/commands.h            |  3 +
 .../sql/alter_database_from_nonmain_db.sql    | 37 +++++++++++-
 4 files changed, 81 insertions(+), 18 deletions(-)

diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c
index c9279b661..fb684a687 100644
--- a/src/backend/distributed/commands/database.c
+++ b/src/backend/distributed/commands/database.c
@@ -293,14 +293,9 @@ PreprocessAlterDatabaseStmt(Node *node, const char *queryString,
 		return NIL;
 	}
 
-	/* Since ALTER DATABASE SET TABLESPACE statement is not supported */
-	/* inside a transaction block, we need to send the command to the */
-	/* main database directly to make it work */
-	if (!IsSetTablespaceStatement(stmt))
-	{
-		EnsureCoordinator();
-	}
 
+	EnsurePropagationToCoordinator();
+	EnsureAllObjectDependenciesExistOnAllNodes(list_make1(dbAddress));
 	SerializeDistributedDDLsOnObjectClassObject(OCLASS_DATABASE, stmt->dbname);
 
 	char *sql = DeparseTreeNode((Node *) stmt);
@@ -317,12 +312,12 @@ PreprocessAlterDatabaseStmt(Node *node, const char *queryString,
 		 * the transaction block.
 		 */
 		bool warnForPartialFailure = true;
-		return NontransactionalNodeDDLTaskList(NON_COORDINATOR_NODES, commands,
+		return NontransactionalNodeDDLTaskList(REMOTE_NODES, commands,
 											   warnForPartialFailure);
 	}
 	else
 	{
-		return NodeDDLTaskList(NON_COORDINATOR_NODES, commands);
+		return NodeDDLTaskList(REMOTE_NODES, commands);
 	}
 }
 
@@ -353,7 +348,9 @@ PreprocessAlterDatabaseRefreshCollStmt(Node *node, const char *queryString,
 		return NIL;
 	}
 
-	EnsureCoordinator();
+	EnsurePropagationToCoordinator();
+	EnsureAllObjectDependenciesExistOnAllNodes(list_make1(dbAddress));
+
 	SerializeDistributedDDLsOnObjectClassObject(OCLASS_DATABASE, stmt->dbname);
 
 	char *sql = DeparseTreeNode((Node *) stmt);
@@ -362,7 +359,7 @@ PreprocessAlterDatabaseRefreshCollStmt(Node *node, const char *queryString,
 								(void *) sql,
 								ENABLE_DDL_PROPAGATION);
 
-	return NodeDDLTaskList(NON_COORDINATOR_NODES, commands);
+	return NodeDDLTaskList(REMOTE_NODES, commands);
 }
 
 
@@ -395,7 +392,8 @@ PreprocessAlterDatabaseRenameStmt(Node *node, const char *queryString,
 		return NIL;
 	}
 
-	EnsureCoordinator();
+	EnsurePropagationToCoordinator();
+	EnsureAllObjectDependenciesExistOnAllNodes(list_make1(dbAddress));
 
 	/*
 	 * Different than other ALTER DATABASE commands, we first acquire a lock
@@ -429,7 +427,8 @@ PostprocessAlterDatabaseRenameStmt(Node *node, const char *queryString)
 		return NIL;
 	}
 
-	EnsureCoordinator();
+	EnsurePropagationToCoordinator();
+	EnsureAllObjectDependenciesExistOnAllNodes(list_make1(dbAddress));
 
 	char *sql = DeparseTreeNode((Node *) stmt);
 
@@ -437,7 +436,7 @@ PostprocessAlterDatabaseRenameStmt(Node *node, const char *queryString)
 								(void *) sql,
 								ENABLE_DDL_PROPAGATION);
 
-	return NodeDDLTaskList(NON_COORDINATOR_NODES, commands);
+	return NodeDDLTaskList(REMOTE_NODES, commands);
 }
 
 
@@ -465,7 +464,8 @@ PreprocessAlterDatabaseSetStmt(Node *node, const char *queryString,
 		return NIL;
 	}
 
-	EnsureCoordinator();
+	EnsurePropagationToCoordinator();
+	EnsureAllObjectDependenciesExistOnAllNodes(list_make1(dbAddress));
 	SerializeDistributedDDLsOnObjectClassObject(OCLASS_DATABASE, stmt->dbname);
 
 	char *sql = DeparseTreeNode((Node *) stmt);
@@ -474,7 +474,7 @@ PreprocessAlterDatabaseSetStmt(Node *node, const char *queryString,
 								(void *) sql,
 								ENABLE_DDL_PROPAGATION);
 
-	return NodeDDLTaskList(NON_COORDINATOR_NODES, commands);
+	return NodeDDLTaskList(REMOTE_NODES, commands);
 }
 
 
@@ -694,6 +694,31 @@ PreprocessDropDatabaseStmt(Node *node, const char *queryString,
 	return dropDatabaseDDLJobList;
 }
 
+List *
+PreprocessAlterDatabaseOwnerStmt(Node *node, const char *queryString,
+						   ProcessUtilityContext processUtilityContext){
+	List *addresses = GetObjectAddressListFromParseTree(node, false, false);
+
+	if (!ShouldPropagateAnyObject(addresses))
+	{
+		return NIL;
+	}
+	EnsurePropagationToCoordinator();
+
+	AlterOwnerStmt *stmt = (AlterOwnerStmt *) node;
+	EnsureSequentialMode(stmt->objectType);
+	QualifyTreeNode(node);
+	char *databaseName = strVal((String *) stmt->object);
+	SerializeDistributedDDLsOnObjectClassObject(OCLASS_DATABASE, databaseName);
+	const char *sql = DeparseTreeNode((Node *) node);
+
+	List *commands = list_make3(DISABLE_DDL_PROPAGATION,
+								(void *) sql,
+								ENABLE_DDL_PROPAGATION);
+
+	return NodeDDLTaskList(REMOTE_NODES, commands);
+}
+
 
 /*
  * DropDatabaseStmtObjectAddress gets the ObjectAddress of the database that is the
diff --git a/src/backend/distributed/commands/distribute_object_ops.c b/src/backend/distributed/commands/distribute_object_ops.c
index 5a62dd2c8..c40738cc2 100644
--- a/src/backend/distributed/commands/distribute_object_ops.c
+++ b/src/backend/distributed/commands/distribute_object_ops.c
@@ -468,7 +468,7 @@ static DistributeObjectOps Collation_Rename = {
 static DistributeObjectOps Database_AlterOwner = {
 	.deparse = DeparseAlterDatabaseOwnerStmt,
 	.qualify = NULL,
-	.preprocess = PreprocessAlterDistributedObjectStmt,
+	.preprocess = PreprocessAlterDatabaseOwnerStmt,
 	.postprocess = PostprocessAlterDistributedObjectStmt,
 	.objectType = OBJECT_DATABASE,
 	.operationType = DIST_OPS_ALTER,
diff --git a/src/include/distributed/commands.h b/src/include/distributed/commands.h
index 8ce5c72ac..62dc23684 100644
--- a/src/include/distributed/commands.h
+++ b/src/include/distributed/commands.h
@@ -234,6 +234,9 @@ extern List * PreprocessAlterDatabaseStmt(Node *node, const char *queryString,
 extern List * PreprocessAlterDatabaseRefreshCollStmt(Node *node, const char *queryString,
 													 ProcessUtilityContext
 													 processUtilityContext);
+extern List * PreprocessAlterDatabaseOwnerStmt(Node *node, const char *queryString,
+						   					   ProcessUtilityContext processUtilityContext);
+											   
 extern List * GetDatabaseMetadataSyncCommands(Oid dbOid);
 
 
diff --git a/src/test/regress/sql/alter_database_from_nonmain_db.sql b/src/test/regress/sql/alter_database_from_nonmain_db.sql
index b8a99a495..7e075c07a 100644
--- a/src/test/regress/sql/alter_database_from_nonmain_db.sql
+++ b/src/test/regress/sql/alter_database_from_nonmain_db.sql
@@ -34,6 +34,16 @@ set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database altered_database_renamed rename to "altered_database!'2";
 
+
+alter database "altered_database!'2" with
+    ALLOW_CONNECTIONS true
+    CONNECTION LIMIT 0
+    IS_TEMPLATE false;
+
+\c regression - - :worker_2_port
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
+
 alter database "altered_database!'2" with
     ALLOW_CONNECTIONS false
     CONNECTION LIMIT 1
@@ -43,6 +53,14 @@ alter database "altered_database!'2" with
     CONNECTION LIMIT 0
     IS_TEMPLATE false;
 
+alter database "altered_database!'2" rename to altered_database_renamed;
+
+alter database altered_database_renamed rename to "altered_database!'2";
+
+
+
+
+
 \c regression
 create role test_owner_non_main_db;
 
@@ -56,9 +74,23 @@ alter database "altered_database!'2" owner to test_owner_non_main_db;
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" owner to CURRENT_USER;
-alter database "altered_database!'2" set default_transaction_read_only = true;
 set default_transaction_read_only = false;
 
+\c regression - - :worker_1_port
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
+alter database "altered_database!'2" owner to test_owner_non_main_db;
+set default_transaction_read_only = false;
+
+
+\c regression - - :master_port
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
+alter database "altered_database!'2" owner to CURRENT_USER;
+set default_transaction_read_only = false;
+
+
+
 \c test_alter_db_from_nonmain_db - - :worker_2_port
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
@@ -76,6 +108,9 @@ alter database "altered_database!'2" RESET TIME ZONE;
 alter database "altered_database!'2" SET TIME ZONE INTERVAL '-08:00' HOUR TO MINUTE;
 alter database "altered_database!'2" RESET TIME ZONE;
 alter database "altered_database!'2" set default_transaction_isolation = 'serializable';
+\c regression - - :worker_2_port
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
 set default_transaction_isolation = 'read committed';
 alter database "altered_database!'2" set default_transaction_isolation from current;
 alter database "altered_database!'2" set default_transaction_isolation to DEFAULT;

From 0e02faccc46cad7846c77c8df8335d86f42c6537 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Fri, 29 Mar 2024 13:32:49 +0300
Subject: [PATCH 67/78] Fixes test output

---
 .../alter_database_from_nonmain_db.out        | 97 ++++++++++++++++---
 1 file changed, 81 insertions(+), 16 deletions(-)

diff --git a/src/test/regress/expected/alter_database_from_nonmain_db.out b/src/test/regress/expected/alter_database_from_nonmain_db.out
index bb1f97612..0b93849c0 100644
--- a/src/test/regress/expected/alter_database_from_nonmain_db.out
+++ b/src/test/regress/expected/alter_database_from_nonmain_db.out
@@ -34,17 +34,40 @@ set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database altered_database_renamed rename to "altered_database!'2";
 NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database_renamed RENAME TO "altered_database!''2"', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database "altered_database!'2" with
+    ALLOW_CONNECTIONS true
+    CONNECTION LIMIT 0
+    IS_TEMPLATE false;
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" WITH ALLOW_CONNECTIONS true CONNECTION LIMIT 0 IS_TEMPLATE false;', 'postgres')
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+\c regression - - :worker_2_port
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" with
     ALLOW_CONNECTIONS false
     CONNECTION LIMIT 1
     IS_TEMPLATE true;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" WITH ALLOW_CONNECTIONS false CONNECTION LIMIT 1 IS_TEMPLATE true;', 'postgres')
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" WITH ALLOW_CONNECTIONS false CONNECTION LIMIT 1 IS_TEMPLATE true;
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" WITH ALLOW_CONNECTIONS false CONNECTION LIMIT 1 IS_TEMPLATE true;
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 alter database "altered_database!'2" with
     ALLOW_CONNECTIONS true
     CONNECTION LIMIT 0
     IS_TEMPLATE false;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" WITH ALLOW_CONNECTIONS true CONNECTION LIMIT 0 IS_TEMPLATE false;', 'postgres')
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" WITH ALLOW_CONNECTIONS true CONNECTION LIMIT 0 IS_TEMPLATE false;
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" WITH ALLOW_CONNECTIONS true CONNECTION LIMIT 0 IS_TEMPLATE false;
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database "altered_database!'2" rename to altered_database_renamed;
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" RENAME TO altered_database_renamed
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" RENAME TO altered_database_renamed
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database altered_database_renamed rename to "altered_database!'2";
+NOTICE:  issuing ALTER DATABASE altered_database_renamed RENAME TO "altered_database!'2"
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE altered_database_renamed RENAME TO "altered_database!'2"
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 \c regression
 create role test_owner_non_main_db;
@@ -61,8 +84,23 @@ set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" owner to CURRENT_USER;
 NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" OWNER TO postgres;', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-alter database "altered_database!'2" set default_transaction_read_only = true;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET default_transaction_read_only = ''true''', 'postgres')
+set default_transaction_read_only = false;
+\c regression - - :worker_1_port
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
+alter database "altered_database!'2" owner to test_owner_non_main_db;
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" OWNER TO test_owner_non_main_db;
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" OWNER TO test_owner_non_main_db;
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+set default_transaction_read_only = false;
+\c regression - - :master_port
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
+alter database "altered_database!'2" owner to CURRENT_USER;
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" OWNER TO postgres;
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" OWNER TO postgres;
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 set default_transaction_read_only = false;
 \c test_alter_db_from_nonmain_db - - :worker_2_port
@@ -101,44 +139,71 @@ DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 alter database "altered_database!'2" set default_transaction_isolation = 'serializable';
 NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET default_transaction_isolation = ''serializable''', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+\c regression - - :worker_2_port
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
 set default_transaction_isolation = 'read committed';
 alter database "altered_database!'2" set default_transaction_isolation from current;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET default_transaction_isolation FROM CURRENT', 'postgres')
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET default_transaction_isolation FROM CURRENT
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET default_transaction_isolation FROM CURRENT
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 alter database "altered_database!'2" set default_transaction_isolation to DEFAULT;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET default_transaction_isolation TO DEFAULT', 'postgres')
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET default_transaction_isolation TO DEFAULT
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET default_transaction_isolation TO DEFAULT
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 alter database "altered_database!'2" RESET default_transaction_isolation;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" RESET default_transaction_isolation', 'postgres')
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" RESET default_transaction_isolation
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" RESET default_transaction_isolation
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 alter database "altered_database!'2" set statement_timeout = 1000;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET statement_timeout = 1000', 'postgres')
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET statement_timeout = 1000
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET statement_timeout = 1000
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 set statement_timeout = 2000;
 alter database "altered_database!'2" set statement_timeout from current;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET statement_timeout FROM CURRENT', 'postgres')
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET statement_timeout FROM CURRENT
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET statement_timeout FROM CURRENT
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 alter database "altered_database!'2" set statement_timeout to DEFAULT;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET statement_timeout TO DEFAULT', 'postgres')
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET statement_timeout TO DEFAULT
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET statement_timeout TO DEFAULT
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 alter database "altered_database!'2" RESET statement_timeout;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" RESET statement_timeout', 'postgres')
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" RESET statement_timeout
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" RESET statement_timeout
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 alter database "altered_database!'2" set lock_timeout = 1201.5;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET lock_timeout = 1201.5', 'postgres')
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET lock_timeout = 1201.5
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET lock_timeout = 1201.5
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 set lock_timeout = 1202.5;
 alter database "altered_database!'2" set lock_timeout from current;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET lock_timeout FROM CURRENT', 'postgres')
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET lock_timeout FROM CURRENT
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET lock_timeout FROM CURRENT
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 alter database "altered_database!'2" set lock_timeout to DEFAULT;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET lock_timeout TO DEFAULT', 'postgres')
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET lock_timeout TO DEFAULT
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET lock_timeout TO DEFAULT
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 alter database "altered_database!'2" RESET lock_timeout;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" RESET lock_timeout', 'postgres')
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" RESET lock_timeout
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" RESET lock_timeout
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 ALTER DATABASE "altered_database!'2" RESET ALL;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" RESET ALL', 'postgres')
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" RESET ALL
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" RESET ALL
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 \c regression
 set citus.enable_create_database_propagation=on;

From 6d964baa0c59428bd942da7f50baa4ac9e894616 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Fri, 29 Mar 2024 14:06:40 +0300
Subject: [PATCH 68/78] Adds additional tests for alter tablespace

---
 .../alter_database_from_nonmain_db.out        | 234 ++++++++++++++++--
 .../sql/alter_database_from_nonmain_db.sql    |  59 ++++-
 2 files changed, 271 insertions(+), 22 deletions(-)

diff --git a/src/test/regress/expected/alter_database_from_nonmain_db.out b/src/test/regress/expected/alter_database_from_nonmain_db.out
index 0b93849c0..9348d8976 100644
--- a/src/test/regress/expected/alter_database_from_nonmain_db.out
+++ b/src/test/regress/expected/alter_database_from_nonmain_db.out
@@ -25,14 +25,197 @@ set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" set tablespace "pg_default";
 NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-alter database "altered_database!'2" rename to altered_database_renamed;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" RENAME TO altered_database_renamed', 'postgres')
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database "altered_database!'2" set tablespace "pg_default";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database "altered_database!'2" set tablespace "pg_default";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database "altered_database!'2" set tablespace "pg_default";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database "altered_database!'2" set tablespace "pg_default";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+\c regression - - :worker_1_port
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
+alter database "altered_database!'2" set tablespace "pg_default";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database "altered_database!'2" set tablespace "pg_default";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database "altered_database!'2" set tablespace "pg_default";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database "altered_database!'2" set tablespace "pg_default";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database "altered_database!'2" set tablespace "pg_default";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+\c regression - - :worker_2_port
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
+alter database "altered_database!'2" set tablespace "pg_default";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database "altered_database!'2" set tablespace "pg_default";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database "altered_database!'2" set tablespace "pg_default";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database "altered_database!'2" set tablespace "pg_default";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database "altered_database!'2" set tablespace "pg_default";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+\c regression - - :master_port
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
+alter database "altered_database!'2" set tablespace "pg_default";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database "altered_database!'2" set tablespace "pg_default";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database "altered_database!'2" set tablespace "pg_default";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database "altered_database!'2" set tablespace "pg_default";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database "altered_database!'2" set tablespace "pg_default";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 \c test_alter_db_from_nonmain_db - - :worker_2_port
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
-alter database altered_database_renamed rename to "altered_database!'2";
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database_renamed RENAME TO "altered_database!''2"', 'postgres')
+alter database "altered_database!'2"  rename to altered_database_renamed;
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" RENAME TO altered_database_renamed', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 alter database "altered_database!'2" with
     ALLOW_CONNECTIONS true
@@ -40,35 +223,50 @@ alter database "altered_database!'2" with
     IS_TEMPLATE false;
 NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" WITH ALLOW_CONNECTIONS true CONNECTION LIMIT 0 IS_TEMPLATE false;', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+ERROR:  database "altered_database!'2" does not exist
 \c regression - - :worker_2_port
 set citus.log_remote_commands = true;
-set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" with
     ALLOW_CONNECTIONS false
     CONNECTION LIMIT 1
     IS_TEMPLATE true;
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" WITH ALLOW_CONNECTIONS false CONNECTION LIMIT 1 IS_TEMPLATE true;
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" WITH ALLOW_CONNECTIONS false CONNECTION LIMIT 1 IS_TEMPLATE true;
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+ERROR:  database "altered_database!'2" does not exist
 alter database "altered_database!'2" with
     ALLOW_CONNECTIONS true
     CONNECTION LIMIT 0
     IS_TEMPLATE false;
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" WITH ALLOW_CONNECTIONS true CONNECTION LIMIT 0 IS_TEMPLATE false;
+ERROR:  database "altered_database!'2" does not exist
+alter database altered_database_renamed  rename to "altered_database!'2";
+NOTICE:  issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx');
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" WITH ALLOW_CONNECTIONS true CONNECTION LIMIT 0 IS_TEMPLATE false;
+NOTICE:  issuing SELECT citus_internal.acquire_citus_advisory_object_class_lock(26, NULL)
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-alter database "altered_database!'2" rename to altered_database_renamed;
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" RENAME TO altered_database_renamed
+NOTICE:  issuing SELECT citus_internal.acquire_citus_advisory_object_class_lock(26, 'altered_database_renamed')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" RENAME TO altered_database_renamed
+NOTICE:  issuing SET citus.enable_ddl_propagation TO 'off'
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE altered_database_renamed RENAME TO "altered_database!'2"
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing SET citus.enable_ddl_propagation TO 'on'
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx');
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing SET citus.enable_ddl_propagation TO 'off'
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE altered_database_renamed RENAME TO "altered_database!'2"
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing SET citus.enable_ddl_propagation TO 'on'
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing PREPARE TRANSACTION 'citus_xx_xx_xx_xx'
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing PREPARE TRANSACTION 'citus_xx_xx_xx_xx'
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing COMMIT PREPARED 'citus_xx_xx_xx_xx'
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing COMMIT PREPARED 'citus_xx_xx_xx_xx'
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 alter database altered_database_renamed rename to "altered_database!'2";
-NOTICE:  issuing ALTER DATABASE altered_database_renamed RENAME TO "altered_database!'2"
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-NOTICE:  issuing ALTER DATABASE altered_database_renamed RENAME TO "altered_database!'2"
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+ERROR:  database "altered_database_renamed" does not exist
 \c regression
 create role test_owner_non_main_db;
 \c test_alter_db_from_nonmain_db
diff --git a/src/test/regress/sql/alter_database_from_nonmain_db.sql b/src/test/regress/sql/alter_database_from_nonmain_db.sql
index 7e075c07a..31e8f113c 100644
--- a/src/test/regress/sql/alter_database_from_nonmain_db.sql
+++ b/src/test/regress/sql/alter_database_from_nonmain_db.sql
@@ -27,12 +27,64 @@ alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" set tablespace "pg_default";
-alter database "altered_database!'2" rename to altered_database_renamed;
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+alter database "altered_database!'2" set tablespace "pg_default";
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+alter database "altered_database!'2" set tablespace "pg_default";
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+alter database "altered_database!'2" set tablespace "pg_default";
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+alter database "altered_database!'2" set tablespace "pg_default";
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+
+\c regression - - :worker_1_port
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
+alter database "altered_database!'2" set tablespace "pg_default";
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+alter database "altered_database!'2" set tablespace "pg_default";
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+alter database "altered_database!'2" set tablespace "pg_default";
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+alter database "altered_database!'2" set tablespace "pg_default";
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+alter database "altered_database!'2" set tablespace "pg_default";
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+
+\c regression - - :worker_2_port
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
+alter database "altered_database!'2" set tablespace "pg_default";
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+alter database "altered_database!'2" set tablespace "pg_default";
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+alter database "altered_database!'2" set tablespace "pg_default";
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+alter database "altered_database!'2" set tablespace "pg_default";
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+alter database "altered_database!'2" set tablespace "pg_default";
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+
+\c regression - - :master_port
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
+alter database "altered_database!'2" set tablespace "pg_default";
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+alter database "altered_database!'2" set tablespace "pg_default";
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+alter database "altered_database!'2" set tablespace "pg_default";
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+alter database "altered_database!'2" set tablespace "pg_default";
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+alter database "altered_database!'2" set tablespace "pg_default";
+alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+
+
 
 \c test_alter_db_from_nonmain_db - - :worker_2_port
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
-alter database altered_database_renamed rename to "altered_database!'2";
+alter database "altered_database!'2"  rename to altered_database_renamed;
 
 
 alter database "altered_database!'2" with
@@ -42,7 +94,6 @@ alter database "altered_database!'2" with
 
 \c regression - - :worker_2_port
 set citus.log_remote_commands = true;
-set citus.grep_remote_commands = "%ALTER DATABASE%";
 
 alter database "altered_database!'2" with
     ALLOW_CONNECTIONS false
@@ -53,7 +104,7 @@ alter database "altered_database!'2" with
     CONNECTION LIMIT 0
     IS_TEMPLATE false;
 
-alter database "altered_database!'2" rename to altered_database_renamed;
+alter database altered_database_renamed  rename to "altered_database!'2";
 
 alter database altered_database_renamed rename to "altered_database!'2";
 

From 02998973950d2c78ae800f270a279faf8a260347 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Sat, 30 Mar 2024 12:00:55 +0300
Subject: [PATCH 69/78] Fixes test errors

---
 src/backend/distributed/commands/database.c   | 12 ++++++
 src/include/distributed/commands.h            |  4 +-
 .../alter_database_from_nonmain_db.out        | 42 +++++--------------
 .../sql/alter_database_from_nonmain_db.sql    |  6 +--
 4 files changed, 27 insertions(+), 37 deletions(-)

diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c
index fb684a687..118395cbb 100644
--- a/src/backend/distributed/commands/database.c
+++ b/src/backend/distributed/commands/database.c
@@ -697,7 +697,19 @@ PreprocessDropDatabaseStmt(Node *node, const char *queryString,
 List *
 PreprocessAlterDatabaseOwnerStmt(Node *node, const char *queryString,
 						   ProcessUtilityContext processUtilityContext){
+
+	const DistributeObjectOps *ops = GetDistributeObjectOps(node);
+	Assert(ops != NULL);
+
+	if (ops->featureFlag && *ops->featureFlag == false)
+	{
+		/* not propagating when a configured feature flag is turned off by the user */
+		return NIL;
+	}
+
 	List *addresses = GetObjectAddressListFromParseTree(node, false, false);
+	/*  the code-path only supports a single object */
+	Assert(list_length(addresses) == 1);
 
 	if (!ShouldPropagateAnyObject(addresses))
 	{
diff --git a/src/include/distributed/commands.h b/src/include/distributed/commands.h
index 62dc23684..72d4f8c27 100644
--- a/src/include/distributed/commands.h
+++ b/src/include/distributed/commands.h
@@ -235,8 +235,8 @@ extern List * PreprocessAlterDatabaseRefreshCollStmt(Node *node, const char *que
 													 ProcessUtilityContext
 													 processUtilityContext);
 extern List * PreprocessAlterDatabaseOwnerStmt(Node *node, const char *queryString,
-						   					   ProcessUtilityContext processUtilityContext);
-											   
+											   ProcessUtilityContext processUtilityContext);
+
 extern List * GetDatabaseMetadataSyncCommands(Oid dbOid);
 
 
diff --git a/src/test/regress/expected/alter_database_from_nonmain_db.out b/src/test/regress/expected/alter_database_from_nonmain_db.out
index 9348d8976..e5eb85c16 100644
--- a/src/test/regress/expected/alter_database_from_nonmain_db.out
+++ b/src/test/regress/expected/alter_database_from_nonmain_db.out
@@ -217,56 +217,34 @@ set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2"  rename to altered_database_renamed;
 NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" RENAME TO altered_database_renamed', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+alter database altered_database_renamed  rename to "altered_database!'2";
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE altered_database_renamed RENAME TO "altered_database!''2"', 'postgres')
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 alter database "altered_database!'2" with
     ALLOW_CONNECTIONS true
     CONNECTION LIMIT 0
     IS_TEMPLATE false;
 NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" WITH ALLOW_CONNECTIONS true CONNECTION LIMIT 0 IS_TEMPLATE false;', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-ERROR:  database "altered_database!'2" does not exist
 \c regression - - :worker_2_port
 set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" with
     ALLOW_CONNECTIONS false
     CONNECTION LIMIT 1
     IS_TEMPLATE true;
-ERROR:  database "altered_database!'2" does not exist
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" WITH ALLOW_CONNECTIONS false CONNECTION LIMIT 1 IS_TEMPLATE true;
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" WITH ALLOW_CONNECTIONS false CONNECTION LIMIT 1 IS_TEMPLATE true;
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 alter database "altered_database!'2" with
     ALLOW_CONNECTIONS true
     CONNECTION LIMIT 0
     IS_TEMPLATE false;
-ERROR:  database "altered_database!'2" does not exist
-alter database altered_database_renamed  rename to "altered_database!'2";
-NOTICE:  issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx');
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" WITH ALLOW_CONNECTIONS true CONNECTION LIMIT 0 IS_TEMPLATE false;
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-NOTICE:  issuing SELECT citus_internal.acquire_citus_advisory_object_class_lock(26, NULL)
+NOTICE:  issuing ALTER DATABASE "altered_database!'2" WITH ALLOW_CONNECTIONS true CONNECTION LIMIT 0 IS_TEMPLATE false;
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-NOTICE:  issuing SELECT citus_internal.acquire_citus_advisory_object_class_lock(26, 'altered_database_renamed')
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-NOTICE:  issuing SET citus.enable_ddl_propagation TO 'off'
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-NOTICE:  issuing ALTER DATABASE altered_database_renamed RENAME TO "altered_database!'2"
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-NOTICE:  issuing SET citus.enable_ddl_propagation TO 'on'
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-NOTICE:  issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx');
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-NOTICE:  issuing SET citus.enable_ddl_propagation TO 'off'
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-NOTICE:  issuing ALTER DATABASE altered_database_renamed RENAME TO "altered_database!'2"
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-NOTICE:  issuing SET citus.enable_ddl_propagation TO 'on'
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-NOTICE:  issuing PREPARE TRANSACTION 'citus_xx_xx_xx_xx'
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-NOTICE:  issuing PREPARE TRANSACTION 'citus_xx_xx_xx_xx'
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-NOTICE:  issuing COMMIT PREPARED 'citus_xx_xx_xx_xx'
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-NOTICE:  issuing COMMIT PREPARED 'citus_xx_xx_xx_xx'
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-alter database altered_database_renamed rename to "altered_database!'2";
-ERROR:  database "altered_database_renamed" does not exist
 \c regression
 create role test_owner_non_main_db;
 \c test_alter_db_from_nonmain_db
diff --git a/src/test/regress/sql/alter_database_from_nonmain_db.sql b/src/test/regress/sql/alter_database_from_nonmain_db.sql
index 31e8f113c..c9eb8f57f 100644
--- a/src/test/regress/sql/alter_database_from_nonmain_db.sql
+++ b/src/test/regress/sql/alter_database_from_nonmain_db.sql
@@ -86,6 +86,8 @@ set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2"  rename to altered_database_renamed;
 
+alter database altered_database_renamed  rename to "altered_database!'2";
+
 
 alter database "altered_database!'2" with
     ALLOW_CONNECTIONS true
@@ -94,6 +96,7 @@ alter database "altered_database!'2" with
 
 \c regression - - :worker_2_port
 set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
 
 alter database "altered_database!'2" with
     ALLOW_CONNECTIONS false
@@ -104,9 +107,6 @@ alter database "altered_database!'2" with
     CONNECTION LIMIT 0
     IS_TEMPLATE false;
 
-alter database altered_database_renamed  rename to "altered_database!'2";
-
-alter database altered_database_renamed rename to "altered_database!'2";
 
 
 

From 179f2e5fad45faa18d850fad5daf660f62080334 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Sat, 30 Mar 2024 12:36:26 +0300
Subject: [PATCH 70/78] Fixes indentation

---
 src/backend/distributed/commands/database.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c
index 118395cbb..9ea6c17ba 100644
--- a/src/backend/distributed/commands/database.c
+++ b/src/backend/distributed/commands/database.c
@@ -694,10 +694,11 @@ PreprocessDropDatabaseStmt(Node *node, const char *queryString,
 	return dropDatabaseDDLJobList;
 }
 
+
 List *
 PreprocessAlterDatabaseOwnerStmt(Node *node, const char *queryString,
-						   ProcessUtilityContext processUtilityContext){
-
+								 ProcessUtilityContext processUtilityContext)
+{
 	const DistributeObjectOps *ops = GetDistributeObjectOps(node);
 	Assert(ops != NULL);
 
@@ -708,6 +709,7 @@ PreprocessAlterDatabaseOwnerStmt(Node *node, const char *queryString,
 	}
 
 	List *addresses = GetObjectAddressListFromParseTree(node, false, false);
+
 	/*  the code-path only supports a single object */
 	Assert(list_length(addresses) == 1);
 

From 62de6dc0b4ce12372291144b7a7a5480995ad2fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=BCrkan=20=C4=B0ndibay?= <gindibay@microsoft.com>
Date: Tue, 2 Apr 2024 09:14:35 +0300
Subject: [PATCH 71/78] Apply suggestions from code review

Co-authored-by: Onur Tirtir <onurcantirtir@gmail.com>
---
 src/backend/distributed/commands/database.c                | 7 ++-----
 .../commands/non_main_db_distribute_object_ops.c           | 4 ++++
 src/include/distributed/commands.h                         | 1 -
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c
index 9ea6c17ba..9fe576c4b 100644
--- a/src/backend/distributed/commands/database.c
+++ b/src/backend/distributed/commands/database.c
@@ -699,12 +699,8 @@ List *
 PreprocessAlterDatabaseOwnerStmt(Node *node, const char *queryString,
 								 ProcessUtilityContext processUtilityContext)
 {
-	const DistributeObjectOps *ops = GetDistributeObjectOps(node);
-	Assert(ops != NULL);
-
-	if (ops->featureFlag && *ops->featureFlag == false)
+	if (!EnableAlterDatabaseOwner)
 	{
-		/* not propagating when a configured feature flag is turned off by the user */
 		return NIL;
 	}
 
@@ -717,6 +713,7 @@ PreprocessAlterDatabaseOwnerStmt(Node *node, const char *queryString,
 	{
 		return NIL;
 	}
+
 	EnsurePropagationToCoordinator();
 
 	AlterOwnerStmt *stmt = (AlterOwnerStmt *) node;
diff --git a/src/backend/distributed/commands/non_main_db_distribute_object_ops.c b/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
index 9cbe43162..3c4d87017 100644
--- a/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
+++ b/src/backend/distributed/commands/non_main_db_distribute_object_ops.c
@@ -76,6 +76,10 @@ static bool GrantStmtCheckSupportedObjectType(Node *node);
 static bool SecLabelStmtCheckSupportedObjectType(Node *node);
 static bool AlterDbRenameCheckSupportedObjectType(Node *node);
 static bool AlterDbOwnerCheckSupportedObjectType(Node *node);
+
+/*
+ * cannotBeExecutedInTransaction callbacks for OperationArray.
+ */
 static bool CannotBeExecutedInTransaction_True(Node *node);
 static bool CannotBeExecutedInTransaction_False(Node *node);
 static bool AlterDbCannotBeExecutedInTransaction(Node *node);
diff --git a/src/include/distributed/commands.h b/src/include/distributed/commands.h
index 72d4f8c27..64c8d5441 100644
--- a/src/include/distributed/commands.h
+++ b/src/include/distributed/commands.h
@@ -236,7 +236,6 @@ extern List * PreprocessAlterDatabaseRefreshCollStmt(Node *node, const char *que
 													 processUtilityContext);
 extern List * PreprocessAlterDatabaseOwnerStmt(Node *node, const char *queryString,
 											   ProcessUtilityContext processUtilityContext);
-
 extern List * GetDatabaseMetadataSyncCommands(Oid dbOid);
 
 

From 3c4f66a8c403f71b96104f0ca0521923bdde6398 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Tue, 2 Apr 2024 09:25:06 +0300
Subject: [PATCH 72/78] Fixes review comments

---
 src/backend/distributed/commands/database.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c
index 9fe576c4b..9c9b600c7 100644
--- a/src/backend/distributed/commands/database.c
+++ b/src/backend/distributed/commands/database.c
@@ -715,12 +715,14 @@ PreprocessAlterDatabaseOwnerStmt(Node *node, const char *queryString,
 	}
 
 	EnsurePropagationToCoordinator();
-
+	
 	AlterOwnerStmt *stmt = (AlterOwnerStmt *) node;
-	EnsureSequentialMode(stmt->objectType);
-	QualifyTreeNode(node);
 	char *databaseName = strVal((String *) stmt->object);
 	SerializeDistributedDDLsOnObjectClassObject(OCLASS_DATABASE, databaseName);
+
+	EnsureSequentialMode(stmt->objectType);
+	QualifyTreeNode(node);
+
 	const char *sql = DeparseTreeNode((Node *) node);
 
 	List *commands = list_make3(DISABLE_DDL_PROPAGATION,

From 152b46da0a5d60c89034410590024e74bc00f793 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Tue, 2 Apr 2024 09:27:23 +0300
Subject: [PATCH 73/78] Fixes indentation

---
 src/backend/distributed/commands/database.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/backend/distributed/commands/database.c b/src/backend/distributed/commands/database.c
index 9c9b600c7..e6fff6eba 100644
--- a/src/backend/distributed/commands/database.c
+++ b/src/backend/distributed/commands/database.c
@@ -715,7 +715,7 @@ PreprocessAlterDatabaseOwnerStmt(Node *node, const char *queryString,
 	}
 
 	EnsurePropagationToCoordinator();
-	
+
 	AlterOwnerStmt *stmt = (AlterOwnerStmt *) node;
 	char *databaseName = strVal((String *) stmt->object);
 	SerializeDistributedDDLsOnObjectClassObject(OCLASS_DATABASE, databaseName);

From 7b280315c1de9dd00f256f33019140601ff3d5f3 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Tue, 2 Apr 2024 12:49:59 +0300
Subject: [PATCH 74/78] Adds detailed logs

---
 .../alter_database_from_nonmain_db.out        | 404 ++++++++++++++----
 .../sql/alter_database_from_nonmain_db.sql    | 174 ++++++--
 2 files changed, 465 insertions(+), 113 deletions(-)

diff --git a/src/test/regress/expected/alter_database_from_nonmain_db.out b/src/test/regress/expected/alter_database_from_nonmain_db.out
index e5eb85c16..50908c68f 100644
--- a/src/test/regress/expected/alter_database_from_nonmain_db.out
+++ b/src/test/regress/expected/alter_database_from_nonmain_db.out
@@ -28,15 +28,51 @@ DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
 NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+\c regression - - :master_port
+SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
+      node_type       |                                                                                                                                                                                                                                result
+---------------------------------------------------------------------
+ coordinator (local)  | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+ worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+ worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+(3 rows)
+
+\c test_alter_db_from_nonmain_db - - :worker_1_port
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" set tablespace "pg_default";
 NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
 NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+\c regression - - :master_port
+SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
+      node_type       |                                                                                                                                                                                                                                result
+---------------------------------------------------------------------
+ coordinator (local)  | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+ worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+ worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+(3 rows)
+
+\c test_alter_db_from_nonmain_db - - :worker_1_port
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" set tablespace "pg_default";
 NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+\c regression - - :master_port
+SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
+      node_type       |                                                                                                                                                                                                                            result
+---------------------------------------------------------------------
+ coordinator (local)  | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+ worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+ worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+(3 rows)
+
+\c test_alter_db_from_nonmain_db - - :worker_1_port
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
 NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
@@ -65,11 +101,35 @@ NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+\c regression - - :master_port
+SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
+      node_type       |                                                                                                                                                                                                                                result
+---------------------------------------------------------------------
+ coordinator (local)  | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+ worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+ worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+(3 rows)
+
+\c regression - - :worker_1_port
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" set tablespace "pg_default";
 NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+\c regression - - :master_port
+SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
+      node_type       |                                                                                                                                                                                                                            result
+---------------------------------------------------------------------
+ coordinator (local)  | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+ worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+ worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+(3 rows)
+
+\c regression - - :worker_1_port
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
 NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
@@ -118,11 +178,29 @@ NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+\c regression - - :master_port
+SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
+      node_type       |                                                                                                                                                                                                                                result
+---------------------------------------------------------------------
+ coordinator (local)  | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+ worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+ worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+(3 rows)
+
+\c regression - - :worker_2_port
 alter database "altered_database!'2" set tablespace "pg_default";
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+\c regression - - :master_port
+SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
+      node_type       |                                                                                                                                                                                                                            result
+---------------------------------------------------------------------
+ coordinator (local)  | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+ worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+ worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+(3 rows)
+
+\c regression - - :worker_2_port
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
 NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
@@ -171,11 +249,27 @@ NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
+      node_type       |                                                                                                                                                                                                                                result
+---------------------------------------------------------------------
+ coordinator (local)  | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+ worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+ worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+(3 rows)
+
 alter database "altered_database!'2" set tablespace "pg_default";
 NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
+      node_type       |                                                                                                                                                                                                                            result
+---------------------------------------------------------------------
+ coordinator (local)  | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+ worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+ worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+(3 rows)
+
 alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
 NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
@@ -226,6 +320,15 @@ alter database "altered_database!'2" with
     IS_TEMPLATE false;
 NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" WITH ALLOW_CONNECTIONS true CONNECTION LIMIT 0 IS_TEMPLATE false;', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+\c regression - - :master_port
+SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
+      node_type       |                                                                                                                                                                                                                               result
+---------------------------------------------------------------------
+ coordinator (local)  | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": 0, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+ worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": 0, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+ worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": 0, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+(3 rows)
+
 \c regression - - :worker_2_port
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
@@ -237,6 +340,18 @@ NOTICE:  issuing ALTER DATABASE "altered_database!'2" WITH ALLOW_CONNECTIONS fal
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 NOTICE:  issuing ALTER DATABASE "altered_database!'2" WITH ALLOW_CONNECTIONS false CONNECTION LIMIT 1 IS_TEMPLATE true;
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+\c regression - - :master_port
+SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
+      node_type       |                                                                                                                                                                                                                               result
+---------------------------------------------------------------------
+ coordinator (local)  | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": false, "datconnlimit": 1, "daticulocale": null, "datistemplate": true, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+ worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": false, "datconnlimit": 1, "daticulocale": null, "datistemplate": true, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+ worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": false, "datconnlimit": 1, "daticulocale": null, "datistemplate": true, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+(3 rows)
+
+\c regression - - :worker_2_port
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" with
     ALLOW_CONNECTIONS true
     CONNECTION LIMIT 0
@@ -254,6 +369,15 @@ set citus.enable_create_database_propagation=on;
 alter database "altered_database!'2" owner to test_owner_non_main_db;
 NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" OWNER TO test_owner_non_main_db;', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+\c regression - - :master_port
+SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
+      node_type       |                                                                                                                                                                                                                                      result
+---------------------------------------------------------------------
+ coordinator (local)  | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": 0, "daticulocale": null, "datistemplate": false, "database_owner": "test_owner_non_main_db", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+ worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": 0, "daticulocale": null, "datistemplate": false, "database_owner": "test_owner_non_main_db", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+ worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": 0, "daticulocale": null, "datistemplate": false, "database_owner": "test_owner_non_main_db", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+(3 rows)
+
 \c test_alter_db_from_nonmain_db - - :worker_1_port
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
@@ -261,6 +385,15 @@ alter database "altered_database!'2" owner to CURRENT_USER;
 NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" OWNER TO postgres;', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 set default_transaction_read_only = false;
+\c regression - - :master_port
+SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
+      node_type       |                                                                                                                                                                                                                               result
+---------------------------------------------------------------------
+ coordinator (local)  | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": 0, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+ worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": 0, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+ worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": 0, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+(3 rows)
+
 \c regression - - :worker_1_port
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
@@ -270,6 +403,15 @@ DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 NOTICE:  issuing ALTER DATABASE "altered_database!'2" OWNER TO test_owner_non_main_db;
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 set default_transaction_read_only = false;
+\c regression - - :master_port
+SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
+      node_type       |                                                                                                                                                                                                                                      result
+---------------------------------------------------------------------
+ coordinator (local)  | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": 0, "daticulocale": null, "datistemplate": false, "database_owner": "test_owner_non_main_db", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+ worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": 0, "daticulocale": null, "datistemplate": false, "database_owner": "test_owner_non_main_db", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+ worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": 0, "daticulocale": null, "datistemplate": false, "database_owner": "test_owner_non_main_db", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+(3 rows)
+
 \c regression - - :master_port
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
@@ -279,72 +421,176 @@ DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 NOTICE:  issuing ALTER DATABASE "altered_database!'2" OWNER TO postgres;
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 set default_transaction_read_only = false;
-\c test_alter_db_from_nonmain_db - - :worker_2_port
-set citus.log_remote_commands = true;
-set citus.grep_remote_commands = "%ALTER DATABASE%";
-alter database "altered_database!'2" set default_transaction_read_only from current;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET default_transaction_read_only FROM CURRENT', 'postgres')
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-alter database "altered_database!'2" set default_transaction_read_only to DEFAULT;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET default_transaction_read_only TO DEFAULT', 'postgres')
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-alter database "altered_database!'2" RESET default_transaction_read_only;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" RESET default_transaction_read_only', 'postgres')
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-alter database "altered_database!'2" SET TIME ZONE '-7';
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET timezone = ''-7''', 'postgres')
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-alter database "altered_database!'2" set TIME ZONE LOCAL;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET timezone TO DEFAULT', 'postgres')
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+\c regression - - :master_port
+SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
+      node_type       |                                                                                                                                                                                                                               result
+---------------------------------------------------------------------
+ coordinator (local)  | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": 0, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+ worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": 0, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+ worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": 0, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+(3 rows)
+
 \c test_alter_db_from_nonmain_db
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
-alter database "altered_database!'2" set TIME ZONE DEFAULT;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET timezone TO DEFAULT', 'postgres')
+alter database "altered_database!'2" set default_transaction_read_only to true;
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET default_transaction_read_only = ''true''', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-alter database "altered_database!'2" RESET TIME ZONE;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" RESET timezone', 'postgres')
+\c "altered_database!'2" - - :master_port
+select name,setting from pg_settings
+        where name ='default_transaction_read_only';
+             name              | setting
+---------------------------------------------------------------------
+ default_transaction_read_only | on
+(1 row)
+
+\c "altered_database!'2" - - :worker_1_port
+select name,setting from pg_settings
+        where name ='default_transaction_read_only';
+             name              | setting
+---------------------------------------------------------------------
+ default_transaction_read_only | on
+(1 row)
+
+\c "altered_database!'2" - - :worker_2_port
+select name,setting from pg_settings
+        where name ='default_transaction_read_only';
+             name              | setting
+---------------------------------------------------------------------
+ default_transaction_read_only | on
+(1 row)
+
+\c test_alter_db_from_nonmain_db
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
+alter database "altered_database!'2" reset default_transaction_read_only;
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" RESET default_transaction_read_only', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-alter database "altered_database!'2" SET TIME ZONE INTERVAL '-08:00' HOUR TO MINUTE;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET TIME ZONE INTERVAL ''@ 8 hours ago''', 'postgres')
+\c "altered_database!'2" - - :master_port
+select name,setting from pg_settings
+        where name ='default_transaction_read_only';
+             name              | setting
+---------------------------------------------------------------------
+ default_transaction_read_only | off
+(1 row)
+
+\c "altered_database!'2" - - :worker_1_port
+select name,setting from pg_settings
+        where name ='default_transaction_read_only';
+             name              | setting
+---------------------------------------------------------------------
+ default_transaction_read_only | off
+(1 row)
+
+\c "altered_database!'2" - - :worker_2_port
+select name,setting from pg_settings
+        where name ='default_transaction_read_only';
+             name              | setting
+---------------------------------------------------------------------
+ default_transaction_read_only | off
+(1 row)
+
+\c test_alter_db_from_nonmain_db - - :worker_2_port
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
+alter database "altered_database!'2" set log_duration from current;
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET log_duration FROM CURRENT', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-alter database "altered_database!'2" RESET TIME ZONE;
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" RESET timezone', 'postgres')
+alter database "altered_database!'2" set log_duration to DEFAULT;
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET log_duration TO DEFAULT', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-alter database "altered_database!'2" set default_transaction_isolation = 'serializable';
-NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET default_transaction_isolation = ''serializable''', 'postgres')
+alter database "altered_database!'2" set log_duration = true;
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" SET log_duration = ''true''', 'postgres')
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+\c "altered_database!'2" - - :master_port
+select name,setting from pg_settings
+        where name ='log_duration';
+     name     | setting
+---------------------------------------------------------------------
+ log_duration | on
+(1 row)
+
+\c "altered_database!'2" - - :worker_1_port
+select name,setting from pg_settings
+        where name ='log_duration';
+     name     | setting
+---------------------------------------------------------------------
+ log_duration | on
+(1 row)
+
+\c "altered_database!'2" - - :worker_2_port
+select name,setting from pg_settings
+        where name ='log_duration';
+     name     | setting
+---------------------------------------------------------------------
+ log_duration | on
+(1 row)
+
+\c test_alter_db_from_nonmain_db - - :worker_2_port
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
+alter database "altered_database!'2" RESET log_duration;
+NOTICE:  issuing SELECT citus_internal.execute_command_on_remote_nodes_as_user('ALTER DATABASE "altered_database!''2" RESET log_duration', 'postgres')
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+\c "altered_database!'2" - - :master_port
+select name,setting from pg_settings
+        where name ='log_duration';
+     name     | setting
+---------------------------------------------------------------------
+ log_duration | off
+(1 row)
+
+\c "altered_database!'2" - - :worker_1_port
+select name,setting from pg_settings
+        where name ='log_duration';
+     name     | setting
+---------------------------------------------------------------------
+ log_duration | off
+(1 row)
+
+\c "altered_database!'2" - - :worker_2_port
+select name,setting from pg_settings
+        where name ='log_duration';
+     name     | setting
+---------------------------------------------------------------------
+ log_duration | off
+(1 row)
+
 \c regression - - :worker_2_port
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
-set default_transaction_isolation = 'read committed';
-alter database "altered_database!'2" set default_transaction_isolation from current;
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET default_transaction_isolation FROM CURRENT
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET default_transaction_isolation FROM CURRENT
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-alter database "altered_database!'2" set default_transaction_isolation to DEFAULT;
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET default_transaction_isolation TO DEFAULT
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET default_transaction_isolation TO DEFAULT
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-alter database "altered_database!'2" RESET default_transaction_isolation;
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" RESET default_transaction_isolation
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" RESET default_transaction_isolation
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 alter database "altered_database!'2" set statement_timeout = 1000;
 NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET statement_timeout = 1000
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET statement_timeout = 1000
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-set statement_timeout = 2000;
-alter database "altered_database!'2" set statement_timeout from current;
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET statement_timeout FROM CURRENT
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET statement_timeout FROM CURRENT
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+\c "altered_database!'2" - - :master_port
+select name,setting from pg_settings
+        where name ='statement_timeout';
+       name        | setting
+---------------------------------------------------------------------
+ statement_timeout | 1000
+(1 row)
+
+\c "altered_database!'2" - - :worker_1_port
+select name,setting from pg_settings
+        where name ='statement_timeout';
+       name        | setting
+---------------------------------------------------------------------
+ statement_timeout | 1000
+(1 row)
+
+\c "altered_database!'2" - - :worker_2_port
+select name,setting from pg_settings
+        where name ='statement_timeout';
+       name        | setting
+---------------------------------------------------------------------
+ statement_timeout | 1000
+(1 row)
+
+\c regression - - :worker_2_port
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" set statement_timeout to DEFAULT;
 NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET statement_timeout TO DEFAULT
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
@@ -355,32 +601,30 @@ NOTICE:  issuing ALTER DATABASE "altered_database!'2" RESET statement_timeout
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 NOTICE:  issuing ALTER DATABASE "altered_database!'2" RESET statement_timeout
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-alter database "altered_database!'2" set lock_timeout = 1201.5;
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET lock_timeout = 1201.5
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET lock_timeout = 1201.5
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-set lock_timeout = 1202.5;
-alter database "altered_database!'2" set lock_timeout from current;
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET lock_timeout FROM CURRENT
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET lock_timeout FROM CURRENT
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-alter database "altered_database!'2" set lock_timeout to DEFAULT;
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET lock_timeout TO DEFAULT
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET lock_timeout TO DEFAULT
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-alter database "altered_database!'2" RESET lock_timeout;
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" RESET lock_timeout
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" RESET lock_timeout
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-ALTER DATABASE "altered_database!'2" RESET ALL;
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" RESET ALL
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" RESET ALL
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+\c "altered_database!'2" - - :master_port
+select name,setting from pg_settings
+        where name ='statement_timeout';
+       name        | setting
+---------------------------------------------------------------------
+ statement_timeout | 0
+(1 row)
+
+\c "altered_database!'2" - - :worker_1_port
+select name,setting from pg_settings
+        where name ='statement_timeout';
+       name        | setting
+---------------------------------------------------------------------
+ statement_timeout | 0
+(1 row)
+
+\c "altered_database!'2" - - :worker_2_port
+select name,setting from pg_settings
+        where name ='statement_timeout';
+       name        | setting
+---------------------------------------------------------------------
+ statement_timeout | 0
+(1 row)
+
 \c regression
 set citus.enable_create_database_propagation=on;
 drop database "altered_database!'2";
diff --git a/src/test/regress/sql/alter_database_from_nonmain_db.sql b/src/test/regress/sql/alter_database_from_nonmain_db.sql
index c9eb8f57f..453c5961e 100644
--- a/src/test/regress/sql/alter_database_from_nonmain_db.sql
+++ b/src/test/regress/sql/alter_database_from_nonmain_db.sql
@@ -28,9 +28,26 @@ set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" set tablespace "pg_default";
 alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+\c regression - - :master_port
+SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
+\c test_alter_db_from_nonmain_db - - :worker_1_port
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" set tablespace "pg_default";
 alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+
+\c regression - - :master_port
+SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
+\c test_alter_db_from_nonmain_db - - :worker_1_port
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" set tablespace "pg_default";
+
+\c regression - - :master_port
+SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
+\c test_alter_db_from_nonmain_db - - :worker_1_port
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
 alter database "altered_database!'2" set tablespace "pg_default";
 alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
@@ -42,7 +59,17 @@ set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" set tablespace "pg_default";
 alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+\c regression - - :master_port
+SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
+\c regression - - :worker_1_port
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" set tablespace "pg_default";
+\c regression - - :master_port
+SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
+\c regression - - :worker_1_port
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
 alter database "altered_database!'2" set tablespace "pg_default";
 alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
@@ -56,7 +83,15 @@ set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" set tablespace "pg_default";
 alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+\c regression - - :master_port
+SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
+\c regression - - :worker_2_port
 alter database "altered_database!'2" set tablespace "pg_default";
+\c regression - - :master_port
+SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
+\c regression - - :worker_2_port
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
 alter database "altered_database!'2" set tablespace "pg_default";
 alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
@@ -70,7 +105,9 @@ set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" set tablespace "pg_default";
 alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
+SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
 alter database "altered_database!'2" set tablespace "pg_default";
+SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
 alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
 alter database "altered_database!'2" set tablespace "pg_default";
 alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
@@ -79,21 +116,20 @@ alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
 alter database "altered_database!'2" set tablespace "pg_default";
 alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
 
-
-
 \c test_alter_db_from_nonmain_db - - :worker_2_port
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2"  rename to altered_database_renamed;
-
 alter database altered_database_renamed  rename to "altered_database!'2";
 
-
 alter database "altered_database!'2" with
     ALLOW_CONNECTIONS true
     CONNECTION LIMIT 0
     IS_TEMPLATE false;
 
+\c regression - - :master_port
+SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
+
 \c regression - - :worker_2_port
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
@@ -102,16 +138,18 @@ alter database "altered_database!'2" with
     ALLOW_CONNECTIONS false
     CONNECTION LIMIT 1
     IS_TEMPLATE true;
+
+\c regression - - :master_port
+SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
+
+\c regression - - :worker_2_port
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" with
     ALLOW_CONNECTIONS true
     CONNECTION LIMIT 0
     IS_TEMPLATE false;
 
-
-
-
-
-
 \c regression
 create role test_owner_non_main_db;
 
@@ -120,63 +158,133 @@ set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
 set citus.enable_create_database_propagation=on;
 alter database "altered_database!'2" owner to test_owner_non_main_db;
+\c regression - - :master_port
+SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
 
 \c test_alter_db_from_nonmain_db - - :worker_1_port
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" owner to CURRENT_USER;
 set default_transaction_read_only = false;
+\c regression - - :master_port
+SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
 
 \c regression - - :worker_1_port
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" owner to test_owner_non_main_db;
 set default_transaction_read_only = false;
-
+\c regression - - :master_port
+SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
 
 \c regression - - :master_port
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" owner to CURRENT_USER;
 set default_transaction_read_only = false;
+\c regression - - :master_port
+SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
 
 
+\c test_alter_db_from_nonmain_db
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
+
+alter database "altered_database!'2" set default_transaction_read_only to true;
+
+\c "altered_database!'2" - - :master_port
+select name,setting from pg_settings
+        where name ='default_transaction_read_only';
+\c "altered_database!'2" - - :worker_1_port
+select name,setting from pg_settings
+        where name ='default_transaction_read_only';
+\c "altered_database!'2" - - :worker_2_port
+select name,setting from pg_settings
+        where name ='default_transaction_read_only';
+
+\c test_alter_db_from_nonmain_db
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
+alter database "altered_database!'2" reset default_transaction_read_only;
+
+\c "altered_database!'2" - - :master_port
+select name,setting from pg_settings
+        where name ='default_transaction_read_only';
+\c "altered_database!'2" - - :worker_1_port
+select name,setting from pg_settings
+        where name ='default_transaction_read_only';
+\c "altered_database!'2" - - :worker_2_port
+select name,setting from pg_settings
+        where name ='default_transaction_read_only';
+
 
 \c test_alter_db_from_nonmain_db - - :worker_2_port
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
-alter database "altered_database!'2" set default_transaction_read_only from current;
-alter database "altered_database!'2" set default_transaction_read_only to DEFAULT;
-alter database "altered_database!'2" RESET default_transaction_read_only;
-alter database "altered_database!'2" SET TIME ZONE '-7';
-alter database "altered_database!'2" set TIME ZONE LOCAL;
+alter database "altered_database!'2" set log_duration from current;
+alter database "altered_database!'2" set log_duration to DEFAULT;
+alter database "altered_database!'2" set log_duration = true;
+\c "altered_database!'2" - - :master_port
+select name,setting from pg_settings
+        where name ='log_duration';
 
-\c test_alter_db_from_nonmain_db
+\c "altered_database!'2" - - :worker_1_port
+select name,setting from pg_settings
+        where name ='log_duration';
+
+\c "altered_database!'2" - - :worker_2_port
+select name,setting from pg_settings
+        where name ='log_duration';
+
+\c test_alter_db_from_nonmain_db - - :worker_2_port
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
-alter database "altered_database!'2" set TIME ZONE DEFAULT;
-alter database "altered_database!'2" RESET TIME ZONE;
-alter database "altered_database!'2" SET TIME ZONE INTERVAL '-08:00' HOUR TO MINUTE;
-alter database "altered_database!'2" RESET TIME ZONE;
-alter database "altered_database!'2" set default_transaction_isolation = 'serializable';
+alter database "altered_database!'2" RESET log_duration;
+
+
+\c "altered_database!'2" - - :master_port
+select name,setting from pg_settings
+        where name ='log_duration';
+\c "altered_database!'2" - - :worker_1_port
+select name,setting from pg_settings
+        where name ='log_duration';
+\c "altered_database!'2" - - :worker_2_port
+select name,setting from pg_settings
+        where name ='log_duration';
+
+
+
 \c regression - - :worker_2_port
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
-set default_transaction_isolation = 'read committed';
-alter database "altered_database!'2" set default_transaction_isolation from current;
-alter database "altered_database!'2" set default_transaction_isolation to DEFAULT;
-alter database "altered_database!'2" RESET default_transaction_isolation;
 alter database "altered_database!'2" set statement_timeout = 1000;
-set statement_timeout = 2000;
-alter database "altered_database!'2" set statement_timeout from current;
+
+\c "altered_database!'2" - - :master_port
+select name,setting from pg_settings
+        where name ='statement_timeout';
+\c "altered_database!'2" - - :worker_1_port
+select name,setting from pg_settings
+        where name ='statement_timeout';
+\c "altered_database!'2" - - :worker_2_port
+select name,setting from pg_settings
+        where name ='statement_timeout';
+
+
+\c regression - - :worker_2_port
+set citus.log_remote_commands = true;
+set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" set statement_timeout to DEFAULT;
 alter database "altered_database!'2" RESET statement_timeout;
-alter database "altered_database!'2" set lock_timeout = 1201.5;
-set lock_timeout = 1202.5;
-alter database "altered_database!'2" set lock_timeout from current;
-alter database "altered_database!'2" set lock_timeout to DEFAULT;
-alter database "altered_database!'2" RESET lock_timeout;
-ALTER DATABASE "altered_database!'2" RESET ALL;
+
+\c "altered_database!'2" - - :master_port
+select name,setting from pg_settings
+        where name ='statement_timeout';
+\c "altered_database!'2" - - :worker_1_port
+select name,setting from pg_settings
+        where name ='statement_timeout';
+\c "altered_database!'2" - - :worker_2_port
+select name,setting from pg_settings
+        where name ='statement_timeout';
 
 \c regression
 set citus.enable_create_database_propagation=on;

From 5aebb585cb80d6e32f92a93e114437f70c7d86a8 Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Tue, 2 Apr 2024 12:59:00 +0300
Subject: [PATCH 75/78] Fixes review issues

---
 .../alter_database_from_nonmain_db.out        | 21 ++++++++++++++++++-
 .../sql/alter_database_from_nonmain_db.sql    |  9 +++++++-
 2 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/src/test/regress/expected/alter_database_from_nonmain_db.out b/src/test/regress/expected/alter_database_from_nonmain_db.out
index 50908c68f..d38975405 100644
--- a/src/test/regress/expected/alter_database_from_nonmain_db.out
+++ b/src/test/regress/expected/alter_database_from_nonmain_db.out
@@ -421,7 +421,6 @@ DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 NOTICE:  issuing ALTER DATABASE "altered_database!'2" OWNER TO postgres;
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 set default_transaction_read_only = false;
-\c regression - - :master_port
 SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
       node_type       |                                                                                                                                                                                                                               result
 ---------------------------------------------------------------------
@@ -430,6 +429,26 @@ SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER
  worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": 0, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
 (3 rows)
 
+set citus.enable_alter_database_owner to off;
+alter database "altered_database!'2" owner to test_owner_non_main_db;
+SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
+      node_type       |                                                                                                                                                                                                                                      result
+---------------------------------------------------------------------
+ coordinator (local)  | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": 0, "daticulocale": null, "datistemplate": false, "database_owner": "test_owner_non_main_db", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+ worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": 0, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+ worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": 0, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+(3 rows)
+
+alter database "altered_database!'2" owner to CURRENT_USER;
+SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
+      node_type       |                                                                                                                                                                                                                               result
+---------------------------------------------------------------------
+ coordinator (local)  | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": 0, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+ worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": 0, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+ worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": 0, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+(3 rows)
+
+reset citus.enable_alter_database_owner;
 \c test_alter_db_from_nonmain_db
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
diff --git a/src/test/regress/sql/alter_database_from_nonmain_db.sql b/src/test/regress/sql/alter_database_from_nonmain_db.sql
index 453c5961e..4be89112e 100644
--- a/src/test/regress/sql/alter_database_from_nonmain_db.sql
+++ b/src/test/regress/sql/alter_database_from_nonmain_db.sql
@@ -182,9 +182,16 @@ set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" owner to CURRENT_USER;
 set default_transaction_read_only = false;
-\c regression - - :master_port
 SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
 
+set citus.enable_alter_database_owner to off;
+alter database "altered_database!'2" owner to test_owner_non_main_db;
+SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
+alter database "altered_database!'2" owner to CURRENT_USER;
+SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
+reset citus.enable_alter_database_owner;
+
+
 
 \c test_alter_db_from_nonmain_db
 set citus.log_remote_commands = true;

From 0f38a2918483aab9467c4be2e1961424eaf3120c Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Mon, 15 Apr 2024 11:35:13 +0300
Subject: [PATCH 76/78] Fixes revire comments

---
 .../alter_database_from_nonmain_db.out        | 77 +++----------------
 .../sql/alter_database_from_nonmain_db.sql    | 40 ++++------
 2 files changed, 23 insertions(+), 94 deletions(-)

diff --git a/src/test/regress/expected/alter_database_from_nonmain_db.out b/src/test/regress/expected/alter_database_from_nonmain_db.out
index d38975405..937dc9b9d 100644
--- a/src/test/regress/expected/alter_database_from_nonmain_db.out
+++ b/src/test/regress/expected/alter_database_from_nonmain_db.out
@@ -13,6 +13,9 @@ CREATE TABLESPACE "ts-needs\!escape2" LOCATION :'alter_db_tablespace';
 \c - - - :worker_2_port
 \set alter_db_tablespace :abs_srcdir '/tmp_check/ts5'
 CREATE TABLESPACE "ts-needs\!escape2" LOCATION :'alter_db_tablespace';
+-- Below tests test the ALTER DATABASE command from main and non-main databases
+-- The tests are run on the master and worker nodes to ensure that the command is
+-- executed on all nodes.
 \c test_alter_db_from_nonmain_db
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
@@ -55,7 +58,7 @@ SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER
  worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
 (3 rows)
 
-\c test_alter_db_from_nonmain_db - - :worker_1_port
+\c test_alter_db_from_nonmain_db - - :worker_2_port
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" set tablespace "pg_default";
@@ -88,6 +91,8 @@ DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
 NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+-- In the below tests, we test the ALTER DATABASE ..set tablespace command
+-- repeatedly to ensure that the command does not stuck when executed multiple times
 \c regression - - :worker_1_port
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
@@ -101,40 +106,6 @@ NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-\c regression - - :master_port
-SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
-      node_type       |                                                                                                                                                                                                                                result
----------------------------------------------------------------------
- coordinator (local)  | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
- worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
- worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
-(3 rows)
-
-\c regression - - :worker_1_port
-set citus.log_remote_commands = true;
-set citus.grep_remote_commands = "%ALTER DATABASE%";
-alter database "altered_database!'2" set tablespace "pg_default";
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-\c regression - - :master_port
-SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
-      node_type       |                                                                                                                                                                                                                            result
----------------------------------------------------------------------
- coordinator (local)  | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
- worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
- worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
-(3 rows)
-
-\c regression - - :worker_1_port
-set citus.log_remote_commands = true;
-set citus.grep_remote_commands = "%ALTER DATABASE%";
-alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 alter database "altered_database!'2" set tablespace "pg_default";
 NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
@@ -178,34 +149,6 @@ NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-\c regression - - :master_port
-SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
-      node_type       |                                                                                                                                                                                                                                result
----------------------------------------------------------------------
- coordinator (local)  | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
- worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
- worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
-(3 rows)
-
-\c regression - - :worker_2_port
-alter database "altered_database!'2" set tablespace "pg_default";
-\c regression - - :master_port
-SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
-      node_type       |                                                                                                                                                                                                                            result
----------------------------------------------------------------------
- coordinator (local)  | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
- worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
- worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
-(3 rows)
-
-\c regression - - :worker_2_port
-set citus.log_remote_commands = true;
-set citus.grep_remote_commands = "%ALTER DATABASE%";
-alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE "ts-needs\!escape2"
-DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 alter database "altered_database!'2" set tablespace "pg_default";
 NOTICE:  issuing ALTER DATABASE "altered_database!'2" SET TABLESPACE pg_default
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
@@ -329,7 +272,7 @@ SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER
  worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": 0, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
 (3 rows)
 
-\c regression - - :worker_2_port
+\c regression - - :worker_1_port
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" with
@@ -394,7 +337,7 @@ SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER
  worker node (remote) | {"database_properties": {"datacl": null, "datname": "altered_database!'2", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "ts-needs\\!escape2", "daticurules": null, "datallowconn": true, "datconnlimit": 0, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
 (3 rows)
 
-\c regression - - :worker_1_port
+\c regression - - :worker_2_port
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" owner to test_owner_non_main_db;
@@ -545,7 +488,7 @@ select name,setting from pg_settings
  log_duration | on
 (1 row)
 
-\c test_alter_db_from_nonmain_db - - :worker_2_port
+\c test_alter_db_from_nonmain_db - - :worker_1_port
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" RESET log_duration;
@@ -607,7 +550,7 @@ select name,setting from pg_settings
  statement_timeout | 1000
 (1 row)
 
-\c regression - - :worker_2_port
+\c regression - - :worker_1_port
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" set statement_timeout to DEFAULT;
diff --git a/src/test/regress/sql/alter_database_from_nonmain_db.sql b/src/test/regress/sql/alter_database_from_nonmain_db.sql
index 4be89112e..1343fba0f 100644
--- a/src/test/regress/sql/alter_database_from_nonmain_db.sql
+++ b/src/test/regress/sql/alter_database_from_nonmain_db.sql
@@ -17,6 +17,10 @@ CREATE TABLESPACE "ts-needs\!escape2" LOCATION :'alter_db_tablespace';
 \set alter_db_tablespace :abs_srcdir '/tmp_check/ts5'
 CREATE TABLESPACE "ts-needs\!escape2" LOCATION :'alter_db_tablespace';
 
+-- Below tests test the ALTER DATABASE command from main and non-main databases
+-- The tests are run on the master and worker nodes to ensure that the command is
+-- executed on all nodes.
+
 \c test_alter_db_from_nonmain_db
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
@@ -38,7 +42,7 @@ alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
 
 \c regression - - :master_port
 SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
-\c test_alter_db_from_nonmain_db - - :worker_1_port
+\c test_alter_db_from_nonmain_db - - :worker_2_port
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" set tablespace "pg_default";
@@ -54,23 +58,14 @@ alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
 alter database "altered_database!'2" set tablespace "pg_default";
 alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
 
+-- In the below tests, we test the ALTER DATABASE ..set tablespace command
+-- repeatedly to ensure that the command does not stuck when executed multiple times
+
 \c regression - - :worker_1_port
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" set tablespace "pg_default";
 alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
-\c regression - - :master_port
-SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
-\c regression - - :worker_1_port
-set citus.log_remote_commands = true;
-set citus.grep_remote_commands = "%ALTER DATABASE%";
-alter database "altered_database!'2" set tablespace "pg_default";
-\c regression - - :master_port
-SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
-\c regression - - :worker_1_port
-set citus.log_remote_commands = true;
-set citus.grep_remote_commands = "%ALTER DATABASE%";
-alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
 alter database "altered_database!'2" set tablespace "pg_default";
 alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
 alter database "altered_database!'2" set tablespace "pg_default";
@@ -81,18 +76,9 @@ alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
 \c regression - - :worker_2_port
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
+
 alter database "altered_database!'2" set tablespace "pg_default";
 alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
-\c regression - - :master_port
-SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
-\c regression - - :worker_2_port
-alter database "altered_database!'2" set tablespace "pg_default";
-\c regression - - :master_port
-SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
-\c regression - - :worker_2_port
-set citus.log_remote_commands = true;
-set citus.grep_remote_commands = "%ALTER DATABASE%";
-alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
 alter database "altered_database!'2" set tablespace "pg_default";
 alter database "altered_database!'2" set tablespace "ts-needs\!escape2";
 alter database "altered_database!'2" set tablespace "pg_default";
@@ -130,7 +116,7 @@ alter database "altered_database!'2" with
 \c regression - - :master_port
 SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
 
-\c regression - - :worker_2_port
+\c regression - - :worker_1_port
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
 
@@ -169,7 +155,7 @@ set default_transaction_read_only = false;
 \c regression - - :master_port
 SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
 
-\c regression - - :worker_1_port
+\c regression - - :worker_2_port
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" owner to test_owner_non_main_db;
@@ -243,7 +229,7 @@ select name,setting from pg_settings
 select name,setting from pg_settings
         where name ='log_duration';
 
-\c test_alter_db_from_nonmain_db - - :worker_2_port
+\c test_alter_db_from_nonmain_db - - :worker_1_port
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" RESET log_duration;
@@ -277,7 +263,7 @@ select name,setting from pg_settings
         where name ='statement_timeout';
 
 
-\c regression - - :worker_2_port
+\c regression - - :worker_1_port
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" set statement_timeout to DEFAULT;

From 43d3b906bba5f656edd6e16b8eac92861b301dbc Mon Sep 17 00:00:00 2001
From: gurkanindibay <gindibay@microsoft.com>
Date: Fri, 19 Apr 2024 11:00:44 +0300
Subject: [PATCH 77/78] Removes unnecessary empty lines

---
 src/test/regress/sql/alter_database_from_nonmain_db.sql | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/src/test/regress/sql/alter_database_from_nonmain_db.sql b/src/test/regress/sql/alter_database_from_nonmain_db.sql
index 1343fba0f..9d1d1f913 100644
--- a/src/test/regress/sql/alter_database_from_nonmain_db.sql
+++ b/src/test/regress/sql/alter_database_from_nonmain_db.sql
@@ -177,8 +177,6 @@ alter database "altered_database!'2" owner to CURRENT_USER;
 SELECT * FROM public.check_database_on_all_nodes($$altered_database!''2$$) ORDER BY node_type;
 reset citus.enable_alter_database_owner;
 
-
-
 \c test_alter_db_from_nonmain_db
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
@@ -234,7 +232,6 @@ set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
 alter database "altered_database!'2" RESET log_duration;
 
-
 \c "altered_database!'2" - - :master_port
 select name,setting from pg_settings
         where name ='log_duration';
@@ -245,8 +242,6 @@ select name,setting from pg_settings
 select name,setting from pg_settings
         where name ='log_duration';
 
-
-
 \c regression - - :worker_2_port
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";
@@ -262,7 +257,6 @@ select name,setting from pg_settings
 select name,setting from pg_settings
         where name ='statement_timeout';
 
-
 \c regression - - :worker_1_port
 set citus.log_remote_commands = true;
 set citus.grep_remote_commands = "%ALTER DATABASE%";

From 824a1724013f9b2486fe4da1f402d6df66e4589e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=BCrkan=20=C4=B0ndibay?= <gindibay@microsoft.com>
Date: Fri, 19 Apr 2024 11:01:53 +0300
Subject: [PATCH 78/78] Apply suggestions from code review

Co-authored-by: Onur Tirtir <onurcantirtir@gmail.com>
---
 src/test/regress/citus_tests/run_test.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/test/regress/citus_tests/run_test.py b/src/test/regress/citus_tests/run_test.py
index f22a097b0..45a52fe2c 100755
--- a/src/test/regress/citus_tests/run_test.py
+++ b/src/test/regress/citus_tests/run_test.py
@@ -422,9 +422,12 @@ def find_test_schedule_and_line(test_name, args):
 
 def test_dependencies(test_name, test_schedule, schedule_line, args):
     if test_name in DEPS:
-        # since enable_ddl_propagation is a must to execute tests
+        # Since enable_ddl_propagation is a must to execute tests,
         # below block adds enable_ddl_propagation as a dependency
-        # as the first element of extra tests if schedule is not configured
+        # as the first element of extra tests if schedule is not
+        # configured. We don't do so if the schedule is confugured
+        # because all base schedules include enable_ddl_propagation
+        # anyway.
         test_deps = DEPS[test_name]
         ddl_propagation_test = (
             "isolation_enable_ddl_propagation"