Bump version to 7.5.3

In recent postgres builds you cannot set client_min_messages to
values higher then ERROR, if will silently set it to ERROR if so.

During some tests we would set it to fatal to hide random values
(eg. pid's of processes) from the test output. This patch will use
different tactics for hiding these values.

CHANGELOG.md

@@ -1,3 +1,59 @@
+### citus v7.5.3 (November 27, 2018) ###
+
+* Execute SQL tasks using worker_execute_sql_task UDF when using task-tracker
+
+### citus v7.5.2 (November 14, 2018) ###
+
+* Fixes inconsistent metadata error when shard metadata caching get interrupted
+
+* Fixes a bug that could cause memory leak
+
+* Fixes a bug that prevents recovering wrong transactions in MX
+
+* Fixes a bug to prevent wrong memory accesses on Citus MX under very high load
+
+* Fixes crashes caused by stack size increase under high memory load
+
+### citus v7.5.1 (August 28, 2018) ###
+
+* Improves query pushdown planning performance
+
+* Fixes a bug that could cause modifying CTEs to select wrong execution mode
+
+### citus v7.4.2 (July 27, 2018) ###
+
+* Fixes a segfault in real-time executor during online shard move
+
+### citus v7.5.0 (July 25, 2018) ###
+
+* Adds foreign key support from hash distributed to reference tables
+
+* Adds SELECT ... FOR UPDATE support for router plannable queries
+
+* Adds support for non-partition columns in count distinct
+
+* Fixes a segfault in real-time executor during online shard move
+
+* Fixes ALTER TABLE ADD COLUMN constraint check
+
+* Fixes a bug where INSERT ... SELECT was allowed to update distribution column
+
+* Allows DDL commands to be sequentialized via `citus.multi_shard_modify_mode`
+
+* Adds support for topn_union_agg and topn_add_agg across shards
+
+* Adds support for hll_union_agg and hll_add_agg across shards
+
+* Fixes a bug that might cause shards to have a wrong owner
+
+* GUC select_opens_transaction_block defers opening transaction block on workers
+
+* Utils to implement DDLs for policies in future, warn about being unsupported
+
+* Intermediate results use separate connections to avoid interfering with tasks
+
+* Adds a node_conninfo GUC to set outgoing connection settings
+
 ### citus v6.2.6 (July 06, 2018) ###
 
 * Adds support for respecting enable_hashagg in the master planner

Makefile

@@ -113,6 +113,7 @@ OBJS = src/backend/distributed/shared_library_init.o \
      src/backend/distributed/worker/worker_file_access_protocol.o \
      src/backend/distributed/worker/worker_merge_protocol.o \
      src/backend/distributed/worker/worker_partition_protocol.o \
+     src/backend/distributed/worker/worker_sql_task_protocol.o \
      src/backend/distributed/worker/worker_truncate_trigger_protocol.o \
      $(WIN32RES)
 

configure

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for Citus 7.5devel.
+# Generated by GNU Autoconf 2.69 for Citus 7.5.3.
 #
 #
 # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@@ -579,8 +579,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='Citus'
 PACKAGE_TARNAME='citus'
-PACKAGE_VERSION='7.5devel'
-PACKAGE_STRING='Citus 7.5devel'
+PACKAGE_VERSION='7.5.3'
+PACKAGE_STRING='Citus 7.5.3'
 PACKAGE_BUGREPORT=''
 PACKAGE_URL=''
 
@@ -1239,7 +1239,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures Citus 7.5devel to adapt to many kinds of systems.
+\`configure' configures Citus 7.5.3 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1300,7 +1300,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of Citus 7.5devel:";;
+     short | recursive ) echo "Configuration of Citus 7.5.3:";;
    esac
   cat <<\_ACEOF
 
@@ -1400,7 +1400,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-Citus configure 7.5devel
+Citus configure 7.5.3
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -1883,7 +1883,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by Citus $as_me 7.5devel, which was
+It was created by Citus $as_me 7.5.3, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -4701,7 +4701,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by Citus $as_me 7.5devel, which was
+This file was extended by Citus $as_me 7.5.3, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -4763,7 +4763,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-Citus config.status 7.5devel
+Citus config.status 7.5.3
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 

configure.in

@@ -5,7 +5,7 @@
 # everyone needing autoconf installed, the resulting files are checked
 # into the SCM.
 
-AC_INIT([Citus], [7.5devel])
+AC_INIT([Citus], [7.5.3])
 AC_COPYRIGHT([Copyright (c) 2012-2017, Citus Data, Inc.])
 
 # we'll need sed and awk for some of the version commands

src/backend/distributed/commands/multi_copy.c

@@ -1240,7 +1240,8 @@ ReportCopyError(MultiConnection *connection, PGresult *result)
 		bool haveDetail = remoteDetail != NULL;
 
 		ereport(ERROR, (errmsg("%s", remoteMessage),
-						haveDetail ? errdetail("%s", remoteDetail) : 0));
+						haveDetail ? errdetail("%s", ApplyLogRedaction(remoteDetail)) :
+						0));
 	}
 	else
 	{
@@ -1250,7 +1251,7 @@ ReportCopyError(MultiConnection *connection, PGresult *result)
 		ereport(ERROR, (errcode(ERRCODE_IO_ERROR),
 						errmsg("failed to complete COPY on %s:%d", connection->hostname,
 							   connection->port),
-						errdetail("%s", remoteMessage)));
+						errdetail("%s", ApplyLogRedaction(remoteMessage))));
 	}
 }
 

src/backend/distributed/connection/connection_management.c

@@ -22,6 +22,7 @@
 #include "access/hash.h"
 #include "commands/dbcommands.h"
 #include "distributed/connection_management.h"
+#include "distributed/errormessage.h"
 #include "distributed/metadata_cache.h"
 #include "distributed/hash_helpers.h"
 #include "distributed/placement_connection.h"
@@ -824,9 +825,9 @@ DefaultCitusNoticeProcessor(void *arg, const char *message)
 	char *trimmedMessage = TrimLogLevel(message);
 	char *level = strtok((char *) message, ":");
 
-	ereport(CitusNoticeLogLevel, (errmsg("%s", trimmedMessage),
-								  errdetail("%s from %s:%d",
-											level, nodeName, nodePort)));
+	ereport(CitusNoticeLogLevel,
+			(errmsg("%s", ApplyLogRedaction(trimmedMessage)),
+			 errdetail("%s from %s:%d", level, nodeName, nodePort)));
 }
 
 

src/backend/distributed/connection/placement_connection.c

@@ -311,7 +311,8 @@ StartPlacementListConnection(uint32 flags, List *placementAccessList,
 		 * No suitable connection in the placement->connection mapping, get one from
 		 * the node->connection pool.
 		 */
-		chosenConnection = StartNodeConnection(flags, nodeName, nodePort);
+		chosenConnection = StartNodeUserDatabaseConnection(flags, nodeName, nodePort,
+														   userName, NULL);
 
 		if (flags & CONNECTION_PER_PLACEMENT &&
 			ConnectionAccessedDifferentPlacement(chosenConnection, placement))
@@ -328,8 +329,10 @@ StartPlacementListConnection(uint32 flags, List *placementAccessList,
 			 * ID as the current placement, then we'd no longer able to write to
 			 * placement B later in the COPY.
 			 */
-			chosenConnection = StartNodeConnection(flags | FORCE_NEW_CONNECTION, nodeName,
-												   nodePort);
+			chosenConnection = StartNodeUserDatabaseConnection(flags |
+															   FORCE_NEW_CONNECTION,
+															   nodeName, nodePort,
+															   userName, NULL);
 
 			Assert(!ConnectionAccessedDifferentPlacement(chosenConnection, placement));
 		}

src/backend/distributed/connection/remote_commands.c

@@ -14,6 +14,7 @@
 #include "libpq-fe.h"
 
 #include "distributed/connection_management.h"
+#include "distributed/errormessage.h"
 #include "distributed/remote_commands.h"
 #include "lib/stringinfo.h"
 #include "miscadmin.h"
@@ -253,7 +254,8 @@ ReportConnectionError(MultiConnection *connection, int elevel)
 
 	ereport(elevel, (errcode(ERRCODE_CONNECTION_FAILURE),
 					 errmsg("connection error: %s:%d", nodeName, nodePort),
-					 messageDetail != NULL ? errdetail("%s", messageDetail) : 0));
+					 messageDetail != NULL ?
+					 errdetail("%s", ApplyLogRedaction(messageDetail)) : 0));
 }
 
 
@@ -295,7 +297,8 @@ ReportResultError(MultiConnection *connection, PGresult *result, int elevel)
 		}
 
 		ereport(elevel, (errcode(sqlState), errmsg("%s", messagePrimary),
-						 messageDetail ? errdetail("%s", messageDetail) : 0,
+						 messageDetail ?
+						 errdetail("%s", ApplyLogRedaction(messageDetail)) : 0,
 						 messageHint ? errhint("%s", messageHint) : 0,
 						 messageContext ? errcontext("%s", messageContext) : 0,
 						 errcontext("while executing command on %s:%d",
@@ -344,7 +347,7 @@ LogRemoteCommand(MultiConnection *connection, const char *command)
 		return;
 	}
 
-	ereport(LOG, (errmsg("issuing %s", command),
+	ereport(LOG, (errmsg("issuing %s", ApplyLogRedaction(command)),
 				  errdetail("on server %s:%d", connection->hostname, connection->port)));
 }
 

src/backend/distributed/executor/citus_custom_scan.c

@@ -88,19 +88,10 @@ static CustomExecMethods TaskTrackerCustomExecMethods = {
 	.ExplainCustomScan = CitusExplainScan
 };
 
-static CustomExecMethods RouterSequentialModifyCustomExecMethods = {
-	.CustomName = "RouterSequentialModifyScan",
+static CustomExecMethods RouterModifyCustomExecMethods = {
+	.CustomName = "RouterModifyScan",
 	.BeginCustomScan = CitusModifyBeginScan,
-	.ExecCustomScan = RouterSequentialModifyExecScan,
-	.EndCustomScan = CitusEndScan,
-	.ReScanCustomScan = CitusReScan,
-	.ExplainCustomScan = CitusExplainScan
-};
-
-static CustomExecMethods RouterMultiModifyCustomExecMethods = {
-	.CustomName = "RouterMultiModifyScan",
-	.BeginCustomScan = CitusModifyBeginScan,
-	.ExecCustomScan = RouterMultiModifyExecScan,
+	.ExecCustomScan = RouterModifyExecScan,
 	.EndCustomScan = CitusEndScan,
 	.ReScanCustomScan = CitusReScan,
 	.ExplainCustomScan = CitusExplainScan
@@ -187,6 +178,8 @@ RouterCreateScan(CustomScan *scan)
 	List *taskList = NIL;
 	bool isModificationQuery = false;
 
+	List *relationRowLockList = NIL;
+
 	scanState->executorType = MULTI_EXECUTOR_ROUTER;
 	scanState->customScanState.ss.ps.type = T_CustomScanState;
 	scanState->distributedPlan = GetDistributedPlan(scan);
@@ -194,47 +187,22 @@ RouterCreateScan(CustomScan *scan)
 	distributedPlan = scanState->distributedPlan;
 	workerJob = distributedPlan->workerJob;
 	taskList = workerJob->taskList;
-
 	isModificationQuery = IsModifyDistributedPlan(distributedPlan);
 
-	/* check whether query has at most one shard */
-	if (list_length(taskList) <= 1)
+	if (list_length(taskList) == 1)
 	{
-		List *relationRowLockList = NIL;
-		if (list_length(taskList) == 1)
-		{
-			Task *task = (Task *) linitial(taskList);
-			relationRowLockList = task->relationRowLockList;
-		}
+		Task *task = (Task *) linitial(taskList);
+		relationRowLockList = task->relationRowLockList;
+	}
 
-		/* if query is SELECT ... FOR UPDATE query, use modify logic */
-		if (isModificationQuery || relationRowLockList != NIL)
-		{
-			scanState->customScanState.methods = &RouterSequentialModifyCustomExecMethods;
-		}
-		else
-		{
-			scanState->customScanState.methods = &RouterSelectCustomExecMethods;
-		}
+	/* if query is SELECT ... FOR UPDATE query, use modify logic */
+	if (isModificationQuery || relationRowLockList != NIL)
+	{
+		scanState->customScanState.methods = &RouterModifyCustomExecMethods;
 	}
 	else
 	{
-		Assert(isModificationQuery);
-
-		if (IsMultiRowInsert(workerJob->jobQuery) ||
-			MultiShardConnectionType == SEQUENTIAL_CONNECTION)
-		{
-			/*
-			 * Multi shard modifications while multi_shard_modify_mode equals
-			 * to 'sequential' or Multi-row INSERT are executed sequentially
-			 * instead of using parallel connections.
-			 */
-			scanState->customScanState.methods = &RouterSequentialModifyCustomExecMethods;
-		}
-		else
-		{
-			scanState->customScanState.methods = &RouterMultiModifyCustomExecMethods;
-		}
+		scanState->customScanState.methods = &RouterSelectCustomExecMethods;
 	}
 
 	return (Node *) scanState;

src/backend/distributed/executor/multi_client_executor.c

@@ -294,6 +294,14 @@ MultiClientConnectPoll(int32 connectionId)
 MultiConnection *
 MultiClientGetConnection(int32 connectionId)
 {
+	if (connectionId == INVALID_CONNECTION_ID)
+	{
+		return NULL;
+	}
+
+	Assert(connectionId >= 0);
+	Assert(connectionId < MAX_CONNECTION_COUNT);
+
 	return ClientConnectionArray[connectionId];
 }
 
@@ -411,8 +419,17 @@ MultiClientSendQuery(int32 connectionId, const char *query)
 	if (querySent == 0)
 	{
 		char *errorMessage = pchomp(PQerrorMessage(connection->pgConn));
-		ereport(WARNING, (errmsg("could not send remote query \"%s\"", query),
-						  errdetail("Client error: %s", errorMessage)));
+
+		/*
+		 * query might include the user query coming from the taskTracker
+		 * code path, that's why we hash it, too. Otherwise, this code
+		 * path is generally exercised for the kind of errors that
+		 * we cannot send the queries that Citus itself produced.
+		 */
+		ereport(WARNING, (errmsg("could not send remote query \"%s\"",
+								 ApplyLogRedaction(query)),
+						  errdetail("Client error: %s",
+									ApplyLogRedaction(errorMessage))));
 
 		success = false;
 	}

src/backend/distributed/executor/multi_executor.c

@@ -226,6 +226,18 @@ StubRelation(TupleDesc tupleDescriptor)
 void
 ExecuteQueryStringIntoDestReceiver(const char *queryString, ParamListInfo params,
 								   DestReceiver *dest)
+{
+	Query *query = ParseQueryString(queryString);
+
+	ExecuteQueryIntoDestReceiver(query, params, dest);
+}
+
+
+/*
+ * ParseQuery parses query string and returns a Query struct.
+ */
+Query *
+ParseQueryString(const char *queryString)
 {
 	Query *query = NULL;
 
@@ -244,7 +256,7 @@ ExecuteQueryStringIntoDestReceiver(const char *queryString, ParamListInfo params
 
 	query = (Query *) linitial(queryTreeList);
 
-	ExecuteQueryIntoDestReceiver(query, params, dest);
+	return query;
 }
 
 

src/backend/distributed/executor/multi_real_time_executor.c

@@ -425,6 +425,7 @@ ManageTaskExecution(Task *task, TaskExecution *taskExecution,
 		case EXEC_TASK_FAILED:
 		{
 			bool raiseError = false;
+			bool isCritical = false;
 
 			/*
 			 * On task failure, we close the connection. We also reset our execution
@@ -434,7 +435,35 @@ ManageTaskExecution(Task *task, TaskExecution *taskExecution,
 			 */
 			int32 connectionId = connectionIdArray[currentIndex];
 			MultiConnection *connection = MultiClientGetConnection(connectionId);
-			bool isCritical = connection->remoteTransaction.transactionCritical;
+
+			/* next time we try this worker node, start from the beginning */
+			taskStatusArray[currentIndex] = EXEC_TASK_CONNECT_START;
+
+			/* try next worker node */
+			AdjustStateForFailure(taskExecution);
+
+			/*
+			 * Add a delay in MultiClientWait, to avoid potentially excerbating problems
+			 * by looping quickly
+			 */
+			*executionStatus = TASK_STATUS_ERROR;
+
+			if (connection == NULL)
+			{
+				/*
+				 * The task failed before we even managed to connect. This happens when
+				 * the metadata is out of sync due to a rebalance. It may be that only
+				 * one placement was moved, in that case the other one might still work.
+				 */
+				break;
+			}
+
+			isCritical = connection->remoteTransaction.transactionCritical;
+			if (isCritical)
+			{
+				/* cannot recover when error occurs in a critical transaction */
+				taskExecution->criticalErrorOccurred = true;
+			}
 
 			/*
 			 * Mark the connection as failed in case it was already used to perform
@@ -447,27 +476,9 @@ ManageTaskExecution(Task *task, TaskExecution *taskExecution,
 
 			MultiClientDisconnect(connectionId);
 			connectionIdArray[currentIndex] = INVALID_CONNECTION_ID;
+
 			connectAction = CONNECT_ACTION_CLOSED;
 
-			taskStatusArray[currentIndex] = EXEC_TASK_CONNECT_START;
-
-			if (isCritical)
-			{
-				/* cannot recover when error occurs in a critical transaction */
-				taskExecution->criticalErrorOccurred = true;
-			}
-			else
-			{
-				/* try next worker node */
-				AdjustStateForFailure(taskExecution);
-			}
-
-			/*
-			 * Add a delay, to avoid potentially excerbating problems by
-			 * looping quickly
-			 */
-			*executionStatus = TASK_STATUS_ERROR;
-
 			break;
 		}
 

src/backend/distributed/executor/multi_router_executor.c

@@ -96,6 +96,8 @@ static int64 ExecuteModifyTasks(List *taskList, bool expectResults,
 static void AcquireExecutorShardLock(Task *task, CmdType commandType);
 static void AcquireExecutorMultiShardLocks(List *taskList);
 static bool RequiresConsistentSnapshot(Task *task);
+static void RouterMultiModifyExecScan(CustomScanState *node);
+static void RouterSequentialModifyExecScan(CustomScanState *node);
 static void ExtractParametersFromParamListInfo(ParamListInfo paramListInfo,
 											   Oid **parameterTypes,
 											   const char ***parameterValues);
@@ -534,12 +536,15 @@ CitusModifyBeginScan(CustomScanState *node, EState *estate, int eflags)
 
 
 /*
- * RouterSequentialModifyExecScan executes 0 or more modifications on a
- * distributed table sequentially and returns results if there are any.
- * Note that we also use this path for SELECT ... FOR UPDATE queries.
+ * RouterModifyExecScan executes a list of tasks on remote nodes, retrieves
+ * the results and, if RETURNING is used or SELECT FOR UPDATE executed,
+ * returns the results with a TupleTableSlot.
+ *
+ * The function can handle both single task query executions,
+ * sequential or parallel multi-task query executions.
  */
 TupleTableSlot *
-RouterSequentialModifyExecScan(CustomScanState *node)
+RouterModifyExecScan(CustomScanState *node)
 {
 	CitusScanState *scanState = (CitusScanState *) node;
 	TupleTableSlot *resultSlot = NULL;
@@ -547,63 +552,26 @@ RouterSequentialModifyExecScan(CustomScanState *node)
 	if (!scanState->finishedRemoteScan)
 	{
 		DistributedPlan *distributedPlan = scanState->distributedPlan;
-		bool hasReturning = distributedPlan->hasReturning;
 		Job *workerJob = distributedPlan->workerJob;
 		List *taskList = workerJob->taskList;
-		ListCell *taskCell = NULL;
-		bool multipleTasks = list_length(taskList) > 1;
-		EState *executorState = scanState->customScanState.ss.ps.state;
-		bool taskListRequires2PC = TaskListRequires2PC(taskList);
-		bool alwaysThrowErrorOnFailure = false;
-		CmdType operation = scanState->distributedPlan->operation;
-
-		/*
-		 * We could naturally handle function-based transactions (i.e. those using
-		 * PL/pgSQL or similar) by checking the type of queryDesc->dest, but some
-		 * customers already use functions that touch multiple shards from within
-		 * a function, so we'll ignore functions for now.
-		 */
-		if (IsTransactionBlock() || multipleTasks || taskListRequires2PC)
-		{
-			BeginOrContinueCoordinatedTransaction();
-
-			/*
-			 * Although using two phase commit protocol is an independent decision than
-			 * failing on any error, we prefer to couple them. Our motivation is that
-			 * the failures are rare, and we prefer to avoid marking placements invalid
-			 * in case of failures.
-			 *
-			 * For reference tables, we always set alwaysThrowErrorOnFailure since we
-			 * absolutely want to avoid marking any placements invalid.
-			 *
-			 * We also cannot handle failures when there is RETURNING and there are more
-			 * than one task to execute.
-			 */
-			if (taskListRequires2PC)
-			{
-				CoordinatedTransactionUse2PC();
-
-				alwaysThrowErrorOnFailure = true;
-			}
-			else if (multipleTasks && hasReturning)
-			{
-				alwaysThrowErrorOnFailure = true;
-			}
-		}
+		bool parallelExecution = true;
 
 		ExecuteSubPlans(distributedPlan);
 
-		foreach(taskCell, taskList)
+		if (list_length(taskList) <= 1 ||
+			IsMultiRowInsert(workerJob->jobQuery) ||
+			MultiShardConnectionType == SEQUENTIAL_CONNECTION)
 		{
-			Task *task = (Task *) lfirst(taskCell);
+			parallelExecution = false;
+		}
 
-			/*
-			 * Result is expected for SELECT ... FOR UPDATE queries as well.
-			 */
-			executorState->es_processed +=
-				ExecuteSingleModifyTask(scanState, task, operation,
-										alwaysThrowErrorOnFailure,
-										hasReturning || task->relationRowLockList != NIL);
+		if (parallelExecution)
+		{
+			RouterMultiModifyExecScan(node);
+		}
+		else
+		{
+			RouterSequentialModifyExecScan(node);
 		}
 
 		scanState->finishedRemoteScan = true;
@@ -615,6 +583,75 @@ RouterSequentialModifyExecScan(CustomScanState *node)
 }
 
 
+/*
+ * RouterSequentialModifyExecScan executes 0 or more modifications on a
+ * distributed table sequentially and stores them in custom scan's tuple
+ * store. Note that we also use this path for SELECT ... FOR UPDATE queries.
+ */
+static void
+RouterSequentialModifyExecScan(CustomScanState *node)
+{
+	CitusScanState *scanState = (CitusScanState *) node;
+	DistributedPlan *distributedPlan = scanState->distributedPlan;
+	bool hasReturning = distributedPlan->hasReturning;
+	Job *workerJob = distributedPlan->workerJob;
+	List *taskList = workerJob->taskList;
+	ListCell *taskCell = NULL;
+	bool multipleTasks = list_length(taskList) > 1;
+	EState *executorState = scanState->customScanState.ss.ps.state;
+	bool taskListRequires2PC = TaskListRequires2PC(taskList);
+	bool alwaysThrowErrorOnFailure = false;
+	CmdType operation = scanState->distributedPlan->operation;
+
+	Assert(!scanState->finishedRemoteScan);
+
+	/*
+	 * We could naturally handle function-based transactions (i.e. those using
+	 * PL/pgSQL or similar) by checking the type of queryDesc->dest, but some
+	 * customers already use functions that touch multiple shards from within
+	 * a function, so we'll ignore functions for now.
+	 */
+	if (IsTransactionBlock() || multipleTasks || taskListRequires2PC)
+	{
+		BeginOrContinueCoordinatedTransaction();
+
+		/*
+		 * Although using two phase commit protocol is an independent decision than
+		 * failing on any error, we prefer to couple them. Our motivation is that
+		 * the failures are rare, and we prefer to avoid marking placements invalid
+		 * in case of failures.
+		 *
+		 * For reference tables, we always set alwaysThrowErrorOnFailure since we
+		 * absolutely want to avoid marking any placements invalid.
+		 *
+		 * We also cannot handle failures when there is RETURNING and there are more
+		 * than one task to execute.
+		 */
+		if (taskListRequires2PC)
+		{
+			CoordinatedTransactionUse2PC();
+
+			alwaysThrowErrorOnFailure = true;
+		}
+		else if (multipleTasks && hasReturning)
+		{
+			alwaysThrowErrorOnFailure = true;
+		}
+	}
+
+
+	foreach(taskCell, taskList)
+	{
+		Task *task = (Task *) lfirst(taskCell);
+		bool expectResults = (hasReturning || task->relationRowLockList != NIL);
+
+		executorState->es_processed +=
+			ExecuteSingleModifyTask(scanState, task, operation,
+									alwaysThrowErrorOnFailure, expectResults);
+	}
+}
+
+
 /*
  * TaskListRequires2PC determines whether the given task list requires 2PC
  * because the tasks provided operates on a reference table or there are multiple
@@ -667,31 +704,20 @@ TaskListRequires2PC(List *taskList)
 /*
  * RouterMultiModifyExecScan executes a list of tasks on remote nodes, retrieves
  * the results and, if RETURNING is used, stores them in custom scan's tuple store.
- * Then, it returns tuples one by one from this tuple store.
  */
-TupleTableSlot *
+static void
 RouterMultiModifyExecScan(CustomScanState *node)
 {
 	CitusScanState *scanState = (CitusScanState *) node;
-	TupleTableSlot *resultSlot = NULL;
+	DistributedPlan *distributedPlan = scanState->distributedPlan;
+	Job *workerJob = distributedPlan->workerJob;
+	List *taskList = workerJob->taskList;
+	bool hasReturning = distributedPlan->hasReturning;
+	bool isModificationQuery = true;
 
-	if (!scanState->finishedRemoteScan)
-	{
-		DistributedPlan *distributedPlan = scanState->distributedPlan;
-		Job *workerJob = distributedPlan->workerJob;
-		List *taskList = workerJob->taskList;
-		bool hasReturning = distributedPlan->hasReturning;
-		bool isModificationQuery = true;
+	Assert(!scanState->finishedRemoteScan);
 
-		ExecuteSubPlans(distributedPlan);
-		ExecuteMultipleTasks(scanState, taskList, isModificationQuery, hasReturning);
-
-		scanState->finishedRemoteScan = true;
-	}
-
-	resultSlot = ReturnTupleFromTuplestore(scanState);
-
-	return resultSlot;
+	ExecuteMultipleTasks(scanState, taskList, isModificationQuery, hasReturning);
 }
 
 

src/backend/distributed/executor/multi_server_executor.c

@@ -67,7 +67,7 @@ JobExecutorType(DistributedPlan *distributedPlan)
 
 				ereport(DEBUG2, (errmsg("Plan is router executable"),
 								 errdetail("distribution column value: %s",
-										   partitionColumnString)));
+										   ApplyLogRedaction(partitionColumnString))));
 			}
 			else
 			{

src/backend/distributed/executor/multi_utility.c

@@ -112,10 +112,11 @@ static bool IsCitusExtensionStmt(Node *parsetree);
 static bool IsTransmitStmt(Node *parsetree);
 static void VerifyTransmitStmt(CopyStmt *copyStatement);
 static bool IsCopyResultStmt(CopyStmt *copyStatement);
+static bool CopyStatementHasFormat(CopyStmt *copyStatement, char *formatName);
 
 /* Local functions forward declarations for processing distributed table commands */
 static Node * ProcessCopyStmt(CopyStmt *copyStatement, char *completionTag,
-							  bool *commandMustRunAsOwner);
+							  const char *queryString);
 static void ProcessCreateTableStmtPartitionOf(CreateStmt *createStatement);
 static void ProcessAlterTableStmtAttachPartition(AlterTableStmt *alterTableStatement);
 static List * PlanIndexStmt(IndexStmt *createIndexStatement,
@@ -164,7 +165,6 @@ static List * InterShardDDLTaskList(Oid leftRelationId, Oid rightRelationId,
 									const char *commandString);
 static void RangeVarCallbackForDropIndex(const RangeVar *rel, Oid relOid, Oid oldRelOid,
 										 void *arg);
-static void CheckCopyPermissions(CopyStmt *copyStatement);
 static List * CopyGetAttnums(TupleDesc tupDesc, Relation rel, List *attnamelist);
 static void PostProcessUtility(Node *parsetree);
 static List * CollectGrantTableIdList(GrantStmt *grantStmt);
@@ -173,6 +173,8 @@ static void ProcessDropTableStmt(DropStmt *dropTableStatement);
 static void ProcessDropSchemaStmt(DropStmt *dropSchemaStatement);
 static void InvalidateForeignKeyGraphForDDL(void);
 
+static void ErrorUnsupportedAlterTableAddColumn(Oid relationId, AlterTableCmd *command,
+												Constraint *constraint);
 
 /*
  * We need to run some of the commands sequentially if there is a foreign constraint
@@ -243,9 +245,6 @@ multi_ProcessUtility(PlannedStmt *pstmt,
 					 char *completionTag)
 {
 	Node *parsetree = pstmt->utilityStmt;
-	bool commandMustRunAsOwner = false;
-	Oid savedUserId = InvalidOid;
-	int savedSecurityContext = 0;
 	List *ddlJobs = NIL;
 	bool checkExtensionVersion = false;
 
@@ -324,7 +323,7 @@ multi_ProcessUtility(PlannedStmt *pstmt,
 
 		parsetree = copyObject(parsetree);
 		parsetree = ProcessCopyStmt((CopyStmt *) parsetree, completionTag,
-									&commandMustRunAsOwner);
+									queryString);
 
 		previousContext = MemoryContextSwitchTo(planContext);
 		parsetree = copyObject(parsetree);
@@ -510,13 +509,6 @@ multi_ProcessUtility(PlannedStmt *pstmt,
 		StopMaintenanceDaemon(databaseOid);
 	}
 
-	/* set user if needed and go ahead and run local utility using standard hook */
-	if (commandMustRunAsOwner)
-	{
-		GetUserIdAndSecContext(&savedUserId, &savedSecurityContext);
-		SetUserIdAndSecContext(CitusExtensionOwner(), SECURITY_LOCAL_USERID_CHANGE);
-	}
-
 #if (PG_VERSION_NUM >= 100000)
 	pstmt->utilityStmt = parsetree;
 	standard_ProcessUtility(pstmt, queryString, context,
@@ -555,11 +547,6 @@ multi_ProcessUtility(PlannedStmt *pstmt,
 		PostProcessUtility(parsetree);
 	}
 
-	if (commandMustRunAsOwner)
-	{
-		SetUserIdAndSecContext(savedUserId, savedSecurityContext);
-	}
-
 	/*
 	 * Re-forming the foreign key graph relies on the command being executed
 	 * on the local table first. However, in order to decide whether the
@@ -613,11 +600,47 @@ multi_ProcessUtility(PlannedStmt *pstmt,
 					}
 
 					constraint = (Constraint *) command->def;
-					if (ConstraintIsAForeignKey(constraint->conname, relationId))
+					if (constraint->contype == CONSTR_FOREIGN)
 					{
 						InvalidateForeignKeyGraph();
 					}
 				}
+				else if (alterTableType == AT_AddColumn)
+				{
+					List *columnConstraints = NIL;
+					ListCell *columnConstraint = NULL;
+					Oid relationId = InvalidOid;
+					LOCKMODE lockmode = NoLock;
+
+					ColumnDef *columnDefinition = (ColumnDef *) command->def;
+					columnConstraints = columnDefinition->constraints;
+					if (columnConstraints)
+					{
+						ErrorIfUnsupportedAlterAddConstraintStmt(alterTableStatement);
+					}
+
+					lockmode = AlterTableGetLockLevel(alterTableStatement->cmds);
+					relationId = AlterTableLookupRelation(alterTableStatement, lockmode);
+					if (!OidIsValid(relationId))
+					{
+						continue;
+					}
+
+					foreach(columnConstraint, columnConstraints)
+					{
+						Constraint *constraint = (Constraint *) lfirst(columnConstraint);
+
+						if (constraint->conname == NULL &&
+							(constraint->contype == CONSTR_PRIMARY ||
+							 constraint->contype == CONSTR_UNIQUE ||
+							 constraint->contype == CONSTR_FOREIGN ||
+							 constraint->contype == CONSTR_CHECK))
+						{
+							ErrorUnsupportedAlterTableAddColumn(relationId, command,
+																constraint);
+						}
+					}
+				}
 			}
 		}
 
@@ -676,6 +699,89 @@ multi_ProcessUtility(PlannedStmt *pstmt,
 }
 
 
+static void
+ErrorUnsupportedAlterTableAddColumn(Oid relationId, AlterTableCmd *command,
+									Constraint *constraint)
+{
+	ColumnDef *columnDefinition = (ColumnDef *) command->def;
+	char *colName = columnDefinition->colname;
+	char *errMsg =
+		"cannot execute ADD COLUMN command with PRIMARY KEY, UNIQUE, FOREIGN and CHECK constraints";
+	StringInfo errHint = makeStringInfo();
+	appendStringInfo(errHint, "You can issue each command separately such as ");
+	appendStringInfo(errHint,
+					 "ALTER TABLE %s ADD COLUMN %s data_type; ALTER TABLE %s ADD CONSTRAINT constraint_name ",
+					 get_rel_name(relationId),
+					 colName, get_rel_name(relationId));
+
+	if (constraint->contype == CONSTR_UNIQUE)
+	{
+		appendStringInfo(errHint, "UNIQUE (%s)", colName);
+	}
+	else if (constraint->contype == CONSTR_PRIMARY)
+	{
+		appendStringInfo(errHint, "PRIMARY KEY (%s)", colName);
+	}
+	else if (constraint->contype == CONSTR_CHECK)
+	{
+		appendStringInfo(errHint, "CHECK (check_expression)");
+	}
+	else if (constraint->contype == CONSTR_FOREIGN)
+	{
+		RangeVar *referencedTable = constraint->pktable;
+		char *referencedColumn = strVal(lfirst(list_head(constraint->pk_attrs)));
+		Oid referencedRelationId = RangeVarGetRelid(referencedTable, NoLock, false);
+
+		appendStringInfo(errHint, "FOREIGN KEY (%s) REFERENCES %s(%s)", colName,
+						 get_rel_name(referencedRelationId), referencedColumn);
+
+		if (constraint->fk_del_action == FKCONSTR_ACTION_SETNULL)
+		{
+			appendStringInfo(errHint, " %s", "ON DELETE SET NULL");
+		}
+		else if (constraint->fk_del_action == FKCONSTR_ACTION_CASCADE)
+		{
+			appendStringInfo(errHint, " %s", "ON DELETE CASCADE");
+		}
+		else if (constraint->fk_del_action == FKCONSTR_ACTION_SETDEFAULT)
+		{
+			appendStringInfo(errHint, " %s", "ON DELETE SET DEFAULT");
+		}
+		else if (constraint->fk_del_action == FKCONSTR_ACTION_RESTRICT)
+		{
+			appendStringInfo(errHint, " %s", "ON DELETE RESTRICT");
+		}
+
+		if (constraint->fk_upd_action == FKCONSTR_ACTION_SETNULL)
+		{
+			appendStringInfo(errHint, " %s", "ON UPDATE SET NULL");
+		}
+		else if (constraint->fk_upd_action == FKCONSTR_ACTION_CASCADE)
+		{
+			appendStringInfo(errHint, " %s", "ON UPDATE CASCADE");
+		}
+		else if (constraint->fk_upd_action == FKCONSTR_ACTION_SETDEFAULT)
+		{
+			appendStringInfo(errHint, " %s", "ON UPDATE SET DEFAULT");
+		}
+		else if (constraint->fk_upd_action == FKCONSTR_ACTION_RESTRICT)
+		{
+			appendStringInfo(errHint, " %s", "ON UPDATE RESTRICT");
+		}
+	}
+
+	appendStringInfo(errHint, "%s", ";");
+
+	ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+					errmsg("%s", errMsg),
+					errhint("%s", errHint->data),
+					errdetail("Adding a column with a constraint in "
+							  "one command is not supported because "
+							  "all constraints in Citus must have "
+							  "explicit names")));
+}
+
+
 /*
  * InvalidateForeignKeyGraphForDDL simply keeps track of whether
  * the foreign key graph should be invalidated due to a DDL.
@@ -802,9 +908,20 @@ VerifyTransmitStmt(CopyStmt *copyStatement)
  */
 static bool
 IsCopyResultStmt(CopyStmt *copyStatement)
+{
+	return CopyStatementHasFormat(copyStatement, "result");
+}
+
+
+/*
+ * CopyStatementHasFormat checks whether the COPY statement has the given
+ * format.
+ */
+static bool
+CopyStatementHasFormat(CopyStmt *copyStatement, char *formatName)
 {
 	ListCell *optionCell = NULL;
-	bool hasFormatReceive = false;
+	bool hasFormat = false;
 
 	/* extract WITH (...) options from the COPY statement */
 	foreach(optionCell, copyStatement->options)
@@ -812,14 +929,14 @@ IsCopyResultStmt(CopyStmt *copyStatement)
 		DefElem *defel = (DefElem *) lfirst(optionCell);
 
 		if (strncmp(defel->defname, "format", NAMEDATALEN) == 0 &&
-			strncmp(defGetString(defel), "result", NAMEDATALEN) == 0)
+			strncmp(defGetString(defel), formatName, NAMEDATALEN) == 0)
 		{
-			hasFormatReceive = true;
+			hasFormat = true;
 			break;
 		}
 	}
 
-	return hasFormatReceive;
+	return hasFormat;
 }
 
 
@@ -828,18 +945,10 @@ IsCopyResultStmt(CopyStmt *copyStatement)
  * COPYing from distributed tables and preventing unsupported actions. The
  * function returns a modified COPY statement to be executed, or NULL if no
  * further processing is needed.
- *
- * commandMustRunAsOwner is an output parameter used to communicate to the caller whether
- * the copy statement should be executed using elevated privileges. If
- * ProcessCopyStmt that is required, a call to CheckCopyPermissions will take
- * care of verifying the current user's permissions before ProcessCopyStmt
- * returns.
  */
 static Node *
-ProcessCopyStmt(CopyStmt *copyStatement, char *completionTag, bool *commandMustRunAsOwner)
+ProcessCopyStmt(CopyStmt *copyStatement, char *completionTag, const char *queryString)
 {
-	*commandMustRunAsOwner = false; /* make sure variable is initialized */
-
 	/*
 	 * Handle special COPY "resultid" FROM STDIN WITH (format result) commands
 	 * for sending intermediate results to workers.
@@ -945,50 +1054,48 @@ ProcessCopyStmt(CopyStmt *copyStatement, char *completionTag, bool *commandMustR
 		}
 	}
 
-
 	if (copyStatement->filename != NULL && !copyStatement->is_program)
 	{
-		const char *filename = copyStatement->filename;
+		char *filename = copyStatement->filename;
 
-		if (CacheDirectoryElement(filename))
+		/*
+		 * We execute COPY commands issued by the task-tracker executor here
+		 * because we're not normally allowed to write to a file as a regular
+		 * user and we don't want to execute the query as superuser.
+		 */
+		if (CacheDirectoryElement(filename) && copyStatement->query != NULL &&
+			!copyStatement->is_from && !is_absolute_path(filename))
 		{
-			/*
-			 * Only superusers are allowed to copy from a file, so we have to
-			 * become superuser to execute copies to/from files used by citus'
-			 * query execution.
-			 *
-			 * XXX: This is a decidedly suboptimal solution, as that means
-			 * that triggers, input functions, etc. run with elevated
-			 * privileges.  But this is better than not being able to run
-			 * queries as normal user.
-			 */
-			*commandMustRunAsOwner = true;
+			bool binaryCopyFormat = CopyStatementHasFormat(copyStatement, "binary");
+			int64 tuplesSent = 0;
+			Query *query = NULL;
+			Node *queryNode = copyStatement->query;
+			List *queryTreeList = NIL;
 
-			/*
-			 * Have to manually check permissions here as the COPY is will be
-			 * run as a superuser.
-			 */
-			if (copyStatement->relation != NULL)
+#if (PG_VERSION_NUM >= 100000)
+			RawStmt *rawStmt = makeNode(RawStmt);
+			rawStmt->stmt = queryNode;
+
+			queryTreeList = pg_analyze_and_rewrite(rawStmt, queryString, NULL, 0, NULL);
+#else
+			queryTreeList = pg_analyze_and_rewrite(queryNode, queryString, NULL, 0);
+#endif
+
+			if (list_length(queryTreeList) != 1)
 			{
-				CheckCopyPermissions(copyStatement);
+				ereport(ERROR, (errmsg("can only execute a single query")));
 			}
 
-			/*
-			 * Check if we have a "COPY (query) TO filename". If we do, copy
-			 * doesn't accept relative file paths. However, SQL tasks that get
-			 * assigned to worker nodes have relative paths. We therefore
-			 * convert relative paths to absolute ones here.
-			 */
-			if (copyStatement->relation == NULL &&
-				!copyStatement->is_from &&
-				!is_absolute_path(filename))
-			{
-				copyStatement->filename = make_absolute_path(filename);
-			}
+			query = (Query *) linitial(queryTreeList);
+			tuplesSent = WorkerExecuteSqlTask(query, filename, binaryCopyFormat);
+
+			snprintf(completionTag, COMPLETION_TAG_BUFSIZE,
+					 "COPY " UINT64_FORMAT, tuplesSent);
+
+			return NULL;
 		}
 	}
 
-
 	return (Node *) copyStatement;
 }
 
@@ -1414,6 +1521,39 @@ PlanAlterTableStmt(AlterTableStmt *alterTableStatement, const char *alterTableCo
 				constraint->skip_validation = true;
 			}
 		}
+		else if (alterTableType == AT_AddColumn)
+		{
+			/*
+			 * TODO: This code path is nothing beneficial since we do not
+			 * support ALTER TABLE %s ADD COLUMN %s [constraint] for foreign keys.
+			 * However, the code is kept in case we fix the constraint
+			 * creation without a name and allow foreign key creation with the mentioned
+			 * command.
+			 */
+			ColumnDef *columnDefinition = (ColumnDef *) command->def;
+			List *columnConstraints = columnDefinition->constraints;
+
+			ListCell *columnConstraint = NULL;
+			foreach(columnConstraint, columnConstraints)
+			{
+				Constraint *constraint = (Constraint *) lfirst(columnConstraint);
+				if (constraint->contype == CONSTR_FOREIGN)
+				{
+					rightRelationId = RangeVarGetRelid(constraint->pktable, lockmode,
+													   alterTableStatement->missing_ok);
+
+					/*
+					 * Foreign constraint validations will be done in workers. If we do not
+					 * set this flag, PostgreSQL tries to do additional checking when we drop
+					 * to standard_ProcessUtility. standard_ProcessUtility tries to open new
+					 * connections to workers to verify foreign constraints while original
+					 * transaction is in process, which causes deadlock.
+					 */
+					constraint->skip_validation = true;
+					break;
+				}
+			}
+		}
 #if (PG_VERSION_NUM >= 100000)
 		else if (alterTableType == AT_AttachPartition)
 		{
@@ -3382,7 +3522,7 @@ RangeVarCallbackForDropIndex(const RangeVar *rel, Oid relOid, Oid oldRelOid, voi
  *
  * Copied from postgres, where it's part of DoCopy().
  */
-static void
+void
 CheckCopyPermissions(CopyStmt *copyStatement)
 {
 	/* *INDENT-OFF* */
@@ -4037,6 +4177,34 @@ SetupExecutionModeForAlterTable(Oid relationId, AlterTableCmd *command)
 			executeSequentially = true;
 		}
 	}
+	else if (alterTableType == AT_AddColumn)
+	{
+		/*
+		 * TODO: This code path will never be executed since we do not
+		 * support foreign constraint creation via
+		 * ALTER TABLE %s ADD COLUMN %s [constraint]. However, the code
+		 * is kept in case we fix the constraint creation without a name
+		 * and allow foreign key creation with the mentioned command.
+		 */
+		ColumnDef *columnDefinition = (ColumnDef *) command->def;
+		List *columnConstraints = columnDefinition->constraints;
+
+		ListCell *columnConstraint = NULL;
+		foreach(columnConstraint, columnConstraints)
+		{
+			Constraint *constraint = (Constraint *) lfirst(columnConstraint);
+			if (constraint->contype == CONSTR_FOREIGN)
+			{
+				Oid rightRelationId = RangeVarGetRelid(constraint->pktable, NoLock,
+													   false);
+				if (IsDistributedTable(rightRelationId) &&
+					PartitionMethod(rightRelationId) == DISTRIBUTE_BY_NONE)
+				{
+					executeSequentially = true;
+				}
+			}
+		}
+	}
 	else if (alterTableType == AT_DropColumn || alterTableType == AT_AlterColumnType)
 	{
 		char *affectedColumnName = command->name;
@@ -4086,7 +4254,7 @@ SetupExecutionModeForAlterTable(Oid relationId, AlterTableCmd *command)
 		char *relationName = get_rel_name(relationId);
 
 		ereport(ERROR, (errmsg("cannot modify table \"%s\" because there "
-							   "was a parallel operation on a distributed table"
+							   "was a parallel operation on a distributed table "
 							   "in the transaction", relationName),
 						errdetail("When there is a foreign key to a reference "
 								  "table, Citus needs to perform all operations "

src/backend/distributed/master/master_delete_protocol.c

@@ -122,7 +122,7 @@ master_apply_delete_command(PG_FUNCTION_ARGS)
 	if (!IsA(queryTreeNode, DeleteStmt))
 	{
 		ereport(ERROR, (errmsg("query \"%s\" is not a delete statement",
-							   queryString)));
+							   ApplyLogRedaction(queryString))));
 	}
 
 	deleteStatement = (DeleteStmt *) queryTreeNode;

src/backend/distributed/master/master_modify_multiple_shards.c

@@ -145,7 +145,7 @@ master_modify_multiple_shards(PG_FUNCTION_ARGS)
 	else
 	{
 		ereport(ERROR, (errmsg("query \"%s\" is not a delete, update, or truncate "
-							   "statement", queryString)));
+							   "statement", ApplyLogRedaction(queryString))));
 	}
 
 	CheckDistributedTable(relationId);

src/backend/distributed/planner/deparse_shard_query.c

@@ -109,11 +109,13 @@ RebuildQueryStrings(Query *originalQuery, List *taskList)
 			}
 		}
 
-		ereport(DEBUG4, (errmsg("query before rebuilding: %s", task->queryString)));
+		ereport(DEBUG4, (errmsg("query before rebuilding: %s",
+								ApplyLogRedaction(task->queryString))));
 
 		UpdateTaskQueryString(query, relationId, valuesRTE, task);
 
-		ereport(DEBUG4, (errmsg("query after rebuilding:  %s", task->queryString)));
+		ereport(DEBUG4, (errmsg("query after rebuilding:  %s",
+								ApplyLogRedaction(task->queryString))));
 	}
 }
 

src/backend/distributed/planner/distributed_planner.c

@@ -147,6 +147,10 @@ distributed_planner(Query *parse, int cursorOptions, ParamListInfo boundParams)
 
 			result = CreateDistributedPlannedStmt(planId, result, originalQuery, parse,
 												  boundParams, plannerRestrictionContext);
+
+			setPartitionedTablesInherited = true;
+			AdjustPartitioningForDistributedPlanning(parse,
+													 setPartitionedTablesInherited);
 		}
 	}
 	PG_CATCH();
@@ -156,13 +160,6 @@ distributed_planner(Query *parse, int cursorOptions, ParamListInfo boundParams)
 	}
 	PG_END_TRY();
 
-	if (needsDistributedPlanning)
-	{
-		setPartitionedTablesInherited = true;
-
-		AdjustPartitioningForDistributedPlanning(parse, setPartitionedTablesInherited);
-	}
-
 	/* remove the context from the context list */
 	PopPlannerRestrictionContext();
 
@@ -1446,6 +1443,13 @@ CurrentPlannerRestrictionContext(void)
 	plannerRestrictionContext =
 		(PlannerRestrictionContext *) linitial(plannerRestrictionContextList);
 
+	if (plannerRestrictionContext == NULL)
+	{
+		ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR),
+						errmsg("planner restriction context stack was empty"),
+						errdetail("Please report this to the Citus core team.")));
+	}
+
 	return plannerRestrictionContext;
 }
 

src/backend/distributed/planner/insert_select_planner.c

@@ -176,6 +176,14 @@ CreateInsertSelectPlan(Query *originalQuery,
 					   PlannerRestrictionContext *plannerRestrictionContext)
 {
 	DistributedPlan *distributedPlan = NULL;
+	DeferredErrorMessage *deferredError = NULL;
+
+	deferredError = ErrorIfOnConflictNotSupported(originalQuery);
+	if (deferredError != NULL)
+	{
+		/* raising the error as there is no possible solution for the unsupported on conflict statements */
+		RaiseDeferredError(deferredError, ERROR);
+	}
 
 	distributedPlan = CreateDistributedInsertSelectPlan(originalQuery,
 														plannerRestrictionContext);
@@ -568,7 +576,8 @@ RouterModifyTaskForShardInterval(Query *originalQuery, ShardInterval *shardInter
 	/* and generate the full query string */
 	deparse_shard_query(copiedQuery, distributedTableId, shardInterval->shardId,
 						queryString);
-	ereport(DEBUG2, (errmsg("distributed statement: %s", queryString->data)));
+	ereport(DEBUG2, (errmsg("distributed statement: %s",
+							ApplyLogRedaction(queryString->data))));
 
 	modifyTask = CreateBasicTask(jobId, taskIdIndex, MODIFY_TASK, queryString->data);
 	modifyTask->dependedTaskList = NULL;

src/backend/distributed/planner/multi_join_order.c

@@ -714,7 +714,8 @@ PrintJoinOrderList(List *joinOrder)
 		}
 	}
 
-	ereport(LOG, (errmsg("join order: %s", printBuffer->data)));
+	ereport(LOG, (errmsg("join order: %s",
+						 ApplyLogRedaction(printBuffer->data))));
 }
 
 

src/backend/distributed/planner/multi_logical_planner.c

@@ -2080,12 +2080,6 @@ ExtractRangeTableRelationWalker(Node *node, List **rangeTableRelationList)
 
 			walkIsComplete = false;
 		}
-		else
-		{
-			walkIsComplete = range_table_walker(list_make1(rangeTable),
-												ExtractRangeTableRelationWalker,
-												rangeTableRelationList, 0);
-		}
 	}
 	else if (IsA(node, Query))
 	{

src/backend/distributed/planner/multi_physical_planner.c

@@ -2418,7 +2418,8 @@ QueryPushdownTaskCreate(Query *originalQuery, int shardIndex,
 		taskType == SQL_TASK)
 	{
 		pg_get_query_def(taskQuery, queryString);
-		ereport(DEBUG4, (errmsg("distributed statement: %s", queryString->data)));
+		ereport(DEBUG4, (errmsg("distributed statement: %s",
+								ApplyLogRedaction(queryString->data))));
 		subqueryTask->queryString = queryString->data;
 	}
 
@@ -2698,7 +2699,8 @@ SqlTaskList(Job *job)
 
 		/* log the query string we generated */
 		ereport(DEBUG4, (errmsg("generated sql query for task %d", sqlTask->taskId),
-						 errdetail("query string: \"%s\"", sqlQueryString->data)));
+						 errdetail("query string: \"%s\"",
+								   ApplyLogRedaction(sqlQueryString->data))));
 
 		sqlTask->anchorShardId = INVALID_SHARD_ID;
 		if (anchorRangeTableBasedAssignment)

src/backend/distributed/planner/multi_router_planner.c

@@ -530,12 +530,8 @@ ModifyQuerySupported(Query *queryTree, Query *originalQuery, bool multiShardQuer
 	List *rangeTableList = NIL;
 	ListCell *rangeTableCell = NULL;
 	uint32 queryTableCount = 0;
-	bool specifiesPartitionValue = false;
-	ListCell *setTargetCell = NULL;
-	List *onConflictSet = NIL;
-	Node *arbiterWhere = NULL;
-	Node *onConflictWhere = NULL;
 	CmdType commandType = queryTree->commandType;
+	DeferredErrorMessage *deferredError = NULL;
 
 	/*
 	 * Here, we check if a recursively planned query tries to modify
@@ -769,7 +765,10 @@ ModifyQuerySupported(Query *queryTree, Query *originalQuery, bool multiShardQuer
 				TargetEntryChangesValue(targetEntry, partitionColumn,
 										queryTree->jointree))
 			{
-				specifiesPartitionValue = true;
+				return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED,
+									 "modifying the partition value of rows is not "
+									 "allowed",
+									 NULL, NULL);
 			}
 
 			if (commandType == CMD_UPDATE &&
@@ -829,13 +828,46 @@ ModifyQuerySupported(Query *queryTree, Query *originalQuery, bool multiShardQuer
 		}
 	}
 
-	if (commandType == CMD_INSERT && queryTree->onConflict != NULL)
+	deferredError = ErrorIfOnConflictNotSupported(queryTree);
+	if (deferredError != NULL)
 	{
-		onConflictSet = queryTree->onConflict->onConflictSet;
-		arbiterWhere = queryTree->onConflict->arbiterWhere;
-		onConflictWhere = queryTree->onConflict->onConflictWhere;
+		return deferredError;
 	}
 
+	return NULL;
+}
+
+
+/*
+ * ErrorIfOnConflictNotSupprted returns an error if an INSERT query has an
+ * unsupported ON CONFLICT clause. In particular, changing the partition
+ * column value or using volatile functions is not allowed.
+ */
+DeferredErrorMessage *
+ErrorIfOnConflictNotSupported(Query *queryTree)
+{
+	Oid distributedTableId = InvalidOid;
+	uint32 rangeTableId = 1;
+	Var *partitionColumn = NULL;
+	List *onConflictSet = NIL;
+	Node *arbiterWhere = NULL;
+	Node *onConflictWhere = NULL;
+	ListCell *setTargetCell = NULL;
+	bool specifiesPartitionValue = false;
+
+	CmdType commandType = queryTree->commandType;
+	if (commandType != CMD_INSERT || queryTree->onConflict == NULL)
+	{
+		return NULL;
+	}
+
+	distributedTableId = ExtractFirstDistributedTableId(queryTree);
+	partitionColumn = PartitionColumn(distributedTableId, rangeTableId);
+
+	onConflictSet = queryTree->onConflict->onConflictSet;
+	arbiterWhere = queryTree->onConflict->arbiterWhere;
+	onConflictWhere = queryTree->onConflict->onConflictWhere;
+
 	/*
 	 * onConflictSet is expanded via expand_targetlist() on the standard planner.
 	 * This ends up adding all the columns to the onConflictSet even if the user
@@ -2513,6 +2545,12 @@ IntersectPlacementList(List *lhsPlacementList, List *rhsPlacementList)
 						WORKER_LENGTH) == 0)
 			{
 				placementList = lappend(placementList, rhsPlacement);
+
+				/*
+				 * We don't need to add the same placement over and over again. This
+				 * could happen if both placements of a shard appear on the same node.
+				 */
+				break;
 			}
 		}
 	}

src/backend/distributed/planner/recursive_planning.c

@@ -218,7 +218,7 @@ GenerateSubplansForSubqueriesAndCTEs(uint64 planId, Query *originalQuery,
 		ereport(DEBUG1, (errmsg(
 							 "Plan " UINT64_FORMAT
 							 " query after replacing subqueries and CTEs: %s", planId,
-							 subPlanString->data)));
+							 ApplyLogRedaction(subPlanString->data))));
 	}
 
 	return context.subPlanList;
@@ -715,9 +715,10 @@ RecursivelyPlanCTEs(Query *query, RecursivePlanningContext *planningContext)
 		{
 			StringInfo subPlanString = makeStringInfo();
 			pg_get_query_def(subquery, subPlanString);
-			ereport(DEBUG1, (errmsg("generating subplan " UINT64_FORMAT "_%u for "
-																		"CTE %s: %s",
-									planId, subPlanId, cteName, subPlanString->data)));
+			ereport(DEBUG1, (errmsg("generating subplan " UINT64_FORMAT
+									"_%u for CTE %s: %s", planId, subPlanId,
+									cteName,
+									ApplyLogRedaction(subPlanString->data))));
 		}
 
 		/* build a sub plan for the CTE */
@@ -1120,9 +1121,9 @@ RecursivelyPlanSubquery(Query *subquery, RecursivePlanningContext *planningConte
 
 		pg_get_query_def(debugQuery, subqueryString);
 
-		ereport(DEBUG1, (errmsg("generating subplan " UINT64_FORMAT "_%u for "
-																	"subquery %s",
-								planId, subPlanId, subqueryString->data)));
+		ereport(DEBUG1, (errmsg("generating subplan " UINT64_FORMAT
+								"_%u for subquery %s", planId, subPlanId,
+								ApplyLogRedaction(subqueryString->data))));
 	}
 
 	/* finally update the input subquery to point the result query */

src/backend/distributed/planner/relation_restriction_equivalence.c

@@ -1825,8 +1825,21 @@ RangeTableArrayContainsAnyRTEIdentities(RangeTblEntry **rangeTableEntries, int
 		 * (i.e.,rangeTableEntry could be a subquery where we're interested
 		 * in relations).
 		 */
-		ExtractRangeTableRelationWalker((Node *) rangeTableEntry,
-										&rangeTableRelationList);
+		if (rangeTableEntry->rtekind == RTE_SUBQUERY)
+		{
+			ExtractRangeTableRelationWalker((Node *) rangeTableEntry->subquery,
+											&rangeTableRelationList);
+		}
+		else if (rangeTableEntry->rtekind == RTE_RELATION)
+		{
+			ExtractRangeTableRelationWalker((Node *) rangeTableEntry,
+											&rangeTableRelationList);
+		}
+		else
+		{
+			/* we currently do not accept any other RTE types here */
+			continue;
+		}
 
 		foreach(rteRelationCell, rangeTableRelationList)
 		{

src/backend/distributed/relay/relay_event_utility.c

@@ -520,6 +520,29 @@ RelayEventExtendNamesForInterShardCommands(Node *parseTree, uint64 leftShardId,
 						relationSchemaName = &(constraint->pktable->schemaname);
 					}
 				}
+				else if (command->subtype == AT_AddColumn)
+				{
+					/*
+					 * TODO: This code path will never be executed since we do not
+					 * support foreign constraint creation via
+					 * ALTER TABLE %s ADD COLUMN %s [constraint]. However, the code
+					 * is kept in case we fix the constraint creation without a name
+					 * and allow foreign key creation with the mentioned command.
+					 */
+					ColumnDef *columnDefinition = (ColumnDef *) command->def;
+					List *columnConstraints = columnDefinition->constraints;
+
+					ListCell *columnConstraint = NULL;
+					foreach(columnConstraint, columnConstraints)
+					{
+						Constraint *constraint = (Constraint *) lfirst(columnConstraint);
+						if (constraint->contype == CONSTR_FOREIGN)
+						{
+							referencedTableName = &(constraint->pktable->relname);
+							relationSchemaName = &(constraint->pktable->schemaname);
+						}
+					}
+				}
 #if (PG_VERSION_NUM >= 100000)
 				else if (command->subtype == AT_AttachPartition ||
 						 command->subtype == AT_DetachPartition)

src/backend/distributed/shared_library_init.c

@@ -53,6 +53,7 @@
 #include "postmaster/postmaster.h"
 #include "optimizer/planner.h"
 #include "optimizer/paths.h"
+#include "tcop/tcopprot.h"
 #include "utils/guc.h"
 #include "utils/guc_tables.h"
 
@@ -63,6 +64,7 @@ static char *CitusVersion = CITUS_VERSION;
 
 void _PG_init(void);
 
+static void ResizeStackToMaximumDepth(void);
 static void multi_log_hook(ErrorData *edata);
 static void CreateRequiredDirectories(void);
 static void RegisterCitusConfigVariables(void);
@@ -167,6 +169,8 @@ _PG_init(void)
 								"shared_preload_libraries.")));
 	}
 
+	ResizeStackToMaximumDepth();
+
 	/*
 	 * Extend the database directory structure before continuing with
 	 * initialization - one of the later steps might require them to exist.
@@ -231,6 +235,35 @@ _PG_init(void)
 }
 
 
+/*
+ * Stack size increase during high memory load may cause unexpected crashes.
+ * With this alloca call, we are increasing stack size explicitly, so that if
+ * it is not possible to increase stack size, we will get an OOM error instead
+ * of a crash.
+ *
+ * This function is called on backend startup. The allocated memory will
+ * automatically be released at the end of the function's scope. However, we'd
+ * have already expanded the stack and it wouldn't shrink back. So, in a sense,
+ * per backend we're securing max_stack_depth kB's of memory on the stack upfront.
+ *
+ * Not all the backends require max_stack_depth kB's on the stack, so we might end
+ * up with unnecessary allocations. However, the default value is 2MB, which seems
+ * an acceptable trade-off. Also, allocating memory upfront may perform better
+ * under some circumstances.
+ */
+static void
+ResizeStackToMaximumDepth(void)
+{
+#ifndef WIN32
+	volatile char *stack_resizer = NULL;
+	long max_stack_depth_bytes = max_stack_depth * 1024L;
+
+	stack_resizer = alloca(max_stack_depth_bytes);
+	stack_resizer[max_stack_depth_bytes - 1] = 0;
+#endif
+}
+
+
 /*
  * multi_log_hook intercepts postgres log commands. We use this to override
  * postgres error messages when they're not specific enough for the users.

src/backend/distributed/test/deparse_shard_query.c

@@ -76,7 +76,7 @@ deparse_shard_query_test(PG_FUNCTION_ARGS)
 
 			deparse_shard_query(query, InvalidOid, 0, buffer);
 
-			elog(INFO, "query: %s", buffer->data);
+			elog(INFO, "query: %s", ApplyLogRedaction(buffer->data));
 		}
 	}
 

src/backend/distributed/test/distribution_metadata.c

@@ -32,6 +32,7 @@
 #include "nodes/pg_list.h"
 #include "nodes/primnodes.h"
 #include "storage/lock.h"
+#include "tcop/tcopprot.h"
 #include "utils/array.h"
 #include "utils/elog.h"
 #include "utils/errcodes.h"
@@ -48,6 +49,7 @@ PG_FUNCTION_INFO_V1(partition_type);
 PG_FUNCTION_INFO_V1(is_distributed_table);
 PG_FUNCTION_INFO_V1(create_monolithic_shard_row);
 PG_FUNCTION_INFO_V1(acquire_shared_shard_lock);
+PG_FUNCTION_INFO_V1(relation_count_in_query);
 
 
 /*
@@ -249,3 +251,43 @@ acquire_shared_shard_lock(PG_FUNCTION_ARGS)
 
 	PG_RETURN_VOID();
 }
+
+
+/*
+ * relation_count_in_query return the first query's relation count.
+ */
+Datum
+relation_count_in_query(PG_FUNCTION_ARGS)
+{
+	text *queryString = PG_GETARG_TEXT_P(0);
+
+	char *queryStringChar = text_to_cstring(queryString);
+	List *parseTreeList = pg_parse_query(queryStringChar);
+	ListCell *parseTreeCell = NULL;
+
+	foreach(parseTreeCell, parseTreeList)
+	{
+		Node *parsetree = (Node *) lfirst(parseTreeCell);
+		ListCell *queryTreeCell = NULL;
+		List *queryTreeList = NIL;
+
+#if (PG_VERSION_NUM >= 100000)
+		queryTreeList = pg_analyze_and_rewrite((RawStmt *) parsetree, queryStringChar,
+											   NULL, 0, NULL);
+#else
+		queryTreeList = pg_analyze_and_rewrite(parsetree, queryStringChar, NULL, 0);
+#endif
+
+		foreach(queryTreeCell, queryTreeList)
+		{
+			Query *query = lfirst(queryTreeCell);
+			List *rangeTableList = NIL;
+
+			ExtractRangeTableRelationWalker((Node *) query, &rangeTableList);
+
+			PG_RETURN_INT32(list_length(rangeTableList));
+		}
+	}
+
+	PG_RETURN_INT32(0);
+}

src/backend/distributed/transaction/backend_data.c

@@ -365,9 +365,11 @@ BackendManagementShmemInit(void)
 
 		/*
 		 * We need to init per backend's spinlock before any backend
-		 * starts its execution.
+		 * starts its execution. Note that we initialize TotalProcs (e.g., not
+		 * MaxBackends) since some of the blocking processes could be prepared
+		 * transactions, which aren't covered by MaxBackends.
 		 */
-		for (backendIndex = 0; backendIndex < MaxBackends; ++backendIndex)
+		for (backendIndex = 0; backendIndex < TotalProcs; ++backendIndex)
 		{
 			SpinLockInit(&backendManagementShmemData->backends[backendIndex].mutex);
 		}
@@ -392,7 +394,7 @@ BackendManagementShmemSize(void)
 	Size size = 0;
 
 	size = add_size(size, sizeof(BackendManagementShmemData));
-	size = add_size(size, mul_size(sizeof(BackendData), MaxBackends));
+	size = add_size(size, mul_size(sizeof(BackendData), TotalProcs));
 
 	return size;
 }

src/backend/distributed/transaction/distributed_deadlock_detection.c

@@ -17,6 +17,7 @@
 #include "access/hash.h"
 #include "distributed/backend_data.h"
 #include "distributed/distributed_deadlock_detection.h"
+#include "distributed/errormessage.h"
 #include "distributed/hash_helpers.h"
 #include "distributed/listutils.h"
 #include "distributed/lock_graph.h"
@@ -673,7 +674,7 @@ LogDistributedDeadlockDebugMessage(const char *errorMessage)
 	}
 
 	ereport(LOG, (errmsg("[%s] %s", timestamptz_to_str(GetCurrentTimestamp()),
-						 errorMessage)));
+						 ApplyLogRedaction(errorMessage))));
 }
 
 

src/backend/distributed/transaction/lock_graph.c

@@ -401,12 +401,12 @@ BuildLocalWaitGraph(void)
 	 */
 	waitGraph = (WaitGraph *) palloc0(sizeof(WaitGraph));
 	waitGraph->localNodeId = GetLocalGroupId();
-	waitGraph->allocatedSize = MaxBackends * 3;
+	waitGraph->allocatedSize = TotalProcs * 3;
 	waitGraph->edgeCount = 0;
 	waitGraph->edges = (WaitEdge *) palloc(waitGraph->allocatedSize * sizeof(WaitEdge));
 
-	remaining.procs = (PGPROC **) palloc(sizeof(PGPROC *) * MaxBackends);
-	remaining.procAdded = (bool *) palloc0(sizeof(bool *) * MaxBackends);
+	remaining.procs = (PGPROC **) palloc(sizeof(PGPROC *) * TotalProcs);
+	remaining.procAdded = (bool *) palloc0(sizeof(bool *) * TotalProcs);
 	remaining.procCount = 0;
 
 	LockLockData();
@@ -419,7 +419,7 @@ BuildLocalWaitGraph(void)
 	 */
 
 	/* build list of starting procs */
-	for (curBackend = 0; curBackend < MaxBackends; curBackend++)
+	for (curBackend = 0; curBackend < TotalProcs; curBackend++)
 	{
 		PGPROC *currentProc = &ProcGlobal->allProcs[curBackend];
 		BackendData currentBackendData;
@@ -765,7 +765,7 @@ AddProcToVisit(PROCStack *remaining, PGPROC *proc)
 		return;
 	}
 
-	Assert(remaining->procCount < MaxBackends);
+	Assert(remaining->procCount < TotalProcs);
 
 	remaining->procs[remaining->procCount++] = proc;
 	remaining->procAdded[proc->pgprocno] = true;

src/backend/distributed/transaction/remote_transaction.c

@@ -528,8 +528,22 @@ FinishRemoteTransactionPrepare(struct MultiConnection *connection)
 		transaction->transactionState = REMOTE_TRANS_PREPARED;
 	}
 
-	result = GetRemoteCommandResult(connection, raiseErrors);
-	Assert(!result);
+	PQclear(result);
+
+	/*
+	 * Try to consume results of PREPARE TRANSACTION command. If we don't
+	 * succeed, rollback the transaction. Note that we've not committed on
+	 * any node yet, and we're not sure about the state of the worker node.
+	 * So rollbacking seems to be the safest action if the worker is
+	 * in a state where it can actually rollback.
+	 */
+	if (!ClearResults(connection, raiseErrors))
+	{
+		ereport(ERROR, (errmsg("failed to prepare transaction '%s' on host %s:%d",
+							   transaction->preparedName, connection->hostname,
+							   connection->port),
+						errhint("Try re-running the command.")));
+	}
 }
 
 

src/backend/distributed/transaction/transaction_recovery.c

@@ -420,7 +420,7 @@ PendingWorkerTransactionList(MultiConnection *connection)
 	int coordinatorId = GetLocalGroupId();
 
 	appendStringInfo(command, "SELECT gid FROM pg_prepared_xacts "
-							  "WHERE gid LIKE 'citus_%d_%%'",
+							  "WHERE gid LIKE 'citus\\_%d\\_%%'",
 					 coordinatorId);
 
 	querySent = SendRemoteCommand(connection, command->data);

src/backend/distributed/utils/errormessage.c

@@ -12,6 +12,16 @@
 #include "distributed/errormessage.h"
 
 
+/*
+ * ApplyLogRedaction is only supported in Citus Enterprise
+ */
+char *
+ApplyLogRedaction(const char *text)
+{
+	return (char *) text;
+}
+
+
 /*
  * DeferredErrorInternal is a helper function for DeferredError().
  */

src/backend/distributed/utils/metadata_cache.c

@@ -858,12 +858,22 @@ LookupDistTableCacheEntry(Oid relationId)
 	memset(((char *) cacheEntry) + sizeof(Oid), 0,
 		   sizeof(DistTableCacheEntry) - sizeof(Oid));
 
+	/*
+	 * We disable interrupts while creating the cache entry because loading
+	 * shard metadata can take a while, and if statement_timeout is too low,
+	 * this will get canceled on each call and we won't be able to run any
+	 * queries on the table.
+	 */
+	HOLD_INTERRUPTS();
+
 	/* actually fill out entry */
 	BuildDistTableCacheEntry(cacheEntry);
 
 	/* and finally mark as valid */
 	cacheEntry->isValid = true;
 
+	RESUME_INTERRUPTS();
+
 	return cacheEntry;
 }
 
@@ -1168,6 +1178,13 @@ BuildCachedShardList(DistTableCacheEntry *cacheEntry)
 		}
 	}
 
+	/*
+	 * We set these here, so ResetDistTableCacheEntry() can see what has been
+	 * entered into DistShardCacheHash even if the following loop is interrupted
+	 * by throwing errors, etc.
+	 */
+	cacheEntry->sortedShardIntervalArray = sortedShardIntervalArray;
+	cacheEntry->shardIntervalArrayLength = 0;
 
 	/* maintain shardId->(table,ShardInterval) cache */
 	for (shardIndex = 0; shardIndex < shardIntervalArrayLength; shardIndex++)
@@ -1192,6 +1209,13 @@ BuildCachedShardList(DistTableCacheEntry *cacheEntry)
 							errhint("Reconnect and try again.")));
 		}
 
+		/*
+		 * We should increment this only after we are sure this hasn't already
+		 * been assigned to any other relations. ResetDistTableCacheEntry()
+		 * depends on this.
+		 */
+		cacheEntry->shardIntervalArrayLength++;
+
 		shardEntry->shardIndex = shardIndex;
 		shardEntry->tableEntry = cacheEntry;
 
@@ -1220,8 +1244,6 @@ BuildCachedShardList(DistTableCacheEntry *cacheEntry)
 		shardInterval->shardIndex = shardIndex;
 	}
 
-	cacheEntry->shardIntervalArrayLength = shardIntervalArrayLength;
-	cacheEntry->sortedShardIntervalArray = sortedShardIntervalArray;
 	cacheEntry->shardColumnCompareFunction = shardColumnCompareFunction;
 	cacheEntry->shardIntervalCompareFunction = shardIntervalCompareFunction;
 }
@@ -2896,7 +2918,10 @@ ResetDistTableCacheEntry(DistTableCacheEntry *cacheEntry)
 		bool foundInCache = false;
 
 		/* delete the shard's placements */
-		pfree(placementArray);
+		if (placementArray != NULL)
+		{
+			pfree(placementArray);
+		}
 
 		/* delete per-shard cache-entry */
 		hash_search(DistShardCacheHash, &shardInterval->shardId, HASH_REMOVE,

src/backend/distributed/worker/worker_data_fetch_protocol.c

@@ -31,6 +31,7 @@
 #include "distributed/master_protocol.h"
 #include "distributed/metadata_cache.h"
 #include "distributed/multi_client_executor.h"
+#include "distributed/multi_copy.h"
 #include "distributed/multi_logical_optimizer.h"
 #include "distributed/multi_server_executor.h"
 #include "distributed/multi_utility.h"
@@ -695,7 +696,8 @@ ParseTreeRawStmt(const char *ddlCommand)
 	/* log immediately if dictated by log statement */
 	if (check_log_statement(parseTreeList))
 	{
-		ereport(LOG, (errmsg("statement: %s", ddlCommand), errhidestmt(true)));
+		ereport(LOG, (errmsg("statement: %s", ApplyLogRedaction(ddlCommand)),
+					  errhidestmt(true)));
 	}
 
 	parseTreeCount = list_length(parseTreeList);
@@ -747,6 +749,8 @@ worker_append_table_to_shard(PG_FUNCTION_ARGS)
 	uint64 shardId = INVALID_SHARD_ID;
 	bool received = false;
 	StringInfo queryString = NULL;
+	Oid savedUserId = InvalidOid;
+	int savedSecurityContext = 0;
 
 	CheckCitusVersion(ERROR);
 
@@ -792,9 +796,18 @@ worker_append_table_to_shard(PG_FUNCTION_ARGS)
 	appendStringInfo(queryString, COPY_IN_COMMAND, shardQualifiedName,
 					 localFilePath->data);
 
+	/* make sure we are allowed to execute the COPY command */
+	CheckCopyPermissions(localCopyCommand);
+
+	/* need superuser to copy from files */
+	GetUserIdAndSecContext(&savedUserId, &savedSecurityContext);
+	SetUserIdAndSecContext(CitusExtensionOwner(), SECURITY_LOCAL_USERID_CHANGE);
+
 	CitusProcessUtility((Node *) localCopyCommand, queryString->data,
 						PROCESS_UTILITY_TOPLEVEL, NULL, None_Receiver, NULL);
 
+	SetUserIdAndSecContext(savedUserId, savedSecurityContext);
+
 	/* finally delete the temporary file we created */
 	CitusDeleteFile(localFilePath->data);
 

src/backend/distributed/worker/worker_partition_protocol.c

@@ -923,7 +923,7 @@ FilterAndPartitionTable(const char *filterQuery,
 	if (queryPortal == NULL)
 	{
 		ereport(ERROR, (errmsg("could not open implicit cursor for query \"%s\"",
-							   filterQuery)));
+							   ApplyLogRedaction(filterQuery))));
 	}
 
 	rowOutputState = InitRowOutputState();

src/backend/distributed/worker/worker_sql_task_protocol.c

@@ -0,0 +1,280 @@
+/*-------------------------------------------------------------------------
+ *
+ * worker_sql_task_protocol.c
+ *
+ * Routines for executing SQL tasks during task-tracker execution.
+ *
+ * Copyright (c) 2012-2018, Citus Data, Inc.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+#include "funcapi.h"
+#include "pgstat.h"
+
+#include "distributed/multi_copy.h"
+#include "distributed/multi_executor.h"
+#include "distributed/transmit.h"
+#include "distributed/worker_protocol.h"
+#include "utils/builtins.h"
+#include "utils/memutils.h"
+
+
+/* TaskFileDestReceiver can be used to stream results into a file */
+typedef struct TaskFileDestReceiver
+{
+	/* public DestReceiver interface */
+	DestReceiver pub;
+
+	/* descriptor of the tuples that are sent to the worker */
+	TupleDesc tupleDescriptor;
+
+	/* EState for per-tuple memory allocation */
+	EState *executorState;
+
+	/* MemoryContext for DestReceiver session */
+	MemoryContext memoryContext;
+
+	/* output file */
+	char *filePath;
+	File fileDesc;
+	bool binaryCopyFormat;
+
+	/* state on how to copy out data types */
+	CopyOutState copyOutState;
+	FmgrInfo *columnOutputFunctions;
+
+	/* number of tuples sent */
+	uint64 tuplesSent;
+} TaskFileDestReceiver;
+
+
+static DestReceiver * CreateTaskFileDestReceiver(char *filePath, EState *executorState,
+												 bool binaryCopyFormat);
+static void TaskFileDestReceiverStartup(DestReceiver *dest, int operation,
+										TupleDesc inputTupleDescriptor);
+static bool TaskFileDestReceiverReceive(TupleTableSlot *slot, DestReceiver *dest);
+static void WriteToLocalFile(StringInfo copyData, File fileDesc);
+static void TaskFileDestReceiverShutdown(DestReceiver *destReceiver);
+static void TaskFileDestReceiverDestroy(DestReceiver *destReceiver);
+
+
+/*
+ * WorkerExecuteSqlTask executes an already-parsed query and writes the result
+ * to the given task file.
+ */
+int64
+WorkerExecuteSqlTask(Query *query, char *taskFilename, bool binaryCopyFormat)
+{
+	EState *estate = NULL;
+	TaskFileDestReceiver *taskFileDest = NULL;
+	ParamListInfo paramListInfo = NULL;
+	int64 tuplesSent = 0L;
+
+	estate = CreateExecutorState();
+	taskFileDest =
+		(TaskFileDestReceiver *) CreateTaskFileDestReceiver(taskFilename, estate,
+															binaryCopyFormat);
+
+	ExecuteQueryIntoDestReceiver(query, paramListInfo, (DestReceiver *) taskFileDest);
+
+	tuplesSent = taskFileDest->tuplesSent;
+
+	taskFileDest->pub.rDestroy((DestReceiver *) taskFileDest);
+	FreeExecutorState(estate);
+
+	return tuplesSent;
+}
+
+
+/*
+ * CreateTaskFileDestReceiver creates a DestReceiver for writing query results
+ * to a task file.
+ */
+static DestReceiver *
+CreateTaskFileDestReceiver(char *filePath, EState *executorState, bool binaryCopyFormat)
+{
+	TaskFileDestReceiver *taskFileDest = NULL;
+
+	taskFileDest = (TaskFileDestReceiver *) palloc0(sizeof(TaskFileDestReceiver));
+
+	/* set up the DestReceiver function pointers */
+	taskFileDest->pub.receiveSlot = TaskFileDestReceiverReceive;
+	taskFileDest->pub.rStartup = TaskFileDestReceiverStartup;
+	taskFileDest->pub.rShutdown = TaskFileDestReceiverShutdown;
+	taskFileDest->pub.rDestroy = TaskFileDestReceiverDestroy;
+	taskFileDest->pub.mydest = DestCopyOut;
+
+	/* set up output parameters */
+	taskFileDest->executorState = executorState;
+	taskFileDest->memoryContext = CurrentMemoryContext;
+	taskFileDest->filePath = pstrdup(filePath);
+	taskFileDest->binaryCopyFormat = binaryCopyFormat;
+
+	return (DestReceiver *) taskFileDest;
+}
+
+
+/*
+ * TaskFileDestReceiverStartup implements the rStartup interface of
+ * TaskFileDestReceiver. It opens the destination file and sets up
+ * the CopyOutState.
+ */
+static void
+TaskFileDestReceiverStartup(DestReceiver *dest, int operation,
+							TupleDesc inputTupleDescriptor)
+{
+	TaskFileDestReceiver *taskFileDest = (TaskFileDestReceiver *) dest;
+
+	CopyOutState copyOutState = NULL;
+	const char *delimiterCharacter = "\t";
+	const char *nullPrintCharacter = "\\N";
+
+	const int fileFlags = (O_APPEND | O_CREAT | O_RDWR | O_TRUNC | PG_BINARY);
+	const int fileMode = (S_IRUSR | S_IWUSR);
+
+	/* use the memory context that was in place when the DestReceiver was created */
+	MemoryContext oldContext = MemoryContextSwitchTo(taskFileDest->memoryContext);
+
+	taskFileDest->tupleDescriptor = inputTupleDescriptor;
+
+	/* define how tuples will be serialised */
+	copyOutState = (CopyOutState) palloc0(sizeof(CopyOutStateData));
+	copyOutState->delim = (char *) delimiterCharacter;
+	copyOutState->null_print = (char *) nullPrintCharacter;
+	copyOutState->null_print_client = (char *) nullPrintCharacter;
+	copyOutState->binary = taskFileDest->binaryCopyFormat;
+	copyOutState->fe_msgbuf = makeStringInfo();
+	copyOutState->rowcontext = GetPerTupleMemoryContext(taskFileDest->executorState);
+	taskFileDest->copyOutState = copyOutState;
+
+	taskFileDest->columnOutputFunctions = ColumnOutputFunctions(inputTupleDescriptor,
+																copyOutState->binary);
+
+	taskFileDest->fileDesc = FileOpenForTransmit(taskFileDest->filePath, fileFlags,
+												 fileMode);
+
+	if (copyOutState->binary)
+	{
+		/* write headers when using binary encoding */
+		resetStringInfo(copyOutState->fe_msgbuf);
+		AppendCopyBinaryHeaders(copyOutState);
+
+		WriteToLocalFile(copyOutState->fe_msgbuf, taskFileDest->fileDesc);
+	}
+
+	MemoryContextSwitchTo(oldContext);
+}
+
+
+/*
+ * TaskFileDestReceiverReceive implements the receiveSlot function of
+ * TaskFileDestReceiver. It takes a TupleTableSlot and writes the contents
+ * to a local file.
+ */
+static bool
+TaskFileDestReceiverReceive(TupleTableSlot *slot, DestReceiver *dest)
+{
+	TaskFileDestReceiver *taskFileDest = (TaskFileDestReceiver *) dest;
+
+	TupleDesc tupleDescriptor = taskFileDest->tupleDescriptor;
+
+	CopyOutState copyOutState = taskFileDest->copyOutState;
+	FmgrInfo *columnOutputFunctions = taskFileDest->columnOutputFunctions;
+
+	Datum *columnValues = NULL;
+	bool *columnNulls = NULL;
+	StringInfo copyData = copyOutState->fe_msgbuf;
+
+	EState *executorState = taskFileDest->executorState;
+	MemoryContext executorTupleContext = GetPerTupleMemoryContext(executorState);
+	MemoryContext oldContext = MemoryContextSwitchTo(executorTupleContext);
+
+	slot_getallattrs(slot);
+
+	columnValues = slot->tts_values;
+	columnNulls = slot->tts_isnull;
+
+	resetStringInfo(copyData);
+
+	/* construct row in COPY format */
+	AppendCopyRowData(columnValues, columnNulls, tupleDescriptor,
+					  copyOutState, columnOutputFunctions, NULL);
+
+	WriteToLocalFile(copyOutState->fe_msgbuf, taskFileDest->fileDesc);
+
+	MemoryContextSwitchTo(oldContext);
+
+	taskFileDest->tuplesSent++;
+
+	ResetPerTupleExprContext(executorState);
+
+	return true;
+}
+
+
+/*
+ * WriteToLocalResultsFile writes the bytes in a StringInfo to a local file.
+ */
+static void
+WriteToLocalFile(StringInfo copyData, File fileDesc)
+{
+#if (PG_VERSION_NUM >= 100000)
+	int bytesWritten = FileWrite(fileDesc, copyData->data, copyData->len, PG_WAIT_IO);
+#else
+	int bytesWritten = FileWrite(fileDesc, copyData->data, copyData->len);
+#endif
+	if (bytesWritten < 0)
+	{
+		ereport(ERROR, (errcode_for_file_access(),
+						errmsg("could not append to file: %m")));
+	}
+}
+
+
+/*
+ * TaskFileDestReceiverShutdown implements the rShutdown interface of
+ * TaskFileDestReceiver. It writes the footer and closes the file.
+ * the relation.
+ */
+static void
+TaskFileDestReceiverShutdown(DestReceiver *destReceiver)
+{
+	TaskFileDestReceiver *taskFileDest = (TaskFileDestReceiver *) destReceiver;
+	CopyOutState copyOutState = taskFileDest->copyOutState;
+
+	if (copyOutState->binary)
+	{
+		/* write footers when using binary encoding */
+		resetStringInfo(copyOutState->fe_msgbuf);
+		AppendCopyBinaryFooters(copyOutState);
+		WriteToLocalFile(copyOutState->fe_msgbuf, taskFileDest->fileDesc);
+	}
+
+	FileClose(taskFileDest->fileDesc);
+}
+
+
+/*
+ * TaskFileDestReceiverDestroy frees memory allocated as part of the
+ * TaskFileDestReceiver and closes file descriptors.
+ */
+static void
+TaskFileDestReceiverDestroy(DestReceiver *destReceiver)
+{
+	TaskFileDestReceiver *taskFileDest = (TaskFileDestReceiver *) destReceiver;
+
+	if (taskFileDest->copyOutState)
+	{
+		pfree(taskFileDest->copyOutState);
+	}
+
+	if (taskFileDest->columnOutputFunctions)
+	{
+		pfree(taskFileDest->columnOutputFunctions);
+	}
+
+	pfree(taskFileDest->filePath);
+	pfree(taskFileDest);
+}

src/include/distributed/backend_data.h

@@ -13,6 +13,7 @@
 #define BACKEND_DATA_H
 
 
+#include "access/twophase.h"
 #include "datatype/timestamp.h"
 #include "distributed/transaction_identifier.h"
 #include "nodes/pg_list.h"
@@ -21,6 +22,9 @@
 #include "storage/s_lock.h"
 
 
+#define TotalProcs (MaxBackends + NUM_AUXILIARY_PROCS + max_prepared_xacts)
+
+
 /*
  * Each backend's active distributed transaction information is tracked via
  * BackendData in shared memory.

src/include/distributed/errormessage.h

@@ -70,3 +70,5 @@ DeferredErrorMessage * DeferredErrorInternal(int code, const char *message, cons
 void RaiseDeferredErrorInternal(DeferredErrorMessage *error, int elevel);
 
 #endif
+
+extern char * ApplyLogRedaction(const char *text);

src/include/distributed/multi_copy.h

@@ -131,6 +131,7 @@ extern void EndRemoteCopy(int64 shardId, List *connectionList, bool stopOnFailur
 extern void CitusCopyFrom(CopyStmt *copyStatement, char *completionTag);
 extern bool IsCopyFromWorker(CopyStmt *copyStatement);
 extern NodeAddress * MasterNodeAddress(CopyStmt *copyStatement);
+extern void CheckCopyPermissions(CopyStmt *copyStatement);
 
 
 #endif /* MULTI_COPY_H */

src/include/distributed/multi_executor.h

@@ -32,6 +32,7 @@ extern TupleTableSlot * ReturnTupleFromTuplestore(CitusScanState *scanState);
 extern void LoadTuplesIntoTupleStore(CitusScanState *citusScanState, Job *workerJob);
 extern void ReadFileIntoTupleStore(char *fileName, char *copyFormat, TupleDesc
 								   tupleDescriptor, Tuplestorestate *tupstore);
+extern Query * ParseQueryString(const char *queryString);
 extern void ExecuteQueryStringIntoDestReceiver(const char *queryString, ParamListInfo
 											   params,
 											   DestReceiver *dest);

src/include/distributed/multi_router_executor.h

@@ -37,9 +37,8 @@ extern bool AllModificationsCommutative;
 extern bool EnableDeadlockPrevention;
 
 extern void CitusModifyBeginScan(CustomScanState *node, EState *estate, int eflags);
-extern TupleTableSlot * RouterSequentialModifyExecScan(CustomScanState *node);
 extern TupleTableSlot * RouterSelectExecScan(CustomScanState *node);
-extern TupleTableSlot * RouterMultiModifyExecScan(CustomScanState *node);
+extern TupleTableSlot * RouterModifyExecScan(CustomScanState *node);
 
 extern int64 ExecuteModifyTasksWithoutResults(List *taskList);
 extern int64 ExecuteModifyTasksSequentiallyWithoutResults(List *taskList,

src/include/distributed/multi_router_planner.h

@@ -52,6 +52,7 @@ extern DeferredErrorMessage * ModifyQuerySupported(Query *queryTree, Query *orig
 												   bool multiShardQuery,
 												   PlannerRestrictionContext *
 												   plannerRestrictionContext);
+extern DeferredErrorMessage * ErrorIfOnConflictNotSupported(Query *queryTree);
 extern List * ShardIntervalOpExpressions(ShardInterval *shardInterval, Index rteIndex);
 extern RelationRestrictionContext * CopyRelationRestrictionContext(
 	RelationRestrictionContext *oldContext);

src/include/distributed/worker_protocol.h

@@ -120,6 +120,8 @@ extern int32 ArrayObjectCount(ArrayType *arrayObject);
 extern FmgrInfo * GetFunctionInfo(Oid typeId, Oid accessMethodId, int16 procedureId);
 extern List * TableDDLCommandList(const char *nodeName, uint32 nodePort,
 								  const char *tableName);
+extern int64 WorkerExecuteSqlTask(Query *query, char *taskFilename,
+								  bool binaryCopyFormat);
 
 /* Function declarations shared with the master planner */
 extern StringInfo TaskFilename(StringInfo directoryName, uint32 taskId);

src/test/regress/.gitignore

@@ -9,3 +9,6 @@
 # Regression test output
 /regression.diffs
 /regression.out
+
+# Failure test side effets
+/proxy.output

src/test/regress/expected/foreign_key_restriction_enforcement.out

@@ -161,7 +161,7 @@ BEGIN;
 (1 row)
 
 	ALTER TABLE on_update_fkey_table DROP COLUMN value_1 CASCADE;
-ERROR:  cannot modify table "on_update_fkey_table" because there was a parallel operation on a distributed tablein the transaction
+ERROR:  cannot modify table "on_update_fkey_table" because there was a parallel operation on a distributed table in the transaction
 DETAIL:  When there is a foreign key to a reference table, Citus needs to perform all operations over a single connection per node to ensure consistency.
 HINT:  Try re-running the transaction with "SET LOCAL citus.multi_shard_modify_mode TO 'sequential';"
 ROLLBACK;
@@ -549,7 +549,7 @@ ROLLBACK;
 BEGIN;
 	ALTER TABLE on_update_fkey_table ADD COLUMN X int;
 	ALTER TABLE on_update_fkey_table ALTER COLUMN value_1 SET DATA TYPE smallint;
-ERROR:  cannot modify table "on_update_fkey_table" because there was a parallel operation on a distributed tablein the transaction
+ERROR:  cannot modify table "on_update_fkey_table" because there was a parallel operation on a distributed table in the transaction
 DETAIL:  When there is a foreign key to a reference table, Citus needs to perform all operations over a single connection per node to ensure consistency.
 HINT:  Try re-running the transaction with "SET LOCAL citus.multi_shard_modify_mode TO 'sequential';"
 ROLLBACK;
@@ -796,7 +796,7 @@ DETAIL:  NOTICE from localhost:57638
 (1 row)
 
 	ALTER TABLE test_table_2 ADD CONSTRAINT c_check FOREIGN KEY (value_1) REFERENCES test_table_1(id);
-ERROR:  cannot modify table "test_table_2" because there was a parallel operation on a distributed tablein the transaction
+ERROR:  cannot modify table "test_table_2" because there was a parallel operation on a distributed table in the transaction
 DETAIL:  When there is a foreign key to a reference table, Citus needs to perform all operations over a single connection per node to ensure consistency.
 HINT:  Try re-running the transaction with "SET LOCAL citus.multi_shard_modify_mode TO 'sequential';"
 	-- make sure that the output isn't too verbose
@@ -895,7 +895,7 @@ DETAIL:  NOTICE from localhost:57638
 (1 row)
 
 	ALTER TABLE test_table_2 ADD CONSTRAINT c_check FOREIGN KEY (value_1) REFERENCES test_table_1(id);
-ERROR:  cannot modify table "test_table_2" because there was a parallel operation on a distributed tablein the transaction
+ERROR:  cannot modify table "test_table_2" because there was a parallel operation on a distributed table in the transaction
 DETAIL:  When there is a foreign key to a reference table, Citus needs to perform all operations over a single connection per node to ensure consistency.
 HINT:  Try re-running the transaction with "SET LOCAL citus.multi_shard_modify_mode TO 'sequential';"
 	-- make sure that the output isn't too verbose
@@ -1136,10 +1136,155 @@ DEBUG:  verifying table "test_table_2"
  	SET LOCAL client_min_messages TO ERROR;
 	DROP TABLE test_table_2, test_table_1;
 COMMIT;
+-- make sure that modifications to reference tables in a CTE can
+-- set the mode to sequential for the next operations
+CREATE TABLE reference_table(id int PRIMARY KEY);
+DEBUG:  CREATE TABLE / PRIMARY KEY will create implicit index "reference_table_pkey" for table "reference_table"
+DEBUG:  building index "reference_table_pkey" on table "reference_table"
+SELECT create_reference_table('reference_table');
+DEBUG:  schema "test_fkey_to_ref_in_tx" already exists, skipping
+DETAIL:  NOTICE from localhost:57638
+DEBUG:  schema "test_fkey_to_ref_in_tx" already exists, skipping
+DETAIL:  NOTICE from localhost:57637
+DEBUG:  schema "test_fkey_to_ref_in_tx" already exists, skipping
+DETAIL:  NOTICE from localhost:57637
+DEBUG:  schema "test_fkey_to_ref_in_tx" already exists, skipping
+DETAIL:  NOTICE from localhost:57638
+ create_reference_table 
+------------------------
+ 
+(1 row)
+
+CREATE TABLE distributed_table(id int PRIMARY KEY, value_1 int);
+DEBUG:  CREATE TABLE / PRIMARY KEY will create implicit index "distributed_table_pkey" for table "distributed_table"
+DEBUG:  building index "distributed_table_pkey" on table "distributed_table"
+SELECT create_distributed_table('distributed_table', 'id');
+DEBUG:  schema "test_fkey_to_ref_in_tx" already exists, skipping
+DETAIL:  NOTICE from localhost:57638
+DEBUG:  schema "test_fkey_to_ref_in_tx" already exists, skipping
+DETAIL:  NOTICE from localhost:57637
+DEBUG:  schema "test_fkey_to_ref_in_tx" already exists, skipping
+DETAIL:  NOTICE from localhost:57637
+DEBUG:  schema "test_fkey_to_ref_in_tx" already exists, skipping
+DETAIL:  NOTICE from localhost:57638
+DEBUG:  schema "test_fkey_to_ref_in_tx" already exists, skipping
+DETAIL:  NOTICE from localhost:57637
+DEBUG:  schema "test_fkey_to_ref_in_tx" already exists, skipping
+DETAIL:  NOTICE from localhost:57638
+ create_distributed_table 
+--------------------------
+ 
+(1 row)
+
+ALTER TABLE 
+	distributed_table 
+ADD CONSTRAINT 
+	fkey_delete FOREIGN KEY(value_1) 
+REFERENCES 
+	reference_table(id) ON DELETE CASCADE;
+DEBUG:  switching to sequential query execution mode
+DETAIL:  Reference relation "reference_table" is modified, which might lead to data inconsistencies or distributed deadlocks via parallel accesses to hash distributed relations due to foreign keys. Any parallel modification to those hash distributed relations in the same transaction can only be executed in sequential query execution mode
+INSERT INTO reference_table SELECT i FROM generate_series(0, 10) i;
+DEBUG:  distributed INSERT ... SELECT can only select from distributed tables
+DEBUG:  Collecting INSERT ... SELECT results on coordinator
+DEBUG:  switching to sequential query execution mode
+DETAIL:  Reference relation "reference_table" is modified, which might lead to data inconsistencies or distributed deadlocks via parallel accesses to hash distributed relations due to foreign keys. Any parallel modification to those hash distributed relations in the same transaction can only be executed in sequential query execution mode
+INSERT INTO distributed_table SELECT i, i % 10  FROM generate_series(0, 100) i;
+DEBUG:  distributed INSERT ... SELECT can only select from distributed tables
+DEBUG:  Collecting INSERT ... SELECT results on coordinator
+-- this query returns 100 rows in Postgres, but not in Citus
+-- see https://github.com/citusdata/citus_docs/issues/664 for the discussion
+WITH t1 AS (DELETE FROM reference_table RETURNING id) 
+	DELETE FROM distributed_table USING t1 WHERE value_1 = t1.id RETURNING *;
+DEBUG:  common table expressions are not supported in distributed modifications
+DEBUG:  generating subplan 92_1 for CTE t1: DELETE FROM test_fkey_to_ref_in_tx.reference_table RETURNING id
+DEBUG:  Plan 92 query after replacing subqueries and CTEs: DELETE FROM test_fkey_to_ref_in_tx.distributed_table USING (SELECT intermediate_result.id FROM read_intermediate_result('92_1'::text, 'binary'::citus_copy_format) intermediate_result(id integer)) t1 WHERE (distributed_table.value_1 OPERATOR(pg_catalog.=) t1.id) RETURNING distributed_table.id, distributed_table.value_1, t1.id
+DEBUG:  switching to sequential query execution mode
+DETAIL:  Reference relation "reference_table" is modified, which might lead to data inconsistencies or distributed deadlocks via parallel accesses to hash distributed relations due to foreign keys. Any parallel modification to those hash distributed relations in the same transaction can only be executed in sequential query execution mode
+ id | value_1 | id 
+----+---------+----
+(0 rows)
+
+-- load some more data for one more test with real-time selects
+INSERT INTO reference_table SELECT i FROM generate_series(0, 10) i;
+DEBUG:  distributed INSERT ... SELECT can only select from distributed tables
+DEBUG:  Collecting INSERT ... SELECT results on coordinator
+DEBUG:  switching to sequential query execution mode
+DETAIL:  Reference relation "reference_table" is modified, which might lead to data inconsistencies or distributed deadlocks via parallel accesses to hash distributed relations due to foreign keys. Any parallel modification to those hash distributed relations in the same transaction can only be executed in sequential query execution mode
+INSERT INTO distributed_table SELECT i, i % 10  FROM generate_series(0, 100) i;
+DEBUG:  distributed INSERT ... SELECT can only select from distributed tables
+DEBUG:  Collecting INSERT ... SELECT results on coordinator
+-- this query returns 100 rows in Postgres, but not in Citus
+-- see https://github.com/citusdata/citus_docs/issues/664 for the discussion
+WITH t1 AS (DELETE FROM reference_table RETURNING id) 
+	SELECT count(*) FROM distributed_table, t1 WHERE  value_1 = t1.id;
+DEBUG:  data-modifying statements are not supported in the WITH clauses of distributed queries
+DEBUG:  generating subplan 96_1 for CTE t1: DELETE FROM test_fkey_to_ref_in_tx.reference_table RETURNING id
+DEBUG:  Plan 96 query after replacing subqueries and CTEs: SELECT count(*) AS count FROM test_fkey_to_ref_in_tx.distributed_table, (SELECT intermediate_result.id FROM read_intermediate_result('96_1'::text, 'binary'::citus_copy_format) intermediate_result(id integer)) t1 WHERE (distributed_table.value_1 OPERATOR(pg_catalog.=) t1.id)
+DEBUG:  switching to sequential query execution mode
+DETAIL:  Reference relation "reference_table" is modified, which might lead to data inconsistencies or distributed deadlocks via parallel accesses to hash distributed relations due to foreign keys. Any parallel modification to those hash distributed relations in the same transaction can only be executed in sequential query execution mode
+ count 
+-------
+     0
+(1 row)
+
+-- this query should fail since we first to a parallel access to a distributed table 
+-- with t1, and then access to t2
+WITH t1 AS (DELETE FROM distributed_table RETURNING id),
+	t2 AS (DELETE FROM reference_table RETURNING id)
+	SELECT count(*) FROM distributed_table, t1, t2 WHERE  value_1 = t1.id AND value_1 = t2.id;
+DEBUG:  data-modifying statements are not supported in the WITH clauses of distributed queries
+DEBUG:  generating subplan 98_1 for CTE t1: DELETE FROM test_fkey_to_ref_in_tx.distributed_table RETURNING id
+DEBUG:  generating subplan 98_2 for CTE t2: DELETE FROM test_fkey_to_ref_in_tx.reference_table RETURNING id
+DEBUG:  Plan 98 query after replacing subqueries and CTEs: SELECT count(*) AS count FROM test_fkey_to_ref_in_tx.distributed_table, (SELECT intermediate_result.id FROM read_intermediate_result('98_1'::text, 'binary'::citus_copy_format) intermediate_result(id integer)) t1, (SELECT intermediate_result.id FROM read_intermediate_result('98_2'::text, 'binary'::citus_copy_format) intermediate_result(id integer)) t2 WHERE ((distributed_table.value_1 OPERATOR(pg_catalog.=) t1.id) AND (distributed_table.value_1 OPERATOR(pg_catalog.=) t2.id))
+ERROR:  cannot execute DML on reference relation "reference_table" because there was a parallel DML access to distributed relation "distributed_table" in the same transaction
+HINT:  Try re-running the transaction with "SET LOCAL citus.multi_shard_modify_mode TO 'sequential';"
+-- similarly this should fail since we first access to a distributed
+-- table via t1, and then access to the reference table in the main query
+WITH t1 AS (DELETE FROM distributed_table RETURNING id)
+	DELETE FROM reference_table RETURNING id;
+DEBUG:  common table expressions are not supported in distributed modifications
+DEBUG:  generating subplan 101_1 for CTE t1: DELETE FROM test_fkey_to_ref_in_tx.distributed_table RETURNING id
+DEBUG:  Plan 101 query after replacing subqueries and CTEs: DELETE FROM test_fkey_to_ref_in_tx.reference_table RETURNING id
+ERROR:  cannot execute DML on reference relation "reference_table" because there was a parallel DML access to distributed relation "distributed_table" in the same transaction
+HINT:  Try re-running the transaction with "SET LOCAL citus.multi_shard_modify_mode TO 'sequential';"
+-- finally, make sure that we can execute the same queries
+-- in the sequential mode
+BEGIN;
+	
+	SET LOCAL citus.multi_shard_modify_mode TO 'sequential';
+	WITH t1 AS (DELETE FROM distributed_table RETURNING id),
+		t2 AS (DELETE FROM reference_table RETURNING id)
+		SELECT count(*) FROM distributed_table, t1, t2 WHERE  value_1 = t1.id AND value_1 = t2.id;
+DEBUG:  data-modifying statements are not supported in the WITH clauses of distributed queries
+DEBUG:  generating subplan 103_1 for CTE t1: DELETE FROM test_fkey_to_ref_in_tx.distributed_table RETURNING id
+DEBUG:  generating subplan 103_2 for CTE t2: DELETE FROM test_fkey_to_ref_in_tx.reference_table RETURNING id
+DEBUG:  Plan 103 query after replacing subqueries and CTEs: SELECT count(*) AS count FROM test_fkey_to_ref_in_tx.distributed_table, (SELECT intermediate_result.id FROM read_intermediate_result('103_1'::text, 'binary'::citus_copy_format) intermediate_result(id integer)) t1, (SELECT intermediate_result.id FROM read_intermediate_result('103_2'::text, 'binary'::citus_copy_format) intermediate_result(id integer)) t2 WHERE ((distributed_table.value_1 OPERATOR(pg_catalog.=) t1.id) AND (distributed_table.value_1 OPERATOR(pg_catalog.=) t2.id))
+ count 
+-------
+     0
+(1 row)
+
+ROLLBACK;
+BEGIN;
+	
+	SET LOCAL citus.multi_shard_modify_mode TO 'sequential';
+	WITH t1 AS (DELETE FROM distributed_table RETURNING id)
+		DELETE FROM reference_table RETURNING id;
+DEBUG:  common table expressions are not supported in distributed modifications
+DEBUG:  generating subplan 106_1 for CTE t1: DELETE FROM test_fkey_to_ref_in_tx.distributed_table RETURNING id
+DEBUG:  Plan 106 query after replacing subqueries and CTEs: DELETE FROM test_fkey_to_ref_in_tx.reference_table RETURNING id
+ id 
+----
+(0 rows)
+
+ROLLBACK;
 RESET client_min_messages;
 DROP SCHEMA test_fkey_to_ref_in_tx CASCADE;
-NOTICE:  drop cascades to 3 other objects
+NOTICE:  drop cascades to 5 other objects
 DETAIL:  drop cascades to table referece_table
 drop cascades to table on_update_fkey_table
 drop cascades to table unrelated_dist_table
+drop cascades to table reference_table
+drop cascades to table distributed_table
 SET search_path TO public;

src/test/regress/expected/foreign_key_to_reference_table.out

@@ -293,6 +293,24 @@ SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' A
  fkey_ref_7000098 | fkey_reference_table.referencing_table_7000098 | fkey_reference_table.referenced_table_7000042
 (8 rows)
 
+DROP TABLE referencing_table;
+-- check if we can add the foreign key while adding the column
+CREATE TABLE referencing_table(id int, ref_id int);
+SELECT create_distributed_table('referencing_table', 'ref_id');
+ create_distributed_table 
+--------------------------
+ 
+(1 row)
+
+ALTER TABLE referencing_table ADD COLUMN referencing int REFERENCES referenced_table(id) ON UPDATE CASCADE;
+ERROR:  cannot execute ADD COLUMN command with PRIMARY KEY, UNIQUE, FOREIGN and CHECK constraints
+DETAIL:  Adding a column with a constraint in one command is not supported because all constraints in Citus must have explicit names
+HINT:  You can issue each command separately such as ALTER TABLE referencing_table ADD COLUMN referencing data_type; ALTER TABLE referencing_table ADD CONSTRAINT constraint_name FOREIGN KEY (referencing) REFERENCES referenced_table(id) ON UPDATE CASCADE;
+SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
+ name | relid | refd_relid 
+------+-------+------------
+(0 rows)
+
 DROP TABLE referencing_table;
 -- foreign keys are only supported when the replication factor = 1
 SET citus.shard_replication_factor TO 2;
@@ -312,6 +330,24 @@ SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' A
 ------+-------+------------
 (0 rows)
 
+DROP TABLE referencing_table;
+-- should fail when we add the column as well
+CREATE TABLE referencing_table(id int, ref_id int);
+SELECT create_distributed_table('referencing_table', 'ref_id');
+ create_distributed_table 
+--------------------------
+ 
+(1 row)
+
+ALTER TABLE referencing_table ADD COLUMN referencing_col int REFERENCES referenced_table(id) ON DELETE SET NULL;
+ERROR:  cannot create foreign key constraint
+DETAIL:  Citus Community Edition currently supports foreign key constraints only for "citus.shard_replication_factor = 1".
+HINT:  Please change "citus.shard_replication_factor to 1". To learn more about using foreign keys with other replication factors, please contact us at https://citusdata.com/about/contact_us.
+SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
+ name | relid | refd_relid 
+------+-------+------------
+(0 rows)
+
 DROP TABLE referencing_table;
 SET citus.shard_replication_factor TO 1;
 -- simple create_distributed_table should work in/out transactions on tables with foreign key to reference tables
@@ -325,14 +361,14 @@ SELECT create_distributed_table('referencing_table', 'ref_id');
 SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
                name                |                     relid                      |                  refd_relid                   
 -----------------------------------+------------------------------------------------+-----------------------------------------------
- referencing_table_id_fkey_7000107 | fkey_reference_table.referencing_table_7000107 | fkey_reference_table.referenced_table_7000042
- referencing_table_id_fkey_7000108 | fkey_reference_table.referencing_table_7000108 | fkey_reference_table.referenced_table_7000042
- referencing_table_id_fkey_7000109 | fkey_reference_table.referencing_table_7000109 | fkey_reference_table.referenced_table_7000042
- referencing_table_id_fkey_7000110 | fkey_reference_table.referencing_table_7000110 | fkey_reference_table.referenced_table_7000042
- referencing_table_id_fkey_7000111 | fkey_reference_table.referencing_table_7000111 | fkey_reference_table.referenced_table_7000042
- referencing_table_id_fkey_7000112 | fkey_reference_table.referencing_table_7000112 | fkey_reference_table.referenced_table_7000042
- referencing_table_id_fkey_7000113 | fkey_reference_table.referencing_table_7000113 | fkey_reference_table.referenced_table_7000042
- referencing_table_id_fkey_7000114 | fkey_reference_table.referencing_table_7000114 | fkey_reference_table.referenced_table_7000042
+ referencing_table_id_fkey_7000123 | fkey_reference_table.referencing_table_7000123 | fkey_reference_table.referenced_table_7000042
+ referencing_table_id_fkey_7000124 | fkey_reference_table.referencing_table_7000124 | fkey_reference_table.referenced_table_7000042
+ referencing_table_id_fkey_7000125 | fkey_reference_table.referencing_table_7000125 | fkey_reference_table.referenced_table_7000042
+ referencing_table_id_fkey_7000126 | fkey_reference_table.referencing_table_7000126 | fkey_reference_table.referenced_table_7000042
+ referencing_table_id_fkey_7000127 | fkey_reference_table.referencing_table_7000127 | fkey_reference_table.referenced_table_7000042
+ referencing_table_id_fkey_7000128 | fkey_reference_table.referencing_table_7000128 | fkey_reference_table.referenced_table_7000042
+ referencing_table_id_fkey_7000129 | fkey_reference_table.referencing_table_7000129 | fkey_reference_table.referenced_table_7000042
+ referencing_table_id_fkey_7000130 | fkey_reference_table.referencing_table_7000130 | fkey_reference_table.referenced_table_7000042
 (8 rows)
 
 DROP TABLE referencing_table;
@@ -356,14 +392,14 @@ COMMIT;
 SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
                name                |                     relid                      |                  refd_relid                   
 -----------------------------------+------------------------------------------------+-----------------------------------------------
- referencing_table_id_fkey_7000116 | fkey_reference_table.referencing_table_7000116 | fkey_reference_table.referenced_table_7000115
- referencing_table_id_fkey_7000117 | fkey_reference_table.referencing_table_7000117 | fkey_reference_table.referenced_table_7000115
- referencing_table_id_fkey_7000118 | fkey_reference_table.referencing_table_7000118 | fkey_reference_table.referenced_table_7000115
- referencing_table_id_fkey_7000119 | fkey_reference_table.referencing_table_7000119 | fkey_reference_table.referenced_table_7000115
- referencing_table_id_fkey_7000120 | fkey_reference_table.referencing_table_7000120 | fkey_reference_table.referenced_table_7000115
- referencing_table_id_fkey_7000121 | fkey_reference_table.referencing_table_7000121 | fkey_reference_table.referenced_table_7000115
- referencing_table_id_fkey_7000122 | fkey_reference_table.referencing_table_7000122 | fkey_reference_table.referenced_table_7000115
- referencing_table_id_fkey_7000123 | fkey_reference_table.referencing_table_7000123 | fkey_reference_table.referenced_table_7000115
+ referencing_table_id_fkey_7000132 | fkey_reference_table.referencing_table_7000132 | fkey_reference_table.referenced_table_7000131
+ referencing_table_id_fkey_7000133 | fkey_reference_table.referencing_table_7000133 | fkey_reference_table.referenced_table_7000131
+ referencing_table_id_fkey_7000134 | fkey_reference_table.referencing_table_7000134 | fkey_reference_table.referenced_table_7000131
+ referencing_table_id_fkey_7000135 | fkey_reference_table.referencing_table_7000135 | fkey_reference_table.referenced_table_7000131
+ referencing_table_id_fkey_7000136 | fkey_reference_table.referencing_table_7000136 | fkey_reference_table.referenced_table_7000131
+ referencing_table_id_fkey_7000137 | fkey_reference_table.referencing_table_7000137 | fkey_reference_table.referenced_table_7000131
+ referencing_table_id_fkey_7000138 | fkey_reference_table.referencing_table_7000138 | fkey_reference_table.referenced_table_7000131
+ referencing_table_id_fkey_7000139 | fkey_reference_table.referencing_table_7000139 | fkey_reference_table.referenced_table_7000131
 (8 rows)
 
 DROP TABLE referencing_table;
@@ -421,8 +457,8 @@ ALTER TABLE referencing_table ADD CONSTRAINT fkey_ref FOREIGN KEY (ref_id) REFER
 -- test inserts
 -- test insert to referencing table while there is NO corresponding value in referenced table
 INSERT INTO referencing_table VALUES(1, 1);
-ERROR:  insert or update on table "referencing_table_7000125" violates foreign key constraint "fkey_ref_7000125"
-DETAIL:  Key (ref_id)=(1) is not present in table "referenced_table_7000124".
+ERROR:  insert or update on table "referencing_table_7000141" violates foreign key constraint "fkey_ref_7000141"
+DETAIL:  Key (ref_id)=(1) is not present in table "referenced_table_7000140".
 CONTEXT:  while executing command on localhost:57637
 -- test insert to referencing while there is corresponding value in referenced table
 INSERT INTO referenced_table SELECT x, x from generate_series(1,1000) as f(x);
@@ -430,8 +466,8 @@ INSERT INTO referencing_table SELECT x, x from generate_series(1,500) as f(x);
 -- test deletes
 -- test delete from referenced table while there is corresponding value in referencing table
 DELETE FROM referenced_table WHERE id > 3;
-ERROR:  update or delete on table "referenced_table_7000124" violates foreign key constraint "fkey_ref_7000127" on table "referencing_table_7000127"
-DETAIL:  Key (id)=(4) is still referenced from table "referencing_table_7000127".
+ERROR:  update or delete on table "referenced_table_7000140" violates foreign key constraint "fkey_ref_7000143" on table "referencing_table_7000143"
+DETAIL:  Key (id)=(4) is still referenced from table "referencing_table_7000143".
 CONTEXT:  while executing command on localhost:57637
 -- test delete from referenced table while there is NO corresponding value in referencing table
 DELETE FROM referenced_table WHERE id = 501;
@@ -626,8 +662,8 @@ INSERT INTO referenced_table SELECT x,x FROM generate_series(1,1000) AS f(x);
 INSERT INTO referencing_table(id) SELECT x FROM generate_series(1,1000) AS f(x);
 -- Fails for non existing value inserts (serial is already incremented)
 INSERT INTO referencing_table(id) SELECT x FROM generate_series(1,10) AS f(x);
-ERROR:  insert or update on table "referencing_table_7000172" violates foreign key constraint "fkey_ref_7000172"
-DETAIL:  Key (ref_id)=(1001) is not present in table "referenced_table_7000171".
+ERROR:  insert or update on table "referencing_table_7000195" violates foreign key constraint "fkey_ref_7000195"
+DETAIL:  Key (ref_id)=(1009) is not present in table "referenced_table_7000187".
 DROP TABLE referenced_table CASCADE;
 NOTICE:  drop cascades to constraint fkey_ref on table referencing_table
 DROP TABLE referencing_table CASCADE;
@@ -657,8 +693,8 @@ INSERT INTO referenced_table(test_column2) SELECT x FROM generate_series(1,1000)
 INSERT INTO referencing_table(id) SELECT x FROM generate_series(1,1000) AS f(x);
 -- Fails for non existing value inserts (serial is already incremented)
 INSERT INTO referencing_table(id) SELECT x FROM generate_series(1,10) AS f(x);
-ERROR:  insert or update on table "referencing_table_7000181" violates foreign key constraint "fkey_ref_7000181"
-DETAIL:  Key (ref_id)=(1001) is not present in table "referenced_table_7000180".
+ERROR:  insert or update on table "referencing_table_7000197" violates foreign key constraint "fkey_ref_7000197"
+DETAIL:  Key (ref_id)=(1001) is not present in table "referenced_table_7000196".
 DROP TABLE referenced_table CASCADE;
 NOTICE:  drop cascades to constraint fkey_ref on table referencing_table
 DROP TABLE referencing_table CASCADE;
@@ -709,8 +745,8 @@ INSERT INTO referenced_table SELECT x, x FROM generate_series(0,1000) AS f(x);
 INSERT INTO referencing_table SELECT x, x FROM generate_series(0,1000) AS f(x);
 -- we expect this to fail because of the foreign constraint.
 INSERT INTO referencing_table SELECT x, x FROM generate_series(1000,1001) AS f(x);
-ERROR:  insert or update on table "referencing_table_7000204" violates foreign key constraint "fkey_ref_7000204"
-DETAIL:  Key (ref_id)=(1001) is not present in table "referenced_table_7000198".
+ERROR:  insert or update on table "referencing_table_7000220" violates foreign key constraint "fkey_ref_7000220"
+DETAIL:  Key (ref_id)=(1001) is not present in table "referenced_table_7000214".
 -- currently not supported
 ALTER TABLE referencing_table VALIDATE CONSTRAINT fkey_ref;
 ERROR:  alter table command is currently unsupported
@@ -809,38 +845,38 @@ ALTER TABLE referencing_table ADD CONSTRAINT foreign_key_2 FOREIGN KEY (id) REFE
 SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
          name          |                     relid                      |                   refd_relid                   
 -----------------------+------------------------------------------------+------------------------------------------------
- fkey_ref_7000219      | fkey_reference_table.referencing_table_7000219 | fkey_reference_table.referenced_table_7000217
- fkey_ref_7000220      | fkey_reference_table.referencing_table_7000220 | fkey_reference_table.referenced_table_7000217
- fkey_ref_7000221      | fkey_reference_table.referencing_table_7000221 | fkey_reference_table.referenced_table_7000217
- fkey_ref_7000222      | fkey_reference_table.referencing_table_7000222 | fkey_reference_table.referenced_table_7000217
- fkey_ref_7000223      | fkey_reference_table.referencing_table_7000223 | fkey_reference_table.referenced_table_7000217
- fkey_ref_7000224      | fkey_reference_table.referencing_table_7000224 | fkey_reference_table.referenced_table_7000217
- fkey_ref_7000225      | fkey_reference_table.referencing_table_7000225 | fkey_reference_table.referenced_table_7000217
- fkey_ref_7000226      | fkey_reference_table.referencing_table_7000226 | fkey_reference_table.referenced_table_7000217
- foreign_key_2_7000219 | fkey_reference_table.referencing_table_7000219 | fkey_reference_table.referenced_table2_7000218
- foreign_key_2_7000220 | fkey_reference_table.referencing_table_7000220 | fkey_reference_table.referenced_table2_7000218
- foreign_key_2_7000221 | fkey_reference_table.referencing_table_7000221 | fkey_reference_table.referenced_table2_7000218
- foreign_key_2_7000222 | fkey_reference_table.referencing_table_7000222 | fkey_reference_table.referenced_table2_7000218
- foreign_key_2_7000223 | fkey_reference_table.referencing_table_7000223 | fkey_reference_table.referenced_table2_7000218
- foreign_key_2_7000224 | fkey_reference_table.referencing_table_7000224 | fkey_reference_table.referenced_table2_7000218
- foreign_key_2_7000225 | fkey_reference_table.referencing_table_7000225 | fkey_reference_table.referenced_table2_7000218
- foreign_key_2_7000226 | fkey_reference_table.referencing_table_7000226 | fkey_reference_table.referenced_table2_7000218
+ fkey_ref_7000235      | fkey_reference_table.referencing_table_7000235 | fkey_reference_table.referenced_table_7000233
+ fkey_ref_7000236      | fkey_reference_table.referencing_table_7000236 | fkey_reference_table.referenced_table_7000233
+ fkey_ref_7000237      | fkey_reference_table.referencing_table_7000237 | fkey_reference_table.referenced_table_7000233
+ fkey_ref_7000238      | fkey_reference_table.referencing_table_7000238 | fkey_reference_table.referenced_table_7000233
+ fkey_ref_7000239      | fkey_reference_table.referencing_table_7000239 | fkey_reference_table.referenced_table_7000233
+ fkey_ref_7000240      | fkey_reference_table.referencing_table_7000240 | fkey_reference_table.referenced_table_7000233
+ fkey_ref_7000241      | fkey_reference_table.referencing_table_7000241 | fkey_reference_table.referenced_table_7000233
+ fkey_ref_7000242      | fkey_reference_table.referencing_table_7000242 | fkey_reference_table.referenced_table_7000233
+ foreign_key_2_7000235 | fkey_reference_table.referencing_table_7000235 | fkey_reference_table.referenced_table2_7000234
+ foreign_key_2_7000236 | fkey_reference_table.referencing_table_7000236 | fkey_reference_table.referenced_table2_7000234
+ foreign_key_2_7000237 | fkey_reference_table.referencing_table_7000237 | fkey_reference_table.referenced_table2_7000234
+ foreign_key_2_7000238 | fkey_reference_table.referencing_table_7000238 | fkey_reference_table.referenced_table2_7000234
+ foreign_key_2_7000239 | fkey_reference_table.referencing_table_7000239 | fkey_reference_table.referenced_table2_7000234
+ foreign_key_2_7000240 | fkey_reference_table.referencing_table_7000240 | fkey_reference_table.referenced_table2_7000234
+ foreign_key_2_7000241 | fkey_reference_table.referencing_table_7000241 | fkey_reference_table.referenced_table2_7000234
+ foreign_key_2_7000242 | fkey_reference_table.referencing_table_7000242 | fkey_reference_table.referenced_table2_7000234
 (16 rows)
 
 INSERT INTO referenced_table SELECT x, x+1 FROM generate_series(0,1000) AS f(x);
 INSERT INTO referenced_table2 SELECT x, x+1 FROM generate_series(500,1500) AS f(x);
 -- should fail
 INSERT INTO referencing_table SELECT x, x+1 FROM generate_series(0,1500) AS f(x);
-ERROR:  insert or update on table "referencing_table_7000220" violates foreign key constraint "foreign_key_2_7000220"
-DETAIL:  Key (id)=(5) is not present in table "referenced_table2_7000218".
+ERROR:  insert or update on table "referencing_table_7000242" violates foreign key constraint "foreign_key_2_7000242"
+DETAIL:  Key (id)=(9) is not present in table "referenced_table2_7000234".
 -- should fail
 INSERT INTO referencing_table SELECT x, x+1 FROM generate_series(0,400) AS f(x);
-ERROR:  insert or update on table "referencing_table_7000220" violates foreign key constraint "foreign_key_2_7000220"
-DETAIL:  Key (id)=(5) is not present in table "referenced_table2_7000218".
+ERROR:  insert or update on table "referencing_table_7000242" violates foreign key constraint "foreign_key_2_7000242"
+DETAIL:  Key (id)=(9) is not present in table "referenced_table2_7000234".
 -- should fail
 INSERT INTO referencing_table SELECT x, x+1 FROM generate_series(1000,1400) AS f(x);
-ERROR:  insert or update on table "referencing_table_7000220" violates foreign key constraint "fkey_ref_7000220"
-DETAIL:  Key (id)=(1005) is not present in table "referenced_table_7000217".
+ERROR:  insert or update on table "referencing_table_7000242" violates foreign key constraint "fkey_ref_7000242"
+DETAIL:  Key (id)=(1023) is not present in table "referenced_table_7000233".
 -- should success
 INSERT INTO referencing_table SELECT x, x+1 FROM generate_series(600,900) AS f(x);
 SELECT count(*) FROM referencing_table;
@@ -936,38 +972,38 @@ COMMIT;
 SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
          name          |                     relid                      |                   refd_relid                   
 -----------------------+------------------------------------------------+------------------------------------------------
- fkey_ref_7000239      | fkey_reference_table.referencing_table_7000239 | fkey_reference_table.referenced_table_7000237
- fkey_ref_7000240      | fkey_reference_table.referencing_table_7000240 | fkey_reference_table.referenced_table_7000237
- fkey_ref_7000241      | fkey_reference_table.referencing_table_7000241 | fkey_reference_table.referenced_table_7000237
- fkey_ref_7000242      | fkey_reference_table.referencing_table_7000242 | fkey_reference_table.referenced_table_7000237
- fkey_ref_7000243      | fkey_reference_table.referencing_table_7000243 | fkey_reference_table.referenced_table_7000237
- fkey_ref_7000244      | fkey_reference_table.referencing_table_7000244 | fkey_reference_table.referenced_table_7000237
- fkey_ref_7000245      | fkey_reference_table.referencing_table_7000245 | fkey_reference_table.referenced_table_7000237
- fkey_ref_7000246      | fkey_reference_table.referencing_table_7000246 | fkey_reference_table.referenced_table_7000237
- foreign_key_2_7000239 | fkey_reference_table.referencing_table_7000239 | fkey_reference_table.referenced_table2_7000238
- foreign_key_2_7000240 | fkey_reference_table.referencing_table_7000240 | fkey_reference_table.referenced_table2_7000238
- foreign_key_2_7000241 | fkey_reference_table.referencing_table_7000241 | fkey_reference_table.referenced_table2_7000238
- foreign_key_2_7000242 | fkey_reference_table.referencing_table_7000242 | fkey_reference_table.referenced_table2_7000238
- foreign_key_2_7000243 | fkey_reference_table.referencing_table_7000243 | fkey_reference_table.referenced_table2_7000238
- foreign_key_2_7000244 | fkey_reference_table.referencing_table_7000244 | fkey_reference_table.referenced_table2_7000238
- foreign_key_2_7000245 | fkey_reference_table.referencing_table_7000245 | fkey_reference_table.referenced_table2_7000238
- foreign_key_2_7000246 | fkey_reference_table.referencing_table_7000246 | fkey_reference_table.referenced_table2_7000238
+ fkey_ref_7000255      | fkey_reference_table.referencing_table_7000255 | fkey_reference_table.referenced_table_7000253
+ fkey_ref_7000256      | fkey_reference_table.referencing_table_7000256 | fkey_reference_table.referenced_table_7000253
+ fkey_ref_7000257      | fkey_reference_table.referencing_table_7000257 | fkey_reference_table.referenced_table_7000253
+ fkey_ref_7000258      | fkey_reference_table.referencing_table_7000258 | fkey_reference_table.referenced_table_7000253
+ fkey_ref_7000259      | fkey_reference_table.referencing_table_7000259 | fkey_reference_table.referenced_table_7000253
+ fkey_ref_7000260      | fkey_reference_table.referencing_table_7000260 | fkey_reference_table.referenced_table_7000253
+ fkey_ref_7000261      | fkey_reference_table.referencing_table_7000261 | fkey_reference_table.referenced_table_7000253
+ fkey_ref_7000262      | fkey_reference_table.referencing_table_7000262 | fkey_reference_table.referenced_table_7000253
+ foreign_key_2_7000255 | fkey_reference_table.referencing_table_7000255 | fkey_reference_table.referenced_table2_7000254
+ foreign_key_2_7000256 | fkey_reference_table.referencing_table_7000256 | fkey_reference_table.referenced_table2_7000254
+ foreign_key_2_7000257 | fkey_reference_table.referencing_table_7000257 | fkey_reference_table.referenced_table2_7000254
+ foreign_key_2_7000258 | fkey_reference_table.referencing_table_7000258 | fkey_reference_table.referenced_table2_7000254
+ foreign_key_2_7000259 | fkey_reference_table.referencing_table_7000259 | fkey_reference_table.referenced_table2_7000254
+ foreign_key_2_7000260 | fkey_reference_table.referencing_table_7000260 | fkey_reference_table.referenced_table2_7000254
+ foreign_key_2_7000261 | fkey_reference_table.referencing_table_7000261 | fkey_reference_table.referenced_table2_7000254
+ foreign_key_2_7000262 | fkey_reference_table.referencing_table_7000262 | fkey_reference_table.referenced_table2_7000254
 (16 rows)
 
 INSERT INTO referenced_table SELECT x, x+1 FROM generate_series(0,1000) AS f(x);
 INSERT INTO referenced_table2 SELECT x, x+1 FROM generate_series(500,1500) AS f(x);
 -- should fail
 INSERT INTO referencing_table SELECT x, x+1 FROM generate_series(0,1500) AS f(x);
-ERROR:  insert or update on table "referencing_table_7000245" violates foreign key constraint "foreign_key_2_7000245"
-DETAIL:  Key (ref_id)=(3) is not present in table "referenced_table2_7000238".
+ERROR:  insert or update on table "referencing_table_7000260" violates foreign key constraint "foreign_key_2_7000260"
+DETAIL:  Key (ref_id)=(7) is not present in table "referenced_table2_7000254".
 -- should fail
 INSERT INTO referencing_table SELECT x, x+1 FROM generate_series(0,400) AS f(x);
-ERROR:  insert or update on table "referencing_table_7000245" violates foreign key constraint "foreign_key_2_7000245"
-DETAIL:  Key (ref_id)=(3) is not present in table "referenced_table2_7000238".
+ERROR:  insert or update on table "referencing_table_7000260" violates foreign key constraint "foreign_key_2_7000260"
+DETAIL:  Key (ref_id)=(7) is not present in table "referenced_table2_7000254".
 -- should fail
 INSERT INTO referencing_table SELECT x, x+1 FROM generate_series(1000,1400) AS f(x);
-ERROR:  insert or update on table "referencing_table_7000245" violates foreign key constraint "fkey_ref_7000245"
-DETAIL:  Key (id)=(1002) is not present in table "referenced_table_7000237".
+ERROR:  insert or update on table "referencing_table_7000260" violates foreign key constraint "fkey_ref_7000260"
+DETAIL:  Key (id)=(1001) is not present in table "referenced_table_7000253".
 -- should success
 INSERT INTO referencing_table SELECT x, x+501 FROM generate_series(0,1000) AS f(x);
 SELECT count(*) FROM referencing_table;
@@ -1069,43 +1105,43 @@ COMMIT;
 SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
            name           |                      relid                      |                   refd_relid                   
 --------------------------+-------------------------------------------------+------------------------------------------------
- fkey_ref_7000258         | fkey_reference_table.referencing_table_7000258  | fkey_reference_table.referenced_table_7000257
- fkey_ref_7000259         | fkey_reference_table.referencing_table_7000259  | fkey_reference_table.referenced_table_7000257
- fkey_ref_7000260         | fkey_reference_table.referencing_table_7000260  | fkey_reference_table.referenced_table_7000257
- fkey_ref_7000261         | fkey_reference_table.referencing_table_7000261  | fkey_reference_table.referenced_table_7000257
- fkey_ref_7000262         | fkey_reference_table.referencing_table_7000262  | fkey_reference_table.referenced_table_7000257
- fkey_ref_7000263         | fkey_reference_table.referencing_table_7000263  | fkey_reference_table.referenced_table_7000257
- fkey_ref_7000264         | fkey_reference_table.referencing_table_7000264  | fkey_reference_table.referenced_table_7000257
- fkey_ref_7000265         | fkey_reference_table.referencing_table_7000265  | fkey_reference_table.referenced_table_7000257
- fkey_ref_7000266         | fkey_reference_table.referencing_table2_7000266 | fkey_reference_table.referenced_table_7000257
- fkey_ref_7000267         | fkey_reference_table.referencing_table2_7000267 | fkey_reference_table.referenced_table_7000257
- fkey_ref_7000268         | fkey_reference_table.referencing_table2_7000268 | fkey_reference_table.referenced_table_7000257
- fkey_ref_7000269         | fkey_reference_table.referencing_table2_7000269 | fkey_reference_table.referenced_table_7000257
- fkey_ref_7000270         | fkey_reference_table.referencing_table2_7000270 | fkey_reference_table.referenced_table_7000257
- fkey_ref_7000271         | fkey_reference_table.referencing_table2_7000271 | fkey_reference_table.referenced_table_7000257
- fkey_ref_7000272         | fkey_reference_table.referencing_table2_7000272 | fkey_reference_table.referenced_table_7000257
- fkey_ref_7000273         | fkey_reference_table.referencing_table2_7000273 | fkey_reference_table.referenced_table_7000257
- fkey_ref_to_dist_7000266 | fkey_reference_table.referencing_table2_7000266 | fkey_reference_table.referencing_table_7000258
- fkey_ref_to_dist_7000267 | fkey_reference_table.referencing_table2_7000267 | fkey_reference_table.referencing_table_7000259
- fkey_ref_to_dist_7000268 | fkey_reference_table.referencing_table2_7000268 | fkey_reference_table.referencing_table_7000260
- fkey_ref_to_dist_7000269 | fkey_reference_table.referencing_table2_7000269 | fkey_reference_table.referencing_table_7000261
- fkey_ref_to_dist_7000270 | fkey_reference_table.referencing_table2_7000270 | fkey_reference_table.referencing_table_7000262
- fkey_ref_to_dist_7000271 | fkey_reference_table.referencing_table2_7000271 | fkey_reference_table.referencing_table_7000263
- fkey_ref_to_dist_7000272 | fkey_reference_table.referencing_table2_7000272 | fkey_reference_table.referencing_table_7000264
- fkey_ref_to_dist_7000273 | fkey_reference_table.referencing_table2_7000273 | fkey_reference_table.referencing_table_7000265
+ fkey_ref_7000274         | fkey_reference_table.referencing_table_7000274  | fkey_reference_table.referenced_table_7000273
+ fkey_ref_7000275         | fkey_reference_table.referencing_table_7000275  | fkey_reference_table.referenced_table_7000273
+ fkey_ref_7000276         | fkey_reference_table.referencing_table_7000276  | fkey_reference_table.referenced_table_7000273
+ fkey_ref_7000277         | fkey_reference_table.referencing_table_7000277  | fkey_reference_table.referenced_table_7000273
+ fkey_ref_7000278         | fkey_reference_table.referencing_table_7000278  | fkey_reference_table.referenced_table_7000273
+ fkey_ref_7000279         | fkey_reference_table.referencing_table_7000279  | fkey_reference_table.referenced_table_7000273
+ fkey_ref_7000280         | fkey_reference_table.referencing_table_7000280  | fkey_reference_table.referenced_table_7000273
+ fkey_ref_7000281         | fkey_reference_table.referencing_table_7000281  | fkey_reference_table.referenced_table_7000273
+ fkey_ref_7000282         | fkey_reference_table.referencing_table2_7000282 | fkey_reference_table.referenced_table_7000273
+ fkey_ref_7000283         | fkey_reference_table.referencing_table2_7000283 | fkey_reference_table.referenced_table_7000273
+ fkey_ref_7000284         | fkey_reference_table.referencing_table2_7000284 | fkey_reference_table.referenced_table_7000273
+ fkey_ref_7000285         | fkey_reference_table.referencing_table2_7000285 | fkey_reference_table.referenced_table_7000273
+ fkey_ref_7000286         | fkey_reference_table.referencing_table2_7000286 | fkey_reference_table.referenced_table_7000273
+ fkey_ref_7000287         | fkey_reference_table.referencing_table2_7000287 | fkey_reference_table.referenced_table_7000273
+ fkey_ref_7000288         | fkey_reference_table.referencing_table2_7000288 | fkey_reference_table.referenced_table_7000273
+ fkey_ref_7000289         | fkey_reference_table.referencing_table2_7000289 | fkey_reference_table.referenced_table_7000273
+ fkey_ref_to_dist_7000282 | fkey_reference_table.referencing_table2_7000282 | fkey_reference_table.referencing_table_7000274
+ fkey_ref_to_dist_7000283 | fkey_reference_table.referencing_table2_7000283 | fkey_reference_table.referencing_table_7000275
+ fkey_ref_to_dist_7000284 | fkey_reference_table.referencing_table2_7000284 | fkey_reference_table.referencing_table_7000276
+ fkey_ref_to_dist_7000285 | fkey_reference_table.referencing_table2_7000285 | fkey_reference_table.referencing_table_7000277
+ fkey_ref_to_dist_7000286 | fkey_reference_table.referencing_table2_7000286 | fkey_reference_table.referencing_table_7000278
+ fkey_ref_to_dist_7000287 | fkey_reference_table.referencing_table2_7000287 | fkey_reference_table.referencing_table_7000279
+ fkey_ref_to_dist_7000288 | fkey_reference_table.referencing_table2_7000288 | fkey_reference_table.referencing_table_7000280
+ fkey_ref_to_dist_7000289 | fkey_reference_table.referencing_table2_7000289 | fkey_reference_table.referencing_table_7000281
 (24 rows)
 
 INSERT INTO referenced_table SELECT x, x+1 FROM generate_series(0,1000) AS f(x);
 -- should fail
 INSERT INTO referencing_table2 SELECT x, x+1 FROM generate_series(0,100) AS f(x);
-ERROR:  insert or update on table "referencing_table2_7000272" violates foreign key constraint "fkey_ref_to_dist_7000272"
-DETAIL:  Key (id)=(2) is not present in table "referencing_table_7000264".
+ERROR:  insert or update on table "referencing_table2_7000284" violates foreign key constraint "fkey_ref_to_dist_7000284"
+DETAIL:  Key (id)=(4) is not present in table "referencing_table_7000276".
 -- should success
 INSERT INTO referencing_table SELECT x, x+1 FROM generate_series(0,400) AS f(x);
 -- should fail
 INSERT INTO referencing_table2 SELECT x, x+1 FROM generate_series(200,500) AS f(x);
-ERROR:  insert or update on table "referencing_table2_7000272" violates foreign key constraint "fkey_ref_to_dist_7000272"
-DETAIL:  Key (id)=(404) is not present in table "referencing_table_7000264".
+ERROR:  insert or update on table "referencing_table2_7000284" violates foreign key constraint "fkey_ref_to_dist_7000284"
+DETAIL:  Key (id)=(408) is not present in table "referencing_table_7000276".
 -- should success
 INSERT INTO referencing_table2 SELECT x, x+1 FROM generate_series(0,300) AS f(x);
 DELETE FROM referenced_table WHERE test_column < 200;
@@ -1203,22 +1239,22 @@ ALTER TABLE referencing_table ADD CONSTRAINT fkey_ref FOREIGN KEY (ref_id, ref_i
 SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.referencing%' ORDER BY 1,2,3;
                      name                      |                           relid                            |                   refd_relid                   
 -----------------------------------------------+------------------------------------------------------------+------------------------------------------------
- fkey_ref_7000292                              | fkey_reference_table.referencing_table_7000292             | fkey_reference_table.referenced_table_7000291
- fkey_ref_7000293                              | fkey_reference_table.referencing_table_7000293             | fkey_reference_table.referenced_table_7000291
- fkey_ref_7000294                              | fkey_reference_table.referencing_table_7000294             | fkey_reference_table.referenced_table_7000291
- fkey_ref_7000295                              | fkey_reference_table.referencing_table_7000295             | fkey_reference_table.referenced_table_7000291
- fkey_ref_7000296                              | fkey_reference_table.referencing_table_7000296             | fkey_reference_table.referenced_table_7000291
- fkey_ref_7000297                              | fkey_reference_table.referencing_table_7000297             | fkey_reference_table.referenced_table_7000291
- fkey_ref_7000298                              | fkey_reference_table.referencing_table_7000298             | fkey_reference_table.referenced_table_7000291
- fkey_ref_7000299                              | fkey_reference_table.referencing_table_7000299             | fkey_reference_table.referenced_table_7000291
- referencing_referencing_table_id_fkey_7000300 | fkey_reference_table.referencing_referencing_table_7000300 | fkey_reference_table.referencing_table_7000292
- referencing_referencing_table_id_fkey_7000301 | fkey_reference_table.referencing_referencing_table_7000301 | fkey_reference_table.referencing_table_7000293
- referencing_referencing_table_id_fkey_7000302 | fkey_reference_table.referencing_referencing_table_7000302 | fkey_reference_table.referencing_table_7000294
- referencing_referencing_table_id_fkey_7000303 | fkey_reference_table.referencing_referencing_table_7000303 | fkey_reference_table.referencing_table_7000295
- referencing_referencing_table_id_fkey_7000304 | fkey_reference_table.referencing_referencing_table_7000304 | fkey_reference_table.referencing_table_7000296
- referencing_referencing_table_id_fkey_7000305 | fkey_reference_table.referencing_referencing_table_7000305 | fkey_reference_table.referencing_table_7000297
- referencing_referencing_table_id_fkey_7000306 | fkey_reference_table.referencing_referencing_table_7000306 | fkey_reference_table.referencing_table_7000298
- referencing_referencing_table_id_fkey_7000307 | fkey_reference_table.referencing_referencing_table_7000307 | fkey_reference_table.referencing_table_7000299
+ fkey_ref_7000308                              | fkey_reference_table.referencing_table_7000308             | fkey_reference_table.referenced_table_7000307
+ fkey_ref_7000309                              | fkey_reference_table.referencing_table_7000309             | fkey_reference_table.referenced_table_7000307
+ fkey_ref_7000310                              | fkey_reference_table.referencing_table_7000310             | fkey_reference_table.referenced_table_7000307
+ fkey_ref_7000311                              | fkey_reference_table.referencing_table_7000311             | fkey_reference_table.referenced_table_7000307
+ fkey_ref_7000312                              | fkey_reference_table.referencing_table_7000312             | fkey_reference_table.referenced_table_7000307
+ fkey_ref_7000313                              | fkey_reference_table.referencing_table_7000313             | fkey_reference_table.referenced_table_7000307
+ fkey_ref_7000314                              | fkey_reference_table.referencing_table_7000314             | fkey_reference_table.referenced_table_7000307
+ fkey_ref_7000315                              | fkey_reference_table.referencing_table_7000315             | fkey_reference_table.referenced_table_7000307
+ referencing_referencing_table_id_fkey_7000316 | fkey_reference_table.referencing_referencing_table_7000316 | fkey_reference_table.referencing_table_7000308
+ referencing_referencing_table_id_fkey_7000317 | fkey_reference_table.referencing_referencing_table_7000317 | fkey_reference_table.referencing_table_7000309
+ referencing_referencing_table_id_fkey_7000318 | fkey_reference_table.referencing_referencing_table_7000318 | fkey_reference_table.referencing_table_7000310
+ referencing_referencing_table_id_fkey_7000319 | fkey_reference_table.referencing_referencing_table_7000319 | fkey_reference_table.referencing_table_7000311
+ referencing_referencing_table_id_fkey_7000320 | fkey_reference_table.referencing_referencing_table_7000320 | fkey_reference_table.referencing_table_7000312
+ referencing_referencing_table_id_fkey_7000321 | fkey_reference_table.referencing_referencing_table_7000321 | fkey_reference_table.referencing_table_7000313
+ referencing_referencing_table_id_fkey_7000322 | fkey_reference_table.referencing_referencing_table_7000322 | fkey_reference_table.referencing_table_7000314
+ referencing_referencing_table_id_fkey_7000323 | fkey_reference_table.referencing_referencing_table_7000323 | fkey_reference_table.referencing_table_7000315
 (16 rows)
 
 INSERT INTO referenced_table SELECT x, x+1 FROM generate_series(1,1000) AS f(x);
@@ -1283,7 +1319,7 @@ BEGIN;
 (1 row)
 
   ALTER TABLE test_table_2 ADD CONSTRAINT c_check FOREIGN KEY (value_1) REFERENCES test_table_1(id);
-ERROR:  cannot modify table "test_table_2" because there was a parallel operation on a distributed tablein the transaction
+ERROR:  cannot modify table "test_table_2" because there was a parallel operation on a distributed table in the transaction
 DETAIL:  When there is a foreign key to a reference table, Citus needs to perform all operations over a single connection per node to ensure consistency.
 HINT:  Try re-running the transaction with "SET LOCAL citus.multi_shard_modify_mode TO 'sequential';"
   DROP TABLE test_table_1, test_table_2;
@@ -1306,7 +1342,7 @@ BEGIN;
 (1 row)
 
   ALTER TABLE test_table_1 ADD CONSTRAINT c_check FOREIGN KEY (value_1) REFERENCES test_table_2(id);
-ERROR:  cannot modify table "test_table_1" because there was a parallel operation on a distributed tablein the transaction
+ERROR:  cannot modify table "test_table_1" because there was a parallel operation on a distributed table in the transaction
 DETAIL:  When there is a foreign key to a reference table, Citus needs to perform all operations over a single connection per node to ensure consistency.
 HINT:  Try re-running the transaction with "SET LOCAL citus.multi_shard_modify_mode TO 'sequential';"
   DROP TABLE test_table_2 CASCADE;
@@ -1399,7 +1435,7 @@ BEGIN;
 (1 row)
 
   ALTER TABLE test_table_2 ADD CONSTRAINT foreign_key FOREIGN KEY(value_1) REFERENCES test_table_1(id);
-ERROR:  cannot modify table "test_table_2" because there was a parallel operation on a distributed tablein the transaction
+ERROR:  cannot modify table "test_table_2" because there was a parallel operation on a distributed table in the transaction
 DETAIL:  When there is a foreign key to a reference table, Citus needs to perform all operations over a single connection per node to ensure consistency.
 HINT:  Try re-running the transaction with "SET LOCAL citus.multi_shard_modify_mode TO 'sequential';"
   ALTER TABLE test_table_2 DROP CONSTRAINT test_table_2_value_1_fkey;
@@ -1830,8 +1866,8 @@ ALTER TABLE referencing_table_4 ADD CONSTRAINT fkey FOREIGN KEY (id) REFERENCES
 ALTER TABLE referencing_table_4 ADD CONSTRAINT fkey_to_ref FOREIGN KEY (value_1) REFERENCES referenced_table;
 -- should fail since the data will flow to partitioning_test_4 and it has a foreign constraint to partitioning_test_0 on id column
 INSERT INTO referencing_table VALUES (0, 5);
-ERROR:  insert or update on table "referencing_table_4_7000533" violates foreign key constraint "fkey_7000533"
-DETAIL:  Key (id)=(0) is not present in table "referencing_table_0_7000517".
+ERROR:  insert or update on table "referencing_table_4_7000549" violates foreign key constraint "fkey_7000549"
+DETAIL:  Key (id)=(0) is not present in table "referencing_table_0_7000533".
 CONTEXT:  while executing command on localhost:57638
 -- should succeed on partitioning_test_0
 INSERT INTO referencing_table VALUES (0, 1);
@@ -1843,8 +1879,8 @@ SELECT * FROM referencing_table;
 
 -- should fail since partitioning_test_4 has foreign constraint to referenced_table on value_1 column
 INSERT INTO referencing_table VALUES (0, 5);
-ERROR:  insert or update on table "referencing_table_4_7000533" violates foreign key constraint "fkey_to_ref_7000533"
-DETAIL:  Key (value_1)=(5) is not present in table "referenced_table_7000505".
+ERROR:  insert or update on table "referencing_table_4_7000549" violates foreign key constraint "fkey_to_ref_7000549"
+DETAIL:  Key (value_1)=(5) is not present in table "referenced_table_7000521".
 CONTEXT:  while executing command on localhost:57638
 INSERT INTO referenced_table VALUES(5,5);
 -- should succeed since both of the foreign constraints are positive

src/test/regress/expected/foreign_key_to_reference_table_0.out

@@ -293,6 +293,24 @@ SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' A
  fkey_ref_7000098 | fkey_reference_table.referencing_table_7000098 | fkey_reference_table.referenced_table_7000042
 (8 rows)
 
+DROP TABLE referencing_table;
+-- check if we can add the foreign key while adding the column
+CREATE TABLE referencing_table(id int, ref_id int);
+SELECT create_distributed_table('referencing_table', 'ref_id');
+ create_distributed_table 
+--------------------------
+ 
+(1 row)
+
+ALTER TABLE referencing_table ADD COLUMN referencing int REFERENCES referenced_table(id) ON UPDATE CASCADE;
+ERROR:  cannot execute ADD COLUMN command with PRIMARY KEY, UNIQUE, FOREIGN and CHECK constraints
+DETAIL:  Adding a column with a constraint in one command is not supported because all constraints in Citus must have explicit names
+HINT:  You can issue each command separately such as ALTER TABLE referencing_table ADD COLUMN referencing data_type; ALTER TABLE referencing_table ADD CONSTRAINT constraint_name FOREIGN KEY (referencing) REFERENCES referenced_table(id) ON UPDATE CASCADE;
+SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
+ name | relid | refd_relid 
+------+-------+------------
+(0 rows)
+
 DROP TABLE referencing_table;
 -- foreign keys are only supported when the replication factor = 1
 SET citus.shard_replication_factor TO 2;
@@ -312,6 +330,24 @@ SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' A
 ------+-------+------------
 (0 rows)
 
+DROP TABLE referencing_table;
+-- should fail when we add the column as well
+CREATE TABLE referencing_table(id int, ref_id int);
+SELECT create_distributed_table('referencing_table', 'ref_id');
+ create_distributed_table 
+--------------------------
+ 
+(1 row)
+
+ALTER TABLE referencing_table ADD COLUMN referencing_col int REFERENCES referenced_table(id) ON DELETE SET NULL;
+ERROR:  cannot create foreign key constraint
+DETAIL:  Citus Community Edition currently supports foreign key constraints only for "citus.shard_replication_factor = 1".
+HINT:  Please change "citus.shard_replication_factor to 1". To learn more about using foreign keys with other replication factors, please contact us at https://citusdata.com/about/contact_us.
+SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
+ name | relid | refd_relid 
+------+-------+------------
+(0 rows)
+
 DROP TABLE referencing_table;
 SET citus.shard_replication_factor TO 1;
 -- simple create_distributed_table should work in/out transactions on tables with foreign key to reference tables
@@ -325,14 +361,14 @@ SELECT create_distributed_table('referencing_table', 'ref_id');
 SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
                name                |                     relid                      |                  refd_relid                   
 -----------------------------------+------------------------------------------------+-----------------------------------------------
- referencing_table_id_fkey_7000107 | fkey_reference_table.referencing_table_7000107 | fkey_reference_table.referenced_table_7000042
- referencing_table_id_fkey_7000108 | fkey_reference_table.referencing_table_7000108 | fkey_reference_table.referenced_table_7000042
- referencing_table_id_fkey_7000109 | fkey_reference_table.referencing_table_7000109 | fkey_reference_table.referenced_table_7000042
- referencing_table_id_fkey_7000110 | fkey_reference_table.referencing_table_7000110 | fkey_reference_table.referenced_table_7000042
- referencing_table_id_fkey_7000111 | fkey_reference_table.referencing_table_7000111 | fkey_reference_table.referenced_table_7000042
- referencing_table_id_fkey_7000112 | fkey_reference_table.referencing_table_7000112 | fkey_reference_table.referenced_table_7000042
- referencing_table_id_fkey_7000113 | fkey_reference_table.referencing_table_7000113 | fkey_reference_table.referenced_table_7000042
- referencing_table_id_fkey_7000114 | fkey_reference_table.referencing_table_7000114 | fkey_reference_table.referenced_table_7000042
+ referencing_table_id_fkey_7000123 | fkey_reference_table.referencing_table_7000123 | fkey_reference_table.referenced_table_7000042
+ referencing_table_id_fkey_7000124 | fkey_reference_table.referencing_table_7000124 | fkey_reference_table.referenced_table_7000042
+ referencing_table_id_fkey_7000125 | fkey_reference_table.referencing_table_7000125 | fkey_reference_table.referenced_table_7000042
+ referencing_table_id_fkey_7000126 | fkey_reference_table.referencing_table_7000126 | fkey_reference_table.referenced_table_7000042
+ referencing_table_id_fkey_7000127 | fkey_reference_table.referencing_table_7000127 | fkey_reference_table.referenced_table_7000042
+ referencing_table_id_fkey_7000128 | fkey_reference_table.referencing_table_7000128 | fkey_reference_table.referenced_table_7000042
+ referencing_table_id_fkey_7000129 | fkey_reference_table.referencing_table_7000129 | fkey_reference_table.referenced_table_7000042
+ referencing_table_id_fkey_7000130 | fkey_reference_table.referencing_table_7000130 | fkey_reference_table.referenced_table_7000042
 (8 rows)
 
 DROP TABLE referencing_table;
@@ -356,14 +392,14 @@ COMMIT;
 SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
                name                |                     relid                      |                  refd_relid                   
 -----------------------------------+------------------------------------------------+-----------------------------------------------
- referencing_table_id_fkey_7000116 | fkey_reference_table.referencing_table_7000116 | fkey_reference_table.referenced_table_7000115
- referencing_table_id_fkey_7000117 | fkey_reference_table.referencing_table_7000117 | fkey_reference_table.referenced_table_7000115
- referencing_table_id_fkey_7000118 | fkey_reference_table.referencing_table_7000118 | fkey_reference_table.referenced_table_7000115
- referencing_table_id_fkey_7000119 | fkey_reference_table.referencing_table_7000119 | fkey_reference_table.referenced_table_7000115
- referencing_table_id_fkey_7000120 | fkey_reference_table.referencing_table_7000120 | fkey_reference_table.referenced_table_7000115
- referencing_table_id_fkey_7000121 | fkey_reference_table.referencing_table_7000121 | fkey_reference_table.referenced_table_7000115
- referencing_table_id_fkey_7000122 | fkey_reference_table.referencing_table_7000122 | fkey_reference_table.referenced_table_7000115
- referencing_table_id_fkey_7000123 | fkey_reference_table.referencing_table_7000123 | fkey_reference_table.referenced_table_7000115
+ referencing_table_id_fkey_7000132 | fkey_reference_table.referencing_table_7000132 | fkey_reference_table.referenced_table_7000131
+ referencing_table_id_fkey_7000133 | fkey_reference_table.referencing_table_7000133 | fkey_reference_table.referenced_table_7000131
+ referencing_table_id_fkey_7000134 | fkey_reference_table.referencing_table_7000134 | fkey_reference_table.referenced_table_7000131
+ referencing_table_id_fkey_7000135 | fkey_reference_table.referencing_table_7000135 | fkey_reference_table.referenced_table_7000131
+ referencing_table_id_fkey_7000136 | fkey_reference_table.referencing_table_7000136 | fkey_reference_table.referenced_table_7000131
+ referencing_table_id_fkey_7000137 | fkey_reference_table.referencing_table_7000137 | fkey_reference_table.referenced_table_7000131
+ referencing_table_id_fkey_7000138 | fkey_reference_table.referencing_table_7000138 | fkey_reference_table.referenced_table_7000131
+ referencing_table_id_fkey_7000139 | fkey_reference_table.referencing_table_7000139 | fkey_reference_table.referenced_table_7000131
 (8 rows)
 
 DROP TABLE referencing_table;
@@ -421,8 +457,8 @@ ALTER TABLE referencing_table ADD CONSTRAINT fkey_ref FOREIGN KEY (ref_id) REFER
 -- test inserts
 -- test insert to referencing table while there is NO corresponding value in referenced table
 INSERT INTO referencing_table VALUES(1, 1);
-ERROR:  insert or update on table "referencing_table_7000125" violates foreign key constraint "fkey_ref_7000125"
-DETAIL:  Key (ref_id)=(1) is not present in table "referenced_table_7000124".
+ERROR:  insert or update on table "referencing_table_7000141" violates foreign key constraint "fkey_ref_7000141"
+DETAIL:  Key (ref_id)=(1) is not present in table "referenced_table_7000140".
 CONTEXT:  while executing command on localhost:57637
 -- test insert to referencing while there is corresponding value in referenced table
 INSERT INTO referenced_table SELECT x, x from generate_series(1,1000) as f(x);
@@ -430,8 +466,8 @@ INSERT INTO referencing_table SELECT x, x from generate_series(1,500) as f(x);
 -- test deletes
 -- test delete from referenced table while there is corresponding value in referencing table
 DELETE FROM referenced_table WHERE id > 3;
-ERROR:  update or delete on table "referenced_table_7000124" violates foreign key constraint "fkey_ref_7000127" on table "referencing_table_7000127"
-DETAIL:  Key (id)=(4) is still referenced from table "referencing_table_7000127".
+ERROR:  update or delete on table "referenced_table_7000140" violates foreign key constraint "fkey_ref_7000143" on table "referencing_table_7000143"
+DETAIL:  Key (id)=(4) is still referenced from table "referencing_table_7000143".
 CONTEXT:  while executing command on localhost:57637
 -- test delete from referenced table while there is NO corresponding value in referencing table
 DELETE FROM referenced_table WHERE id = 501;
@@ -626,8 +662,8 @@ INSERT INTO referenced_table SELECT x,x FROM generate_series(1,1000) AS f(x);
 INSERT INTO referencing_table(id) SELECT x FROM generate_series(1,1000) AS f(x);
 -- Fails for non existing value inserts (serial is already incremented)
 INSERT INTO referencing_table(id) SELECT x FROM generate_series(1,10) AS f(x);
-ERROR:  insert or update on table "referencing_table_7000172" violates foreign key constraint "fkey_ref_7000172"
-DETAIL:  Key (ref_id)=(1001) is not present in table "referenced_table_7000171".
+ERROR:  insert or update on table "referencing_table_7000195" violates foreign key constraint "fkey_ref_7000195"
+DETAIL:  Key (ref_id)=(1009) is not present in table "referenced_table_7000187".
 DROP TABLE referenced_table CASCADE;
 NOTICE:  drop cascades to constraint fkey_ref on table referencing_table
 DROP TABLE referencing_table CASCADE;
@@ -657,8 +693,8 @@ INSERT INTO referenced_table(test_column2) SELECT x FROM generate_series(1,1000)
 INSERT INTO referencing_table(id) SELECT x FROM generate_series(1,1000) AS f(x);
 -- Fails for non existing value inserts (serial is already incremented)
 INSERT INTO referencing_table(id) SELECT x FROM generate_series(1,10) AS f(x);
-ERROR:  insert or update on table "referencing_table_7000181" violates foreign key constraint "fkey_ref_7000181"
-DETAIL:  Key (ref_id)=(1001) is not present in table "referenced_table_7000180".
+ERROR:  insert or update on table "referencing_table_7000197" violates foreign key constraint "fkey_ref_7000197"
+DETAIL:  Key (ref_id)=(1001) is not present in table "referenced_table_7000196".
 DROP TABLE referenced_table CASCADE;
 NOTICE:  drop cascades to constraint fkey_ref on table referencing_table
 DROP TABLE referencing_table CASCADE;
@@ -709,8 +745,8 @@ INSERT INTO referenced_table SELECT x, x FROM generate_series(0,1000) AS f(x);
 INSERT INTO referencing_table SELECT x, x FROM generate_series(0,1000) AS f(x);
 -- we expect this to fail because of the foreign constraint.
 INSERT INTO referencing_table SELECT x, x FROM generate_series(1000,1001) AS f(x);
-ERROR:  insert or update on table "referencing_table_7000204" violates foreign key constraint "fkey_ref_7000204"
-DETAIL:  Key (ref_id)=(1001) is not present in table "referenced_table_7000198".
+ERROR:  insert or update on table "referencing_table_7000220" violates foreign key constraint "fkey_ref_7000220"
+DETAIL:  Key (ref_id)=(1001) is not present in table "referenced_table_7000214".
 -- currently not supported
 ALTER TABLE referencing_table VALIDATE CONSTRAINT fkey_ref;
 ERROR:  alter table command is currently unsupported
@@ -809,38 +845,38 @@ ALTER TABLE referencing_table ADD CONSTRAINT foreign_key_2 FOREIGN KEY (id) REFE
 SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
          name          |                     relid                      |                   refd_relid                   
 -----------------------+------------------------------------------------+------------------------------------------------
- fkey_ref_7000219      | fkey_reference_table.referencing_table_7000219 | fkey_reference_table.referenced_table_7000217
- fkey_ref_7000220      | fkey_reference_table.referencing_table_7000220 | fkey_reference_table.referenced_table_7000217
- fkey_ref_7000221      | fkey_reference_table.referencing_table_7000221 | fkey_reference_table.referenced_table_7000217
- fkey_ref_7000222      | fkey_reference_table.referencing_table_7000222 | fkey_reference_table.referenced_table_7000217
- fkey_ref_7000223      | fkey_reference_table.referencing_table_7000223 | fkey_reference_table.referenced_table_7000217
- fkey_ref_7000224      | fkey_reference_table.referencing_table_7000224 | fkey_reference_table.referenced_table_7000217
- fkey_ref_7000225      | fkey_reference_table.referencing_table_7000225 | fkey_reference_table.referenced_table_7000217
- fkey_ref_7000226      | fkey_reference_table.referencing_table_7000226 | fkey_reference_table.referenced_table_7000217
- foreign_key_2_7000219 | fkey_reference_table.referencing_table_7000219 | fkey_reference_table.referenced_table2_7000218
- foreign_key_2_7000220 | fkey_reference_table.referencing_table_7000220 | fkey_reference_table.referenced_table2_7000218
- foreign_key_2_7000221 | fkey_reference_table.referencing_table_7000221 | fkey_reference_table.referenced_table2_7000218
- foreign_key_2_7000222 | fkey_reference_table.referencing_table_7000222 | fkey_reference_table.referenced_table2_7000218
- foreign_key_2_7000223 | fkey_reference_table.referencing_table_7000223 | fkey_reference_table.referenced_table2_7000218
- foreign_key_2_7000224 | fkey_reference_table.referencing_table_7000224 | fkey_reference_table.referenced_table2_7000218
- foreign_key_2_7000225 | fkey_reference_table.referencing_table_7000225 | fkey_reference_table.referenced_table2_7000218
- foreign_key_2_7000226 | fkey_reference_table.referencing_table_7000226 | fkey_reference_table.referenced_table2_7000218
+ fkey_ref_7000235      | fkey_reference_table.referencing_table_7000235 | fkey_reference_table.referenced_table_7000233
+ fkey_ref_7000236      | fkey_reference_table.referencing_table_7000236 | fkey_reference_table.referenced_table_7000233
+ fkey_ref_7000237      | fkey_reference_table.referencing_table_7000237 | fkey_reference_table.referenced_table_7000233
+ fkey_ref_7000238      | fkey_reference_table.referencing_table_7000238 | fkey_reference_table.referenced_table_7000233
+ fkey_ref_7000239      | fkey_reference_table.referencing_table_7000239 | fkey_reference_table.referenced_table_7000233
+ fkey_ref_7000240      | fkey_reference_table.referencing_table_7000240 | fkey_reference_table.referenced_table_7000233
+ fkey_ref_7000241      | fkey_reference_table.referencing_table_7000241 | fkey_reference_table.referenced_table_7000233
+ fkey_ref_7000242      | fkey_reference_table.referencing_table_7000242 | fkey_reference_table.referenced_table_7000233
+ foreign_key_2_7000235 | fkey_reference_table.referencing_table_7000235 | fkey_reference_table.referenced_table2_7000234
+ foreign_key_2_7000236 | fkey_reference_table.referencing_table_7000236 | fkey_reference_table.referenced_table2_7000234
+ foreign_key_2_7000237 | fkey_reference_table.referencing_table_7000237 | fkey_reference_table.referenced_table2_7000234
+ foreign_key_2_7000238 | fkey_reference_table.referencing_table_7000238 | fkey_reference_table.referenced_table2_7000234
+ foreign_key_2_7000239 | fkey_reference_table.referencing_table_7000239 | fkey_reference_table.referenced_table2_7000234
+ foreign_key_2_7000240 | fkey_reference_table.referencing_table_7000240 | fkey_reference_table.referenced_table2_7000234
+ foreign_key_2_7000241 | fkey_reference_table.referencing_table_7000241 | fkey_reference_table.referenced_table2_7000234
+ foreign_key_2_7000242 | fkey_reference_table.referencing_table_7000242 | fkey_reference_table.referenced_table2_7000234
 (16 rows)
 
 INSERT INTO referenced_table SELECT x, x+1 FROM generate_series(0,1000) AS f(x);
 INSERT INTO referenced_table2 SELECT x, x+1 FROM generate_series(500,1500) AS f(x);
 -- should fail
 INSERT INTO referencing_table SELECT x, x+1 FROM generate_series(0,1500) AS f(x);
-ERROR:  insert or update on table "referencing_table_7000220" violates foreign key constraint "foreign_key_2_7000220"
-DETAIL:  Key (id)=(5) is not present in table "referenced_table2_7000218".
+ERROR:  insert or update on table "referencing_table_7000242" violates foreign key constraint "foreign_key_2_7000242"
+DETAIL:  Key (id)=(9) is not present in table "referenced_table2_7000234".
 -- should fail
 INSERT INTO referencing_table SELECT x, x+1 FROM generate_series(0,400) AS f(x);
-ERROR:  insert or update on table "referencing_table_7000220" violates foreign key constraint "foreign_key_2_7000220"
-DETAIL:  Key (id)=(5) is not present in table "referenced_table2_7000218".
+ERROR:  insert or update on table "referencing_table_7000242" violates foreign key constraint "foreign_key_2_7000242"
+DETAIL:  Key (id)=(9) is not present in table "referenced_table2_7000234".
 -- should fail
 INSERT INTO referencing_table SELECT x, x+1 FROM generate_series(1000,1400) AS f(x);
-ERROR:  insert or update on table "referencing_table_7000220" violates foreign key constraint "fkey_ref_7000220"
-DETAIL:  Key (id)=(1005) is not present in table "referenced_table_7000217".
+ERROR:  insert or update on table "referencing_table_7000242" violates foreign key constraint "fkey_ref_7000242"
+DETAIL:  Key (id)=(1023) is not present in table "referenced_table_7000233".
 -- should success
 INSERT INTO referencing_table SELECT x, x+1 FROM generate_series(600,900) AS f(x);
 SELECT count(*) FROM referencing_table;
@@ -936,38 +972,38 @@ COMMIT;
 SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
          name          |                     relid                      |                   refd_relid                   
 -----------------------+------------------------------------------------+------------------------------------------------
- fkey_ref_7000239      | fkey_reference_table.referencing_table_7000239 | fkey_reference_table.referenced_table_7000237
- fkey_ref_7000240      | fkey_reference_table.referencing_table_7000240 | fkey_reference_table.referenced_table_7000237
- fkey_ref_7000241      | fkey_reference_table.referencing_table_7000241 | fkey_reference_table.referenced_table_7000237
- fkey_ref_7000242      | fkey_reference_table.referencing_table_7000242 | fkey_reference_table.referenced_table_7000237
- fkey_ref_7000243      | fkey_reference_table.referencing_table_7000243 | fkey_reference_table.referenced_table_7000237
- fkey_ref_7000244      | fkey_reference_table.referencing_table_7000244 | fkey_reference_table.referenced_table_7000237
- fkey_ref_7000245      | fkey_reference_table.referencing_table_7000245 | fkey_reference_table.referenced_table_7000237
- fkey_ref_7000246      | fkey_reference_table.referencing_table_7000246 | fkey_reference_table.referenced_table_7000237
- foreign_key_2_7000239 | fkey_reference_table.referencing_table_7000239 | fkey_reference_table.referenced_table2_7000238
- foreign_key_2_7000240 | fkey_reference_table.referencing_table_7000240 | fkey_reference_table.referenced_table2_7000238
- foreign_key_2_7000241 | fkey_reference_table.referencing_table_7000241 | fkey_reference_table.referenced_table2_7000238
- foreign_key_2_7000242 | fkey_reference_table.referencing_table_7000242 | fkey_reference_table.referenced_table2_7000238
- foreign_key_2_7000243 | fkey_reference_table.referencing_table_7000243 | fkey_reference_table.referenced_table2_7000238
- foreign_key_2_7000244 | fkey_reference_table.referencing_table_7000244 | fkey_reference_table.referenced_table2_7000238
- foreign_key_2_7000245 | fkey_reference_table.referencing_table_7000245 | fkey_reference_table.referenced_table2_7000238
- foreign_key_2_7000246 | fkey_reference_table.referencing_table_7000246 | fkey_reference_table.referenced_table2_7000238
+ fkey_ref_7000255      | fkey_reference_table.referencing_table_7000255 | fkey_reference_table.referenced_table_7000253
+ fkey_ref_7000256      | fkey_reference_table.referencing_table_7000256 | fkey_reference_table.referenced_table_7000253
+ fkey_ref_7000257      | fkey_reference_table.referencing_table_7000257 | fkey_reference_table.referenced_table_7000253
+ fkey_ref_7000258      | fkey_reference_table.referencing_table_7000258 | fkey_reference_table.referenced_table_7000253
+ fkey_ref_7000259      | fkey_reference_table.referencing_table_7000259 | fkey_reference_table.referenced_table_7000253
+ fkey_ref_7000260      | fkey_reference_table.referencing_table_7000260 | fkey_reference_table.referenced_table_7000253
+ fkey_ref_7000261      | fkey_reference_table.referencing_table_7000261 | fkey_reference_table.referenced_table_7000253
+ fkey_ref_7000262      | fkey_reference_table.referencing_table_7000262 | fkey_reference_table.referenced_table_7000253
+ foreign_key_2_7000255 | fkey_reference_table.referencing_table_7000255 | fkey_reference_table.referenced_table2_7000254
+ foreign_key_2_7000256 | fkey_reference_table.referencing_table_7000256 | fkey_reference_table.referenced_table2_7000254
+ foreign_key_2_7000257 | fkey_reference_table.referencing_table_7000257 | fkey_reference_table.referenced_table2_7000254
+ foreign_key_2_7000258 | fkey_reference_table.referencing_table_7000258 | fkey_reference_table.referenced_table2_7000254
+ foreign_key_2_7000259 | fkey_reference_table.referencing_table_7000259 | fkey_reference_table.referenced_table2_7000254
+ foreign_key_2_7000260 | fkey_reference_table.referencing_table_7000260 | fkey_reference_table.referenced_table2_7000254
+ foreign_key_2_7000261 | fkey_reference_table.referencing_table_7000261 | fkey_reference_table.referenced_table2_7000254
+ foreign_key_2_7000262 | fkey_reference_table.referencing_table_7000262 | fkey_reference_table.referenced_table2_7000254
 (16 rows)
 
 INSERT INTO referenced_table SELECT x, x+1 FROM generate_series(0,1000) AS f(x);
 INSERT INTO referenced_table2 SELECT x, x+1 FROM generate_series(500,1500) AS f(x);
 -- should fail
 INSERT INTO referencing_table SELECT x, x+1 FROM generate_series(0,1500) AS f(x);
-ERROR:  insert or update on table "referencing_table_7000245" violates foreign key constraint "foreign_key_2_7000245"
-DETAIL:  Key (ref_id)=(3) is not present in table "referenced_table2_7000238".
+ERROR:  insert or update on table "referencing_table_7000260" violates foreign key constraint "foreign_key_2_7000260"
+DETAIL:  Key (ref_id)=(7) is not present in table "referenced_table2_7000254".
 -- should fail
 INSERT INTO referencing_table SELECT x, x+1 FROM generate_series(0,400) AS f(x);
-ERROR:  insert or update on table "referencing_table_7000245" violates foreign key constraint "foreign_key_2_7000245"
-DETAIL:  Key (ref_id)=(3) is not present in table "referenced_table2_7000238".
+ERROR:  insert or update on table "referencing_table_7000260" violates foreign key constraint "foreign_key_2_7000260"
+DETAIL:  Key (ref_id)=(7) is not present in table "referenced_table2_7000254".
 -- should fail
 INSERT INTO referencing_table SELECT x, x+1 FROM generate_series(1000,1400) AS f(x);
-ERROR:  insert or update on table "referencing_table_7000245" violates foreign key constraint "fkey_ref_7000245"
-DETAIL:  Key (id)=(1002) is not present in table "referenced_table_7000237".
+ERROR:  insert or update on table "referencing_table_7000260" violates foreign key constraint "fkey_ref_7000260"
+DETAIL:  Key (id)=(1001) is not present in table "referenced_table_7000253".
 -- should success
 INSERT INTO referencing_table SELECT x, x+501 FROM generate_series(0,1000) AS f(x);
 SELECT count(*) FROM referencing_table;
@@ -1069,43 +1105,43 @@ COMMIT;
 SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
            name           |                      relid                      |                   refd_relid                   
 --------------------------+-------------------------------------------------+------------------------------------------------
- fkey_ref_7000258         | fkey_reference_table.referencing_table_7000258  | fkey_reference_table.referenced_table_7000257
- fkey_ref_7000259         | fkey_reference_table.referencing_table_7000259  | fkey_reference_table.referenced_table_7000257
- fkey_ref_7000260         | fkey_reference_table.referencing_table_7000260  | fkey_reference_table.referenced_table_7000257
- fkey_ref_7000261         | fkey_reference_table.referencing_table_7000261  | fkey_reference_table.referenced_table_7000257
- fkey_ref_7000262         | fkey_reference_table.referencing_table_7000262  | fkey_reference_table.referenced_table_7000257
- fkey_ref_7000263         | fkey_reference_table.referencing_table_7000263  | fkey_reference_table.referenced_table_7000257
- fkey_ref_7000264         | fkey_reference_table.referencing_table_7000264  | fkey_reference_table.referenced_table_7000257
- fkey_ref_7000265         | fkey_reference_table.referencing_table_7000265  | fkey_reference_table.referenced_table_7000257
- fkey_ref_7000266         | fkey_reference_table.referencing_table2_7000266 | fkey_reference_table.referenced_table_7000257
- fkey_ref_7000267         | fkey_reference_table.referencing_table2_7000267 | fkey_reference_table.referenced_table_7000257
- fkey_ref_7000268         | fkey_reference_table.referencing_table2_7000268 | fkey_reference_table.referenced_table_7000257
- fkey_ref_7000269         | fkey_reference_table.referencing_table2_7000269 | fkey_reference_table.referenced_table_7000257
- fkey_ref_7000270         | fkey_reference_table.referencing_table2_7000270 | fkey_reference_table.referenced_table_7000257
- fkey_ref_7000271         | fkey_reference_table.referencing_table2_7000271 | fkey_reference_table.referenced_table_7000257
- fkey_ref_7000272         | fkey_reference_table.referencing_table2_7000272 | fkey_reference_table.referenced_table_7000257
- fkey_ref_7000273         | fkey_reference_table.referencing_table2_7000273 | fkey_reference_table.referenced_table_7000257
- fkey_ref_to_dist_7000266 | fkey_reference_table.referencing_table2_7000266 | fkey_reference_table.referencing_table_7000258
- fkey_ref_to_dist_7000267 | fkey_reference_table.referencing_table2_7000267 | fkey_reference_table.referencing_table_7000259
- fkey_ref_to_dist_7000268 | fkey_reference_table.referencing_table2_7000268 | fkey_reference_table.referencing_table_7000260
- fkey_ref_to_dist_7000269 | fkey_reference_table.referencing_table2_7000269 | fkey_reference_table.referencing_table_7000261
- fkey_ref_to_dist_7000270 | fkey_reference_table.referencing_table2_7000270 | fkey_reference_table.referencing_table_7000262
- fkey_ref_to_dist_7000271 | fkey_reference_table.referencing_table2_7000271 | fkey_reference_table.referencing_table_7000263
- fkey_ref_to_dist_7000272 | fkey_reference_table.referencing_table2_7000272 | fkey_reference_table.referencing_table_7000264
- fkey_ref_to_dist_7000273 | fkey_reference_table.referencing_table2_7000273 | fkey_reference_table.referencing_table_7000265
+ fkey_ref_7000274         | fkey_reference_table.referencing_table_7000274  | fkey_reference_table.referenced_table_7000273
+ fkey_ref_7000275         | fkey_reference_table.referencing_table_7000275  | fkey_reference_table.referenced_table_7000273
+ fkey_ref_7000276         | fkey_reference_table.referencing_table_7000276  | fkey_reference_table.referenced_table_7000273
+ fkey_ref_7000277         | fkey_reference_table.referencing_table_7000277  | fkey_reference_table.referenced_table_7000273
+ fkey_ref_7000278         | fkey_reference_table.referencing_table_7000278  | fkey_reference_table.referenced_table_7000273
+ fkey_ref_7000279         | fkey_reference_table.referencing_table_7000279  | fkey_reference_table.referenced_table_7000273
+ fkey_ref_7000280         | fkey_reference_table.referencing_table_7000280  | fkey_reference_table.referenced_table_7000273
+ fkey_ref_7000281         | fkey_reference_table.referencing_table_7000281  | fkey_reference_table.referenced_table_7000273
+ fkey_ref_7000282         | fkey_reference_table.referencing_table2_7000282 | fkey_reference_table.referenced_table_7000273
+ fkey_ref_7000283         | fkey_reference_table.referencing_table2_7000283 | fkey_reference_table.referenced_table_7000273
+ fkey_ref_7000284         | fkey_reference_table.referencing_table2_7000284 | fkey_reference_table.referenced_table_7000273
+ fkey_ref_7000285         | fkey_reference_table.referencing_table2_7000285 | fkey_reference_table.referenced_table_7000273
+ fkey_ref_7000286         | fkey_reference_table.referencing_table2_7000286 | fkey_reference_table.referenced_table_7000273
+ fkey_ref_7000287         | fkey_reference_table.referencing_table2_7000287 | fkey_reference_table.referenced_table_7000273
+ fkey_ref_7000288         | fkey_reference_table.referencing_table2_7000288 | fkey_reference_table.referenced_table_7000273
+ fkey_ref_7000289         | fkey_reference_table.referencing_table2_7000289 | fkey_reference_table.referenced_table_7000273
+ fkey_ref_to_dist_7000282 | fkey_reference_table.referencing_table2_7000282 | fkey_reference_table.referencing_table_7000274
+ fkey_ref_to_dist_7000283 | fkey_reference_table.referencing_table2_7000283 | fkey_reference_table.referencing_table_7000275
+ fkey_ref_to_dist_7000284 | fkey_reference_table.referencing_table2_7000284 | fkey_reference_table.referencing_table_7000276
+ fkey_ref_to_dist_7000285 | fkey_reference_table.referencing_table2_7000285 | fkey_reference_table.referencing_table_7000277
+ fkey_ref_to_dist_7000286 | fkey_reference_table.referencing_table2_7000286 | fkey_reference_table.referencing_table_7000278
+ fkey_ref_to_dist_7000287 | fkey_reference_table.referencing_table2_7000287 | fkey_reference_table.referencing_table_7000279
+ fkey_ref_to_dist_7000288 | fkey_reference_table.referencing_table2_7000288 | fkey_reference_table.referencing_table_7000280
+ fkey_ref_to_dist_7000289 | fkey_reference_table.referencing_table2_7000289 | fkey_reference_table.referencing_table_7000281
 (24 rows)
 
 INSERT INTO referenced_table SELECT x, x+1 FROM generate_series(0,1000) AS f(x);
 -- should fail
 INSERT INTO referencing_table2 SELECT x, x+1 FROM generate_series(0,100) AS f(x);
-ERROR:  insert or update on table "referencing_table2_7000272" violates foreign key constraint "fkey_ref_to_dist_7000272"
-DETAIL:  Key (id)=(2) is not present in table "referencing_table_7000264".
+ERROR:  insert or update on table "referencing_table2_7000284" violates foreign key constraint "fkey_ref_to_dist_7000284"
+DETAIL:  Key (id)=(4) is not present in table "referencing_table_7000276".
 -- should success
 INSERT INTO referencing_table SELECT x, x+1 FROM generate_series(0,400) AS f(x);
 -- should fail
 INSERT INTO referencing_table2 SELECT x, x+1 FROM generate_series(200,500) AS f(x);
-ERROR:  insert or update on table "referencing_table2_7000272" violates foreign key constraint "fkey_ref_to_dist_7000272"
-DETAIL:  Key (id)=(404) is not present in table "referencing_table_7000264".
+ERROR:  insert or update on table "referencing_table2_7000284" violates foreign key constraint "fkey_ref_to_dist_7000284"
+DETAIL:  Key (id)=(408) is not present in table "referencing_table_7000276".
 -- should success
 INSERT INTO referencing_table2 SELECT x, x+1 FROM generate_series(0,300) AS f(x);
 DELETE FROM referenced_table WHERE test_column < 200;
@@ -1203,22 +1239,22 @@ ALTER TABLE referencing_table ADD CONSTRAINT fkey_ref FOREIGN KEY (ref_id, ref_i
 SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.referencing%' ORDER BY 1,2,3;
                      name                      |                           relid                            |                   refd_relid                   
 -----------------------------------------------+------------------------------------------------------------+------------------------------------------------
- fkey_ref_7000292                              | fkey_reference_table.referencing_table_7000292             | fkey_reference_table.referenced_table_7000291
- fkey_ref_7000293                              | fkey_reference_table.referencing_table_7000293             | fkey_reference_table.referenced_table_7000291
- fkey_ref_7000294                              | fkey_reference_table.referencing_table_7000294             | fkey_reference_table.referenced_table_7000291
- fkey_ref_7000295                              | fkey_reference_table.referencing_table_7000295             | fkey_reference_table.referenced_table_7000291
- fkey_ref_7000296                              | fkey_reference_table.referencing_table_7000296             | fkey_reference_table.referenced_table_7000291
- fkey_ref_7000297                              | fkey_reference_table.referencing_table_7000297             | fkey_reference_table.referenced_table_7000291
- fkey_ref_7000298                              | fkey_reference_table.referencing_table_7000298             | fkey_reference_table.referenced_table_7000291
- fkey_ref_7000299                              | fkey_reference_table.referencing_table_7000299             | fkey_reference_table.referenced_table_7000291
- referencing_referencing_table_id_fkey_7000300 | fkey_reference_table.referencing_referencing_table_7000300 | fkey_reference_table.referencing_table_7000292
- referencing_referencing_table_id_fkey_7000301 | fkey_reference_table.referencing_referencing_table_7000301 | fkey_reference_table.referencing_table_7000293
- referencing_referencing_table_id_fkey_7000302 | fkey_reference_table.referencing_referencing_table_7000302 | fkey_reference_table.referencing_table_7000294
- referencing_referencing_table_id_fkey_7000303 | fkey_reference_table.referencing_referencing_table_7000303 | fkey_reference_table.referencing_table_7000295
- referencing_referencing_table_id_fkey_7000304 | fkey_reference_table.referencing_referencing_table_7000304 | fkey_reference_table.referencing_table_7000296
- referencing_referencing_table_id_fkey_7000305 | fkey_reference_table.referencing_referencing_table_7000305 | fkey_reference_table.referencing_table_7000297
- referencing_referencing_table_id_fkey_7000306 | fkey_reference_table.referencing_referencing_table_7000306 | fkey_reference_table.referencing_table_7000298
- referencing_referencing_table_id_fkey_7000307 | fkey_reference_table.referencing_referencing_table_7000307 | fkey_reference_table.referencing_table_7000299
+ fkey_ref_7000308                              | fkey_reference_table.referencing_table_7000308             | fkey_reference_table.referenced_table_7000307
+ fkey_ref_7000309                              | fkey_reference_table.referencing_table_7000309             | fkey_reference_table.referenced_table_7000307
+ fkey_ref_7000310                              | fkey_reference_table.referencing_table_7000310             | fkey_reference_table.referenced_table_7000307
+ fkey_ref_7000311                              | fkey_reference_table.referencing_table_7000311             | fkey_reference_table.referenced_table_7000307
+ fkey_ref_7000312                              | fkey_reference_table.referencing_table_7000312             | fkey_reference_table.referenced_table_7000307
+ fkey_ref_7000313                              | fkey_reference_table.referencing_table_7000313             | fkey_reference_table.referenced_table_7000307
+ fkey_ref_7000314                              | fkey_reference_table.referencing_table_7000314             | fkey_reference_table.referenced_table_7000307
+ fkey_ref_7000315                              | fkey_reference_table.referencing_table_7000315             | fkey_reference_table.referenced_table_7000307
+ referencing_referencing_table_id_fkey_7000316 | fkey_reference_table.referencing_referencing_table_7000316 | fkey_reference_table.referencing_table_7000308
+ referencing_referencing_table_id_fkey_7000317 | fkey_reference_table.referencing_referencing_table_7000317 | fkey_reference_table.referencing_table_7000309
+ referencing_referencing_table_id_fkey_7000318 | fkey_reference_table.referencing_referencing_table_7000318 | fkey_reference_table.referencing_table_7000310
+ referencing_referencing_table_id_fkey_7000319 | fkey_reference_table.referencing_referencing_table_7000319 | fkey_reference_table.referencing_table_7000311
+ referencing_referencing_table_id_fkey_7000320 | fkey_reference_table.referencing_referencing_table_7000320 | fkey_reference_table.referencing_table_7000312
+ referencing_referencing_table_id_fkey_7000321 | fkey_reference_table.referencing_referencing_table_7000321 | fkey_reference_table.referencing_table_7000313
+ referencing_referencing_table_id_fkey_7000322 | fkey_reference_table.referencing_referencing_table_7000322 | fkey_reference_table.referencing_table_7000314
+ referencing_referencing_table_id_fkey_7000323 | fkey_reference_table.referencing_referencing_table_7000323 | fkey_reference_table.referencing_table_7000315
 (16 rows)
 
 INSERT INTO referenced_table SELECT x, x+1 FROM generate_series(1,1000) AS f(x);
@@ -1283,7 +1319,7 @@ BEGIN;
 (1 row)
 
   ALTER TABLE test_table_2 ADD CONSTRAINT c_check FOREIGN KEY (value_1) REFERENCES test_table_1(id);
-ERROR:  cannot modify table "test_table_2" because there was a parallel operation on a distributed tablein the transaction
+ERROR:  cannot modify table "test_table_2" because there was a parallel operation on a distributed table in the transaction
 DETAIL:  When there is a foreign key to a reference table, Citus needs to perform all operations over a single connection per node to ensure consistency.
 HINT:  Try re-running the transaction with "SET LOCAL citus.multi_shard_modify_mode TO 'sequential';"
   DROP TABLE test_table_1, test_table_2;
@@ -1306,7 +1342,7 @@ BEGIN;
 (1 row)
 
   ALTER TABLE test_table_1 ADD CONSTRAINT c_check FOREIGN KEY (value_1) REFERENCES test_table_2(id);
-ERROR:  cannot modify table "test_table_1" because there was a parallel operation on a distributed tablein the transaction
+ERROR:  cannot modify table "test_table_1" because there was a parallel operation on a distributed table in the transaction
 DETAIL:  When there is a foreign key to a reference table, Citus needs to perform all operations over a single connection per node to ensure consistency.
 HINT:  Try re-running the transaction with "SET LOCAL citus.multi_shard_modify_mode TO 'sequential';"
   DROP TABLE test_table_2 CASCADE;
@@ -1399,7 +1435,7 @@ BEGIN;
 (1 row)
 
   ALTER TABLE test_table_2 ADD CONSTRAINT foreign_key FOREIGN KEY(value_1) REFERENCES test_table_1(id);
-ERROR:  cannot modify table "test_table_2" because there was a parallel operation on a distributed tablein the transaction
+ERROR:  cannot modify table "test_table_2" because there was a parallel operation on a distributed table in the transaction
 DETAIL:  When there is a foreign key to a reference table, Citus needs to perform all operations over a single connection per node to ensure consistency.
 HINT:  Try re-running the transaction with "SET LOCAL citus.multi_shard_modify_mode TO 'sequential';"
   ALTER TABLE test_table_2 DROP CONSTRAINT test_table_2_value_1_fkey;

src/test/regress/expected/multi_alter_table_add_constraints.out

@@ -513,8 +513,8 @@ ORDER BY
     1,2,3,4;
  nodename  | nodeport | success |       result        
 -----------+----------+---------+---------------------
- localhost |    57637 | t       | alter_pk_idx_220026
- localhost |    57638 | t       | alter_pk_idx_220026
+ localhost |    57637 | t       | alter_pk_idx_220035
+ localhost |    57638 | t       | alter_pk_idx_220035
 (2 rows)
 
 CREATE SCHEMA sc2;
@@ -544,8 +544,8 @@ ORDER BY
     1,2,3,4;
  nodename  | nodeport | success |       result        
 -----------+----------+---------+---------------------
- localhost |    57637 | t       | alter_pk_idx_220028
- localhost |    57638 | t       | alter_pk_idx_220028
+ localhost |    57637 | t       | alter_pk_idx_220037
+ localhost |    57638 | t       | alter_pk_idx_220037
 (2 rows)
 
 -- We are running almost the same test with a slight change on the constraint name because if the constraint has a different name than the index, Postgres renames the index.
@@ -579,8 +579,8 @@ ORDER BY
     1,2,3,4;
  nodename  | nodeport | success |       result        
 -----------+----------+---------+---------------------
- localhost |    57637 | t       | a_constraint_220030
- localhost |    57638 | t       | a_constraint_220030
+ localhost |    57637 | t       | a_constraint_220039
+ localhost |    57638 | t       | a_constraint_220039
 (2 rows)
 
 ALTER TABLE alter_add_prim_key DROP CONSTRAINT a_constraint;

src/test/regress/expected/multi_distribution_metadata.out

@@ -38,6 +38,10 @@ CREATE FUNCTION acquire_shared_shard_lock(bigint)
 	RETURNS void
 	AS 'citus'
 	LANGUAGE C STRICT;
+CREATE FUNCTION relation_count_in_query(text)
+	RETURNS int
+	AS 'citus'
+	LANGUAGE C STRICT;
 -- ===================================================================
 -- test distribution metadata functionality
 -- ===================================================================
@@ -464,5 +468,96 @@ SELECT get_shard_id_for_distribution_column('get_shardid_test_table5', -999);
                                     0
 (1 row)
 
+SET citus.shard_count TO 2;
+CREATE TABLE events_table_count (user_id int, time timestamp, event_type int, value_2 int, value_3 float, value_4 bigint);
+SELECT create_distributed_table('events_table_count', 'user_id');
+ create_distributed_table 
+--------------------------
+ 
+(1 row)
+
+CREATE TABLE users_table_count (user_id int, time timestamp, value_1 int, value_2 int, value_3 float, value_4 bigint);
+SELECT create_distributed_table('users_table_count', 'user_id');
+ create_distributed_table 
+--------------------------
+ 
+(1 row)
+
+SELECT relation_count_in_query($$-- we can support arbitrary subqueries within UNIONs
+SELECT ("final_query"."event_types") as types, count(*) AS sumOfEventType
+FROM
+  ( SELECT 
+      *, random()
+    FROM
+     (SELECT 
+        "t"."user_id", "t"."time", unnest("t"."collected_events") AS "event_types"
+      FROM
+        ( SELECT 
+            "t1"."user_id", min("t1"."time") AS "time", array_agg(("t1"."event") ORDER BY TIME ASC, event DESC) AS collected_events
+          FROM (
+                (SELECT 
+                    *
+                 FROM
+                   (SELECT 
+                          events_table."time", 0 AS event, events_table."user_id"
+                    FROM 
+                       "events_table_count" as events_table
+                    WHERE 
+                      events_table.event_type IN (1, 2) ) events_subquery_1) 
+                UNION 
+                 (SELECT *
+                  FROM
+                    (
+                          SELECT * FROM
+                          (
+                              SELECT 
+                                max("events_table_count"."time"),
+                                0 AS event,
+                                "events_table_count"."user_id"
+                              FROM 
+                                "events_table_count", users_table_count as "users"
+                              WHERE 
+                                 "events_table_count".user_id = users.user_id AND
+                                "events_table_count".event_type IN (1, 2)
+                                GROUP BY   "events_table_count"."user_id"
+                          ) as events_subquery_5
+                     ) events_subquery_2)
+               UNION 
+                 (SELECT *
+                  FROM
+                    (SELECT 
+                        "events_table_count"."time", 2 AS event, "events_table_count"."user_id"
+                     FROM 
+                       "events_table_count"
+                     WHERE 
+                      event_type IN (3, 4) ) events_subquery_3)
+               UNION 
+                 (SELECT *
+                  FROM
+                    (SELECT
+                       "events_table_count"."time", 3 AS event, "events_table_count"."user_id"
+                     FROM 
+                       "events_table_count"
+                     WHERE 
+                      event_type IN (5, 6)) events_subquery_4)
+                 ) t1
+         GROUP BY "t1"."user_id") AS t) "q" 
+INNER JOIN
+     (SELECT 
+        "events_table_count"."user_id"
+      FROM 
+        users_table_count as "events_table_count"
+      WHERE 
+        value_1 > 0 and value_1 < 4) AS t 
+     ON (t.user_id = q.user_id)) as final_query
+GROUP BY 
+  types
+ORDER BY 
+  types;$$);
+ relation_count_in_query 
+-------------------------
+                       6
+(1 row)
+
 -- clear unnecessary tables;
-DROP TABLE get_shardid_test_table1, get_shardid_test_table2, get_shardid_test_table3, get_shardid_test_table4, get_shardid_test_table5;
+DROP TABLE get_shardid_test_table1, get_shardid_test_table2, get_shardid_test_table3, get_shardid_test_table4, get_shardid_test_table5, events_table_count;

src/test/regress/expected/multi_extension.out

@@ -147,7 +147,7 @@ ALTER EXTENSION citus UPDATE TO '7.5-7';
 SHOW citus.version;
  citus.version 
 ---------------
- 7.5devel
+ 7.5.3
 (1 row)
 
 -- ensure no objects were created outside pg_catalog

src/test/regress/expected/multi_foreign_key.out

@@ -465,6 +465,10 @@ DETAIL:  SET NULL, SET DEFAULT or CASCADE is not supported in ON UPDATE operatio
 ALTER TABLE referencing_table ADD CONSTRAINT test_constraint FOREIGN KEY(ref_id) REFERENCES referenced_table(id) ON UPDATE CASCADE;
 ERROR:  cannot create foreign key constraint
 DETAIL:  SET NULL, SET DEFAULT or CASCADE is not supported in ON UPDATE operation  when distribution key included in the foreign constraint.
+-- test foreign constraint creation while adding the column
+ALTER TABLE referencing_table ADD COLUMN referencing_col int REFERENCES referenced_table(id) ON UPDATE CASCADE;;
+ERROR:  cannot create foreign key constraint
+DETAIL:  Foreign keys are supported in two cases, either in between two colocated tables including partition column in the same ordinal in the both tables or from distributed to reference tables
 -- test foreign constraint creation with multiple subcommands
 ALTER TABLE referencing_table ADD CONSTRAINT test_constraint FOREIGN KEY(ref_id) REFERENCES referenced_table(id),
 							  ADD CONSTRAINT test_constraint FOREIGN KEY(id) REFERENCES referenced_table(test_column);

src/test/regress/expected/multi_insert_select.out

@@ -2735,6 +2735,22 @@ EXECUTE prepared_recursive_insert_select;
 EXECUTE prepared_recursive_insert_select;
 EXECUTE prepared_recursive_insert_select;
 ROLLBACK;
+-- upsert with on conflict update distribution column is unsupported
+INSERT INTO agg_events AS ae
+            (
+                        user_id,
+                        value_1_agg,
+                        agg_time
+            )
+SELECT user_id,
+       value_1,
+       time
+FROM   raw_events_first
+ON conflict (user_id, value_1_agg)
+DO UPDATE
+   SET    user_id = 42
+RETURNING user_id, value_1_agg;
+ERROR:  modifying the partition value of rows is not allowed
 -- wrap in a transaction to improve performance
 BEGIN;
 DROP TABLE coerce_events;

src/test/regress/expected/multi_multiuser.out

@@ -93,10 +93,10 @@ SELECT count(*) FROM test WHERE id = 1;
 (1 row)
 
 SET citus.task_executor_type TO 'task-tracker';
-SELECT count(*) FROM test;
- count 
--------
-     2
+SELECT count(*), min(current_user) FROM test;
+ count |     min     
+-------+-------------
+     2 | full_access
 (1 row)
 
 -- test re-partition query (needs to transmit intermediate results)
@@ -140,10 +140,10 @@ SELECT count(*) FROM test WHERE id = 1;
 (1 row)
 
 SET citus.task_executor_type TO 'task-tracker';
-SELECT count(*) FROM test;
- count 
--------
-     2
+SELECT count(*), min(current_user) FROM test;
+ count |     min     
+-------+-------------
+     2 | read_access
 (1 row)
 
 -- test re-partition query (needs to transmit intermediate results)
@@ -171,7 +171,7 @@ ERROR:  permission denied for table test
 SELECT count(*) FROM test WHERE id = 1;
 ERROR:  permission denied for table test
 SET citus.task_executor_type TO 'task-tracker';
-SELECT count(*) FROM test;
+SELECT count(*), min(current_user) FROM test;
 ERROR:  permission denied for table test
 -- test re-partition query
 SELECT count(*) FROM test a JOIN test b ON (a.val = b.val) WHERE a.id = 1 AND b.id = 2;

src/test/regress/expected/multi_multiuser_0.out

@@ -93,10 +93,10 @@ SELECT count(*) FROM test WHERE id = 1;
 (1 row)
 
 SET citus.task_executor_type TO 'task-tracker';
-SELECT count(*) FROM test;
- count 
--------
-     2
+SELECT count(*), min(current_user) FROM test;
+ count |     min     
+-------+-------------
+     2 | full_access
 (1 row)
 
 -- test re-partition query (needs to transmit intermediate results)
@@ -140,10 +140,10 @@ SELECT count(*) FROM test WHERE id = 1;
 (1 row)
 
 SET citus.task_executor_type TO 'task-tracker';
-SELECT count(*) FROM test;
- count 
--------
-     2
+SELECT count(*), min(current_user) FROM test;
+ count |     min     
+-------+-------------
+     2 | read_access
 (1 row)
 
 -- test re-partition query (needs to transmit intermediate results)
@@ -171,7 +171,7 @@ ERROR:  permission denied for relation test
 SELECT count(*) FROM test WHERE id = 1;
 ERROR:  permission denied for relation test
 SET citus.task_executor_type TO 'task-tracker';
-SELECT count(*) FROM test;
+SELECT count(*), min(current_user) FROM test;
 ERROR:  permission denied for relation test
 -- test re-partition query
 SELECT count(*) FROM test a JOIN test b ON (a.val = b.val) WHERE a.id = 1 AND b.id = 2;
@@ -212,7 +212,24 @@ ERROR:  permission denied for relation test
 ABORT;
 SELECT * FROM citus_stat_statements_reset();
 ERROR:  permission denied for function citus_stat_statements_reset
+-- table owner should be the same on the shards, even when distributing the table as superuser
+SET ROLE full_access;
+CREATE TABLE my_table (id integer, val integer);
 RESET ROLE;
+SELECT create_distributed_table('my_table', 'id');
+ create_distributed_table 
+--------------------------
+ 
+(1 row)
+
+SELECT result FROM run_command_on_workers($$SELECT tableowner FROM pg_tables WHERE tablename LIKE 'my_table_%' LIMIT 1$$);
+   result    
+-------------
+ full_access
+ full_access
+(2 rows)
+
+DROP TABLE my_table;
 DROP TABLE test;
 DROP USER full_access;
 DROP USER read_access;

src/test/regress/expected/multi_mx_transaction_recovery.out

@@ -50,15 +50,29 @@ PREPARE TRANSACTION 'citus_12_should_commit';
 BEGIN;
 CREATE TABLE should_be_sorted_into_middle (value int);
 PREPARE TRANSACTION 'citus_12_should_be_sorted_into_middle';
+-- this node (e.g., node id 12) should not touch
+-- transactions with different nodeIds in the gid
+BEGIN;
+CREATE TABLE table_should_do_nothing (value int);
+PREPARE TRANSACTION 'citus_122_should_do_nothing';
 -- Add "fake" pg_dist_transaction records and run recovery
 INSERT INTO pg_dist_transaction VALUES (12, 'citus_12_should_commit');
 INSERT INTO pg_dist_transaction VALUES (12, 'citus_12_should_be_forgotten');
+INSERT INTO pg_dist_transaction VALUES (122, 'citus_122_should_do_nothing');
 SELECT recover_prepared_transactions();
  recover_prepared_transactions 
 -------------------------------
                              3
 (1 row)
 
+-- delete the citus_122_should_do_nothing transaction
+DELETE FROM pg_dist_transaction WHERE gid = 'citus_122_should_do_nothing' RETURNING *;
+ groupid |             gid             
+---------+-----------------------------
+     122 | citus_122_should_do_nothing
+(1 row)
+
+ROLLBACK PREPARED 'citus_122_should_do_nothing';
 SELECT count(*) FROM pg_dist_transaction;
  count 
 -------

src/test/regress/expected/multi_task_string_size.out

@@ -225,8 +225,12 @@ ERROR:  parameter "citus.max_task_string_size" cannot be changed without restart
 -- error message may vary between executions
 -- hiding warning and error message
 -- no output means the query has failed
-SET client_min_messages to FATAL;
+SET client_min_messages to ERROR;
+SELECT raise_failed_execution('
 SELECT u.* FROM wide_table u JOIN wide_table v ON (u.long_column_002 = v.long_column_003);
+');
+ERROR:  Task failed to execute
+CONTEXT:  PL/pgSQL function raise_failed_execution(text) line 6 at RAISE
 -- following will succeed since it fetches few columns
 SELECT u.long_column_001, u.long_column_002, u.long_column_003 FROM wide_table u JOIN wide_table v ON (u.long_column_002 = v.long_column_003);
  long_column_001 | long_column_002 | long_column_003 

src/test/regress/expected/multi_test_helpers.out

@@ -84,3 +84,13 @@ $desc_views$
  
 (1 row)
 
+-- Create a function to make sure that queries returning the same result
+CREATE FUNCTION raise_failed_execution(query text) RETURNS void AS $$
+BEGIN
+	EXECUTE query;
+	EXCEPTION WHEN OTHERS THEN
+	IF SQLERRM LIKE 'failed to execute task%' THEN
+		RAISE 'Task failed to execute';
+	END IF;
+END;
+$$LANGUAGE plpgsql;

src/test/regress/expected/multi_utility_warnings.out

@@ -18,3 +18,10 @@ INSERT INTO pg_dist_authinfo VALUES (0, 'new_user', 'password=1234');
 ERROR:  cannot write to pg_dist_authinfo
 DETAIL:  Citus Community Edition does not support the use of custom authentication options.
 HINT:  To learn more about using advanced authentication schemes with Citus, please contact us at https://citusdata.com/about/contact_us
+BEGIN;
+INSERT INTO pg_dist_node VALUES (1234567890, 1234567890, 'localhost', 5432);
+INSERT INTO pg_dist_poolinfo VALUES (1234567890, 'port=1234');
+ERROR:  cannot write to pg_dist_poolinfo
+DETAIL:  Citus Community Edition does not support the use of pooler options.
+HINT:  To learn more about using advanced pooling schemes with Citus, please contact us at https://citusdata.com/about/contact_us
+ROLLBACK;

src/test/regress/input/multi_alter_table_statements.source

@@ -585,3 +585,24 @@ SELECT relname, reloptions FROM pg_class WHERE relname LIKE 'another_index%' ORD
 
 -- get rid of the index
 DROP INDEX another_index;
+
+-- check if we fail properly when a column with un-supported constraint is added
+-- UNIQUE, PRIMARY KEY on non-distribution column is not supported
+-- CHECK, FOREIGN KEY, UNIQE, PRIMARY KEY cannot be added together with ADD COLUMN
+SET citus.shard_replication_factor TO 1;
+CREATE TABLE test_table_1(id int);
+SELECT create_distributed_table('test_table_1', 'id');
+ALTER TABLE test_table_1 ADD COLUMN test_col int UNIQUE;
+ALTER TABLE test_table_1 ADD COLUMN test_col int PRIMARY KEY;
+ALTER TABLE test_table_1 ADD COLUMN test_col int CHECK (test_col > 3);
+
+CREATE TABLE reference_table(i int UNIQUE);
+SELECT create_reference_table('reference_table');
+ALTER TABLE test_table_1 ADD COLUMN test_col int REFERENCES reference_table(i) ON DELETE CASCADE;
+ALTER TABLE test_table_1 ADD COLUMN test_col int REFERENCES reference_table(i) ON DELETE CASCADE ON UPDATE SET NULL;
+DROP TABLE reference_table;
+
+CREATE TABLE referenced_table(i int UNIQUE);
+SELECT create_distributed_table('referenced_table', 'i');
+ALTER TABLE test_table_1 ADD COLUMN test_col int REFERENCES referenced_table(i);
+DROP TABLE referenced_table, test_table_1;

src/test/regress/output/multi_alter_table_statements.source

@@ -1153,3 +1153,51 @@ SELECT relname, reloptions FROM pg_class WHERE relname LIKE 'another_index%' ORD
 \c - - - :master_port
 -- get rid of the index
 DROP INDEX another_index;
+-- check if we fail properly when a column with un-supported constraint is added
+-- UNIQUE, PRIMARY KEY on non-distribution column is not supported
+-- CHECK, FOREIGN KEY, UNIQE, PRIMARY KEY cannot be added together with ADD COLUMN
+SET citus.shard_replication_factor TO 1;
+CREATE TABLE test_table_1(id int);
+SELECT create_distributed_table('test_table_1', 'id');
+ create_distributed_table 
+--------------------------
+ 
+(1 row)
+
+ALTER TABLE test_table_1 ADD COLUMN test_col int UNIQUE;
+ERROR:  cannot create constraint on "test_table_1"
+DETAIL:  Distributed relations cannot have UNIQUE, EXCLUDE, or PRIMARY KEY constraints that do not include the partition column (with an equality operator if EXCLUDE).
+ALTER TABLE test_table_1 ADD COLUMN test_col int PRIMARY KEY;
+ERROR:  cannot create constraint on "test_table_1"
+DETAIL:  Distributed relations cannot have UNIQUE, EXCLUDE, or PRIMARY KEY constraints that do not include the partition column (with an equality operator if EXCLUDE).
+ALTER TABLE test_table_1 ADD COLUMN test_col int CHECK (test_col > 3);
+ERROR:  cannot execute ADD COLUMN command with PRIMARY KEY, UNIQUE, FOREIGN and CHECK constraints
+DETAIL:  Adding a column with a constraint in one command is not supported because all constraints in Citus must have explicit names
+HINT:  You can issue each command separately such as ALTER TABLE test_table_1 ADD COLUMN test_col data_type; ALTER TABLE test_table_1 ADD CONSTRAINT constraint_name CHECK (check_expression);
+CREATE TABLE reference_table(i int UNIQUE);
+SELECT create_reference_table('reference_table');
+ create_reference_table 
+------------------------
+ 
+(1 row)
+
+ALTER TABLE test_table_1 ADD COLUMN test_col int REFERENCES reference_table(i) ON DELETE CASCADE;
+ERROR:  cannot execute ADD COLUMN command with PRIMARY KEY, UNIQUE, FOREIGN and CHECK constraints
+DETAIL:  Adding a column with a constraint in one command is not supported because all constraints in Citus must have explicit names
+HINT:  You can issue each command separately such as ALTER TABLE test_table_1 ADD COLUMN test_col data_type; ALTER TABLE test_table_1 ADD CONSTRAINT constraint_name FOREIGN KEY (test_col) REFERENCES reference_table(i) ON DELETE CASCADE;
+ALTER TABLE test_table_1 ADD COLUMN test_col int REFERENCES reference_table(i) ON DELETE CASCADE ON UPDATE SET NULL;
+ERROR:  cannot execute ADD COLUMN command with PRIMARY KEY, UNIQUE, FOREIGN and CHECK constraints
+DETAIL:  Adding a column with a constraint in one command is not supported because all constraints in Citus must have explicit names
+HINT:  You can issue each command separately such as ALTER TABLE test_table_1 ADD COLUMN test_col data_type; ALTER TABLE test_table_1 ADD CONSTRAINT constraint_name FOREIGN KEY (test_col) REFERENCES reference_table(i) ON DELETE CASCADE ON UPDATE SET NULL;
+DROP TABLE reference_table;
+CREATE TABLE referenced_table(i int UNIQUE);
+SELECT create_distributed_table('referenced_table', 'i');
+ create_distributed_table 
+--------------------------
+ 
+(1 row)
+
+ALTER TABLE test_table_1 ADD COLUMN test_col int REFERENCES referenced_table(i);
+ERROR:  cannot create foreign key constraint
+DETAIL:  Foreign keys are supported in two cases, either in between two colocated tables including partition column in the same ordinal in the both tables or from distributed to reference tables
+DROP TABLE referenced_table, test_table_1;

src/test/regress/sql/foreign_key_restriction_enforcement.sql

@@ -570,6 +570,69 @@ BEGIN;
 	DROP TABLE test_table_2, test_table_1;
 COMMIT;
 
+-- make sure that modifications to reference tables in a CTE can
+-- set the mode to sequential for the next operations
+CREATE TABLE reference_table(id int PRIMARY KEY);
+SELECT create_reference_table('reference_table');
+
+CREATE TABLE distributed_table(id int PRIMARY KEY, value_1 int);
+SELECT create_distributed_table('distributed_table', 'id');
+
+ALTER TABLE 
+	distributed_table 
+ADD CONSTRAINT 
+	fkey_delete FOREIGN KEY(value_1) 
+REFERENCES 
+	reference_table(id) ON DELETE CASCADE;
+
+INSERT INTO reference_table SELECT i FROM generate_series(0, 10) i;
+INSERT INTO distributed_table SELECT i, i % 10  FROM generate_series(0, 100) i;
+
+-- this query returns 100 rows in Postgres, but not in Citus
+-- see https://github.com/citusdata/citus_docs/issues/664 for the discussion
+WITH t1 AS (DELETE FROM reference_table RETURNING id) 
+	DELETE FROM distributed_table USING t1 WHERE value_1 = t1.id RETURNING *;
+
+-- load some more data for one more test with real-time selects
+INSERT INTO reference_table SELECT i FROM generate_series(0, 10) i;
+INSERT INTO distributed_table SELECT i, i % 10  FROM generate_series(0, 100) i;
+
+-- this query returns 100 rows in Postgres, but not in Citus
+-- see https://github.com/citusdata/citus_docs/issues/664 for the discussion
+WITH t1 AS (DELETE FROM reference_table RETURNING id) 
+	SELECT count(*) FROM distributed_table, t1 WHERE  value_1 = t1.id;
+
+-- this query should fail since we first to a parallel access to a distributed table 
+-- with t1, and then access to t2
+WITH t1 AS (DELETE FROM distributed_table RETURNING id),
+	t2 AS (DELETE FROM reference_table RETURNING id)
+	SELECT count(*) FROM distributed_table, t1, t2 WHERE  value_1 = t1.id AND value_1 = t2.id;
+
+-- similarly this should fail since we first access to a distributed
+-- table via t1, and then access to the reference table in the main query
+WITH t1 AS (DELETE FROM distributed_table RETURNING id)
+	DELETE FROM reference_table RETURNING id;
+
+
+-- finally, make sure that we can execute the same queries
+-- in the sequential mode
+BEGIN;
+	
+	SET LOCAL citus.multi_shard_modify_mode TO 'sequential';
+
+	WITH t1 AS (DELETE FROM distributed_table RETURNING id),
+		t2 AS (DELETE FROM reference_table RETURNING id)
+		SELECT count(*) FROM distributed_table, t1, t2 WHERE  value_1 = t1.id AND value_1 = t2.id;
+ROLLBACK;
+
+BEGIN;
+	
+	SET LOCAL citus.multi_shard_modify_mode TO 'sequential';
+
+	WITH t1 AS (DELETE FROM distributed_table RETURNING id)
+		DELETE FROM reference_table RETURNING id;
+ROLLBACK;
+
 RESET client_min_messages;
 
 DROP SCHEMA test_fkey_to_ref_in_tx CASCADE;

src/test/regress/sql/foreign_key_to_reference_table.sql

@@ -133,6 +133,13 @@ ALTER TABLE referencing_table ADD CONSTRAINT fkey_ref FOREIGN KEY(id) REFERENCES
 SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
 DROP TABLE referencing_table;
 
+-- check if we can add the foreign key while adding the column
+CREATE TABLE referencing_table(id int, ref_id int);
+SELECT create_distributed_table('referencing_table', 'ref_id');
+ALTER TABLE referencing_table ADD COLUMN referencing int REFERENCES referenced_table(id) ON UPDATE CASCADE;
+SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
+DROP TABLE referencing_table;
+
 -- foreign keys are only supported when the replication factor = 1
 SET citus.shard_replication_factor TO 2;
 CREATE TABLE referencing_table(id int, ref_id int);
@@ -140,6 +147,13 @@ SELECT create_distributed_table('referencing_table', 'ref_id');
 ALTER TABLE referencing_table ADD CONSTRAINT fkey_ref FOREIGN KEY (id) REFERENCES referenced_table(id);
 SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
 DROP TABLE referencing_table;
+
+-- should fail when we add the column as well
+CREATE TABLE referencing_table(id int, ref_id int);
+SELECT create_distributed_table('referencing_table', 'ref_id');
+ALTER TABLE referencing_table ADD COLUMN referencing_col int REFERENCES referenced_table(id) ON DELETE SET NULL;
+SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
+DROP TABLE referencing_table;
 SET citus.shard_replication_factor TO 1;
 
 -- simple create_distributed_table should work in/out transactions on tables with foreign key to reference tables

src/test/regress/sql/multi_distribution_metadata.sql

@@ -51,6 +51,11 @@ CREATE FUNCTION acquire_shared_shard_lock(bigint)
 	AS 'citus'
 	LANGUAGE C STRICT;
 
+CREATE FUNCTION relation_count_in_query(text)
+	RETURNS int
+	AS 'citus'
+	LANGUAGE C STRICT;
+
 -- ===================================================================
 -- test distribution metadata functionality
 -- ===================================================================
@@ -259,5 +264,85 @@ SELECT get_shard_id_for_distribution_column('get_shardid_test_table5', 3248);
 SELECT get_shard_id_for_distribution_column('get_shardid_test_table5', 4001);
 SELECT get_shard_id_for_distribution_column('get_shardid_test_table5', -999);
 
+
+SET citus.shard_count TO 2;
+CREATE TABLE events_table_count (user_id int, time timestamp, event_type int, value_2 int, value_3 float, value_4 bigint);
+SELECT create_distributed_table('events_table_count', 'user_id');
+
+CREATE TABLE users_table_count (user_id int, time timestamp, value_1 int, value_2 int, value_3 float, value_4 bigint);
+SELECT create_distributed_table('users_table_count', 'user_id');
+
+SELECT relation_count_in_query($$-- we can support arbitrary subqueries within UNIONs
+SELECT ("final_query"."event_types") as types, count(*) AS sumOfEventType
+FROM
+  ( SELECT 
+      *, random()
+    FROM
+     (SELECT 
+        "t"."user_id", "t"."time", unnest("t"."collected_events") AS "event_types"
+      FROM
+        ( SELECT 
+            "t1"."user_id", min("t1"."time") AS "time", array_agg(("t1"."event") ORDER BY TIME ASC, event DESC) AS collected_events
+          FROM (
+                (SELECT 
+                    *
+                 FROM
+                   (SELECT 
+                          events_table."time", 0 AS event, events_table."user_id"
+                    FROM 
+                       "events_table_count" as events_table
+                    WHERE 
+                      events_table.event_type IN (1, 2) ) events_subquery_1) 
+                UNION 
+                 (SELECT *
+                  FROM
+                    (
+                          SELECT * FROM
+                          (
+                              SELECT 
+                                max("events_table_count"."time"),
+                                0 AS event,
+                                "events_table_count"."user_id"
+                              FROM 
+                                "events_table_count", users_table_count as "users"
+                              WHERE 
+                                 "events_table_count".user_id = users.user_id AND
+                                "events_table_count".event_type IN (1, 2)
+                                GROUP BY   "events_table_count"."user_id"
+                          ) as events_subquery_5
+                     ) events_subquery_2)
+               UNION 
+                 (SELECT *
+                  FROM
+                    (SELECT 
+                        "events_table_count"."time", 2 AS event, "events_table_count"."user_id"
+                     FROM 
+                       "events_table_count"
+                     WHERE 
+                      event_type IN (3, 4) ) events_subquery_3)
+               UNION 
+                 (SELECT *
+                  FROM
+                    (SELECT
+                       "events_table_count"."time", 3 AS event, "events_table_count"."user_id"
+                     FROM 
+                       "events_table_count"
+                     WHERE 
+                      event_type IN (5, 6)) events_subquery_4)
+                 ) t1
+         GROUP BY "t1"."user_id") AS t) "q" 
+INNER JOIN
+     (SELECT 
+        "events_table_count"."user_id"
+      FROM 
+        users_table_count as "events_table_count"
+      WHERE 
+        value_1 > 0 and value_1 < 4) AS t 
+     ON (t.user_id = q.user_id)) as final_query
+GROUP BY 
+  types
+ORDER BY 
+  types;$$);
+
 -- clear unnecessary tables;
-DROP TABLE get_shardid_test_table1, get_shardid_test_table2, get_shardid_test_table3, get_shardid_test_table4, get_shardid_test_table5;
+DROP TABLE get_shardid_test_table1, get_shardid_test_table2, get_shardid_test_table3, get_shardid_test_table4, get_shardid_test_table5, events_table_count;

src/test/regress/sql/multi_foreign_key.sql

@@ -262,6 +262,9 @@ ALTER TABLE referencing_table ADD CONSTRAINT test_constraint FOREIGN KEY(ref_id)
 ALTER TABLE referencing_table ADD CONSTRAINT test_constraint FOREIGN KEY(ref_id) REFERENCES referenced_table(id) ON UPDATE SET DEFAULT;
 ALTER TABLE referencing_table ADD CONSTRAINT test_constraint FOREIGN KEY(ref_id) REFERENCES referenced_table(id) ON UPDATE CASCADE;
 
+-- test foreign constraint creation while adding the column
+ALTER TABLE referencing_table ADD COLUMN referencing_col int REFERENCES referenced_table(id) ON UPDATE CASCADE;;
+
 -- test foreign constraint creation with multiple subcommands
 ALTER TABLE referencing_table ADD CONSTRAINT test_constraint FOREIGN KEY(ref_id) REFERENCES referenced_table(id),
 							  ADD CONSTRAINT test_constraint FOREIGN KEY(id) REFERENCES referenced_table(test_column);

src/test/regress/sql/multi_insert_select.sql

@@ -2104,6 +2104,22 @@ EXECUTE prepared_recursive_insert_select;
 EXECUTE prepared_recursive_insert_select;
 ROLLBACK;
 
+-- upsert with on conflict update distribution column is unsupported
+INSERT INTO agg_events AS ae
+            (
+                        user_id,
+                        value_1_agg,
+                        agg_time
+            )
+SELECT user_id,
+       value_1,
+       time
+FROM   raw_events_first
+ON conflict (user_id, value_1_agg)
+DO UPDATE
+   SET    user_id = 42
+RETURNING user_id, value_1_agg;
+
 -- wrap in a transaction to improve performance
 BEGIN;
 DROP TABLE coerce_events;

src/test/regress/sql/multi_multiuser.sql

@@ -70,7 +70,7 @@ SELECT count(*) FROM test;
 SELECT count(*) FROM test WHERE id = 1;
 
 SET citus.task_executor_type TO 'task-tracker';
-SELECT count(*) FROM test;
+SELECT count(*), min(current_user) FROM test;
 
 -- test re-partition query (needs to transmit intermediate results)
 SELECT count(*) FROM test a JOIN test b ON (a.val = b.val) WHERE a.id = 1 AND b.id = 2;
@@ -94,7 +94,7 @@ SELECT count(*) FROM test;
 SELECT count(*) FROM test WHERE id = 1;
 
 SET citus.task_executor_type TO 'task-tracker';
-SELECT count(*) FROM test;
+SELECT count(*), min(current_user) FROM test;
 
 -- test re-partition query (needs to transmit intermediate results)
 SELECT count(*) FROM test a JOIN test b ON (a.val = b.val) WHERE a.id = 1 AND b.id = 2;
@@ -115,7 +115,7 @@ SELECT count(*) FROM test;
 SELECT count(*) FROM test WHERE id = 1;
 
 SET citus.task_executor_type TO 'task-tracker';
-SELECT count(*) FROM test;
+SELECT count(*), min(current_user) FROM test;
 
 -- test re-partition query
 SELECT count(*) FROM test a JOIN test b ON (a.val = b.val) WHERE a.id = 1 AND b.id = 2;
@@ -138,8 +138,14 @@ ABORT;
 
 SELECT * FROM citus_stat_statements_reset();
 
+-- table owner should be the same on the shards, even when distributing the table as superuser
+SET ROLE full_access;
+CREATE TABLE my_table (id integer, val integer);
 RESET ROLE;
+SELECT create_distributed_table('my_table', 'id');
+SELECT result FROM run_command_on_workers($$SELECT tableowner FROM pg_tables WHERE tablename LIKE 'my_table_%' LIMIT 1$$);
 
+DROP TABLE my_table;
 DROP TABLE test;
 DROP USER full_access;
 DROP USER read_access;

src/test/regress/sql/multi_mx_transaction_recovery.sql

@@ -36,11 +36,23 @@ BEGIN;
 CREATE TABLE should_be_sorted_into_middle (value int);
 PREPARE TRANSACTION 'citus_12_should_be_sorted_into_middle';
 
+-- this node (e.g., node id 12) should not touch
+-- transactions with different nodeIds in the gid
+BEGIN;
+CREATE TABLE table_should_do_nothing (value int);
+PREPARE TRANSACTION 'citus_122_should_do_nothing';
+
 -- Add "fake" pg_dist_transaction records and run recovery
 INSERT INTO pg_dist_transaction VALUES (12, 'citus_12_should_commit');
 INSERT INTO pg_dist_transaction VALUES (12, 'citus_12_should_be_forgotten');
+INSERT INTO pg_dist_transaction VALUES (122, 'citus_122_should_do_nothing');
 
 SELECT recover_prepared_transactions();
+
+-- delete the citus_122_should_do_nothing transaction
+DELETE FROM pg_dist_transaction WHERE gid = 'citus_122_should_do_nothing' RETURNING *;
+ROLLBACK PREPARED 'citus_122_should_do_nothing';
+
 SELECT count(*) FROM pg_dist_transaction;
 
 SELECT count(*) FROM pg_tables WHERE tablename = 'table_should_abort';

src/test/regress/sql/multi_task_string_size.sql

@@ -220,9 +220,11 @@ SET citus.max_task_string_size TO 20000;
 -- error message may vary between executions
 -- hiding warning and error message
 -- no output means the query has failed
-SET client_min_messages to FATAL;
+SET client_min_messages to ERROR;
 
+SELECT raise_failed_execution('
 SELECT u.* FROM wide_table u JOIN wide_table v ON (u.long_column_002 = v.long_column_003);
+');
 
 -- following will succeed since it fetches few columns
 SELECT u.long_column_001, u.long_column_002, u.long_column_003 FROM wide_table u JOIN wide_table v ON (u.long_column_002 = v.long_column_003);

src/test/regress/sql/multi_test_helpers.sql

@@ -81,3 +81,14 @@ WHERE cc.constraint_schema = ccu.constraint_schema AND
 ORDER BY cc.constraint_name ASC;
 $desc_views$
 );
+
+-- Create a function to make sure that queries returning the same result
+CREATE FUNCTION raise_failed_execution(query text) RETURNS void AS $$
+BEGIN
+	EXECUTE query;
+	EXCEPTION WHEN OTHERS THEN
+	IF SQLERRM LIKE 'failed to execute task%' THEN
+		RAISE 'Task failed to execute';
+	END IF;
+END;
+$$LANGUAGE plpgsql;

src/test/regress/sql/multi_utility_warnings.sql

@@ -16,3 +16,8 @@ CREATE ROLE new_role;
 CREATE USER new_user;
 
 INSERT INTO pg_dist_authinfo VALUES (0, 'new_user', 'password=1234');
+
+BEGIN;
+INSERT INTO pg_dist_node VALUES (1234567890, 1234567890, 'localhost', 5432);
+INSERT INTO pg_dist_poolinfo VALUES (1234567890, 'port=1234');
+ROLLBACK;

windows/include/citus_version.h

@@ -11,13 +11,13 @@
 #define CITUS_MAJORVERSION "7.5"
 
 /* Citus version as a string */
-#define CITUS_VERSION "7.5devel"
+#define CITUS_VERSION "7.5.2"
 
 /* Citus version as a number */
 #define CITUS_VERSION_NUM 70500
 
 /* A string containing the version number, platform, and C compiler */
-#define CITUS_VERSION_STR "Citus 7.5devel on x86_64-windows, compiled by Visual Studio"
+#define CITUS_VERSION_STR "Citus 7.5.2 on x86_64-windows, compiled by Visual Studio"
 
 /* Define to 1 if you have the `curl' library (-lcurl). */
 #define HAVE_LIBCURL 0