Merge pull request #3654 from citusdata/2776_modifying_ctes

Modifying ctes in router planner

src/backend/distributed/executor/adaptive_executor.c

@@ -1467,11 +1467,6 @@ ReadOnlyTask(TaskType taskType)
 		case MAP_TASK:
 		case MERGE_TASK:
 		{
-			/*
-			 * TODO: We currently do not execute modifying CTEs via ROUTER_TASK/SQL_TASK.
-			 * When we implement it, we should either not use the mentioned task types for
-			 * modifying CTEs detect them here.
-			 */
 			return true;
 		}
 

src/backend/distributed/executor/citus_custom_scan.c

@@ -54,7 +54,7 @@ static Node * DelayedErrorCreateScan(CustomScan *scan);
 
 /* functions that are common to different scans */
 static void CitusBeginScan(CustomScanState *node, EState *estate, int eflags);
-static void CitusBeginSelectScan(CustomScanState *node, EState *estate, int eflags);
+static void CitusBeginReadOnlyScan(CustomScanState *node, EState *estate, int eflags);
 static void CitusBeginModifyScan(CustomScanState *node, EState *estate, int eflags);
 static void CitusPreExecScan(CitusScanState *scanState);
 static bool ModifyJobNeedsEvaluation(Job *workerJob);
@@ -206,7 +206,7 @@ CitusBeginScan(CustomScanState *node, EState *estate, int eflags)
 	}
 	else if (distributedPlan->modLevel == ROW_MODIFY_READONLY)
 	{
-		CitusBeginSelectScan(node, estate, eflags);
+		CitusBeginReadOnlyScan(node, estate, eflags);
 	}
 	else
 	{
@@ -248,14 +248,16 @@ CitusExecScan(CustomScanState *node)
 
 
 /*
- * CitusBeginSelectScan handles deferred pruning and plan caching for SELECTs.
+ * CitusBeginReadOnlyScan handles deferred pruning and plan caching for SELECTs.
  */
 static void
-CitusBeginSelectScan(CustomScanState *node, EState *estate, int eflags)
+CitusBeginReadOnlyScan(CustomScanState *node, EState *estate, int eflags)
 {
 	CitusScanState *scanState = (CitusScanState *) node;
 	DistributedPlan *originalDistributedPlan = scanState->distributedPlan;
 
+	Assert(originalDistributedPlan->workerJob->jobQuery->commandType == CMD_SELECT);
+
 	if (!originalDistributedPlan->workerJob->deferredPruning)
 	{
 		/*
@@ -295,7 +297,7 @@ CitusBeginSelectScan(CustomScanState *node, EState *estate, int eflags)
 	 *
 	 * TODO: evaluate stable functions
 	 */
-	ExecuteMasterEvaluableParameters(jobQuery, planState);
+	ExecuteMasterEvaluableExpressions(jobQuery, planState);
 
 	/* job query no longer has parameters, so we should not send any */
 	workerJob->parametersInJobQueryResolved = true;
@@ -345,8 +347,7 @@ CitusBeginModifyScan(CustomScanState *node, EState *estate, int eflags)
 
 	if (ModifyJobNeedsEvaluation(workerJob))
 	{
-		/* evaluate both functions and parameters */
+		ExecuteMasterEvaluableExpressions(jobQuery, planState);
-		ExecuteMasterEvaluableFunctionsAndParameters(jobQuery, planState);
 
 		/* job query no longer has parameters, so we should not send any */
 		workerJob->parametersInJobQueryResolved = true;

src/backend/distributed/planner/local_plan_cache.c

@@ -190,7 +190,7 @@ IsLocalPlanCachingSupported(Job *currentJob, DistributedPlan *originalDistribute
 		 * We do not cache plans with volatile functions in the query.
 		 *
 		 * The reason we care about volatile functions is primarily that we
-		 * already executed them in ExecuteMasterEvaluableFunctionsAndParameters
+		 * already executed them in ExecuteMasterEvaluableExpressions
 		 * and since we're falling back to the original query tree here we would
 		 * execute them again if we execute the plan.
 		 */

src/backend/distributed/planner/multi_logical_planner.c

@@ -457,6 +457,19 @@ IsDistributedTableRTE(Node *node)
 }
 
 
+/*
+ * IsReferenceTableRTE gets a node and returns true if the node
+ * is a range table relation entry that points to a reference table.
+ */
+bool
+IsReferenceTableRTE(Node *node)
+{
+	Oid relationId = NodeTryGetRteRelid(node);
+	return relationId != InvalidOid && IsCitusTable(relationId) &&
+		   PartitionMethod(relationId) == DISTRIBUTE_BY_NONE;
+}
+
+
 /*
  * FullCompositeFieldList gets a composite field list, and checks if all fields
  * of composite type are used in the list.

src/backend/distributed/planner/multi_physical_planner.c

@@ -4636,6 +4636,22 @@ RowModifyLevelForQuery(Query *query)
 
 	if (commandType == CMD_SELECT)
 	{
+		if (query->hasModifyingCTE)
+		{
+			/* skip checking for INSERT as those CTEs are recursively planned */
+			CommonTableExpr *cte = NULL;
+			foreach_ptr(cte, query->cteList)
+			{
+				Query *cteQuery = (Query *) cte->ctequery;
+
+				if (cteQuery->commandType == CMD_UPDATE ||
+					cteQuery->commandType == CMD_DELETE)
+				{
+					return ROW_MODIFY_NONCOMMUTATIVE;
+				}
+			}
+		}
+
 		return ROW_MODIFY_READONLY;
 	}
 

src/backend/distributed/planner/multi_router_planner.c

@@ -125,6 +125,9 @@ static void CreateSingleTaskRouterSelectPlan(DistributedPlan *distributedPlan,
 											 plannerRestrictionContext);
 static Oid ResultRelationOidForQuery(Query *query);
 static bool IsTidColumn(Node *node);
+static DeferredErrorMessage * ModifyPartialQuerySupported(Query *queryTree, bool
+														  multiShardQuery,
+														  Oid *distributedTableId);
 static DeferredErrorMessage * MultiShardModifyQuerySupported(Query *originalQuery,
 															 PlannerRestrictionContext *
 															 plannerRestrictionContext);
@@ -149,19 +152,16 @@ static List * BuildRoutesForInsert(Query *query, DeferredErrorMessage **planning
 static List * GroupInsertValuesByShardId(List *insertValuesList);
 static List * ExtractInsertValuesList(Query *query, Var *partitionColumn);
 static DeferredErrorMessage * MultiRouterPlannableQuery(Query *query);
-static DeferredErrorMessage * ErrorIfQueryHasModifyingCTE(Query *queryTree);
+static DeferredErrorMessage * ErrorIfQueryHasUnroutableModifyingCTE(Query *queryTree);
 static bool SelectsFromDistributedTable(List *rangeTableList, Query *query);
 static ShardPlacement * CreateDummyPlacement(void);
 static List * get_all_actual_clauses(List *restrictinfo_list);
 static int CompareInsertValuesByShardId(const void *leftElement,
 										const void *rightElement);
-static List * SingleShardSelectTaskList(Query *query, uint64 jobId,
+static List * SingleShardTaskList(Query *query, uint64 jobId,
-										List *relationShardList, List *placementList,
+								  List *relationShardList, List *placementList,
-										uint64 shardId, bool parametersInQueryResolved);
+								  uint64 shardId, bool parametersInQueryResolved);
 static bool RowLocksOnRelations(Node *node, List **rtiLockList);
-static List * SingleShardModifyTaskList(Query *query, uint64 jobId,
-										List *relationShardList, List *placementList,
-										uint64 shardId, bool parametersInQueryResolved);
 static List * RemoveCoordinatorPlacement(List *placementList);
 static void ReorderTaskPlacementsByTaskAssignmentPolicy(Job *job,
 														TaskAssignmentPolicyType
@@ -511,7 +511,12 @@ ResultRelationOidForQuery(Query *query)
 RangeTblEntry *
 ExtractResultRelationRTE(Query *query)
 {
-	return rt_fetch(query->resultRelation, query->rtable);
+	if (query->resultRelation > 0)
+	{
+		return rt_fetch(query->resultRelation, query->rtable);
+	}
+
+	return NULL;
 }
 
 
@@ -535,28 +540,18 @@ IsTidColumn(Node *node)
 
 
 /*
- * ModifyQuerySupported returns NULL if the query only contains supported
+ * ModifyPartialQuerySupported implements a subset of what ModifyQuerySupported checks,
- * features, otherwise it returns an error description.
+ * that subset being what's necessary to check modifying CTEs for.
- * Note that we need both the original query and the modified one because
- * different checks need different versions. In particular, we cannot
- * perform the ContainsReadIntermediateResultFunction check on the
- * rewritten query because it may have been replaced by a subplan,
- * while some of the checks for setting the partition column value rely
- * on the rewritten query.
  */
-DeferredErrorMessage *
+static DeferredErrorMessage *
-ModifyQuerySupported(Query *queryTree, Query *originalQuery, bool multiShardQuery,
+ModifyPartialQuerySupported(Query *queryTree, bool multiShardQuery,
-					 PlannerRestrictionContext *plannerRestrictionContext)
+							Oid *distributedTableIdOutput)
 {
 	uint32 rangeTableId = 1;
-	List *rangeTableList = NIL;
-	ListCell *rangeTableCell = NULL;
-	uint32 queryTableCount = 0;
 	CmdType commandType = queryTree->commandType;
-	bool fastPathRouterQuery =
-		plannerRestrictionContext->fastPathRestrictionContext->fastPathRouterQuery;
 
 	Oid distributedTableId = ModifyQueryResultRelationId(queryTree);
+	*distributedTableIdOutput = distributedTableId;
 	if (!IsCitusTable(distributedTableId))
 	{
 		return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED,
@@ -573,30 +568,6 @@ ModifyQuerySupported(Query *queryTree, Query *originalQuery, bool multiShardQuer
 		return deferredError;
 	}
 
-	/*
-	 * Here, we check if a recursively planned query tries to modify
-	 * rows based on the ctid column. This is a bad idea because ctid of
-	 * the rows could be changed before the modification part of
-	 * the query is executed.
-	 *
-	 * We can exclude fast path queries since they cannot have intermediate
-	 * results by definition.
-	 */
-	if (!fastPathRouterQuery &&
-		ContainsReadIntermediateResultFunction((Node *) originalQuery))
-	{
-		bool hasTidColumn = FindNodeCheck((Node *) originalQuery->jointree, IsTidColumn);
-		if (hasTidColumn)
-		{
-			return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED,
-								 "cannot perform distributed planning for the given "
-								 "modification",
-								 "Recursively planned distributed modifications "
-								 "with ctid on where clause are not supported.",
-								 NULL);
-		}
-	}
-
 	/*
 	 * Reject subqueries which are in SELECT or WHERE clause.
 	 * Queries which include subqueries in FROM clauses are rejected below.
@@ -626,17 +597,38 @@ ModifyQuerySupported(Query *queryTree, Query *originalQuery, bool multiShardQuer
 			Query *cteQuery = (Query *) cte->ctequery;
 
 			if (cteQuery->commandType != CMD_SELECT)
+			{
+				/* Modifying CTEs still not supported for INSERTs & multi shard queries. */
+				if (queryTree->commandType == CMD_INSERT)
+				{
+					return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED,
+										 "Router planner doesn't support non-select common table expressions with non-select queries.",
+										 NULL, NULL);
+				}
+
+				if (multiShardQuery)
+				{
+					return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED,
+										 "Router planner doesn't support non-select common table expressions with multi shard queries.",
+										 NULL, NULL);
+				}
+			}
+
+			/* Modifying CTEs exclude both INSERT CTEs & INSERT queries. */
+			if (cteQuery->commandType == CMD_INSERT)
 			{
 				return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED,
-									 "Router planner doesn't support non-select common table expressions.",
+									 "Router planner doesn't support INSERT common table expressions.",
 									 NULL, NULL);
 			}
 
-			if (cteQuery->hasForUpdate)
+
+			if (cteQuery->hasForUpdate &&
+				FindNodeCheckInRangeTableList(cteQuery->rtable, IsReferenceTableRTE))
 			{
 				return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED,
 									 "Router planner doesn't support SELECT FOR UPDATE"
-									 " in common table expressions.",
+									 " in common table expressions involving reference tables.",
 									 NULL, NULL);
 			}
 
@@ -648,14 +640,184 @@ ModifyQuerySupported(Query *queryTree, Query *originalQuery, bool multiShardQuer
 									 NULL, NULL);
 			}
 
-			DeferredErrorMessage *cteError = MultiRouterPlannableQuery(cteQuery);
+			if (cteQuery->commandType == CMD_SELECT)
-			if (cteError)
 			{
-				return cteError;
+				DeferredErrorMessage *cteError = MultiRouterPlannableQuery(cteQuery);
+				if (cteError)
+				{
+					return cteError;
+				}
 			}
 		}
 	}
 
+	if (commandType == CMD_INSERT || commandType == CMD_UPDATE ||
+		commandType == CMD_DELETE)
+	{
+		bool hasVarArgument = false; /* A STABLE function is passed a Var argument */
+		bool hasBadCoalesce = false; /* CASE/COALESCE passed a mutable function */
+		FromExpr *joinTree = queryTree->jointree;
+		ListCell *targetEntryCell = NULL;
+
+		foreach(targetEntryCell, queryTree->targetList)
+		{
+			TargetEntry *targetEntry = (TargetEntry *) lfirst(targetEntryCell);
+			bool targetEntryPartitionColumn = false;
+
+			/* reference tables do not have partition column */
+			if (partitionColumn == NULL)
+			{
+				targetEntryPartitionColumn = false;
+			}
+			else if (targetEntry->resno == partitionColumn->varattno)
+			{
+				targetEntryPartitionColumn = true;
+			}
+
+			/* skip resjunk entries: UPDATE adds some for ctid, etc. */
+			if (targetEntry->resjunk)
+			{
+				continue;
+			}
+
+			if (commandType == CMD_UPDATE &&
+				FindNodeCheck((Node *) targetEntry->expr, CitusIsVolatileFunction))
+			{
+				return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED,
+									 "functions used in UPDATE queries on distributed "
+									 "tables must not be VOLATILE",
+									 NULL, NULL);
+			}
+
+			if (commandType == CMD_UPDATE && targetEntryPartitionColumn &&
+				TargetEntryChangesValue(targetEntry, partitionColumn,
+										queryTree->jointree))
+			{
+				return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED,
+									 "modifying the partition value of rows is not "
+									 "allowed",
+									 NULL, NULL);
+			}
+
+			if (commandType == CMD_UPDATE &&
+				MasterIrreducibleExpression((Node *) targetEntry->expr,
+											&hasVarArgument, &hasBadCoalesce))
+			{
+				Assert(hasVarArgument || hasBadCoalesce);
+			}
+		}
+
+		if (joinTree != NULL)
+		{
+			if (FindNodeCheck((Node *) joinTree->quals, CitusIsVolatileFunction))
+			{
+				return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED,
+									 "functions used in the WHERE clause of modification "
+									 "queries on distributed tables must not be VOLATILE",
+									 NULL, NULL);
+			}
+			else if (MasterIrreducibleExpression(joinTree->quals, &hasVarArgument,
+												 &hasBadCoalesce))
+			{
+				Assert(hasVarArgument || hasBadCoalesce);
+			}
+		}
+
+		if (hasVarArgument)
+		{
+			return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED,
+								 "STABLE functions used in UPDATE queries "
+								 "cannot be called with column references",
+								 NULL, NULL);
+		}
+
+		if (hasBadCoalesce)
+		{
+			return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED,
+								 "non-IMMUTABLE functions are not allowed in CASE or "
+								 "COALESCE statements",
+								 NULL, NULL);
+		}
+
+		if (contain_mutable_functions((Node *) queryTree->returningList))
+		{
+			return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED,
+								 "non-IMMUTABLE functions are not allowed in the "
+								 "RETURNING clause",
+								 NULL, NULL);
+		}
+
+		if (queryTree->jointree->quals != NULL &&
+			nodeTag(queryTree->jointree->quals) == T_CurrentOfExpr)
+		{
+			return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED,
+								 "cannot run DML queries with cursors", NULL,
+								 NULL);
+		}
+	}
+
+	deferredError = ErrorIfOnConflictNotSupported(queryTree);
+	if (deferredError != NULL)
+	{
+		return deferredError;
+	}
+
+	return NULL;
+}
+
+
+/*
+ * ModifyQuerySupported returns NULL if the query only contains supported
+ * features, otherwise it returns an error description.
+ * Note that we need both the original query and the modified one because
+ * different checks need different versions. In particular, we cannot
+ * perform the ContainsReadIntermediateResultFunction check on the
+ * rewritten query because it may have been replaced by a subplan,
+ * while some of the checks for setting the partition column value rely
+ * on the rewritten query.
+ */
+DeferredErrorMessage *
+ModifyQuerySupported(Query *queryTree, Query *originalQuery, bool multiShardQuery,
+					 PlannerRestrictionContext *plannerRestrictionContext)
+{
+	Oid distributedTableId = InvalidOid;
+	DeferredErrorMessage *error = ModifyPartialQuerySupported(queryTree, multiShardQuery,
+															  &distributedTableId);
+	if (error)
+	{
+		return error;
+	}
+
+	List *rangeTableList = NIL;
+	uint32 queryTableCount = 0;
+	CmdType commandType = queryTree->commandType;
+	bool fastPathRouterQuery =
+		plannerRestrictionContext->fastPathRestrictionContext->fastPathRouterQuery;
+
+	/*
+	 * Here, we check if a recursively planned query tries to modify
+	 * rows based on the ctid column. This is a bad idea because ctid of
+	 * the rows could be changed before the modification part of
+	 * the query is executed.
+	 *
+	 * We can exclude fast path queries since they cannot have intermediate
+	 * results by definition.
+	 */
+	if (!fastPathRouterQuery &&
+		ContainsReadIntermediateResultFunction((Node *) originalQuery))
+	{
+		bool hasTidColumn = FindNodeCheck((Node *) originalQuery->jointree, IsTidColumn);
+		if (hasTidColumn)
+		{
+			return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED,
+								 "cannot perform distributed planning for the given "
+								 "modification",
+								 "Recursively planned distributed modifications "
+								 "with ctid on where clause are not supported.",
+								 NULL);
+		}
+	}
+
 	/*
 	 * Extract range table entries for queries that are not fast path. We can skip fast
 	 * path queries because their definition is a single RTE entry, which is a relation,
@@ -666,10 +828,9 @@ ModifyQuerySupported(Query *queryTree, Query *originalQuery, bool multiShardQuer
 		ExtractRangeTableEntryWalker((Node *) originalQuery, &rangeTableList);
 	}
 
-	foreach(rangeTableCell, rangeTableList)
+	RangeTblEntry *rangeTableEntry = NULL;
+	foreach_ptr(rangeTableEntry, rangeTableList)
 	{
-		RangeTblEntry *rangeTableEntry = (RangeTblEntry *) lfirst(rangeTableCell);
-
 		if (rangeTableEntry->rtekind == RTE_RELATION)
 		{
 			/* we do not expect to see a view in modify query */
@@ -789,117 +950,6 @@ ModifyQuerySupported(Query *queryTree, Query *originalQuery, bool multiShardQuer
 		}
 	}
 
-	if (commandType == CMD_INSERT || commandType == CMD_UPDATE ||
-		commandType == CMD_DELETE)
-	{
-		bool hasVarArgument = false; /* A STABLE function is passed a Var argument */
-		bool hasBadCoalesce = false; /* CASE/COALESCE passed a mutable function */
-		FromExpr *joinTree = queryTree->jointree;
-		ListCell *targetEntryCell = NULL;
-
-		foreach(targetEntryCell, queryTree->targetList)
-		{
-			TargetEntry *targetEntry = (TargetEntry *) lfirst(targetEntryCell);
-			bool targetEntryPartitionColumn = false;
-
-			/* reference tables do not have partition column */
-			if (partitionColumn == NULL)
-			{
-				targetEntryPartitionColumn = false;
-			}
-			else if (targetEntry->resno == partitionColumn->varattno)
-			{
-				targetEntryPartitionColumn = true;
-			}
-
-			/* skip resjunk entries: UPDATE adds some for ctid, etc. */
-			if (targetEntry->resjunk)
-			{
-				continue;
-			}
-
-			if (commandType == CMD_UPDATE &&
-				FindNodeCheck((Node *) targetEntry->expr, CitusIsVolatileFunction))
-			{
-				return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED,
-									 "functions used in UPDATE queries on distributed "
-									 "tables must not be VOLATILE",
-									 NULL, NULL);
-			}
-
-			if (commandType == CMD_UPDATE && targetEntryPartitionColumn &&
-				TargetEntryChangesValue(targetEntry, partitionColumn,
-										queryTree->jointree))
-			{
-				return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED,
-									 "modifying the partition value of rows is not "
-									 "allowed",
-									 NULL, NULL);
-			}
-
-			if (commandType == CMD_UPDATE &&
-				MasterIrreducibleExpression((Node *) targetEntry->expr,
-											&hasVarArgument, &hasBadCoalesce))
-			{
-				Assert(hasVarArgument || hasBadCoalesce);
-			}
-		}
-
-		if (joinTree != NULL)
-		{
-			if (FindNodeCheck((Node *) joinTree->quals, CitusIsVolatileFunction))
-			{
-				return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED,
-									 "functions used in the WHERE clause of modification "
-									 "queries on distributed tables must not be VOLATILE",
-									 NULL, NULL);
-			}
-			else if (MasterIrreducibleExpression(joinTree->quals, &hasVarArgument,
-												 &hasBadCoalesce))
-			{
-				Assert(hasVarArgument || hasBadCoalesce);
-			}
-		}
-
-		if (hasVarArgument)
-		{
-			return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED,
-								 "STABLE functions used in UPDATE queries "
-								 "cannot be called with column references",
-								 NULL, NULL);
-		}
-
-		if (hasBadCoalesce)
-		{
-			return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED,
-								 "non-IMMUTABLE functions are not allowed in CASE or "
-								 "COALESCE statements",
-								 NULL, NULL);
-		}
-
-		if (contain_mutable_functions((Node *) queryTree->returningList))
-		{
-			return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED,
-								 "non-IMMUTABLE functions are not allowed in the "
-								 "RETURNING clause",
-								 NULL, NULL);
-		}
-
-		if (queryTree->jointree->quals != NULL &&
-			nodeTag(queryTree->jointree->quals) == T_CurrentOfExpr)
-		{
-			return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED,
-								 "cannot run DML queries with cursors", NULL,
-								 NULL);
-		}
-	}
-
-	deferredError = ErrorIfOnConflictNotSupported(queryTree);
-	if (deferredError != NULL)
-	{
-		return deferredError;
-	}
-
 	return NULL;
 }
 
@@ -1053,8 +1103,7 @@ MultiShardModifyQuerySupported(Query *originalQuery,
 							   PlannerRestrictionContext *plannerRestrictionContext)
 {
 	DeferredErrorMessage *errorMessage = NULL;
-	RangeTblEntry *resultRangeTable = rt_fetch(originalQuery->resultRelation,
+	RangeTblEntry *resultRangeTable = ExtractResultRelationRTE(originalQuery);
-											   originalQuery->rtable);
 	Oid resultRelationOid = resultRangeTable->relid;
 	char resultPartitionMethod = PartitionMethod(resultRelationOid);
 
@@ -1666,10 +1715,10 @@ GenerateSingleShardRouterTaskList(Job *job, List *relationShardList,
 
 	if (originalQuery->commandType == CMD_SELECT)
 	{
-		job->taskList = SingleShardSelectTaskList(originalQuery, job->jobId,
+		job->taskList = SingleShardTaskList(originalQuery, job->jobId,
-												  relationShardList, placementList,
+											relationShardList, placementList,
-												  shardId,
+											shardId,
-												  job->parametersInJobQueryResolved);
+											job->parametersInJobQueryResolved);
 
 		/*
 		 * Queries to reference tables, or distributed tables with multiple replica's have
@@ -1693,10 +1742,10 @@ GenerateSingleShardRouterTaskList(Job *job, List *relationShardList,
 	}
 	else
 	{
-		job->taskList = SingleShardModifyTaskList(originalQuery, job->jobId,
+		job->taskList = SingleShardTaskList(originalQuery, job->jobId,
-												  relationShardList, placementList,
+											relationShardList, placementList,
-												  shardId,
+											shardId,
-												  job->parametersInJobQueryResolved);
+											job->parametersInJobQueryResolved);
 	}
 }
 
@@ -1783,15 +1832,65 @@ RemoveCoordinatorPlacement(List *placementList)
 
 
 /*
- * SingleShardSelectTaskList generates a task for single shard select query
+ * SingleShardTaskList generates a task for single shard query
  * and returns it as a list.
  */
 static List *
-SingleShardSelectTaskList(Query *query, uint64 jobId, List *relationShardList,
+SingleShardTaskList(Query *query, uint64 jobId, List *relationShardList,
-						  List *placementList, uint64 shardId,
+					List *placementList, uint64 shardId,
-						  bool parametersInQueryResolved)
+					bool parametersInQueryResolved)
 {
-	Task *task = CreateTask(READ_TASK);
+	TaskType taskType = READ_TASK;
+	char replicationModel = 0;
+
+	if (query->commandType != CMD_SELECT)
+	{
+		List *rangeTableList = NIL;
+		ExtractRangeTableEntryWalker((Node *) query, &rangeTableList);
+
+		RangeTblEntry *updateOrDeleteRTE = ExtractResultRelationRTE(query);
+		Assert(updateOrDeleteRTE != NULL);
+
+		CitusTableCacheEntry *modificationTableCacheEntry = GetCitusTableCacheEntry(
+			updateOrDeleteRTE->relid);
+		char modificationPartitionMethod = modificationTableCacheEntry->partitionMethod;
+
+		if (modificationPartitionMethod == DISTRIBUTE_BY_NONE &&
+			SelectsFromDistributedTable(rangeTableList, query))
+		{
+			ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+							errmsg("cannot perform select on a distributed table "
+								   "and modify a reference table")));
+		}
+
+		taskType = MODIFY_TASK;
+		replicationModel = modificationTableCacheEntry->replicationModel;
+	}
+
+	if (taskType == READ_TASK && query->hasModifyingCTE)
+	{
+		/* assume ErrorIfQueryHasUnroutableModifyingCTE checked query already */
+
+		CommonTableExpr *cte = NULL;
+		foreach_ptr(cte, query->cteList)
+		{
+			Query *cteQuery = (Query *) cte->ctequery;
+
+			if (cteQuery->commandType != CMD_SELECT)
+			{
+				RangeTblEntry *updateOrDeleteRTE = ExtractResultRelationRTE(cteQuery);
+				CitusTableCacheEntry *modificationTableCacheEntry =
+					GetCitusTableCacheEntry(
+						updateOrDeleteRTE->relid);
+
+				taskType = MODIFY_TASK;
+				replicationModel = modificationTableCacheEntry->replicationModel;
+				break;
+			}
+		}
+	}
+
+	Task *task = CreateTask(taskType);
 	List *relationRowLockList = NIL;
 
 	RowLocksOnRelations((Node *) query, &relationRowLockList);
@@ -1807,6 +1906,7 @@ SingleShardSelectTaskList(Query *query, uint64 jobId, List *relationShardList,
 	task->jobId = jobId;
 	task->relationShardList = relationShardList;
 	task->relationRowLockList = relationRowLockList;
+	task->replicationModel = replicationModel;
 	task->parametersInQueryStringResolved = parametersInQueryResolved;
 
 	return list_make1(task);
@@ -1854,45 +1954,6 @@ RowLocksOnRelations(Node *node, List **relationRowLockList)
 }
 
 
-/*
- * SingleShardModifyTaskList generates a task for single shard update/delete query
- * and returns it as a list.
- */
-static List *
-SingleShardModifyTaskList(Query *query, uint64 jobId, List *relationShardList,
-						  List *placementList, uint64 shardId,
-						  bool parametersInQueryResolved)
-{
-	Task *task = CreateTask(MODIFY_TASK);
-	List *rangeTableList = NIL;
-
-	ExtractRangeTableEntryWalker((Node *) query, &rangeTableList);
-	RangeTblEntry *updateOrDeleteRTE = ExtractResultRelationRTE(query);
-
-	CitusTableCacheEntry *modificationTableCacheEntry = GetCitusTableCacheEntry(
-		updateOrDeleteRTE->relid);
-	char modificationPartitionMethod = modificationTableCacheEntry->partitionMethod;
-
-	if (modificationPartitionMethod == DISTRIBUTE_BY_NONE &&
-		SelectsFromDistributedTable(rangeTableList, query))
-	{
-		ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
-						errmsg("cannot perform select on a distributed table "
-							   "and modify a reference table")));
-	}
-
-	task->taskPlacementList = placementList;
-	SetTaskQueryIfShouldLazyDeparse(task, query);
-	task->anchorShardId = shardId;
-	task->jobId = jobId;
-	task->relationShardList = relationShardList;
-	task->replicationModel = modificationTableCacheEntry->replicationModel;
-	task->parametersInQueryStringResolved = parametersInQueryResolved;
-
-	return list_make1(task);
-}
-
-
 /*
  * SelectsFromDistributedTable checks if there is a select on a distributed
  * table by looking into range table entries.
@@ -1901,12 +1962,11 @@ static bool
 SelectsFromDistributedTable(List *rangeTableList, Query *query)
 {
 	ListCell *rangeTableCell = NULL;
-	int resultRelation = query->resultRelation;
 	RangeTblEntry *resultRangeTableEntry = NULL;
 
-	if (resultRelation > 0)
+	if (query->resultRelation > 0)
 	{
-		resultRangeTableEntry = rt_fetch(resultRelation, query->rtable);
+		resultRangeTableEntry = ExtractResultRelationRTE(query);
 	}
 
 	foreach(rangeTableCell, rangeTableList)
@@ -3186,7 +3246,7 @@ MultiRouterPlannableQuery(Query *query)
 		}
 	}
 
-	return ErrorIfQueryHasModifyingCTE(query);
+	return ErrorIfQueryHasUnroutableModifyingCTE(query);
 }
 
 
@@ -3247,19 +3307,25 @@ CopyRelationRestrictionContext(RelationRestrictionContext *oldContext)
 
 
 /*
- * ErrorIfQueryHasModifyingCTE checks if the query contains modifying common table
+ * ErrorIfQueryHasUnroutableModifyingCTE checks if the query contains modifying common table
  * expressions and errors out if it does.
  */
 static DeferredErrorMessage *
-ErrorIfQueryHasModifyingCTE(Query *queryTree)
+ErrorIfQueryHasUnroutableModifyingCTE(Query *queryTree)
 {
-	ListCell *cteCell = NULL;
-
 	Assert(queryTree->commandType == CMD_SELECT);
 
-	foreach(cteCell, queryTree->cteList)
+	if (!queryTree->hasModifyingCTE)
+	{
+		return NULL;
+	}
+
+	/* we can't route conflicting replication models */
+	char replicationModel = 0;
+
+	CommonTableExpr *cte = NULL;
+	foreach_ptr(cte, queryTree->cteList)
 	{
-		CommonTableExpr *cte = (CommonTableExpr *) lfirst(cteCell);
 		Query *cteQuery = (Query *) cte->ctequery;
 
 		/*
@@ -3268,13 +3334,48 @@ ErrorIfQueryHasModifyingCTE(Query *queryTree)
 		 * be at top level of CTE. Therefore it is OK to just check for top level.
 		 * Similarly, we do not need to check for subqueries.
 		 */
-		if (cteQuery->commandType != CMD_SELECT)
+		if (cteQuery->commandType != CMD_SELECT &&
+			cteQuery->commandType != CMD_UPDATE &&
+			cteQuery->commandType != CMD_DELETE)
 		{
 			return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED,
-								 "data-modifying statements are not supported in "
+								 "only SELECT, UPDATE, or DELETE common table expressions "
-								 "the WITH clauses of distributed queries",
+								 "may be router planned",
 								 NULL, NULL);
 		}
+
+		if (cteQuery->commandType != CMD_SELECT)
+		{
+			Oid distributedTableId = InvalidOid;
+			DeferredErrorMessage *cteError =
+				ModifyPartialQuerySupported(cteQuery, false, &distributedTableId);
+			if (cteError)
+			{
+				return cteError;
+			}
+
+			CitusTableCacheEntry *modificationTableCacheEntry =
+				GetCitusTableCacheEntry(distributedTableId);
+			char modificationPartitionMethod =
+				modificationTableCacheEntry->partitionMethod;
+
+			if (modificationPartitionMethod == DISTRIBUTE_BY_NONE)
+			{
+				return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED,
+									 "cannot router plan modification of a reference table",
+									 NULL, NULL);
+			}
+
+			if (replicationModel &&
+				modificationTableCacheEntry->replicationModel != replicationModel)
+			{
+				return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED,
+									 "cannot route mixed replication models",
+									 NULL, NULL);
+			}
+
+			replicationModel = modificationTableCacheEntry->replicationModel;
+		}
 	}
 
 	/* everything OK */

src/backend/distributed/utils/citus_clauses.c

@@ -38,14 +38,14 @@ static bool ShouldEvaluateFunctionWithMasterContext(MasterEvaluationContext *
 													evaluationContext);
 
 /*
- * RequiresMastereEvaluation returns the executor needs to reparse and
+ * RequiresMasterEvaluation returns the executor needs to reparse and
  * try to execute this query, which is the case if the query contains
  * any stable or volatile function.
  */
 bool
 RequiresMasterEvaluation(Query *query)
 {
-	if (query->commandType == CMD_SELECT)
+	if (query->commandType == CMD_SELECT && !query->hasModifyingCTE)
 	{
 		return false;
 	}
@@ -55,32 +55,23 @@ RequiresMasterEvaluation(Query *query)
 
 
 /*
- * ExecuteMasterEvaluableFunctionsAndParameters evaluates expressions and parameters
+ * ExecuteMasterEvaluableExpressions evaluates expressions and parameters
  * that can be resolved to a constant.
  */
 void
-ExecuteMasterEvaluableFunctionsAndParameters(Query *query, PlanState *planState)
+ExecuteMasterEvaluableExpressions(Query *query, PlanState *planState)
 {
 	MasterEvaluationContext masterEvaluationContext;
 
 	masterEvaluationContext.planState = planState;
-	masterEvaluationContext.evaluationMode = EVALUATE_FUNCTIONS_PARAMS;
+	if (query->commandType == CMD_SELECT)
-
+	{
-	PartiallyEvaluateExpression((Node *) query, &masterEvaluationContext);
+		masterEvaluationContext.evaluationMode = EVALUATE_PARAMS;
-}
+	}
-
+	else
-
+	{
-/*
+		masterEvaluationContext.evaluationMode = EVALUATE_FUNCTIONS_PARAMS;
- * ExecuteMasterEvaluableParameters evaluates external parameters that can be
+	}
- * resolved to a constant.
- */
-void
-ExecuteMasterEvaluableParameters(Query *query, PlanState *planState)
-{
-	MasterEvaluationContext masterEvaluationContext;
-
-	masterEvaluationContext.planState = planState;
-	masterEvaluationContext.evaluationMode = EVALUATE_PARAMS;
 
 	PartiallyEvaluateExpression((Node *) query, &masterEvaluationContext);
 }
@@ -146,9 +137,22 @@ PartiallyEvaluateExpression(Node *expression,
 	}
 	else if (nodeTag == T_Query)
 	{
-		return (Node *) query_tree_mutator((Query *) expression,
+		Query *query = (Query *) expression;
+		MasterEvaluationContext subContext = *masterEvaluationContext;
+		if (query->commandType != CMD_SELECT)
+		{
+			/*
+			 * Currently INSERT SELECT evaluates stable functions on master,
+			 * while a plain SELECT does not. For evaluating SELECT evaluationMode is
+			 * EVALUATE_PARAMS, but if recursing into a modifying CTE switch into
+			 * EVALUATE_FUNCTIONS_PARAMS.
+			 */
+			subContext.evaluationMode = EVALUATE_FUNCTIONS_PARAMS;
+		}
+
+		return (Node *) query_tree_mutator(query,
 										   PartiallyEvaluateExpression,
-										   masterEvaluationContext,
+										   &subContext,
 										   QTW_DONT_COPY_QUERY);
 	}
 	else
@@ -359,6 +363,8 @@ citus_evaluate_expr(Expr *expr, Oid result_type, int32 result_typmod,
 							  resultTypByVal);
 }
 
+/* *INDENT-ON* */
+
 
 /*
  * CitusIsVolatileFunctionIdChecker checks if the given function id is
@@ -446,6 +452,3 @@ CitusIsMutableFunction(Node *node)
 
 	return false;
 }
-
-
-/* *INDENT-ON* */

src/include/distributed/citus_clauses.h

@@ -44,9 +44,7 @@ typedef struct MasterEvaluationContext
 
 
 extern bool RequiresMasterEvaluation(Query *query);
-extern void ExecuteMasterEvaluableFunctionsAndParameters(Query *query,
+extern void ExecuteMasterEvaluableExpressions(Query *query, PlanState *planState);
-														 PlanState *planState);
-extern void ExecuteMasterEvaluableParameters(Query *query, PlanState *planState);
 extern Node * PartiallyEvaluateExpression(Node *expression,
 										  MasterEvaluationContext *masterEvaluationContext);
 extern bool CitusIsVolatileFunction(Node *node);

src/include/distributed/multi_executor.h

@@ -123,7 +123,7 @@ extern uint64 ExecuteTaskListExtended(ExecutionParams *executionParams);
 extern uint64 ExecuteTaskListIntoTupleStore(RowModifyLevel modLevel, List *taskList,
 											TupleDesc tupleDescriptor,
 											Tuplestorestate *tupleStore,
-											bool hasReturning);
+											bool expectResults);
 extern bool IsCitusCustomState(PlanState *planState);
 extern TupleTableSlot * CitusExecScan(CustomScanState *node);
 extern TupleTableSlot * ReturnTupleFromTuplestore(CitusScanState *scanState);

src/include/distributed/multi_logical_planner.h

@@ -193,6 +193,8 @@ extern bool TargetListOnPartitionColumn(Query *query, List *targetEntryList);
 extern bool FindNodeCheckInRangeTableList(List *rtable, bool (*check)(Node *));
 extern bool IsCitusTableRTE(Node *node);
 extern bool IsDistributedTableRTE(Node *node);
+extern bool IsReferenceTableRTE(Node *node);
+extern bool QueryContainsDistributedTableRTE(Query *query);
 extern bool IsCitusExtraDataContainerRelation(RangeTblEntry *rte);
 extern bool ContainsReadIntermediateResultFunction(Node *node);
 extern bool ContainsReadIntermediateResultArrayFunction(Node *node);

src/test/regress/bin/normalize.sed

@@ -60,6 +60,9 @@ s/(job_[0-9]+\/task_[0-9]+\/p_[0-9]+\.)[0-9]+/\1xxxx/g
 # isolation_ref2ref_foreign_keys
 s/"(ref_table_[0-9]_|ref_table_[0-9]_value_fkey_)[0-9]+"/"\1xxxxxxx"/g
 
+# pg11/pg12 varies in isolation debug output
+s/s1: DEBUG:/DEBUG:/g
+
 # commands cascading to shard relations
 s/(NOTICE:  .*_)[0-9]{5,}( CASCADE)/\1xxxxx\2/g
 s/(NOTICE:  [a-z]+ cascades to table ".*)_[0-9]{5,}"/\1_xxxxx"/g

src/test/regress/expected/cte_inline.out

@@ -924,7 +924,7 @@ DEBUG:  Plan is router executable
 -- we don't inline CTEs if they are modifying CTEs
 WITH cte_1 AS (DELETE FROM test_table WHERE key % 3  = 1 RETURNING key)
 SELECT * FROM cte_1 ORDER BY 1 DESC LIMIT 3;
-DEBUG:  data-modifying statements are not supported in the WITH clauses of distributed queries
+DEBUG:  Router planner cannot handle multi-shard select queries
 DEBUG:  generating subplan XXX_1 for CTE cte_1: DELETE FROM cte_inline.test_table WHERE ((key OPERATOR(pg_catalog.%) 3) OPERATOR(pg_catalog.=) 1) RETURNING key
 DEBUG:  Creating router plan
 DEBUG:  Plan is router executable
@@ -941,7 +941,7 @@ DEBUG:  Plan is router executable
 -- NOT MATERIALIZED should not affect modifying CTEs
 WITH cte_1 AS NOT MATERIALIZED (DELETE FROM test_table WHERE key % 3 = 0 RETURNING key)
 SELECT count(*) FROM cte_1;
-DEBUG:  data-modifying statements are not supported in the WITH clauses of distributed queries
+DEBUG:  Router planner cannot handle multi-shard select queries
 DEBUG:  generating subplan XXX_1 for CTE cte_1: DELETE FROM cte_inline.test_table WHERE ((key OPERATOR(pg_catalog.%) 3) OPERATOR(pg_catalog.=) 0) RETURNING key
 DEBUG:  Creating router plan
 DEBUG:  Plan is router executable

src/test/regress/expected/cte_inline_0.out

@@ -801,7 +801,7 @@ ERROR:  syntax error at or near "NOT"
 -- we don't inline CTEs if they are modifying CTEs
 WITH cte_1 AS (DELETE FROM test_table WHERE key % 3  = 1 RETURNING key)
 SELECT * FROM cte_1 ORDER BY 1 DESC LIMIT 3;
-DEBUG:  data-modifying statements are not supported in the WITH clauses of distributed queries
+DEBUG:  Router planner cannot handle multi-shard select queries
 DEBUG:  generating subplan XXX_1 for CTE cte_1: DELETE FROM cte_inline.test_table WHERE ((key OPERATOR(pg_catalog.%) 3) OPERATOR(pg_catalog.=) 1) RETURNING key
 DEBUG:  Creating router plan
 DEBUG:  Plan is router executable

src/test/regress/expected/fast_path_router_modify.out

@@ -146,7 +146,7 @@ ERROR:  non-IMMUTABLE functions are not allowed in the RETURNING clause
 -- modifying ctes are not supported via fast-path
 WITH t1 AS (DELETE FROM modify_fast_path WHERE key = 1), t2 AS  (SELECT * FROM modify_fast_path) SELECT * FROM t2;
 DEBUG:  CTE t2 is going to be inlined via distributed planning
-DEBUG:  data-modifying statements are not supported in the WITH clauses of distributed queries
+DEBUG:  Router planner cannot handle multi-shard select queries
 DEBUG:  generating subplan XXX_1 for CTE t1: DELETE FROM fast_path_router_modify.modify_fast_path WHERE (key OPERATOR(pg_catalog.=) 1)
 DEBUG:  Distributed planning for a fast-path router query
 DEBUG:  Creating router plan

src/test/regress/expected/foreign_key_restriction_enforcement.out

@@ -920,6 +920,48 @@ BEGIN;
 ERROR:  cannot execute DML on reference table "transitive_reference_table" because there was a parallel DML access to distributed table "on_update_fkey_table" in the same transaction
 HINT:  Try re-running the transaction with "SET LOCAL citus.multi_shard_modify_mode TO 'sequential';"
 ROLLBACK;
+BEGIN;
+	WITH cte AS (UPDATE on_update_fkey_table SET value_1 = 16 WHERE value_1 = 15 RETURNING *)
+    SELECT * FROM cte;
+ id  | value_1
+---------------------------------------------------------------------
+  15 |      16
+ 115 |      16
+ 215 |      16
+ 315 |      16
+ 415 |      16
+ 515 |      16
+ 615 |      16
+ 715 |      16
+ 815 |      16
+ 915 |      16
+(10 rows)
+
+	UPDATE reference_table SET id = 160 WHERE id = 15;
+ERROR:  cannot execute DML on reference table "reference_table" because there was a parallel DML access to distributed table "on_update_fkey_table" in the same transaction
+HINT:  Try re-running the transaction with "SET LOCAL citus.multi_shard_modify_mode TO 'sequential';"
+ROLLBACK;
+BEGIN;
+	WITH cte AS (UPDATE on_update_fkey_table SET value_1 = 16 WHERE value_1 = 15 RETURNING *)
+    SELECT * FROM cte;
+ id  | value_1
+---------------------------------------------------------------------
+  15 |      16
+ 115 |      16
+ 215 |      16
+ 315 |      16
+ 415 |      16
+ 515 |      16
+ 615 |      16
+ 715 |      16
+ 815 |      16
+ 915 |      16
+(10 rows)
+
+	UPDATE transitive_reference_table SET id = 160 WHERE id = 15;
+ERROR:  cannot execute DML on reference table "transitive_reference_table" because there was a parallel DML access to distributed table "on_update_fkey_table" in the same transaction
+HINT:  Try re-running the transaction with "SET LOCAL citus.multi_shard_modify_mode TO 'sequential';"
+ROLLBACK;
 -- case 5.3: Parallel UPDATE on distributed table follow by an unrelated DDL on reference table
 BEGIN;
 	UPDATE on_update_fkey_table SET value_1 = 16 WHERE value_1 = 15;

src/test/regress/expected/insert_select_repartition.out

@@ -670,7 +670,7 @@ WITH r AS (
 )
 INSERT INTO target_table SELECT source_table.a, max(source_table.b) FROM source_table NATURAL JOIN r GROUP BY source_table.a;
 DEBUG:  INSERT target table and the source relation of the SELECT partition column value must be colocated in distributed INSERT ... SELECT
-DEBUG:  data-modifying statements are not supported in the WITH clauses of distributed queries
+DEBUG:  only SELECT, UPDATE, or DELETE common table expressions may be router planned
 DEBUG:  generating subplan XXX_1 for CTE r: INSERT INTO insert_select_repartition.target_table (a, b) SELECT a, b FROM insert_select_repartition.source_table RETURNING target_table.a, target_table.b
 DEBUG:  INSERT target table and the source relation of the SELECT partition column value must be colocated in distributed INSERT ... SELECT
 DEBUG:  Plan XXX query after replacing subqueries and CTEs: SELECT a, max AS b FROM (SELECT source_table.a, max(source_table.b) AS max FROM (insert_select_repartition.source_table JOIN (SELECT intermediate_result.a, intermediate_result.b FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer)) r USING (a, b)) GROUP BY source_table.a) citus_insert_select_subquery

src/test/regress/expected/isolation_select_for_update.out

@@ -2,30 +2,30 @@ Parsed test spec with 2 sessions
 
 starting permutation: s1-begin s1-select-from-t1-t2-for-update s2-begin s2-update-t1 s1-commit s2-commit
 step s1-begin:
-	BEGIN;
+ BEGIN;
 
 step s1-select-from-t1-t2-for-update:
- 	SELECT * FROM
+  SELECT * FROM
-		test_table_1_rf1 as tt1 INNER JOIN test_table_2_rf1 as tt2 on tt1.id = tt2.id
+  test_table_1_rf1 as tt1 INNER JOIN test_table_2_rf1 as tt2 on tt1.id = tt2.id
-		WHERE tt1.id = 1
+  WHERE tt1.id = 1
-		ORDER BY 1
+  ORDER BY 1
-		FOR UPDATE;
+  FOR UPDATE;
 
 id             val_1          id             val_1
 
 1              2              1              2
 step s2-begin:
-	BEGIN;
+ BEGIN;
 
 step s2-update-t1:
-	UPDATE test_table_1_rf1 SET val_1 = 5 WHERE id = 1;
+ UPDATE test_table_1_rf1 SET val_1 = 5 WHERE id = 1;
  <waiting ...>
 step s1-commit:
-	COMMIT;
+ COMMIT;
 
 step s2-update-t1: <... completed>
 step s2-commit:
-	COMMIT;
+ COMMIT;
 
 restore_isolation_tester_func
 
@@ -33,30 +33,30 @@ restore_isolation_tester_func
 
 starting permutation: s1-begin s1-select-from-t1-t2-for-share s2-begin s2-delete-t1 s1-commit s2-commit
 step s1-begin:
-	BEGIN;
+ BEGIN;
 
 step s1-select-from-t1-t2-for-share:
-	SELECT * FROM
+ SELECT * FROM
-		test_table_1_rf1 as tt1 INNER JOIN test_table_2_rf1 as tt2 on tt1.id = tt2.id
+  test_table_1_rf1 as tt1 INNER JOIN test_table_2_rf1 as tt2 on tt1.id = tt2.id
-		WHERE tt1.id = 1
+  WHERE tt1.id = 1
-		ORDER BY 1
+  ORDER BY 1
-		FOR SHARE;
+  FOR SHARE;
 
 id             val_1          id             val_1
 
 1              2              1              2
 step s2-begin:
-	BEGIN;
+ BEGIN;
 
 step s2-delete-t1:
-	DELETE FROM test_table_1_rf1 WHERE id = 1;
+ DELETE FROM test_table_1_rf1 WHERE id = 1;
  <waiting ...>
 step s1-commit:
-	COMMIT;
+ COMMIT;
 
 step s2-delete-t1: <... completed>
 step s2-commit:
-	COMMIT;
+ COMMIT;
 
 restore_isolation_tester_func
 
@@ -64,30 +64,30 @@ restore_isolation_tester_func
 
 starting permutation: s1-begin s1-select-from-t1-rt-for-update s2-begin s2-update-t1 s1-commit s2-commit
 step s1-begin:
-	BEGIN;
+ BEGIN;
 
 step s1-select-from-t1-rt-for-update:
-	SELECT * FROM
+ SELECT * FROM
-		test_table_1_rf1 as tt1 INNER JOIN ref_table as rt1 on tt1.id = rt1.id
+  test_table_1_rf1 as tt1 INNER JOIN ref_table as rt1 on tt1.id = rt1.id
-		WHERE tt1.id = 1
+  WHERE tt1.id = 1
-		ORDER BY 1
+  ORDER BY 1
-		FOR UPDATE;
+  FOR UPDATE;
 
 id             val_1          id             val_1
 
 1              2              1              2
 step s2-begin:
-	BEGIN;
+ BEGIN;
 
 step s2-update-t1:
-	UPDATE test_table_1_rf1 SET val_1 = 5 WHERE id = 1;
+ UPDATE test_table_1_rf1 SET val_1 = 5 WHERE id = 1;
  <waiting ...>
 step s1-commit:
-	COMMIT;
+ COMMIT;
 
 step s2-update-t1: <... completed>
 step s2-commit:
-	COMMIT;
+ COMMIT;
 
 restore_isolation_tester_func
 
@@ -95,31 +95,31 @@ restore_isolation_tester_func
 
 starting permutation: s1-begin s1-select-from-t1-rt-with-lc-for-update s2-begin s2-update-rt s1-commit s2-commit
 step s1-begin:
-	BEGIN;
+ BEGIN;
 
 step s1-select-from-t1-rt-with-lc-for-update:
-	SELECT * FROM
+ SELECT * FROM
-		test_table_1_rf1 as tt1 INNER JOIN ref_table as rt1 on tt1.id = rt1.id
+  test_table_1_rf1 as tt1 INNER JOIN ref_table as rt1 on tt1.id = rt1.id
-		WHERE tt1.id = 1
+  WHERE tt1.id = 1
-		ORDER BY 1
+  ORDER BY 1
-		FOR UPDATE
+  FOR UPDATE
-		OF rt1;
+  OF rt1;
 
 id             val_1          id             val_1
 
 1              2              1              2
 step s2-begin:
-	BEGIN;
+ BEGIN;
 
 step s2-update-rt:
-	UPDATE ref_table SET val_1 = 5 WHERE id = 1;
+ UPDATE ref_table SET val_1 = 5 WHERE id = 1;
  <waiting ...>
 step s1-commit:
-	COMMIT;
+ COMMIT;
 
 step s2-update-rt: <... completed>
 step s2-commit:
-	COMMIT;
+ COMMIT;
 
 restore_isolation_tester_func
 
@@ -127,30 +127,30 @@ restore_isolation_tester_func
 
 starting permutation: s1-begin s1-select-from-t1-rt-with-lc-for-update s2-begin s2-update-t1 s1-commit s2-commit
 step s1-begin:
-	BEGIN;
+ BEGIN;
 
 step s1-select-from-t1-rt-with-lc-for-update:
-	SELECT * FROM
+ SELECT * FROM
-		test_table_1_rf1 as tt1 INNER JOIN ref_table as rt1 on tt1.id = rt1.id
+  test_table_1_rf1 as tt1 INNER JOIN ref_table as rt1 on tt1.id = rt1.id
-		WHERE tt1.id = 1
+  WHERE tt1.id = 1
-		ORDER BY 1
+  ORDER BY 1
-		FOR UPDATE
+  FOR UPDATE
-		OF rt1;
+  OF rt1;
 
 id             val_1          id             val_1
 
 1              2              1              2
 step s2-begin:
-	BEGIN;
+ BEGIN;
 
 step s2-update-t1:
-	UPDATE test_table_1_rf1 SET val_1 = 5 WHERE id = 1;
+ UPDATE test_table_1_rf1 SET val_1 = 5 WHERE id = 1;
 
 step s1-commit:
-	COMMIT;
+ COMMIT;
 
 step s2-commit:
-	COMMIT;
+ COMMIT;
 
 restore_isolation_tester_func
 
@@ -158,36 +158,36 @@ restore_isolation_tester_func
 
 starting permutation: s1-begin s1-select-from-t1-t2-for-share s2-begin s2-select-from-t1-t2-for-share s1-commit s2-commit
 step s1-begin:
-	BEGIN;
+ BEGIN;
 
 step s1-select-from-t1-t2-for-share:
-	SELECT * FROM
+ SELECT * FROM
-		test_table_1_rf1 as tt1 INNER JOIN test_table_2_rf1 as tt2 on tt1.id = tt2.id
+  test_table_1_rf1 as tt1 INNER JOIN test_table_2_rf1 as tt2 on tt1.id = tt2.id
-		WHERE tt1.id = 1
+  WHERE tt1.id = 1
-		ORDER BY 1
+  ORDER BY 1
-		FOR SHARE;
+  FOR SHARE;
 
 id             val_1          id             val_1
 
 1              2              1              2
 step s2-begin:
-	BEGIN;
+ BEGIN;
 
 step s2-select-from-t1-t2-for-share:
-	SELECT * FROM
+ SELECT * FROM
-		test_table_1_rf1 as tt1 INNER JOIN test_table_1_rf1 as tt2 on tt1.id = tt2.id
+  test_table_1_rf1 as tt1 INNER JOIN test_table_1_rf1 as tt2 on tt1.id = tt2.id
-		WHERE tt1.id = 1
+  WHERE tt1.id = 1
-		ORDER BY 1
+  ORDER BY 1
-		FOR SHARE;
+  FOR SHARE;
 
 id             val_1          id             val_1
 
 1              2              1              2
 step s1-commit:
-	COMMIT;
+ COMMIT;
 
 step s2-commit:
-	COMMIT;
+ COMMIT;
 
 restore_isolation_tester_func
 
@@ -195,37 +195,37 @@ restore_isolation_tester_func
 
 starting permutation: s1-begin s1-select-from-t1-rt-for-update s2-begin s2-select-from-t1-t2-for-update s1-commit s2-commit
 step s1-begin:
-	BEGIN;
+ BEGIN;
 
 step s1-select-from-t1-rt-for-update:
-	SELECT * FROM
+ SELECT * FROM
-		test_table_1_rf1 as tt1 INNER JOIN ref_table as rt1 on tt1.id = rt1.id
+  test_table_1_rf1 as tt1 INNER JOIN ref_table as rt1 on tt1.id = rt1.id
-		WHERE tt1.id = 1
+  WHERE tt1.id = 1
-		ORDER BY 1
+  ORDER BY 1
-		FOR UPDATE;
+  FOR UPDATE;
 
 id             val_1          id             val_1
 
 1              2              1              2
 step s2-begin:
-	BEGIN;
+ BEGIN;
 
 step s2-select-from-t1-t2-for-update:
- 	SELECT * FROM
+  SELECT * FROM
-		test_table_1_rf1 as tt1 INNER JOIN test_table_1_rf1 as tt2 on tt1.id = tt2.id
+  test_table_1_rf1 as tt1 INNER JOIN test_table_1_rf1 as tt2 on tt1.id = tt2.id
-		WHERE tt1.id = 1
+  WHERE tt1.id = 1
-		ORDER BY 1
+  ORDER BY 1
-		FOR UPDATE;
+  FOR UPDATE;
  <waiting ...>
 step s1-commit:
-	COMMIT;
+ COMMIT;
 
 step s2-select-from-t1-t2-for-update: <... completed>
 id             val_1          id             val_1
 
 1              2              1              2
 step s2-commit:
-	COMMIT;
+ COMMIT;
 
 restore_isolation_tester_func
 
@@ -233,34 +233,34 @@ restore_isolation_tester_func
 
 starting permutation: s1-begin s1-select-from-t1-within-cte s2-begin s2-select-from-t1-t2-for-update s1-commit s2-commit
 step s1-begin:
-	BEGIN;
+ BEGIN;
 
 step s1-select-from-t1-within-cte:
-	WITH first_value AS ( SELECT val_1 FROM test_table_1_rf1 WHERE id = 1 FOR UPDATE)
+ WITH first_value AS (SELECT val_1 FROM test_table_1_rf1 WHERE id = 1 FOR UPDATE)
-	SELECT * FROM first_value;
+ SELECT * FROM first_value WHERE EXISTS (SELECT * FROM first_value);
 
 val_1
 
 2
 step s2-begin:
-	BEGIN;
+ BEGIN;
 
 step s2-select-from-t1-t2-for-update:
- 	SELECT * FROM
+  SELECT * FROM
-		test_table_1_rf1 as tt1 INNER JOIN test_table_1_rf1 as tt2 on tt1.id = tt2.id
+  test_table_1_rf1 as tt1 INNER JOIN test_table_1_rf1 as tt2 on tt1.id = tt2.id
-		WHERE tt1.id = 1
+  WHERE tt1.id = 1
-		ORDER BY 1
+  ORDER BY 1
-		FOR UPDATE;
+  FOR UPDATE;
  <waiting ...>
 step s1-commit:
-	COMMIT;
+ COMMIT;
 
 step s2-select-from-t1-t2-for-update: <... completed>
 id             val_1          id             val_1
 
 1              2              1              2
 step s2-commit:
-	COMMIT;
+ COMMIT;
 
 restore_isolation_tester_func
 
@@ -268,27 +268,27 @@ restore_isolation_tester_func
 
 starting permutation: s1-begin s1-select-from-t1-within-cte s2-begin s2-update-t1 s1-commit s2-commit
 step s1-begin:
-	BEGIN;
+ BEGIN;
 
 step s1-select-from-t1-within-cte:
-	WITH first_value AS ( SELECT val_1 FROM test_table_1_rf1 WHERE id = 1 FOR UPDATE)
+ WITH first_value AS (SELECT val_1 FROM test_table_1_rf1 WHERE id = 1 FOR UPDATE)
-	SELECT * FROM first_value;
+ SELECT * FROM first_value WHERE EXISTS (SELECT * FROM first_value);
 
 val_1
 
 2
 step s2-begin:
-	BEGIN;
+ BEGIN;
 
 step s2-update-t1:
-	UPDATE test_table_1_rf1 SET val_1 = 5 WHERE id = 1;
+ UPDATE test_table_1_rf1 SET val_1 = 5 WHERE id = 1;
  <waiting ...>
 step s1-commit:
-	COMMIT;
+ COMMIT;
 
 step s2-update-t1: <... completed>
 step s2-commit:
-	COMMIT;
+ COMMIT;
 
 restore_isolation_tester_func
 
@@ -296,26 +296,31 @@ restore_isolation_tester_func
 
 starting permutation: s1-begin s1-select-from-t1-with-subquery s2-begin s2-update-t1 s1-commit s2-commit
 step s1-begin:
-	BEGIN;
+ BEGIN;
 
+DEBUG:  Creating router plan
 step s1-select-from-t1-with-subquery:
-	SELECT * FROM (SELECT * FROM test_table_1_rf1 FOR UPDATE) foo WHERE id = 1;
+ SET client_min_messages TO DEBUG2;
+ SELECT * FROM (SELECT * FROM test_table_1_rf1 FOR UPDATE) foo WHERE id = 1;
+ RESET client_min_messages;
 
+DEBUG:  Plan is router executable
+DETAIL:  distribution column value: 1
 id             val_1
 
 1              2
 step s2-begin:
-	BEGIN;
+ BEGIN;
 
 step s2-update-t1:
-	UPDATE test_table_1_rf1 SET val_1 = 5 WHERE id = 1;
+ UPDATE test_table_1_rf1 SET val_1 = 5 WHERE id = 1;
  <waiting ...>
 step s1-commit:
-	COMMIT;
+ COMMIT;
 
 step s2-update-t1: <... completed>
 step s2-commit:
-	COMMIT;
+ COMMIT;
 
 restore_isolation_tester_func
 
@@ -323,26 +328,26 @@ restore_isolation_tester_func
 
 starting permutation: s1-begin s1-select-from-rt-with-subquery s2-begin s2-update-rt s1-commit s2-commit
 step s1-begin:
-	BEGIN;
+ BEGIN;
 
 step s1-select-from-rt-with-subquery:
-	SELECT * FROM (SELECT * FROM ref_table FOR UPDATE) foo WHERE id = 1;
+ SELECT * FROM (SELECT * FROM ref_table FOR UPDATE) foo WHERE id = 1;
 
 id             val_1
 
 1              2
 step s2-begin:
-	BEGIN;
+ BEGIN;
 
 step s2-update-rt:
-	UPDATE ref_table SET val_1 = 5 WHERE id = 1;
+ UPDATE ref_table SET val_1 = 5 WHERE id = 1;
  <waiting ...>
 step s1-commit:
-	COMMIT;
+ COMMIT;
 
 step s2-update-rt: <... completed>
 step s2-commit:
-	COMMIT;
+ COMMIT;
 
 restore_isolation_tester_func
 
@@ -350,26 +355,26 @@ restore_isolation_tester_func
 
 starting permutation: s1-begin s1-select-from-t1-with-view s2-begin s2-update-t1 s1-commit s2-commit
 step s1-begin:
-	BEGIN;
+ BEGIN;
 
 step s1-select-from-t1-with-view:
-	SELECT * FROM test_1 WHERE id = 1 FOR UPDATE;
+ SELECT * FROM test_1 WHERE id = 1 FOR UPDATE;
 
 id             val_1
 
 1              2
 step s2-begin:
-	BEGIN;
+ BEGIN;
 
 step s2-update-t1:
-	UPDATE test_table_1_rf1 SET val_1 = 5 WHERE id = 1;
+ UPDATE test_table_1_rf1 SET val_1 = 5 WHERE id = 1;
  <waiting ...>
 step s1-commit:
-	COMMIT;
+ COMMIT;
 
 step s2-update-t1: <... completed>
 step s2-commit:
-	COMMIT;
+ COMMIT;
 
 restore_isolation_tester_func
 
@@ -377,24 +382,24 @@ restore_isolation_tester_func
 
 starting permutation: s1-begin s1-update-rt-with-cte-select-from-rt s2-begin s2-update-rt s1-commit s2-commit
 step s1-begin:
-	BEGIN;
+ BEGIN;
 
 step s1-update-rt-with-cte-select-from-rt:
-	WITH foo AS (SELECT * FROM ref_table FOR UPDATE)
+ WITH foo AS (SELECT * FROM ref_table FOR UPDATE)
-	UPDATE ref_table SET val_1 = 4 FROM foo WHERE ref_table.id = foo.id;
+ UPDATE ref_table SET val_1 = 4 FROM foo WHERE ref_table.id = foo.id AND EXISTS (SELECT * FROM foo);
 
 step s2-begin:
-	BEGIN;
+ BEGIN;
 
 step s2-update-rt:
-	UPDATE ref_table SET val_1 = 5 WHERE id = 1;
+ UPDATE ref_table SET val_1 = 5 WHERE id = 1;
  <waiting ...>
 step s1-commit:
-	COMMIT;
+ COMMIT;
 
 step s2-update-rt: <... completed>
 step s2-commit:
-	COMMIT;
+ COMMIT;
 
 restore_isolation_tester_func
 

src/test/regress/expected/isolation_update_vs_all.out

@@ -20,6 +20,32 @@ restore_isolation_tester_func
 
 
 
+starting permutation: s1-initialize s1-begin s2-begin s1-update-cte s2-update-cte s1-commit s2-commit s1-select-count
+create_distributed_table
+
+
+step s1-initialize: COPY update_hash FROM PROGRAM 'echo 0, a && echo 1, b && echo 2, c && echo 3, d && echo 4, e' WITH CSV;
+step s1-begin: BEGIN;
+step s2-begin: BEGIN;
+step s1-update-cte: WITH cte AS (UPDATE update_hash SET data = 'l' WHERE id = 4 RETURNING *) SELECT * FROM cte WHERE id = 4;
+id             data
+
+4              l
+step s2-update-cte: WITH cte AS (UPDATE update_hash SET data = 'l' WHERE id = 4 RETURNING *) SELECT * FROM cte WHERE id = 4; <waiting ...>
+step s1-commit: COMMIT;
+step s2-update-cte: <... completed>
+id             data
+
+4              l
+step s2-commit: COMMIT;
+step s1-select-count: SELECT COUNT(*) FROM update_hash;
+count
+
+5
+restore_isolation_tester_func
+
+
+
 starting permutation: s1-initialize s1-begin s2-begin s1-update s2-delete s1-commit s2-commit s1-select-count
 create_distributed_table
 
@@ -295,6 +321,289 @@ restore_isolation_tester_func
 
 
 
+starting permutation: s1-initialize s1-begin s2-begin s1-update-cte s2-delete s1-commit s2-commit s1-select-count
+create_distributed_table
+
+
+step s1-initialize: COPY update_hash FROM PROGRAM 'echo 0, a && echo 1, b && echo 2, c && echo 3, d && echo 4, e' WITH CSV;
+step s1-begin: BEGIN;
+step s2-begin: BEGIN;
+step s1-update-cte: WITH cte AS (UPDATE update_hash SET data = 'l' WHERE id = 4 RETURNING *) SELECT * FROM cte WHERE id = 4;
+id             data
+
+4              l
+step s2-delete: DELETE FROM update_hash WHERE id = 4; <waiting ...>
+step s1-commit: COMMIT;
+step s2-delete: <... completed>
+step s2-commit: COMMIT;
+step s1-select-count: SELECT COUNT(*) FROM update_hash;
+count
+
+4
+restore_isolation_tester_func
+
+
+
+starting permutation: s1-initialize s1-begin s2-begin s1-update-cte s2-truncate s1-commit s2-commit s1-select-count
+create_distributed_table
+
+
+step s1-initialize: COPY update_hash FROM PROGRAM 'echo 0, a && echo 1, b && echo 2, c && echo 3, d && echo 4, e' WITH CSV;
+step s1-begin: BEGIN;
+step s2-begin: BEGIN;
+step s1-update-cte: WITH cte AS (UPDATE update_hash SET data = 'l' WHERE id = 4 RETURNING *) SELECT * FROM cte WHERE id = 4;
+id             data
+
+4              l
+step s2-truncate: TRUNCATE update_hash; <waiting ...>
+step s1-commit: COMMIT;
+step s2-truncate: <... completed>
+step s2-commit: COMMIT;
+step s1-select-count: SELECT COUNT(*) FROM update_hash;
+count
+
+0
+restore_isolation_tester_func
+
+
+
+starting permutation: s1-initialize s1-begin s2-begin s1-update-cte s2-drop s1-commit s2-commit s1-select-count
+create_distributed_table
+
+
+step s1-initialize: COPY update_hash FROM PROGRAM 'echo 0, a && echo 1, b && echo 2, c && echo 3, d && echo 4, e' WITH CSV;
+step s1-begin: BEGIN;
+step s2-begin: BEGIN;
+step s1-update-cte: WITH cte AS (UPDATE update_hash SET data = 'l' WHERE id = 4 RETURNING *) SELECT * FROM cte WHERE id = 4;
+id             data
+
+4              l
+step s2-drop: DROP TABLE update_hash; <waiting ...>
+step s1-commit: COMMIT;
+step s2-drop: <... completed>
+step s2-commit: COMMIT;
+step s1-select-count: SELECT COUNT(*) FROM update_hash;
+ERROR:  relation "update_hash" does not exist
+restore_isolation_tester_func
+
+
+
+starting permutation: s1-initialize s1-begin s2-begin s1-update-cte s2-ddl-create-index s1-commit s2-commit s1-select-count s1-show-indexes
+create_distributed_table
+
+
+step s1-initialize: COPY update_hash FROM PROGRAM 'echo 0, a && echo 1, b && echo 2, c && echo 3, d && echo 4, e' WITH CSV;
+step s1-begin: BEGIN;
+step s2-begin: BEGIN;
+step s1-update-cte: WITH cte AS (UPDATE update_hash SET data = 'l' WHERE id = 4 RETURNING *) SELECT * FROM cte WHERE id = 4;
+id             data
+
+4              l
+step s2-ddl-create-index: CREATE INDEX update_hash_index ON update_hash(id); <waiting ...>
+step s1-commit: COMMIT;
+step s2-ddl-create-index: <... completed>
+step s2-commit: COMMIT;
+step s1-select-count: SELECT COUNT(*) FROM update_hash;
+count
+
+5
+step s1-show-indexes: SELECT run_command_on_workers('SELECT COUNT(*) FROM pg_indexes WHERE tablename LIKE ''update_hash%''');
+run_command_on_workers
+
+(localhost,57637,t,2)
+(localhost,57638,t,2)
+restore_isolation_tester_func
+
+
+
+starting permutation: s1-initialize s1-ddl-create-index s1-begin s2-begin s1-update-cte s2-ddl-drop-index s1-commit s2-commit s1-select-count s1-show-indexes
+create_distributed_table
+
+
+step s1-initialize: COPY update_hash FROM PROGRAM 'echo 0, a && echo 1, b && echo 2, c && echo 3, d && echo 4, e' WITH CSV;
+step s1-ddl-create-index: CREATE INDEX update_hash_index ON update_hash(id);
+step s1-begin: BEGIN;
+step s2-begin: BEGIN;
+step s1-update-cte: WITH cte AS (UPDATE update_hash SET data = 'l' WHERE id = 4 RETURNING *) SELECT * FROM cte WHERE id = 4;
+id             data
+
+4              l
+step s2-ddl-drop-index: DROP INDEX update_hash_index; <waiting ...>
+step s1-commit: COMMIT;
+step s2-ddl-drop-index: <... completed>
+step s2-commit: COMMIT;
+step s1-select-count: SELECT COUNT(*) FROM update_hash;
+count
+
+5
+step s1-show-indexes: SELECT run_command_on_workers('SELECT COUNT(*) FROM pg_indexes WHERE tablename LIKE ''update_hash%''');
+run_command_on_workers
+
+(localhost,57637,t,0)
+(localhost,57638,t,0)
+restore_isolation_tester_func
+
+
+
+starting permutation: s1-initialize s1-begin s1-update-cte s2-ddl-create-index-concurrently s1-commit s1-select-count s1-show-indexes
+create_distributed_table
+
+
+step s1-initialize: COPY update_hash FROM PROGRAM 'echo 0, a && echo 1, b && echo 2, c && echo 3, d && echo 4, e' WITH CSV;
+step s1-begin: BEGIN;
+step s1-update-cte: WITH cte AS (UPDATE update_hash SET data = 'l' WHERE id = 4 RETURNING *) SELECT * FROM cte WHERE id = 4;
+id             data
+
+4              l
+step s2-ddl-create-index-concurrently: CREATE INDEX CONCURRENTLY update_hash_index ON update_hash(id); <waiting ...>
+step s1-commit: COMMIT;
+step s2-ddl-create-index-concurrently: <... completed>
+step s1-select-count: SELECT COUNT(*) FROM update_hash;
+count
+
+5
+step s1-show-indexes: SELECT run_command_on_workers('SELECT COUNT(*) FROM pg_indexes WHERE tablename LIKE ''update_hash%''');
+run_command_on_workers
+
+(localhost,57637,t,2)
+(localhost,57638,t,2)
+restore_isolation_tester_func
+
+
+
+starting permutation: s1-initialize s1-begin s2-begin s1-update-cte s2-ddl-add-column s1-commit s2-commit s1-select-count s1-show-columns
+create_distributed_table
+
+
+step s1-initialize: COPY update_hash FROM PROGRAM 'echo 0, a && echo 1, b && echo 2, c && echo 3, d && echo 4, e' WITH CSV;
+step s1-begin: BEGIN;
+step s2-begin: BEGIN;
+step s1-update-cte: WITH cte AS (UPDATE update_hash SET data = 'l' WHERE id = 4 RETURNING *) SELECT * FROM cte WHERE id = 4;
+id             data
+
+4              l
+step s2-ddl-add-column: ALTER TABLE update_hash ADD new_column int DEFAULT 0; <waiting ...>
+step s1-commit: COMMIT;
+step s2-ddl-add-column: <... completed>
+step s2-commit: COMMIT;
+step s1-select-count: SELECT COUNT(*) FROM update_hash;
+count
+
+5
+step s1-show-columns: SELECT run_command_on_workers('SELECT column_name FROM information_schema.columns WHERE table_name LIKE ''update_hash%'' AND column_name = ''new_column'' ORDER BY 1 LIMIT 1');
+run_command_on_workers
+
+(localhost,57637,t,new_column)
+(localhost,57638,t,new_column)
+restore_isolation_tester_func
+
+
+
+starting permutation: s1-initialize s1-ddl-add-column s1-begin s2-begin s1-update-cte s2-ddl-drop-column s1-commit s2-commit s1-select-count s1-show-columns
+create_distributed_table
+
+
+step s1-initialize: COPY update_hash FROM PROGRAM 'echo 0, a && echo 1, b && echo 2, c && echo 3, d && echo 4, e' WITH CSV;
+step s1-ddl-add-column: ALTER TABLE update_hash ADD new_column int DEFAULT 0;
+step s1-begin: BEGIN;
+step s2-begin: BEGIN;
+step s1-update-cte: WITH cte AS (UPDATE update_hash SET data = 'l' WHERE id = 4 RETURNING *) SELECT * FROM cte WHERE id = 4;
+id             data           new_column
+
+4              l              0
+step s2-ddl-drop-column: ALTER TABLE update_hash DROP new_column; <waiting ...>
+step s1-commit: COMMIT;
+step s2-ddl-drop-column: <... completed>
+step s2-commit: COMMIT;
+step s1-select-count: SELECT COUNT(*) FROM update_hash;
+count
+
+5
+step s1-show-columns: SELECT run_command_on_workers('SELECT column_name FROM information_schema.columns WHERE table_name LIKE ''update_hash%'' AND column_name = ''new_column'' ORDER BY 1 LIMIT 1');
+run_command_on_workers
+
+(localhost,57637,t,"")
+(localhost,57638,t,"")
+restore_isolation_tester_func
+
+
+
+starting permutation: s1-initialize s1-begin s2-begin s1-update-cte s2-ddl-rename-column s1-commit s2-commit s1-select-count s1-show-columns
+create_distributed_table
+
+
+step s1-initialize: COPY update_hash FROM PROGRAM 'echo 0, a && echo 1, b && echo 2, c && echo 3, d && echo 4, e' WITH CSV;
+step s1-begin: BEGIN;
+step s2-begin: BEGIN;
+step s1-update-cte: WITH cte AS (UPDATE update_hash SET data = 'l' WHERE id = 4 RETURNING *) SELECT * FROM cte WHERE id = 4;
+id             data
+
+4              l
+step s2-ddl-rename-column: ALTER TABLE update_hash RENAME data TO new_column; <waiting ...>
+step s1-commit: COMMIT;
+step s2-ddl-rename-column: <... completed>
+step s2-commit: COMMIT;
+step s1-select-count: SELECT COUNT(*) FROM update_hash;
+count
+
+5
+step s1-show-columns: SELECT run_command_on_workers('SELECT column_name FROM information_schema.columns WHERE table_name LIKE ''update_hash%'' AND column_name = ''new_column'' ORDER BY 1 LIMIT 1');
+run_command_on_workers
+
+(localhost,57637,t,new_column)
+(localhost,57638,t,new_column)
+restore_isolation_tester_func
+
+
+
+starting permutation: s1-initialize s1-begin s2-begin s1-update-cte s2-table-size s1-commit s2-commit s1-select-count
+create_distributed_table
+
+
+step s1-initialize: COPY update_hash FROM PROGRAM 'echo 0, a && echo 1, b && echo 2, c && echo 3, d && echo 4, e' WITH CSV;
+step s1-begin: BEGIN;
+step s2-begin: BEGIN;
+step s1-update-cte: WITH cte AS (UPDATE update_hash SET data = 'l' WHERE id = 4 RETURNING *) SELECT * FROM cte WHERE id = 4;
+id             data
+
+4              l
+step s2-table-size: SELECT citus_total_relation_size('update_hash');
+citus_total_relation_size
+
+57344
+step s1-commit: COMMIT;
+step s2-commit: COMMIT;
+step s1-select-count: SELECT COUNT(*) FROM update_hash;
+count
+
+5
+restore_isolation_tester_func
+
+
+
+starting permutation: s1-initialize s1-begin s2-begin s1-update-cte s2-master-modify-multiple-shards s1-commit s2-commit s1-select-count
+create_distributed_table
+
+
+step s1-initialize: COPY update_hash FROM PROGRAM 'echo 0, a && echo 1, b && echo 2, c && echo 3, d && echo 4, e' WITH CSV;
+step s1-begin: BEGIN;
+step s2-begin: BEGIN;
+step s1-update-cte: WITH cte AS (UPDATE update_hash SET data = 'l' WHERE id = 4 RETURNING *) SELECT * FROM cte WHERE id = 4;
+id             data
+
+4              l
+step s2-master-modify-multiple-shards: DELETE FROM update_hash; <waiting ...>
+step s1-commit: COMMIT;
+step s2-master-modify-multiple-shards: <... completed>
+step s2-commit: COMMIT;
+step s1-select-count: SELECT COUNT(*) FROM update_hash;
+count
+
+0
+restore_isolation_tester_func
+
+
+
 starting permutation: s1-initialize s1-begin s2-begin s1-delete s2-update s1-commit s2-commit s1-select-count
 create_distributed_table
 
@@ -548,3 +857,282 @@ count
 restore_isolation_tester_func
 
 
+
+starting permutation: s1-initialize s1-begin s2-begin s1-delete s2-update-cte s1-commit s2-commit s1-select-count
+create_distributed_table
+
+
+step s1-initialize: COPY update_hash FROM PROGRAM 'echo 0, a && echo 1, b && echo 2, c && echo 3, d && echo 4, e' WITH CSV;
+step s1-begin: BEGIN;
+step s2-begin: BEGIN;
+step s1-delete: DELETE FROM update_hash WHERE id = 4;
+step s2-update-cte: WITH cte AS (UPDATE update_hash SET data = 'l' WHERE id = 4 RETURNING *) SELECT * FROM cte WHERE id = 4; <waiting ...>
+step s1-commit: COMMIT;
+step s2-update-cte: <... completed>
+id             data
+
+step s2-commit: COMMIT;
+step s1-select-count: SELECT COUNT(*) FROM update_hash;
+count
+
+4
+restore_isolation_tester_func
+
+
+
+starting permutation: s1-initialize s1-begin s2-begin s1-truncate s2-update-cte s1-commit s2-commit s1-select-count
+create_distributed_table
+
+
+step s1-initialize: COPY update_hash FROM PROGRAM 'echo 0, a && echo 1, b && echo 2, c && echo 3, d && echo 4, e' WITH CSV;
+step s1-begin: BEGIN;
+step s2-begin: BEGIN;
+step s1-truncate: TRUNCATE update_hash;
+step s2-update-cte: WITH cte AS (UPDATE update_hash SET data = 'l' WHERE id = 4 RETURNING *) SELECT * FROM cte WHERE id = 4; <waiting ...>
+step s1-commit: COMMIT;
+step s2-update-cte: <... completed>
+id             data
+
+step s2-commit: COMMIT;
+step s1-select-count: SELECT COUNT(*) FROM update_hash;
+count
+
+0
+restore_isolation_tester_func
+
+
+
+starting permutation: s1-initialize s1-begin s2-begin s1-drop s2-update-cte s1-commit s2-commit s1-select-count
+create_distributed_table
+
+
+step s1-initialize: COPY update_hash FROM PROGRAM 'echo 0, a && echo 1, b && echo 2, c && echo 3, d && echo 4, e' WITH CSV;
+step s1-begin: BEGIN;
+step s2-begin: BEGIN;
+step s1-drop: DROP TABLE update_hash;
+step s2-update-cte: WITH cte AS (UPDATE update_hash SET data = 'l' WHERE id = 4 RETURNING *) SELECT * FROM cte WHERE id = 4; <waiting ...>
+step s1-commit: COMMIT;
+step s2-update-cte: <... completed>
+error in steps s1-commit s2-update-cte: ERROR:  relation "update_hash" does not exist
+step s2-commit: COMMIT;
+step s1-select-count: SELECT COUNT(*) FROM update_hash;
+ERROR:  relation "update_hash" does not exist
+restore_isolation_tester_func
+
+
+
+starting permutation: s1-initialize s1-begin s2-begin s1-ddl-create-index s2-update-cte s1-commit s2-commit s1-select-count s1-show-indexes
+create_distributed_table
+
+
+step s1-initialize: COPY update_hash FROM PROGRAM 'echo 0, a && echo 1, b && echo 2, c && echo 3, d && echo 4, e' WITH CSV;
+step s1-begin: BEGIN;
+step s2-begin: BEGIN;
+step s1-ddl-create-index: CREATE INDEX update_hash_index ON update_hash(id);
+step s2-update-cte: WITH cte AS (UPDATE update_hash SET data = 'l' WHERE id = 4 RETURNING *) SELECT * FROM cte WHERE id = 4; <waiting ...>
+step s1-commit: COMMIT;
+step s2-update-cte: <... completed>
+id             data
+
+4              l
+step s2-commit: COMMIT;
+step s1-select-count: SELECT COUNT(*) FROM update_hash;
+count
+
+5
+step s1-show-indexes: SELECT run_command_on_workers('SELECT COUNT(*) FROM pg_indexes WHERE tablename LIKE ''update_hash%''');
+run_command_on_workers
+
+(localhost,57637,t,2)
+(localhost,57638,t,2)
+restore_isolation_tester_func
+
+
+
+starting permutation: s1-initialize s1-ddl-create-index s1-begin s2-begin s1-ddl-drop-index s2-update-cte s1-commit s2-commit s1-select-count s1-show-indexes
+create_distributed_table
+
+
+step s1-initialize: COPY update_hash FROM PROGRAM 'echo 0, a && echo 1, b && echo 2, c && echo 3, d && echo 4, e' WITH CSV;
+step s1-ddl-create-index: CREATE INDEX update_hash_index ON update_hash(id);
+step s1-begin: BEGIN;
+step s2-begin: BEGIN;
+step s1-ddl-drop-index: DROP INDEX update_hash_index;
+step s2-update-cte: WITH cte AS (UPDATE update_hash SET data = 'l' WHERE id = 4 RETURNING *) SELECT * FROM cte WHERE id = 4; <waiting ...>
+step s1-commit: COMMIT;
+step s2-update-cte: <... completed>
+id             data
+
+4              l
+step s2-commit: COMMIT;
+step s1-select-count: SELECT COUNT(*) FROM update_hash;
+count
+
+5
+step s1-show-indexes: SELECT run_command_on_workers('SELECT COUNT(*) FROM pg_indexes WHERE tablename LIKE ''update_hash%''');
+run_command_on_workers
+
+(localhost,57637,t,0)
+(localhost,57638,t,0)
+restore_isolation_tester_func
+
+
+
+starting permutation: s1-initialize s1-begin s2-begin s1-ddl-add-column s2-update-cte s1-commit s2-commit s1-select-count s1-show-columns
+create_distributed_table
+
+
+step s1-initialize: COPY update_hash FROM PROGRAM 'echo 0, a && echo 1, b && echo 2, c && echo 3, d && echo 4, e' WITH CSV;
+step s1-begin: BEGIN;
+step s2-begin: BEGIN;
+step s1-ddl-add-column: ALTER TABLE update_hash ADD new_column int DEFAULT 0;
+step s2-update-cte: WITH cte AS (UPDATE update_hash SET data = 'l' WHERE id = 4 RETURNING *) SELECT * FROM cte WHERE id = 4; <waiting ...>
+step s1-commit: COMMIT;
+step s2-update-cte: <... completed>
+id             data           new_column
+
+4              l              0
+step s2-commit: COMMIT;
+step s1-select-count: SELECT COUNT(*) FROM update_hash;
+count
+
+5
+step s1-show-columns: SELECT run_command_on_workers('SELECT column_name FROM information_schema.columns WHERE table_name LIKE ''update_hash%'' AND column_name = ''new_column'' ORDER BY 1 LIMIT 1');
+run_command_on_workers
+
+(localhost,57637,t,new_column)
+(localhost,57638,t,new_column)
+restore_isolation_tester_func
+
+
+
+starting permutation: s1-initialize s1-ddl-add-column s1-begin s2-begin s1-ddl-drop-column s2-update-cte s1-commit s2-commit s1-select-count s1-show-columns
+create_distributed_table
+
+
+step s1-initialize: COPY update_hash FROM PROGRAM 'echo 0, a && echo 1, b && echo 2, c && echo 3, d && echo 4, e' WITH CSV;
+step s1-ddl-add-column: ALTER TABLE update_hash ADD new_column int DEFAULT 0;
+step s1-begin: BEGIN;
+step s2-begin: BEGIN;
+step s1-ddl-drop-column: ALTER TABLE update_hash DROP new_column;
+step s2-update-cte: WITH cte AS (UPDATE update_hash SET data = 'l' WHERE id = 4 RETURNING *) SELECT * FROM cte WHERE id = 4; <waiting ...>
+step s1-commit: COMMIT;
+step s2-update-cte: <... completed>
+id             data
+
+4              l
+step s2-commit: COMMIT;
+step s1-select-count: SELECT COUNT(*) FROM update_hash;
+count
+
+5
+step s1-show-columns: SELECT run_command_on_workers('SELECT column_name FROM information_schema.columns WHERE table_name LIKE ''update_hash%'' AND column_name = ''new_column'' ORDER BY 1 LIMIT 1');
+run_command_on_workers
+
+(localhost,57637,t,"")
+(localhost,57638,t,"")
+restore_isolation_tester_func
+
+
+
+starting permutation: s1-initialize s1-begin s2-begin s1-ddl-rename-column s2-update-cte s1-commit s2-commit s1-select-count s1-show-columns
+create_distributed_table
+
+
+step s1-initialize: COPY update_hash FROM PROGRAM 'echo 0, a && echo 1, b && echo 2, c && echo 3, d && echo 4, e' WITH CSV;
+step s1-begin: BEGIN;
+step s2-begin: BEGIN;
+step s1-ddl-rename-column: ALTER TABLE update_hash RENAME data TO new_column;
+step s2-update-cte: WITH cte AS (UPDATE update_hash SET data = 'l' WHERE id = 4 RETURNING *) SELECT * FROM cte WHERE id = 4; <waiting ...>
+step s1-commit: COMMIT;
+step s2-update-cte: <... completed>
+error in steps s1-commit s2-update-cte: ERROR:  column "data" of relation "update_hash" does not exist
+step s2-commit: COMMIT;
+step s1-select-count: SELECT COUNT(*) FROM update_hash;
+count
+
+5
+step s1-show-columns: SELECT run_command_on_workers('SELECT column_name FROM information_schema.columns WHERE table_name LIKE ''update_hash%'' AND column_name = ''new_column'' ORDER BY 1 LIMIT 1');
+run_command_on_workers
+
+(localhost,57637,t,new_column)
+(localhost,57638,t,new_column)
+restore_isolation_tester_func
+
+
+
+starting permutation: s1-initialize s1-begin s2-begin s1-table-size s2-update-cte s1-commit s2-commit s1-select-count
+create_distributed_table
+
+
+step s1-initialize: COPY update_hash FROM PROGRAM 'echo 0, a && echo 1, b && echo 2, c && echo 3, d && echo 4, e' WITH CSV;
+step s1-begin: BEGIN;
+step s2-begin: BEGIN;
+step s1-table-size: SELECT citus_total_relation_size('update_hash');
+citus_total_relation_size
+
+57344
+step s2-update-cte: WITH cte AS (UPDATE update_hash SET data = 'l' WHERE id = 4 RETURNING *) SELECT * FROM cte WHERE id = 4;
+id             data
+
+4              l
+step s1-commit: COMMIT;
+step s2-commit: COMMIT;
+step s1-select-count: SELECT COUNT(*) FROM update_hash;
+count
+
+5
+restore_isolation_tester_func
+
+
+
+starting permutation: s1-initialize s1-begin s2-begin s1-master-modify-multiple-shards s2-update-cte s1-commit s2-commit s1-select-count
+create_distributed_table
+
+
+step s1-initialize: COPY update_hash FROM PROGRAM 'echo 0, a && echo 1, b && echo 2, c && echo 3, d && echo 4, e' WITH CSV;
+step s1-begin: BEGIN;
+step s2-begin: BEGIN;
+step s1-master-modify-multiple-shards: DELETE FROM update_hash;
+step s2-update-cte: WITH cte AS (UPDATE update_hash SET data = 'l' WHERE id = 4 RETURNING *) SELECT * FROM cte WHERE id = 4; <waiting ...>
+step s1-commit: COMMIT;
+step s2-update-cte: <... completed>
+id             data
+
+step s2-commit: COMMIT;
+step s1-select-count: SELECT COUNT(*) FROM update_hash;
+count
+
+0
+restore_isolation_tester_func
+
+
+
+starting permutation: s1-drop s1-create-non-distributed-table s1-initialize s1-begin s2-begin s1-distribute-table s2-update-cte s1-commit s2-commit s1-select-count
+create_distributed_table
+
+
+step s1-drop: DROP TABLE update_hash;
+step s1-create-non-distributed-table: CREATE TABLE update_hash(id integer, data text); COPY update_hash FROM PROGRAM 'echo 0, a && echo 1, b && echo 2, c && echo 3, d && echo 4, e' WITH CSV;
+step s1-initialize: COPY update_hash FROM PROGRAM 'echo 0, a && echo 1, b && echo 2, c && echo 3, d && echo 4, e' WITH CSV;
+step s1-begin: BEGIN;
+step s2-begin: BEGIN;
+step s1-distribute-table: SELECT create_distributed_table('update_hash', 'id');
+create_distributed_table
+
+
+step s2-update-cte: WITH cte AS (UPDATE update_hash SET data = 'l' WHERE id = 4 RETURNING *) SELECT * FROM cte WHERE id = 4; <waiting ...>
+step s1-commit: COMMIT;
+step s2-update-cte: <... completed>
+id             data
+
+4              l
+4              l
+step s2-commit: COMMIT;
+step s1-select-count: SELECT COUNT(*) FROM update_hash;
+count
+
+10
+restore_isolation_tester_func
+
+

src/test/regress/expected/multi_router_planner.out

@@ -314,6 +314,44 @@ DETAIL:  distribution column value: 1
  41
 (5 rows)
 
+-- SELECT FOR UPDATE is supported if not involving reference table
+BEGIN;
+WITH first_author AS (
+    SELECT articles_hash.id, auref.name FROM articles_hash, authors_reference auref
+    WHERE author_id = 2 AND auref.id = author_id
+    FOR UPDATE
+)
+UPDATE articles_hash SET title = first_author.name
+FROM first_author WHERE articles_hash.author_id = 2 AND articles_hash.id = first_author.id;
+DEBUG:  Router planner doesn't support SELECT FOR UPDATE in common table expressions involving reference tables.
+DEBUG:  generating subplan XXX_1 for CTE first_author: SELECT articles_hash.id, auref.name FROM public.articles_hash, public.authors_reference auref WHERE ((articles_hash.author_id OPERATOR(pg_catalog.=) 2) AND (auref.id OPERATOR(pg_catalog.=) articles_hash.author_id)) FOR UPDATE OF articles_hash FOR UPDATE OF auref
+DEBUG:  Creating router plan
+DEBUG:  Plan is router executable
+DETAIL:  distribution column value: 2
+DEBUG:  Plan XXX query after replacing subqueries and CTEs: UPDATE public.articles_hash SET title = first_author.name FROM (SELECT intermediate_result.id, intermediate_result.name FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(id bigint, name character varying(20))) first_author WHERE ((articles_hash.author_id OPERATOR(pg_catalog.=) 2) AND (articles_hash.id OPERATOR(pg_catalog.=) first_author.id))
+DEBUG:  Creating router plan
+DEBUG:  Plan is router executable
+DETAIL:  distribution column value: 2
+WITH first_author AS (
+    SELECT id, word_count FROM articles_hash WHERE author_id = 2
+    FOR UPDATE
+)
+UPDATE articles_hash SET title = first_author.word_count::text
+FROM first_author WHERE articles_hash.author_id = 2 AND articles_hash.id = first_author.id;
+DEBUG:  Creating router plan
+DEBUG:  Plan is router executable
+DETAIL:  distribution column value: 2
+-- Without FOR UPDATE this is router plannable
+WITH first_author AS (
+    SELECT articles_hash.id, auref.name FROM articles_hash, authors_reference auref
+    WHERE author_id = 2 AND auref.id = author_id
+)
+UPDATE articles_hash SET title = first_author.name
+FROM first_author WHERE articles_hash.author_id = 2 AND articles_hash.id = first_author.id;
+DEBUG:  Creating router plan
+DEBUG:  Plan is router executable
+DETAIL:  distribution column value: 2
+ROLLBACK;
 -- queries with CTEs are supported even if CTE is not referenced inside query
 WITH first_author AS ( SELECT id FROM articles_hash WHERE author_id = 1)
 SELECT title FROM articles_hash WHERE author_id = 1;
@@ -476,7 +514,7 @@ WITH new_article AS (
     INSERT INTO articles_hash VALUES (1,  1, 'arsenous', 9) RETURNING *
 )
 SELECT * FROM new_article;
-DEBUG:  data-modifying statements are not supported in the WITH clauses of distributed queries
+DEBUG:  only SELECT, UPDATE, or DELETE common table expressions may be router planned
 DEBUG:  generating subplan XXX_1 for CTE new_article: INSERT INTO public.articles_hash (id, author_id, title, word_count) VALUES (1, 1, 'arsenous'::character varying, 9) RETURNING id, author_id, title, word_count
 DEBUG:  Creating router plan
 DEBUG:  Plan is router executable
@@ -493,7 +531,7 @@ WITH update_article AS (
     UPDATE articles_hash SET word_count = 10 WHERE id = 1 AND word_count = 9 RETURNING *
 )
 SELECT * FROM update_article;
-DEBUG:  data-modifying statements are not supported in the WITH clauses of distributed queries
+DEBUG:  Router planner cannot handle multi-shard select queries
 DEBUG:  generating subplan XXX_1 for CTE update_article: UPDATE public.articles_hash SET word_count = 10 WHERE ((id OPERATOR(pg_catalog.=) 1) AND (word_count OPERATOR(pg_catalog.=) 9)) RETURNING id, author_id, title, word_count
 DEBUG:  Creating router plan
 DEBUG:  Plan is router executable
@@ -509,7 +547,7 @@ WITH delete_article AS (
     DELETE FROM articles_hash WHERE id = 1 AND word_count = 10 RETURNING *
 )
 SELECT * FROM delete_article;
-DEBUG:  data-modifying statements are not supported in the WITH clauses of distributed queries
+DEBUG:  Router planner cannot handle multi-shard select queries
 DEBUG:  generating subplan XXX_1 for CTE delete_article: DELETE FROM public.articles_hash WHERE ((id OPERATOR(pg_catalog.=) 1) AND (word_count OPERATOR(pg_catalog.=) 10)) RETURNING id, author_id, title, word_count
 DEBUG:  Creating router plan
 DEBUG:  Plan is router executable
@@ -2241,7 +2279,8 @@ DEBUG:  Router planner cannot handle multi-shard select queries
 PREPARE author_1_articles as
 	SELECT *
 	FROM articles_hash
-	WHERE author_id = 1;
+	WHERE author_id = 1
+	ORDER BY 1;
 EXECUTE author_1_articles;
 DEBUG:  Creating router plan
 DEBUG:  Plan is router executable
@@ -2259,7 +2298,8 @@ DETAIL:  distribution column value: 1
 PREPARE author_articles(int) as
 	SELECT *
 	FROM articles_hash
-	WHERE author_id = $1;
+	WHERE author_id = $1
+	ORDER BY 1;
 EXECUTE author_articles(1);
 DEBUG:  Creating router plan
 DEBUG:  Plan is router executable
@@ -2310,7 +2350,7 @@ BEGIN
 
 END;
 $$ LANGUAGE plpgsql;
-SELECT * FROM author_articles_id_word_count();
+SELECT * FROM author_articles_id_word_count() ORDER BY 1;
 DEBUG:  Creating router plan
 CONTEXT:  SQL statement "SELECT ah.id, ah.word_count
 		FROM articles_hash ah
@@ -2369,7 +2409,8 @@ SELECT * FROM mv_articles_hash_data ORDER BY 1, 2, 3, 4;
 SET citus.task_executor_type to 'task-tracker';
 SELECT id
 	FROM articles_hash
-	WHERE author_id = 1;
+	WHERE author_id = 1
+	ORDER BY 1;
 DEBUG:  Creating router plan
 DEBUG:  Plan is router executable
 DETAIL:  distribution column value: 1
@@ -2390,7 +2431,8 @@ DETAIL:  distribution column value: 1
 -- verify insert is successful (not router plannable and executable)
 SELECT id
 	FROM articles_hash
-	WHERE author_id = 1;
+	WHERE author_id = 1
+	ORDER BY 1;
 DEBUG:  Creating router plan
 DEBUG:  Plan is router executable
 DETAIL:  distribution column value: 1

src/test/regress/expected/multi_router_planner_fast_path.out

@@ -319,7 +319,7 @@ WITH update_article AS (
     UPDATE articles_hash SET word_count = 10 WHERE id = 1 AND word_count = 9 RETURNING *
 )
 SELECT * FROM update_article;
-DEBUG:  data-modifying statements are not supported in the WITH clauses of distributed queries
+DEBUG:  Router planner cannot handle multi-shard select queries
 DEBUG:  generating subplan XXX_1 for CTE update_article: UPDATE fast_path_router_select.articles_hash SET word_count = 10 WHERE ((id OPERATOR(pg_catalog.=) 1) AND (word_count OPERATOR(pg_catalog.=) 9)) RETURNING id, author_id, title, word_count
 DEBUG:  Creating router plan
 DEBUG:  Plan is router executable
@@ -334,7 +334,7 @@ WITH delete_article AS (
     DELETE FROM articles_hash WHERE id = 1 AND word_count = 10 RETURNING *
 )
 SELECT * FROM delete_article;
-DEBUG:  data-modifying statements are not supported in the WITH clauses of distributed queries
+DEBUG:  Router planner cannot handle multi-shard select queries
 DEBUG:  generating subplan XXX_1 for CTE delete_article: DELETE FROM fast_path_router_select.articles_hash WHERE ((id OPERATOR(pg_catalog.=) 1) AND (word_count OPERATOR(pg_catalog.=) 10)) RETURNING id, author_id, title, word_count
 DEBUG:  Creating router plan
 DEBUG:  Plan is router executable

src/test/regress/expected/relation_access_tracking.out

@@ -44,7 +44,7 @@ BEGIN
 	 END IF;
 END;
 $$ LANGUAGE 'plpgsql' IMMUTABLE;
-CREATE VIEW relation_acesses AS
+CREATE VIEW relation_accesses AS
 	SELECT table_name,
 			relation_access_mode_to_text(table_name, relation_select_access_mode(table_name::regclass)) as select_access,
 			relation_access_mode_to_text(table_name, relation_dml_access_mode(table_name::regclass)) as dml_access,
@@ -109,21 +109,21 @@ BEGIN;
 
 (1 row)
 
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_7')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_7')  ORDER BY 1;
  table_name |     select_access     |      dml_access       |   ddl_access
 ---------------------------------------------------------------------
  table_7    | not_parallel_accessed | not_parallel_accessed | parallel_access
 (1 row)
 
 COMMIT;
--- outisde the transaction blocks, the function always returns zero
+-- outside the transaction blocks, the function always returns zero
 SELECT count(*) FROM table_1;
  count
 ---------------------------------------------------------------------
    101
 (1 row)
 
-SELECT * FROM relation_acesses WHERE table_name = 'table_1';
+SELECT * FROM relation_accesses WHERE table_name = 'table_1';
  table_name |     select_access     |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  table_1    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
@@ -132,7 +132,7 @@ SELECT * FROM relation_acesses WHERE table_name = 'table_1';
 -- a very simple test that first checks sequential
 -- and parallel SELECTs,DMLs, and DDLs
 BEGIN;
-	SELECT * FROM relation_acesses WHERE table_name = 'table_1';
+	SELECT * FROM relation_accesses WHERE table_name = 'table_1';
  table_name |     select_access     |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  table_1    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
@@ -144,7 +144,7 @@ BEGIN;
      1
 (1 row)
 
-	SELECT * FROM relation_acesses WHERE table_name = 'table_1';
+	SELECT * FROM relation_accesses WHERE table_name = 'table_1';
  table_name |     select_access     |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  table_1    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
@@ -156,21 +156,21 @@ BEGIN;
      2
 (1 row)
 
-	SELECT * FROM relation_acesses WHERE table_name = 'table_1';
+	SELECT * FROM relation_accesses WHERE table_name = 'table_1';
  table_name |  select_access  |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  table_1    | parallel_access | not_parallel_accessed | not_parallel_accessed
 (1 row)
 
 	INSERT INTO table_1 VALUES (1,1);
-	SELECT * FROM relation_acesses WHERE table_name = 'table_1';
+	SELECT * FROM relation_accesses WHERE table_name = 'table_1';
  table_name |  select_access  |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  table_1    | parallel_access | not_parallel_accessed | not_parallel_accessed
 (1 row)
 
 	INSERT INTO table_1 VALUES (1,1), (2,2);
-	SELECT * FROM relation_acesses WHERE table_name = 'table_1';
+	SELECT * FROM relation_accesses WHERE table_name = 'table_1';
  table_name |  select_access  |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  table_1    | parallel_access | not_parallel_accessed | not_parallel_accessed
@@ -178,7 +178,7 @@ BEGIN;
 
 	ALTER TABLE table_1 ADD COLUMN test_col INT;
 	-- now see that the other tables are not accessed at all
-	SELECT * FROM relation_acesses WHERE table_name = 'table_1';
+	SELECT * FROM relation_accesses WHERE table_name = 'table_1';
  table_name |  select_access  |      dml_access       |   ddl_access
 ---------------------------------------------------------------------
  table_1    | parallel_access | not_parallel_accessed | parallel_access
@@ -194,7 +194,7 @@ BEGIN;
      1
 (1 row)
 
-	SELECT * FROM relation_acesses WHERE table_name = 'table_1';
+	SELECT * FROM relation_accesses WHERE table_name = 'table_1';
  table_name |     select_access     |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  table_1    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
@@ -206,21 +206,21 @@ BEGIN;
      1
 (1 row)
 
-	SELECT * FROM relation_acesses WHERE table_name = 'table_1';
+	SELECT * FROM relation_accesses WHERE table_name = 'table_1';
  table_name |     select_access     |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  table_1    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
 (1 row)
 
 	INSERT INTO table_1 VALUES (1,1);
-	SELECT * FROM relation_acesses WHERE table_name = 'table_1';
+	SELECT * FROM relation_accesses WHERE table_name = 'table_1';
  table_name |     select_access     |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  table_1    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
 (1 row)
 
 	INSERT INTO table_1 VALUES (2,2);
-	SELECT * FROM relation_acesses WHERE table_name = 'table_1';
+	SELECT * FROM relation_accesses WHERE table_name = 'table_1';
  table_name |     select_access     |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  table_1    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
@@ -230,7 +230,7 @@ ROLLBACK;
 -- a sample DDL example
 BEGIN;
 	ALTER TABLE table_1 ADD CONSTRAINT table_1_u UNIQUE (key);
-	SELECT * FROM relation_acesses WHERE table_name = 'table_1';
+	SELECT * FROM relation_accesses WHERE table_name = 'table_1';
  table_name |     select_access     |      dml_access       |   ddl_access
 ---------------------------------------------------------------------
  table_1    | not_parallel_accessed | not_parallel_accessed | parallel_access
@@ -252,7 +252,7 @@ BEGIN;
      1
 (1 row)
 
-		SELECT * FROM relation_acesses WHERE table_name LIKE 'table_%' ORDER BY 1;
+		SELECT * FROM relation_accesses WHERE table_name LIKE 'table_%' ORDER BY 1;
  table_name |     select_access     |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  table_1    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
@@ -278,7 +278,7 @@ BEGIN;
    101
 (1 row)
 
-		SELECT * FROM relation_acesses WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
+		SELECT * FROM relation_accesses WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
  table_name |  select_access  |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  table_1    | parallel_access | not_parallel_accessed | not_parallel_accessed
@@ -301,7 +301,7 @@ BEGIN;
    101
 (1 row)
 
-		SELECT * FROM relation_acesses WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
+		SELECT * FROM relation_accesses WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
  table_name |     select_access     |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  table_1    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
@@ -328,7 +328,7 @@ BEGIN;
    101
 (1 row)
 
-	SELECT * FROM relation_acesses WHERE table_name LIKE 'table_%' ORDER BY 1;
+	SELECT * FROM relation_accesses WHERE table_name LIKE 'table_%' ORDER BY 1;
  table_name |     select_access     |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  table_1    | parallel_access       | not_parallel_accessed | not_parallel_accessed
@@ -346,7 +346,7 @@ ROLLBACK;
 -- access for all the shards accessed. But, sequential mode is OK
 BEGIN;
 	UPDATE table_1 SET value = 15;
-	SELECT * FROM relation_acesses WHERE table_name = 'table_1';
+	SELECT * FROM relation_accesses WHERE table_name = 'table_1';
  table_name |  select_access  |   dml_access    |      ddl_access
 ---------------------------------------------------------------------
  table_1    | parallel_access | parallel_access | not_parallel_accessed
@@ -354,7 +354,7 @@ BEGIN;
 
 	SET LOCAL citus.multi_shard_modify_mode = 'sequential';
 	UPDATE table_2 SET value = 15;
-	SELECT * FROM relation_acesses WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
+	SELECT * FROM relation_accesses WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
  table_name |     select_access     |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  table_1    | parallel_access       | parallel_access       | not_parallel_accessed
@@ -367,7 +367,7 @@ BEGIN;
 	UPDATE
 		table_1 SET value = 15
 	WHERE key IN (SELECT key FROM table_2 JOIN table_3 USING (key) WHERE table_2.value = 15);
-	SELECT * FROM relation_acesses WHERE table_name IN ('table_1', 'table_2', 'table_3')  ORDER BY 1;
+	SELECT * FROM relation_accesses WHERE table_name IN ('table_1', 'table_2', 'table_3')  ORDER BY 1;
  table_name |  select_access  |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  table_1    | parallel_access | parallel_access       | not_parallel_accessed
@@ -379,7 +379,7 @@ ROLLBACK;
 -- INSERT .. SELECT pushdown
 BEGIN;
 	INSERT INTO table_2 SELECT * FROM table_1;
-	SELECT * FROM relation_acesses WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
+	SELECT * FROM relation_accesses WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
  table_name |     select_access     |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  table_1    | parallel_access       | not_parallel_accessed | not_parallel_accessed
@@ -391,7 +391,7 @@ ROLLBACK;
 BEGIN;
 	SET LOCAL citus.multi_shard_modify_mode = 'sequential';
 	INSERT INTO table_2 SELECT * FROM table_1;
-	SELECT * FROM relation_acesses WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
+	SELECT * FROM relation_accesses WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
  table_name |     select_access     |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  table_1    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
@@ -403,7 +403,7 @@ ROLLBACK;
 BEGIN;
     -- We use offset 1 to make sure the result needs to be pulled to the coordinator, offset 0 would be optimized away
 	INSERT INTO table_2 SELECT * FROM table_1 OFFSET 1;
-	SELECT * FROM relation_acesses WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
+	SELECT * FROM relation_accesses WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
  table_name |     select_access     |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  table_1    | parallel_access       | not_parallel_accessed | not_parallel_accessed
@@ -430,7 +430,7 @@ BEGIN;
    101
 (1 row)
 
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
  table_name |  select_access  |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  table_1    | parallel_access | not_parallel_accessed | not_parallel_accessed
@@ -453,7 +453,7 @@ BEGIN;
 			table_1.key = table_2.key
 			OFFSET 0
 	) as foo;
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1', 'table_2', 'table_3')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_1', 'table_2', 'table_3')  ORDER BY 1;
  table_name |     select_access     |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  table_1    | parallel_access       | not_parallel_accessed | not_parallel_accessed
@@ -479,7 +479,7 @@ BEGIN;
 		AND table_1.key = 1
 			OFFSET 0
 	) as foo;
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1', 'table_2', 'table_3')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_1', 'table_2', 'table_3')  ORDER BY 1;
  table_name |     select_access     |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  table_1    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
@@ -505,7 +505,7 @@ BEGIN;
 				OFFSET 0
 		) as foo
 	) AND value IN (SELECT key FROM table_4);
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1', 'table_2', 'table_3', 'table_4')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_1', 'table_2', 'table_3', 'table_4')  ORDER BY 1;
  table_name |  select_access  |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  table_1    | parallel_access | not_parallel_accessed | not_parallel_accessed
@@ -521,7 +521,7 @@ BEGIN;
 1	1
 2	2
 3	3
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_1')  ORDER BY 1;
  table_name |  select_access  |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  table_1    | parallel_access | not_parallel_accessed | not_parallel_accessed
@@ -531,7 +531,7 @@ ROLLBACK;
 -- copy in
 BEGIN;
 	COPY table_1 FROM STDIN WITH CSV;
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_1')  ORDER BY 1;
  table_name |     select_access     |   dml_access    |      ddl_access
 ---------------------------------------------------------------------
  table_1    | not_parallel_accessed | parallel_access | not_parallel_accessed
@@ -541,7 +541,7 @@ ROLLBACK;
 -- copy in single shard
 BEGIN;
 	COPY table_1 FROM STDIN WITH CSV;
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_1')  ORDER BY 1;
  table_name |     select_access     |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  table_1    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
@@ -556,21 +556,21 @@ BEGIN;
    101
 (1 row)
 
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_6');
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_6');
  table_name |     select_access      |  dml_access  |  ddl_access
 ---------------------------------------------------------------------
  table_6    | reference_table_access | not_accessed | not_accessed
 (1 row)
 
 	UPDATE table_6 SET value = 15;
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_6');
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_6');
  table_name |     select_access      |       dml_access       |  ddl_access
 ---------------------------------------------------------------------
  table_6    | reference_table_access | reference_table_access | not_accessed
 (1 row)
 
 	ALTER TABLE table_6 ADD COLUMN x INT;
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_6');
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_6');
  table_name |     select_access      |       dml_access       |       ddl_access
 ---------------------------------------------------------------------
  table_6    | reference_table_access | reference_table_access | reference_table_access
@@ -585,7 +585,7 @@ BEGIN;
    101
 (1 row)
 
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_6', 'table_1') ORDER BY 1,2;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_6', 'table_1') ORDER BY 1,2;
  table_name |  select_access  |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  table_1    | parallel_access | not_parallel_accessed | not_parallel_accessed
@@ -596,7 +596,7 @@ ROLLBACK;
 -- TRUNCATE should be DDL
 BEGIN;
 	TRUNCATE table_1;
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_1')  ORDER BY 1;
  table_name |     select_access     |      dml_access       |   ddl_access
 ---------------------------------------------------------------------
  table_1    | not_parallel_accessed | not_parallel_accessed | parallel_access
@@ -607,7 +607,7 @@ ROLLBACK;
 BEGIN;
 	SET LOCAL citus.multi_shard_modify_mode = 'sequential';
 	TRUNCATE table_1;
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_1')  ORDER BY 1;
  table_name |     select_access     |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  table_1    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
@@ -617,7 +617,7 @@ ROLLBACK;
 -- TRUNCATE on a reference table should be sequential
 BEGIN;
 	TRUNCATE table_6;
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_6')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_6')  ORDER BY 1;
  table_name | select_access |  dml_access  |       ddl_access
 ---------------------------------------------------------------------
  table_6    | not_accessed  | not_accessed | reference_table_access
@@ -628,7 +628,7 @@ ROLLBACK;
 ALTER TABLE table_1 ADD CONSTRAINT table_1_u UNIQUE (key);
 BEGIN;
 	ALTER TABLE table_2 ADD CONSTRAINT table_2_u FOREIGN KEY (key) REFERENCES table_1(key);
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
  table_name |     select_access     |      dml_access       |   ddl_access
 ---------------------------------------------------------------------
  table_1    | not_parallel_accessed | not_parallel_accessed | parallel_access
@@ -641,7 +641,7 @@ ROLLBACK;
 BEGIN;
 	SET LOCAL citus.multi_shard_modify_mode = 'sequential';
 	ALTER TABLE table_2 ADD CONSTRAINT table_2_u FOREIGN KEY (key) REFERENCES table_1(key);
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
  table_name |     select_access     |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  table_1    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
@@ -659,7 +659,7 @@ SELECT create_distributed_table('partitioning_test', 'id');
 -- Adding partition tables via CREATE TABLE should have DDL access the partitioned table as well
 BEGIN;
 	CREATE TABLE partitioning_test_2009 PARTITION OF partitioning_test FOR VALUES FROM ('2009-01-01') TO ('2010-01-01');
-	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009')  ORDER BY 1;
        table_name       |     select_access     |      dml_access       |   ddl_access
 ---------------------------------------------------------------------
  partitioning_test      | not_parallel_accessed | not_parallel_accessed | parallel_access
@@ -671,7 +671,7 @@ ROLLBACK;
 CREATE TABLE partitioning_test_2009 AS SELECT * FROM partitioning_test;
 BEGIN;
 	ALTER TABLE partitioning_test ATTACH PARTITION partitioning_test_2009 FOR VALUES FROM ('2009-01-01') TO ('2010-01-01');
-	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009')  ORDER BY 1;
        table_name       |     select_access     |      dml_access       |   ddl_access
 ---------------------------------------------------------------------
  partitioning_test      | not_parallel_accessed | not_parallel_accessed | parallel_access
@@ -689,7 +689,7 @@ SELECT create_distributed_table('partitioning_test_2010', 'id');
 
 BEGIN;
 	ALTER TABLE partitioning_test ATTACH PARTITION partitioning_test_2010 FOR VALUES FROM ('2010-01-01') TO ('2011-01-01');
-	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2010')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2010')  ORDER BY 1;
        table_name       |     select_access     |      dml_access       |   ddl_access
 ---------------------------------------------------------------------
  partitioning_test      | not_parallel_accessed | not_parallel_accessed | parallel_access
@@ -705,7 +705,7 @@ BEGIN;
      0
 (1 row)
 
-	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
        table_name       |  select_access  |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  partitioning_test      | parallel_access | not_parallel_accessed | not_parallel_accessed
@@ -723,7 +723,7 @@ BEGIN;
      0
 (1 row)
 
-	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
        table_name       |     select_access     |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  partitioning_test      | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
@@ -735,7 +735,7 @@ COMMIT;
 -- updating partitioned table marks all of its partitions
 BEGIN;
 	UPDATE partitioning_test SET time = now();
-	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
        table_name       |  select_access  |   dml_access    |      ddl_access
 ---------------------------------------------------------------------
  partitioning_test      | parallel_access | parallel_access | not_parallel_accessed
@@ -748,7 +748,7 @@ COMMIT;
 BEGIN;
 	SET LOCAL citus.multi_shard_modify_mode TO 'sequential';
 	UPDATE partitioning_test SET time = now();
-	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
        table_name       |     select_access     |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  partitioning_test      | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
@@ -760,7 +760,7 @@ COMMIT;
 -- DDLs on partitioned table marks all of its partitions
 BEGIN;
 	ALTER TABLE partitioning_test ADD COLUMN X INT;
-	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
        table_name       |     select_access     |      dml_access       |   ddl_access
 ---------------------------------------------------------------------
  partitioning_test      | not_parallel_accessed | not_parallel_accessed | parallel_access
@@ -773,7 +773,7 @@ ROLLBACK;
 BEGIN;
 	SET LOCAL citus.multi_shard_modify_mode TO 'sequential';
 	ALTER TABLE partitioning_test ADD COLUMN X INT;
-	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
        table_name       |     select_access     |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  partitioning_test      | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
@@ -790,7 +790,7 @@ BEGIN;
      0
 (1 row)
 
-	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
        table_name       |     select_access     |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  partitioning_test      | parallel_access       | not_parallel_accessed | not_parallel_accessed
@@ -808,7 +808,7 @@ BEGIN;
      0
 (1 row)
 
-	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
        table_name       |     select_access     |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  partitioning_test      | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
@@ -820,7 +820,7 @@ COMMIT;
 -- updating from partition table marks its parent
 BEGIN;
 	UPDATE partitioning_test_2009 SET time = now();
-	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
        table_name       |     select_access     |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  partitioning_test      | parallel_access       | parallel_access       | not_parallel_accessed
@@ -833,7 +833,7 @@ COMMIT;
 BEGIN;
 	SET LOCAL citus.multi_shard_modify_mode TO 'sequential';
 	UPDATE partitioning_test_2009 SET time = now();
-	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
        table_name       |     select_access     |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  partitioning_test      | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
@@ -845,7 +845,7 @@ COMMIT;
 -- DDLs on partition table marks its parent
 BEGIN;
 	CREATE INDEX i1000000 ON partitioning_test_2009 (id);
-	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
        table_name       |     select_access     |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  partitioning_test      | not_parallel_accessed | not_parallel_accessed | parallel_access
@@ -858,7 +858,7 @@ ROLLBACK;
 BEGIN;
 	SET LOCAL citus.multi_shard_modify_mode TO 'sequential';
 	CREATE INDEX i1000000 ON partitioning_test_2009 (id);
-	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
        table_name       |     select_access     |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  partitioning_test      | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
@@ -872,7 +872,7 @@ ALTER TABLE table_2 ADD CONSTRAINT table_2_u FOREIGN KEY (key) REFERENCES table_
 BEGIN;
 	TRUNCATE table_1 CASCADE;
 NOTICE:  truncate cascades to table "table_2"
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
  table_name |     select_access     |      dml_access       |   ddl_access
 ---------------------------------------------------------------------
  table_1    | not_parallel_accessed | not_parallel_accessed | parallel_access
@@ -889,7 +889,7 @@ BEGIN;
    101
 (1 row)
 
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_1')  ORDER BY 1;
  table_name |  select_access  |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  table_1    | parallel_access | not_parallel_accessed | not_parallel_accessed
@@ -906,7 +906,7 @@ BEGIN;
    101
 (1 row)
 
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_1')  ORDER BY 1;
  table_name |     select_access     |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  table_1    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
@@ -924,7 +924,7 @@ BEGIN;
  1002 |  1002
 (3 rows)
 
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_1')  ORDER BY 1;
  table_name |     select_access     |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
  table_1    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
@@ -940,7 +940,7 @@ BEGIN;
    101
 (1 row)
 
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_1')  ORDER BY 1;
  table_name |  select_access  |   dml_access    |      ddl_access
 ---------------------------------------------------------------------
  table_1    | parallel_access | parallel_access | not_parallel_accessed
@@ -949,6 +949,7 @@ BEGIN;
 ROLLBACK;
 -- modifying CTEs should work fine with sequential mode
 BEGIN;
+	SET LOCAL citus.multi_shard_modify_mode = 'sequential';
 	WITH cte_1 AS (UPDATE table_1 SET value = 15 RETURNING *)
 	SELECT count(*) FROM cte_1 ORDER BY 1;
  count
@@ -956,10 +957,43 @@ BEGIN;
    101
 (1 row)
 
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_1')  ORDER BY 1;
- table_name |  select_access  |   dml_access    |      ddl_access
+ table_name |     select_access     |      dml_access       |      ddl_access
 ---------------------------------------------------------------------
- table_1    | parallel_access | parallel_access | not_parallel_accessed
+ table_1    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
+(1 row)
+
+ROLLBACK;
+-- router planned modifying CTEs should work fine with parallel mode
+BEGIN;
+	WITH cte_1 AS (UPDATE table_1 SET value = 15 WHERE key = 6 RETURNING *)
+	SELECT count(*) FROM cte_1 ORDER BY 1;
+ count
+---------------------------------------------------------------------
+     1
+(1 row)
+
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_1')  ORDER BY 1;
+ table_name |     select_access     |      dml_access       |      ddl_access
+---------------------------------------------------------------------
+ table_1    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
+(1 row)
+
+ROLLBACK;
+-- router planned modifying CTEs should work fine with sequential mode
+BEGIN;
+	SET LOCAL citus.multi_shard_modify_mode = 'sequential';
+	WITH cte_1 AS (UPDATE table_1 SET value = 15 WHERE key = 6 RETURNING *)
+	SELECT count(*) FROM cte_1 ORDER BY 1;
+ count
+---------------------------------------------------------------------
+     1
+(1 row)
+
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_1')  ORDER BY 1;
+ table_name |     select_access     |      dml_access       |      ddl_access
+---------------------------------------------------------------------
+ table_1    | not_parallel_accessed | not_parallel_accessed | not_parallel_accessed
 (1 row)
 
 ROLLBACK;
@@ -979,7 +1013,7 @@ HINT:  To remove the local data, run: SELECT truncate_local_data_after_distribut
 
 (1 row)
 
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_3')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_3')  ORDER BY 1;
  table_name |     select_access     |   dml_access    |   ddl_access
 ---------------------------------------------------------------------
  table_3    | not_parallel_accessed | parallel_access | parallel_access
@@ -994,7 +1028,7 @@ drop cascades to function access_tracking.relation_dml_access_mode(oid)
 drop cascades to function access_tracking.relation_ddl_access_mode(oid)
 drop cascades to function access_tracking.distributed_relation(text)
 drop cascades to function access_tracking.relation_access_mode_to_text(text,integer)
-drop cascades to view access_tracking.relation_acesses
+drop cascades to view access_tracking.relation_accesses
 drop cascades to table access_tracking.table_1
 drop cascades to table access_tracking.table_2
 drop cascades to table access_tracking.table_4

src/test/regress/expected/with_modifying.out

@@ -2,6 +2,7 @@
 SET citus.next_shard_id TO 1502000;
 CREATE SCHEMA with_modifying;
 SET search_path TO with_modifying, public;
+CREATE TABLE with_modifying.local_table (id int, val int);
 CREATE TABLE with_modifying.modify_table (id int, val int);
 SELECT create_distributed_table('modify_table', 'id');
  create_distributed_table
@@ -192,13 +193,13 @@ INSERT INTO modify_table (SELECT cte_1.user_id FROM cte_1 join cte_2 on cte_1.va
 -- between different executors
 CREATE FUNCTION raise_failed_execution_cte(query text) RETURNS void AS $$
 BEGIN
-        EXECUTE query;
+	EXECUTE query;
-        EXCEPTION WHEN OTHERS THEN
+	EXCEPTION WHEN OTHERS THEN
-        IF SQLERRM LIKE '%more than one row returned by a subquery used as an expression%' THEN
+	IF SQLERRM LIKE '%more than one row returned by a subquery used as an expression%' THEN
-                RAISE 'Task failed to execute';
+		RAISE 'Task failed to execute';
-        ELSIF SQLERRM LIKE '%could not receive query results%' THEN
+	ELSIF SQLERRM LIKE '%could not receive query results%' THEN
-          RAISE 'Task failed to execute';
+		RAISE 'Task failed to execute';
-        END IF;
+	END IF;
 END;
 $$LANGUAGE plpgsql;
 SET client_min_messages TO ERROR;
@@ -378,11 +379,11 @@ INSERT INTO summary_table SELECT id, COUNT(*) AS counter FROM raw_data GROUP BY
 INSERT INTO modify_table VALUES (21, 1), (22, 2), (23, 3);
 -- read ids from the same table
 WITH distinct_ids AS (
-  SELECT DISTINCT id FROM modify_table
+	SELECT DISTINCT id FROM modify_table
 ),
 update_data AS (
-  UPDATE modify_table SET val = 100 WHERE id > 10 AND
+	UPDATE modify_table SET val = 100 WHERE id > 10 AND
-  	id IN (SELECT * FROM distinct_ids) RETURNING *
+		id IN (SELECT * FROM distinct_ids) RETURNING *
 )
 SELECT count(*) FROM update_data;
  count
@@ -392,11 +393,11 @@ SELECT count(*) FROM update_data;
 
 -- read ids from a different table
 WITH distinct_ids AS (
-  SELECT DISTINCT id FROM summary_table
+	SELECT DISTINCT id FROM summary_table
 ),
 update_data AS (
-  UPDATE modify_table SET val = 100 WHERE id > 10 AND
+	UPDATE modify_table SET val = 100 WHERE id > 10 AND
-  	id IN (SELECT * FROM distinct_ids) RETURNING *
+		id IN (SELECT * FROM distinct_ids) RETURNING *
 )
 SELECT count(*) FROM update_data;
  count
@@ -667,6 +668,157 @@ SELECT * FROM raw_data ORDER BY val;
 ---------------------------------------------------------------------
 (0 rows)
 
+-- Test that local tables are barred
+UPDATE local_table lt SET val = mt.val
+FROM modify_table mt WHERE mt.id = lt.id;
+ERROR:  cannot plan modifications of local tables involving distributed tables
+-- Including inside CTEs
+WITH cte AS (
+	UPDATE local_table lt SET val = mt.val
+	FROM modify_table mt WHERE mt.id = lt.id
+	RETURNING lt.id, lt.val
+) SELECT * FROM cte JOIN modify_table mt ON mt.id = cte.id ORDER BY 1,2;
+ERROR:  cannot plan modifications of local tables involving distributed tables
+-- Make sure checks for volatile functions apply to CTEs too
+WITH cte AS (UPDATE modify_table SET val = random() WHERE id = 3 RETURNING *)
+SELECT * FROM cte JOIN modify_table mt ON mt.id = 3 AND mt.id = cte.id ORDER BY 1,2;
+ERROR:  functions used in UPDATE queries on distributed tables must not be VOLATILE
+-- Two queries from HammerDB:
+-- 1
+CREATE TABLE with_modifying.stock (s_i_id numeric(6,0) NOT NULL, s_w_id numeric(4,0) NOT NULL, s_quantity numeric(6,0), s_dist_01 character(24)) WITH (fillfactor='50');
+ALTER TABLE with_modifying.stock ADD CONSTRAINT stock_i1 PRIMARY KEY (s_i_id, s_w_id);
+SELECT create_distributed_table('stock', 's_w_id');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+INSERT INTO with_modifying.stock VALUES
+	(64833, 10, 3, 'test1'),
+	(64834, 10, 3, 'test2'),
+	(63867, 10, 3, 'test3');
+PREPARE su_after(INT[], SMALLINT[], SMALLINT[], NUMERIC(5,2)[], NUMERIC, NUMERIC, NUMERIC) AS
+	WITH stock_update AS (
+		UPDATE stock
+		SET s_quantity = ( CASE WHEN s_quantity < (item_stock.quantity + 10) THEN s_quantity + 91 ELSE s_quantity END) - item_stock.quantity
+		FROM UNNEST($1, $2, $3, $4) AS item_stock (item_id, supply_wid, quantity, price)
+		WHERE stock.s_i_id = item_stock.item_id
+			AND stock.s_w_id = item_stock.supply_wid
+			AND stock.s_w_id = ANY ($2)
+		RETURNING stock.s_dist_01 as s_dist, stock.s_quantity, ( item_stock.quantity + item_stock.price * ( 1 + $5 + $6 ) * ( 1 - $7) ) amount
+	)
+	SELECT array_agg ( s_dist ), array_agg ( s_quantity ), array_agg ( amount )
+	FROM stock_update;
+EXECUTE su_after('{64833,63857,13941,76514,35858,10004,88553,34483,91251,28144,51687,36407,54436,72873}', '{10,10,10,10,10,10,10,10,10,10,10,10,10,10}', '{8,2,2,6,7,4,6,1,1,5,6,7,6,2}', '{26.04,4.79,67.84,77.66,47.06,23.12,32.74,56.99,84.75,37.52,73.52,98.86,49.96,29.47}', 0.1800, 0.1100, 0.5000);
+          array_agg           | array_agg |    array_agg
+---------------------------------------------------------------------
+ {"test1                   "} | {86}      | {24.7958000000}
+(1 row)
+
+EXECUTE su_after('{64833,63857,13941,76514,35858,10004,88553,34483,91251,28144,51687,36407,54436,72873}', '{10,10,10,10,10,10,10,10,10,10,10,10,10,10}', '{8,2,2,6,7,4,6,1,1,5,6,7,6,2}', '{26.04,4.79,67.84,77.66,47.06,23.12,32.74,56.99,84.75,37.52,73.52,98.86,49.96,29.47}', 0.1800, 0.1100, 0.5000);
+          array_agg           | array_agg |    array_agg
+---------------------------------------------------------------------
+ {"test1                   "} | {78}      | {24.7958000000}
+(1 row)
+
+EXECUTE su_after('{64833,63857,13941,76514,35858,10004,88553,34483,91251,28144,51687,36407,54436,72873}', '{10,10,10,10,10,10,10,10,10,10,10,10,10,10}', '{8,2,2,6,7,4,6,1,1,5,6,7,6,2}', '{26.04,4.79,67.84,77.66,47.06,23.12,32.74,56.99,84.75,37.52,73.52,98.86,49.96,29.47}', 0.1800, 0.1100, 0.5000);
+          array_agg           | array_agg |    array_agg
+---------------------------------------------------------------------
+ {"test1                   "} | {70}      | {24.7958000000}
+(1 row)
+
+EXECUTE su_after('{64833,63857,13941,76514,35858,10004,88553,34483,91251,28144,51687,36407,54436,72873}', '{10,10,10,10,10,10,10,10,10,10,10,10,10,10}', '{8,2,2,6,7,4,6,1,1,5,6,7,6,2}', '{26.04,4.79,67.84,77.66,47.06,23.12,32.74,56.99,84.75,37.52,73.52,98.86,49.96,29.47}', 0.1800, 0.1100, 0.5000);
+          array_agg           | array_agg |    array_agg
+---------------------------------------------------------------------
+ {"test1                   "} | {62}      | {24.7958000000}
+(1 row)
+
+EXECUTE su_after('{64833,63857,13941,76514,35858,10004,88553,34483,91251,28144,51687,36407,54436,72873}', '{10,10,10,10,10,10,10,10,10,10,10,10,10,10}', '{8,2,2,6,7,4,6,1,1,5,6,7,6,2}', '{26.04,4.79,67.84,77.66,47.06,23.12,32.74,56.99,84.75,37.52,73.52,98.86,49.96,29.47}', 0.1800, 0.1100, 0.5000);
+          array_agg           | array_agg |    array_agg
+---------------------------------------------------------------------
+ {"test1                   "} | {54}      | {24.7958000000}
+(1 row)
+
+EXECUTE su_after('{64833,63857,13941,76514,35858,10004,88553,34483,91251,28144,51687,36407,54436,72873}', '{10,10,10,10,10,10,10,10,10,10,10,10,10,10}', '{8,2,2,6,7,4,6,1,1,5,6,7,6,2}', '{26.04,4.79,67.84,77.66,47.06,23.12,32.74,56.99,84.75,37.52,73.52,98.86,49.96,29.47}', 0.1800, 0.1100, 0.5000);
+          array_agg           | array_agg |    array_agg
+---------------------------------------------------------------------
+ {"test1                   "} | {46}      | {24.7958000000}
+(1 row)
+
+-- 2
+CREATE TABLE with_modifying.orders (o_id numeric NOT NULL, o_w_id numeric NOT NULL, o_d_id numeric NOT NULL, o_c_id numeric) WITH (fillfactor='50');
+CREATE UNIQUE INDEX orders_i2 ON with_modifying.orders USING btree (o_w_id, o_d_id, o_c_id, o_id) TABLESPACE pg_default;
+ALTER TABLE with_modifying.orders ADD CONSTRAINT orders_i1 PRIMARY KEY (o_w_id, o_d_id, o_id);
+CREATE TABLE with_modifying.order_line (ol_w_id numeric NOT NULL, ol_d_id numeric NOT NULL, ol_o_id numeric NOT NULL, ol_number numeric NOT NULL, ol_delivery_d timestamp without time zone, ol_amount numeric) WITH (fillfactor='50');
+ALTER TABLE with_modifying.order_line ADD CONSTRAINT order_line_i1 PRIMARY KEY (ol_w_id, ol_d_id, ol_o_id, ol_number);
+SELECT create_distributed_table('orders', 'o_w_id');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+SELECT create_distributed_table('order_line', 'ol_w_id');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+INSERT INTO orders VALUES (1, 1, 1, 1), (2, 2, 2, 2), (3, 3, 3, 3);
+INSERT INTO order_line VALUES (1, 1, 1, 10), (2, 2, 2, 20), (3, 3, 3, 30);
+PREPARE olu(int,int[],int[]) AS
+	WITH order_line_update AS (
+		UPDATE order_line
+		SET ol_delivery_d = current_timestamp
+		FROM UNNEST($2, $3) AS ids(o_id, d_id)
+		WHERE ol_o_id = ids.o_id
+			AND ol_d_id = ids.d_id
+			AND ol_w_id = $1
+		RETURNING ol_d_id, ol_o_id, ol_amount
+	)
+	SELECT array_agg(ol_d_id), array_agg(c_id), array_agg(sum_amount)
+	FROM (
+		SELECT ol_d_id,
+			(SELECT DISTINCT o_c_id FROM orders WHERE o_id = ol_o_id AND o_d_id = ol_d_id AND o_w_id = $1) AS c_id,
+			sum(ol_amount) AS sum_amount
+		FROM order_line_update
+		GROUP BY ol_d_id, ol_o_id
+	) AS inner_sum;
+EXECUTE olu(1,ARRAY[1,2],ARRAY[1,2]);
+ array_agg | array_agg | array_agg
+---------------------------------------------------------------------
+ {1}       | {1}       | {NULL}
+(1 row)
+
+EXECUTE olu(1,ARRAY[1,2],ARRAY[1,2]);
+ array_agg | array_agg | array_agg
+---------------------------------------------------------------------
+ {1}       | {1}       | {NULL}
+(1 row)
+
+EXECUTE olu(1,ARRAY[1,2],ARRAY[1,2]);
+ array_agg | array_agg | array_agg
+---------------------------------------------------------------------
+ {1}       | {1}       | {NULL}
+(1 row)
+
+EXECUTE olu(1,ARRAY[1,2],ARRAY[1,2]);
+ array_agg | array_agg | array_agg
+---------------------------------------------------------------------
+ {1}       | {1}       | {NULL}
+(1 row)
+
+EXECUTE olu(1,ARRAY[1,2],ARRAY[1,2]);
+ array_agg | array_agg | array_agg
+---------------------------------------------------------------------
+ {1}       | {1}       | {NULL}
+(1 row)
+
+EXECUTE olu(1,ARRAY[1,2],ARRAY[1,2]);
+ array_agg | array_agg | array_agg
+---------------------------------------------------------------------
+ {1}       | {1}       | {NULL}
+(1 row)
+
 -- Test with replication factor 2
 SET citus.shard_replication_factor to 2;
 DROP TABLE modify_table;
@@ -730,17 +882,88 @@ BEGIN;
 
 ROLLBACK;
 -- similarly, make sure that the intermediate result uses a seperate connection
- WITH first_query AS (INSERT INTO modify_table (id) VALUES (10001)),
+WITH first_query AS (INSERT INTO modify_table (id) VALUES (10001)),
  	second_query AS (SELECT * FROM modify_table) SELECT count(*) FROM second_query;
  count
 ---------------------------------------------------------------------
      1
 (1 row)
 
+SET client_min_messages TO debug2;
+-- pushed down without the insert
+WITH mb AS (UPDATE modify_table SET val = 3 WHERE id = 3 RETURNING NULL) INSERT INTO modify_table WITH ma AS (SELECT * FROM modify_table LIMIT 10) SELECT count(*) FROM mb;
+DEBUG:  LIMIT clauses are not allowed in distributed INSERT ... SELECT queries
+DEBUG:  Creating router plan
+DEBUG:  Plan is router executable
+DETAIL:  distribution column value: 3
+DEBUG:  Collecting INSERT ... SELECT results on coordinator
+-- not pushed down due to volatile
+WITH ma AS (SELECT count(*) FROM modify_table where id = 1), mu AS (WITH allref AS (SELECT random() a FROM modify_table limit 4) UPDATE modify_table SET val = 3 WHERE id = 1 AND val IN (SELECT a FROM allref) RETURNING id+1) SELECT count(*) FROM mu, ma;
+DEBUG:  CTE ma is going to be inlined via distributed planning
+DEBUG:  Router planner doesn't support VOLATILE functions in common table expressions.
+DEBUG:  generating subplan XXX_1 for CTE mu: WITH allref AS (SELECT random() AS a FROM with_modifying.modify_table modify_table_1 LIMIT 4) UPDATE with_modifying.modify_table SET val = 3 WHERE ((id OPERATOR(pg_catalog.=) 1) AND ((val)::double precision OPERATOR(pg_catalog.=) ANY (SELECT allref.a FROM allref))) RETURNING (id OPERATOR(pg_catalog.+) 1)
+DEBUG:  Router planner doesn't support VOLATILE functions in common table expressions.
+DEBUG:  generating subplan XXX_1 for CTE allref: SELECT random() AS a FROM with_modifying.modify_table LIMIT 4
+DEBUG:  Router planner cannot handle multi-shard select queries
+DEBUG:  push down of limit count: 4
+DEBUG:  Plan XXX query after replacing subqueries and CTEs: UPDATE with_modifying.modify_table SET val = 3 WHERE ((id OPERATOR(pg_catalog.=) 1) AND ((val)::double precision OPERATOR(pg_catalog.=) ANY (SELECT allref.a FROM (SELECT intermediate_result.a FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(a double precision)) allref))) RETURNING (id OPERATOR(pg_catalog.+) 1)
+DEBUG:  Creating router plan
+DEBUG:  Plan is router executable
+DETAIL:  distribution column value: 1
+DEBUG:  Distributed planning for a fast-path router query
+DEBUG:  Creating router plan
+DEBUG:  Plan is router executable
+DETAIL:  distribution column value: 1
+DEBUG:  generating subplan XXX_2 for subquery SELECT count(*) AS count FROM with_modifying.modify_table WHERE (id OPERATOR(pg_catalog.=) 1)
+DEBUG:  Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result."?column?" FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result("?column?" integer)) mu, (SELECT intermediate_result.count FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(count bigint)) ma
+DEBUG:  Creating router plan
+DEBUG:  Plan is router executable
+ count
+---------------------------------------------------------------------
+     0
+(1 row)
+
+WITH mu AS (WITH allref AS (SELECT random() a FROM anchor_table) UPDATE modify_table SET val = 3 WHERE id = 1 AND val IN (SELECT a FROM allref) RETURNING id+1) SELECT count(*) FROM mu;
+DEBUG:  Router planner doesn't support VOLATILE functions in common table expressions.
+DEBUG:  generating subplan XXX_1 for CTE mu: WITH allref AS (SELECT random() AS a FROM with_modifying.anchor_table) UPDATE with_modifying.modify_table SET val = 3 WHERE ((id OPERATOR(pg_catalog.=) 1) AND ((val)::double precision OPERATOR(pg_catalog.=) ANY (SELECT allref.a FROM allref))) RETURNING (id OPERATOR(pg_catalog.+) 1)
+DEBUG:  Router planner doesn't support VOLATILE functions in common table expressions.
+DEBUG:  generating subplan XXX_1 for CTE allref: SELECT random() AS a FROM with_modifying.anchor_table
+DEBUG:  Distributed planning for a fast-path router query
+DEBUG:  Creating router plan
+DEBUG:  Plan is router executable
+DEBUG:  Plan XXX query after replacing subqueries and CTEs: UPDATE with_modifying.modify_table SET val = 3 WHERE ((id OPERATOR(pg_catalog.=) 1) AND ((val)::double precision OPERATOR(pg_catalog.=) ANY (SELECT allref.a FROM (SELECT intermediate_result.a FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(a double precision)) allref))) RETURNING (id OPERATOR(pg_catalog.+) 1)
+DEBUG:  Creating router plan
+DEBUG:  Plan is router executable
+DETAIL:  distribution column value: 1
+DEBUG:  Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result."?column?" FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result("?column?" integer)) mu
+DEBUG:  Creating router plan
+DEBUG:  Plan is router executable
+ count
+---------------------------------------------------------------------
+     0
+(1 row)
+
+-- pushed down
+WITH mu AS (WITH allref AS (SELECT id a FROM anchor_table) UPDATE modify_table SET val = 3 WHERE id = 1 AND val IN (SELECT a FROM allref) RETURNING id+1) SELECT count(*) FROM mu;
+DEBUG:  Creating router plan
+DEBUG:  Plan is router executable
+DETAIL:  distribution column value: 1
+ count
+---------------------------------------------------------------------
+     0
+(1 row)
+
+-- pushed down and stable function evaluated
+WITH mu AS (WITH allref AS (SELECT now() a FROM anchor_table) UPDATE modify_table SET val = 3 WHERE id = 1 AND now() IN (SELECT a FROM allref) RETURNING id+1) SELECT count(*) FROM mu;
+DEBUG:  Creating router plan
+DEBUG:  Plan is router executable
+DETAIL:  distribution column value: 1
+ count
+---------------------------------------------------------------------
+     0
+(1 row)
+
+RESET client_min_messages;
+\set VERBOSITY terse
 DROP SCHEMA with_modifying CASCADE;
-NOTICE:  drop cascades to 5 other objects
+NOTICE:  drop cascades to 9 other objects
-DETAIL:  drop cascades to table users_table
-drop cascades to table anchor_table
-drop cascades to function raise_failed_execution_cte(text)
-drop cascades to table modify_table
-drop cascades to table summary_table

src/test/regress/expected/with_prepare.out

@@ -186,6 +186,31 @@ FROM
 	event_id
 WHERE
 	events_user_id IN (SELECT user_id FROM users_table);
+-- Prepare routable modifying CTEs
+PREPARE prepared_test_7 AS
+WITH basic_delete AS (
+	DELETE FROM users_table WHERE user_id=6 RETURNING *
+)
+SELECT
+	*
+FROM
+	basic_delete
+ORDER BY
+	user_id,
+	time
+LIMIT 10;
+PREPARE prepared_test_8 AS
+WITH basic_delete AS (
+	UPDATE users_table SET value_1 = value_1 + 1 WHERE user_id=6 RETURNING *
+)
+SELECT
+	*
+FROM
+	basic_delete
+ORDER BY
+	user_id,
+	time
+LIMIT 10;
 EXECUTE prepared_test_1;
  user_id |              time               | value_1 | value_2 | value_3 | value_4
 ---------------------------------------------------------------------
@@ -590,6 +615,274 @@ EXECUTE prepared_test_6;
    101
 (1 row)
 
+BEGIN;
+EXECUTE prepared_test_7;
+ user_id |              time               | value_1 | value_2 | value_3 | value_4
+---------------------------------------------------------------------
+       6 | Wed Nov 22 20:15:53.317797 2017 |       1 |       1 |       1 |
+       6 | Wed Nov 22 23:01:24.82289 2017  |       2 |       4 |       1 |
+       6 | Thu Nov 23 00:07:11.068353 2017 |       1 |       1 |       4 |
+       6 | Thu Nov 23 00:09:44.19812 2017  |       5 |       2 |       0 |
+       6 | Thu Nov 23 01:13:50.526322 2017 |       2 |       4 |       1 |
+       6 | Thu Nov 23 01:14:55.769581 2017 |       0 |       0 |       5 |
+       6 | Thu Nov 23 10:22:11.02918 2017  |       5 |       0 |       5 |
+       6 | Thu Nov 23 11:08:04.244582 2017 |       2 |       3 |       2 |
+       6 | Thu Nov 23 13:51:16.92838 2017  |       0 |       4 |       2 |
+       6 | Thu Nov 23 14:43:18.024104 2017 |       3 |       2 |       5 |
+(10 rows)
+
+EXECUTE prepared_test_7;
+ user_id | time | value_1 | value_2 | value_3 | value_4
+---------------------------------------------------------------------
+(0 rows)
+
+EXECUTE prepared_test_7;
+ user_id | time | value_1 | value_2 | value_3 | value_4
+---------------------------------------------------------------------
+(0 rows)
+
+EXECUTE prepared_test_7;
+ user_id | time | value_1 | value_2 | value_3 | value_4
+---------------------------------------------------------------------
+(0 rows)
+
+EXECUTE prepared_test_7;
+ user_id | time | value_1 | value_2 | value_3 | value_4
+---------------------------------------------------------------------
+(0 rows)
+
+EXECUTE prepared_test_7;
+ user_id | time | value_1 | value_2 | value_3 | value_4
+---------------------------------------------------------------------
+(0 rows)
+
+ROLLBACK;
+BEGIN;
+EXECUTE prepared_test_7;
+ user_id |              time               | value_1 | value_2 | value_3 | value_4
+---------------------------------------------------------------------
+       6 | Wed Nov 22 20:15:53.317797 2017 |       1 |       1 |       1 |
+       6 | Wed Nov 22 23:01:24.82289 2017  |       2 |       4 |       1 |
+       6 | Thu Nov 23 00:07:11.068353 2017 |       1 |       1 |       4 |
+       6 | Thu Nov 23 00:09:44.19812 2017  |       5 |       2 |       0 |
+       6 | Thu Nov 23 01:13:50.526322 2017 |       2 |       4 |       1 |
+       6 | Thu Nov 23 01:14:55.769581 2017 |       0 |       0 |       5 |
+       6 | Thu Nov 23 10:22:11.02918 2017  |       5 |       0 |       5 |
+       6 | Thu Nov 23 11:08:04.244582 2017 |       2 |       3 |       2 |
+       6 | Thu Nov 23 13:51:16.92838 2017  |       0 |       4 |       2 |
+       6 | Thu Nov 23 14:43:18.024104 2017 |       3 |       2 |       5 |
+(10 rows)
+
+EXECUTE prepared_test_7;
+ user_id | time | value_1 | value_2 | value_3 | value_4
+---------------------------------------------------------------------
+(0 rows)
+
+EXECUTE prepared_test_7;
+ user_id | time | value_1 | value_2 | value_3 | value_4
+---------------------------------------------------------------------
+(0 rows)
+
+EXECUTE prepared_test_7;
+ user_id | time | value_1 | value_2 | value_3 | value_4
+---------------------------------------------------------------------
+(0 rows)
+
+EXECUTE prepared_test_7;
+ user_id | time | value_1 | value_2 | value_3 | value_4
+---------------------------------------------------------------------
+(0 rows)
+
+EXECUTE prepared_test_7;
+ user_id | time | value_1 | value_2 | value_3 | value_4
+---------------------------------------------------------------------
+(0 rows)
+
+ROLLBACK;
+BEGIN;
+EXECUTE prepared_test_8;
+ user_id |              time               | value_1 | value_2 | value_3 | value_4
+---------------------------------------------------------------------
+       6 | Wed Nov 22 20:15:53.317797 2017 |       2 |       1 |       1 |
+       6 | Wed Nov 22 23:01:24.82289 2017  |       3 |       4 |       1 |
+       6 | Thu Nov 23 00:07:11.068353 2017 |       2 |       1 |       4 |
+       6 | Thu Nov 23 00:09:44.19812 2017  |       6 |       2 |       0 |
+       6 | Thu Nov 23 01:13:50.526322 2017 |       3 |       4 |       1 |
+       6 | Thu Nov 23 01:14:55.769581 2017 |       1 |       0 |       5 |
+       6 | Thu Nov 23 10:22:11.02918 2017  |       6 |       0 |       5 |
+       6 | Thu Nov 23 11:08:04.244582 2017 |       3 |       3 |       2 |
+       6 | Thu Nov 23 13:51:16.92838 2017  |       1 |       4 |       2 |
+       6 | Thu Nov 23 14:43:18.024104 2017 |       4 |       2 |       5 |
+(10 rows)
+
+EXECUTE prepared_test_8;
+ user_id |              time               | value_1 | value_2 | value_3 | value_4
+---------------------------------------------------------------------
+       6 | Wed Nov 22 20:15:53.317797 2017 |       3 |       1 |       1 |
+       6 | Wed Nov 22 23:01:24.82289 2017  |       4 |       4 |       1 |
+       6 | Thu Nov 23 00:07:11.068353 2017 |       3 |       1 |       4 |
+       6 | Thu Nov 23 00:09:44.19812 2017  |       7 |       2 |       0 |
+       6 | Thu Nov 23 01:13:50.526322 2017 |       4 |       4 |       1 |
+       6 | Thu Nov 23 01:14:55.769581 2017 |       2 |       0 |       5 |
+       6 | Thu Nov 23 10:22:11.02918 2017  |       7 |       0 |       5 |
+       6 | Thu Nov 23 11:08:04.244582 2017 |       4 |       3 |       2 |
+       6 | Thu Nov 23 13:51:16.92838 2017  |       2 |       4 |       2 |
+       6 | Thu Nov 23 14:43:18.024104 2017 |       5 |       2 |       5 |
+(10 rows)
+
+EXECUTE prepared_test_8;
+ user_id |              time               | value_1 | value_2 | value_3 | value_4
+---------------------------------------------------------------------
+       6 | Wed Nov 22 20:15:53.317797 2017 |       4 |       1 |       1 |
+       6 | Wed Nov 22 23:01:24.82289 2017  |       5 |       4 |       1 |
+       6 | Thu Nov 23 00:07:11.068353 2017 |       4 |       1 |       4 |
+       6 | Thu Nov 23 00:09:44.19812 2017  |       8 |       2 |       0 |
+       6 | Thu Nov 23 01:13:50.526322 2017 |       5 |       4 |       1 |
+       6 | Thu Nov 23 01:14:55.769581 2017 |       3 |       0 |       5 |
+       6 | Thu Nov 23 10:22:11.02918 2017  |       8 |       0 |       5 |
+       6 | Thu Nov 23 11:08:04.244582 2017 |       5 |       3 |       2 |
+       6 | Thu Nov 23 13:51:16.92838 2017  |       3 |       4 |       2 |
+       6 | Thu Nov 23 14:43:18.024104 2017 |       6 |       2 |       5 |
+(10 rows)
+
+EXECUTE prepared_test_8;
+ user_id |              time               | value_1 | value_2 | value_3 | value_4
+---------------------------------------------------------------------
+       6 | Wed Nov 22 20:15:53.317797 2017 |       5 |       1 |       1 |
+       6 | Wed Nov 22 23:01:24.82289 2017  |       6 |       4 |       1 |
+       6 | Thu Nov 23 00:07:11.068353 2017 |       5 |       1 |       4 |
+       6 | Thu Nov 23 00:09:44.19812 2017  |       9 |       2 |       0 |
+       6 | Thu Nov 23 01:13:50.526322 2017 |       6 |       4 |       1 |
+       6 | Thu Nov 23 01:14:55.769581 2017 |       4 |       0 |       5 |
+       6 | Thu Nov 23 10:22:11.02918 2017  |       9 |       0 |       5 |
+       6 | Thu Nov 23 11:08:04.244582 2017 |       6 |       3 |       2 |
+       6 | Thu Nov 23 13:51:16.92838 2017  |       4 |       4 |       2 |
+       6 | Thu Nov 23 14:43:18.024104 2017 |       7 |       2 |       5 |
+(10 rows)
+
+EXECUTE prepared_test_8;
+ user_id |              time               | value_1 | value_2 | value_3 | value_4
+---------------------------------------------------------------------
+       6 | Wed Nov 22 20:15:53.317797 2017 |       6 |       1 |       1 |
+       6 | Wed Nov 22 23:01:24.82289 2017  |       7 |       4 |       1 |
+       6 | Thu Nov 23 00:07:11.068353 2017 |       6 |       1 |       4 |
+       6 | Thu Nov 23 00:09:44.19812 2017  |      10 |       2 |       0 |
+       6 | Thu Nov 23 01:13:50.526322 2017 |       7 |       4 |       1 |
+       6 | Thu Nov 23 01:14:55.769581 2017 |       5 |       0 |       5 |
+       6 | Thu Nov 23 10:22:11.02918 2017  |      10 |       0 |       5 |
+       6 | Thu Nov 23 11:08:04.244582 2017 |       7 |       3 |       2 |
+       6 | Thu Nov 23 13:51:16.92838 2017  |       5 |       4 |       2 |
+       6 | Thu Nov 23 14:43:18.024104 2017 |       8 |       2 |       5 |
+(10 rows)
+
+EXECUTE prepared_test_8;
+ user_id |              time               | value_1 | value_2 | value_3 | value_4
+---------------------------------------------------------------------
+       6 | Wed Nov 22 20:15:53.317797 2017 |       7 |       1 |       1 |
+       6 | Wed Nov 22 23:01:24.82289 2017  |       8 |       4 |       1 |
+       6 | Thu Nov 23 00:07:11.068353 2017 |       7 |       1 |       4 |
+       6 | Thu Nov 23 00:09:44.19812 2017  |      11 |       2 |       0 |
+       6 | Thu Nov 23 01:13:50.526322 2017 |       8 |       4 |       1 |
+       6 | Thu Nov 23 01:14:55.769581 2017 |       6 |       0 |       5 |
+       6 | Thu Nov 23 10:22:11.02918 2017  |      11 |       0 |       5 |
+       6 | Thu Nov 23 11:08:04.244582 2017 |       8 |       3 |       2 |
+       6 | Thu Nov 23 13:51:16.92838 2017  |       6 |       4 |       2 |
+       6 | Thu Nov 23 14:43:18.024104 2017 |       9 |       2 |       5 |
+(10 rows)
+
+ROLLBACK;
+BEGIN;
+EXECUTE prepared_test_8;
+ user_id |              time               | value_1 | value_2 | value_3 | value_4
+---------------------------------------------------------------------
+       6 | Wed Nov 22 20:15:53.317797 2017 |       2 |       1 |       1 |
+       6 | Wed Nov 22 23:01:24.82289 2017  |       3 |       4 |       1 |
+       6 | Thu Nov 23 00:07:11.068353 2017 |       2 |       1 |       4 |
+       6 | Thu Nov 23 00:09:44.19812 2017  |       6 |       2 |       0 |
+       6 | Thu Nov 23 01:13:50.526322 2017 |       3 |       4 |       1 |
+       6 | Thu Nov 23 01:14:55.769581 2017 |       1 |       0 |       5 |
+       6 | Thu Nov 23 10:22:11.02918 2017  |       6 |       0 |       5 |
+       6 | Thu Nov 23 11:08:04.244582 2017 |       3 |       3 |       2 |
+       6 | Thu Nov 23 13:51:16.92838 2017  |       1 |       4 |       2 |
+       6 | Thu Nov 23 14:43:18.024104 2017 |       4 |       2 |       5 |
+(10 rows)
+
+EXECUTE prepared_test_8;
+ user_id |              time               | value_1 | value_2 | value_3 | value_4
+---------------------------------------------------------------------
+       6 | Wed Nov 22 20:15:53.317797 2017 |       3 |       1 |       1 |
+       6 | Wed Nov 22 23:01:24.82289 2017  |       4 |       4 |       1 |
+       6 | Thu Nov 23 00:07:11.068353 2017 |       3 |       1 |       4 |
+       6 | Thu Nov 23 00:09:44.19812 2017  |       7 |       2 |       0 |
+       6 | Thu Nov 23 01:13:50.526322 2017 |       4 |       4 |       1 |
+       6 | Thu Nov 23 01:14:55.769581 2017 |       2 |       0 |       5 |
+       6 | Thu Nov 23 10:22:11.02918 2017  |       7 |       0 |       5 |
+       6 | Thu Nov 23 11:08:04.244582 2017 |       4 |       3 |       2 |
+       6 | Thu Nov 23 13:51:16.92838 2017  |       2 |       4 |       2 |
+       6 | Thu Nov 23 14:43:18.024104 2017 |       5 |       2 |       5 |
+(10 rows)
+
+EXECUTE prepared_test_8;
+ user_id |              time               | value_1 | value_2 | value_3 | value_4
+---------------------------------------------------------------------
+       6 | Wed Nov 22 20:15:53.317797 2017 |       4 |       1 |       1 |
+       6 | Wed Nov 22 23:01:24.82289 2017  |       5 |       4 |       1 |
+       6 | Thu Nov 23 00:07:11.068353 2017 |       4 |       1 |       4 |
+       6 | Thu Nov 23 00:09:44.19812 2017  |       8 |       2 |       0 |
+       6 | Thu Nov 23 01:13:50.526322 2017 |       5 |       4 |       1 |
+       6 | Thu Nov 23 01:14:55.769581 2017 |       3 |       0 |       5 |
+       6 | Thu Nov 23 10:22:11.02918 2017  |       8 |       0 |       5 |
+       6 | Thu Nov 23 11:08:04.244582 2017 |       5 |       3 |       2 |
+       6 | Thu Nov 23 13:51:16.92838 2017  |       3 |       4 |       2 |
+       6 | Thu Nov 23 14:43:18.024104 2017 |       6 |       2 |       5 |
+(10 rows)
+
+EXECUTE prepared_test_8;
+ user_id |              time               | value_1 | value_2 | value_3 | value_4
+---------------------------------------------------------------------
+       6 | Wed Nov 22 20:15:53.317797 2017 |       5 |       1 |       1 |
+       6 | Wed Nov 22 23:01:24.82289 2017  |       6 |       4 |       1 |
+       6 | Thu Nov 23 00:07:11.068353 2017 |       5 |       1 |       4 |
+       6 | Thu Nov 23 00:09:44.19812 2017  |       9 |       2 |       0 |
+       6 | Thu Nov 23 01:13:50.526322 2017 |       6 |       4 |       1 |
+       6 | Thu Nov 23 01:14:55.769581 2017 |       4 |       0 |       5 |
+       6 | Thu Nov 23 10:22:11.02918 2017  |       9 |       0 |       5 |
+       6 | Thu Nov 23 11:08:04.244582 2017 |       6 |       3 |       2 |
+       6 | Thu Nov 23 13:51:16.92838 2017  |       4 |       4 |       2 |
+       6 | Thu Nov 23 14:43:18.024104 2017 |       7 |       2 |       5 |
+(10 rows)
+
+EXECUTE prepared_test_8;
+ user_id |              time               | value_1 | value_2 | value_3 | value_4
+---------------------------------------------------------------------
+       6 | Wed Nov 22 20:15:53.317797 2017 |       6 |       1 |       1 |
+       6 | Wed Nov 22 23:01:24.82289 2017  |       7 |       4 |       1 |
+       6 | Thu Nov 23 00:07:11.068353 2017 |       6 |       1 |       4 |
+       6 | Thu Nov 23 00:09:44.19812 2017  |      10 |       2 |       0 |
+       6 | Thu Nov 23 01:13:50.526322 2017 |       7 |       4 |       1 |
+       6 | Thu Nov 23 01:14:55.769581 2017 |       5 |       0 |       5 |
+       6 | Thu Nov 23 10:22:11.02918 2017  |      10 |       0 |       5 |
+       6 | Thu Nov 23 11:08:04.244582 2017 |       7 |       3 |       2 |
+       6 | Thu Nov 23 13:51:16.92838 2017  |       5 |       4 |       2 |
+       6 | Thu Nov 23 14:43:18.024104 2017 |       8 |       2 |       5 |
+(10 rows)
+
+EXECUTE prepared_test_8;
+ user_id |              time               | value_1 | value_2 | value_3 | value_4
+---------------------------------------------------------------------
+       6 | Wed Nov 22 20:15:53.317797 2017 |       7 |       1 |       1 |
+       6 | Wed Nov 22 23:01:24.82289 2017  |       8 |       4 |       1 |
+       6 | Thu Nov 23 00:07:11.068353 2017 |       7 |       1 |       4 |
+       6 | Thu Nov 23 00:09:44.19812 2017  |      11 |       2 |       0 |
+       6 | Thu Nov 23 01:13:50.526322 2017 |       8 |       4 |       1 |
+       6 | Thu Nov 23 01:14:55.769581 2017 |       6 |       0 |       5 |
+       6 | Thu Nov 23 10:22:11.02918 2017  |      11 |       0 |       5 |
+       6 | Thu Nov 23 11:08:04.244582 2017 |       8 |       3 |       2 |
+       6 | Thu Nov 23 13:51:16.92838 2017  |       6 |       4 |       2 |
+       6 | Thu Nov 23 14:43:18.024104 2017 |       9 |       2 |       5 |
+(10 rows)
+
+ROLLBACK;
 EXECUTE prepared_partition_column_insert(1);
  user_id | time | value_1 | value_2 | value_3 | value_4
 ---------------------------------------------------------------------

src/test/regress/spec/isolation_select_for_update.spec

@@ -76,19 +76,21 @@ step "s1-select-from-t1-rt-with-lc-for-update"
 
 step "s1-select-from-t1-within-cte"
 {
-	WITH first_value AS ( SELECT val_1 FROM test_table_1_rf1 WHERE id = 1 FOR UPDATE)
+	WITH first_value AS (SELECT val_1 FROM test_table_1_rf1 WHERE id = 1 FOR UPDATE)
-	SELECT * FROM first_value;
+	SELECT * FROM first_value WHERE EXISTS (SELECT * FROM first_value);
 }
 
 step "s1-update-rt-with-cte-select-from-rt"
 {
 	WITH foo AS (SELECT * FROM ref_table FOR UPDATE)
-	UPDATE ref_table SET val_1 = 4 FROM foo WHERE ref_table.id = foo.id;
+	UPDATE ref_table SET val_1 = 4 FROM foo WHERE ref_table.id = foo.id AND EXISTS (SELECT * FROM foo);
 }
 
 step "s1-select-from-t1-with-subquery"
 {
+	SET client_min_messages TO DEBUG2;
 	SELECT * FROM (SELECT * FROM test_table_1_rf1 FOR UPDATE) foo WHERE id = 1;
+	RESET client_min_messages;
 }
 
 step "s1-select-from-rt-with-subquery"

src/test/regress/spec/isolation_update_vs_all.spec

@@ -26,6 +26,7 @@ session "s1"
 step "s1-initialize" { COPY update_hash FROM PROGRAM 'echo 0, a && echo 1, b && echo 2, c && echo 3, d && echo 4, e' WITH CSV; }
 step "s1-begin" { BEGIN; }
 step "s1-update" { UPDATE update_hash SET data = 'l' WHERE id = 4; }
+step "s1-update-cte" { WITH cte AS (UPDATE update_hash SET data = 'l' WHERE id = 4 RETURNING *) SELECT * FROM cte WHERE id = 4; }
 step "s1-delete" { DELETE FROM update_hash WHERE id = 4; }
 step "s1-truncate" { TRUNCATE update_hash; }
 step "s1-drop" { DROP TABLE update_hash; }
@@ -47,6 +48,7 @@ step "s1-commit" { COMMIT; }
 session "s2"
 step "s2-begin" { BEGIN; }
 step "s2-update" { UPDATE update_hash SET data = 'l' WHERE id = 4; }
+step "s2-update-cte" { WITH cte AS (UPDATE update_hash SET data = 'l' WHERE id = 4 RETURNING *) SELECT * FROM cte WHERE id = 4; }
 step "s2-delete" { DELETE FROM update_hash WHERE id = 4; }
 step "s2-truncate" { TRUNCATE update_hash; }
 step "s2-drop" { DROP TABLE update_hash; }
@@ -63,6 +65,7 @@ step "s2-commit" { COMMIT; }
 
 // permutations - UPDATE vs UPDATE
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-update" "s2-update" "s1-commit" "s2-commit" "s1-select-count"
+permutation "s1-initialize" "s1-begin" "s2-begin" "s1-update-cte" "s2-update-cte" "s1-commit" "s2-commit" "s1-select-count"
 
 // permutations - UPDATE first
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-update" "s2-delete" "s1-commit" "s2-commit" "s1-select-count"
@@ -78,6 +81,18 @@ permutation "s1-initialize" "s1-begin" "s2-begin" "s1-update" "s2-table-size" "s
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-update" "s2-master-modify-multiple-shards" "s1-commit" "s2-commit" "s1-select-count"
 permutation "s1-drop" "s1-create-non-distributed-table" "s1-initialize" "s1-begin" "s2-begin" "s1-update" "s2-distribute-table" "s1-commit" "s2-commit" "s1-select-count"
 
+permutation "s1-initialize" "s1-begin" "s2-begin" "s1-update-cte" "s2-delete" "s1-commit" "s2-commit" "s1-select-count"
+permutation "s1-initialize" "s1-begin" "s2-begin" "s1-update-cte" "s2-truncate" "s1-commit" "s2-commit" "s1-select-count"
+permutation "s1-initialize" "s1-begin" "s2-begin" "s1-update-cte" "s2-drop" "s1-commit" "s2-commit" "s1-select-count"
+permutation "s1-initialize" "s1-begin" "s2-begin" "s1-update-cte" "s2-ddl-create-index" "s1-commit" "s2-commit" "s1-select-count" "s1-show-indexes"
+permutation "s1-initialize" "s1-ddl-create-index" "s1-begin" "s2-begin" "s1-update-cte" "s2-ddl-drop-index" "s1-commit" "s2-commit" "s1-select-count" "s1-show-indexes"
+permutation "s1-initialize" "s1-begin" "s1-update-cte" "s2-ddl-create-index-concurrently" "s1-commit" "s1-select-count" "s1-show-indexes"
+permutation "s1-initialize" "s1-begin" "s2-begin" "s1-update-cte" "s2-ddl-add-column" "s1-commit" "s2-commit" "s1-select-count" "s1-show-columns"
+permutation "s1-initialize" "s1-ddl-add-column" "s1-begin" "s2-begin" "s1-update-cte" "s2-ddl-drop-column" "s1-commit" "s2-commit" "s1-select-count" "s1-show-columns"
+permutation "s1-initialize" "s1-begin" "s2-begin" "s1-update-cte" "s2-ddl-rename-column" "s1-commit" "s2-commit" "s1-select-count" "s1-show-columns"
+permutation "s1-initialize" "s1-begin" "s2-begin" "s1-update-cte" "s2-table-size" "s1-commit" "s2-commit" "s1-select-count"
+permutation "s1-initialize" "s1-begin" "s2-begin" "s1-update-cte" "s2-master-modify-multiple-shards" "s1-commit" "s2-commit" "s1-select-count"
+
 // permutations - UPDATE second
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-delete" "s2-update" "s1-commit" "s2-commit" "s1-select-count"
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-truncate" "s2-update" "s1-commit" "s2-commit" "s1-select-count"
@@ -90,3 +105,15 @@ permutation "s1-initialize" "s1-begin" "s2-begin" "s1-ddl-rename-column" "s2-upd
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-table-size" "s2-update" "s1-commit" "s2-commit" "s1-select-count"
 permutation "s1-initialize" "s1-begin" "s2-begin" "s1-master-modify-multiple-shards" "s2-update" "s1-commit" "s2-commit" "s1-select-count"
 permutation "s1-drop" "s1-create-non-distributed-table" "s1-initialize" "s1-begin" "s2-begin" "s1-distribute-table" "s2-update" "s1-commit" "s2-commit" "s1-select-count"
+
+permutation "s1-initialize" "s1-begin" "s2-begin" "s1-delete" "s2-update-cte" "s1-commit" "s2-commit" "s1-select-count"
+permutation "s1-initialize" "s1-begin" "s2-begin" "s1-truncate" "s2-update-cte" "s1-commit" "s2-commit" "s1-select-count"
+permutation "s1-initialize" "s1-begin" "s2-begin" "s1-drop" "s2-update-cte" "s1-commit" "s2-commit" "s1-select-count"
+permutation "s1-initialize" "s1-begin" "s2-begin" "s1-ddl-create-index" "s2-update-cte" "s1-commit" "s2-commit" "s1-select-count" "s1-show-indexes"
+permutation "s1-initialize" "s1-ddl-create-index" "s1-begin" "s2-begin" "s1-ddl-drop-index" "s2-update-cte" "s1-commit" "s2-commit" "s1-select-count" "s1-show-indexes"
+permutation "s1-initialize" "s1-begin" "s2-begin" "s1-ddl-add-column" "s2-update-cte" "s1-commit" "s2-commit" "s1-select-count" "s1-show-columns"
+permutation "s1-initialize" "s1-ddl-add-column" "s1-begin" "s2-begin" "s1-ddl-drop-column" "s2-update-cte" "s1-commit" "s2-commit" "s1-select-count" "s1-show-columns"
+permutation "s1-initialize" "s1-begin" "s2-begin" "s1-ddl-rename-column" "s2-update-cte" "s1-commit" "s2-commit" "s1-select-count" "s1-show-columns"
+permutation "s1-initialize" "s1-begin" "s2-begin" "s1-table-size" "s2-update-cte" "s1-commit" "s2-commit" "s1-select-count"
+permutation "s1-initialize" "s1-begin" "s2-begin" "s1-master-modify-multiple-shards" "s2-update-cte" "s1-commit" "s2-commit" "s1-select-count"
+permutation "s1-drop" "s1-create-non-distributed-table" "s1-initialize" "s1-begin" "s2-begin" "s1-distribute-table" "s2-update-cte" "s1-commit" "s2-commit" "s1-select-count"

src/test/regress/sql/foreign_key_restriction_enforcement.sql

@@ -532,6 +532,18 @@ BEGIN;
 	UPDATE transitive_reference_table SET id = 160 WHERE id = 15;
 ROLLBACK;
 
+BEGIN;
+	WITH cte AS (UPDATE on_update_fkey_table SET value_1 = 16 WHERE value_1 = 15 RETURNING *)
+    SELECT * FROM cte;
+	UPDATE reference_table SET id = 160 WHERE id = 15;
+ROLLBACK;
+
+BEGIN;
+	WITH cte AS (UPDATE on_update_fkey_table SET value_1 = 16 WHERE value_1 = 15 RETURNING *)
+    SELECT * FROM cte;
+	UPDATE transitive_reference_table SET id = 160 WHERE id = 15;
+ROLLBACK;
+
 -- case 5.3: Parallel UPDATE on distributed table follow by an unrelated DDL on reference table
 BEGIN;
 	UPDATE on_update_fkey_table SET value_1 = 16 WHERE value_1 = 15;

src/test/regress/sql/multi_router_planner.sql

@@ -171,6 +171,32 @@ SELECT * FROM articles_hash WHERE author_id IN (1, NULL) ORDER BY id;
 WITH first_author AS ( SELECT id FROM articles_hash WHERE author_id = 1)
 SELECT * FROM first_author;
 
+-- SELECT FOR UPDATE is supported if not involving reference table
+BEGIN;
+WITH first_author AS (
+    SELECT articles_hash.id, auref.name FROM articles_hash, authors_reference auref
+    WHERE author_id = 2 AND auref.id = author_id
+    FOR UPDATE
+)
+UPDATE articles_hash SET title = first_author.name
+FROM first_author WHERE articles_hash.author_id = 2 AND articles_hash.id = first_author.id;
+
+WITH first_author AS (
+    SELECT id, word_count FROM articles_hash WHERE author_id = 2
+    FOR UPDATE
+)
+UPDATE articles_hash SET title = first_author.word_count::text
+FROM first_author WHERE articles_hash.author_id = 2 AND articles_hash.id = first_author.id;
+
+-- Without FOR UPDATE this is router plannable
+WITH first_author AS (
+    SELECT articles_hash.id, auref.name FROM articles_hash, authors_reference auref
+    WHERE author_id = 2 AND auref.id = author_id
+)
+UPDATE articles_hash SET title = first_author.name
+FROM first_author WHERE articles_hash.author_id = 2 AND articles_hash.id = first_author.id;
+ROLLBACK;
+
 -- queries with CTEs are supported even if CTE is not referenced inside query
 WITH first_author AS ( SELECT id FROM articles_hash WHERE author_id = 1)
 SELECT title FROM articles_hash WHERE author_id = 1;
@@ -1038,7 +1064,8 @@ SELECT count(*), count(*) FILTER (WHERE id < 3)
 PREPARE author_1_articles as
 	SELECT *
 	FROM articles_hash
-	WHERE author_id = 1;
+	WHERE author_id = 1
+	ORDER BY 1;
 
 EXECUTE author_1_articles;
 
@@ -1046,7 +1073,8 @@ EXECUTE author_1_articles;
 PREPARE author_articles(int) as
 	SELECT *
 	FROM articles_hash
-	WHERE author_id = $1;
+	WHERE author_id = $1
+	ORDER BY 1;
 
 EXECUTE author_articles(1);
 
@@ -1076,7 +1104,7 @@ BEGIN
 END;
 $$ LANGUAGE plpgsql;
 
-SELECT * FROM author_articles_id_word_count();
+SELECT * FROM author_articles_id_word_count() ORDER BY 1;
 
 -- materialized views can be created for router plannable queries
 CREATE MATERIALIZED VIEW mv_articles_hash_empty AS
@@ -1091,7 +1119,8 @@ SELECT * FROM mv_articles_hash_data ORDER BY 1, 2, 3, 4;
 SET citus.task_executor_type to 'task-tracker';
 SELECT id
 	FROM articles_hash
-	WHERE author_id = 1;
+	WHERE author_id = 1
+	ORDER BY 1;
 
 -- insert query is router plannable even under task-tracker
 INSERT INTO articles_hash VALUES (51, 1, 'amateus', 1814), (52, 1, 'second amateus', 2824);
@@ -1099,7 +1128,8 @@ INSERT INTO articles_hash VALUES (51, 1, 'amateus', 1814), (52, 1, 'second amate
 -- verify insert is successful (not router plannable and executable)
 SELECT id
 	FROM articles_hash
-	WHERE author_id = 1;
+	WHERE author_id = 1
+	ORDER BY 1;
 
 -- https://github.com/citusdata/citus/issues/3624
 UPDATE articles_hash SET id = id

src/test/regress/sql/relation_access_tracking.sql

@@ -55,7 +55,7 @@ $$ LANGUAGE 'plpgsql' IMMUTABLE;
 
 
 
-CREATE VIEW relation_acesses AS
+CREATE VIEW relation_accesses AS
 	SELECT table_name,
 			relation_access_mode_to_text(table_name, relation_select_access_mode(table_name::regclass)) as select_access,
 			relation_access_mode_to_text(table_name, relation_dml_access_mode(table_name::regclass)) as dml_access,
@@ -93,29 +93,29 @@ INSERT INTO table_6 SELECT i, i FROM generate_series(0,100) i;
 BEGIN;
 	CREATE TABLE table_7 (key int, value int);
 	SELECT create_distributed_table('table_7', 'key');
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_7')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_7')  ORDER BY 1;
 COMMIT;
 
--- outisde the transaction blocks, the function always returns zero
+-- outside the transaction blocks, the function always returns zero
 SELECT count(*) FROM table_1;
-SELECT * FROM relation_acesses WHERE table_name = 'table_1';
+SELECT * FROM relation_accesses WHERE table_name = 'table_1';
 
 -- a very simple test that first checks sequential
 -- and parallel SELECTs,DMLs, and DDLs
 BEGIN;
-	SELECT * FROM relation_acesses WHERE table_name = 'table_1';
+	SELECT * FROM relation_accesses WHERE table_name = 'table_1';
 	SELECT count(*) FROM table_1 WHERE key = 1;
-	SELECT * FROM relation_acesses WHERE table_name = 'table_1';
+	SELECT * FROM relation_accesses WHERE table_name = 'table_1';
 	SELECT count(*) FROM table_1 WHERE key = 1 OR key = 2;
-	SELECT * FROM relation_acesses WHERE table_name = 'table_1';
+	SELECT * FROM relation_accesses WHERE table_name = 'table_1';
 	INSERT INTO table_1 VALUES (1,1);
-	SELECT * FROM relation_acesses WHERE table_name = 'table_1';
+	SELECT * FROM relation_accesses WHERE table_name = 'table_1';
 	INSERT INTO table_1 VALUES (1,1), (2,2);
-	SELECT * FROM relation_acesses WHERE table_name = 'table_1';
+	SELECT * FROM relation_accesses WHERE table_name = 'table_1';
 	ALTER TABLE table_1 ADD COLUMN test_col INT;
 
 	-- now see that the other tables are not accessed at all
-	SELECT * FROM relation_acesses WHERE table_name = 'table_1';
+	SELECT * FROM relation_accesses WHERE table_name = 'table_1';
 
 ROLLBACK;
 
@@ -124,19 +124,19 @@ ROLLBACK;
 -- commands executed, we can treat the transaction as sequential
 BEGIN;
 	SELECT count(*) FROM table_1 WHERE key = 1;
-	SELECT * FROM relation_acesses WHERE table_name = 'table_1';
+	SELECT * FROM relation_accesses WHERE table_name = 'table_1';
 	SELECT count(*) FROM table_1 WHERE key = 2;
-	SELECT * FROM relation_acesses WHERE table_name = 'table_1';
+	SELECT * FROM relation_accesses WHERE table_name = 'table_1';
 	INSERT INTO table_1 VALUES (1,1);
-	SELECT * FROM relation_acesses WHERE table_name = 'table_1';
+	SELECT * FROM relation_accesses WHERE table_name = 'table_1';
 	INSERT INTO table_1 VALUES (2,2);
-	SELECT * FROM relation_acesses WHERE table_name = 'table_1';
+	SELECT * FROM relation_accesses WHERE table_name = 'table_1';
 ROLLBACK;
 
 -- a sample DDL example
 BEGIN;
 	ALTER TABLE table_1 ADD CONSTRAINT table_1_u UNIQUE (key);
-	SELECT * FROM relation_acesses WHERE table_name = 'table_1';
+	SELECT * FROM relation_accesses WHERE table_name = 'table_1';
 ROLLBACK;
 
 -- a simple join touches single shard per table
@@ -150,7 +150,7 @@ BEGIN;
 		table_3.key = table_4.key AND table_4.key = table_5.key AND
 		table_1.key = 1;
 
-		SELECT * FROM relation_acesses WHERE table_name LIKE 'table_%' ORDER BY 1;
+		SELECT * FROM relation_accesses WHERE table_name LIKE 'table_%' ORDER BY 1;
 ROLLBACK;
 
 -- a simple real-time join touches all shard per table
@@ -162,7 +162,7 @@ BEGIN;
 	WHERE
 		table_1.key = table_2.key;
 
-		SELECT * FROM relation_acesses WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
+		SELECT * FROM relation_accesses WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
 ROLLBACK;
 
 -- a simple real-time join touches all shard per table
@@ -177,7 +177,7 @@ BEGIN;
 	WHERE
 		table_1.key = table_2.key;
 
-		SELECT * FROM relation_acesses WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
+		SELECT * FROM relation_accesses WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
 ROLLBACK;
 
 -- a simple subquery pushdown that touches all shards
@@ -196,7 +196,7 @@ BEGIN;
 			table_3.key = table_4.key AND table_4.key = table_5.key
 	) as foo;
 
-	SELECT * FROM relation_acesses WHERE table_name LIKE 'table_%' ORDER BY 1;
+	SELECT * FROM relation_accesses WHERE table_name LIKE 'table_%' ORDER BY 1;
 ROLLBACK;
 
 -- simple multi shard update both sequential and parallel modes
@@ -204,10 +204,10 @@ ROLLBACK;
 -- access for all the shards accessed. But, sequential mode is OK
 BEGIN;
 	UPDATE table_1 SET value = 15;
-	SELECT * FROM relation_acesses WHERE table_name = 'table_1';
+	SELECT * FROM relation_accesses WHERE table_name = 'table_1';
 	SET LOCAL citus.multi_shard_modify_mode = 'sequential';
 	UPDATE table_2 SET value = 15;
-	SELECT * FROM relation_acesses WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
+	SELECT * FROM relation_accesses WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
 ROLLBACK;
 
 -- now UPDATE/DELETE with subselect pushdown
@@ -215,13 +215,13 @@ BEGIN;
 	UPDATE
 		table_1 SET value = 15
 	WHERE key IN (SELECT key FROM table_2 JOIN table_3 USING (key) WHERE table_2.value = 15);
-	SELECT * FROM relation_acesses WHERE table_name IN ('table_1', 'table_2', 'table_3')  ORDER BY 1;
+	SELECT * FROM relation_accesses WHERE table_name IN ('table_1', 'table_2', 'table_3')  ORDER BY 1;
 ROLLBACK;
 
 -- INSERT .. SELECT pushdown
 BEGIN;
 	INSERT INTO table_2 SELECT * FROM table_1;
-	SELECT * FROM relation_acesses WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
+	SELECT * FROM relation_accesses WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
 ROLLBACK;
 
 -- INSERT .. SELECT pushdown in sequential mode should be OK
@@ -229,14 +229,14 @@ BEGIN;
 	SET LOCAL citus.multi_shard_modify_mode = 'sequential';
 
 	INSERT INTO table_2 SELECT * FROM table_1;
-	SELECT * FROM relation_acesses WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
+	SELECT * FROM relation_accesses WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
 ROLLBACK;
 
 -- coordinator INSERT .. SELECT
 BEGIN;
     -- We use offset 1 to make sure the result needs to be pulled to the coordinator, offset 0 would be optimized away
 	INSERT INTO table_2 SELECT * FROM table_1 OFFSET 1;
-	SELECT * FROM relation_acesses WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
+	SELECT * FROM relation_accesses WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
 ROLLBACK;
 
 
@@ -257,7 +257,7 @@ BEGIN;
 			OFFSET 0
 	) as foo;
 
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
 ROLLBACK;
 
 -- recursively planned SELECT and coordinator INSERT .. SELECT
@@ -277,7 +277,7 @@ BEGIN;
 			OFFSET 0
 	) as foo;
 
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1', 'table_2', 'table_3')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_1', 'table_2', 'table_3')  ORDER BY 1;
 ROLLBACK;
 
 -- recursively planned SELECT and coordinator INSERT .. SELECT
@@ -299,7 +299,7 @@ BEGIN;
 			OFFSET 0
 	) as foo;
 
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1', 'table_2', 'table_3')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_1', 'table_2', 'table_3')  ORDER BY 1;
 ROLLBACK;
 
 -- recursively planned SELECT and recursively planned multi-shard DELETE
@@ -320,13 +320,13 @@ BEGIN;
 		) as foo
 	) AND value IN (SELECT key FROM table_4);
 
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1', 'table_2', 'table_3', 'table_4')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_1', 'table_2', 'table_3', 'table_4')  ORDER BY 1;
 ROLLBACK;
 
 -- copy out
 BEGIN;
 	COPY (SELECT * FROM table_1 WHERE key IN (1,2,3) ORDER BY 1) TO stdout;
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_1')  ORDER BY 1;
 ROLLBACK;
 
 -- copy in
@@ -336,7 +336,7 @@ BEGIN;
 2,2
 3,3
 \.
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_1')  ORDER BY 1;
 ROLLBACK;
 
 -- copy in single shard
@@ -346,51 +346,51 @@ BEGIN;
 1,2
 1,3
 \.
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_1')  ORDER BY 1;
 ROLLBACK;
 
 -- reference table accesses should always be a sequential
 BEGIN;
 	SELECT count(*) FROM table_6;
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_6');
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_6');
 
 	UPDATE table_6 SET value = 15;
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_6');
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_6');
 
 	ALTER TABLE table_6 ADD COLUMN x INT;
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_6');
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_6');
 ROLLBACK;
 
 -- reference table join with a distributed table
 BEGIN;
 	SELECT count(*) FROM table_1 JOIN table_6 USING(key);
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_6', 'table_1') ORDER BY 1,2;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_6', 'table_1') ORDER BY 1,2;
 ROLLBACK;
 
 -- TRUNCATE should be DDL
 BEGIN;
 	TRUNCATE table_1;
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_1')  ORDER BY 1;
 ROLLBACK;
 
 -- TRUNCATE can be a sequential DDL
 BEGIN;
 	SET LOCAL citus.multi_shard_modify_mode = 'sequential';
 	TRUNCATE table_1;
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_1')  ORDER BY 1;
 ROLLBACK;
 
 -- TRUNCATE on a reference table should be sequential
 BEGIN;
 	TRUNCATE table_6;
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_6')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_6')  ORDER BY 1;
 ROLLBACK;
 
 -- creating foreign keys should consider adding the placement accesses for the referenced table
 ALTER TABLE table_1 ADD CONSTRAINT table_1_u UNIQUE (key);
 BEGIN;
 	ALTER TABLE table_2 ADD CONSTRAINT table_2_u FOREIGN KEY (key) REFERENCES table_1(key);
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
 ROLLBACK;
 
 -- creating foreign keys should consider adding the placement accesses for the referenced table
@@ -399,7 +399,7 @@ BEGIN;
 	SET LOCAL citus.multi_shard_modify_mode = 'sequential';
 
 	ALTER TABLE table_2 ADD CONSTRAINT table_2_u FOREIGN KEY (key) REFERENCES table_1(key);
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
 ROLLBACK;
 
 CREATE TABLE partitioning_test(id int, time date) PARTITION BY RANGE (time);
@@ -408,14 +408,14 @@ SELECT create_distributed_table('partitioning_test', 'id');
 -- Adding partition tables via CREATE TABLE should have DDL access the partitioned table as well
 BEGIN;
 	CREATE TABLE partitioning_test_2009 PARTITION OF partitioning_test FOR VALUES FROM ('2009-01-01') TO ('2010-01-01');
-	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009')  ORDER BY 1;
 ROLLBACK;
 
 -- Adding partition tables via ATTACH PARTITION on local tables should have DDL access the partitioned table as well
 CREATE TABLE partitioning_test_2009 AS SELECT * FROM partitioning_test;
 BEGIN;
 	ALTER TABLE partitioning_test ATTACH PARTITION partitioning_test_2009 FOR VALUES FROM ('2009-01-01') TO ('2010-01-01');
-	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009')  ORDER BY 1;
 COMMIT;
 
 -- Adding partition tables via ATTACH PARTITION on distributed tables should have DDL access the partitioned table as well
@@ -423,95 +423,95 @@ CREATE TABLE partitioning_test_2010 AS SELECT * FROM partitioning_test;
 SELECT create_distributed_table('partitioning_test_2010', 'id');
 BEGIN;
 	ALTER TABLE partitioning_test ATTACH PARTITION partitioning_test_2010 FOR VALUES FROM ('2010-01-01') TO ('2011-01-01');
-	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2010')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2010')  ORDER BY 1;
 COMMIT;
 
 -- reading from partitioned table marks all of its partitions
 BEGIN;
 	SELECT count(*) FROM partitioning_test;
-	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
 COMMIT;
 
 -- reading from partitioned table sequentially marks all of its partitions with sequential accesses
 BEGIN;
 	SET LOCAL citus.multi_shard_modify_mode TO 'sequential';
 	SELECT count(*) FROM partitioning_test;
-	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
 COMMIT;
 
 -- updating partitioned table marks all of its partitions
 BEGIN;
 	UPDATE partitioning_test SET time = now();
-	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
 COMMIT;
 
 -- updating partitioned table sequentially marks all of its partitions with sequential accesses
 BEGIN;
 	SET LOCAL citus.multi_shard_modify_mode TO 'sequential';
 	UPDATE partitioning_test SET time = now();
-	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
 COMMIT;
 
 
 -- DDLs on partitioned table marks all of its partitions
 BEGIN;
 	ALTER TABLE partitioning_test ADD COLUMN X INT;
-	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
 ROLLBACK;
 
 -- DDLs on partitioned table sequentially marks all of its partitions with sequential accesses
 BEGIN;
 	SET LOCAL citus.multi_shard_modify_mode TO 'sequential';
 	ALTER TABLE partitioning_test ADD COLUMN X INT;
-	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
 ROLLBACK;
 
 
 -- reading from partition table marks its parent
 BEGIN;
 	SELECT count(*) FROM partitioning_test_2009;
-	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
 COMMIT;
 
 -- rreading from partition table marks its parent with sequential accesses
 BEGIN;
 	SET LOCAL citus.multi_shard_modify_mode TO 'sequential';
 	SELECT count(*) FROM partitioning_test_2009;
-	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
 COMMIT;
 
 -- updating from partition table marks its parent
 BEGIN;
 	UPDATE partitioning_test_2009 SET time = now();
-	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
 COMMIT;
 
 -- updating from partition table marks its parent sequential accesses
 BEGIN;
 	SET LOCAL citus.multi_shard_modify_mode TO 'sequential';
 	UPDATE partitioning_test_2009 SET time = now();
-	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
 COMMIT;
 
 
 -- DDLs on partition table marks its parent
 BEGIN;
 	CREATE INDEX i1000000 ON partitioning_test_2009 (id);
-	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
 ROLLBACK;
 
 -- DDLs on partition table marks its parent in sequential mode
 BEGIN;
 	SET LOCAL citus.multi_shard_modify_mode TO 'sequential';
 	CREATE INDEX i1000000 ON partitioning_test_2009 (id);
-	SELECT * FROM relation_acesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010')  ORDER BY 1;
 ROLLBACK;
 
 -- TRUNCATE CASCADE works fine
 ALTER TABLE table_2 ADD CONSTRAINT table_2_u FOREIGN KEY (key) REFERENCES table_1(key);
 BEGIN;
 	TRUNCATE table_1 CASCADE;
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_1', 'table_2')  ORDER BY 1;
 ROLLBACK;
 
 -- CTEs with SELECT only should work fine
@@ -519,7 +519,7 @@ BEGIN;
 
 	WITH cte AS (SELECT count(*) FROM table_1)
 	SELECT * FROM cte;
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_1')  ORDER BY 1;
 COMMIT;
 
 -- CTEs with SELECT only in sequential mode should work fine
@@ -528,7 +528,7 @@ BEGIN;
 
 	WITH cte AS (SELECT count(*) FROM table_1)
 	SELECT * FROM cte;
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_1')  ORDER BY 1;
 COMMIT;
 
 -- modifying CTEs should work fine with multi-row inserts, which are by default in sequential
@@ -536,7 +536,7 @@ BEGIN;
 
 	WITH cte_1 AS (INSERT INTO table_1 VALUES (1000,1000), (1001, 1001), (1002, 1002) RETURNING *)
 	SELECT * FROM cte_1 ORDER BY 1;
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_1')  ORDER BY 1;
 ROLLBACK;
 
 -- modifying CTEs should work fine with parallel mode
@@ -544,15 +544,33 @@ BEGIN;
 
 	WITH cte_1 AS (UPDATE table_1 SET value = 15 RETURNING *)
 	SELECT count(*) FROM cte_1 ORDER BY 1;
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_1')  ORDER BY 1;
 ROLLBACK;
 
 -- modifying CTEs should work fine with sequential mode
 BEGIN;
+	SET LOCAL citus.multi_shard_modify_mode = 'sequential';
 
 	WITH cte_1 AS (UPDATE table_1 SET value = 15 RETURNING *)
 	SELECT count(*) FROM cte_1 ORDER BY 1;
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_1')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_1')  ORDER BY 1;
+ROLLBACK;
+
+-- router planned modifying CTEs should work fine with parallel mode
+BEGIN;
+
+	WITH cte_1 AS (UPDATE table_1 SET value = 15 WHERE key = 6 RETURNING *)
+	SELECT count(*) FROM cte_1 ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_1')  ORDER BY 1;
+ROLLBACK;
+
+-- router planned modifying CTEs should work fine with sequential mode
+BEGIN;
+	SET LOCAL citus.multi_shard_modify_mode = 'sequential';
+
+	WITH cte_1 AS (UPDATE table_1 SET value = 15 WHERE key = 6 RETURNING *)
+	SELECT count(*) FROM cte_1 ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_1')  ORDER BY 1;
 ROLLBACK;
 
 -- create distributed table with data loading
@@ -562,7 +580,7 @@ CREATE TABLE table_3 (key int, value int);
 INSERT INTO table_3 SELECT i, i FROM generate_series(0,100) i;
 BEGIN;
 	SELECT create_distributed_table('table_3', 'key');
-	SELECT * FROM relation_acesses  WHERE table_name IN ('table_3')  ORDER BY 1;
+	SELECT * FROM relation_accesses  WHERE table_name IN ('table_3')  ORDER BY 1;
 COMMIT;
 
 SET search_path TO 'public';

src/test/regress/sql/with_modifying.sql

@@ -4,6 +4,8 @@ SET citus.next_shard_id TO 1502000;
 CREATE SCHEMA with_modifying;
 SET search_path TO with_modifying, public;
 
+CREATE TABLE with_modifying.local_table (id int, val int);
+
 CREATE TABLE with_modifying.modify_table (id int, val int);
 SELECT create_distributed_table('modify_table', 'id');
 
@@ -113,13 +115,13 @@ INSERT INTO modify_table (SELECT cte_1.user_id FROM cte_1 join cte_2 on cte_1.va
 -- between different executors
 CREATE FUNCTION raise_failed_execution_cte(query text) RETURNS void AS $$
 BEGIN
-        EXECUTE query;
+	EXECUTE query;
-        EXCEPTION WHEN OTHERS THEN
+	EXCEPTION WHEN OTHERS THEN
-        IF SQLERRM LIKE '%more than one row returned by a subquery used as an expression%' THEN
+	IF SQLERRM LIKE '%more than one row returned by a subquery used as an expression%' THEN
-                RAISE 'Task failed to execute';
+		RAISE 'Task failed to execute';
-        ELSIF SQLERRM LIKE '%could not receive query results%' THEN
+	ELSIF SQLERRM LIKE '%could not receive query results%' THEN
-          RAISE 'Task failed to execute';
+		RAISE 'Task failed to execute';
-        END IF;
+	END IF;
 END;
 $$LANGUAGE plpgsql;
 
@@ -236,21 +238,21 @@ INSERT INTO modify_table VALUES (21, 1), (22, 2), (23, 3);
 
 -- read ids from the same table
 WITH distinct_ids AS (
-  SELECT DISTINCT id FROM modify_table
+	SELECT DISTINCT id FROM modify_table
 ),
 update_data AS (
-  UPDATE modify_table SET val = 100 WHERE id > 10 AND
+	UPDATE modify_table SET val = 100 WHERE id > 10 AND
-  	id IN (SELECT * FROM distinct_ids) RETURNING *
+		id IN (SELECT * FROM distinct_ids) RETURNING *
 )
 SELECT count(*) FROM update_data;
 
 -- read ids from a different table
 WITH distinct_ids AS (
-  SELECT DISTINCT id FROM summary_table
+	SELECT DISTINCT id FROM summary_table
 ),
 update_data AS (
-  UPDATE modify_table SET val = 100 WHERE id > 10 AND
+	UPDATE modify_table SET val = 100 WHERE id > 10 AND
-  	id IN (SELECT * FROM distinct_ids) RETURNING *
+		id IN (SELECT * FROM distinct_ids) RETURNING *
 )
 SELECT count(*) FROM update_data;
 
@@ -410,6 +412,84 @@ raw_data AS (
 )
 SELECT * FROM raw_data ORDER BY val;
 
+-- Test that local tables are barred
+UPDATE local_table lt SET val = mt.val
+FROM modify_table mt WHERE mt.id = lt.id;
+
+-- Including inside CTEs
+WITH cte AS (
+	UPDATE local_table lt SET val = mt.val
+	FROM modify_table mt WHERE mt.id = lt.id
+	RETURNING lt.id, lt.val
+) SELECT * FROM cte JOIN modify_table mt ON mt.id = cte.id ORDER BY 1,2;
+
+-- Make sure checks for volatile functions apply to CTEs too
+WITH cte AS (UPDATE modify_table SET val = random() WHERE id = 3 RETURNING *)
+SELECT * FROM cte JOIN modify_table mt ON mt.id = 3 AND mt.id = cte.id ORDER BY 1,2;
+
+-- Two queries from HammerDB:
+-- 1
+CREATE TABLE with_modifying.stock (s_i_id numeric(6,0) NOT NULL, s_w_id numeric(4,0) NOT NULL, s_quantity numeric(6,0), s_dist_01 character(24)) WITH (fillfactor='50');
+ALTER TABLE with_modifying.stock ADD CONSTRAINT stock_i1 PRIMARY KEY (s_i_id, s_w_id);
+SELECT create_distributed_table('stock', 's_w_id');
+INSERT INTO with_modifying.stock VALUES
+	(64833, 10, 3, 'test1'),
+	(64834, 10, 3, 'test2'),
+	(63867, 10, 3, 'test3');
+PREPARE su_after(INT[], SMALLINT[], SMALLINT[], NUMERIC(5,2)[], NUMERIC, NUMERIC, NUMERIC) AS
+	WITH stock_update AS (
+		UPDATE stock
+		SET s_quantity = ( CASE WHEN s_quantity < (item_stock.quantity + 10) THEN s_quantity + 91 ELSE s_quantity END) - item_stock.quantity
+		FROM UNNEST($1, $2, $3, $4) AS item_stock (item_id, supply_wid, quantity, price)
+		WHERE stock.s_i_id = item_stock.item_id
+			AND stock.s_w_id = item_stock.supply_wid
+			AND stock.s_w_id = ANY ($2)
+		RETURNING stock.s_dist_01 as s_dist, stock.s_quantity, ( item_stock.quantity + item_stock.price * ( 1 + $5 + $6 ) * ( 1 - $7) ) amount
+	)
+	SELECT array_agg ( s_dist ), array_agg ( s_quantity ), array_agg ( amount )
+	FROM stock_update;
+EXECUTE su_after('{64833,63857,13941,76514,35858,10004,88553,34483,91251,28144,51687,36407,54436,72873}', '{10,10,10,10,10,10,10,10,10,10,10,10,10,10}', '{8,2,2,6,7,4,6,1,1,5,6,7,6,2}', '{26.04,4.79,67.84,77.66,47.06,23.12,32.74,56.99,84.75,37.52,73.52,98.86,49.96,29.47}', 0.1800, 0.1100, 0.5000);
+EXECUTE su_after('{64833,63857,13941,76514,35858,10004,88553,34483,91251,28144,51687,36407,54436,72873}', '{10,10,10,10,10,10,10,10,10,10,10,10,10,10}', '{8,2,2,6,7,4,6,1,1,5,6,7,6,2}', '{26.04,4.79,67.84,77.66,47.06,23.12,32.74,56.99,84.75,37.52,73.52,98.86,49.96,29.47}', 0.1800, 0.1100, 0.5000);
+EXECUTE su_after('{64833,63857,13941,76514,35858,10004,88553,34483,91251,28144,51687,36407,54436,72873}', '{10,10,10,10,10,10,10,10,10,10,10,10,10,10}', '{8,2,2,6,7,4,6,1,1,5,6,7,6,2}', '{26.04,4.79,67.84,77.66,47.06,23.12,32.74,56.99,84.75,37.52,73.52,98.86,49.96,29.47}', 0.1800, 0.1100, 0.5000);
+EXECUTE su_after('{64833,63857,13941,76514,35858,10004,88553,34483,91251,28144,51687,36407,54436,72873}', '{10,10,10,10,10,10,10,10,10,10,10,10,10,10}', '{8,2,2,6,7,4,6,1,1,5,6,7,6,2}', '{26.04,4.79,67.84,77.66,47.06,23.12,32.74,56.99,84.75,37.52,73.52,98.86,49.96,29.47}', 0.1800, 0.1100, 0.5000);
+EXECUTE su_after('{64833,63857,13941,76514,35858,10004,88553,34483,91251,28144,51687,36407,54436,72873}', '{10,10,10,10,10,10,10,10,10,10,10,10,10,10}', '{8,2,2,6,7,4,6,1,1,5,6,7,6,2}', '{26.04,4.79,67.84,77.66,47.06,23.12,32.74,56.99,84.75,37.52,73.52,98.86,49.96,29.47}', 0.1800, 0.1100, 0.5000);
+EXECUTE su_after('{64833,63857,13941,76514,35858,10004,88553,34483,91251,28144,51687,36407,54436,72873}', '{10,10,10,10,10,10,10,10,10,10,10,10,10,10}', '{8,2,2,6,7,4,6,1,1,5,6,7,6,2}', '{26.04,4.79,67.84,77.66,47.06,23.12,32.74,56.99,84.75,37.52,73.52,98.86,49.96,29.47}', 0.1800, 0.1100, 0.5000);
+
+-- 2
+CREATE TABLE with_modifying.orders (o_id numeric NOT NULL, o_w_id numeric NOT NULL, o_d_id numeric NOT NULL, o_c_id numeric) WITH (fillfactor='50');
+CREATE UNIQUE INDEX orders_i2 ON with_modifying.orders USING btree (o_w_id, o_d_id, o_c_id, o_id) TABLESPACE pg_default;
+ALTER TABLE with_modifying.orders ADD CONSTRAINT orders_i1 PRIMARY KEY (o_w_id, o_d_id, o_id);
+CREATE TABLE with_modifying.order_line (ol_w_id numeric NOT NULL, ol_d_id numeric NOT NULL, ol_o_id numeric NOT NULL, ol_number numeric NOT NULL, ol_delivery_d timestamp without time zone, ol_amount numeric) WITH (fillfactor='50');
+ALTER TABLE with_modifying.order_line ADD CONSTRAINT order_line_i1 PRIMARY KEY (ol_w_id, ol_d_id, ol_o_id, ol_number);
+SELECT create_distributed_table('orders', 'o_w_id');
+SELECT create_distributed_table('order_line', 'ol_w_id');
+INSERT INTO orders VALUES (1, 1, 1, 1), (2, 2, 2, 2), (3, 3, 3, 3);
+INSERT INTO order_line VALUES (1, 1, 1, 10), (2, 2, 2, 20), (3, 3, 3, 30);
+PREPARE olu(int,int[],int[]) AS
+	WITH order_line_update AS (
+		UPDATE order_line
+		SET ol_delivery_d = current_timestamp
+		FROM UNNEST($2, $3) AS ids(o_id, d_id)
+		WHERE ol_o_id = ids.o_id
+			AND ol_d_id = ids.d_id
+			AND ol_w_id = $1
+		RETURNING ol_d_id, ol_o_id, ol_amount
+	)
+	SELECT array_agg(ol_d_id), array_agg(c_id), array_agg(sum_amount)
+	FROM (
+		SELECT ol_d_id,
+			(SELECT DISTINCT o_c_id FROM orders WHERE o_id = ol_o_id AND o_d_id = ol_d_id AND o_w_id = $1) AS c_id,
+			sum(ol_amount) AS sum_amount
+		FROM order_line_update
+		GROUP BY ol_d_id, ol_o_id
+	) AS inner_sum;
+EXECUTE olu(1,ARRAY[1,2],ARRAY[1,2]);
+EXECUTE olu(1,ARRAY[1,2],ARRAY[1,2]);
+EXECUTE olu(1,ARRAY[1,2],ARRAY[1,2]);
+EXECUTE olu(1,ARRAY[1,2],ARRAY[1,2]);
+EXECUTE olu(1,ARRAY[1,2],ARRAY[1,2]);
+EXECUTE olu(1,ARRAY[1,2],ARRAY[1,2]);
+
 -- Test with replication factor 2
 SET citus.shard_replication_factor to 2;
 
@@ -441,7 +521,23 @@ BEGIN;
 ROLLBACK;
 
 -- similarly, make sure that the intermediate result uses a seperate connection
- WITH first_query AS (INSERT INTO modify_table (id) VALUES (10001)),
+WITH first_query AS (INSERT INTO modify_table (id) VALUES (10001)),
  	second_query AS (SELECT * FROM modify_table) SELECT count(*) FROM second_query;
 
+SET client_min_messages TO debug2;
+-- pushed down without the insert
+WITH mb AS (UPDATE modify_table SET val = 3 WHERE id = 3 RETURNING NULL) INSERT INTO modify_table WITH ma AS (SELECT * FROM modify_table LIMIT 10) SELECT count(*) FROM mb;
+
+-- not pushed down due to volatile
+WITH ma AS (SELECT count(*) FROM modify_table where id = 1), mu AS (WITH allref AS (SELECT random() a FROM modify_table limit 4) UPDATE modify_table SET val = 3 WHERE id = 1 AND val IN (SELECT a FROM allref) RETURNING id+1) SELECT count(*) FROM mu, ma;
+WITH mu AS (WITH allref AS (SELECT random() a FROM anchor_table) UPDATE modify_table SET val = 3 WHERE id = 1 AND val IN (SELECT a FROM allref) RETURNING id+1) SELECT count(*) FROM mu;
+
+-- pushed down
+WITH mu AS (WITH allref AS (SELECT id a FROM anchor_table) UPDATE modify_table SET val = 3 WHERE id = 1 AND val IN (SELECT a FROM allref) RETURNING id+1) SELECT count(*) FROM mu;
+
+-- pushed down and stable function evaluated
+WITH mu AS (WITH allref AS (SELECT now() a FROM anchor_table) UPDATE modify_table SET val = 3 WHERE id = 1 AND now() IN (SELECT a FROM allref) RETURNING id+1) SELECT count(*) FROM mu;
+RESET client_min_messages;
+
+\set VERBOSITY terse
 DROP SCHEMA with_modifying CASCADE;

src/test/regress/sql/with_prepare.sql

@@ -198,6 +198,33 @@ FROM
 WHERE
 	events_user_id IN (SELECT user_id FROM users_table);
 
+-- Prepare routable modifying CTEs
+PREPARE prepared_test_7 AS
+WITH basic_delete AS (
+	DELETE FROM users_table WHERE user_id=6 RETURNING *
+)
+SELECT
+	*
+FROM
+	basic_delete
+ORDER BY
+	user_id,
+	time
+LIMIT 10;
+
+PREPARE prepared_test_8 AS
+WITH basic_delete AS (
+	UPDATE users_table SET value_1 = value_1 + 1 WHERE user_id=6 RETURNING *
+)
+SELECT
+	*
+FROM
+	basic_delete
+ORDER BY
+	user_id,
+	time
+LIMIT 10;
+
 EXECUTE prepared_test_1;
 EXECUTE prepared_test_1;
 EXECUTE prepared_test_1;
@@ -240,6 +267,40 @@ EXECUTE prepared_test_6;
 EXECUTE prepared_test_6;
 EXECUTE prepared_test_6;
 
+BEGIN;
+EXECUTE prepared_test_7;
+EXECUTE prepared_test_7;
+EXECUTE prepared_test_7;
+EXECUTE prepared_test_7;
+EXECUTE prepared_test_7;
+EXECUTE prepared_test_7;
+ROLLBACK;
+BEGIN;
+EXECUTE prepared_test_7;
+EXECUTE prepared_test_7;
+EXECUTE prepared_test_7;
+EXECUTE prepared_test_7;
+EXECUTE prepared_test_7;
+EXECUTE prepared_test_7;
+ROLLBACK;
+
+BEGIN;
+EXECUTE prepared_test_8;
+EXECUTE prepared_test_8;
+EXECUTE prepared_test_8;
+EXECUTE prepared_test_8;
+EXECUTE prepared_test_8;
+EXECUTE prepared_test_8;
+ROLLBACK;
+BEGIN;
+EXECUTE prepared_test_8;
+EXECUTE prepared_test_8;
+EXECUTE prepared_test_8;
+EXECUTE prepared_test_8;
+EXECUTE prepared_test_8;
+EXECUTE prepared_test_8;
+ROLLBACK;
+
 EXECUTE prepared_partition_column_insert(1);
 EXECUTE prepared_partition_column_insert(2);
 EXECUTE prepared_partition_column_insert(3);