From 2460e4768336b8751999db9acf033d2da5d8e529 Mon Sep 17 00:00:00 2001 From: Onder Kalaci Date: Mon, 30 Jan 2017 13:41:45 +0200 Subject: [PATCH] Add back partition column equality check --- .../planner/multi_router_planner.c | 77 ++++++- .../regress/expected/multi_insert_select.out | 194 +++++++++++------- src/test/regress/sql/multi_insert_select.sql | 49 +++-- 3 files changed, 234 insertions(+), 86 deletions(-) diff --git a/src/backend/distributed/planner/multi_router_planner.c b/src/backend/distributed/planner/multi_router_planner.c index 67f9c5c8d..ad5c92a04 100644 --- a/src/backend/distributed/planner/multi_router_planner.c +++ b/src/backend/distributed/planner/multi_router_planner.c @@ -78,6 +78,12 @@ typedef struct InstantiateQualWalker Var *relationPartitionColumn; }InstantiateQualWalker; +typedef struct PartitionColumnEqualityCheckWalker +{ + Query *subquery; + bool partitionColumnEqualityExists; +}PartitionColumnEqualityCheckWalker; + bool EnableRouterExecution = true; @@ -135,6 +141,7 @@ static DeferredErrorMessage * InsertPartitionColumnMatchesSelect(Query *query, Oid * selectPartitionColumnTableId); static void AddUninstantiatedEqualityQual(Query *query, Var *targetPartitionColumnVar); +static bool PartitionColumnEqualityWalker(Node *inputNode, void *context); static DeferredErrorMessage * ErrorIfQueryHasModifyingCTE(Query *queryTree); @@ -475,8 +482,7 @@ RouterModifyTaskForShardInterval(Query *originalQuery, ShardInterval *shardInter /* * It doesn't make sense that the anchor shard interval and target shard interval * have different ranges. This case is valid once original query already - * includes a partition column equality qual and there is a JOIN on the - * same column. + * includes a partition column equality qual. * * We actually could skip adding this check here since the subquery would return zero * rows given that we already have AddShardIntervalRestrictionToSelect(). However, by @@ -534,7 +540,7 @@ RouterModifyTaskForShardInterval(Query *originalQuery, ShardInterval *shardInter modifyTask = CreateBasicTask(jobId, taskIdIndex, MODIFY_TASK, queryString->data); modifyTask->dependedTaskList = NULL; modifyTask->anchorShardId = shardId; - modifyTask->taskPlacementList = FinalizedShardPlacementList(shardId); + modifyTask->taskPlacementList = insertShardPlacementList; modifyTask->upsertQuery = upsertQuery; modifyTask->relationShardList = relationShardList; modifyTask->replicationModel = cacheEntry->replicationModel; @@ -1197,6 +1203,22 @@ AddUninstantiatedEqualityQual(Query *query, Var *partitionColumn) Oid equalsOperator = InvalidOid; Oid greaterOperator = InvalidOid; bool hashable = false; + PartitionColumnEqualityCheckWalker *walker = + palloc0(sizeof(PartitionColumnEqualityCheckWalker)); + + AssertArg(query->commandType == CMD_SELECT); + + walker->partitionColumnEqualityExists = false; + walker->subquery = query; + + /* if the query already includes part_column = const, we don't need to add another */ + PartitionColumnEqualityWalker((Node *) query->jointree->quals, walker); + if (walker->partitionColumnEqualityExists) + { + ereport(DEBUG2, (errmsg("skipping to add uninstantiated equality qual"))); + + return; + } /* get the necessary equality operator */ get_sort_group_operators(partitionColumn->vartype, false, true, false, @@ -1239,6 +1261,53 @@ AddUninstantiatedEqualityQual(Query *query, Var *partitionColumn) } +/* + * PartitionColumnEqualityWalker walks over the quals of a join tree and checks + * whether the quals include any (partitionColumn = Const) expression. If so, context + * is marked accordingly and the iteration is terminated. + */ +static bool +PartitionColumnEqualityWalker(Node *inputNode, void *context) +{ + PartitionColumnEqualityCheckWalker *walker = + (PartitionColumnEqualityCheckWalker *) context; + + if (inputNode == NULL) + { + return false; + } + + if (IsA(inputNode, OpExpr) && list_length(((OpExpr *) inputNode)->args) == 2) + { + OpExpr *op = (OpExpr *) inputNode; + Node *leftop = get_leftop((Expr *) op); + Node *rightop = get_rightop((Expr *) op); + Var *column = NULL; + + if (IsA(leftop, Var) && IsA(rightop, Const) && + OperatorImplementsEquality(op->opno)) + { + column = (Var *) leftop; + } + else if (IsA(leftop, Const) && IsA(rightop, Var) && + OperatorImplementsEquality(op->opno)) + { + column = (Var *) rightop; + } + + if (column && IsPartitionColumn((Expr *) column, walker->subquery)) + { + walker->partitionColumnEqualityExists = true; + + return true; + } + } + + return expression_tree_walker(inputNode, PartitionColumnEqualityWalker, + (void *) context); +} + + /* * ModifyQuerySupported returns NULL if the query only contains supported * features, otherwise it returns an error description. @@ -3052,7 +3121,7 @@ InstantiatePartitionQual(Node *node, void *context) if (IsA(rightop, Var)) { - currentColumn = (Var *) rightop; + currentColumn = (Var *) rightop; } } else if (IsA(rightop, Param)) diff --git a/src/test/regress/expected/multi_insert_select.out b/src/test/regress/expected/multi_insert_select.out index 7ab6252a8..3aa760f50 100644 --- a/src/test/regress/expected/multi_insert_select.out +++ b/src/test/regress/expected/multi_insert_select.out @@ -197,25 +197,23 @@ INSERT INTO raw_events_second (user_id, time) SELECT user_id, time FROM raw_even DEBUG: StartTransactionCommand DEBUG: StartTransaction DEBUG: name: unnamed; blockState: DEFAULT; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +DEBUG: skipping to add uninstantiated equality qual DEBUG: predicate pruning for shardId 13300000 -DEBUG: predicate pruning for shardId 13300001 DEBUG: predicate pruning for shardId 13300002 DEBUG: predicate pruning for shardId 13300003 -DEBUG: Skipping target shard interval 13300004 since SELECT query for it pruned away +DEBUG: Skipping target shard interval 13300004 since it doesn't have the same shard range with the select anchor shard interval 13300001 DEBUG: predicate pruning for shardId 13300000 DEBUG: predicate pruning for shardId 13300002 DEBUG: predicate pruning for shardId 13300003 DEBUG: distributed statement: INSERT INTO public.raw_events_second_13300005 AS citus_table_alias (user_id, "time") SELECT user_id, "time" FROM public.raw_events_first_13300001 raw_events_first WHERE ((user_id = 7) AND ((hashint4(user_id) >= '-1073741824'::integer) AND (hashint4(user_id) <= '-1'::integer))) DEBUG: predicate pruning for shardId 13300000 -DEBUG: predicate pruning for shardId 13300001 DEBUG: predicate pruning for shardId 13300002 DEBUG: predicate pruning for shardId 13300003 -DEBUG: Skipping target shard interval 13300006 since SELECT query for it pruned away +DEBUG: Skipping target shard interval 13300006 since it doesn't have the same shard range with the select anchor shard interval 13300001 DEBUG: predicate pruning for shardId 13300000 -DEBUG: predicate pruning for shardId 13300001 DEBUG: predicate pruning for shardId 13300002 DEBUG: predicate pruning for shardId 13300003 -DEBUG: Skipping target shard interval 13300007 since SELECT query for it pruned away +DEBUG: Skipping target shard interval 13300007 since it doesn't have the same shard range with the select anchor shard interval 13300001 DEBUG: ProcessQuery DEBUG: Plan is router executable DEBUG: CommitTransactionCommand @@ -244,25 +242,23 @@ WHERE DEBUG: StartTransactionCommand DEBUG: StartTransaction DEBUG: name: unnamed; blockState: DEFAULT; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +DEBUG: skipping to add uninstantiated equality qual DEBUG: predicate pruning for shardId 13300001 DEBUG: predicate pruning for shardId 13300002 DEBUG: predicate pruning for shardId 13300003 DEBUG: distributed statement: INSERT INTO public.raw_events_second_13300004 AS citus_table_alias (user_id, "time", value_1, value_2, value_3, value_4) SELECT user_id, "time", value_1, value_2, value_3, value_4 FROM public.raw_events_first_13300000 raw_events_first WHERE ((user_id = 8) AND ((hashint4(user_id) >= '-2147483648'::integer) AND (hashint4(user_id) <= '-1073741825'::integer))) -DEBUG: predicate pruning for shardId 13300000 DEBUG: predicate pruning for shardId 13300001 DEBUG: predicate pruning for shardId 13300002 DEBUG: predicate pruning for shardId 13300003 -DEBUG: Skipping target shard interval 13300005 since SELECT query for it pruned away -DEBUG: predicate pruning for shardId 13300000 +DEBUG: Skipping target shard interval 13300005 since it doesn't have the same shard range with the select anchor shard interval 13300000 DEBUG: predicate pruning for shardId 13300001 DEBUG: predicate pruning for shardId 13300002 DEBUG: predicate pruning for shardId 13300003 -DEBUG: Skipping target shard interval 13300006 since SELECT query for it pruned away -DEBUG: predicate pruning for shardId 13300000 +DEBUG: Skipping target shard interval 13300006 since it doesn't have the same shard range with the select anchor shard interval 13300000 DEBUG: predicate pruning for shardId 13300001 DEBUG: predicate pruning for shardId 13300002 DEBUG: predicate pruning for shardId 13300003 -DEBUG: Skipping target shard interval 13300007 since SELECT query for it pruned away +DEBUG: Skipping target shard interval 13300007 since it doesn't have the same shard range with the select anchor shard interval 13300000 DEBUG: ProcessQuery DEBUG: Plan is router executable DEBUG: CommitTransactionCommand @@ -370,29 +366,11 @@ RETURNING *; DEBUG: StartTransactionCommand DEBUG: StartTransaction DEBUG: name: unnamed; blockState: DEFAULT; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: -DEBUG: predicate pruning for shardId 13300000 -DEBUG: predicate pruning for shardId 13300001 -DEBUG: predicate pruning for shardId 13300002 -DEBUG: predicate pruning for shardId 13300003 -DEBUG: Skipping target shard interval 13300004 since SELECT query for it pruned away +DEBUG: skipping to add uninstantiated equality qual DEBUG: predicate pruning for shardId 13300000 DEBUG: predicate pruning for shardId 13300002 -DEBUG: predicate pruning for shardId 13300003 -DEBUG: distributed statement: INSERT INTO public.raw_events_second_13300005 AS citus_table_alias (user_id, value_1, value_3) SELECT user_id, value_1, value_3 FROM public.raw_events_first_13300001 raw_events_first WHERE (((user_id = 9) OR (user_id = 16)) AND ((hashint4(user_id) >= '-1073741824'::integer) AND (hashint4(user_id) <= '-1'::integer))) RETURNING citus_table_alias.user_id, citus_table_alias."time", citus_table_alias.value_1, citus_table_alias.value_2, citus_table_alias.value_3, citus_table_alias.value_4 -DEBUG: predicate pruning for shardId 13300000 -DEBUG: predicate pruning for shardId 13300001 -DEBUG: predicate pruning for shardId 13300002 -DEBUG: predicate pruning for shardId 13300003 -DEBUG: Skipping target shard interval 13300006 since SELECT query for it pruned away -DEBUG: predicate pruning for shardId 13300000 -DEBUG: predicate pruning for shardId 13300001 -DEBUG: predicate pruning for shardId 13300002 -DEBUG: distributed statement: INSERT INTO public.raw_events_second_13300007 AS citus_table_alias (user_id, value_1, value_3) SELECT user_id, value_1, value_3 FROM public.raw_events_first_13300003 raw_events_first WHERE (((user_id = 9) OR (user_id = 16)) AND ((hashint4(user_id) >= 1073741824) AND (hashint4(user_id) <= 2147483647))) RETURNING citus_table_alias.user_id, citus_table_alias."time", citus_table_alias.value_1, citus_table_alias.value_2, citus_table_alias.value_3, citus_table_alias.value_4 -DEBUG: ProcessQuery -DEBUG: Plan is router executable -ERROR: duplicate key value violates unique constraint "raw_events_second_user_id_value_1_key_13300007" -DETAIL: Key (user_id, value_1)=(9, 90) already exists. -CONTEXT: while executing command on localhost:57638 +ERROR: cannot perform distributed planning for the given modification +DETAIL: Select query cannot be pushed down to the worker. -- now do some aggregations INSERT INTO agg_events SELECT @@ -1226,6 +1204,7 @@ FROM DEBUG: StartTransactionCommand DEBUG: StartTransaction DEBUG: name: unnamed; blockState: DEFAULT; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +DEBUG: skipping to add uninstantiated equality qual DEBUG: predicate pruning for shardId 13300001 DEBUG: predicate pruning for shardId 13300002 DEBUG: predicate pruning for shardId 13300003 @@ -1236,7 +1215,6 @@ DEBUG: distributed statement: INSERT INTO public.agg_events_13300008 AS citus_t DEBUG: predicate pruning for shardId 13300001 DEBUG: predicate pruning for shardId 13300002 DEBUG: predicate pruning for shardId 13300003 -DEBUG: predicate pruning for shardId 13300004 DEBUG: predicate pruning for shardId 13300005 DEBUG: predicate pruning for shardId 13300006 DEBUG: predicate pruning for shardId 13300007 @@ -1244,7 +1222,6 @@ DEBUG: Skipping target shard interval 13300009 since it doesn't have the same s DEBUG: predicate pruning for shardId 13300001 DEBUG: predicate pruning for shardId 13300002 DEBUG: predicate pruning for shardId 13300003 -DEBUG: predicate pruning for shardId 13300004 DEBUG: predicate pruning for shardId 13300005 DEBUG: predicate pruning for shardId 13300006 DEBUG: predicate pruning for shardId 13300007 @@ -1252,7 +1229,6 @@ DEBUG: Skipping target shard interval 13300010 since it doesn't have the same s DEBUG: predicate pruning for shardId 13300001 DEBUG: predicate pruning for shardId 13300002 DEBUG: predicate pruning for shardId 13300003 -DEBUG: predicate pruning for shardId 13300004 DEBUG: predicate pruning for shardId 13300005 DEBUG: predicate pruning for shardId 13300006 DEBUG: predicate pruning for shardId 13300007 @@ -1271,6 +1247,7 @@ FROM DEBUG: StartTransactionCommand DEBUG: StartTransaction DEBUG: name: unnamed; blockState: DEFAULT; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +DEBUG: skipping to add uninstantiated equality qual DEBUG: predicate pruning for shardId 13300001 DEBUG: predicate pruning for shardId 13300002 DEBUG: predicate pruning for shardId 13300003 @@ -1313,6 +1290,7 @@ FROM DEBUG: StartTransactionCommand DEBUG: StartTransaction DEBUG: name: unnamed; blockState: DEFAULT; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +DEBUG: skipping to add uninstantiated equality qual DEBUG: Skipping target shard interval 13300008 since SELECT query for it pruned away DEBUG: Skipping target shard interval 13300009 since SELECT query for it pruned away DEBUG: Skipping target shard interval 13300010 since SELECT query for it pruned away @@ -1331,6 +1309,7 @@ FROM DEBUG: StartTransactionCommand DEBUG: StartTransaction DEBUG: name: unnamed; blockState: DEFAULT; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +DEBUG: skipping to add uninstantiated equality qual NOTICE: cannot use shard pruning with ANY/ALL (array expression) HINT: Consider rewriting the expression with OR/AND clauses. DEBUG: predicate pruning for shardId 13300001 @@ -1345,7 +1324,6 @@ HINT: Consider rewriting the expression with OR/AND clauses. DEBUG: predicate pruning for shardId 13300001 DEBUG: predicate pruning for shardId 13300002 DEBUG: predicate pruning for shardId 13300003 -DEBUG: predicate pruning for shardId 13300004 DEBUG: predicate pruning for shardId 13300005 DEBUG: predicate pruning for shardId 13300006 DEBUG: predicate pruning for shardId 13300007 @@ -1355,7 +1333,6 @@ HINT: Consider rewriting the expression with OR/AND clauses. DEBUG: predicate pruning for shardId 13300001 DEBUG: predicate pruning for shardId 13300002 DEBUG: predicate pruning for shardId 13300003 -DEBUG: predicate pruning for shardId 13300004 DEBUG: predicate pruning for shardId 13300005 DEBUG: predicate pruning for shardId 13300006 DEBUG: predicate pruning for shardId 13300007 @@ -1365,7 +1342,6 @@ HINT: Consider rewriting the expression with OR/AND clauses. DEBUG: predicate pruning for shardId 13300001 DEBUG: predicate pruning for shardId 13300002 DEBUG: predicate pruning for shardId 13300003 -DEBUG: predicate pruning for shardId 13300004 DEBUG: predicate pruning for shardId 13300005 DEBUG: predicate pruning for shardId 13300006 DEBUG: predicate pruning for shardId 13300007 @@ -1384,6 +1360,7 @@ FROM DEBUG: StartTransactionCommand DEBUG: StartTransaction DEBUG: name: unnamed; blockState: DEFAULT; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +DEBUG: skipping to add uninstantiated equality qual DEBUG: predicate pruning for shardId 13300001 DEBUG: predicate pruning for shardId 13300002 DEBUG: predicate pruning for shardId 13300003 @@ -1425,6 +1402,53 @@ DEBUG: Plan is router executable DEBUG: CommitTransactionCommand DEBUG: CommitTransaction DEBUG: name: unnamed; blockState: STARTED; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +-- the following is a very tricky query for Citus +-- although we do not support pushing down JOINs on non-partition +-- columns here it is safe to push it down given that we're looking for +-- a specific value (i.e., value_1 = 12) on the joning column. +INSERT INTO agg_events + (user_id) +SELECT raw_events_first.user_id +FROM raw_events_first, + raw_events_second +WHERE raw_events_second.user_id = raw_events_first.value_1 + AND raw_events_first.value_1 = 12; +DEBUG: StartTransactionCommand +DEBUG: StartTransaction +DEBUG: name: unnamed; blockState: DEFAULT; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 +DEBUG: predicate pruning for shardId 13300004 +DEBUG: predicate pruning for shardId 13300005 +DEBUG: predicate pruning for shardId 13300006 +DEBUG: distributed statement: INSERT INTO public.agg_events_13300008 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM public.raw_events_first_13300000 raw_events_first, public.raw_events_second_13300007 raw_events_second WHERE (((raw_events_second.user_id = raw_events_first.value_1) AND (raw_events_first.value_1 = 12)) AND ((hashint4(raw_events_first.user_id) >= '-2147483648'::integer) AND (hashint4(raw_events_first.user_id) <= '-1073741825'::integer))) +DEBUG: predicate pruning for shardId 13300000 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 +DEBUG: predicate pruning for shardId 13300004 +DEBUG: predicate pruning for shardId 13300005 +DEBUG: predicate pruning for shardId 13300006 +DEBUG: distributed statement: INSERT INTO public.agg_events_13300009 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM public.raw_events_first_13300001 raw_events_first, public.raw_events_second_13300007 raw_events_second WHERE (((raw_events_second.user_id = raw_events_first.value_1) AND (raw_events_first.value_1 = 12)) AND ((hashint4(raw_events_first.user_id) >= '-1073741824'::integer) AND (hashint4(raw_events_first.user_id) <= '-1'::integer))) +DEBUG: predicate pruning for shardId 13300000 +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300003 +DEBUG: predicate pruning for shardId 13300004 +DEBUG: predicate pruning for shardId 13300005 +DEBUG: predicate pruning for shardId 13300006 +DEBUG: distributed statement: INSERT INTO public.agg_events_13300010 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM public.raw_events_first_13300002 raw_events_first, public.raw_events_second_13300007 raw_events_second WHERE (((raw_events_second.user_id = raw_events_first.value_1) AND (raw_events_first.value_1 = 12)) AND ((hashint4(raw_events_first.user_id) >= 0) AND (hashint4(raw_events_first.user_id) <= 1073741823))) +DEBUG: predicate pruning for shardId 13300000 +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300004 +DEBUG: predicate pruning for shardId 13300005 +DEBUG: predicate pruning for shardId 13300006 +DEBUG: distributed statement: INSERT INTO public.agg_events_13300011 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM public.raw_events_first_13300003 raw_events_first, public.raw_events_second_13300007 raw_events_second WHERE (((raw_events_second.user_id = raw_events_first.value_1) AND (raw_events_first.value_1 = 12)) AND ((hashint4(raw_events_first.user_id) >= 1073741824) AND (hashint4(raw_events_first.user_id) <= 2147483647))) +DEBUG: ProcessQuery +DEBUG: Plan is router executable +DEBUG: CommitTransactionCommand +DEBUG: CommitTransaction +DEBUG: name: unnamed; blockState: STARTED; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: -- some unsupported LEFT/INNER JOINs -- JOIN on one table with partition column other is not INSERT INTO agg_events (user_id) @@ -1484,19 +1508,38 @@ ERROR: cannot perform distributed planning for the given modification DETAIL: Select query cannot be pushed down to the worker. -- JOIN on one table with partition column other is not -- also an equality qual is added on the partition key ---INSERT INTO agg_events (user_id) ---SELECT --- raw_events_first.user_id ---FROM --- raw_events_first LEFT JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.value_1 --- WHERE raw_events_first.user_id = 10; +INSERT INTO agg_events (user_id) +SELECT + raw_events_first.user_id +FROM + raw_events_first LEFT JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.value_1 +WHERE + raw_events_first.user_id = 10; +DEBUG: StartTransactionCommand +DEBUG: StartTransaction +DEBUG: name: unnamed; blockState: DEFAULT; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +DEBUG: skipping to add uninstantiated equality qual +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 +ERROR: cannot perform distributed planning for the given modification +DETAIL: Select query cannot be pushed down to the worker. -- same as the above with INNER JOIN ---INSERT INTO agg_events (user_id) ---SELECT --- raw_events_first.user_id ---FROM --- raw_events_first INNER JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.value_1 --- WHERE raw_events_first.user_id = 10; +INSERT INTO agg_events (user_id) +SELECT + raw_events_first.user_id +FROM + raw_events_first INNER JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.value_1 +WHERE raw_events_first.user_id = 10; +DEBUG: StartTransactionCommand +DEBUG: StartTransaction +DEBUG: name: unnamed; blockState: DEFAULT; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +DEBUG: skipping to add uninstantiated equality qual +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 +ERROR: cannot perform distributed planning for the given modification +DETAIL: Select query cannot be pushed down to the worker. -- make things a bit more complicate with IN clauses INSERT INTO agg_events (user_id) SELECT @@ -1527,6 +1570,24 @@ DEBUG: predicate pruning for shardId 13300002 DEBUG: predicate pruning for shardId 13300003 ERROR: cannot perform distributed planning for the given modification DETAIL: Select query cannot be pushed down to the worker. +-- the following is again a very tricky query for Citus +-- if the given filter was on value_1 as shown in the above, Citus could +-- push it down. But here the query is refused +INSERT INTO agg_events + (user_id) +SELECT raw_events_first.user_id +FROM raw_events_first, + raw_events_second +WHERE raw_events_second.user_id = raw_events_first.value_1 + AND raw_events_first.value_2 = 12; +DEBUG: StartTransactionCommand +DEBUG: StartTransaction +DEBUG: name: unnamed; blockState: DEFAULT; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 +ERROR: cannot perform distributed planning for the given modification +DETAIL: Select query cannot be pushed down to the worker. -- unsupported JOIN INSERT INTO agg_events (value_4_agg, @@ -1903,25 +1964,23 @@ INSERT INTO raw_events_first SELECT * FROM raw_events_second WHERE user_id = 5; DEBUG: StartTransactionCommand DEBUG: StartTransaction DEBUG: name: unnamed; blockState: DEFAULT; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +DEBUG: skipping to add uninstantiated equality qual DEBUG: predicate pruning for shardId 13300005 DEBUG: predicate pruning for shardId 13300006 DEBUG: predicate pruning for shardId 13300007 DEBUG: distributed statement: INSERT INTO public.raw_events_first_13300000 AS citus_table_alias (user_id, "time", value_1, value_2, value_3, value_4) SELECT user_id, "time", value_1, value_2, value_3, value_4 FROM public.raw_events_second_13300004 raw_events_second WHERE ((user_id = 5) AND ((hashint4(user_id) >= '-2147483648'::integer) AND (hashint4(user_id) <= '-1073741825'::integer))) -DEBUG: predicate pruning for shardId 13300004 DEBUG: predicate pruning for shardId 13300005 DEBUG: predicate pruning for shardId 13300006 DEBUG: predicate pruning for shardId 13300007 -DEBUG: Skipping target shard interval 13300001 since SELECT query for it pruned away -DEBUG: predicate pruning for shardId 13300004 +DEBUG: Skipping target shard interval 13300001 since it doesn't have the same shard range with the select anchor shard interval 13300004 DEBUG: predicate pruning for shardId 13300005 DEBUG: predicate pruning for shardId 13300006 DEBUG: predicate pruning for shardId 13300007 -DEBUG: Skipping target shard interval 13300002 since SELECT query for it pruned away -DEBUG: predicate pruning for shardId 13300004 +DEBUG: Skipping target shard interval 13300002 since it doesn't have the same shard range with the select anchor shard interval 13300004 DEBUG: predicate pruning for shardId 13300005 DEBUG: predicate pruning for shardId 13300006 DEBUG: predicate pruning for shardId 13300007 -DEBUG: Skipping target shard interval 13300003 since SELECT query for it pruned away +DEBUG: Skipping target shard interval 13300003 since it doesn't have the same shard range with the select anchor shard interval 13300004 DEBUG: ProcessQuery DEBUG: Plan is router executable DEBUG: CommitTransactionCommand @@ -1954,6 +2013,7 @@ INSERT INTO raw_events_first SELECT * FROM raw_events_second WHERE user_id = 5; DEBUG: StartTransactionCommand DEBUG: StartTransaction DEBUG: name: unnamed; blockState: DEFAULT; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +DEBUG: skipping to add uninstantiated equality qual DEBUG: predicate pruning for shardId 13300005 DEBUG: predicate pruning for shardId 13300006 DEBUG: predicate pruning for shardId 13300007 @@ -1964,25 +2024,23 @@ INSERT INTO raw_events_first SELECT * FROM raw_events_second WHERE user_id = 6; DEBUG: StartTransactionCommand DEBUG: StartTransaction DEBUG: name: unnamed; blockState: DEFAULT; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +DEBUG: skipping to add uninstantiated equality qual DEBUG: predicate pruning for shardId 13300004 DEBUG: predicate pruning for shardId 13300005 -DEBUG: predicate pruning for shardId 13300006 DEBUG: predicate pruning for shardId 13300007 -DEBUG: Skipping target shard interval 13300000 since SELECT query for it pruned away +DEBUG: Skipping target shard interval 13300000 since it doesn't have the same shard range with the select anchor shard interval 13300006 DEBUG: predicate pruning for shardId 13300004 DEBUG: predicate pruning for shardId 13300005 -DEBUG: predicate pruning for shardId 13300006 DEBUG: predicate pruning for shardId 13300007 -DEBUG: Skipping target shard interval 13300001 since SELECT query for it pruned away +DEBUG: Skipping target shard interval 13300001 since it doesn't have the same shard range with the select anchor shard interval 13300006 DEBUG: predicate pruning for shardId 13300004 DEBUG: predicate pruning for shardId 13300005 DEBUG: predicate pruning for shardId 13300007 DEBUG: distributed statement: INSERT INTO public.raw_events_first_13300002 AS citus_table_alias (user_id, "time", value_1, value_2, value_3, value_4) SELECT user_id, "time", value_1, value_2, value_3, value_4 FROM public.raw_events_second_13300006 raw_events_second WHERE ((user_id = 6) AND ((hashint4(user_id) >= 0) AND (hashint4(user_id) <= 1073741823))) DEBUG: predicate pruning for shardId 13300004 DEBUG: predicate pruning for shardId 13300005 -DEBUG: predicate pruning for shardId 13300006 DEBUG: predicate pruning for shardId 13300007 -DEBUG: Skipping target shard interval 13300003 since SELECT query for it pruned away +DEBUG: Skipping target shard interval 13300003 since it doesn't have the same shard range with the select anchor shard interval 13300006 DEBUG: ProcessQuery DEBUG: Plan is router executable DEBUG: CommitTransactionCommand @@ -2043,25 +2101,23 @@ INSERT INTO raw_events_first SELECT * FROM raw_events_second WHERE user_id = 5; DEBUG: StartTransactionCommand DEBUG: StartTransaction DEBUG: name: unnamed; blockState: DEFAULT; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +DEBUG: skipping to add uninstantiated equality qual DEBUG: predicate pruning for shardId 13300005 DEBUG: predicate pruning for shardId 13300006 DEBUG: predicate pruning for shardId 13300007 DEBUG: distributed statement: INSERT INTO public.raw_events_first_13300000 AS citus_table_alias (user_id, "time", value_1, value_2, value_3, value_4) SELECT user_id, "time", value_1, value_2, value_3, value_4 FROM public.raw_events_second_13300004 raw_events_second WHERE ((user_id = 5) AND ((hashint4(user_id) >= '-2147483648'::integer) AND (hashint4(user_id) <= '-1073741825'::integer))) -DEBUG: predicate pruning for shardId 13300004 DEBUG: predicate pruning for shardId 13300005 DEBUG: predicate pruning for shardId 13300006 DEBUG: predicate pruning for shardId 13300007 -DEBUG: Skipping target shard interval 13300001 since SELECT query for it pruned away -DEBUG: predicate pruning for shardId 13300004 +DEBUG: Skipping target shard interval 13300001 since it doesn't have the same shard range with the select anchor shard interval 13300004 DEBUG: predicate pruning for shardId 13300005 DEBUG: predicate pruning for shardId 13300006 DEBUG: predicate pruning for shardId 13300007 -DEBUG: Skipping target shard interval 13300002 since SELECT query for it pruned away -DEBUG: predicate pruning for shardId 13300004 +DEBUG: Skipping target shard interval 13300002 since it doesn't have the same shard range with the select anchor shard interval 13300004 DEBUG: predicate pruning for shardId 13300005 DEBUG: predicate pruning for shardId 13300006 DEBUG: predicate pruning for shardId 13300007 -DEBUG: Skipping target shard interval 13300003 since SELECT query for it pruned away +DEBUG: Skipping target shard interval 13300003 since it doesn't have the same shard range with the select anchor shard interval 13300004 DEBUG: ProcessQuery DEBUG: Plan is router executable DEBUG: CommitTransactionCommand diff --git a/src/test/regress/sql/multi_insert_select.sql b/src/test/regress/sql/multi_insert_select.sql index 342382ff7..51ec8ff85 100644 --- a/src/test/regress/sql/multi_insert_select.sql +++ b/src/test/regress/sql/multi_insert_select.sql @@ -539,8 +539,19 @@ FROM raw_events_first INNER JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.user_id WHERE raw_events_first.user_id = 10 AND raw_events_second.user_id IN (19, 20, 21); --- some unsupported LEFT/INNER JOINs +-- the following is a very tricky query for Citus +-- although we do not support pushing down JOINs on non-partition +-- columns here it is safe to push it down given that we're looking for +-- a specific value (i.e., value_1 = 12) on the joning column. +INSERT INTO agg_events + (user_id) +SELECT raw_events_first.user_id +FROM raw_events_first, + raw_events_second +WHERE raw_events_second.user_id = raw_events_first.value_1 + AND raw_events_first.value_1 = 12; +-- some unsupported LEFT/INNER JOINs -- JOIN on one table with partition column other is not INSERT INTO agg_events (user_id) SELECT @@ -571,20 +582,21 @@ FROM -- JOIN on one table with partition column other is not -- also an equality qual is added on the partition key ---INSERT INTO agg_events (user_id) ---SELECT --- raw_events_first.user_id ---FROM --- raw_events_first LEFT JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.value_1 --- WHERE raw_events_first.user_id = 10; +INSERT INTO agg_events (user_id) +SELECT + raw_events_first.user_id +FROM + raw_events_first LEFT JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.value_1 +WHERE + raw_events_first.user_id = 10; -- same as the above with INNER JOIN ---INSERT INTO agg_events (user_id) ---SELECT --- raw_events_first.user_id ---FROM --- raw_events_first INNER JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.value_1 --- WHERE raw_events_first.user_id = 10; +INSERT INTO agg_events (user_id) +SELECT + raw_events_first.user_id +FROM + raw_events_first INNER JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.value_1 +WHERE raw_events_first.user_id = 10; -- make things a bit more complicate with IN clauses INSERT INTO agg_events (user_id) @@ -602,6 +614,17 @@ FROM raw_events_first, raw_events_second WHERE raw_events_second.user_id = raw_events_first.value_1; +-- the following is again a very tricky query for Citus +-- if the given filter was on value_1 as shown in the above, Citus could +-- push it down. But here the query is refused +INSERT INTO agg_events + (user_id) +SELECT raw_events_first.user_id +FROM raw_events_first, + raw_events_second +WHERE raw_events_second.user_id = raw_events_first.value_1 + AND raw_events_first.value_2 = 12; + -- unsupported JOIN INSERT INTO agg_events (value_4_agg,