diff --git a/src/backend/distributed/planner/multi_router_planner.c b/src/backend/distributed/planner/multi_router_planner.c index 69b057b93..120d74585 100644 --- a/src/backend/distributed/planner/multi_router_planner.c +++ b/src/backend/distributed/planner/multi_router_planner.c @@ -48,6 +48,8 @@ #include "nodes/pg_list.h" #include "nodes/primnodes.h" #include "optimizer/clauses.h" +#include "optimizer/paths.h" +#include "optimizer/pathnode.h" #include "optimizer/predtest.h" #include "optimizer/restrictinfo.h" #include "optimizer/var.h" @@ -72,6 +74,13 @@ typedef struct WalkerState bool badCoalesce; } WalkerState; +typedef struct InstantiateQualWalker +{ + ShardInterval *targetShardInterval; + Var *relationPartitionColumn; +}InstantiateQualWalker; + + bool EnableRouterExecution = true; /* planner functions forward declarations */ @@ -87,6 +96,8 @@ static Task * RouterModifyTaskForShardInterval(Query *originalQuery, RelationRestrictionContext * restrictionContext, uint32 taskIdIndex); +static List * GetAllActualClausesForRelation(RelOptInfo *relInfo, + PlannerInfo *plannerInfo); static bool MasterIrreducibleExpression(Node *expression, bool *varArgument, bool *badCoalesce); static bool MasterIrreducibleExpressionWalker(Node *expression, WalkerState *state); @@ -377,6 +388,10 @@ RouterModifyTaskForShardInterval(Query *originalQuery, ShardInterval *shardInter { RelationRestriction *restriction = lfirst(restrictionCell); List *originalBaserestrictInfo = restriction->relOptInfo->baserestrictinfo; + List *originalJoinInfo = restriction->relOptInfo->joininfo; + InstantiateQualWalker *instantiateQualWalker = palloc0( + sizeof(InstantiateQualWalker)); + Var *relationPartitionKey = PartitionKey(restriction->relationId); /* * We haven't added the quals if all participating tables are reference @@ -387,9 +402,15 @@ RouterModifyTaskForShardInterval(Query *originalQuery, ShardInterval *shardInter break; } + instantiateQualWalker->relationPartitionColumn = relationPartitionKey; + instantiateQualWalker->targetShardInterval = shardInterval; + originalBaserestrictInfo = (List *) InstantiatePartitionQual((Node *) originalBaserestrictInfo, - shardInterval); + instantiateQualWalker); + originalJoinInfo = + (List *) InstantiatePartitionQual((Node *) originalJoinInfo, + instantiateQualWalker); } /* @@ -454,7 +475,6 @@ RouterModifyTaskForShardInterval(Query *originalQuery, ShardInterval *shardInter "for shard %ld", shardId))); } - /* this is required for correct deparsing of the query */ ReorderInsertSelectTargetLists(copiedQuery, copiedInsertRte, copiedSubqueryRte); @@ -488,6 +508,88 @@ RouterModifyTaskForShardInterval(Query *originalQuery, ShardInterval *shardInter } +/* + * GetAllActualClausesForRelation returns all the restrictions that a relation + * has including the baserestrictinfo and the related parts of joininfo. + * + * This function is heavily inspired by check_index_predicates() from the PostgreSQL's + * source file src/backend/optimizer/path/indexpath.c. + */ +static List * +GetAllActualClausesForRelation(RelOptInfo *relInfo, PlannerInfo *plannerInfo) +{ + List *allRestrictInfos = NULL; + ListCell *joinInfoCell = NULL; + Relids otherRelIds = NULL; + List *allActualClauses = NIL; + + /* we should definitely add all the entries in baserestrictinfo */ + allRestrictInfos = relInfo->baserestrictinfo; + + /* Scan the rel's join clauses and get the necessary ones */ + foreach(joinInfoCell, relInfo->joininfo) + { + RestrictInfo *restrictInfo = (RestrictInfo *) lfirst(joinInfoCell); + List *columnList = pull_var_clause_default((Node *) restrictInfo->clause); + + /* check if the restrict info already contains a hashed restrict info */ + if (list_length(columnList) > 0) + { + Var *firstColumn = linitial(columnList); + Var *hashedColumn = MakeInt4Column(); + + if (equal(firstColumn, hashedColumn)) + { + /* + * This means that we've already instantiated the restrict info + * so this should be included in the final restrict info list. + */ + allRestrictInfos = lappend(allRestrictInfos, restrictInfo); + continue; + } + } + + /* check if clause can be moved to this rel */ + if (join_clause_is_movable_to(restrictInfo, relInfo)) + { + allRestrictInfos = lappend(allRestrictInfos, restrictInfo); + } + } + + /* + * Add on any equivalence-derivable join clauses. Computing the correct + * relid sets for generate_join_implied_equalities is slightly tricky + * because the rel could be a child rel rather than a true baserel, and in + * that case we must remove its parents' relid(s) from all_baserels. + */ + if (relInfo->reloptkind == RELOPT_OTHER_MEMBER_REL) + { + otherRelIds = bms_difference(plannerInfo->all_baserels, + find_childrel_parents(plannerInfo, relInfo)); + } + else + { + otherRelIds = bms_difference(plannerInfo->all_baserels, relInfo->relids); + } + + if (!bms_is_empty(otherRelIds)) + { + List *impliedJoinEqualities = generate_join_implied_equalities(plannerInfo, + bms_union( + relInfo->relids, + otherRelIds), + otherRelIds, + relInfo); + allRestrictInfos = list_concat(allRestrictInfos, impliedJoinEqualities); + } + + /* finally get the actual clauses from the restrict infos */ + allActualClauses = get_all_actual_clauses(allRestrictInfos); + + return allActualClauses; +} + + /* * AddShardIntervalRestrictionToSelect adds the following range boundaries * with the given subquery and shardInterval: @@ -2395,8 +2497,6 @@ TargetShardIntervalsForSelect(Query *query, Index tableId = relationRestriction->index; DistTableCacheEntry *cacheEntry = DistributedTableCacheEntry(relationId); int shardCount = cacheEntry->shardIntervalArrayLength; - List *baseRestrictionList = relationRestriction->relOptInfo->baserestrictinfo; - List *restrictClauseList = get_all_actual_clauses(baseRestrictionList); List *prunedShardList = NIL; int shardIndex = 0; List *joinInfoList = relationRestriction->relOptInfo->joininfo; @@ -2415,6 +2515,7 @@ TargetShardIntervalsForSelect(Query *query, if (!whereFalseQuery && shardCount > 0) { List *shardIntervalList = NIL; + List *allActualClauses = NIL; for (shardIndex = 0; shardIndex < shardCount; shardIndex++) { @@ -2423,8 +2524,12 @@ TargetShardIntervalsForSelect(Query *query, shardIntervalList = lappend(shardIntervalList, shardInterval); } + allActualClauses = + GetAllActualClausesForRelation(relationRestriction->relOptInfo, + relationRestriction->plannerInfo); + prunedShardList = PruneShardList(relationId, tableId, - restrictClauseList, + allActualClauses, shardIntervalList); /* @@ -2950,7 +3055,10 @@ CopyRelationRestrictionContext(RelationRestrictionContext *oldContext) static Node * InstantiatePartitionQual(Node *node, void *context) { - ShardInterval *shardInterval = (ShardInterval *) context; + InstantiateQualWalker *instantiateContext = ((InstantiateQualWalker *) context); + ShardInterval *shardInterval = instantiateContext->targetShardInterval; + Var *relationPartitionColumn = instantiateContext->relationPartitionColumn; + Assert(shardInterval->minValueExists); Assert(shardInterval->maxValueExists); @@ -2974,6 +3082,7 @@ InstantiatePartitionQual(Node *node, void *context) Node *leftop = get_leftop((Expr *) op); Node *rightop = get_rightop((Expr *) op); Param *param = NULL; + Var *currentColumn = NULL; Var *hashedGEColumn = NULL; OpExpr *hashedGEOpExpr = NULL; @@ -2992,10 +3101,20 @@ InstantiatePartitionQual(Node *node, void *context) if (IsA(leftop, Param)) { param = (Param *) leftop; + + if (IsA(rightop, Var)) + { + currentColumn = (Var *) rightop; + } } else if (IsA(rightop, Param)) { param = (Param *) rightop; + + if (IsA(leftop, Var)) + { + currentColumn = (Var *) leftop; + } } /* not an interesting param for our purpose, so return */ @@ -3004,6 +3123,13 @@ InstantiatePartitionQual(Node *node, void *context) return node; } + /* if the qual is not on the partition column, do not instantiate */ + if (relationPartitionColumn && currentColumn && + currentColumn->varattno != relationPartitionColumn->varattno) + { + return node; + } + /* get the integer >=, <= operators from the catalog */ integer4GEoperatorId = get_opfamily_member(INTEGER_BTREE_FAM_OID, INT4OID, INT4OID, diff --git a/src/test/regress/expected/multi_insert_select.out b/src/test/regress/expected/multi_insert_select.out index f28f6f472..9760a1282 100644 --- a/src/test/regress/expected/multi_insert_select.out +++ b/src/test/regress/expected/multi_insert_select.out @@ -719,24 +719,30 @@ DEBUG: name: unnamed; blockState: DEFAULT; state: INPROGR, xid/subid/cid: DEBUG: predicate pruning for shardId 13300000 DEBUG: predicate pruning for shardId 13300001 DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 DEBUG: predicate pruning for shardId 13300004 DEBUG: predicate pruning for shardId 13300005 DEBUG: predicate pruning for shardId 13300006 -DEBUG: distributed statement: INSERT INTO public.raw_events_second_13300004 AS citus_table_alias (user_id) SELECT user_id FROM public.raw_events_first_13300003 raw_events_first WHERE ((user_id IN (SELECT raw_events_second.user_id FROM public.raw_events_second_13300007 raw_events_second WHERE (raw_events_second.user_id = 2))) AND ((hashint4(user_id) >= '-2147483648'::integer) AND (hashint4(user_id) <= '-1073741825'::integer))) +DEBUG: predicate pruning for shardId 13300007 +DEBUG: Skipping target shard interval 13300004 since SELECT query for it pruned away DEBUG: predicate pruning for shardId 13300000 DEBUG: predicate pruning for shardId 13300001 DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 DEBUG: predicate pruning for shardId 13300004 DEBUG: predicate pruning for shardId 13300005 DEBUG: predicate pruning for shardId 13300006 -DEBUG: distributed statement: INSERT INTO public.raw_events_second_13300005 AS citus_table_alias (user_id) SELECT user_id FROM public.raw_events_first_13300003 raw_events_first WHERE ((user_id IN (SELECT raw_events_second.user_id FROM public.raw_events_second_13300007 raw_events_second WHERE (raw_events_second.user_id = 2))) AND ((hashint4(user_id) >= '-1073741824'::integer) AND (hashint4(user_id) <= '-1'::integer))) +DEBUG: predicate pruning for shardId 13300007 +DEBUG: Skipping target shard interval 13300005 since SELECT query for it pruned away DEBUG: predicate pruning for shardId 13300000 DEBUG: predicate pruning for shardId 13300001 DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 DEBUG: predicate pruning for shardId 13300004 DEBUG: predicate pruning for shardId 13300005 DEBUG: predicate pruning for shardId 13300006 -DEBUG: distributed statement: INSERT INTO public.raw_events_second_13300006 AS citus_table_alias (user_id) SELECT user_id FROM public.raw_events_first_13300003 raw_events_first WHERE ((user_id IN (SELECT raw_events_second.user_id FROM public.raw_events_second_13300007 raw_events_second WHERE (raw_events_second.user_id = 2))) AND ((hashint4(user_id) >= 0) AND (hashint4(user_id) <= 1073741823))) +DEBUG: predicate pruning for shardId 13300007 +DEBUG: Skipping target shard interval 13300006 since SELECT query for it pruned away DEBUG: predicate pruning for shardId 13300000 DEBUG: predicate pruning for shardId 13300001 DEBUG: predicate pruning for shardId 13300002 @@ -1175,6 +1181,537 @@ DEBUG: StartTransactionCommand DEBUG: StartTransaction DEBUG: name: unnamed; blockState: DEFAULT; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: ERROR: set operations are not allowed in INSERT ... SELECT queries +-- some supported LEFT JOINs +INSERT INTO agg_events (user_id) +SELECT + raw_events_first.user_id +FROM + raw_events_first LEFT JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.user_id; +DEBUG: StartTransactionCommand +DEBUG: StartTransaction +DEBUG: name: unnamed; blockState: DEFAULT; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 +DEBUG: predicate pruning for shardId 13300005 +DEBUG: predicate pruning for shardId 13300006 +DEBUG: predicate pruning for shardId 13300007 +DEBUG: distributed statement: INSERT INTO public.agg_events_13300008 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM (public.raw_events_first_13300000 raw_events_first LEFT JOIN public.raw_events_second_13300004 raw_events_second ON ((raw_events_first.user_id = raw_events_second.user_id))) WHERE ((hashint4(raw_events_first.user_id) >= '-2147483648'::integer) AND (hashint4(raw_events_first.user_id) <= '-1073741825'::integer)) +DEBUG: predicate pruning for shardId 13300000 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 +DEBUG: predicate pruning for shardId 13300004 +DEBUG: predicate pruning for shardId 13300006 +DEBUG: predicate pruning for shardId 13300007 +DEBUG: distributed statement: INSERT INTO public.agg_events_13300009 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM (public.raw_events_first_13300001 raw_events_first LEFT JOIN public.raw_events_second_13300005 raw_events_second ON ((raw_events_first.user_id = raw_events_second.user_id))) WHERE ((hashint4(raw_events_first.user_id) >= '-1073741824'::integer) AND (hashint4(raw_events_first.user_id) <= '-1'::integer)) +DEBUG: predicate pruning for shardId 13300000 +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300003 +DEBUG: predicate pruning for shardId 13300004 +DEBUG: predicate pruning for shardId 13300005 +DEBUG: predicate pruning for shardId 13300007 +DEBUG: distributed statement: INSERT INTO public.agg_events_13300010 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM (public.raw_events_first_13300002 raw_events_first LEFT JOIN public.raw_events_second_13300006 raw_events_second ON ((raw_events_first.user_id = raw_events_second.user_id))) WHERE ((hashint4(raw_events_first.user_id) >= 0) AND (hashint4(raw_events_first.user_id) <= 1073741823)) +DEBUG: predicate pruning for shardId 13300000 +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300004 +DEBUG: predicate pruning for shardId 13300005 +DEBUG: predicate pruning for shardId 13300006 +DEBUG: distributed statement: INSERT INTO public.agg_events_13300011 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM (public.raw_events_first_13300003 raw_events_first LEFT JOIN public.raw_events_second_13300007 raw_events_second ON ((raw_events_first.user_id = raw_events_second.user_id))) WHERE ((hashint4(raw_events_first.user_id) >= 1073741824) AND (hashint4(raw_events_first.user_id) <= 2147483647)) +DEBUG: ProcessQuery +DEBUG: Plan is router executable +DEBUG: CommitTransactionCommand +DEBUG: CommitTransaction +DEBUG: name: unnamed; blockState: STARTED; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +INSERT INTO agg_events (user_id) +SELECT + raw_events_first.user_id +FROM + raw_events_first LEFT JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.user_id + WHERE raw_events_first.user_id = 10; +DEBUG: StartTransactionCommand +DEBUG: StartTransaction +DEBUG: name: unnamed; blockState: DEFAULT; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 +DEBUG: predicate pruning for shardId 13300005 +DEBUG: predicate pruning for shardId 13300006 +DEBUG: predicate pruning for shardId 13300007 +DEBUG: distributed statement: INSERT INTO public.agg_events_13300008 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM (public.raw_events_first_13300000 raw_events_first LEFT JOIN public.raw_events_second_13300004 raw_events_second ON ((raw_events_first.user_id = raw_events_second.user_id))) WHERE ((raw_events_first.user_id = 10) AND ((hashint4(raw_events_first.user_id) >= '-2147483648'::integer) AND (hashint4(raw_events_first.user_id) <= '-1073741825'::integer))) +DEBUG: predicate pruning for shardId 13300000 +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 +DEBUG: predicate pruning for shardId 13300004 +DEBUG: predicate pruning for shardId 13300005 +DEBUG: predicate pruning for shardId 13300006 +DEBUG: predicate pruning for shardId 13300007 +DEBUG: Skipping target shard interval 13300009 since SELECT query for it pruned away +DEBUG: predicate pruning for shardId 13300000 +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 +DEBUG: predicate pruning for shardId 13300004 +DEBUG: predicate pruning for shardId 13300005 +DEBUG: predicate pruning for shardId 13300006 +DEBUG: predicate pruning for shardId 13300007 +DEBUG: Skipping target shard interval 13300010 since SELECT query for it pruned away +DEBUG: predicate pruning for shardId 13300000 +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 +DEBUG: predicate pruning for shardId 13300004 +DEBUG: predicate pruning for shardId 13300005 +DEBUG: predicate pruning for shardId 13300006 +DEBUG: predicate pruning for shardId 13300007 +DEBUG: Skipping target shard interval 13300011 since SELECT query for it pruned away +DEBUG: ProcessQuery +DEBUG: Plan is router executable +DEBUG: CommitTransactionCommand +DEBUG: CommitTransaction +DEBUG: name: unnamed; blockState: STARTED; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +INSERT INTO agg_events (user_id) +SELECT + raw_events_first.user_id +FROM + raw_events_first LEFT JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.user_id + WHERE raw_events_second.user_id = 10; +DEBUG: StartTransactionCommand +DEBUG: StartTransaction +DEBUG: name: unnamed; blockState: DEFAULT; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 +DEBUG: predicate pruning for shardId 13300005 +DEBUG: predicate pruning for shardId 13300006 +DEBUG: predicate pruning for shardId 13300007 +DEBUG: distributed statement: INSERT INTO public.agg_events_13300008 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM (public.raw_events_first_13300000 raw_events_first LEFT JOIN public.raw_events_second_13300004 raw_events_second ON ((raw_events_first.user_id = raw_events_second.user_id))) WHERE ((raw_events_second.user_id = 10) AND ((hashint4(raw_events_first.user_id) >= '-2147483648'::integer) AND (hashint4(raw_events_first.user_id) <= '-1073741825'::integer))) +DEBUG: predicate pruning for shardId 13300000 +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 +DEBUG: predicate pruning for shardId 13300004 +DEBUG: predicate pruning for shardId 13300005 +DEBUG: predicate pruning for shardId 13300006 +DEBUG: predicate pruning for shardId 13300007 +DEBUG: Skipping target shard interval 13300009 since SELECT query for it pruned away +DEBUG: predicate pruning for shardId 13300000 +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 +DEBUG: predicate pruning for shardId 13300004 +DEBUG: predicate pruning for shardId 13300005 +DEBUG: predicate pruning for shardId 13300006 +DEBUG: predicate pruning for shardId 13300007 +DEBUG: Skipping target shard interval 13300010 since SELECT query for it pruned away +DEBUG: predicate pruning for shardId 13300000 +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 +DEBUG: predicate pruning for shardId 13300004 +DEBUG: predicate pruning for shardId 13300005 +DEBUG: predicate pruning for shardId 13300006 +DEBUG: predicate pruning for shardId 13300007 +DEBUG: Skipping target shard interval 13300011 since SELECT query for it pruned away +DEBUG: ProcessQuery +DEBUG: Plan is router executable +DEBUG: CommitTransactionCommand +DEBUG: CommitTransaction +DEBUG: name: unnamed; blockState: STARTED; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +INSERT INTO agg_events (user_id) +SELECT + raw_events_first.user_id +FROM + raw_events_first LEFT JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.user_id + WHERE raw_events_first.user_id = 10 AND raw_events_first.user_id = 20; +DEBUG: StartTransactionCommand +DEBUG: StartTransaction +DEBUG: name: unnamed; blockState: DEFAULT; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +DEBUG: Skipping target shard interval 13300008 since SELECT query for it pruned away +DEBUG: Skipping target shard interval 13300009 since SELECT query for it pruned away +DEBUG: Skipping target shard interval 13300010 since SELECT query for it pruned away +DEBUG: Skipping target shard interval 13300011 since SELECT query for it pruned away +DEBUG: ProcessQuery +DEBUG: Plan is router executable +DEBUG: CommitTransactionCommand +DEBUG: CommitTransaction +DEBUG: name: unnamed; blockState: STARTED; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +INSERT INTO agg_events (user_id) +SELECT + raw_events_first.user_id +FROM + raw_events_first LEFT JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.user_id + WHERE raw_events_first.user_id = 10 AND raw_events_first.user_id IN (19, 20, 21); +DEBUG: StartTransactionCommand +DEBUG: StartTransaction +DEBUG: name: unnamed; blockState: DEFAULT; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +NOTICE: cannot use shard pruning with ANY/ALL (array expression) +HINT: Consider rewriting the expression with OR/AND clauses. +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 +DEBUG: predicate pruning for shardId 13300005 +DEBUG: predicate pruning for shardId 13300006 +DEBUG: predicate pruning for shardId 13300007 +DEBUG: distributed statement: INSERT INTO public.agg_events_13300008 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM (public.raw_events_first_13300000 raw_events_first LEFT JOIN public.raw_events_second_13300004 raw_events_second ON ((raw_events_first.user_id = raw_events_second.user_id))) WHERE (((raw_events_first.user_id = 10) AND (raw_events_first.user_id = ANY (ARRAY[19, 20, 21]))) AND ((hashint4(raw_events_first.user_id) >= '-2147483648'::integer) AND (hashint4(raw_events_first.user_id) <= '-1073741825'::integer))) +NOTICE: cannot use shard pruning with ANY/ALL (array expression) +HINT: Consider rewriting the expression with OR/AND clauses. +DEBUG: predicate pruning for shardId 13300000 +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 +DEBUG: predicate pruning for shardId 13300004 +DEBUG: predicate pruning for shardId 13300005 +DEBUG: predicate pruning for shardId 13300006 +DEBUG: predicate pruning for shardId 13300007 +DEBUG: Skipping target shard interval 13300009 since SELECT query for it pruned away +NOTICE: cannot use shard pruning with ANY/ALL (array expression) +HINT: Consider rewriting the expression with OR/AND clauses. +DEBUG: predicate pruning for shardId 13300000 +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 +DEBUG: predicate pruning for shardId 13300004 +DEBUG: predicate pruning for shardId 13300005 +DEBUG: predicate pruning for shardId 13300006 +DEBUG: predicate pruning for shardId 13300007 +DEBUG: Skipping target shard interval 13300010 since SELECT query for it pruned away +NOTICE: cannot use shard pruning with ANY/ALL (array expression) +HINT: Consider rewriting the expression with OR/AND clauses. +DEBUG: predicate pruning for shardId 13300000 +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 +DEBUG: predicate pruning for shardId 13300004 +DEBUG: predicate pruning for shardId 13300005 +DEBUG: predicate pruning for shardId 13300006 +DEBUG: predicate pruning for shardId 13300007 +DEBUG: Skipping target shard interval 13300011 since SELECT query for it pruned away +DEBUG: ProcessQuery +DEBUG: Plan is router executable +DEBUG: CommitTransactionCommand +DEBUG: CommitTransaction +DEBUG: name: unnamed; blockState: STARTED; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +INSERT INTO agg_events (user_id) +SELECT + raw_events_first.user_id +FROM + raw_events_first INNER JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.user_id + WHERE raw_events_first.user_id = 10 AND raw_events_second.user_id IN (19, 20, 21); +DEBUG: StartTransactionCommand +DEBUG: StartTransaction +DEBUG: name: unnamed; blockState: DEFAULT; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 +NOTICE: cannot use shard pruning with ANY/ALL (array expression) +HINT: Consider rewriting the expression with OR/AND clauses. +DEBUG: predicate pruning for shardId 13300005 +DEBUG: predicate pruning for shardId 13300006 +DEBUG: predicate pruning for shardId 13300007 +DEBUG: distributed statement: INSERT INTO public.agg_events_13300008 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM (public.raw_events_first_13300000 raw_events_first JOIN public.raw_events_second_13300004 raw_events_second ON ((raw_events_first.user_id = raw_events_second.user_id))) WHERE (((raw_events_first.user_id = 10) AND (raw_events_second.user_id = ANY (ARRAY[19, 20, 21]))) AND ((hashint4(raw_events_first.user_id) >= '-2147483648'::integer) AND (hashint4(raw_events_first.user_id) <= '-1073741825'::integer))) +DEBUG: predicate pruning for shardId 13300000 +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 +NOTICE: cannot use shard pruning with ANY/ALL (array expression) +HINT: Consider rewriting the expression with OR/AND clauses. +DEBUG: predicate pruning for shardId 13300004 +DEBUG: predicate pruning for shardId 13300005 +DEBUG: predicate pruning for shardId 13300006 +DEBUG: predicate pruning for shardId 13300007 +DEBUG: Skipping target shard interval 13300009 since SELECT query for it pruned away +DEBUG: predicate pruning for shardId 13300000 +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 +NOTICE: cannot use shard pruning with ANY/ALL (array expression) +HINT: Consider rewriting the expression with OR/AND clauses. +DEBUG: predicate pruning for shardId 13300004 +DEBUG: predicate pruning for shardId 13300005 +DEBUG: predicate pruning for shardId 13300006 +DEBUG: predicate pruning for shardId 13300007 +DEBUG: Skipping target shard interval 13300010 since SELECT query for it pruned away +DEBUG: predicate pruning for shardId 13300000 +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 +NOTICE: cannot use shard pruning with ANY/ALL (array expression) +HINT: Consider rewriting the expression with OR/AND clauses. +DEBUG: predicate pruning for shardId 13300004 +DEBUG: predicate pruning for shardId 13300005 +DEBUG: predicate pruning for shardId 13300006 +DEBUG: predicate pruning for shardId 13300007 +DEBUG: Skipping target shard interval 13300011 since SELECT query for it pruned away +DEBUG: ProcessQuery +DEBUG: Plan is router executable +DEBUG: CommitTransactionCommand +DEBUG: CommitTransaction +DEBUG: name: unnamed; blockState: STARTED; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +-- the following is a very tricky query for Citus +-- although we do not support pushing down JOINs on non-partition +-- columns here it is safe to push it down given that we're looking for +-- a specific value (i.e., value_1 = 12) on the joining column. +-- Note that the query always hits the same shard on raw_events_second. +-- Since that shard is on both nodes the query can be pushed down +-- but for instance if there were 3 worker nodes the query would fail +INSERT INTO agg_events + (user_id) +SELECT raw_events_first.user_id +FROM raw_events_first, + raw_events_second +WHERE raw_events_second.user_id = raw_events_first.value_1 + AND raw_events_first.value_1 = 12; +DEBUG: StartTransactionCommand +DEBUG: StartTransaction +DEBUG: name: unnamed; blockState: DEFAULT; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 +DEBUG: predicate pruning for shardId 13300004 +DEBUG: predicate pruning for shardId 13300005 +DEBUG: predicate pruning for shardId 13300006 +DEBUG: distributed statement: INSERT INTO public.agg_events_13300008 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM public.raw_events_first_13300000 raw_events_first, public.raw_events_second_13300007 raw_events_second WHERE (((raw_events_second.user_id = raw_events_first.value_1) AND (raw_events_first.value_1 = 12)) AND ((hashint4(raw_events_first.user_id) >= '-2147483648'::integer) AND (hashint4(raw_events_first.user_id) <= '-1073741825'::integer))) +DEBUG: predicate pruning for shardId 13300000 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 +DEBUG: predicate pruning for shardId 13300004 +DEBUG: predicate pruning for shardId 13300005 +DEBUG: predicate pruning for shardId 13300006 +DEBUG: distributed statement: INSERT INTO public.agg_events_13300009 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM public.raw_events_first_13300001 raw_events_first, public.raw_events_second_13300007 raw_events_second WHERE (((raw_events_second.user_id = raw_events_first.value_1) AND (raw_events_first.value_1 = 12)) AND ((hashint4(raw_events_first.user_id) >= '-1073741824'::integer) AND (hashint4(raw_events_first.user_id) <= '-1'::integer))) +DEBUG: predicate pruning for shardId 13300000 +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300003 +DEBUG: predicate pruning for shardId 13300004 +DEBUG: predicate pruning for shardId 13300005 +DEBUG: predicate pruning for shardId 13300006 +DEBUG: distributed statement: INSERT INTO public.agg_events_13300010 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM public.raw_events_first_13300002 raw_events_first, public.raw_events_second_13300007 raw_events_second WHERE (((raw_events_second.user_id = raw_events_first.value_1) AND (raw_events_first.value_1 = 12)) AND ((hashint4(raw_events_first.user_id) >= 0) AND (hashint4(raw_events_first.user_id) <= 1073741823))) +DEBUG: predicate pruning for shardId 13300000 +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300004 +DEBUG: predicate pruning for shardId 13300005 +DEBUG: predicate pruning for shardId 13300006 +DEBUG: distributed statement: INSERT INTO public.agg_events_13300011 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM public.raw_events_first_13300003 raw_events_first, public.raw_events_second_13300007 raw_events_second WHERE (((raw_events_second.user_id = raw_events_first.value_1) AND (raw_events_first.value_1 = 12)) AND ((hashint4(raw_events_first.user_id) >= 1073741824) AND (hashint4(raw_events_first.user_id) <= 2147483647))) +DEBUG: ProcessQuery +DEBUG: Plan is router executable +DEBUG: CommitTransactionCommand +DEBUG: CommitTransaction +DEBUG: name: unnamed; blockState: STARTED; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +-- some unsupported LEFT/INNER JOINs +-- JOIN on one table with partition column other is not +INSERT INTO agg_events (user_id) +SELECT + raw_events_first.user_id +FROM + raw_events_first LEFT JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.value_1; +DEBUG: StartTransactionCommand +DEBUG: StartTransaction +DEBUG: name: unnamed; blockState: DEFAULT; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 +ERROR: cannot perform distributed planning for the given modification +DETAIL: Select query cannot be pushed down to the worker. +-- same as the above with INNER JOIN +INSERT INTO agg_events (user_id) +SELECT + raw_events_first.user_id +FROM + raw_events_first INNER JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.value_1; +DEBUG: StartTransactionCommand +DEBUG: StartTransaction +DEBUG: name: unnamed; blockState: DEFAULT; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 +ERROR: cannot perform distributed planning for the given modification +DETAIL: Select query cannot be pushed down to the worker. +-- a not meaningful query +INSERT INTO agg_events + (user_id) +SELECT raw_events_second.user_id +FROM raw_events_first, + raw_events_second +WHERE raw_events_first.user_id = raw_events_first.value_1; +DEBUG: StartTransactionCommand +DEBUG: StartTransaction +DEBUG: name: unnamed; blockState: DEFAULT; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +ERROR: cannot perform distributed planning for the given modification +DETAIL: Select query cannot be pushed down to the worker. +-- both tables joined on non-partition columns +INSERT INTO agg_events (user_id) +SELECT + raw_events_first.user_id +FROM + raw_events_first LEFT JOIN raw_events_second ON raw_events_first.value_1 = raw_events_second.value_1; +DEBUG: StartTransactionCommand +DEBUG: StartTransaction +DEBUG: name: unnamed; blockState: DEFAULT; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 +ERROR: cannot perform distributed planning for the given modification +DETAIL: Select query cannot be pushed down to the worker. +-- same as the above with INNER JOIN +INSERT INTO agg_events (user_id) +SELECT + raw_events_first.user_id +FROM + raw_events_first INNER JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.value_1; +DEBUG: StartTransactionCommand +DEBUG: StartTransaction +DEBUG: name: unnamed; blockState: DEFAULT; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 +ERROR: cannot perform distributed planning for the given modification +DETAIL: Select query cannot be pushed down to the worker. +-- JOIN on one table with partition column other is not +-- also an equality qual is added on the partition key +INSERT INTO agg_events (user_id) +SELECT + raw_events_first.user_id +FROM + raw_events_first LEFT JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.value_1 +WHERE + raw_events_first.user_id = 10; +DEBUG: StartTransactionCommand +DEBUG: StartTransaction +DEBUG: name: unnamed; blockState: DEFAULT; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 +DEBUG: predicate pruning for shardId 13300005 +DEBUG: predicate pruning for shardId 13300006 +DEBUG: predicate pruning for shardId 13300007 +DEBUG: distributed statement: INSERT INTO public.agg_events_13300008 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM (public.raw_events_first_13300000 raw_events_first LEFT JOIN public.raw_events_second_13300004 raw_events_second ON ((raw_events_first.user_id = raw_events_second.value_1))) WHERE ((raw_events_first.user_id = 10) AND ((hashint4(raw_events_first.user_id) >= '-2147483648'::integer) AND (hashint4(raw_events_first.user_id) <= '-1073741825'::integer))) +DEBUG: predicate pruning for shardId 13300000 +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 +DEBUG: predicate pruning for shardId 13300004 +DEBUG: predicate pruning for shardId 13300006 +DEBUG: predicate pruning for shardId 13300007 +DEBUG: distributed statement: INSERT INTO public.agg_events_13300009 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM ((SELECT NULL::integer AS user_id, NULL::timestamp without time zone AS "time", NULL::integer AS value_1, NULL::integer AS value_2, NULL::double precision AS value_3, NULL::bigint AS value_4 WHERE false) raw_events_first(user_id, "time", value_1, value_2, value_3, value_4) LEFT JOIN public.raw_events_second_13300005 raw_events_second ON ((raw_events_first.user_id = raw_events_second.value_1))) WHERE ((raw_events_first.user_id = 10) AND ((hashint4(raw_events_first.user_id) >= '-1073741824'::integer) AND (hashint4(raw_events_first.user_id) <= '-1'::integer))) +DEBUG: predicate pruning for shardId 13300000 +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 +DEBUG: predicate pruning for shardId 13300004 +DEBUG: predicate pruning for shardId 13300005 +DEBUG: predicate pruning for shardId 13300007 +DEBUG: distributed statement: INSERT INTO public.agg_events_13300010 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM ((SELECT NULL::integer AS user_id, NULL::timestamp without time zone AS "time", NULL::integer AS value_1, NULL::integer AS value_2, NULL::double precision AS value_3, NULL::bigint AS value_4 WHERE false) raw_events_first(user_id, "time", value_1, value_2, value_3, value_4) LEFT JOIN public.raw_events_second_13300006 raw_events_second ON ((raw_events_first.user_id = raw_events_second.value_1))) WHERE ((raw_events_first.user_id = 10) AND ((hashint4(raw_events_first.user_id) >= 0) AND (hashint4(raw_events_first.user_id) <= 1073741823))) +DEBUG: predicate pruning for shardId 13300000 +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 +DEBUG: predicate pruning for shardId 13300004 +DEBUG: predicate pruning for shardId 13300005 +DEBUG: predicate pruning for shardId 13300006 +DEBUG: distributed statement: INSERT INTO public.agg_events_13300011 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM ((SELECT NULL::integer AS user_id, NULL::timestamp without time zone AS "time", NULL::integer AS value_1, NULL::integer AS value_2, NULL::double precision AS value_3, NULL::bigint AS value_4 WHERE false) raw_events_first(user_id, "time", value_1, value_2, value_3, value_4) LEFT JOIN public.raw_events_second_13300007 raw_events_second ON ((raw_events_first.user_id = raw_events_second.value_1))) WHERE ((raw_events_first.user_id = 10) AND ((hashint4(raw_events_first.user_id) >= 1073741824) AND (hashint4(raw_events_first.user_id) <= 2147483647))) +DEBUG: ProcessQuery +DEBUG: Plan is router executable +DEBUG: CommitTransactionCommand +DEBUG: CommitTransaction +DEBUG: name: unnamed; blockState: STARTED; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +-- same as the above with INNER JOIN +INSERT INTO agg_events (user_id) +SELECT + raw_events_first.user_id +FROM + raw_events_first INNER JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.value_1 +WHERE raw_events_first.user_id = 10; +DEBUG: StartTransactionCommand +DEBUG: StartTransaction +DEBUG: name: unnamed; blockState: DEFAULT; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 +DEBUG: predicate pruning for shardId 13300005 +DEBUG: predicate pruning for shardId 13300006 +DEBUG: predicate pruning for shardId 13300007 +DEBUG: distributed statement: INSERT INTO public.agg_events_13300008 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM (public.raw_events_first_13300000 raw_events_first JOIN public.raw_events_second_13300004 raw_events_second ON ((raw_events_first.user_id = raw_events_second.value_1))) WHERE ((raw_events_first.user_id = 10) AND ((hashint4(raw_events_first.user_id) >= '-2147483648'::integer) AND (hashint4(raw_events_first.user_id) <= '-1073741825'::integer))) +DEBUG: predicate pruning for shardId 13300000 +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 +DEBUG: predicate pruning for shardId 13300004 +DEBUG: predicate pruning for shardId 13300006 +DEBUG: predicate pruning for shardId 13300007 +DEBUG: distributed statement: INSERT INTO public.agg_events_13300009 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM ((SELECT NULL::integer AS user_id, NULL::timestamp without time zone AS "time", NULL::integer AS value_1, NULL::integer AS value_2, NULL::double precision AS value_3, NULL::bigint AS value_4 WHERE false) raw_events_first(user_id, "time", value_1, value_2, value_3, value_4) JOIN public.raw_events_second_13300005 raw_events_second ON ((raw_events_first.user_id = raw_events_second.value_1))) WHERE ((raw_events_first.user_id = 10) AND ((hashint4(raw_events_first.user_id) >= '-1073741824'::integer) AND (hashint4(raw_events_first.user_id) <= '-1'::integer))) +DEBUG: predicate pruning for shardId 13300000 +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 +DEBUG: predicate pruning for shardId 13300004 +DEBUG: predicate pruning for shardId 13300005 +DEBUG: predicate pruning for shardId 13300007 +DEBUG: distributed statement: INSERT INTO public.agg_events_13300010 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM ((SELECT NULL::integer AS user_id, NULL::timestamp without time zone AS "time", NULL::integer AS value_1, NULL::integer AS value_2, NULL::double precision AS value_3, NULL::bigint AS value_4 WHERE false) raw_events_first(user_id, "time", value_1, value_2, value_3, value_4) JOIN public.raw_events_second_13300006 raw_events_second ON ((raw_events_first.user_id = raw_events_second.value_1))) WHERE ((raw_events_first.user_id = 10) AND ((hashint4(raw_events_first.user_id) >= 0) AND (hashint4(raw_events_first.user_id) <= 1073741823))) +DEBUG: predicate pruning for shardId 13300000 +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 +DEBUG: predicate pruning for shardId 13300004 +DEBUG: predicate pruning for shardId 13300005 +DEBUG: predicate pruning for shardId 13300006 +DEBUG: distributed statement: INSERT INTO public.agg_events_13300011 AS citus_table_alias (user_id) SELECT raw_events_first.user_id FROM ((SELECT NULL::integer AS user_id, NULL::timestamp without time zone AS "time", NULL::integer AS value_1, NULL::integer AS value_2, NULL::double precision AS value_3, NULL::bigint AS value_4 WHERE false) raw_events_first(user_id, "time", value_1, value_2, value_3, value_4) JOIN public.raw_events_second_13300007 raw_events_second ON ((raw_events_first.user_id = raw_events_second.value_1))) WHERE ((raw_events_first.user_id = 10) AND ((hashint4(raw_events_first.user_id) >= 1073741824) AND (hashint4(raw_events_first.user_id) <= 2147483647))) +DEBUG: ProcessQuery +DEBUG: Plan is router executable +DEBUG: CommitTransactionCommand +DEBUG: CommitTransaction +DEBUG: name: unnamed; blockState: STARTED; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +-- make things a bit more complicate with IN clauses +INSERT INTO agg_events (user_id) +SELECT + raw_events_first.user_id +FROM + raw_events_first INNER JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.value_1 + WHERE raw_events_first.value_1 IN (10, 11,12) OR raw_events_second.user_id IN (1,2,3,4); +DEBUG: StartTransactionCommand +DEBUG: StartTransaction +DEBUG: name: unnamed; blockState: DEFAULT; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 +NOTICE: cannot use shard pruning with ANY/ALL (array expression) +HINT: Consider rewriting the expression with OR/AND clauses. +ERROR: cannot perform distributed planning for the given modification +DETAIL: Select query cannot be pushed down to the worker. +-- implicit join on non partition column should also not be pushed down +INSERT INTO agg_events + (user_id) +SELECT raw_events_first.user_id +FROM raw_events_first, + raw_events_second +WHERE raw_events_second.user_id = raw_events_first.value_1; +DEBUG: StartTransactionCommand +DEBUG: StartTransaction +DEBUG: name: unnamed; blockState: DEFAULT; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 +ERROR: cannot perform distributed planning for the given modification +DETAIL: Select query cannot be pushed down to the worker. +-- the following is again a very tricky query for Citus +-- if the given filter was on value_1 as shown in the above, Citus could +-- push it down. But here the query is refused +INSERT INTO agg_events + (user_id) +SELECT raw_events_first.user_id +FROM raw_events_first, + raw_events_second +WHERE raw_events_second.user_id = raw_events_first.value_1 + AND raw_events_first.value_2 = 12; +DEBUG: StartTransactionCommand +DEBUG: StartTransaction +DEBUG: name: unnamed; blockState: DEFAULT; state: INPROGR, xid/subid/cid: 0/1/0, nestlvl: 1, children: +DEBUG: predicate pruning for shardId 13300001 +DEBUG: predicate pruning for shardId 13300002 +DEBUG: predicate pruning for shardId 13300003 +ERROR: cannot perform distributed planning for the given modification +DETAIL: Select query cannot be pushed down to the worker. -- unsupported JOIN INSERT INTO agg_events (value_4_agg, @@ -2073,3 +2610,10 @@ DROP TABLE table_with_serial; DROP TABLE text_table; DROP TABLE char_table; DROP TABLE table_with_starts_with_defaults; +-- clean any 2PC transactions we've used +SELECT recover_prepared_transactions(); + recover_prepared_transactions +------------------------------- + 0 +(1 row) + diff --git a/src/test/regress/sql/multi_insert_select.sql b/src/test/regress/sql/multi_insert_select.sql index ea11ef1b7..c78387fea 100644 --- a/src/test/regress/sql/multi_insert_select.sql +++ b/src/test/regress/sql/multi_insert_select.sql @@ -496,6 +496,146 @@ FROM ((SELECT user_id FROM raw_events_first WHERE user_id = 15) EXCEPT (SELECT user_id FROM raw_events_second where user_id = 17)) as foo; + +-- some supported LEFT JOINs +INSERT INTO agg_events (user_id) +SELECT + raw_events_first.user_id +FROM + raw_events_first LEFT JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.user_id; + +INSERT INTO agg_events (user_id) +SELECT + raw_events_first.user_id +FROM + raw_events_first LEFT JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.user_id + WHERE raw_events_first.user_id = 10; + +INSERT INTO agg_events (user_id) +SELECT + raw_events_first.user_id +FROM + raw_events_first LEFT JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.user_id + WHERE raw_events_second.user_id = 10; + +INSERT INTO agg_events (user_id) +SELECT + raw_events_first.user_id +FROM + raw_events_first LEFT JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.user_id + WHERE raw_events_first.user_id = 10 AND raw_events_first.user_id = 20; + +INSERT INTO agg_events (user_id) +SELECT + raw_events_first.user_id +FROM + raw_events_first LEFT JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.user_id + WHERE raw_events_first.user_id = 10 AND raw_events_first.user_id IN (19, 20, 21); + +INSERT INTO agg_events (user_id) +SELECT + raw_events_first.user_id +FROM + raw_events_first INNER JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.user_id + WHERE raw_events_first.user_id = 10 AND raw_events_second.user_id IN (19, 20, 21); + +-- the following is a very tricky query for Citus +-- although we do not support pushing down JOINs on non-partition +-- columns here it is safe to push it down given that we're looking for +-- a specific value (i.e., value_1 = 12) on the joining column. +-- Note that the query always hits the same shard on raw_events_second. +-- Since that shard is on both nodes the query can be pushed down +-- but for instance if there were 3 worker nodes the query would fail +INSERT INTO agg_events + (user_id) +SELECT raw_events_first.user_id +FROM raw_events_first, + raw_events_second +WHERE raw_events_second.user_id = raw_events_first.value_1 + AND raw_events_first.value_1 = 12; + +-- some unsupported LEFT/INNER JOINs +-- JOIN on one table with partition column other is not +INSERT INTO agg_events (user_id) +SELECT + raw_events_first.user_id +FROM + raw_events_first LEFT JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.value_1; + +-- same as the above with INNER JOIN +INSERT INTO agg_events (user_id) +SELECT + raw_events_first.user_id +FROM + raw_events_first INNER JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.value_1; + +-- a not meaningful query +INSERT INTO agg_events + (user_id) +SELECT raw_events_second.user_id +FROM raw_events_first, + raw_events_second +WHERE raw_events_first.user_id = raw_events_first.value_1; + +-- both tables joined on non-partition columns +INSERT INTO agg_events (user_id) +SELECT + raw_events_first.user_id +FROM + raw_events_first LEFT JOIN raw_events_second ON raw_events_first.value_1 = raw_events_second.value_1; + +-- same as the above with INNER JOIN +INSERT INTO agg_events (user_id) +SELECT + raw_events_first.user_id +FROM + raw_events_first INNER JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.value_1; + +-- JOIN on one table with partition column other is not +-- also an equality qual is added on the partition key +INSERT INTO agg_events (user_id) +SELECT + raw_events_first.user_id +FROM + raw_events_first LEFT JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.value_1 +WHERE + raw_events_first.user_id = 10; + +-- same as the above with INNER JOIN +INSERT INTO agg_events (user_id) +SELECT + raw_events_first.user_id +FROM + raw_events_first INNER JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.value_1 +WHERE raw_events_first.user_id = 10; + +-- make things a bit more complicate with IN clauses +INSERT INTO agg_events (user_id) +SELECT + raw_events_first.user_id +FROM + raw_events_first INNER JOIN raw_events_second ON raw_events_first.user_id = raw_events_second.value_1 + WHERE raw_events_first.value_1 IN (10, 11,12) OR raw_events_second.user_id IN (1,2,3,4); + +-- implicit join on non partition column should also not be pushed down +INSERT INTO agg_events + (user_id) +SELECT raw_events_first.user_id +FROM raw_events_first, + raw_events_second +WHERE raw_events_second.user_id = raw_events_first.value_1; + +-- the following is again a very tricky query for Citus +-- if the given filter was on value_1 as shown in the above, Citus could +-- push it down. But here the query is refused +INSERT INTO agg_events + (user_id) +SELECT raw_events_first.user_id +FROM raw_events_first, + raw_events_second +WHERE raw_events_second.user_id = raw_events_first.value_1 + AND raw_events_first.value_2 = 12; + -- unsupported JOIN INSERT INTO agg_events (value_4_agg, @@ -979,3 +1119,6 @@ DROP TABLE table_with_serial; DROP TABLE text_table; DROP TABLE char_table; DROP TABLE table_with_starts_with_defaults; + +-- clean any 2PC transactions we've used +SELECT recover_prepared_transactions();