diff --git a/src/backend/distributed/planner/shard_pruning.c b/src/backend/distributed/planner/shard_pruning.c index f0529357e..71d09ae6f 100644 --- a/src/backend/distributed/planner/shard_pruning.c +++ b/src/backend/distributed/planner/shard_pruning.c @@ -6,8 +6,27 @@ * The goal of shard pruning is to find a minimal (super)set of shards that * need to be queried to find rows matching the expression in a query. * - * In PruneShards, we first compute a simplified disjunctive normal form (DNF) - * of the expression as a list of pruning instances. Each pruning instance + * In PruneShards we first make a compact representation of the given + * query logical tree. This tree represent boolean operators and its + * associated valid constrainst (expression nodes) and whether boolean + * operator has associated unknown constraints. This allows essentially + * unknown constraints to be replaced by a simple placeholder flag. + * + * For example query: WHERE (hash_col IN (1,2)) AND (other_col=1 OR other_col=2) + * Gets transformed by steps: + * 1. AND(hash_col IN (1,2), OR(X, X)) + * 2. AND(hash_col IN (1,2), OR(X)) + * 3. AND(hash_col IN (1,2), X) + * Where X represents any (set of) unrecognized unprunable constraint(s). + * + * Above allows the following pruning machinery to understand that + * the target shard is determined solely by constraint: hash_col IN (1,2). + * Here it does not matter what X is as its ANDed by a valid constraint. + * Pruning machinery will fail, returning all shards, if it encounters + * eg. OR(hash_col=1, X) as this condition does not limit the target shards. + * + * PruneShards secondly computes a simplified disjunctive normal form (DNF) + * of the logical tree as a list of pruning instances. Each pruning instance * contains all AND-ed constraints on the partition column. An OR expression * will result in two or more new pruning instances being added for the * subexpressions. The "parent" instance is marked isPartial and ignored @@ -62,6 +81,7 @@ #include "distributed/pg_dist_partition.h" #include "distributed/version_compat.h" #include "distributed/worker_protocol.h" +#include "distributed/log_utils.h" #include "nodes/nodeFuncs.h" #include "nodes/makefuncs.h" #include "optimizer/clauses.h" @@ -71,6 +91,39 @@ #include "utils/catcache.h" #include "utils/lsyscache.h" #include "utils/memutils.h" +#include "utils/ruleutils.h" + +/* + * Tree node for compact representation of the given query logical tree. + * Represent a single boolean operator node and its associated + * valid constraints (expression nodes) and invalid constraint flag. + */ +typedef struct PruningTreeNode +{ + /* Indicates is this AND/OR boolean operator */ + bool isAnd; + + /* Does this boolean operator have unknown/unprunable constraint(s) */ + bool hasInvalidConstraints; + + /* List of recognized valid prunable constraints of this boolean opearator */ + List *validConstraints; + + /* + * Child boolean operators. + * Parent is always different boolean operator from its children. + */ + List *childBooleanNodes; +} PruningTreeNode; + +/* + * Context used for expression_tree_walker + */ +typedef struct PruningTreeBuildContext +{ + Var *partitionColumn; + PruningTreeNode *current; +} PruningTreeBuildContext; /* * A pruning instance is a set of ANDed constraints on a partition key. @@ -104,14 +157,6 @@ typedef struct PruningInstance */ Const *hashedEqualConsts; - /* - * Types of constraints not understood. We could theoretically try more - * expensive methods of pruning if any such restrictions are found. - * - * TODO: any actual use for this? Right now there seems little point. - */ - List *otherRestrictions; - /* * Has this PruningInstance been added to * ClauseWalkerContext->pruningInstances? This is not done immediately, @@ -137,7 +182,7 @@ typedef struct PruningInstance typedef struct PendingPruningInstance { PruningInstance *instance; - Node *continueAt; + PruningTreeNode *continueAt; } PendingPruningInstance; #if PG_VERSION_NUM >= 120000 @@ -180,19 +225,29 @@ typedef struct ClauseWalkerContext FunctionCall2InfoData compareIntervalFunctionCall; } ClauseWalkerContext; -static void PrunableExpressions(Node *originalNode, ClauseWalkerContext *context); -static bool PrunableExpressionsWalker(Node *originalNode, ClauseWalkerContext *context); +static bool BuildPruningTree(Node *node, PruningTreeBuildContext *context); +static void SimplifyPruningTree(PruningTreeNode *node, PruningTreeNode *parent); +static void PrunableExpressions(PruningTreeNode *node, ClauseWalkerContext *context); +static void PrunableExpressionsWalker(PruningTreeNode *node, + ClauseWalkerContext *context); +static bool IsValidPartitionKeyRestriction(OpExpr *opClause); static void AddPartitionKeyRestrictionToInstance(ClauseWalkerContext *context, OpExpr *opClause, Var *varClause, Const *constantClause); +static bool VarConstOpExprClause(OpExpr *opClause, Var *partitionColumn, + Var **varClause, Const **constantClause); static Const * TransformPartitionRestrictionValue(Var *partitionColumn, Const *restrictionValue); static void AddSAOPartitionKeyRestrictionToInstance(ClauseWalkerContext *context, ScalarArrayOpExpr * arrayOperatorExpression); +static bool SAORestrictions(ScalarArrayOpExpr *arrayOperatorExpression, + Var *partitionColumn, + List **requestedRestrictions); +static bool IsValidHashRestriction(OpExpr *opClause); static void AddHashRestrictionToInstance(ClauseWalkerContext *context, OpExpr *opClause, Var *varClause, Const *constantClause); -static void AddNewConjuction(ClauseWalkerContext *context, OpExpr *op); +static void AddNewConjuction(ClauseWalkerContext *context, PruningTreeNode *node); static PruningInstance * CopyPartialPruningInstance(PruningInstance *sourceInstance); static List * ShardArrayToList(ShardInterval **shardArray, int length); static List * DeepCopyShardIntervalList(List *originalShardIntervalList); @@ -219,7 +274,32 @@ static int LowerShardBoundary(Datum partitionColumnValue, ShardInterval **shardIntervalCache, int shardCount, FunctionCallInfo compareFunction, bool includeMax); +static inline PruningTreeNode * CreatePruningNode(bool isAnd); +static inline OpExpr * SAORestrictionArrayEqualityOp( + ScalarArrayOpExpr *arrayOperatorExpression, + Var *partitionColumn); +static inline void DebugLogNode(char *fmt, Node *node, List *deparseCtx); +#define AndBooleanNode() (CreatePruningNode(true)) +#define OrBooleanNode() (CreatePruningNode(false)) +#define IsAndOp(node) ((node)->isAnd) +#define IsOrOp(node) (!(node)->isAnd) +#define ConstraintCount(node) \ + (list_length((node)->childBooleanNodes) + \ + list_length((node)->validConstraints) + \ + ((node)->hasInvalidConstraints ? 1 : 0)) + +#define DebugLogPruningInstance(prune, deparseCtx) \ + DebugLogNode("constraint value: %s, ", \ + (Node *) (prune)->equalConsts, (deparseCtx)); \ + DebugLogNode("constraint (lt) value: %s, ", \ + (Node *) (prune)->lessConsts, (deparseCtx)); \ + DebugLogNode("constraint (lteq) value: %s, ", \ + (Node *) (prune)->lessEqualConsts, (deparseCtx)); \ + DebugLogNode("constraint (gt) value: %s, ", \ + (Node *) (prune)->greaterConsts, (deparseCtx)); \ + DebugLogNode("constraint (gteq) value: %s, ", \ + (Node *) (prune)->greaterEqualConsts, (deparseCtx)); /* * PruneShards returns all shards from a distributed table that cannot be @@ -298,8 +378,22 @@ PruneShards(Oid relationId, Index rangeTableId, List *whereClauseList, "a partition column comparator"))); } + PruningTreeNode *tree = AndBooleanNode(); + + PruningTreeBuildContext treeBuildContext = { 0 }; + treeBuildContext.current = tree; + treeBuildContext.partitionColumn = PartitionColumn(relationId, rangeTableId); + + /* Build logical tree of prunable restrictions and invalid restrictions */ + BuildPruningTree((Node *) whereClauseList, &treeBuildContext); + + /* Simplify logic tree of prunable restrictions */ + SimplifyPruningTree(tree, NULL); + /* Figure out what we can prune on */ - PrunableExpressions((Node *) whereClauseList, &context); + PrunableExpressions(tree, &context); + + List *debugLoggedPruningInstances = NULL; /* * Prune using each of the PrunableInstances we found, and OR results @@ -375,6 +469,11 @@ PruneShards(Oid relationId, Index rangeTableId, List *whereClauseList, prunedList = pruneOneList; } foundRestriction = true; + + if (IsLoggableLevel(DEBUG3) && pruneOneList) + { + debugLoggedPruningInstances = lappend(debugLoggedPruningInstances, prune); + } } /* found no valid restriction, build list of all shards */ @@ -384,6 +483,25 @@ PruneShards(Oid relationId, Index rangeTableId, List *whereClauseList, cacheEntry->shardIntervalArrayLength); } + if (IsLoggableLevel(DEBUG3)) + { + if (foundRestriction && debugLoggedPruningInstances) + { + List *deparseCtx = deparse_context_for("unknown", relationId); + foreach(pruneCell, debugLoggedPruningInstances) + { + PruningInstance *prune = (PruningInstance *) lfirst(pruneCell); + DebugLogPruningInstance(prune, deparseCtx); + } + } + else + { + ereport(DEBUG3, (errmsg("no valid constraints found"))); + } + + ereport(DEBUG3, (errmsg("shard count: %d", list_length(prunedList)))); + } + /* if requested, copy the partition value constant */ if (partitionValueConst != NULL) { @@ -405,6 +523,150 @@ PruneShards(Oid relationId, Index rangeTableId, List *whereClauseList, } +/* + * Check whether node is a valid constraint for pruning + */ +static bool +IsValidConditionNode(Node *node, Var *partitionColumn) +{ + if (IsA(node, OpExpr)) + { + OpExpr *opClause = (OpExpr *) node; + Var *varClause = NULL; + if (VarConstOpExprClause(opClause, partitionColumn, &varClause, NULL)) + { + if (equal(varClause, partitionColumn)) + { + return IsValidPartitionKeyRestriction(opClause); + } + else if (varClause->varattno == RESERVED_HASHED_COLUMN_ID) + { + return IsValidHashRestriction(opClause); + } + } + + return false; + } + else if (IsA(node, ScalarArrayOpExpr)) + { + ScalarArrayOpExpr *arrayOperatorExpression = (ScalarArrayOpExpr *) node; + if (SAORestrictions(arrayOperatorExpression, partitionColumn, NULL)) + { + return true; + } + + return false; + } + else + { + return false; + } +} + + +/* + * Build a logical tree of valid constraints and invalid constaints for pruning. + */ +static bool +BuildPruningTree(Node *node, PruningTreeBuildContext *context) +{ + if (node == NULL) + { + return false; + } + + if (IsA(node, List)) + { + return expression_tree_walker(node, BuildPruningTree, context); + } + else if (IsA(node, BoolExpr)) + { + BoolExpr *boolExpr = (BoolExpr *) node; + bool isAnded = boolExpr->boolop == AND_EXPR; + + if (boolExpr->boolop == NOT_EXPR) + { + return false; + } + else if (context->current->isAnd != isAnded) + { + PruningTreeNode *child = CreatePruningNode(isAnded); + + context->current->childBooleanNodes = lappend( + context->current->childBooleanNodes, child); + + PruningTreeBuildContext newContext = { 0 }; + newContext.partitionColumn = context->partitionColumn; + newContext.current = child; + + return expression_tree_walker((Node *) boolExpr->args, + BuildPruningTree, &newContext); + } + else + { + return expression_tree_walker(node, BuildPruningTree, context); + } + } + else if (IsValidConditionNode(node, context->partitionColumn)) + { + context->current->validConstraints = lappend(context->current->validConstraints, + node); + + return false; + } + else + { + context->current->hasInvalidConstraints = true; + + return false; + } +} + + +/* + * Simplifies the logical tree of valid and invalid constraints for pruning. + * The goal is to remove any node having just a single constraint associated with it. + * This constraint is assigned to the parent logical node. + * Removal of nodes is done by traversing from tree leafs upward. + * + * For example logical tree of + * AND(hash_col = 1, OR(X)) gets simplified into AND(hash_col = 1, X) + * Where X is any unknown condition. + */ +static void +SimplifyPruningTree(PruningTreeNode *node, PruningTreeNode *parent) +{ + /* Copy list of children as its mutated inside the loop */ + List *childBooleanNodes = list_copy(node->childBooleanNodes); + + ListCell *cell; + foreach(cell, childBooleanNodes) + { + PruningTreeNode *child = (PruningTreeNode *) lfirst(cell); + SimplifyPruningTree(child, node); + } + + if (!parent) + { + /* Root is always ANDed expressions */ + Assert(IsAndOp(node)); + return; + } + + /* Boolean operator with just a single (regocnized/unknown) constraints gets simplified */ + if (ConstraintCount(node) <= 1) + { + parent->validConstraints = list_concat(parent->validConstraints, + node->validConstraints); + parent->hasInvalidConstraints = parent->hasInvalidConstraints || + node->hasInvalidConstraints; + + /* Remove current node from parent. Its constraint was assigned to the parent above */ + parent->childBooleanNodes = list_delete_ptr(parent->childBooleanNodes, node); + } +} + + /* * ContainsFalseClause returns whether the flattened where clause list * contains false as a clause. @@ -439,7 +701,7 @@ ContainsFalseClause(List *whereClauseList) * storing them in context->pruningInstances. */ static void -PrunableExpressions(Node *node, ClauseWalkerContext *context) +PrunableExpressions(PruningTreeNode *tree, ClauseWalkerContext *context) { /* * Build initial list of prunable expressions. As long as only, @@ -451,7 +713,7 @@ PrunableExpressions(Node *node, ClauseWalkerContext *context) * expressions - that allows us to find all ANDed expressions, before * recursing into an ORed expression. */ - PrunableExpressionsWalker(node, context); + PrunableExpressionsWalker(tree, context); /* * Process all pending instances. While processing, new ones might be @@ -484,127 +746,129 @@ PrunableExpressions(Node *node, ClauseWalkerContext *context) * PrunableExpressionsWalker() is the main work horse for * PrunableExpressions(). */ -static bool -PrunableExpressionsWalker(Node *node, ClauseWalkerContext *context) +static void +PrunableExpressionsWalker(PruningTreeNode *node, ClauseWalkerContext *context) { + ListCell *cell = NULL; + if (node == NULL) { - return false; + return; } - /* - * Check for expressions understood by this routine. - */ - if (IsA(node, List)) + if (IsOrOp(node)) { - /* at the top of quals we'll frequently see lists, those are to be treated as ANDs */ + /* + * "Queue" partial pruning instances. This is used to convert + * expressions like (A AND (B OR C) AND D) into (A AND B AND D), + * (A AND C AND D), with A, B, C, D being restrictions. When the + * OR is encountered, a reference to the partially built + * PruningInstance (containing A at this point), is added to + * context->pendingInstances once for B and once for C. Once a + * full tree-walk completed, PrunableExpressions() will complete + * the pending instances, which'll now also know about restriction + * D, by calling PrunableExpressionsWalker() once for B and once + * for C. + */ + + if (node->hasInvalidConstraints) + { + PruningTreeNode *child = AndBooleanNode(); + child->hasInvalidConstraints = true; + + AddNewConjuction(context, child); + } + + foreach(cell, node->validConstraints) + { + Node *constraint = (Node *) lfirst(cell); + + PruningTreeNode *child = AndBooleanNode(); + child->validConstraints = list_make1(constraint); + + AddNewConjuction(context, child); + } + + foreach(cell, node->childBooleanNodes) + { + PruningTreeNode *child = (PruningTreeNode *) lfirst(cell); + Assert(IsAndOp(child)); + AddNewConjuction(context, child); + } + + return; } - else if (IsA(node, BoolExpr)) + + Assert(IsAndOp(node)); + + foreach(cell, node->validConstraints) { - BoolExpr *boolExpr = (BoolExpr *) node; + Node *constraint = (Node *) lfirst(cell); - if (boolExpr->boolop == NOT_EXPR) + if (IsA(constraint, OpExpr)) { - return false; - } - else if (boolExpr->boolop == AND_EXPR) - { - return expression_tree_walker((Node *) boolExpr->args, - PrunableExpressionsWalker, context); - } - else if (boolExpr->boolop == OR_EXPR) - { - ListCell *opCell = NULL; + OpExpr *opClause = (OpExpr *) constraint; + PruningInstance *prune = context->currentPruningInstance; + Var *varClause = NULL; + Const *constantClause = NULL; - /* - * "Queue" partial pruning instances. This is used to convert - * expressions like (A AND (B OR C) AND D) into (A AND B AND D), - * (A AND C AND D), with A, B, C, D being restrictions. When the - * OR is encountered, a reference to the partially built - * PruningInstance (containing A at this point), is added to - * context->pendingInstances once for B and once for C. Once a - * full tree-walk completed, PrunableExpressions() will complete - * the pending instances, which'll now also know about restriction - * D, by calling PrunableExpressionsWalker() once for B and once - * for C. - */ - foreach(opCell, boolExpr->args) + if (!prune->addedToPruningInstances) { - AddNewConjuction(context, lfirst(opCell)); + context->pruningInstances = lappend(context->pruningInstances, prune); + prune->addedToPruningInstances = true; } - return false; - } - } - else if (IsA(node, OpExpr)) - { - OpExpr *opClause = (OpExpr *) node; - PruningInstance *prune = context->currentPruningInstance; - Node *leftOperand = NULL; - Node *rightOperand = NULL; - Const *constantClause = NULL; - Var *varClause = NULL; - - if (!prune->addedToPruningInstances) - { - context->pruningInstances = lappend(context->pruningInstances, prune); - prune->addedToPruningInstances = true; - } - - if (list_length(opClause->args) == 2) - { - leftOperand = get_leftop((Expr *) opClause); - rightOperand = get_rightop((Expr *) opClause); - - leftOperand = strip_implicit_coercions(leftOperand); - rightOperand = strip_implicit_coercions(rightOperand); - - if (IsA(rightOperand, Const) && IsA(leftOperand, Var)) + if (VarConstOpExprClause(opClause, context->partitionColumn, &varClause, + &constantClause)) { - constantClause = (Const *) rightOperand; - varClause = (Var *) leftOperand; - } - else if (IsA(leftOperand, Const) && IsA(rightOperand, Var)) - { - constantClause = (Const *) leftOperand; - varClause = (Var *) rightOperand; - } - } - - if (constantClause && varClause && equal(varClause, context->partitionColumn)) - { - /* - * Found a restriction on the partition column itself. Update the - * current constraint with the new information. - */ - AddPartitionKeyRestrictionToInstance(context, opClause, varClause, + if (equal(varClause, context->partitionColumn)) + { + /* + * Found a restriction on the partition column itself. Update the + * current constraint with the new information. + */ + AddPartitionKeyRestrictionToInstance(context, opClause, varClause, + constantClause); + } + else if (varClause->varattno == RESERVED_HASHED_COLUMN_ID) + { + /* + * Found restriction that directly specifies the boundaries of a + * hashed column. + */ + AddHashRestrictionToInstance(context, opClause, varClause, constantClause); + } + else + { + /* We encounter here only valid constraints */ + Assert(false); + } + } + else + { + /* We encounter here only valid constraints */ + Assert(false); + } } - else if (constantClause && varClause && - varClause->varattno == RESERVED_HASHED_COLUMN_ID) + else if (IsA(constraint, ScalarArrayOpExpr)) { - /* - * Found restriction that directly specifies the boundaries of a - * hashed column. - */ - AddHashRestrictionToInstance(context, opClause, varClause, constantClause); + ScalarArrayOpExpr *arrayOperatorExpression = (ScalarArrayOpExpr *) constraint; + AddSAOPartitionKeyRestrictionToInstance(context, arrayOperatorExpression); + } + else + { + /* We encounter here only valid constraints */ + Assert(false); } - - return false; } - else if (IsA(node, ScalarArrayOpExpr)) - { - ScalarArrayOpExpr *arrayOperatorExpression = (ScalarArrayOpExpr *) node; - AddSAOPartitionKeyRestrictionToInstance(context, arrayOperatorExpression); - return false; - } - else + if (node->hasInvalidConstraints) { PruningInstance *prune = context->currentPruningInstance; /* - * Mark expression as added, so we'll fail pruning if there's no ANDed + * Mark unknown expression as added, so we'll fail pruning if there's no ANDed * restrictions that we know how to deal with. */ if (!prune->addedToPruningInstances) @@ -612,11 +876,62 @@ PrunableExpressionsWalker(Node *node, ClauseWalkerContext *context) context->pruningInstances = lappend(context->pruningInstances, prune); prune->addedToPruningInstances = true; } - - return false; } - return expression_tree_walker(node, PrunableExpressionsWalker, context); + foreach(cell, node->childBooleanNodes) + { + PruningTreeNode *child = (PruningTreeNode *) lfirst(cell); + Assert(IsOrOp(child)); + PrunableExpressionsWalker(child, context); + } +} + + +/* + * Check whether expression is a valid comparison of a var to a constant. + * Also obtaining the var with constant when valid. + */ +static bool +VarConstOpExprClause(OpExpr *opClause, Var *partitionColumn, Var **varClause, + Const **constantClause) +{ + Var *foundVarClause = NULL; + Const *foundConstantClause = NULL; + + if (list_length(opClause->args) == 2) + { + Node *leftOperand = get_leftop((Expr *) opClause); + Node *rightOperand = get_rightop((Expr *) opClause); + + leftOperand = strip_implicit_coercions(leftOperand); + rightOperand = strip_implicit_coercions(rightOperand); + + if (IsA(rightOperand, Const) && IsA(leftOperand, Var)) + { + foundVarClause = (Var *) leftOperand; + foundConstantClause = (Const *) rightOperand; + } + else if (IsA(leftOperand, Const) && IsA(rightOperand, Var)) + { + foundVarClause = (Var *) rightOperand; + foundConstantClause = (Const *) leftOperand; + } + } + + if (foundVarClause && foundConstantClause) + { + if (varClause) + { + *varClause = foundVarClause; + } + if (constantClause) + { + *constantClause = foundConstantClause; + } + return true; + } + + return false; } @@ -630,7 +945,27 @@ static void AddSAOPartitionKeyRestrictionToInstance(ClauseWalkerContext *context, ScalarArrayOpExpr *arrayOperatorExpression) { - PruningInstance *prune = context->currentPruningInstance; + List *restrictions = NULL; + if (SAORestrictions(arrayOperatorExpression, context->partitionColumn, &restrictions)) + { + PruningTreeNode *node = OrBooleanNode(); + node->validConstraints = restrictions; + AddNewConjuction(context, node); + } + else + { + Assert(false); + } +} + + +/* + * Check whether SAO constraint is valid. Also obtaining the built equality restrictions. + */ +static bool +SAORestrictions(ScalarArrayOpExpr *arrayOperatorExpression, Var *partitionColumn, + List **requestedRestrictions) +{ Node *leftOpExpression = linitial(arrayOperatorExpression->args); Node *strippedLeftOpExpression = strip_implicit_coercions(leftOpExpression); bool usingEqualityOperator = OperatorImplementsEquality( @@ -639,7 +974,7 @@ AddSAOPartitionKeyRestrictionToInstance(ClauseWalkerContext *context, /* checking for partcol = ANY(const, value, s); or partcol IN (const,b,c); */ if (usingEqualityOperator && strippedLeftOpExpression != NULL && - equal(strippedLeftOpExpression, context->partitionColumn) && + equal(strippedLeftOpExpression, partitionColumn) && IsA(arrayArgument, Const)) { int16 typlen = 0; @@ -648,11 +983,12 @@ AddSAOPartitionKeyRestrictionToInstance(ClauseWalkerContext *context, Datum arrayElement = 0; Datum inArray = ((Const *) arrayArgument)->constvalue; bool isNull = false; + bool foundValid = false; /* check for the NULL right-hand expression*/ if (inArray == 0) { - return; + return false; } ArrayType *array = DatumGetArrayTypeP(((Const *) arrayArgument)->constvalue); @@ -677,33 +1013,34 @@ AddSAOPartitionKeyRestrictionToInstance(ClauseWalkerContext *context, continue; } - Const *constElement = makeConst(elementType, -1, - DEFAULT_COLLATION_OID, typlen, arrayElement, - isNull, typbyval); + foundValid = true; - /* build partcol = arrayelem operator */ - OpExpr *arrayEqualityOp = makeNode(OpExpr); - arrayEqualityOp->opno = arrayOperatorExpression->opno; - arrayEqualityOp->opfuncid = arrayOperatorExpression->opfuncid; - arrayEqualityOp->inputcollid = arrayOperatorExpression->inputcollid; - arrayEqualityOp->opresulttype = get_func_rettype( - arrayOperatorExpression->opfuncid); - arrayEqualityOp->opcollid = context->partitionColumn->varcollid; - arrayEqualityOp->location = -1; - arrayEqualityOp->args = list_make2(strippedLeftOpExpression, constElement); + if (requestedRestrictions) + { + Const *constElement = makeConst(elementType, -1, + DEFAULT_COLLATION_OID, typlen, + arrayElement, + isNull, typbyval); - AddNewConjuction(context, arrayEqualityOp); + /* build partcol = arrayelem operator */ + OpExpr *arrayEqualityOp = SAORestrictionArrayEqualityOp( + arrayOperatorExpression, + partitionColumn); + arrayEqualityOp->args = list_make2(strippedLeftOpExpression, + constElement); + + *requestedRestrictions = lappend(*requestedRestrictions, arrayEqualityOp); + } + else + { + break; + } } + + return foundValid; } - /* Since we could not deal with the constraint, add the pruning instance to - * pruning instance list and labeled it as added. - */ - else if (!prune->addedToPruningInstances) - { - context->pruningInstances = lappend(context->pruningInstances, prune); - prune->addedToPruningInstances = true; - } + return false; } @@ -712,12 +1049,12 @@ AddSAOPartitionKeyRestrictionToInstance(ClauseWalkerContext *context, * as conjunction as partial instance. */ static void -AddNewConjuction(ClauseWalkerContext *context, OpExpr *op) +AddNewConjuction(ClauseWalkerContext *context, PruningTreeNode *node) { PendingPruningInstance *instance = palloc0(sizeof(PendingPruningInstance)); instance->instance = context->currentPruningInstance; - instance->continueAt = (Node *) op; + instance->continueAt = node; /* * Signal that this instance is not to be used for pruning on @@ -729,6 +1066,35 @@ AddNewConjuction(ClauseWalkerContext *context, OpExpr *op) } +/* + * Check whether operator clause is valid restriction for partition column. + */ +static bool +IsValidPartitionKeyRestriction(OpExpr *opClause) +{ + ListCell *btreeInterpretationCell = NULL; + bool matchedOp = false; + + List *btreeInterpretationList = + get_op_btree_interpretation(opClause->opno); + foreach(btreeInterpretationCell, btreeInterpretationList) + { + OpBtreeInterpretation *btreeInterpretation = + (OpBtreeInterpretation *) lfirst(btreeInterpretationCell); + + if (btreeInterpretation->strategy == ROWCOMPARE_NE) + { + /* TODO: could add support for this, if we feel like it */ + return false; + } + + matchedOp = true; + } + + return matchedOp; +} + + /* * AddPartitionKeyRestrictionToInstance adds information about a PartitionKey * $op Const restriction to the current pruning instance. @@ -739,7 +1105,6 @@ AddPartitionKeyRestrictionToInstance(ClauseWalkerContext *context, OpExpr *opCla { PruningInstance *prune = context->currentPruningInstance; ListCell *btreeInterpretationCell = NULL; - bool matchedOp = false; /* only have extra work to do if const isn't same type as partition column */ if (constantClause->consttype != partitionColumn->vartype) @@ -749,18 +1114,14 @@ AddPartitionKeyRestrictionToInstance(ClauseWalkerContext *context, OpExpr *opCla constantClause); if (constantClause == NULL) { - /* couldn't coerce value, so we save it in otherRestrictions */ - prune->otherRestrictions = lappend(prune->otherRestrictions, opClause); - + /* couldn't coerce value, its invalid restriction */ return; } } if (constantClause->constisnull) { - /* we cannot do pruning for NULL values, so we save it in otherRestrictions */ - prune->otherRestrictions = lappend(prune->otherRestrictions, opClause); - + /* we cannot do pruning on NULL values */ return; } @@ -785,7 +1146,6 @@ AddPartitionKeyRestrictionToInstance(ClauseWalkerContext *context, OpExpr *opCla { prune->lessConsts = constantClause; } - matchedOp = true; break; } @@ -799,7 +1159,6 @@ AddPartitionKeyRestrictionToInstance(ClauseWalkerContext *context, OpExpr *opCla { prune->lessEqualConsts = constantClause; } - matchedOp = true; break; } @@ -817,7 +1176,6 @@ AddPartitionKeyRestrictionToInstance(ClauseWalkerContext *context, OpExpr *opCla /* key can't be equal to two values */ prune->evaluatesToFalse = true; } - matchedOp = true; break; } @@ -832,7 +1190,6 @@ AddPartitionKeyRestrictionToInstance(ClauseWalkerContext *context, OpExpr *opCla { prune->greaterEqualConsts = constantClause; } - matchedOp = true; break; } @@ -846,27 +1203,6 @@ AddPartitionKeyRestrictionToInstance(ClauseWalkerContext *context, OpExpr *opCla { prune->greaterConsts = constantClause; } - matchedOp = true; - break; - } - - case ROWCOMPARE_NE: - { - /* - * This case should only arise when ALL list elements have this - * "strategy" number set. Skipping to the end of the list might - * protect us if that assumption is violated, and an Assert can - * notify us if it ever is... - */ - - /* should see this value immediately */ - Assert(btreeInterpretationCell == btreeInterpretationList->head); - - /* stop processing early, would only see unsupported nodes anyhow */ - btreeInterpretationCell = btreeInterpretationList->tail; - - /* TODO: could add support for this, if we feel like it */ - matchedOp = false; break; } @@ -875,14 +1211,7 @@ AddPartitionKeyRestrictionToInstance(ClauseWalkerContext *context, OpExpr *opCla } } - if (!matchedOp) - { - prune->otherRestrictions = lappend(prune->otherRestrictions, opClause); - } - else - { - prune->hasValidConstraint = true; - } + prune->hasValidConstraint = true; } @@ -926,20 +1255,13 @@ TransformPartitionRestrictionValue(Var *partitionColumn, Const *restrictionValue /* - * AddHashRestrictionToInstance adds information about a - * RESERVED_HASHED_COLUMN_ID = Const restriction to the current pruning - * instance. + * Check whether operator clause is valid restriction for hashed column. */ -static void -AddHashRestrictionToInstance(ClauseWalkerContext *context, OpExpr *opClause, - Var *varClause, Const *constantClause) +static bool +IsValidHashRestriction(OpExpr *opClause) { - PruningInstance *prune = context->currentPruningInstance; ListCell *btreeInterpretationCell = NULL; - /* be paranoid */ - Assert(IsBinaryCoercible(constantClause->consttype, INT4OID)); - List *btreeInterpretationList = get_op_btree_interpretation(opClause->opno); foreach(btreeInterpretationCell, btreeInterpretationList) @@ -947,21 +1269,41 @@ AddHashRestrictionToInstance(ClauseWalkerContext *context, OpExpr *opClause, OpBtreeInterpretation *btreeInterpretation = (OpBtreeInterpretation *) lfirst(btreeInterpretationCell); - /* - * Ladidadida, dirty hackety hack. We only add such - * constraints (in ShardIntervalOpExpressions()) to select a - * shard based on its exact boundaries. For efficient binary - * search it's better to simply use one representative value - * to look up the shard. In practice, this is sufficient for - * now. - */ if (btreeInterpretation->strategy == BTGreaterEqualStrategyNumber) { - Assert(!prune->hashedEqualConsts); - prune->hashedEqualConsts = constantClause; - prune->hasValidConstraint = true; + return true; } } + + return false; +} + + +/* + * AddHashRestrictionToInstance adds information about a + * RESERVED_HASHED_COLUMN_ID = Const restriction to the current pruning + * instance. + */ +static void +AddHashRestrictionToInstance(ClauseWalkerContext *context, OpExpr *opClause, + Var *varClause, Const *constantClause) +{ + /* be paranoid */ + Assert(IsBinaryCoercible(constantClause->consttype, INT4OID)); + Assert(IsValidHashRestriction(opClause)); + + /* + * Ladidadida, dirty hackety hack. We only add such + * constraints (in ShardIntervalOpExpressions()) to select a + * shard based on its exact boundaries. For efficient binary + * search it's better to simply use one representative value + * to look up the shard. In practice, this is sufficient for + * now. + */ + PruningInstance *prune = context->currentPruningInstance; + Assert(!prune->hashedEqualConsts); + prune->hashedEqualConsts = constantClause; + prune->hasValidConstraint = true; } @@ -1565,3 +1907,53 @@ ExhaustivePruneOne(ShardInterval *curInterval, return false; } + + +/* + * Helper for creating a node for pruning tree + */ +static inline PruningTreeNode * +CreatePruningNode(bool isAnd) +{ + PruningTreeNode *node = palloc0(sizeof(PruningTreeNode)); + node->isAnd = isAnd; + node->childBooleanNodes = NULL; + node->validConstraints = NULL; + node->hasInvalidConstraints = false; + return node; +} + + +/* + * Create equality operator for a single element of scalar array constraint. + */ +static inline OpExpr * +SAORestrictionArrayEqualityOp(ScalarArrayOpExpr *arrayOperatorExpression, + Var *partitionColumn) +{ + OpExpr *arrayEqualityOp = makeNode(OpExpr); + arrayEqualityOp->opno = arrayOperatorExpression->opno; + arrayEqualityOp->opfuncid = arrayOperatorExpression->opfuncid; + arrayEqualityOp->inputcollid = arrayOperatorExpression->inputcollid; + arrayEqualityOp->opresulttype = get_func_rettype( + arrayOperatorExpression->opfuncid); + arrayEqualityOp->opcollid = partitionColumn->varcollid; + arrayEqualityOp->location = -1; + return arrayEqualityOp; +} + + +/* + * Debug helper for logging expression nodes + */ +static inline void +DebugLogNode(char *fmt, Node *node, List *deparseCtx) +{ + if (!node) + { + return; + } + + char *deparsed = deparse_expression(node, deparseCtx, false, false); + ereport(DEBUG3, (errmsg(fmt, deparsed))); +} diff --git a/src/test/regress/expected/fast_path_router_modify.out b/src/test/regress/expected/fast_path_router_modify.out index 0780e20ce..f07dc600b 100644 --- a/src/test/regress/expected/fast_path_router_modify.out +++ b/src/test/regress/expected/fast_path_router_modify.out @@ -87,7 +87,6 @@ DEBUG: Plan is router executable DELETE FROM modify_fast_path WHERE value_1 = 15 AND (key = 1 OR value_2 = 'citus'); DEBUG: Creating router plan DEBUG: Plan is router executable -DETAIL: distribution column value: 1 -- goes through fast-path planning even if the key is updated to the same value UPDATE modify_fast_path SET key = 1 WHERE key = 1; DEBUG: Distributed planning for a fast-path router query diff --git a/src/test/regress/expected/multi_hash_pruning.out b/src/test/regress/expected/multi_hash_pruning.out index f61625c85..103f46026 100644 --- a/src/test/regress/expected/multi_hash_pruning.out +++ b/src/test/regress/expected/multi_hash_pruning.out @@ -29,6 +29,11 @@ SELECT create_distributed_table('orders_hash_partitioned', 'o_orderkey'); (1 row) +INSERT INTO orders_hash_partitioned (o_orderkey, o_custkey, o_totalprice, o_shippriority, o_clerk) VALUES + (1, 11, 10, 111, 'aaa'), + (2, 22, 20, 222, 'bbb'), + (3, 33, 30, 333, 'ccc'), + (4, 44, 40, 444, 'ddd'); SET client_min_messages TO DEBUG2; -- Check that we can prune shards for simple cases, boolean expressions and -- immutable functions. @@ -36,7 +41,7 @@ SELECT count(*) FROM orders_hash_partitioned; DEBUG: Router planner cannot handle multi-shard select queries count --------------------------------------------------------------------- - 0 + 4 (1 row) SELECT count(*) FROM orders_hash_partitioned WHERE o_orderkey = 1; @@ -45,7 +50,7 @@ DEBUG: Plan is router executable DETAIL: distribution column value: 1 count --------------------------------------------------------------------- - 0 + 1 (1 row) SELECT count(*) FROM orders_hash_partitioned WHERE o_orderkey = 2; @@ -54,7 +59,7 @@ DEBUG: Plan is router executable DETAIL: distribution column value: 2 count --------------------------------------------------------------------- - 0 + 1 (1 row) SELECT count(*) FROM orders_hash_partitioned WHERE o_orderkey = 3; @@ -63,7 +68,7 @@ DEBUG: Plan is router executable DETAIL: distribution column value: 3 count --------------------------------------------------------------------- - 0 + 1 (1 row) SELECT count(*) FROM orders_hash_partitioned WHERE o_orderkey = 4; @@ -72,7 +77,7 @@ DEBUG: Plan is router executable DETAIL: distribution column value: 4 count --------------------------------------------------------------------- - 0 + 1 (1 row) SELECT count(*) FROM orders_hash_partitioned @@ -82,7 +87,7 @@ DEBUG: Plan is router executable DETAIL: distribution column value: 1 count --------------------------------------------------------------------- - 0 + 1 (1 row) SELECT count(*) FROM orders_hash_partitioned WHERE o_orderkey = abs(-1); @@ -91,7 +96,7 @@ DEBUG: Plan is router executable DETAIL: distribution column value: 1 count --------------------------------------------------------------------- - 0 + 1 (1 row) -- disable router planning @@ -100,35 +105,35 @@ SELECT count(*) FROM orders_hash_partitioned; DEBUG: Router planner not enabled. count --------------------------------------------------------------------- - 0 + 4 (1 row) SELECT count(*) FROM orders_hash_partitioned WHERE o_orderkey = 1; DEBUG: Router planner not enabled. count --------------------------------------------------------------------- - 0 + 1 (1 row) SELECT count(*) FROM orders_hash_partitioned WHERE o_orderkey = 2; DEBUG: Router planner not enabled. count --------------------------------------------------------------------- - 0 + 1 (1 row) SELECT count(*) FROM orders_hash_partitioned WHERE o_orderkey = 3; DEBUG: Router planner not enabled. count --------------------------------------------------------------------- - 0 + 1 (1 row) SELECT count(*) FROM orders_hash_partitioned WHERE o_orderkey = 4; DEBUG: Router planner not enabled. count --------------------------------------------------------------------- - 0 + 1 (1 row) SELECT count(*) FROM orders_hash_partitioned @@ -136,14 +141,14 @@ SELECT count(*) FROM orders_hash_partitioned DEBUG: Router planner not enabled. count --------------------------------------------------------------------- - 0 + 1 (1 row) SELECT count(*) FROM orders_hash_partitioned WHERE o_orderkey = abs(-1); DEBUG: Router planner not enabled. count --------------------------------------------------------------------- - 0 + 1 (1 row) SET citus.enable_router_execution TO DEFAULT; @@ -158,14 +163,14 @@ SELECT count(*) FROM orders_hash_partitioned WHERE o_orderkey is not NULL; DEBUG: Router planner cannot handle multi-shard select queries count --------------------------------------------------------------------- - 0 + 4 (1 row) SELECT count(*) FROM orders_hash_partitioned WHERE o_orderkey > 2; DEBUG: Router planner cannot handle multi-shard select queries count --------------------------------------------------------------------- - 0 + 2 (1 row) SELECT count(*) FROM orders_hash_partitioned @@ -173,7 +178,7 @@ SELECT count(*) FROM orders_hash_partitioned DEBUG: Router planner cannot handle multi-shard select queries count --------------------------------------------------------------------- - 0 + 2 (1 row) SELECT count(*) FROM orders_hash_partitioned @@ -181,15 +186,15 @@ SELECT count(*) FROM orders_hash_partitioned DEBUG: Router planner cannot handle multi-shard select queries count --------------------------------------------------------------------- - 0 + 1 (1 row) SELECT count(*) FROM orders_hash_partitioned - WHERE o_orderkey = 1 OR (o_orderkey = 3 AND o_clerk = 'aaa'); + WHERE o_orderkey = 1 OR (o_orderkey = 3 AND o_clerk = 'ccc'); DEBUG: Router planner cannot handle multi-shard select queries count --------------------------------------------------------------------- - 0 + 2 (1 row) SELECT count(*) FROM orders_hash_partitioned @@ -197,7 +202,7 @@ SELECT count(*) FROM orders_hash_partitioned DEBUG: Router planner cannot handle multi-shard select queries count --------------------------------------------------------------------- - 0 + 1 (1 row) SELECT count(*) FROM @@ -207,13 +212,25 @@ DEBUG: Plan is router executable DETAIL: distribution column value: 1 count --------------------------------------------------------------------- - 0 + 1 (1 row) -SET client_min_messages TO DEFAULT; +SET client_min_messages TO DEBUG3; -- Check that we support runing for ANY/IN with literal. SELECT count(*) FROM lineitem_hash_part WHERE l_orderkey = ANY ('{1,2,3}'); +DEBUG: constraint value: '1'::bigint COLLATE "default", +DEBUG: constraint value: '2'::bigint COLLATE "default", +DEBUG: constraint value: '3'::bigint COLLATE "default", +DEBUG: shard count: 3 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: constraint value: '1'::bigint COLLATE "default", +DEBUG: constraint value: '2'::bigint COLLATE "default", +DEBUG: constraint value: '3'::bigint COLLATE "default", +DEBUG: shard count: 3 +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx count --------------------------------------------------------------------- 13 @@ -221,6 +238,18 @@ SELECT count(*) FROM lineitem_hash_part SELECT count(*) FROM lineitem_hash_part WHERE l_orderkey IN (1,2,3); +DEBUG: constraint value: '1'::bigint COLLATE "default", +DEBUG: constraint value: '2'::bigint COLLATE "default", +DEBUG: constraint value: '3'::bigint COLLATE "default", +DEBUG: shard count: 3 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: constraint value: '1'::bigint COLLATE "default", +DEBUG: constraint value: '2'::bigint COLLATE "default", +DEBUG: constraint value: '3'::bigint COLLATE "default", +DEBUG: shard count: 3 +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx count --------------------------------------------------------------------- 13 @@ -229,6 +258,8 @@ SELECT count(*) FROM lineitem_hash_part -- Check whether we can deal with null arrays SELECT count(*) FROM lineitem_hash_part WHERE l_orderkey IN (NULL); +DEBUG: Creating router plan +DEBUG: Plan is router executable count --------------------------------------------------------------------- 0 @@ -236,6 +267,15 @@ SELECT count(*) FROM lineitem_hash_part SELECT count(*) FROM lineitem_hash_part WHERE l_orderkey = ANY (NULL); +DEBUG: no valid constraints found +DEBUG: shard count: 4 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: no valid constraints found +DEBUG: shard count: 4 +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx count --------------------------------------------------------------------- 0 @@ -243,13 +283,31 @@ SELECT count(*) FROM lineitem_hash_part SELECT count(*) FROM lineitem_hash_part WHERE l_orderkey IN (NULL) OR TRUE; +DEBUG: no valid constraints found +DEBUG: shard count: 4 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: no valid constraints found +DEBUG: shard count: 4 +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx count --------------------------------------------------------------------- 12000 (1 row) SELECT count(*) FROM lineitem_hash_part - WHERE l_orderkey = ANY (NULL) OR TRUE; + WHERE l_orderkey = ANY (NULL) OR TRUE; +DEBUG: no valid constraints found +DEBUG: shard count: 4 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: no valid constraints found +DEBUG: shard count: 4 +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx count --------------------------------------------------------------------- 12000 @@ -257,20 +315,77 @@ SELECT count(*) FROM lineitem_hash_part -- Check whether we support IN/ANY in subquery SELECT count(*) FROM lineitem_hash_part WHERE l_orderkey IN (SELECT l_orderkey FROM lineitem_hash_part); +DEBUG: no valid constraints found +DEBUG: shard count: 4 +DEBUG: no valid constraints found +DEBUG: shard count: 4 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: no valid constraints found +DEBUG: shard count: 4 +DEBUG: no valid constraints found +DEBUG: shard count: 4 +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx count --------------------------------------------------------------------- 12000 (1 row) SELECT count(*) FROM lineitem_hash_part WHERE l_orderkey = ANY (SELECT l_orderkey FROM lineitem_hash_part); +DEBUG: no valid constraints found +DEBUG: shard count: 4 +DEBUG: no valid constraints found +DEBUG: shard count: 4 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: no valid constraints found +DEBUG: shard count: 4 +DEBUG: no valid constraints found +DEBUG: shard count: 4 +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx count --------------------------------------------------------------------- 12000 (1 row) +-- Check whether we support range queries with append distributed table +SELECT count(*) FROM lineitem + WHERE l_orderkey >= 1 AND l_orderkey <= 3; +DEBUG: Router planner does not support append-partitioned tables. +DEBUG: constraint (lteq) value: '3'::bigint, +DEBUG: constraint (gteq) value: '1'::bigint, +DEBUG: shard count: 1 +DEBUG: assigned task to node localhost:xxxxx + count +--------------------------------------------------------------------- + 13 +(1 row) + +SELECT count(*) FROM lineitem + WHERE (l_orderkey >= 1 AND l_orderkey <= 3) AND (l_quantity > 11 AND l_quantity < 22); +DEBUG: Router planner does not support append-partitioned tables. +DEBUG: constraint (lteq) value: '3'::bigint, +DEBUG: constraint (gteq) value: '1'::bigint, +DEBUG: shard count: 1 +DEBUG: assigned task to node localhost:xxxxx + count +--------------------------------------------------------------------- + 1 +(1 row) + -- Check whether we support IN/ANY in subquery with append and range distributed table SELECT count(*) FROM lineitem WHERE l_orderkey = ANY ('{1,2,3}'); +DEBUG: Router planner does not support append-partitioned tables. +DEBUG: constraint value: '1'::bigint COLLATE "default", +DEBUG: constraint value: '2'::bigint COLLATE "default", +DEBUG: constraint value: '3'::bigint COLLATE "default", +DEBUG: shard count: 1 +DEBUG: assigned task to node localhost:xxxxx count --------------------------------------------------------------------- 13 @@ -278,13 +393,24 @@ SELECT count(*) FROM lineitem SELECT count(*) FROM lineitem WHERE l_orderkey IN (1,2,3); +DEBUG: Router planner does not support append-partitioned tables. +DEBUG: constraint value: '1'::bigint COLLATE "default", +DEBUG: constraint value: '2'::bigint COLLATE "default", +DEBUG: constraint value: '3'::bigint COLLATE "default", +DEBUG: shard count: 1 +DEBUG: assigned task to node localhost:xxxxx count --------------------------------------------------------------------- 13 (1 row) SELECT count(*) FROM lineitem - WHERE l_orderkey = ANY(NULL) OR TRUE; + WHERE l_orderkey = ANY(NULL) OR TRUE; +DEBUG: Router planner does not support append-partitioned tables. +DEBUG: no valid constraints found +DEBUG: shard count: 2 +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx count --------------------------------------------------------------------- 12000 @@ -292,6 +418,12 @@ SELECT count(*) FROM lineitem SELECT count(*) FROM lineitem_range WHERE l_orderkey = ANY ('{1,2,3}'); +DEBUG: constraint value: '1'::bigint COLLATE "default", +DEBUG: constraint value: '2'::bigint COLLATE "default", +DEBUG: constraint value: '3'::bigint COLLATE "default", +DEBUG: shard count: 1 +DEBUG: Creating router plan +DEBUG: Plan is router executable count --------------------------------------------------------------------- 13 @@ -299,24 +431,44 @@ SELECT count(*) FROM lineitem_range SELECT count(*) FROM lineitem_range WHERE l_orderkey IN (1,2,3); +DEBUG: constraint value: '1'::bigint COLLATE "default", +DEBUG: constraint value: '2'::bigint COLLATE "default", +DEBUG: constraint value: '3'::bigint COLLATE "default", +DEBUG: shard count: 1 +DEBUG: Creating router plan +DEBUG: Plan is router executable count --------------------------------------------------------------------- 13 (1 row) SELECT count(*) FROM lineitem_range - WHERE l_orderkey = ANY(NULL) OR TRUE; + WHERE l_orderkey = ANY(NULL) OR TRUE; +DEBUG: no valid constraints found +DEBUG: shard count: 2 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: no valid constraints found +DEBUG: shard count: 2 +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx count --------------------------------------------------------------------- 12000 (1 row) -SET client_min_messages TO DEBUG2; -- Check that we don't show the message if the operator is not -- equality operator SELECT count(*) FROM orders_hash_partitioned WHERE o_orderkey < ALL ('{1,2,3}'); +DEBUG: no valid constraints found +DEBUG: shard count: 4 DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: no valid constraints found +DEBUG: shard count: 4 +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx count --------------------------------------------------------------------- 0 @@ -325,31 +477,57 @@ DEBUG: Router planner cannot handle multi-shard select queries -- Check that we don't give a spurious hint message when non-partition -- columns are used with ANY/IN/ALL SELECT count(*) FROM orders_hash_partitioned - WHERE o_orderkey = 1 OR o_totalprice IN (2, 5); + WHERE o_orderkey = 1 OR o_totalprice IN (20, 30); +DEBUG: no valid constraints found +DEBUG: shard count: 4 DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: no valid constraints found +DEBUG: shard count: 4 +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx count --------------------------------------------------------------------- - 0 + 3 (1 row) -- Check that we cannot prune for mutable functions. -SELECT count(*) FROM orders_hash_partitioned WHERE o_orderkey = random(); +SELECT count(*) FROM orders_hash_partitioned WHERE o_orderkey = (random() + 100); +DEBUG: no valid constraints found +DEBUG: shard count: 4 DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: no valid constraints found +DEBUG: shard count: 4 +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx count --------------------------------------------------------------------- 0 (1 row) SELECT count(*) FROM orders_hash_partitioned - WHERE o_orderkey = random() OR o_orderkey = 1; + WHERE o_orderkey = (random() + 100) OR o_orderkey = 1; +DEBUG: no valid constraints found +DEBUG: shard count: 4 DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: no valid constraints found +DEBUG: shard count: 4 +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx count --------------------------------------------------------------------- - 0 + 1 (1 row) SELECT count(*) FROM orders_hash_partitioned - WHERE o_orderkey = random() AND o_orderkey = 1; + WHERE o_orderkey = (random() + 100) AND o_orderkey = 1; +DEBUG: constraint value: 1, +DEBUG: shard count: 1 DEBUG: Creating router plan DEBUG: Plan is router executable DETAIL: distribution column value: 1 @@ -362,7 +540,15 @@ DETAIL: distribution column value: 1 SELECT count(*) FROM orders_hash_partitioned orders1, orders_hash_partitioned orders2 WHERE orders1.o_orderkey = orders2.o_orderkey; +DEBUG: no valid constraints found +DEBUG: shard count: 4 +DEBUG: no valid constraints found +DEBUG: shard count: 4 DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: no valid constraints found +DEBUG: shard count: 4 +DEBUG: no valid constraints found +DEBUG: shard count: 4 DEBUG: join prunable for intervals [-2147483648,-1073741825] and [-1073741824,-1] DEBUG: join prunable for intervals [-2147483648,-1073741825] and [0,1073741823] DEBUG: join prunable for intervals [-2147483648,-1073741825] and [1073741824,2147483647] @@ -375,9 +561,13 @@ DEBUG: join prunable for intervals [0,1073741823] and [1073741824,2147483647] DEBUG: join prunable for intervals [1073741824,2147483647] and [-2147483648,-1073741825] DEBUG: join prunable for intervals [1073741824,2147483647] and [-1073741824,-1] DEBUG: join prunable for intervals [1073741824,2147483647] and [0,1073741823] +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx count --------------------------------------------------------------------- - 0 + 4 (1 row) SELECT count(*) @@ -385,6 +575,10 @@ SELECT count(*) WHERE orders1.o_orderkey = orders2.o_orderkey AND orders1.o_orderkey = 1 AND orders2.o_orderkey is NULL; +DEBUG: constraint value: 1, +DEBUG: shard count: 1 +DEBUG: constraint value: 1, +DEBUG: shard count: 1 DEBUG: Creating router plan DEBUG: Plan is router executable DETAIL: distribution column value: 1 @@ -393,3 +587,412 @@ DETAIL: distribution column value: 1 0 (1 row) +-- All shards used without constraints +SELECT count(*) FROM orders_hash_partitioned; +DEBUG: no valid constraints found +DEBUG: shard count: 4 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: no valid constraints found +DEBUG: shard count: 4 +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx + count +--------------------------------------------------------------------- + 4 +(1 row) + +-- Shards restricted correctly with prunable constraint +SELECT count(*) FROM orders_hash_partitioned + WHERE o_orderkey = 1; +DEBUG: constraint value: 1, +DEBUG: shard count: 1 +DEBUG: Creating router plan +DEBUG: Plan is router executable +DETAIL: distribution column value: 1 + count +--------------------------------------------------------------------- + 1 +(1 row) + +-- Shards restricted correctly with prunable constraint ANDed with unprunable expression using OR +SELECT count(*) FROM orders_hash_partitioned + WHERE o_orderkey = 1 AND (o_custkey = 11 OR o_custkey = 22); +DEBUG: constraint value: 1, +DEBUG: shard count: 1 +DEBUG: Creating router plan +DEBUG: Plan is router executable +DETAIL: distribution column value: 1 + count +--------------------------------------------------------------------- + 1 +(1 row) + +-- Shards restricted correctly with prunable constraints ORed +SELECT count(*) FROM orders_hash_partitioned + WHERE (o_orderkey = 1 OR o_orderkey = 2); +DEBUG: constraint value: 1, +DEBUG: constraint value: 2, +DEBUG: shard count: 2 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: constraint value: 1, +DEBUG: constraint value: 2, +DEBUG: shard count: 2 +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx + count +--------------------------------------------------------------------- + 2 +(1 row) + +-- Shards restricted correctly with prunable constraints ANDed with unprunable expression using OR +SELECT count(*) FROM orders_hash_partitioned + WHERE (o_orderkey = 1 OR o_orderkey = 2) AND (o_custkey = 11 OR o_custkey = 22); +DEBUG: constraint value: 1, +DEBUG: constraint value: 2, +DEBUG: shard count: 2 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: constraint value: 1, +DEBUG: constraint value: 2, +DEBUG: shard count: 2 +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx + count +--------------------------------------------------------------------- + 2 +(1 row) + +-- Shards restricted correctly with many different prunable constraints ORed +SELECT count(*) FROM orders_hash_partitioned + WHERE (o_orderkey = 1 AND o_custkey = 11) OR (o_orderkey = 1 AND o_custkey = 33) OR (o_orderkey = 2 AND o_custkey = 22) OR (o_orderkey = 2 AND o_custkey = 44); +DEBUG: constraint value: 1, +DEBUG: constraint value: 1, +DEBUG: constraint value: 2, +DEBUG: constraint value: 2, +DEBUG: shard count: 2 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: constraint value: 1, +DEBUG: constraint value: 1, +DEBUG: constraint value: 2, +DEBUG: constraint value: 2, +DEBUG: shard count: 2 +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx + count +--------------------------------------------------------------------- + 2 +(1 row) + +-- Shards restricted correctly with prunable SAO constraint ANDed with unprunable expression using OR +SELECT count(*) FROM orders_hash_partitioned + WHERE (o_orderkey IN (1,2)) AND (o_custkey = 11 OR o_custkey = 22 OR o_custkey = 33); +DEBUG: constraint value: 1 COLLATE "default", +DEBUG: constraint value: 2 COLLATE "default", +DEBUG: shard count: 2 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: constraint value: 1 COLLATE "default", +DEBUG: constraint value: 2 COLLATE "default", +DEBUG: shard count: 2 +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx + count +--------------------------------------------------------------------- + 2 +(1 row) + +-- Shards restricted correctly with prunable SAO constraint ANDed with multiple unprunable expressions +SELECT count(*) FROM orders_hash_partitioned + WHERE (o_orderkey IN (1,2)) AND (o_totalprice < 11 OR o_totalprice > 19) AND o_shippriority > 100 AND (o_custkey = 11 OR o_custkey = 22); +DEBUG: constraint value: 1 COLLATE "default", +DEBUG: constraint value: 2 COLLATE "default", +DEBUG: shard count: 2 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: constraint value: 1 COLLATE "default", +DEBUG: constraint value: 2 COLLATE "default", +DEBUG: shard count: 2 +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx + count +--------------------------------------------------------------------- + 2 +(1 row) + +-- Shards restricted correctly with prunable SAO constraints ORed +SELECT count(*) FROM orders_hash_partitioned + WHERE (o_orderkey IN (1,2) AND o_custkey = 11) OR (o_orderkey IN (2,3) AND o_custkey = 22); +DEBUG: constraint value: 1 COLLATE "default", +DEBUG: constraint value: 2 COLLATE "default", +DEBUG: constraint value: 2 COLLATE "default", +DEBUG: constraint value: 3 COLLATE "default", +DEBUG: shard count: 3 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: constraint value: 1 COLLATE "default", +DEBUG: constraint value: 2 COLLATE "default", +DEBUG: constraint value: 2 COLLATE "default", +DEBUG: constraint value: 3 COLLATE "default", +DEBUG: shard count: 3 +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx + count +--------------------------------------------------------------------- + 2 +(1 row) + +-- All shards used with prunable expression ORed with unprunable expression +SELECT count(*) FROM orders_hash_partitioned + WHERE o_orderkey IN (1,2) OR o_custkey = 33; +DEBUG: no valid constraints found +DEBUG: shard count: 4 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: no valid constraints found +DEBUG: shard count: 4 +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx + count +--------------------------------------------------------------------- + 3 +(1 row) + +-- Shards restricted correctly with prunable constraint ORed +SELECT count(*) FROM orders_hash_partitioned + WHERE o_orderkey = 1 OR ((o_orderkey = 2 AND o_custkey = 22) OR (o_orderkey = 3 AND o_custkey = 33)); +DEBUG: constraint value: 1, +DEBUG: constraint value: 2, +DEBUG: constraint value: 3, +DEBUG: shard count: 3 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: constraint value: 1, +DEBUG: constraint value: 2, +DEBUG: constraint value: 3, +DEBUG: shard count: 3 +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx + count +--------------------------------------------------------------------- + 3 +(1 row) + +-- Shards restricted correctly with prunable constraint ORed with falsy expression +SELECT count(*) FROM orders_hash_partitioned + WHERE o_orderkey = 1 OR (o_orderkey = 2 AND (o_custkey = 11 OR (o_orderkey = 3 AND o_custkey = 44))); +DEBUG: constraint value: 1, +DEBUG: constraint value: 2, +DEBUG: shard count: 2 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: constraint value: 1, +DEBUG: constraint value: 2, +DEBUG: shard count: 2 +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx + count +--------------------------------------------------------------------- + 1 +(1 row) + +-- Shards restricted correctly with prunable SAO constraint ORed with prunable nested EQ constraint +SELECT count(*) FROM orders_hash_partitioned + WHERE (o_orderkey IN (1,2)) AND (o_custkey = 11 OR o_custkey = 22 OR o_custkey = 33) AND o_totalprice <= 20; +DEBUG: constraint value: 1 COLLATE "default", +DEBUG: constraint value: 2 COLLATE "default", +DEBUG: shard count: 2 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: constraint value: 1 COLLATE "default", +DEBUG: constraint value: 2 COLLATE "default", +DEBUG: shard count: 2 +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx + count +--------------------------------------------------------------------- + 2 +(1 row) + +-- Shards restricted correctly with prunable SAO constraint ANDed with unprunable expressions +SELECT count(*) FROM orders_hash_partitioned + WHERE (o_orderkey IN (1,2)) AND (o_custkey = 11 OR o_custkey = 33) AND o_custkey = 22; +DEBUG: constraint value: 1 COLLATE "default", +DEBUG: constraint value: 2 COLLATE "default", +DEBUG: shard count: 2 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: constraint value: 1 COLLATE "default", +DEBUG: constraint value: 2 COLLATE "default", +DEBUG: shard count: 2 +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx + count +--------------------------------------------------------------------- + 0 +(1 row) + +-- All shards used with prunable SAO constraint ORed with unprunable nested expression +SELECT count(*) FROM orders_hash_partitioned + WHERE ((o_orderkey IN (1,2)) AND (o_custkey = 11 OR o_custkey = 22)) OR o_custkey = 33; +DEBUG: no valid constraints found +DEBUG: shard count: 4 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: no valid constraints found +DEBUG: shard count: 4 +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx + count +--------------------------------------------------------------------- + 3 +(1 row) + +-- Shards restricted correctly with prunable SAO constraint ORed with prunable nested EQ constraint +SELECT count(*) FROM orders_hash_partitioned + WHERE ((o_orderkey IN (1,2)) AND (o_custkey = 11 OR o_custkey = 22)) OR (o_orderkey = 3 AND o_custkey = 33); +DEBUG: constraint value: 3, +DEBUG: constraint value: 1 COLLATE "default", +DEBUG: constraint value: 2 COLLATE "default", +DEBUG: shard count: 3 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: constraint value: 3, +DEBUG: constraint value: 1 COLLATE "default", +DEBUG: constraint value: 2 COLLATE "default", +DEBUG: shard count: 3 +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx + count +--------------------------------------------------------------------- + 3 +(1 row) + +-- All shards used with ORed top level unprunable expression +SELECT count(*) FROM orders_hash_partitioned + WHERE o_custkey = 11 OR (o_orderkey = 2 AND o_custkey = 22); +DEBUG: no valid constraints found +DEBUG: shard count: 4 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: no valid constraints found +DEBUG: shard count: 4 +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx + count +--------------------------------------------------------------------- + 2 +(1 row) + +-- Single shard used when deeply nested prunable expression is restrictive with nested ANDs +SELECT count(*) FROM orders_hash_partitioned + WHERE o_orderkey = 1 OR (o_orderkey = 2 AND (o_orderkey = 3 OR (o_orderkey = 1 AND o_custkey = 11))); +DEBUG: constraint value: 1, +DEBUG: shard count: 1 +DEBUG: Creating router plan +DEBUG: Plan is router executable + count +--------------------------------------------------------------------- + 1 +(1 row) + +-- Single shard used when top prunable expression is restrictive with nested ANDs +SELECT count(*) FROM orders_hash_partitioned + WHERE o_orderkey = 1 AND ((o_orderkey = 2 OR o_orderkey = 3) AND (o_custkey = 11 OR o_custkey = 22)); +DEBUG: no valid constraints found +DEBUG: shard count: 0 +DEBUG: Creating router plan +DEBUG: Plan is router executable +DETAIL: distribution column value: 1 + count +--------------------------------------------------------------------- + 0 +(1 row) + +-- Deeply nested prunable expression affects used shards +SELECT count(*) FROM orders_hash_partitioned + WHERE o_orderkey = 1 OR ((o_orderkey = 2 OR o_orderkey = 3) AND (o_custkey = 22 OR o_custkey = 33)); +DEBUG: constraint value: 1, +DEBUG: constraint value: 2, +DEBUG: constraint value: 3, +DEBUG: shard count: 3 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: constraint value: 1, +DEBUG: constraint value: 2, +DEBUG: constraint value: 3, +DEBUG: shard count: 3 +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx + count +--------------------------------------------------------------------- + 3 +(1 row) + +-- Deeply nested non prunable expression uses all shards +SELECT count(*) FROM orders_hash_partitioned + WHERE o_orderkey = 1 OR ((o_orderkey = 2 OR o_custkey = 11) AND (o_custkey = 22 OR o_custkey = 33)); +DEBUG: no valid constraints found +DEBUG: shard count: 4 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: no valid constraints found +DEBUG: shard count: 4 +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx + count +--------------------------------------------------------------------- + 2 +(1 row) + +-- a OR partkey != x Uses all shards +SELECT count(*) FROM orders_hash_partitioned + WHERE o_orderkey = 1 OR o_orderkey != 2; +DEBUG: no valid constraints found +DEBUG: shard count: 4 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: no valid constraints found +DEBUG: shard count: 4 +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx + count +--------------------------------------------------------------------- + 3 +(1 row) + +-- a OR partkey IS NULL Uses all shards +SELECT count(*) FROM orders_hash_partitioned + WHERE o_orderkey = 1 OR o_orderkey IS NULL; +DEBUG: no valid constraints found +DEBUG: shard count: 4 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: no valid constraints found +DEBUG: shard count: 4 +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx + count +--------------------------------------------------------------------- + 1 +(1 row) + +-- a OR partkey IS NOT NULL Uses all shards +SELECT count(*) FROM orders_hash_partitioned + WHERE o_orderkey = 1 OR o_orderkey IS NOT NULL; +DEBUG: no valid constraints found +DEBUG: shard count: 4 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: no valid constraints found +DEBUG: shard count: 4 +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx +DEBUG: assigned task to node localhost:xxxxx + count +--------------------------------------------------------------------- + 4 +(1 row) + +SET client_min_messages TO DEFAULT; diff --git a/src/test/regress/expected/multi_repartition_join_planning.out b/src/test/regress/expected/multi_repartition_join_planning.out index 3f3340469..6990eaeb0 100644 --- a/src/test/regress/expected/multi_repartition_join_planning.out +++ b/src/test/regress/expected/multi_repartition_join_planning.out @@ -65,6 +65,10 @@ GROUP BY ORDER BY l_partkey, o_orderkey; DEBUG: Router planner does not support append-partitioned tables. +DEBUG: no valid constraints found +DEBUG: shard count: 2 +DEBUG: no valid constraints found +DEBUG: shard count: 2 DEBUG: join prunable for intervals [1,5986] and [8997,14947] DEBUG: join prunable for intervals [8997,14947] and [1,5986] DEBUG: generated sql query for task 1 @@ -73,6 +77,8 @@ DEBUG: generated sql query for task 2 DETAIL: query string: "SELECT lineitem.l_partkey, orders.o_orderkey, lineitem.l_quantity, lineitem.l_extendedprice, orders.o_custkey FROM (lineitem_290001 lineitem JOIN orders_290003 orders ON ((lineitem.l_orderkey OPERATOR(pg_catalog.=) orders.o_orderkey))) WHERE ((lineitem.l_partkey OPERATOR(pg_catalog.<) 1000) AND (orders.o_totalprice OPERATOR(pg_catalog.>) 10.0))" DEBUG: assigned task to node localhost:xxxxx DEBUG: assigned task to node localhost:xxxxx +DEBUG: no valid constraints found +DEBUG: shard count: 2 DEBUG: join prunable for intervals [1,1000] and [6001,7000] DEBUG: join prunable for intervals [6001,7000] and [1,1000] DEBUG: generated sql query for task 2 @@ -85,6 +91,8 @@ DEBUG: pruning merge fetch taskId 3 DETAIL: Creating dependency on merge taskId 6 DEBUG: assigned task to node localhost:xxxxx DEBUG: assigned task to node localhost:xxxxx +DEBUG: no valid constraints found +DEBUG: shard count: 3 DEBUG: join prunable for intervals [1,1000] and [1001,2000] DEBUG: join prunable for intervals [1,1000] and [6001,7000] DEBUG: join prunable for intervals [1001,2000] and [1,1000] @@ -157,12 +165,16 @@ GROUP BY ORDER BY l_partkey, o_orderkey; DEBUG: Router planner does not support append-partitioned tables. +DEBUG: no valid constraints found +DEBUG: shard count: 2 DEBUG: generated sql query for task 1 DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290000 lineitem WHERE (l_quantity OPERATOR(pg_catalog.<) 5.0)" DEBUG: generated sql query for task 2 DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290001 lineitem WHERE (l_quantity OPERATOR(pg_catalog.<) 5.0)" DEBUG: assigned task to node localhost:xxxxx DEBUG: assigned task to node localhost:xxxxx +DEBUG: no valid constraints found +DEBUG: shard count: 2 DEBUG: generated sql query for task 1 DETAIL: query string: "SELECT o_orderkey, o_shippriority FROM orders_290002 orders WHERE (o_totalprice OPERATOR(pg_catalog.<>) 4.0)" DEBUG: generated sql query for task 2 @@ -231,12 +243,16 @@ GROUP BY ORDER BY o_orderkey; DEBUG: Router planner does not support append-partitioned tables. +DEBUG: no valid constraints found +DEBUG: shard count: 2 DEBUG: generated sql query for task 1 DETAIL: query string: "SELECT l_suppkey FROM lineitem_290000 lineitem WHERE true" DEBUG: generated sql query for task 2 DETAIL: query string: "SELECT l_suppkey FROM lineitem_290001 lineitem WHERE true" DEBUG: assigned task to node localhost:xxxxx DEBUG: assigned task to node localhost:xxxxx +DEBUG: no valid constraints found +DEBUG: shard count: 2 DEBUG: generated sql query for task 1 DETAIL: query string: "SELECT o_orderkey, o_shippriority FROM orders_290002 orders WHERE true" DEBUG: generated sql query for task 2 @@ -307,12 +323,16 @@ GROUP BY ORDER BY o_orderkey; DEBUG: Router planner does not support append-partitioned tables. +DEBUG: no valid constraints found +DEBUG: shard count: 2 DEBUG: generated sql query for task 1 DETAIL: query string: "SELECT l_suppkey FROM lineitem_290000 lineitem WHERE true" DEBUG: generated sql query for task 2 DETAIL: query string: "SELECT l_suppkey FROM lineitem_290001 lineitem WHERE true" DEBUG: assigned task to node localhost:xxxxx DEBUG: assigned task to node localhost:xxxxx +DEBUG: no valid constraints found +DEBUG: shard count: 2 DEBUG: generated sql query for task 1 DETAIL: query string: "SELECT o_orderkey, o_shippriority FROM orders_290002 orders WHERE true" DEBUG: generated sql query for task 2 @@ -381,12 +401,16 @@ GROUP BY ORDER BY o_orderkey; DEBUG: Router planner does not support append-partitioned tables. +DEBUG: no valid constraints found +DEBUG: shard count: 2 DEBUG: generated sql query for task 1 DETAIL: query string: "SELECT l_suppkey FROM lineitem_290000 lineitem WHERE true" DEBUG: generated sql query for task 2 DETAIL: query string: "SELECT l_suppkey FROM lineitem_290001 lineitem WHERE true" DEBUG: assigned task to node localhost:xxxxx DEBUG: assigned task to node localhost:xxxxx +DEBUG: no valid constraints found +DEBUG: shard count: 2 DEBUG: generated sql query for task 1 DETAIL: query string: "SELECT o_orderkey, o_shippriority FROM orders_290002 orders WHERE true" DEBUG: generated sql query for task 2 @@ -452,7 +476,13 @@ select s_i_id group by s_i_id, s_w_id, s_quantity having s_quantity > random() ; +DEBUG: no valid constraints found +DEBUG: shard count: 4 +DEBUG: no valid constraints found +DEBUG: shard count: 4 DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: no valid constraints found +DEBUG: shard count: 4 DEBUG: generated sql query for task 1 DETAIL: query string: "SELECT s_i_id, s_w_id, s_quantity FROM stock_690004 stock WHERE true" DEBUG: generated sql query for task 2 @@ -465,6 +495,8 @@ DEBUG: assigned task to node localhost:xxxxx DEBUG: assigned task to node localhost:xxxxx DEBUG: assigned task to node localhost:xxxxx DEBUG: assigned task to node localhost:xxxxx +DEBUG: no valid constraints found +DEBUG: shard count: 4 DEBUG: generated sql query for task 1 DETAIL: query string: "SELECT ol_i_id FROM order_line_690000 order_line WHERE true" DEBUG: generated sql query for task 2 diff --git a/src/test/regress/expected/multi_repartition_join_task_assignment.out b/src/test/regress/expected/multi_repartition_join_task_assignment.out index d3146999d..78d0216a8 100644 --- a/src/test/regress/expected/multi_repartition_join_task_assignment.out +++ b/src/test/regress/expected/multi_repartition_join_task_assignment.out @@ -18,8 +18,12 @@ FROM WHERE o_custkey = c_custkey; DEBUG: Router planner does not support append-partitioned tables. +DEBUG: no valid constraints found +DEBUG: shard count: 2 DEBUG: assigned task to node localhost:xxxxx DEBUG: assigned task to node localhost:xxxxx +DEBUG: no valid constraints found +DEBUG: shard count: 3 DEBUG: join prunable for intervals [1,1000] and [1001,2000] DEBUG: join prunable for intervals [1,1000] and [6001,7000] DEBUG: join prunable for intervals [1001,2000] and [1,1000] @@ -52,9 +56,13 @@ WHERE o_custkey = c_custkey AND o_orderkey = l_orderkey; DEBUG: Router planner does not support append-partitioned tables. +DEBUG: no valid constraints found +DEBUG: shard count: 3 DEBUG: assigned task to node localhost:xxxxx DEBUG: assigned task to node localhost:xxxxx DEBUG: assigned task to node localhost:xxxxx +DEBUG: no valid constraints found +DEBUG: shard count: 2 DEBUG: join prunable for intervals [1,5986] and [8997,14947] DEBUG: join prunable for intervals [8997,14947] and [1,5986] DEBUG: pruning merge fetch taskId 1 @@ -77,8 +85,12 @@ FROM WHERE l_partkey = c_nationkey; DEBUG: Router planner does not support append-partitioned tables. +DEBUG: no valid constraints found +DEBUG: shard count: 2 DEBUG: assigned task to node localhost:xxxxx DEBUG: assigned task to node localhost:xxxxx +DEBUG: no valid constraints found +DEBUG: shard count: 3 DEBUG: assigned task to node localhost:xxxxx DEBUG: assigned task to node localhost:xxxxx DEBUG: assigned task to node localhost:xxxxx diff --git a/src/test/regress/expected/multi_task_assignment_policy.out b/src/test/regress/expected/multi_task_assignment_policy.out index e44d8ee9a..5fe888c8d 100644 --- a/src/test/regress/expected/multi_task_assignment_policy.out +++ b/src/test/regress/expected/multi_task_assignment_policy.out @@ -70,6 +70,8 @@ SET client_min_messages TO DEBUG3; SET citus.task_assignment_policy TO 'greedy'; EXPLAIN SELECT count(*) FROM task_assignment_test_table; DEBUG: Router planner does not support append-partitioned tables. +DEBUG: no valid constraints found +DEBUG: shard count: 3 DEBUG: assigned task to node localhost:xxxxx DEBUG: assigned task to node localhost:xxxxx DEBUG: assigned task to node localhost:xxxxx @@ -82,6 +84,8 @@ DEBUG: assigned task to node localhost:xxxxx EXPLAIN SELECT count(*) FROM task_assignment_test_table; DEBUG: Router planner does not support append-partitioned tables. +DEBUG: no valid constraints found +DEBUG: shard count: 3 DEBUG: assigned task to node localhost:xxxxx DEBUG: assigned task to node localhost:xxxxx DEBUG: assigned task to node localhost:xxxxx @@ -96,6 +100,8 @@ DEBUG: assigned task to node localhost:xxxxx SET citus.task_assignment_policy TO 'first-replica'; EXPLAIN SELECT count(*) FROM task_assignment_test_table; DEBUG: Router planner does not support append-partitioned tables. +DEBUG: no valid constraints found +DEBUG: shard count: 3 DEBUG: assigned task to node localhost:xxxxx DEBUG: assigned task to node localhost:xxxxx DEBUG: assigned task to node localhost:xxxxx @@ -108,6 +114,8 @@ DEBUG: assigned task to node localhost:xxxxx EXPLAIN SELECT count(*) FROM task_assignment_test_table; DEBUG: Router planner does not support append-partitioned tables. +DEBUG: no valid constraints found +DEBUG: shard count: 3 DEBUG: assigned task to node localhost:xxxxx DEBUG: assigned task to node localhost:xxxxx DEBUG: assigned task to node localhost:xxxxx diff --git a/src/test/regress/sql/multi_hash_pruning.sql b/src/test/regress/sql/multi_hash_pruning.sql index 2b5964ad9..19e024510 100644 --- a/src/test/regress/sql/multi_hash_pruning.sql +++ b/src/test/regress/sql/multi_hash_pruning.sql @@ -32,6 +32,12 @@ CREATE TABLE orders_hash_partitioned ( o_comment varchar(79) ); SELECT create_distributed_table('orders_hash_partitioned', 'o_orderkey'); +INSERT INTO orders_hash_partitioned (o_orderkey, o_custkey, o_totalprice, o_shippriority, o_clerk) VALUES + (1, 11, 10, 111, 'aaa'), + (2, 22, 20, 222, 'bbb'), + (3, 33, 30, 333, 'ccc'), + (4, 44, 40, 444, 'ddd'); + SET client_min_messages TO DEBUG2; -- Check that we can prune shards for simple cases, boolean expressions and @@ -68,13 +74,13 @@ SELECT count(*) FROM orders_hash_partitioned SELECT count(*) FROM orders_hash_partitioned WHERE o_orderkey = 1 OR o_clerk = 'aaa'; SELECT count(*) FROM orders_hash_partitioned - WHERE o_orderkey = 1 OR (o_orderkey = 3 AND o_clerk = 'aaa'); + WHERE o_orderkey = 1 OR (o_orderkey = 3 AND o_clerk = 'ccc'); SELECT count(*) FROM orders_hash_partitioned WHERE o_orderkey = 1 OR o_orderkey is NULL; SELECT count(*) FROM (SELECT o_orderkey FROM orders_hash_partitioned WHERE o_orderkey = 1) AS orderkeys; -SET client_min_messages TO DEFAULT; +SET client_min_messages TO DEBUG3; -- Check that we support runing for ANY/IN with literal. SELECT count(*) FROM lineitem_hash_part @@ -100,6 +106,13 @@ SELECT count(*) FROM lineitem_hash_part SELECT count(*) FROM lineitem_hash_part WHERE l_orderkey IN (SELECT l_orderkey FROM lineitem_hash_part); SELECT count(*) FROM lineitem_hash_part WHERE l_orderkey = ANY (SELECT l_orderkey FROM lineitem_hash_part); +-- Check whether we support range queries with append distributed table +SELECT count(*) FROM lineitem + WHERE l_orderkey >= 1 AND l_orderkey <= 3; + +SELECT count(*) FROM lineitem + WHERE (l_orderkey >= 1 AND l_orderkey <= 3) AND (l_quantity > 11 AND l_quantity < 22); + -- Check whether we support IN/ANY in subquery with append and range distributed table SELECT count(*) FROM lineitem WHERE l_orderkey = ANY ('{1,2,3}'); @@ -119,8 +132,6 @@ SELECT count(*) FROM lineitem_range SELECT count(*) FROM lineitem_range WHERE l_orderkey = ANY(NULL) OR TRUE; -SET client_min_messages TO DEBUG2; - -- Check that we don't show the message if the operator is not -- equality operator SELECT count(*) FROM orders_hash_partitioned @@ -129,15 +140,15 @@ SELECT count(*) FROM orders_hash_partitioned -- Check that we don't give a spurious hint message when non-partition -- columns are used with ANY/IN/ALL SELECT count(*) FROM orders_hash_partitioned - WHERE o_orderkey = 1 OR o_totalprice IN (2, 5); + WHERE o_orderkey = 1 OR o_totalprice IN (20, 30); -- Check that we cannot prune for mutable functions. -SELECT count(*) FROM orders_hash_partitioned WHERE o_orderkey = random(); +SELECT count(*) FROM orders_hash_partitioned WHERE o_orderkey = (random() + 100); SELECT count(*) FROM orders_hash_partitioned - WHERE o_orderkey = random() OR o_orderkey = 1; + WHERE o_orderkey = (random() + 100) OR o_orderkey = 1; SELECT count(*) FROM orders_hash_partitioned - WHERE o_orderkey = random() AND o_orderkey = 1; + WHERE o_orderkey = (random() + 100) AND o_orderkey = 1; -- Check that we can do join pruning. @@ -150,3 +161,101 @@ SELECT count(*) WHERE orders1.o_orderkey = orders2.o_orderkey AND orders1.o_orderkey = 1 AND orders2.o_orderkey is NULL; + + +-- All shards used without constraints +SELECT count(*) FROM orders_hash_partitioned; + +-- Shards restricted correctly with prunable constraint +SELECT count(*) FROM orders_hash_partitioned + WHERE o_orderkey = 1; + +-- Shards restricted correctly with prunable constraint ANDed with unprunable expression using OR +SELECT count(*) FROM orders_hash_partitioned + WHERE o_orderkey = 1 AND (o_custkey = 11 OR o_custkey = 22); + +-- Shards restricted correctly with prunable constraints ORed +SELECT count(*) FROM orders_hash_partitioned + WHERE (o_orderkey = 1 OR o_orderkey = 2); + +-- Shards restricted correctly with prunable constraints ANDed with unprunable expression using OR +SELECT count(*) FROM orders_hash_partitioned + WHERE (o_orderkey = 1 OR o_orderkey = 2) AND (o_custkey = 11 OR o_custkey = 22); + +-- Shards restricted correctly with many different prunable constraints ORed +SELECT count(*) FROM orders_hash_partitioned + WHERE (o_orderkey = 1 AND o_custkey = 11) OR (o_orderkey = 1 AND o_custkey = 33) OR (o_orderkey = 2 AND o_custkey = 22) OR (o_orderkey = 2 AND o_custkey = 44); + +-- Shards restricted correctly with prunable SAO constraint ANDed with unprunable expression using OR +SELECT count(*) FROM orders_hash_partitioned + WHERE (o_orderkey IN (1,2)) AND (o_custkey = 11 OR o_custkey = 22 OR o_custkey = 33); + +-- Shards restricted correctly with prunable SAO constraint ANDed with multiple unprunable expressions +SELECT count(*) FROM orders_hash_partitioned + WHERE (o_orderkey IN (1,2)) AND (o_totalprice < 11 OR o_totalprice > 19) AND o_shippriority > 100 AND (o_custkey = 11 OR o_custkey = 22); + +-- Shards restricted correctly with prunable SAO constraints ORed +SELECT count(*) FROM orders_hash_partitioned + WHERE (o_orderkey IN (1,2) AND o_custkey = 11) OR (o_orderkey IN (2,3) AND o_custkey = 22); + +-- All shards used with prunable expression ORed with unprunable expression +SELECT count(*) FROM orders_hash_partitioned + WHERE o_orderkey IN (1,2) OR o_custkey = 33; + +-- Shards restricted correctly with prunable constraint ORed +SELECT count(*) FROM orders_hash_partitioned + WHERE o_orderkey = 1 OR ((o_orderkey = 2 AND o_custkey = 22) OR (o_orderkey = 3 AND o_custkey = 33)); + +-- Shards restricted correctly with prunable constraint ORed with falsy expression +SELECT count(*) FROM orders_hash_partitioned + WHERE o_orderkey = 1 OR (o_orderkey = 2 AND (o_custkey = 11 OR (o_orderkey = 3 AND o_custkey = 44))); + +-- Shards restricted correctly with prunable SAO constraint ORed with prunable nested EQ constraint +SELECT count(*) FROM orders_hash_partitioned + WHERE (o_orderkey IN (1,2)) AND (o_custkey = 11 OR o_custkey = 22 OR o_custkey = 33) AND o_totalprice <= 20; + +-- Shards restricted correctly with prunable SAO constraint ANDed with unprunable expressions +SELECT count(*) FROM orders_hash_partitioned + WHERE (o_orderkey IN (1,2)) AND (o_custkey = 11 OR o_custkey = 33) AND o_custkey = 22; + +-- All shards used with prunable SAO constraint ORed with unprunable nested expression +SELECT count(*) FROM orders_hash_partitioned + WHERE ((o_orderkey IN (1,2)) AND (o_custkey = 11 OR o_custkey = 22)) OR o_custkey = 33; + +-- Shards restricted correctly with prunable SAO constraint ORed with prunable nested EQ constraint +SELECT count(*) FROM orders_hash_partitioned + WHERE ((o_orderkey IN (1,2)) AND (o_custkey = 11 OR o_custkey = 22)) OR (o_orderkey = 3 AND o_custkey = 33); + +-- All shards used with ORed top level unprunable expression +SELECT count(*) FROM orders_hash_partitioned + WHERE o_custkey = 11 OR (o_orderkey = 2 AND o_custkey = 22); + +-- Single shard used when deeply nested prunable expression is restrictive with nested ANDs +SELECT count(*) FROM orders_hash_partitioned + WHERE o_orderkey = 1 OR (o_orderkey = 2 AND (o_orderkey = 3 OR (o_orderkey = 1 AND o_custkey = 11))); + +-- Single shard used when top prunable expression is restrictive with nested ANDs +SELECT count(*) FROM orders_hash_partitioned + WHERE o_orderkey = 1 AND ((o_orderkey = 2 OR o_orderkey = 3) AND (o_custkey = 11 OR o_custkey = 22)); + +-- Deeply nested prunable expression affects used shards +SELECT count(*) FROM orders_hash_partitioned + WHERE o_orderkey = 1 OR ((o_orderkey = 2 OR o_orderkey = 3) AND (o_custkey = 22 OR o_custkey = 33)); + +-- Deeply nested non prunable expression uses all shards +SELECT count(*) FROM orders_hash_partitioned + WHERE o_orderkey = 1 OR ((o_orderkey = 2 OR o_custkey = 11) AND (o_custkey = 22 OR o_custkey = 33)); + +-- a OR partkey != x Uses all shards +SELECT count(*) FROM orders_hash_partitioned + WHERE o_orderkey = 1 OR o_orderkey != 2; + +-- a OR partkey IS NULL Uses all shards +SELECT count(*) FROM orders_hash_partitioned + WHERE o_orderkey = 1 OR o_orderkey IS NULL; + +-- a OR partkey IS NOT NULL Uses all shards +SELECT count(*) FROM orders_hash_partitioned + WHERE o_orderkey = 1 OR o_orderkey IS NOT NULL; + +SET client_min_messages TO DEFAULT;