From 58643a409851376d05e49346f11391067abe8ae2 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Sat, 16 May 2020 21:29:03 +0300 Subject: [PATCH 01/52] Enable postgres 13 in configure --- configure | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure b/configure index 12d292baf..de36e580e 100755 --- a/configure +++ b/configure @@ -2533,7 +2533,7 @@ if test -z "$version_num"; then as_fn_error $? "Could not detect PostgreSQL version from pg_config." "$LINENO" 5 fi -if test "$version_num" != '11' -a "$version_num" != '12'; then +if test "$version_num" != '11' -a "$version_num" != '12' -a "$version_num" != '13'; then as_fn_error $? "Citus is not compatible with the detected PostgreSQL version ${version_num}." "$LINENO" 5 else { $as_echo "$as_me:${as_lineno-$LINENO}: building against PostgreSQL $version_num" >&5 From 30549dc0e2a724c662886023453c6ea9b7935772 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Mon, 18 May 2020 11:43:53 +0300 Subject: [PATCH 02/52] add copy of ruleutils_12 as ruleutils_13 --- .gitattributes | 2 + .../distributed/deparser/ruleutils_11.c | 6 +- .../distributed/deparser/ruleutils_12.c | 6 +- .../distributed/deparser/ruleutils_13.c | 8001 +++++++++++++++++ .../distributed/pg_version_constants.h | 1 + 5 files changed, 8012 insertions(+), 4 deletions(-) create mode 100644 src/backend/distributed/deparser/ruleutils_13.c diff --git a/.gitattributes b/.gitattributes index 37f1ef2bb..4a012b4c1 100644 --- a/.gitattributes +++ b/.gitattributes @@ -30,6 +30,8 @@ src/backend/distributed/utils/pg11_snprintf.c -citus-style src/backend/distributed/deparser/ruleutils_10.c -citus-style src/backend/distributed/deparser/ruleutils_11.c -citus-style src/backend/distributed/deparser/ruleutils_12.c -citus-style +src/backend/distributed/deparser/ruleutils_13.c -citus-style + src/include/distributed/citus_nodes.h -citus-style /vendor/** -citus-style diff --git a/src/backend/distributed/deparser/ruleutils_11.c b/src/backend/distributed/deparser/ruleutils_11.c index 393240276..f854003e6 100644 --- a/src/backend/distributed/deparser/ruleutils_11.c +++ b/src/backend/distributed/deparser/ruleutils_11.c @@ -15,12 +15,14 @@ *------------------------------------------------------------------------- */ -#include "postgres.h" - #include "distributed/pg_version_constants.h" +#include "pg_config.h" + #if (PG_VERSION_NUM >= PG_VERSION_11) && (PG_VERSION_NUM < PG_VERSION_12) +#include "postgres.h" + #include #include #include diff --git a/src/backend/distributed/deparser/ruleutils_12.c b/src/backend/distributed/deparser/ruleutils_12.c index 5ce6fc5e9..b3b7b2151 100644 --- a/src/backend/distributed/deparser/ruleutils_12.c +++ b/src/backend/distributed/deparser/ruleutils_12.c @@ -14,12 +14,14 @@ * This needs to be closely in sync with the core code. *------------------------------------------------------------------------- */ - -#include "postgres.h" #include "distributed/pg_version_constants.h" +#include "pg_config.h" + #if (PG_VERSION_NUM >= PG_VERSION_12) && (PG_VERSION_NUM < PG_VERSION_13) +#include "postgres.h" + #include #include #include diff --git a/src/backend/distributed/deparser/ruleutils_13.c b/src/backend/distributed/deparser/ruleutils_13.c new file mode 100644 index 000000000..3ca2da67b --- /dev/null +++ b/src/backend/distributed/deparser/ruleutils_13.c @@ -0,0 +1,8001 @@ +/*------------------------------------------------------------------------- + * + * ruleutils_13.c + * Functions to convert stored expressions/querytrees back to + * source text + * + * Portions Copyright (c) 1996-2018, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/backend/distributed/deparser/ruleutils_13.c + * + * This needs to be closely in sync with the core code. + *------------------------------------------------------------------------- + */ +#include "distributed/pg_version_constants.h" + +#include "pg_config.h" + +#if (PG_VERSION_NUM >= PG_VERSION_13) && (PG_VERSION_NUM < PG_VERSION_14) + +#include "postgres.h" + +#include +#include +#include + +#include "access/amapi.h" +#include "access/htup_details.h" +#include "access/relation.h" +#include "access/sysattr.h" +#include "access/table.h" +#include "catalog/dependency.h" +#include "catalog/indexing.h" +#include "catalog/pg_aggregate.h" +#include "catalog/pg_am.h" +#include "catalog/pg_authid.h" +#include "catalog/pg_collation.h" +#include "catalog/pg_constraint.h" +#include "catalog/pg_depend.h" +#include "catalog/pg_extension.h" +#include "catalog/pg_foreign_data_wrapper.h" +#include "catalog/pg_language.h" +#include "catalog/pg_opclass.h" +#include "catalog/pg_operator.h" +#include "catalog/pg_partitioned_table.h" +#include "catalog/pg_proc.h" +#include "catalog/pg_statistic_ext.h" +#include "catalog/pg_trigger.h" +#include "catalog/pg_type.h" +#include "commands/defrem.h" +#include "commands/extension.h" +#include "commands/tablespace.h" +#include "common/keywords.h" +#include "distributed/citus_nodefuncs.h" +#include "distributed/citus_ruleutils.h" +#include "executor/spi.h" +#include "foreign/foreign.h" +#include "funcapi.h" +#include "mb/pg_wchar.h" +#include "miscadmin.h" +#include "nodes/makefuncs.h" +#include "nodes/nodeFuncs.h" +#include "nodes/pathnodes.h" +#include "optimizer/optimizer.h" +#include "parser/parse_node.h" +#include "parser/parse_agg.h" +#include "parser/parse_func.h" +#include "parser/parse_node.h" +#include "parser/parse_oper.h" +#include "parser/parser.h" +#include "parser/parsetree.h" +#include "rewrite/rewriteHandler.h" +#include "rewrite/rewriteManip.h" +#include "rewrite/rewriteSupport.h" +#include "utils/array.h" +#include "utils/builtins.h" +#include "utils/fmgroids.h" +#include "utils/hsearch.h" +#include "utils/lsyscache.h" +#include "utils/rel.h" +#include "utils/ruleutils.h" +#include "utils/snapmgr.h" +#include "utils/syscache.h" +#include "utils/typcache.h" +#include "utils/varlena.h" +#include "utils/xml.h" + + +/* ---------- + * Pretty formatting constants + * ---------- + */ + +/* Indent counts */ +#define PRETTYINDENT_STD 8 +#define PRETTYINDENT_JOIN 4 +#define PRETTYINDENT_VAR 4 + +#define PRETTYINDENT_LIMIT 40 /* wrap limit */ + +/* Pretty flags */ +#define PRETTYFLAG_PAREN 0x0001 +#define PRETTYFLAG_INDENT 0x0002 + +/* Default line length for pretty-print wrapping: 0 means wrap always */ +#define WRAP_COLUMN_DEFAULT 0 + +/* macros to test if pretty action needed */ +#define PRETTY_PAREN(context) ((context)->prettyFlags & PRETTYFLAG_PAREN) +#define PRETTY_INDENT(context) ((context)->prettyFlags & PRETTYFLAG_INDENT) + + +/* ---------- + * Local data types + * ---------- + */ + +/* Context info needed for invoking a recursive querytree display routine */ +typedef struct +{ + StringInfo buf; /* output buffer to append to */ + List *namespaces; /* List of deparse_namespace nodes */ + List *windowClause; /* Current query level's WINDOW clause */ + List *windowTList; /* targetlist for resolving WINDOW clause */ + int prettyFlags; /* enabling of pretty-print functions */ + int wrapColumn; /* max line length, or -1 for no limit */ + int indentLevel; /* current indent level for prettyprint */ + bool varprefix; /* true to print prefixes on Vars */ + Oid distrelid; /* the distributed table being modified, if valid */ + int64 shardid; /* a distributed table's shardid, if positive */ + ParseExprKind special_exprkind; /* set only for exprkinds needing special + * handling */ + Bitmapset *appendparents; +} deparse_context; + +/* + * Each level of query context around a subtree needs a level of Var namespace. + * A Var having varlevelsup=N refers to the N'th item (counting from 0) in + * the current context's namespaces list. + * + * The rangetable is the list of actual RTEs from the query tree, and the + * cte list is the list of actual CTEs. + * + * rtable_names holds the alias name to be used for each RTE (either a C + * string, or NULL for nameless RTEs such as unnamed joins). + * rtable_columns holds the column alias names to be used for each RTE. + * + * In some cases we need to make names of merged JOIN USING columns unique + * across the whole query, not only per-RTE. If so, unique_using is true + * and using_names is a list of C strings representing names already assigned + * to USING columns. + * + * When deparsing plan trees, there is always just a single item in the + * deparse_namespace list (since a plan tree never contains Vars with + * varlevelsup > 0). We store the PlanState node that is the immediate + * parent of the expression to be deparsed, as well as a list of that + * PlanState's ancestors. In addition, we store its outer and inner subplan + * state nodes, as well as their plan nodes' targetlists, and the index tlist + * if the current plan node might contain INDEX_VAR Vars. (These fields could + * be derived on-the-fly from the current PlanState, but it seems notationally + * clearer to set them up as separate fields.) + */ +typedef struct +{ + List *rtable; /* List of RangeTblEntry nodes */ + List *rtable_names; /* Parallel list of names for RTEs */ + List *rtable_columns; /* Parallel list of deparse_columns structs */ + List *ctes; /* List of CommonTableExpr nodes */ + /* Workspace for column alias assignment: */ + bool unique_using; /* Are we making USING names globally unique */ + List *using_names; /* List of assigned names for USING columns */ + /* Remaining fields are used only when deparsing a Plan tree: */ + PlanState *planstate; /* immediate parent of current expression */ + List *ancestors; /* ancestors of planstate */ + PlanState *outer_planstate; /* outer subplan state, or NULL if none */ + PlanState *inner_planstate; /* inner subplan state, or NULL if none */ + List *outer_tlist; /* referent for OUTER_VAR Vars */ + List *inner_tlist; /* referent for INNER_VAR Vars */ + List *index_tlist; /* referent for INDEX_VAR Vars */ +} deparse_namespace; + +/* + * Per-relation data about column alias names. + * + * Selecting aliases is unreasonably complicated because of the need to dump + * rules/views whose underlying tables may have had columns added, deleted, or + * renamed since the query was parsed. We must nonetheless print the rule/view + * in a form that can be reloaded and will produce the same results as before. + * + * For each RTE used in the query, we must assign column aliases that are + * unique within that RTE. SQL does not require this of the original query, + * but due to factors such as *-expansion we need to be able to uniquely + * reference every column in a decompiled query. As long as we qualify all + * column references, per-RTE uniqueness is sufficient for that. + * + * However, we can't ensure per-column name uniqueness for unnamed join RTEs, + * since they just inherit column names from their input RTEs, and we can't + * rename the columns at the join level. Most of the time this isn't an issue + * because we don't need to reference the join's output columns as such; we + * can reference the input columns instead. That approach can fail for merged + * JOIN USING columns, however, so when we have one of those in an unnamed + * join, we have to make that column's alias globally unique across the whole + * query to ensure it can be referenced unambiguously. + * + * Another problem is that a JOIN USING clause requires the columns to be + * merged to have the same aliases in both input RTEs, and that no other + * columns in those RTEs or their children conflict with the USING names. + * To handle that, we do USING-column alias assignment in a recursive + * traversal of the query's jointree. When descending through a JOIN with + * USING, we preassign the USING column names to the child columns, overriding + * other rules for column alias assignment. We also mark each RTE with a list + * of all USING column names selected for joins containing that RTE, so that + * when we assign other columns' aliases later, we can avoid conflicts. + * + * Another problem is that if a JOIN's input tables have had columns added or + * deleted since the query was parsed, we must generate a column alias list + * for the join that matches the current set of input columns --- otherwise, a + * change in the number of columns in the left input would throw off matching + * of aliases to columns of the right input. Thus, positions in the printable + * column alias list are not necessarily one-for-one with varattnos of the + * JOIN, so we need a separate new_colnames[] array for printing purposes. + */ +typedef struct +{ + /* + * colnames is an array containing column aliases to use for columns that + * existed when the query was parsed. Dropped columns have NULL entries. + * This array can be directly indexed by varattno to get a Var's name. + * + * Non-NULL entries are guaranteed unique within the RTE, *except* when + * this is for an unnamed JOIN RTE. In that case we merely copy up names + * from the two input RTEs. + * + * During the recursive descent in set_using_names(), forcible assignment + * of a child RTE's column name is represented by pre-setting that element + * of the child's colnames array. So at that stage, NULL entries in this + * array just mean that no name has been preassigned, not necessarily that + * the column is dropped. + */ + int num_cols; /* length of colnames[] array */ + char **colnames; /* array of C strings and NULLs */ + + /* + * new_colnames is an array containing column aliases to use for columns + * that would exist if the query was re-parsed against the current + * definitions of its base tables. This is what to print as the column + * alias list for the RTE. This array does not include dropped columns, + * but it will include columns added since original parsing. Indexes in + * it therefore have little to do with current varattno values. As above, + * entries are unique unless this is for an unnamed JOIN RTE. (In such an + * RTE, we never actually print this array, but we must compute it anyway + * for possible use in computing column names of upper joins.) The + * parallel array is_new_col marks which of these columns are new since + * original parsing. Entries with is_new_col false must match the + * non-NULL colnames entries one-for-one. + */ + int num_new_cols; /* length of new_colnames[] array */ + char **new_colnames; /* array of C strings */ + bool *is_new_col; /* array of bool flags */ + + /* This flag tells whether we should actually print a column alias list */ + bool printaliases; + + /* This list has all names used as USING names in joins above this RTE */ + List *parentUsing; /* names assigned to parent merged columns */ + + /* + * If this struct is for a JOIN RTE, we fill these fields during the + * set_using_names() pass to describe its relationship to its child RTEs. + * + * leftattnos and rightattnos are arrays with one entry per existing + * output column of the join (hence, indexable by join varattno). For a + * simple reference to a column of the left child, leftattnos[i] is the + * child RTE's attno and rightattnos[i] is zero; and conversely for a + * column of the right child. But for merged columns produced by JOIN + * USING/NATURAL JOIN, both leftattnos[i] and rightattnos[i] are nonzero. + * Also, if the column has been dropped, both are zero. + * + * If it's a JOIN USING, usingNames holds the alias names selected for the + * merged columns (these might be different from the original USING list, + * if we had to modify names to achieve uniqueness). + */ + int leftrti; /* rangetable index of left child */ + int rightrti; /* rangetable index of right child */ + int *leftattnos; /* left-child varattnos of join cols, or 0 */ + int *rightattnos; /* right-child varattnos of join cols, or 0 */ + List *usingNames; /* names assigned to merged columns */ +} deparse_columns; + +/* This macro is analogous to rt_fetch(), but for deparse_columns structs */ +#define deparse_columns_fetch(rangetable_index, dpns) \ + ((deparse_columns *) list_nth((dpns)->rtable_columns, (rangetable_index)-1)) + +/* + * Entry in set_rtable_names' hash table + */ +typedef struct +{ + char name[NAMEDATALEN]; /* Hash key --- must be first */ + int counter; /* Largest addition used so far for name */ +} NameHashEntry; + + +/* ---------- + * Local functions + * + * Most of these functions used to use fixed-size buffers to build their + * results. Now, they take an (already initialized) StringInfo object + * as a parameter, and append their text output to its contents. + * ---------- + */ +static void set_rtable_names(deparse_namespace *dpns, List *parent_namespaces, + Bitmapset *rels_used); +static void set_deparse_for_query(deparse_namespace *dpns, Query *query, + List *parent_namespaces); +static bool has_dangerous_join_using(deparse_namespace *dpns, Node *jtnode); +static void set_using_names(deparse_namespace *dpns, Node *jtnode, + List *parentUsing); +static void set_relation_column_names(deparse_namespace *dpns, + RangeTblEntry *rte, + deparse_columns *colinfo); +static void set_join_column_names(deparse_namespace *dpns, RangeTblEntry *rte, + deparse_columns *colinfo); +static bool colname_is_unique(const char *colname, deparse_namespace *dpns, + deparse_columns *colinfo); +static char *make_colname_unique(char *colname, deparse_namespace *dpns, + deparse_columns *colinfo); +static void expand_colnames_array_to(deparse_columns *colinfo, int n); +static void identify_join_columns(JoinExpr *j, RangeTblEntry *jrte, + deparse_columns *colinfo); +static void flatten_join_using_qual(Node *qual, + List **leftvars, List **rightvars); +static char *get_rtable_name(int rtindex, deparse_context *context); +static void set_deparse_planstate(deparse_namespace *dpns, PlanState *ps); +static void push_child_plan(deparse_namespace *dpns, PlanState *ps, + deparse_namespace *save_dpns); +static void pop_child_plan(deparse_namespace *dpns, + deparse_namespace *save_dpns); +static void push_ancestor_plan(deparse_namespace *dpns, ListCell *ancestor_cell, + deparse_namespace *save_dpns); +static void pop_ancestor_plan(deparse_namespace *dpns, + deparse_namespace *save_dpns); +static void get_query_def(Query *query, StringInfo buf, List *parentnamespace, + TupleDesc resultDesc, + int prettyFlags, int wrapColumn, int startIndent); +static void get_query_def_extended(Query *query, StringInfo buf, + List *parentnamespace, Oid distrelid, int64 shardid, + TupleDesc resultDesc, int prettyFlags, int wrapColumn, + int startIndent); +static void get_values_def(List *values_lists, deparse_context *context); +static void get_with_clause(Query *query, deparse_context *context); +static void get_select_query_def(Query *query, deparse_context *context, + TupleDesc resultDesc); +static void get_insert_query_def(Query *query, deparse_context *context); +static void get_update_query_def(Query *query, deparse_context *context); +static void get_update_query_targetlist_def(Query *query, List *targetList, + deparse_context *context, + RangeTblEntry *rte); +static void get_delete_query_def(Query *query, deparse_context *context); +static void get_utility_query_def(Query *query, deparse_context *context); +static void get_basic_select_query(Query *query, deparse_context *context, + TupleDesc resultDesc); +static void get_target_list(List *targetList, deparse_context *context, + TupleDesc resultDesc); +static void get_setop_query(Node *setOp, Query *query, + deparse_context *context, + TupleDesc resultDesc); +static Node *get_rule_sortgroupclause(Index ref, List *tlist, + bool force_colno, + deparse_context *context); +static void get_rule_groupingset(GroupingSet *gset, List *targetlist, + bool omit_parens, deparse_context *context); +static void get_rule_orderby(List *orderList, List *targetList, + bool force_colno, deparse_context *context); +static void get_rule_windowclause(Query *query, deparse_context *context); +static void get_rule_windowspec(WindowClause *wc, List *targetList, + deparse_context *context); +static char *get_variable(Var *var, int levelsup, bool istoplevel, + deparse_context *context); +static void get_special_variable(Node *node, deparse_context *context, + void *private); +static void resolve_special_varno(Node *node, deparse_context *context, + void *private, + void (*callback) (Node *, deparse_context *, void *)); +static Node *find_param_referent(Param *param, deparse_context *context, + deparse_namespace **dpns_p, ListCell **ancestor_cell_p); +static void get_parameter(Param *param, deparse_context *context); +static const char *get_simple_binary_op_name(OpExpr *expr); +static bool isSimpleNode(Node *node, Node *parentNode, int prettyFlags); +static void appendContextKeyword(deparse_context *context, const char *str, + int indentBefore, int indentAfter, int indentPlus); +static void removeStringInfoSpaces(StringInfo str); +static void get_rule_expr(Node *node, deparse_context *context, + bool showimplicit); +static void get_rule_expr_toplevel(Node *node, deparse_context *context, + bool showimplicit); +static void get_rule_expr_funccall(Node *node, deparse_context *context, + bool showimplicit); +static bool looks_like_function(Node *node); +static void get_oper_expr(OpExpr *expr, deparse_context *context); +static void get_func_expr(FuncExpr *expr, deparse_context *context, + bool showimplicit); +static void get_agg_expr(Aggref *aggref, deparse_context *context, + Aggref *original_aggref); +static void get_agg_combine_expr(Node *node, deparse_context *context, + void *private); +static void get_windowfunc_expr(WindowFunc *wfunc, deparse_context *context); +static void get_coercion_expr(Node *arg, deparse_context *context, + Oid resulttype, int32 resulttypmod, + Node *parentNode); +static void get_const_expr(Const *constval, deparse_context *context, + int showtype); +static void get_const_collation(Const *constval, deparse_context *context); +static void simple_quote_literal(StringInfo buf, const char *val); +static void get_sublink_expr(SubLink *sublink, deparse_context *context); +static void get_tablefunc(TableFunc *tf, deparse_context *context, + bool showimplicit); +static void get_from_clause(Query *query, const char *prefix, + deparse_context *context); +static void get_from_clause_item(Node *jtnode, Query *query, + deparse_context *context); +static void get_column_alias_list(deparse_columns *colinfo, + deparse_context *context); +static void get_from_clause_coldeflist(RangeTblFunction *rtfunc, + deparse_columns *colinfo, + deparse_context *context); +static void get_tablesample_def(TableSampleClause *tablesample, + deparse_context *context); +static void get_opclass_name(Oid opclass, Oid actual_datatype, + StringInfo buf); +static Node *processIndirection(Node *node, deparse_context *context); +static void printSubscripts(SubscriptingRef *aref, deparse_context *context); +static char *get_relation_name(Oid relid); +static char *generate_relation_or_shard_name(Oid relid, Oid distrelid, + int64 shardid, List *namespaces); +static char *generate_rte_shard_name(RangeTblEntry *rangeTableEntry); +static char *generate_fragment_name(char *schemaName, char *tableName); +static char *generate_function_name(Oid funcid, int nargs, + List *argnames, Oid *argtypes, + bool has_variadic, bool *use_variadic_p, + ParseExprKind special_exprkind); + +#define only_marker(rte) ((rte)->inh ? "" : "ONLY ") + + + +/* + * pg_get_query_def parses back one query tree, and outputs the resulting query + * string into given buffer. + */ +void +pg_get_query_def(Query *query, StringInfo buffer) +{ + get_query_def(query, buffer, NIL, NULL, 0, WRAP_COLUMN_DEFAULT, 0); +} + + +/* + * pg_get_rule_expr deparses an expression and returns the result as a string. + */ +char * +pg_get_rule_expr(Node *expression) +{ + bool showImplicitCasts = true; + deparse_context context; + OverrideSearchPath *overridePath = NULL; + StringInfo buffer = makeStringInfo(); + + /* + * Set search_path to NIL so that all objects outside of pg_catalog will be + * schema-prefixed. pg_catalog will be added automatically when we call + * PushOverrideSearchPath(), since we set addCatalog to true; + */ + overridePath = GetOverrideSearchPath(CurrentMemoryContext); + overridePath->schemas = NIL; + overridePath->addCatalog = true; + PushOverrideSearchPath(overridePath); + + context.buf = buffer; + context.namespaces = NIL; + context.windowClause = NIL; + context.windowTList = NIL; + context.varprefix = false; + context.prettyFlags = 0; + context.wrapColumn = WRAP_COLUMN_DEFAULT; + context.indentLevel = 0; + context.special_exprkind = EXPR_KIND_NONE; + context.distrelid = InvalidOid; + context.shardid = INVALID_SHARD_ID; + + get_rule_expr(expression, &context, showImplicitCasts); + + /* revert back to original search_path */ + PopOverrideSearchPath(); + + return buffer->data; +} + + +/* + * set_rtable_names: select RTE aliases to be used in printing a query + * + * We fill in dpns->rtable_names with a list of names that is one-for-one with + * the already-filled dpns->rtable list. Each RTE name is unique among those + * in the new namespace plus any ancestor namespaces listed in + * parent_namespaces. + * + * If rels_used isn't NULL, only RTE indexes listed in it are given aliases. + * + * Note that this function is only concerned with relation names, not column + * names. + */ +static void +set_rtable_names(deparse_namespace *dpns, List *parent_namespaces, + Bitmapset *rels_used) +{ + HASHCTL hash_ctl; + HTAB *names_hash; + NameHashEntry *hentry; + bool found; + int rtindex; + ListCell *lc; + + dpns->rtable_names = NIL; + /* nothing more to do if empty rtable */ + if (dpns->rtable == NIL) + return; + + /* + * We use a hash table to hold known names, so that this process is O(N) + * not O(N^2) for N names. + */ + MemSet(&hash_ctl, 0, sizeof(hash_ctl)); + hash_ctl.keysize = NAMEDATALEN; + hash_ctl.entrysize = sizeof(NameHashEntry); + hash_ctl.hcxt = CurrentMemoryContext; + names_hash = hash_create("set_rtable_names names", + list_length(dpns->rtable), + &hash_ctl, + HASH_ELEM | HASH_CONTEXT); + /* Preload the hash table with names appearing in parent_namespaces */ + foreach(lc, parent_namespaces) + { + deparse_namespace *olddpns = (deparse_namespace *) lfirst(lc); + ListCell *lc2; + + foreach(lc2, olddpns->rtable_names) + { + char *oldname = (char *) lfirst(lc2); + + if (oldname == NULL) + continue; + hentry = (NameHashEntry *) hash_search(names_hash, + oldname, + HASH_ENTER, + &found); + /* we do not complain about duplicate names in parent namespaces */ + hentry->counter = 0; + } + } + + /* Now we can scan the rtable */ + rtindex = 1; + foreach(lc, dpns->rtable) + { + RangeTblEntry *rte = (RangeTblEntry *) lfirst(lc); + char *refname; + + /* Just in case this takes an unreasonable amount of time ... */ + CHECK_FOR_INTERRUPTS(); + + if (rels_used && !bms_is_member(rtindex, rels_used)) + { + /* Ignore unreferenced RTE */ + refname = NULL; + } + else if (rte->alias) + { + /* If RTE has a user-defined alias, prefer that */ + refname = rte->alias->aliasname; + } + else if (rte->rtekind == RTE_RELATION) + { + /* Use the current actual name of the relation */ + refname = get_rel_name(rte->relid); + } + else if (rte->rtekind == RTE_JOIN) + { + /* Unnamed join has no refname */ + refname = NULL; + } + else + { + /* Otherwise use whatever the parser assigned */ + refname = rte->eref->aliasname; + } + + /* + * If the selected name isn't unique, append digits to make it so, and + * make a new hash entry for it once we've got a unique name. For a + * very long input name, we might have to truncate to stay within + * NAMEDATALEN. + */ + if (refname) + { + hentry = (NameHashEntry *) hash_search(names_hash, + refname, + HASH_ENTER, + &found); + if (found) + { + /* Name already in use, must choose a new one */ + int refnamelen = strlen(refname); + char *modname = (char *) palloc(refnamelen + 16); + NameHashEntry *hentry2; + + do + { + hentry->counter++; + for (;;) + { + /* + * We avoid using %.*s here because it can misbehave + * if the data is not valid in what libc thinks is the + * prevailing encoding. + */ + memcpy(modname, refname, refnamelen); + sprintf(modname + refnamelen, "_%d", hentry->counter); + if (strlen(modname) < NAMEDATALEN) + break; + /* drop chars from refname to keep all the digits */ + refnamelen = pg_mbcliplen(refname, refnamelen, + refnamelen - 1); + } + hentry2 = (NameHashEntry *) hash_search(names_hash, + modname, + HASH_ENTER, + &found); + } while (found); + hentry2->counter = 0; /* init new hash entry */ + refname = modname; + } + else + { + /* Name not previously used, need only initialize hentry */ + hentry->counter = 0; + } + } + + dpns->rtable_names = lappend(dpns->rtable_names, refname); + rtindex++; + } + + hash_destroy(names_hash); +} + +/* + * set_deparse_for_query: set up deparse_namespace for deparsing a Query tree + * + * For convenience, this is defined to initialize the deparse_namespace struct + * from scratch. + */ +static void +set_deparse_for_query(deparse_namespace *dpns, Query *query, + List *parent_namespaces) +{ + ListCell *lc; + ListCell *lc2; + + /* Initialize *dpns and fill rtable/ctes links */ + memset(dpns, 0, sizeof(deparse_namespace)); + dpns->rtable = query->rtable; + dpns->ctes = query->cteList; + + /* Assign a unique relation alias to each RTE */ + set_rtable_names(dpns, parent_namespaces, NULL); + + /* Initialize dpns->rtable_columns to contain zeroed structs */ + dpns->rtable_columns = NIL; + while (list_length(dpns->rtable_columns) < list_length(dpns->rtable)) + dpns->rtable_columns = lappend(dpns->rtable_columns, + palloc0(sizeof(deparse_columns))); + + /* If it's a utility query, it won't have a jointree */ + if (query->jointree) + { + /* Detect whether global uniqueness of USING names is needed */ + dpns->unique_using = + has_dangerous_join_using(dpns, (Node *) query->jointree); + + /* + * Select names for columns merged by USING, via a recursive pass over + * the query jointree. + */ + set_using_names(dpns, (Node *) query->jointree, NIL); + } + + /* + * Now assign remaining column aliases for each RTE. We do this in a + * linear scan of the rtable, so as to process RTEs whether or not they + * are in the jointree (we mustn't miss NEW.*, INSERT target relations, + * etc). JOIN RTEs must be processed after their children, but this is + * okay because they appear later in the rtable list than their children + * (cf Asserts in identify_join_columns()). + */ + forboth(lc, dpns->rtable, lc2, dpns->rtable_columns) + { + RangeTblEntry *rte = (RangeTblEntry *) lfirst(lc); + deparse_columns *colinfo = (deparse_columns *) lfirst(lc2); + + if (rte->rtekind == RTE_JOIN) + set_join_column_names(dpns, rte, colinfo); + else + set_relation_column_names(dpns, rte, colinfo); + } +} + +/* + * has_dangerous_join_using: search jointree for unnamed JOIN USING + * + * Merged columns of a JOIN USING may act differently from either of the input + * columns, either because they are merged with COALESCE (in a FULL JOIN) or + * because an implicit coercion of the underlying input column is required. + * In such a case the column must be referenced as a column of the JOIN not as + * a column of either input. And this is problematic if the join is unnamed + * (alias-less): we cannot qualify the column's name with an RTE name, since + * there is none. (Forcibly assigning an alias to the join is not a solution, + * since that will prevent legal references to tables below the join.) + * To ensure that every column in the query is unambiguously referenceable, + * we must assign such merged columns names that are globally unique across + * the whole query, aliasing other columns out of the way as necessary. + * + * Because the ensuing re-aliasing is fairly damaging to the readability of + * the query, we don't do this unless we have to. So, we must pre-scan + * the join tree to see if we have to, before starting set_using_names(). + */ +static bool +has_dangerous_join_using(deparse_namespace *dpns, Node *jtnode) +{ + if (IsA(jtnode, RangeTblRef)) + { + /* nothing to do here */ + } + else if (IsA(jtnode, FromExpr)) + { + FromExpr *f = (FromExpr *) jtnode; + ListCell *lc; + + foreach(lc, f->fromlist) + { + if (has_dangerous_join_using(dpns, (Node *) lfirst(lc))) + return true; + } + } + else if (IsA(jtnode, JoinExpr)) + { + JoinExpr *j = (JoinExpr *) jtnode; + + /* Is it an unnamed JOIN with USING? */ + if (j->alias == NULL && j->usingClause) + { + /* + * Yes, so check each join alias var to see if any of them are not + * simple references to underlying columns. If so, we have a + * dangerous situation and must pick unique aliases. + */ + RangeTblEntry *jrte = rt_fetch(j->rtindex, dpns->rtable); + ListCell *lc; + + foreach(lc, jrte->joinaliasvars) + { + Var *aliasvar = (Var *) lfirst(lc); + + if (aliasvar != NULL && !IsA(aliasvar, Var)) + return true; + } + } + + /* Nope, but inspect children */ + if (has_dangerous_join_using(dpns, j->larg)) + return true; + if (has_dangerous_join_using(dpns, j->rarg)) + return true; + } + else + elog(ERROR, "unrecognized node type: %d", + (int) nodeTag(jtnode)); + return false; +} + +/* + * set_using_names: select column aliases to be used for merged USING columns + * + * We do this during a recursive descent of the query jointree. + * dpns->unique_using must already be set to determine the global strategy. + * + * Column alias info is saved in the dpns->rtable_columns list, which is + * assumed to be filled with pre-zeroed deparse_columns structs. + * + * parentUsing is a list of all USING aliases assigned in parent joins of + * the current jointree node. (The passed-in list must not be modified.) + */ +static void +set_using_names(deparse_namespace *dpns, Node *jtnode, List *parentUsing) +{ + if (IsA(jtnode, RangeTblRef)) + { + /* nothing to do now */ + } + else if (IsA(jtnode, FromExpr)) + { + FromExpr *f = (FromExpr *) jtnode; + ListCell *lc; + + foreach(lc, f->fromlist) + set_using_names(dpns, (Node *) lfirst(lc), parentUsing); + } + else if (IsA(jtnode, JoinExpr)) + { + JoinExpr *j = (JoinExpr *) jtnode; + RangeTblEntry *rte = rt_fetch(j->rtindex, dpns->rtable); + deparse_columns *colinfo = deparse_columns_fetch(j->rtindex, dpns); + int *leftattnos; + int *rightattnos; + deparse_columns *leftcolinfo; + deparse_columns *rightcolinfo; + int i; + ListCell *lc; + + /* Get info about the shape of the join */ + identify_join_columns(j, rte, colinfo); + leftattnos = colinfo->leftattnos; + rightattnos = colinfo->rightattnos; + + /* Look up the not-yet-filled-in child deparse_columns structs */ + leftcolinfo = deparse_columns_fetch(colinfo->leftrti, dpns); + rightcolinfo = deparse_columns_fetch(colinfo->rightrti, dpns); + + /* + * If this join is unnamed, then we cannot substitute new aliases at + * this level, so any name requirements pushed down to here must be + * pushed down again to the children. + */ + if (rte->alias == NULL) + { + for (i = 0; i < colinfo->num_cols; i++) + { + char *colname = colinfo->colnames[i]; + + if (colname == NULL) + continue; + + /* Push down to left column, unless it's a system column */ + if (leftattnos[i] > 0) + { + expand_colnames_array_to(leftcolinfo, leftattnos[i]); + leftcolinfo->colnames[leftattnos[i] - 1] = colname; + } + + /* Same on the righthand side */ + if (rightattnos[i] > 0) + { + expand_colnames_array_to(rightcolinfo, rightattnos[i]); + rightcolinfo->colnames[rightattnos[i] - 1] = colname; + } + } + } + + /* + * If there's a USING clause, select the USING column names and push + * those names down to the children. We have two strategies: + * + * If dpns->unique_using is true, we force all USING names to be + * unique across the whole query level. In principle we'd only need + * the names of dangerous USING columns to be globally unique, but to + * safely assign all USING names in a single pass, we have to enforce + * the same uniqueness rule for all of them. However, if a USING + * column's name has been pushed down from the parent, we should use + * it as-is rather than making a uniqueness adjustment. This is + * necessary when we're at an unnamed join, and it creates no risk of + * ambiguity. Also, if there's a user-written output alias for a + * merged column, we prefer to use that rather than the input name; + * this simplifies the logic and seems likely to lead to less aliasing + * overall. + * + * If dpns->unique_using is false, we only need USING names to be + * unique within their own join RTE. We still need to honor + * pushed-down names, though. + * + * Though significantly different in results, these two strategies are + * implemented by the same code, with only the difference of whether + * to put assigned names into dpns->using_names. + */ + if (j->usingClause) + { + /* Copy the input parentUsing list so we don't modify it */ + parentUsing = list_copy(parentUsing); + + /* USING names must correspond to the first join output columns */ + expand_colnames_array_to(colinfo, list_length(j->usingClause)); + i = 0; + foreach(lc, j->usingClause) + { + char *colname = strVal(lfirst(lc)); + + /* Assert it's a merged column */ + Assert(leftattnos[i] != 0 && rightattnos[i] != 0); + + /* Adopt passed-down name if any, else select unique name */ + if (colinfo->colnames[i] != NULL) + colname = colinfo->colnames[i]; + else + { + /* Prefer user-written output alias if any */ + if (rte->alias && i < list_length(rte->alias->colnames)) + colname = strVal(list_nth(rte->alias->colnames, i)); + /* Make it appropriately unique */ + colname = make_colname_unique(colname, dpns, colinfo); + if (dpns->unique_using) + dpns->using_names = lappend(dpns->using_names, + colname); + /* Save it as output column name, too */ + colinfo->colnames[i] = colname; + } + + /* Remember selected names for use later */ + colinfo->usingNames = lappend(colinfo->usingNames, colname); + parentUsing = lappend(parentUsing, colname); + + /* Push down to left column, unless it's a system column */ + if (leftattnos[i] > 0) + { + expand_colnames_array_to(leftcolinfo, leftattnos[i]); + leftcolinfo->colnames[leftattnos[i] - 1] = colname; + } + + /* Same on the righthand side */ + if (rightattnos[i] > 0) + { + expand_colnames_array_to(rightcolinfo, rightattnos[i]); + rightcolinfo->colnames[rightattnos[i] - 1] = colname; + } + + i++; + } + } + + /* Mark child deparse_columns structs with correct parentUsing info */ + leftcolinfo->parentUsing = parentUsing; + rightcolinfo->parentUsing = parentUsing; + + /* Now recursively assign USING column names in children */ + set_using_names(dpns, j->larg, parentUsing); + set_using_names(dpns, j->rarg, parentUsing); + } + else + elog(ERROR, "unrecognized node type: %d", + (int) nodeTag(jtnode)); +} + +/* + * set_relation_column_names: select column aliases for a non-join RTE + * + * Column alias info is saved in *colinfo, which is assumed to be pre-zeroed. + * If any colnames entries are already filled in, those override local + * choices. + */ +static void +set_relation_column_names(deparse_namespace *dpns, RangeTblEntry *rte, + deparse_columns *colinfo) +{ + int ncolumns; + char **real_colnames; + bool changed_any; + int noldcolumns; + int i; + int j; + + /* + * Extract the RTE's "real" column names. This is comparable to + * get_rte_attribute_name, except that it's important to disregard dropped + * columns. We put NULL into the array for a dropped column. + */ + if (rte->rtekind == RTE_RELATION) + { + /* Relation --- look to the system catalogs for up-to-date info */ + Relation rel; + TupleDesc tupdesc; + + rel = relation_open(rte->relid, AccessShareLock); + tupdesc = RelationGetDescr(rel); + + ncolumns = tupdesc->natts; + real_colnames = (char **) palloc(ncolumns * sizeof(char *)); + + for (i = 0; i < ncolumns; i++) + { + Form_pg_attribute attr = TupleDescAttr(tupdesc, i); + + if (attr->attisdropped) + real_colnames[i] = NULL; + else + real_colnames[i] = pstrdup(NameStr(attr->attname)); + } + relation_close(rel, AccessShareLock); + } + else + { + /* Otherwise use the column names from eref */ + ListCell *lc; + + ncolumns = list_length(rte->eref->colnames); + real_colnames = (char **) palloc(ncolumns * sizeof(char *)); + + i = 0; + foreach(lc, rte->eref->colnames) + { + /* + * If the column name shown in eref is an empty string, then it's + * a column that was dropped at the time of parsing the query, so + * treat it as dropped. + */ + char *cname = strVal(lfirst(lc)); + + if (cname[0] == '\0') + cname = NULL; + real_colnames[i] = cname; + i++; + } + } + + /* + * Ensure colinfo->colnames has a slot for each column. (It could be long + * enough already, if we pushed down a name for the last column.) Note: + * it's possible that there are now more columns than there were when the + * query was parsed, ie colnames could be longer than rte->eref->colnames. + * We must assign unique aliases to the new columns too, else there could + * be unresolved conflicts when the view/rule is reloaded. + */ + expand_colnames_array_to(colinfo, ncolumns); + Assert(colinfo->num_cols == ncolumns); + + /* + * Make sufficiently large new_colnames and is_new_col arrays, too. + * + * Note: because we leave colinfo->num_new_cols zero until after the loop, + * colname_is_unique will not consult that array, which is fine because it + * would only be duplicate effort. + */ + colinfo->new_colnames = (char **) palloc(ncolumns * sizeof(char *)); + colinfo->is_new_col = (bool *) palloc(ncolumns * sizeof(bool)); + + /* + * Scan the columns, select a unique alias for each one, and store it in + * colinfo->colnames and colinfo->new_colnames. The former array has NULL + * entries for dropped columns, the latter omits them. Also mark + * new_colnames entries as to whether they are new since parse time; this + * is the case for entries beyond the length of rte->eref->colnames. + */ + noldcolumns = list_length(rte->eref->colnames); + changed_any = false; + j = 0; + for (i = 0; i < ncolumns; i++) + { + char *real_colname = real_colnames[i]; + char *colname = colinfo->colnames[i]; + + /* Skip dropped columns */ + if (real_colname == NULL) + { + Assert(colname == NULL); /* colnames[i] is already NULL */ + continue; + } + + /* If alias already assigned, that's what to use */ + if (colname == NULL) + { + /* If user wrote an alias, prefer that over real column name */ + if (rte->alias && i < list_length(rte->alias->colnames)) + colname = strVal(list_nth(rte->alias->colnames, i)); + else + colname = real_colname; + + /* Unique-ify and insert into colinfo */ + colname = make_colname_unique(colname, dpns, colinfo); + + colinfo->colnames[i] = colname; + } + + /* Put names of non-dropped columns in new_colnames[] too */ + colinfo->new_colnames[j] = colname; + /* And mark them as new or not */ + colinfo->is_new_col[j] = (i >= noldcolumns); + j++; + + /* Remember if any assigned aliases differ from "real" name */ + if (!changed_any && strcmp(colname, real_colname) != 0) + changed_any = true; + } + + /* + * Set correct length for new_colnames[] array. (Note: if columns have + * been added, colinfo->num_cols includes them, which is not really quite + * right but is harmless, since any new columns must be at the end where + * they won't affect varattnos of pre-existing columns.) + */ + colinfo->num_new_cols = j; + + /* + * For a relation RTE, we need only print the alias column names if any + * are different from the underlying "real" names. For a function RTE, + * always emit a complete column alias list; this is to protect against + * possible instability of the default column names (eg, from altering + * parameter names). For tablefunc RTEs, we never print aliases, because + * the column names are part of the clause itself. For other RTE types, + * print if we changed anything OR if there were user-written column + * aliases (since the latter would be part of the underlying "reality"). + */ + if (rte->rtekind == RTE_RELATION) + colinfo->printaliases = changed_any; + else if (rte->rtekind == RTE_FUNCTION) + colinfo->printaliases = true; + else if (rte->rtekind == RTE_TABLEFUNC) + colinfo->printaliases = false; + else if (rte->alias && rte->alias->colnames != NIL) + colinfo->printaliases = true; + else + colinfo->printaliases = changed_any; +} + +/* + * set_join_column_names: select column aliases for a join RTE + * + * Column alias info is saved in *colinfo, which is assumed to be pre-zeroed. + * If any colnames entries are already filled in, those override local + * choices. Also, names for USING columns were already chosen by + * set_using_names(). We further expect that column alias selection has been + * completed for both input RTEs. + */ +static void +set_join_column_names(deparse_namespace *dpns, RangeTblEntry *rte, + deparse_columns *colinfo) +{ + deparse_columns *leftcolinfo; + deparse_columns *rightcolinfo; + bool changed_any; + int noldcolumns; + int nnewcolumns; + Bitmapset *leftmerged = NULL; + Bitmapset *rightmerged = NULL; + int i; + int j; + int ic; + int jc; + + /* Look up the previously-filled-in child deparse_columns structs */ + leftcolinfo = deparse_columns_fetch(colinfo->leftrti, dpns); + rightcolinfo = deparse_columns_fetch(colinfo->rightrti, dpns); + + /* + * Ensure colinfo->colnames has a slot for each column. (It could be long + * enough already, if we pushed down a name for the last column.) Note: + * it's possible that one or both inputs now have more columns than there + * were when the query was parsed, but we'll deal with that below. We + * only need entries in colnames for pre-existing columns. + */ + noldcolumns = list_length(rte->eref->colnames); + expand_colnames_array_to(colinfo, noldcolumns); + Assert(colinfo->num_cols == noldcolumns); + + /* + * Scan the join output columns, select an alias for each one, and store + * it in colinfo->colnames. If there are USING columns, set_using_names() + * already selected their names, so we can start the loop at the first + * non-merged column. + */ + changed_any = false; + for (i = list_length(colinfo->usingNames); i < noldcolumns; i++) + { + char *colname = colinfo->colnames[i]; + char *real_colname; + + /* Ignore dropped column (only possible for non-merged column) */ + if (colinfo->leftattnos[i] == 0 && colinfo->rightattnos[i] == 0) + { + Assert(colname == NULL); + continue; + } + + /* Get the child column name */ + if (colinfo->leftattnos[i] > 0) + real_colname = leftcolinfo->colnames[colinfo->leftattnos[i] - 1]; + else if (colinfo->rightattnos[i] > 0) + real_colname = rightcolinfo->colnames[colinfo->rightattnos[i] - 1]; + else + { + /* We're joining system columns --- use eref name */ + real_colname = strVal(list_nth(rte->eref->colnames, i)); + } + Assert(real_colname != NULL); + + /* In an unnamed join, just report child column names as-is */ + if (rte->alias == NULL) + { + colinfo->colnames[i] = real_colname; + continue; + } + + /* If alias already assigned, that's what to use */ + if (colname == NULL) + { + /* If user wrote an alias, prefer that over real column name */ + if (rte->alias && i < list_length(rte->alias->colnames)) + colname = strVal(list_nth(rte->alias->colnames, i)); + else + colname = real_colname; + + /* Unique-ify and insert into colinfo */ + colname = make_colname_unique(colname, dpns, colinfo); + + colinfo->colnames[i] = colname; + } + + /* Remember if any assigned aliases differ from "real" name */ + if (!changed_any && strcmp(colname, real_colname) != 0) + changed_any = true; + } + + /* + * Calculate number of columns the join would have if it were re-parsed + * now, and create storage for the new_colnames and is_new_col arrays. + * + * Note: colname_is_unique will be consulting new_colnames[] during the + * loops below, so its not-yet-filled entries must be zeroes. + */ + nnewcolumns = leftcolinfo->num_new_cols + rightcolinfo->num_new_cols - + list_length(colinfo->usingNames); + colinfo->num_new_cols = nnewcolumns; + colinfo->new_colnames = (char **) palloc0(nnewcolumns * sizeof(char *)); + colinfo->is_new_col = (bool *) palloc0(nnewcolumns * sizeof(bool)); + + /* + * Generating the new_colnames array is a bit tricky since any new columns + * added since parse time must be inserted in the right places. This code + * must match the parser, which will order a join's columns as merged + * columns first (in USING-clause order), then non-merged columns from the + * left input (in attnum order), then non-merged columns from the right + * input (ditto). If one of the inputs is itself a join, its columns will + * be ordered according to the same rule, which means newly-added columns + * might not be at the end. We can figure out what's what by consulting + * the leftattnos and rightattnos arrays plus the input is_new_col arrays. + * + * In these loops, i indexes leftattnos/rightattnos (so it's join varattno + * less one), j indexes new_colnames/is_new_col, and ic/jc have similar + * meanings for the current child RTE. + */ + + /* Handle merged columns; they are first and can't be new */ + i = j = 0; + while (i < noldcolumns && + colinfo->leftattnos[i] != 0 && + colinfo->rightattnos[i] != 0) + { + /* column name is already determined and known unique */ + colinfo->new_colnames[j] = colinfo->colnames[i]; + colinfo->is_new_col[j] = false; + + /* build bitmapsets of child attnums of merged columns */ + if (colinfo->leftattnos[i] > 0) + leftmerged = bms_add_member(leftmerged, colinfo->leftattnos[i]); + if (colinfo->rightattnos[i] > 0) + rightmerged = bms_add_member(rightmerged, colinfo->rightattnos[i]); + + i++, j++; + } + + /* Handle non-merged left-child columns */ + ic = 0; + for (jc = 0; jc < leftcolinfo->num_new_cols; jc++) + { + char *child_colname = leftcolinfo->new_colnames[jc]; + + if (!leftcolinfo->is_new_col[jc]) + { + /* Advance ic to next non-dropped old column of left child */ + while (ic < leftcolinfo->num_cols && + leftcolinfo->colnames[ic] == NULL) + ic++; + Assert(ic < leftcolinfo->num_cols); + ic++; + /* If it is a merged column, we already processed it */ + if (bms_is_member(ic, leftmerged)) + continue; + /* Else, advance i to the corresponding existing join column */ + while (i < colinfo->num_cols && + colinfo->colnames[i] == NULL) + i++; + Assert(i < colinfo->num_cols); + Assert(ic == colinfo->leftattnos[i]); + /* Use the already-assigned name of this column */ + colinfo->new_colnames[j] = colinfo->colnames[i]; + i++; + } + else + { + /* + * Unique-ify the new child column name and assign, unless we're + * in an unnamed join, in which case just copy + */ + if (rte->alias != NULL) + { + colinfo->new_colnames[j] = + make_colname_unique(child_colname, dpns, colinfo); + if (!changed_any && + strcmp(colinfo->new_colnames[j], child_colname) != 0) + changed_any = true; + } + else + colinfo->new_colnames[j] = child_colname; + } + + colinfo->is_new_col[j] = leftcolinfo->is_new_col[jc]; + j++; + } + + /* Handle non-merged right-child columns in exactly the same way */ + ic = 0; + for (jc = 0; jc < rightcolinfo->num_new_cols; jc++) + { + char *child_colname = rightcolinfo->new_colnames[jc]; + + if (!rightcolinfo->is_new_col[jc]) + { + /* Advance ic to next non-dropped old column of right child */ + while (ic < rightcolinfo->num_cols && + rightcolinfo->colnames[ic] == NULL) + ic++; + Assert(ic < rightcolinfo->num_cols); + ic++; + /* If it is a merged column, we already processed it */ + if (bms_is_member(ic, rightmerged)) + continue; + /* Else, advance i to the corresponding existing join column */ + while (i < colinfo->num_cols && + colinfo->colnames[i] == NULL) + i++; + Assert(i < colinfo->num_cols); + Assert(ic == colinfo->rightattnos[i]); + /* Use the already-assigned name of this column */ + colinfo->new_colnames[j] = colinfo->colnames[i]; + i++; + } + else + { + /* + * Unique-ify the new child column name and assign, unless we're + * in an unnamed join, in which case just copy + */ + if (rte->alias != NULL) + { + colinfo->new_colnames[j] = + make_colname_unique(child_colname, dpns, colinfo); + if (!changed_any && + strcmp(colinfo->new_colnames[j], child_colname) != 0) + changed_any = true; + } + else + colinfo->new_colnames[j] = child_colname; + } + + colinfo->is_new_col[j] = rightcolinfo->is_new_col[jc]; + j++; + } + + /* Assert we processed the right number of columns */ +#ifdef USE_ASSERT_CHECKING + while (i < colinfo->num_cols && colinfo->colnames[i] == NULL) + i++; + Assert(i == colinfo->num_cols); + Assert(j == nnewcolumns); +#endif + + /* + * For a named join, print column aliases if we changed any from the child + * names. Unnamed joins cannot print aliases. + */ + if (rte->alias != NULL) + colinfo->printaliases = changed_any; + else + colinfo->printaliases = false; +} + +/* + * colname_is_unique: is colname distinct from already-chosen column names? + * + * dpns is query-wide info, colinfo is for the column's RTE + */ +static bool +colname_is_unique(const char *colname, deparse_namespace *dpns, + deparse_columns *colinfo) +{ + int i; + ListCell *lc; + + /* Check against already-assigned column aliases within RTE */ + for (i = 0; i < colinfo->num_cols; i++) + { + char *oldname = colinfo->colnames[i]; + + if (oldname && strcmp(oldname, colname) == 0) + return false; + } + + /* + * If we're building a new_colnames array, check that too (this will be + * partially but not completely redundant with the previous checks) + */ + for (i = 0; i < colinfo->num_new_cols; i++) + { + char *oldname = colinfo->new_colnames[i]; + + if (oldname && strcmp(oldname, colname) == 0) + return false; + } + + /* Also check against USING-column names that must be globally unique */ + foreach(lc, dpns->using_names) + { + char *oldname = (char *) lfirst(lc); + + if (strcmp(oldname, colname) == 0) + return false; + } + + /* Also check against names already assigned for parent-join USING cols */ + foreach(lc, colinfo->parentUsing) + { + char *oldname = (char *) lfirst(lc); + + if (strcmp(oldname, colname) == 0) + return false; + } + + return true; +} + +/* + * make_colname_unique: modify colname if necessary to make it unique + * + * dpns is query-wide info, colinfo is for the column's RTE + */ +static char * +make_colname_unique(char *colname, deparse_namespace *dpns, + deparse_columns *colinfo) +{ + /* + * If the selected name isn't unique, append digits to make it so. For a + * very long input name, we might have to truncate to stay within + * NAMEDATALEN. + */ + if (!colname_is_unique(colname, dpns, colinfo)) + { + int colnamelen = strlen(colname); + char *modname = (char *) palloc(colnamelen + 16); + int i = 0; + + do + { + i++; + for (;;) + { + /* + * We avoid using %.*s here because it can misbehave if the + * data is not valid in what libc thinks is the prevailing + * encoding. + */ + memcpy(modname, colname, colnamelen); + sprintf(modname + colnamelen, "_%d", i); + if (strlen(modname) < NAMEDATALEN) + break; + /* drop chars from colname to keep all the digits */ + colnamelen = pg_mbcliplen(colname, colnamelen, + colnamelen - 1); + } + } while (!colname_is_unique(modname, dpns, colinfo)); + colname = modname; + } + return colname; +} + +/* + * expand_colnames_array_to: make colinfo->colnames at least n items long + * + * Any added array entries are initialized to zero. + */ +static void +expand_colnames_array_to(deparse_columns *colinfo, int n) +{ + if (n > colinfo->num_cols) + { + if (colinfo->colnames == NULL) + colinfo->colnames = (char **) palloc0(n * sizeof(char *)); + else + { + colinfo->colnames = (char **) repalloc(colinfo->colnames, + n * sizeof(char *)); + memset(colinfo->colnames + colinfo->num_cols, 0, + (n - colinfo->num_cols) * sizeof(char *)); + } + colinfo->num_cols = n; + } +} + +/* + * identify_join_columns: figure out where columns of a join come from + * + * Fills the join-specific fields of the colinfo struct, except for + * usingNames which is filled later. + */ +static void +identify_join_columns(JoinExpr *j, RangeTblEntry *jrte, + deparse_columns *colinfo) +{ + int numjoincols; + int i; + ListCell *lc; + + /* Extract left/right child RT indexes */ + if (IsA(j->larg, RangeTblRef)) + colinfo->leftrti = ((RangeTblRef *) j->larg)->rtindex; + else if (IsA(j->larg, JoinExpr)) + colinfo->leftrti = ((JoinExpr *) j->larg)->rtindex; + else + elog(ERROR, "unrecognized node type in jointree: %d", + (int) nodeTag(j->larg)); + if (IsA(j->rarg, RangeTblRef)) + colinfo->rightrti = ((RangeTblRef *) j->rarg)->rtindex; + else if (IsA(j->rarg, JoinExpr)) + colinfo->rightrti = ((JoinExpr *) j->rarg)->rtindex; + else + elog(ERROR, "unrecognized node type in jointree: %d", + (int) nodeTag(j->rarg)); + + /* Assert children will be processed earlier than join in second pass */ + Assert(colinfo->leftrti < j->rtindex); + Assert(colinfo->rightrti < j->rtindex); + + /* Initialize result arrays with zeroes */ + numjoincols = list_length(jrte->joinaliasvars); + Assert(numjoincols == list_length(jrte->eref->colnames)); + colinfo->leftattnos = (int *) palloc0(numjoincols * sizeof(int)); + colinfo->rightattnos = (int *) palloc0(numjoincols * sizeof(int)); + + /* Scan the joinaliasvars list to identify simple column references */ + i = 0; + foreach(lc, jrte->joinaliasvars) + { + Var *aliasvar = (Var *) lfirst(lc); + + /* get rid of any implicit coercion above the Var */ + aliasvar = (Var *) strip_implicit_coercions((Node *) aliasvar); + + if (aliasvar == NULL) + { + /* It's a dropped column; nothing to do here */ + } + else if (IsA(aliasvar, Var)) + { + Assert(aliasvar->varlevelsup == 0); + Assert(aliasvar->varattno != 0); + if (aliasvar->varno == colinfo->leftrti) + colinfo->leftattnos[i] = aliasvar->varattno; + else if (aliasvar->varno == colinfo->rightrti) + colinfo->rightattnos[i] = aliasvar->varattno; + else + elog(ERROR, "unexpected varno %d in JOIN RTE", + aliasvar->varno); + } + else if (IsA(aliasvar, CoalesceExpr)) + { + /* + * It's a merged column in FULL JOIN USING. Ignore it for now and + * let the code below identify the merged columns. + */ + } + else + elog(ERROR, "unrecognized node type in join alias vars: %d", + (int) nodeTag(aliasvar)); + + i++; + } + + /* + * If there's a USING clause, deconstruct the join quals to identify the + * merged columns. This is a tad painful but if we cannot rely on the + * column names, there is no other representation of which columns were + * joined by USING. (Unless the join type is FULL, we can't tell from the + * joinaliasvars list which columns are merged.) Note: we assume that the + * merged columns are the first output column(s) of the join. + */ + if (j->usingClause) + { + List *leftvars = NIL; + List *rightvars = NIL; + ListCell *lc2; + + /* Extract left- and right-side Vars from the qual expression */ + flatten_join_using_qual(j->quals, &leftvars, &rightvars); + Assert(list_length(leftvars) == list_length(j->usingClause)); + Assert(list_length(rightvars) == list_length(j->usingClause)); + + /* Mark the output columns accordingly */ + i = 0; + forboth(lc, leftvars, lc2, rightvars) + { + Var *leftvar = (Var *) lfirst(lc); + Var *rightvar = (Var *) lfirst(lc2); + + Assert(leftvar->varlevelsup == 0); + Assert(leftvar->varattno != 0); + if (leftvar->varno != colinfo->leftrti) + elog(ERROR, "unexpected varno %d in JOIN USING qual", + leftvar->varno); + colinfo->leftattnos[i] = leftvar->varattno; + + Assert(rightvar->varlevelsup == 0); + Assert(rightvar->varattno != 0); + if (rightvar->varno != colinfo->rightrti) + elog(ERROR, "unexpected varno %d in JOIN USING qual", + rightvar->varno); + colinfo->rightattnos[i] = rightvar->varattno; + + i++; + } + } +} + +/* + * flatten_join_using_qual: extract Vars being joined from a JOIN/USING qual + * + * We assume that transformJoinUsingClause won't have produced anything except + * AND nodes, equality operator nodes, and possibly implicit coercions, and + * that the AND node inputs match left-to-right with the original USING list. + * + * Caller must initialize the result lists to NIL. + */ +static void +flatten_join_using_qual(Node *qual, List **leftvars, List **rightvars) +{ + if (IsA(qual, BoolExpr)) + { + /* Handle AND nodes by recursion */ + BoolExpr *b = (BoolExpr *) qual; + ListCell *lc; + + Assert(b->boolop == AND_EXPR); + foreach(lc, b->args) + { + flatten_join_using_qual((Node *) lfirst(lc), + leftvars, rightvars); + } + } + else if (IsA(qual, OpExpr)) + { + /* Otherwise we should have an equality operator */ + OpExpr *op = (OpExpr *) qual; + Var *var; + + if (list_length(op->args) != 2) + elog(ERROR, "unexpected unary operator in JOIN/USING qual"); + /* Arguments should be Vars with perhaps implicit coercions */ + var = (Var *) strip_implicit_coercions((Node *) linitial(op->args)); + if (!IsA(var, Var)) + elog(ERROR, "unexpected node type in JOIN/USING qual: %d", + (int) nodeTag(var)); + *leftvars = lappend(*leftvars, var); + var = (Var *) strip_implicit_coercions((Node *) lsecond(op->args)); + if (!IsA(var, Var)) + elog(ERROR, "unexpected node type in JOIN/USING qual: %d", + (int) nodeTag(var)); + *rightvars = lappend(*rightvars, var); + } + else + { + /* Perhaps we have an implicit coercion to boolean? */ + Node *q = strip_implicit_coercions(qual); + + if (q != qual) + flatten_join_using_qual(q, leftvars, rightvars); + else + elog(ERROR, "unexpected node type in JOIN/USING qual: %d", + (int) nodeTag(qual)); + } +} + +/* + * get_rtable_name: convenience function to get a previously assigned RTE alias + * + * The RTE must belong to the topmost namespace level in "context". + */ +static char * +get_rtable_name(int rtindex, deparse_context *context) +{ + deparse_namespace *dpns = (deparse_namespace *) linitial(context->namespaces); + + Assert(rtindex > 0 && rtindex <= list_length(dpns->rtable_names)); + return (char *) list_nth(dpns->rtable_names, rtindex - 1); +} + +/* + * set_deparse_planstate: set up deparse_namespace to parse subexpressions + * of a given PlanState node + * + * This sets the planstate, outer_planstate, inner_planstate, outer_tlist, + * inner_tlist, and index_tlist fields. Caller is responsible for adjusting + * the ancestors list if necessary. Note that the rtable and ctes fields do + * not need to change when shifting attention to different plan nodes in a + * single plan tree. + */ +static void +set_deparse_planstate(deparse_namespace *dpns, PlanState *ps) +{ + dpns->planstate = ps; + + /* + * We special-case Append and MergeAppend to pretend that the first child + * plan is the OUTER referent; we have to interpret OUTER Vars in their + * tlists according to one of the children, and the first one is the most + * natural choice. Likewise special-case ModifyTable to pretend that the + * first child plan is the OUTER referent; this is to support RETURNING + * lists containing references to non-target relations. + */ + if (IsA(ps, AppendState)) + dpns->outer_planstate = ((AppendState *) ps)->appendplans[0]; + else if (IsA(ps, MergeAppendState)) + dpns->outer_planstate = ((MergeAppendState *) ps)->mergeplans[0]; + else if (IsA(ps, ModifyTableState)) + dpns->outer_planstate = ((ModifyTableState *) ps)->mt_plans[0]; + else + dpns->outer_planstate = outerPlanState(ps); + + if (dpns->outer_planstate) + dpns->outer_tlist = dpns->outer_planstate->plan->targetlist; + else + dpns->outer_tlist = NIL; + + /* + * For a SubqueryScan, pretend the subplan is INNER referent. (We don't + * use OUTER because that could someday conflict with the normal meaning.) + * Likewise, for a CteScan, pretend the subquery's plan is INNER referent. + * For ON CONFLICT .. UPDATE we just need the inner tlist to point to the + * excluded expression's tlist. (Similar to the SubqueryScan we don't want + * to reuse OUTER, it's used for RETURNING in some modify table cases, + * although not INSERT .. CONFLICT). + */ + if (IsA(ps, SubqueryScanState)) + dpns->inner_planstate = ((SubqueryScanState *) ps)->subplan; + else if (IsA(ps, CteScanState)) + dpns->inner_planstate = ((CteScanState *) ps)->cteplanstate; + else if (IsA(ps, ModifyTableState)) + dpns->inner_planstate = ps; + else + dpns->inner_planstate = innerPlanState(ps); + + if (IsA(ps, ModifyTableState)) + dpns->inner_tlist = ((ModifyTableState *) ps)->mt_excludedtlist; + else if (dpns->inner_planstate) + dpns->inner_tlist = dpns->inner_planstate->plan->targetlist; + else + dpns->inner_tlist = NIL; + + /* Set up referent for INDEX_VAR Vars, if needed */ + if (IsA(ps->plan, IndexOnlyScan)) + dpns->index_tlist = ((IndexOnlyScan *) ps->plan)->indextlist; + else if (IsA(ps->plan, ForeignScan)) + dpns->index_tlist = ((ForeignScan *) ps->plan)->fdw_scan_tlist; + else if (IsA(ps->plan, CustomScan)) + dpns->index_tlist = ((CustomScan *) ps->plan)->custom_scan_tlist; + else + dpns->index_tlist = NIL; +} + +/* + * push_child_plan: temporarily transfer deparsing attention to a child plan + * + * When expanding an OUTER_VAR or INNER_VAR reference, we must adjust the + * deparse context in case the referenced expression itself uses + * OUTER_VAR/INNER_VAR. We modify the top stack entry in-place to avoid + * affecting levelsup issues (although in a Plan tree there really shouldn't + * be any). + * + * Caller must provide a local deparse_namespace variable to save the + * previous state for pop_child_plan. + */ +static void +push_child_plan(deparse_namespace *dpns, PlanState *ps, + deparse_namespace *save_dpns) +{ + /* Save state for restoration later */ + *save_dpns = *dpns; + + /* Link current plan node into ancestors list */ + dpns->ancestors = lcons(dpns->planstate, dpns->ancestors); + + /* Set attention on selected child */ + set_deparse_planstate(dpns, ps); +} + +/* + * pop_child_plan: undo the effects of push_child_plan + */ +static void +pop_child_plan(deparse_namespace *dpns, deparse_namespace *save_dpns) +{ + List *ancestors; + + /* Get rid of ancestors list cell added by push_child_plan */ + ancestors = list_delete_first(dpns->ancestors); + + /* Restore fields changed by push_child_plan */ + *dpns = *save_dpns; + + /* Make sure dpns->ancestors is right (may be unnecessary) */ + dpns->ancestors = ancestors; +} + +/* + * push_ancestor_plan: temporarily transfer deparsing attention to an + * ancestor plan + * + * When expanding a Param reference, we must adjust the deparse context + * to match the plan node that contains the expression being printed; + * otherwise we'd fail if that expression itself contains a Param or + * OUTER_VAR/INNER_VAR/INDEX_VAR variable. + * + * The target ancestor is conveniently identified by the ListCell holding it + * in dpns->ancestors. + * + * Caller must provide a local deparse_namespace variable to save the + * previous state for pop_ancestor_plan. + */ +static void +push_ancestor_plan(deparse_namespace *dpns, ListCell *ancestor_cell, + deparse_namespace *save_dpns) +{ + Plan *plan = (Plan *) lfirst(ancestor_cell); + + /* Save state for restoration later */ + *save_dpns = *dpns; + + /* Build a new ancestor list with just this node's ancestors */ + dpns->ancestors = + list_copy_tail(dpns->ancestors, + list_cell_number(dpns->ancestors, ancestor_cell) + 1); + + /* Set attention on selected ancestor */ + set_deparse_plan(dpns, plan); +} + +/* + * pop_ancestor_plan: undo the effects of push_ancestor_plan + */ +static void +pop_ancestor_plan(deparse_namespace *dpns, deparse_namespace *save_dpns) +{ + /* Free the ancestor list made in push_ancestor_plan */ + list_free(dpns->ancestors); + + /* Restore fields changed by push_ancestor_plan */ + *dpns = *save_dpns; +} + + +/* ---------- + * deparse_shard_query - Parse back a query for execution on a shard + * + * Builds an SQL string to perform the provided query on a specific shard and + * places this string into the provided buffer. + * ---------- + */ +void +deparse_shard_query(Query *query, Oid distrelid, int64 shardid, + StringInfo buffer) +{ + get_query_def_extended(query, buffer, NIL, distrelid, shardid, NULL, 0, + WRAP_COLUMN_DEFAULT, 0); +} + + +/* ---------- + * get_query_def - Parse back one query parsetree + * + * If resultDesc is not NULL, then it is the output tuple descriptor for + * the view represented by a SELECT query. + * ---------- + */ +static void +get_query_def(Query *query, StringInfo buf, List *parentnamespace, + TupleDesc resultDesc, + int prettyFlags, int wrapColumn, int startIndent) +{ + get_query_def_extended(query, buf, parentnamespace, InvalidOid, 0, resultDesc, + prettyFlags, wrapColumn, startIndent); +} + + +/* ---------- + * get_query_def_extended - Parse back one query parsetree, optionally + * with extension using a shard identifier. + * + * If distrelid is valid and shardid is positive, the provided shardid is added + * any time the provided relid is deparsed, so that the query may be executed + * on a placement for the given shard. + * ---------- + */ +static void +get_query_def_extended(Query *query, StringInfo buf, List *parentnamespace, + Oid distrelid, int64 shardid, TupleDesc resultDesc, + int prettyFlags, int wrapColumn, int startIndent) +{ + deparse_context context; + deparse_namespace dpns; + + OverrideSearchPath *overridePath = NULL; + + /* Guard against excessively long or deeply-nested queries */ + CHECK_FOR_INTERRUPTS(); + check_stack_depth(); + + /* + * Before we begin to examine the query, acquire locks on referenced + * relations, and fix up deleted columns in JOIN RTEs. This ensures + * consistent results. Note we assume it's OK to scribble on the passed + * querytree! + * + * We are only deparsing the query (we are not about to execute it), so we + * only need AccessShareLock on the relations it mentions. + */ + AcquireRewriteLocks(query, false, false); + + /* + * Set search_path to NIL so that all objects outside of pg_catalog will be + * schema-prefixed. pg_catalog will be added automatically when we call + * PushOverrideSearchPath(), since we set addCatalog to true; + */ + overridePath = GetOverrideSearchPath(CurrentMemoryContext); + overridePath->schemas = NIL; + overridePath->addCatalog = true; + PushOverrideSearchPath(overridePath); + + context.buf = buf; + context.namespaces = lcons(&dpns, list_copy(parentnamespace)); + context.windowClause = NIL; + context.windowTList = NIL; + context.varprefix = (parentnamespace != NIL || + list_length(query->rtable) != 1); + context.prettyFlags = prettyFlags; + context.wrapColumn = wrapColumn; + context.indentLevel = startIndent; + context.special_exprkind = EXPR_KIND_NONE; + context.distrelid = distrelid; + context.shardid = shardid; + + set_deparse_for_query(&dpns, query, parentnamespace); + + switch (query->commandType) + { + case CMD_SELECT: + get_select_query_def(query, &context, resultDesc); + break; + + case CMD_UPDATE: + get_update_query_def(query, &context); + break; + + case CMD_INSERT: + get_insert_query_def(query, &context); + break; + + case CMD_DELETE: + get_delete_query_def(query, &context); + break; + + case CMD_NOTHING: + appendStringInfoString(buf, "NOTHING"); + break; + + case CMD_UTILITY: + get_utility_query_def(query, &context); + break; + + default: + elog(ERROR, "unrecognized query command type: %d", + query->commandType); + break; + } + + /* revert back to original search_path */ + PopOverrideSearchPath(); +} + +/* ---------- + * get_values_def - Parse back a VALUES list + * ---------- + */ +static void +get_values_def(List *values_lists, deparse_context *context) +{ + StringInfo buf = context->buf; + bool first_list = true; + ListCell *vtl; + + appendStringInfoString(buf, "VALUES "); + + foreach(vtl, values_lists) + { + List *sublist = (List *) lfirst(vtl); + bool first_col = true; + ListCell *lc; + + if (first_list) + first_list = false; + else + appendStringInfoString(buf, ", "); + + appendStringInfoChar(buf, '('); + foreach(lc, sublist) + { + Node *col = (Node *) lfirst(lc); + + if (first_col) + first_col = false; + else + appendStringInfoChar(buf, ','); + + /* + * Print the value. Whole-row Vars need special treatment. + */ + get_rule_expr_toplevel(col, context, false); + } + appendStringInfoChar(buf, ')'); + } +} + +/* ---------- + * get_with_clause - Parse back a WITH clause + * ---------- + */ +static void +get_with_clause(Query *query, deparse_context *context) +{ + StringInfo buf = context->buf; + const char *sep; + ListCell *l; + + if (query->cteList == NIL) + return; + + if (PRETTY_INDENT(context)) + { + context->indentLevel += PRETTYINDENT_STD; + appendStringInfoChar(buf, ' '); + } + + if (query->hasRecursive) + sep = "WITH RECURSIVE "; + else + sep = "WITH "; + foreach(l, query->cteList) + { + CommonTableExpr *cte = (CommonTableExpr *) lfirst(l); + + appendStringInfoString(buf, sep); + appendStringInfoString(buf, quote_identifier(cte->ctename)); + if (cte->aliascolnames) + { + bool first = true; + ListCell *col; + + appendStringInfoChar(buf, '('); + foreach(col, cte->aliascolnames) + { + if (first) + first = false; + else + appendStringInfoString(buf, ", "); + appendStringInfoString(buf, + quote_identifier(strVal(lfirst(col)))); + } + appendStringInfoChar(buf, ')'); + } + appendStringInfoString(buf, " AS "); + switch (cte->ctematerialized) + { + case CTEMaterializeDefault: + break; + case CTEMaterializeAlways: + appendStringInfoString(buf, "MATERIALIZED "); + break; + case CTEMaterializeNever: + appendStringInfoString(buf, "NOT MATERIALIZED "); + break; + } + appendStringInfoChar(buf, '('); + if (PRETTY_INDENT(context)) + appendContextKeyword(context, "", 0, 0, 0); + get_query_def((Query *) cte->ctequery, buf, context->namespaces, NULL, + context->prettyFlags, context->wrapColumn, + context->indentLevel); + if (PRETTY_INDENT(context)) + appendContextKeyword(context, "", 0, 0, 0); + appendStringInfoChar(buf, ')'); + sep = ", "; + } + + if (PRETTY_INDENT(context)) + { + context->indentLevel -= PRETTYINDENT_STD; + appendContextKeyword(context, "", 0, 0, 0); + } + else + appendStringInfoChar(buf, ' '); +} + +/* ---------- + * get_select_query_def - Parse back a SELECT parsetree + * ---------- + */ +static void +get_select_query_def(Query *query, deparse_context *context, + TupleDesc resultDesc) +{ + StringInfo buf = context->buf; + List *save_windowclause; + List *save_windowtlist; + bool force_colno; + ListCell *l; + + /* Insert the WITH clause if given */ + get_with_clause(query, context); + + /* Set up context for possible window functions */ + save_windowclause = context->windowClause; + context->windowClause = query->windowClause; + save_windowtlist = context->windowTList; + context->windowTList = query->targetList; + + /* + * If the Query node has a setOperations tree, then it's the top level of + * a UNION/INTERSECT/EXCEPT query; only the WITH, ORDER BY and LIMIT + * fields are interesting in the top query itself. + */ + if (query->setOperations) + { + get_setop_query(query->setOperations, query, context, resultDesc); + /* ORDER BY clauses must be simple in this case */ + force_colno = true; + } + else + { + get_basic_select_query(query, context, resultDesc); + force_colno = false; + } + + /* Add the ORDER BY clause if given */ + if (query->sortClause != NIL) + { + appendContextKeyword(context, " ORDER BY ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_rule_orderby(query->sortClause, query->targetList, + force_colno, context); + } + + /* Add the LIMIT clause if given */ + if (query->limitOffset != NULL) + { + appendContextKeyword(context, " OFFSET ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); + get_rule_expr(query->limitOffset, context, false); + } + if (query->limitCount != NULL) + { + appendContextKeyword(context, " LIMIT ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); + if (IsA(query->limitCount, Const) && + ((Const *) query->limitCount)->constisnull) + appendStringInfoString(buf, "ALL"); + else + get_rule_expr(query->limitCount, context, false); + } + + /* Add FOR [KEY] UPDATE/SHARE clauses if present */ + if (query->hasForUpdate) + { + foreach(l, query->rowMarks) + { + RowMarkClause *rc = (RowMarkClause *) lfirst(l); + + /* don't print implicit clauses */ + if (rc->pushedDown) + continue; + + switch (rc->strength) + { + case LCS_NONE: + /* we intentionally throw an error for LCS_NONE */ + elog(ERROR, "unrecognized LockClauseStrength %d", + (int) rc->strength); + break; + case LCS_FORKEYSHARE: + appendContextKeyword(context, " FOR KEY SHARE", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); + break; + case LCS_FORSHARE: + appendContextKeyword(context, " FOR SHARE", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); + break; + case LCS_FORNOKEYUPDATE: + appendContextKeyword(context, " FOR NO KEY UPDATE", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); + break; + case LCS_FORUPDATE: + appendContextKeyword(context, " FOR UPDATE", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); + break; + } + + appendStringInfo(buf, " OF %s", + quote_identifier(get_rtable_name(rc->rti, + context))); + if (rc->waitPolicy == LockWaitError) + appendStringInfoString(buf, " NOWAIT"); + else if (rc->waitPolicy == LockWaitSkip) + appendStringInfoString(buf, " SKIP LOCKED"); + } + } + + context->windowClause = save_windowclause; + context->windowTList = save_windowtlist; +} + +/* + * Detect whether query looks like SELECT ... FROM VALUES(); + * if so, return the VALUES RTE. Otherwise return NULL. + */ +static RangeTblEntry * +get_simple_values_rte(Query *query) +{ + RangeTblEntry *result = NULL; + ListCell *lc; + + /* + * We want to return true even if the Query also contains OLD or NEW rule + * RTEs. So the idea is to scan the rtable and see if there is only one + * inFromCl RTE that is a VALUES RTE. + */ + foreach(lc, query->rtable) + { + RangeTblEntry *rte = (RangeTblEntry *) lfirst(lc); + + if (rte->rtekind == RTE_VALUES && rte->inFromCl) + { + if (result) + return NULL; /* multiple VALUES (probably not possible) */ + result = rte; + } + else if (rte->rtekind == RTE_RELATION && !rte->inFromCl) + continue; /* ignore rule entries */ + else + return NULL; /* something else -> not simple VALUES */ + } + + /* + * We don't need to check the targetlist in any great detail, because + * parser/analyze.c will never generate a "bare" VALUES RTE --- they only + * appear inside auto-generated sub-queries with very restricted + * structure. However, DefineView might have modified the tlist by + * injecting new column aliases; so compare tlist resnames against the + * RTE's names to detect that. + */ + if (result) + { + ListCell *lcn; + + if (list_length(query->targetList) != list_length(result->eref->colnames)) + return NULL; /* this probably cannot happen */ + forboth(lc, query->targetList, lcn, result->eref->colnames) + { + TargetEntry *tle = (TargetEntry *) lfirst(lc); + char *cname = strVal(lfirst(lcn)); + + if (tle->resjunk) + return NULL; /* this probably cannot happen */ + if (tle->resname == NULL || strcmp(tle->resname, cname) != 0) + return NULL; /* column name has been changed */ + } + } + + return result; +} + +static void +get_basic_select_query(Query *query, deparse_context *context, + TupleDesc resultDesc) +{ + StringInfo buf = context->buf; + RangeTblEntry *values_rte; + char *sep; + ListCell *l; + + if (PRETTY_INDENT(context)) + { + context->indentLevel += PRETTYINDENT_STD; + appendStringInfoChar(buf, ' '); + } + + /* + * If the query looks like SELECT * FROM (VALUES ...), then print just the + * VALUES part. This reverses what transformValuesClause() did at parse + * time. + */ + values_rte = get_simple_values_rte(query); + if (values_rte) + { + get_values_def(values_rte->values_lists, context); + return; + } + + /* + * Build up the query string - first we say SELECT + */ + appendStringInfoString(buf, "SELECT"); + + /* Add the DISTINCT clause if given */ + if (query->distinctClause != NIL) + { + if (query->hasDistinctOn) + { + appendStringInfoString(buf, " DISTINCT ON ("); + sep = ""; + foreach(l, query->distinctClause) + { + SortGroupClause *srt = (SortGroupClause *) lfirst(l); + + appendStringInfoString(buf, sep); + get_rule_sortgroupclause(srt->tleSortGroupRef, query->targetList, + false, context); + sep = ", "; + } + appendStringInfoChar(buf, ')'); + } + else + appendStringInfoString(buf, " DISTINCT"); + } + + /* Then we tell what to select (the targetlist) */ + get_target_list(query->targetList, context, resultDesc); + + /* Add the FROM clause if needed */ + get_from_clause(query, " FROM ", context); + + /* Add the WHERE clause if given */ + if (query->jointree->quals != NULL) + { + appendContextKeyword(context, " WHERE ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_rule_expr(query->jointree->quals, context, false); + } + + /* Add the GROUP BY clause if given */ + if (query->groupClause != NULL || query->groupingSets != NULL) + { + ParseExprKind save_exprkind; + + appendContextKeyword(context, " GROUP BY ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + + save_exprkind = context->special_exprkind; + context->special_exprkind = EXPR_KIND_GROUP_BY; + + if (query->groupingSets == NIL) + { + sep = ""; + foreach(l, query->groupClause) + { + SortGroupClause *grp = (SortGroupClause *) lfirst(l); + + appendStringInfoString(buf, sep); + get_rule_sortgroupclause(grp->tleSortGroupRef, query->targetList, + false, context); + sep = ", "; + } + } + else + { + sep = ""; + foreach(l, query->groupingSets) + { + GroupingSet *grp = lfirst(l); + + appendStringInfoString(buf, sep); + get_rule_groupingset(grp, query->targetList, true, context); + sep = ", "; + } + } + + context->special_exprkind = save_exprkind; + } + + /* Add the HAVING clause if given */ + if (query->havingQual != NULL) + { + appendContextKeyword(context, " HAVING ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); + get_rule_expr(query->havingQual, context, false); + } + + /* Add the WINDOW clause if needed */ + if (query->windowClause != NIL) + get_rule_windowclause(query, context); +} + +/* ---------- + * get_target_list - Parse back a SELECT target list + * + * This is also used for RETURNING lists in INSERT/UPDATE/DELETE. + * ---------- + */ +static void +get_target_list(List *targetList, deparse_context *context, + TupleDesc resultDesc) +{ + StringInfo buf = context->buf; + StringInfoData targetbuf; + bool last_was_multiline = false; + char *sep; + int colno; + ListCell *l; + + /* we use targetbuf to hold each TLE's text temporarily */ + initStringInfo(&targetbuf); + + sep = " "; + colno = 0; + foreach(l, targetList) + { + TargetEntry *tle = (TargetEntry *) lfirst(l); + char *colname; + char *attname; + + if (tle->resjunk) + continue; /* ignore junk entries */ + + appendStringInfoString(buf, sep); + sep = ", "; + colno++; + + /* + * Put the new field text into targetbuf so we can decide after we've + * got it whether or not it needs to go on a new line. + */ + resetStringInfo(&targetbuf); + context->buf = &targetbuf; + + /* + * We special-case Var nodes rather than using get_rule_expr. This is + * needed because get_rule_expr will display a whole-row Var as + * "foo.*", which is the preferred notation in most contexts, but at + * the top level of a SELECT list it's not right (the parser will + * expand that notation into multiple columns, yielding behavior + * different from a whole-row Var). We need to call get_variable + * directly so that we can tell it to do the right thing, and so that + * we can get the attribute name which is the default AS label. + */ + if (tle->expr && (IsA(tle->expr, Var))) + { + attname = get_variable((Var *) tle->expr, 0, true, context); + } + else + { + get_rule_expr((Node *) tle->expr, context, true); + /* We'll show the AS name unless it's this: */ + attname = "?column?"; + } + + /* + * Figure out what the result column should be called. In the context + * of a view, use the view's tuple descriptor (so as to pick up the + * effects of any column RENAME that's been done on the view). + * Otherwise, just use what we can find in the TLE. + */ + if (resultDesc && colno <= resultDesc->natts) + colname = NameStr(TupleDescAttr(resultDesc, colno - 1)->attname); + else + colname = tle->resname; + + /* Show AS unless the column's name is correct as-is */ + if (colname) /* resname could be NULL */ + { + if (attname == NULL || strcmp(attname, colname) != 0) + appendStringInfo(&targetbuf, " AS %s", quote_identifier(colname)); + } + + /* Restore context's output buffer */ + context->buf = buf; + + /* Consider line-wrapping if enabled */ + if (PRETTY_INDENT(context) && context->wrapColumn >= 0) + { + int leading_nl_pos; + + /* Does the new field start with a new line? */ + if (targetbuf.len > 0 && targetbuf.data[0] == '\n') + leading_nl_pos = 0; + else + leading_nl_pos = -1; + + /* If so, we shouldn't add anything */ + if (leading_nl_pos >= 0) + { + /* instead, remove any trailing spaces currently in buf */ + removeStringInfoSpaces(buf); + } + else + { + char *trailing_nl; + + /* Locate the start of the current line in the output buffer */ + trailing_nl = strrchr(buf->data, '\n'); + if (trailing_nl == NULL) + trailing_nl = buf->data; + else + trailing_nl++; + + /* + * Add a newline, plus some indentation, if the new field is + * not the first and either the new field would cause an + * overflow or the last field used more than one line. + */ + if (colno > 1 && + ((strlen(trailing_nl) + targetbuf.len > context->wrapColumn) || + last_was_multiline)) + appendContextKeyword(context, "", -PRETTYINDENT_STD, + PRETTYINDENT_STD, PRETTYINDENT_VAR); + } + + /* Remember this field's multiline status for next iteration */ + last_was_multiline = + (strchr(targetbuf.data + leading_nl_pos + 1, '\n') != NULL); + } + + /* Add the new field */ + appendStringInfoString(buf, targetbuf.data); + } + + /* clean up */ + pfree(targetbuf.data); +} + +static void +get_setop_query(Node *setOp, Query *query, deparse_context *context, + TupleDesc resultDesc) +{ + StringInfo buf = context->buf; + bool need_paren; + + /* Guard against excessively long or deeply-nested queries */ + CHECK_FOR_INTERRUPTS(); + check_stack_depth(); + + if (IsA(setOp, RangeTblRef)) + { + RangeTblRef *rtr = (RangeTblRef *) setOp; + RangeTblEntry *rte = rt_fetch(rtr->rtindex, query->rtable); + Query *subquery = rte->subquery; + + Assert(subquery != NULL); + Assert(subquery->setOperations == NULL); + /* Need parens if WITH, ORDER BY, FOR UPDATE, or LIMIT; see gram.y */ + need_paren = (subquery->cteList || + subquery->sortClause || + subquery->rowMarks || + subquery->limitOffset || + subquery->limitCount); + if (need_paren) + appendStringInfoChar(buf, '('); + get_query_def(subquery, buf, context->namespaces, resultDesc, + context->prettyFlags, context->wrapColumn, + context->indentLevel); + if (need_paren) + appendStringInfoChar(buf, ')'); + } + else if (IsA(setOp, SetOperationStmt)) + { + SetOperationStmt *op = (SetOperationStmt *) setOp; + int subindent; + + /* + * We force parens when nesting two SetOperationStmts, except when the + * lefthand input is another setop of the same kind. Syntactically, + * we could omit parens in rather more cases, but it seems best to use + * parens to flag cases where the setop operator changes. If we use + * parens, we also increase the indentation level for the child query. + * + * There are some cases in which parens are needed around a leaf query + * too, but those are more easily handled at the next level down (see + * code above). + */ + if (IsA(op->larg, SetOperationStmt)) + { + SetOperationStmt *lop = (SetOperationStmt *) op->larg; + + if (op->op == lop->op && op->all == lop->all) + need_paren = false; + else + need_paren = true; + } + else + need_paren = false; + + if (need_paren) + { + appendStringInfoChar(buf, '('); + subindent = PRETTYINDENT_STD; + appendContextKeyword(context, "", subindent, 0, 0); + } + else + subindent = 0; + + get_setop_query(op->larg, query, context, resultDesc); + + if (need_paren) + appendContextKeyword(context, ") ", -subindent, 0, 0); + else if (PRETTY_INDENT(context)) + appendContextKeyword(context, "", -subindent, 0, 0); + else + appendStringInfoChar(buf, ' '); + + switch (op->op) + { + case SETOP_UNION: + appendStringInfoString(buf, "UNION "); + break; + case SETOP_INTERSECT: + appendStringInfoString(buf, "INTERSECT "); + break; + case SETOP_EXCEPT: + appendStringInfoString(buf, "EXCEPT "); + break; + default: + elog(ERROR, "unrecognized set op: %d", + (int) op->op); + } + if (op->all) + appendStringInfoString(buf, "ALL "); + + /* Always parenthesize if RHS is another setop */ + need_paren = IsA(op->rarg, SetOperationStmt); + + /* + * The indentation code here is deliberately a bit different from that + * for the lefthand input, because we want the line breaks in + * different places. + */ + if (need_paren) + { + appendStringInfoChar(buf, '('); + subindent = PRETTYINDENT_STD; + } + else + subindent = 0; + appendContextKeyword(context, "", subindent, 0, 0); + + get_setop_query(op->rarg, query, context, resultDesc); + + if (PRETTY_INDENT(context)) + context->indentLevel -= subindent; + if (need_paren) + appendContextKeyword(context, ")", 0, 0, 0); + } + else + { + elog(ERROR, "unrecognized node type: %d", + (int) nodeTag(setOp)); + } +} + +/* + * Display a sort/group clause. + * + * Also returns the expression tree, so caller need not find it again. + */ +static Node * +get_rule_sortgroupclause(Index ref, List *tlist, bool force_colno, + deparse_context *context) +{ + StringInfo buf = context->buf; + TargetEntry *tle; + Node *expr; + + tle = get_sortgroupref_tle(ref, tlist); + expr = (Node *) tle->expr; + + /* + * Use column-number form if requested by caller. Otherwise, if + * expression is a constant, force it to be dumped with an explicit cast + * as decoration --- this is because a simple integer constant is + * ambiguous (and will be misinterpreted by findTargetlistEntry()) if we + * dump it without any decoration. If it's anything more complex than a + * simple Var, then force extra parens around it, to ensure it can't be + * misinterpreted as a cube() or rollup() construct. + */ + if (force_colno) + { + Assert(!tle->resjunk); + appendStringInfo(buf, "%d", tle->resno); + } + else if (expr && IsA(expr, Const)) + get_const_expr((Const *) expr, context, 1); + else if (!expr || IsA(expr, Var)) + get_rule_expr(expr, context, true); + else + { + /* + * We must force parens for function-like expressions even if + * PRETTY_PAREN is off, since those are the ones in danger of + * misparsing. For other expressions we need to force them only if + * PRETTY_PAREN is on, since otherwise the expression will output them + * itself. (We can't skip the parens.) + */ + bool need_paren = (PRETTY_PAREN(context) + || IsA(expr, FuncExpr) + ||IsA(expr, Aggref) + ||IsA(expr, WindowFunc)); + + if (need_paren) + appendStringInfoChar(context->buf, '('); + get_rule_expr(expr, context, true); + if (need_paren) + appendStringInfoChar(context->buf, ')'); + } + + return expr; +} + +/* + * Display a GroupingSet + */ +static void +get_rule_groupingset(GroupingSet *gset, List *targetlist, + bool omit_parens, deparse_context *context) +{ + ListCell *l; + StringInfo buf = context->buf; + bool omit_child_parens = true; + char *sep = ""; + + switch (gset->kind) + { + case GROUPING_SET_EMPTY: + appendStringInfoString(buf, "()"); + return; + + case GROUPING_SET_SIMPLE: + { + if (!omit_parens || list_length(gset->content) != 1) + appendStringInfoChar(buf, '('); + + foreach(l, gset->content) + { + Index ref = lfirst_int(l); + + appendStringInfoString(buf, sep); + get_rule_sortgroupclause(ref, targetlist, + false, context); + sep = ", "; + } + + if (!omit_parens || list_length(gset->content) != 1) + appendStringInfoChar(buf, ')'); + } + return; + + case GROUPING_SET_ROLLUP: + appendStringInfoString(buf, "ROLLUP("); + break; + case GROUPING_SET_CUBE: + appendStringInfoString(buf, "CUBE("); + break; + case GROUPING_SET_SETS: + appendStringInfoString(buf, "GROUPING SETS ("); + omit_child_parens = false; + break; + } + + foreach(l, gset->content) + { + appendStringInfoString(buf, sep); + get_rule_groupingset(lfirst(l), targetlist, omit_child_parens, context); + sep = ", "; + } + + appendStringInfoChar(buf, ')'); +} + +/* + * Display an ORDER BY list. + */ +static void +get_rule_orderby(List *orderList, List *targetList, + bool force_colno, deparse_context *context) +{ + StringInfo buf = context->buf; + const char *sep; + ListCell *l; + + sep = ""; + foreach(l, orderList) + { + SortGroupClause *srt = (SortGroupClause *) lfirst(l); + Node *sortexpr; + Oid sortcoltype; + TypeCacheEntry *typentry; + + appendStringInfoString(buf, sep); + sortexpr = get_rule_sortgroupclause(srt->tleSortGroupRef, targetList, + force_colno, context); + sortcoltype = exprType(sortexpr); + /* See whether operator is default < or > for datatype */ + typentry = lookup_type_cache(sortcoltype, + TYPECACHE_LT_OPR | TYPECACHE_GT_OPR); + if (srt->sortop == typentry->lt_opr) + { + /* ASC is default, so emit nothing for it */ + if (srt->nulls_first) + appendStringInfoString(buf, " NULLS FIRST"); + } + else if (srt->sortop == typentry->gt_opr) + { + appendStringInfoString(buf, " DESC"); + /* DESC defaults to NULLS FIRST */ + if (!srt->nulls_first) + appendStringInfoString(buf, " NULLS LAST"); + } + else + { + appendStringInfo(buf, " USING %s", + generate_operator_name(srt->sortop, + sortcoltype, + sortcoltype)); + /* be specific to eliminate ambiguity */ + if (srt->nulls_first) + appendStringInfoString(buf, " NULLS FIRST"); + else + appendStringInfoString(buf, " NULLS LAST"); + } + sep = ", "; + } +} + +/* + * Display a WINDOW clause. + * + * Note that the windowClause list might contain only anonymous window + * specifications, in which case we should print nothing here. + */ +static void +get_rule_windowclause(Query *query, deparse_context *context) +{ + StringInfo buf = context->buf; + const char *sep; + ListCell *l; + + sep = NULL; + foreach(l, query->windowClause) + { + WindowClause *wc = (WindowClause *) lfirst(l); + + if (wc->name == NULL) + continue; /* ignore anonymous windows */ + + if (sep == NULL) + appendContextKeyword(context, " WINDOW ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + else + appendStringInfoString(buf, sep); + + appendStringInfo(buf, "%s AS ", quote_identifier(wc->name)); + + get_rule_windowspec(wc, query->targetList, context); + + sep = ", "; + } +} + +/* + * Display a window definition + */ +static void +get_rule_windowspec(WindowClause *wc, List *targetList, + deparse_context *context) +{ + StringInfo buf = context->buf; + bool needspace = false; + const char *sep; + ListCell *l; + + appendStringInfoChar(buf, '('); + if (wc->refname) + { + appendStringInfoString(buf, quote_identifier(wc->refname)); + needspace = true; + } + /* partition clauses are always inherited, so only print if no refname */ + if (wc->partitionClause && !wc->refname) + { + if (needspace) + appendStringInfoChar(buf, ' '); + appendStringInfoString(buf, "PARTITION BY "); + sep = ""; + foreach(l, wc->partitionClause) + { + SortGroupClause *grp = (SortGroupClause *) lfirst(l); + + appendStringInfoString(buf, sep); + get_rule_sortgroupclause(grp->tleSortGroupRef, targetList, + false, context); + sep = ", "; + } + needspace = true; + } + /* print ordering clause only if not inherited */ + if (wc->orderClause && !wc->copiedOrder) + { + if (needspace) + appendStringInfoChar(buf, ' '); + appendStringInfoString(buf, "ORDER BY "); + get_rule_orderby(wc->orderClause, targetList, false, context); + needspace = true; + } + /* framing clause is never inherited, so print unless it's default */ + if (wc->frameOptions & FRAMEOPTION_NONDEFAULT) + { + if (needspace) + appendStringInfoChar(buf, ' '); + if (wc->frameOptions & FRAMEOPTION_RANGE) + appendStringInfoString(buf, "RANGE "); + else if (wc->frameOptions & FRAMEOPTION_ROWS) + appendStringInfoString(buf, "ROWS "); + else if (wc->frameOptions & FRAMEOPTION_GROUPS) + appendStringInfoString(buf, "GROUPS "); + else + Assert(false); + if (wc->frameOptions & FRAMEOPTION_BETWEEN) + appendStringInfoString(buf, "BETWEEN "); + if (wc->frameOptions & FRAMEOPTION_START_UNBOUNDED_PRECEDING) + appendStringInfoString(buf, "UNBOUNDED PRECEDING "); + else if (wc->frameOptions & FRAMEOPTION_START_CURRENT_ROW) + appendStringInfoString(buf, "CURRENT ROW "); + else if (wc->frameOptions & FRAMEOPTION_START_OFFSET) + { + get_rule_expr(wc->startOffset, context, false); + if (wc->frameOptions & FRAMEOPTION_START_OFFSET_PRECEDING) + appendStringInfoString(buf, " PRECEDING "); + else if (wc->frameOptions & FRAMEOPTION_START_OFFSET_FOLLOWING) + appendStringInfoString(buf, " FOLLOWING "); + else + Assert(false); + } + else + Assert(false); + if (wc->frameOptions & FRAMEOPTION_BETWEEN) + { + appendStringInfoString(buf, "AND "); + if (wc->frameOptions & FRAMEOPTION_END_UNBOUNDED_FOLLOWING) + appendStringInfoString(buf, "UNBOUNDED FOLLOWING "); + else if (wc->frameOptions & FRAMEOPTION_END_CURRENT_ROW) + appendStringInfoString(buf, "CURRENT ROW "); + else if (wc->frameOptions & FRAMEOPTION_END_OFFSET) + { + get_rule_expr(wc->endOffset, context, false); + if (wc->frameOptions & FRAMEOPTION_END_OFFSET_PRECEDING) + appendStringInfoString(buf, " PRECEDING "); + else if (wc->frameOptions & FRAMEOPTION_END_OFFSET_FOLLOWING) + appendStringInfoString(buf, " FOLLOWING "); + else + Assert(false); + } + else + Assert(false); + } + if (wc->frameOptions & FRAMEOPTION_EXCLUDE_CURRENT_ROW) + appendStringInfoString(buf, "EXCLUDE CURRENT ROW "); + else if (wc->frameOptions & FRAMEOPTION_EXCLUDE_GROUP) + appendStringInfoString(buf, "EXCLUDE GROUP "); + else if (wc->frameOptions & FRAMEOPTION_EXCLUDE_TIES) + appendStringInfoString(buf, "EXCLUDE TIES "); + /* we will now have a trailing space; remove it */ + buf->len--; + } + appendStringInfoChar(buf, ')'); +} + +/* ---------- + * get_insert_query_def - Parse back an INSERT parsetree + * ---------- + */ +static void +get_insert_query_def(Query *query, deparse_context *context) +{ + StringInfo buf = context->buf; + RangeTblEntry *select_rte = NULL; + RangeTblEntry *values_rte = NULL; + RangeTblEntry *rte; + char *sep; + ListCell *l; + List *strippedexprs; + + /* Insert the WITH clause if given */ + get_with_clause(query, context); + + /* + * If it's an INSERT ... SELECT or multi-row VALUES, there will be a + * single RTE for the SELECT or VALUES. Plain VALUES has neither. + */ + foreach(l, query->rtable) + { + rte = (RangeTblEntry *) lfirst(l); + + if (rte->rtekind == RTE_SUBQUERY) + { + if (select_rte) + elog(ERROR, "too many subquery RTEs in INSERT"); + select_rte = rte; + } + + if (rte->rtekind == RTE_VALUES) + { + if (values_rte) + elog(ERROR, "too many values RTEs in INSERT"); + values_rte = rte; + } + } + if (select_rte && values_rte) + elog(ERROR, "both subquery and values RTEs in INSERT"); + + /* + * Start the query with INSERT INTO relname + */ + rte = rt_fetch(query->resultRelation, query->rtable); + Assert(rte->rtekind == RTE_RELATION); + + if (PRETTY_INDENT(context)) + { + context->indentLevel += PRETTYINDENT_STD; + appendStringInfoChar(buf, ' '); + } + appendStringInfo(buf, "INSERT INTO %s ", + generate_relation_or_shard_name(rte->relid, + context->distrelid, + context->shardid, NIL)); + /* INSERT requires AS keyword for target alias */ + if (rte->alias != NULL) + appendStringInfo(buf, "AS %s ", + quote_identifier(rte->alias->aliasname)); + + /* + * Add the insert-column-names list. Any indirection decoration needed on + * the column names can be inferred from the top targetlist. + */ + strippedexprs = NIL; + sep = ""; + if (query->targetList) + appendStringInfoChar(buf, '('); + foreach(l, query->targetList) + { + TargetEntry *tle = (TargetEntry *) lfirst(l); + + if (tle->resjunk) + continue; /* ignore junk entries */ + + appendStringInfoString(buf, sep); + sep = ", "; + + /* + * Put out name of target column; look in the catalogs, not at + * tle->resname, since resname will fail to track RENAME. + */ + appendStringInfoString(buf, + quote_identifier(get_attname(rte->relid, + tle->resno, + false))); + + /* + * Print any indirection needed (subfields or subscripts), and strip + * off the top-level nodes representing the indirection assignments. + * Add the stripped expressions to strippedexprs. (If it's a + * single-VALUES statement, the stripped expressions are the VALUES to + * print below. Otherwise they're just Vars and not really + * interesting.) + */ + strippedexprs = lappend(strippedexprs, + processIndirection((Node *) tle->expr, + context)); + } + if (query->targetList) + appendStringInfoString(buf, ") "); + + if (query->override) + { + if (query->override == OVERRIDING_SYSTEM_VALUE) + appendStringInfoString(buf, "OVERRIDING SYSTEM VALUE "); + else if (query->override == OVERRIDING_USER_VALUE) + appendStringInfoString(buf, "OVERRIDING USER VALUE "); + } + + if (select_rte) + { + /* Add the SELECT */ + get_query_def(select_rte->subquery, buf, NIL, NULL, + context->prettyFlags, context->wrapColumn, + context->indentLevel); + } + else if (values_rte) + { + /* Add the multi-VALUES expression lists */ + get_values_def(values_rte->values_lists, context); + } + else if (strippedexprs) + { + /* Add the single-VALUES expression list */ + appendContextKeyword(context, "VALUES (", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 2); + get_rule_expr((Node *) strippedexprs, context, false); + appendStringInfoChar(buf, ')'); + } + else + { + /* No expressions, so it must be DEFAULT VALUES */ + appendStringInfoString(buf, "DEFAULT VALUES"); + } + + /* Add ON CONFLICT if present */ + if (query->onConflict) + { + OnConflictExpr *confl = query->onConflict; + + appendStringInfoString(buf, " ON CONFLICT"); + + if (confl->arbiterElems) + { + /* Add the single-VALUES expression list */ + appendStringInfoChar(buf, '('); + get_rule_expr((Node *) confl->arbiterElems, context, false); + appendStringInfoChar(buf, ')'); + + /* Add a WHERE clause (for partial indexes) if given */ + if (confl->arbiterWhere != NULL) + { + bool save_varprefix; + + /* + * Force non-prefixing of Vars, since parser assumes that they + * belong to target relation. WHERE clause does not use + * InferenceElem, so this is separately required. + */ + save_varprefix = context->varprefix; + context->varprefix = false; + + appendContextKeyword(context, " WHERE ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_rule_expr(confl->arbiterWhere, context, false); + + context->varprefix = save_varprefix; + } + } + else if (OidIsValid(confl->constraint)) + { + char *constraint = get_constraint_name(confl->constraint); + int64 shardId = context->shardid; + + if (shardId > 0) + { + AppendShardIdToName(&constraint, shardId); + } + + if (!constraint) + elog(ERROR, "cache lookup failed for constraint %u", + confl->constraint); + appendStringInfo(buf, " ON CONSTRAINT %s", + quote_identifier(constraint)); + } + + if (confl->action == ONCONFLICT_NOTHING) + { + appendStringInfoString(buf, " DO NOTHING"); + } + else + { + appendStringInfoString(buf, " DO UPDATE SET "); + /* Deparse targetlist */ + get_update_query_targetlist_def(query, confl->onConflictSet, + context, rte); + + /* Add a WHERE clause if given */ + if (confl->onConflictWhere != NULL) + { + appendContextKeyword(context, " WHERE ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_rule_expr(confl->onConflictWhere, context, false); + } + } + } + + /* Add RETURNING if present */ + if (query->returningList) + { + appendContextKeyword(context, " RETURNING", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_target_list(query->returningList, context, NULL); + } +} + + +/* ---------- + * get_update_query_def - Parse back an UPDATE parsetree + * ---------- + */ +static void +get_update_query_def(Query *query, deparse_context *context) +{ + StringInfo buf = context->buf; + RangeTblEntry *rte; + + /* Insert the WITH clause if given */ + get_with_clause(query, context); + + /* + * Start the query with UPDATE relname SET + */ + rte = rt_fetch(query->resultRelation, query->rtable); + + if (PRETTY_INDENT(context)) + { + appendStringInfoChar(buf, ' '); + context->indentLevel += PRETTYINDENT_STD; + } + + /* if it's a shard, do differently */ + if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) + { + char *fragmentSchemaName = NULL; + char *fragmentTableName = NULL; + + ExtractRangeTblExtraData(rte, NULL, &fragmentSchemaName, &fragmentTableName, NULL); + + /* use schema and table name from the remote alias */ + appendStringInfo(buf, "UPDATE %s%s", + only_marker(rte), + generate_fragment_name(fragmentSchemaName, fragmentTableName)); + + if(rte->eref != NULL) + appendStringInfo(buf, " %s", + quote_identifier(rte->eref->aliasname)); + } + else + { + appendStringInfo(buf, "UPDATE %s%s", + only_marker(rte), + generate_relation_or_shard_name(rte->relid, + context->distrelid, + context->shardid, NIL)); + + if (rte->alias != NULL) + appendStringInfo(buf, " %s", + quote_identifier(rte->alias->aliasname)); + } + + appendStringInfoString(buf, " SET "); + + /* Deparse targetlist */ + get_update_query_targetlist_def(query, query->targetList, context, rte); + + /* Add the FROM clause if needed */ + get_from_clause(query, " FROM ", context); + + /* Add a WHERE clause if given */ + if (query->jointree->quals != NULL) + { + appendContextKeyword(context, " WHERE ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_rule_expr(query->jointree->quals, context, false); + } + + /* Add RETURNING if present */ + if (query->returningList) + { + appendContextKeyword(context, " RETURNING", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_target_list(query->returningList, context, NULL); + } +} + + +/* ---------- + * get_update_query_targetlist_def - Parse back an UPDATE targetlist + * ---------- + */ +static void +get_update_query_targetlist_def(Query *query, List *targetList, + deparse_context *context, RangeTblEntry *rte) +{ + StringInfo buf = context->buf; + ListCell *l; + ListCell *next_ma_cell; + int remaining_ma_columns; + const char *sep; + SubLink *cur_ma_sublink; + List *ma_sublinks; + + /* + * Prepare to deal with MULTIEXPR assignments: collect the source SubLinks + * into a list. We expect them to appear, in ID order, in resjunk tlist + * entries. + */ + ma_sublinks = NIL; + if (query->hasSubLinks) /* else there can't be any */ + { + foreach(l, targetList) + { + TargetEntry *tle = (TargetEntry *) lfirst(l); + + if (tle->resjunk && IsA(tle->expr, SubLink)) + { + SubLink *sl = (SubLink *) tle->expr; + + if (sl->subLinkType == MULTIEXPR_SUBLINK) + { + ma_sublinks = lappend(ma_sublinks, sl); + Assert(sl->subLinkId == list_length(ma_sublinks)); + } + } + } + } + next_ma_cell = list_head(ma_sublinks); + cur_ma_sublink = NULL; + remaining_ma_columns = 0; + + /* Add the comma separated list of 'attname = value' */ + sep = ""; + foreach(l, targetList) + { + TargetEntry *tle = (TargetEntry *) lfirst(l); + Node *expr; + + if (tle->resjunk) + continue; /* ignore junk entries */ + + /* Emit separator (OK whether we're in multiassignment or not) */ + appendStringInfoString(buf, sep); + sep = ", "; + + /* + * Check to see if we're starting a multiassignment group: if so, + * output a left paren. + */ + if (next_ma_cell != NULL && cur_ma_sublink == NULL) + { + /* + * We must dig down into the expr to see if it's a PARAM_MULTIEXPR + * Param. That could be buried under FieldStores and + * SubscriptingRefs and CoerceToDomains (cf processIndirection()), + * and underneath those there could be an implicit type coercion. + * Because we would ignore implicit type coercions anyway, we + * don't need to be as careful as processIndirection() is about + * descending past implicit CoerceToDomains. + */ + expr = (Node *) tle->expr; + while (expr) + { + if (IsA(expr, FieldStore)) + { + FieldStore *fstore = (FieldStore *) expr; + + expr = (Node *) linitial(fstore->newvals); + } + else if (IsA(expr, SubscriptingRef)) + { + SubscriptingRef *sbsref = (SubscriptingRef *) expr; + + if (sbsref->refassgnexpr == NULL) + break; + expr = (Node *) sbsref->refassgnexpr; + } + else if (IsA(expr, CoerceToDomain)) + { + CoerceToDomain *cdomain = (CoerceToDomain *) expr; + + if (cdomain->coercionformat != COERCE_IMPLICIT_CAST) + break; + expr = (Node *) cdomain->arg; + } + else + break; + } + expr = strip_implicit_coercions(expr); + + if (expr && IsA(expr, Param) && + ((Param *) expr)->paramkind == PARAM_MULTIEXPR) + { + cur_ma_sublink = (SubLink *) lfirst(next_ma_cell); + next_ma_cell = lnext(ma_sublinks, next_ma_cell); + remaining_ma_columns = count_nonjunk_tlist_entries( + ((Query *) cur_ma_sublink->subselect)->targetList); + Assert(((Param *) expr)->paramid == + ((cur_ma_sublink->subLinkId << 16) | 1)); + appendStringInfoChar(buf, '('); + } + } + + /* + * Put out name of target column; look in the catalogs, not at + * tle->resname, since resname will fail to track RENAME. + */ + appendStringInfoString(buf, + quote_identifier(get_attname(rte->relid, + tle->resno, + false))); + + /* + * Print any indirection needed (subfields or subscripts), and strip + * off the top-level nodes representing the indirection assignments. + */ + expr = processIndirection((Node *) tle->expr, context); + + /* + * If we're in a multiassignment, skip printing anything more, unless + * this is the last column; in which case, what we print should be the + * sublink, not the Param. + */ + if (cur_ma_sublink != NULL) + { + if (--remaining_ma_columns > 0) + continue; /* not the last column of multiassignment */ + appendStringInfoChar(buf, ')'); + expr = (Node *) cur_ma_sublink; + cur_ma_sublink = NULL; + } + + appendStringInfoString(buf, " = "); + + get_rule_expr(expr, context, false); + } +} + + +/* ---------- + * get_delete_query_def - Parse back a DELETE parsetree + * ---------- + */ +static void +get_delete_query_def(Query *query, deparse_context *context) +{ + StringInfo buf = context->buf; + RangeTblEntry *rte; + + /* Insert the WITH clause if given */ + get_with_clause(query, context); + + /* + * Start the query with DELETE FROM relname + */ + rte = rt_fetch(query->resultRelation, query->rtable); + + if (PRETTY_INDENT(context)) + { + appendStringInfoChar(buf, ' '); + context->indentLevel += PRETTYINDENT_STD; + } + + /* if it's a shard, do differently */ + if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) + { + char *fragmentSchemaName = NULL; + char *fragmentTableName = NULL; + + ExtractRangeTblExtraData(rte, NULL, &fragmentSchemaName, &fragmentTableName, NULL); + + /* use schema and table name from the remote alias */ + appendStringInfo(buf, "DELETE FROM %s%s", + only_marker(rte), + generate_fragment_name(fragmentSchemaName, fragmentTableName)); + + if(rte->eref != NULL) + appendStringInfo(buf, " %s", + quote_identifier(rte->eref->aliasname)); + } + else + { + appendStringInfo(buf, "DELETE FROM %s%s", + only_marker(rte), + generate_relation_or_shard_name(rte->relid, + context->distrelid, + context->shardid, NIL)); + + if (rte->alias != NULL) + appendStringInfo(buf, " %s", + quote_identifier(rte->alias->aliasname)); + } + + /* Add the USING clause if given */ + get_from_clause(query, " USING ", context); + + /* Add a WHERE clause if given */ + if (query->jointree->quals != NULL) + { + appendContextKeyword(context, " WHERE ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_rule_expr(query->jointree->quals, context, false); + } + + /* Add RETURNING if present */ + if (query->returningList) + { + appendContextKeyword(context, " RETURNING", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_target_list(query->returningList, context, NULL); + } +} + + +/* ---------- + * get_utility_query_def - Parse back a UTILITY parsetree + * ---------- + */ +static void +get_utility_query_def(Query *query, deparse_context *context) +{ + StringInfo buf = context->buf; + + if (query->utilityStmt && IsA(query->utilityStmt, NotifyStmt)) + { + NotifyStmt *stmt = (NotifyStmt *) query->utilityStmt; + + appendContextKeyword(context, "", + 0, PRETTYINDENT_STD, 1); + appendStringInfo(buf, "NOTIFY %s", + quote_identifier(stmt->conditionname)); + if (stmt->payload) + { + appendStringInfoString(buf, ", "); + simple_quote_literal(buf, stmt->payload); + } + } + else if (query->utilityStmt && IsA(query->utilityStmt, TruncateStmt)) + { + TruncateStmt *stmt = (TruncateStmt *) query->utilityStmt; + List *relationList = stmt->relations; + ListCell *relationCell = NULL; + + appendContextKeyword(context, "", + 0, PRETTYINDENT_STD, 1); + + appendStringInfo(buf, "TRUNCATE TABLE"); + + foreach(relationCell, relationList) + { + RangeVar *relationVar = (RangeVar *) lfirst(relationCell); + Oid relationId = RangeVarGetRelid(relationVar, NoLock, false); + char *relationName = generate_relation_or_shard_name(relationId, + context->distrelid, + context->shardid, NIL); + appendStringInfo(buf, " %s", relationName); + + if (lnext(relationList, relationCell) != NULL) + { + appendStringInfo(buf, ","); + } + } + + if (stmt->restart_seqs) + { + appendStringInfo(buf, " RESTART IDENTITY"); + } + + if (stmt->behavior == DROP_CASCADE) + { + appendStringInfo(buf, " CASCADE"); + } + } + else + { + /* Currently only NOTIFY utility commands can appear in rules */ + elog(ERROR, "unexpected utility statement type"); + } +} + +/* + * Display a Var appropriately. + * + * In some cases (currently only when recursing into an unnamed join) + * the Var's varlevelsup has to be interpreted with respect to a context + * above the current one; levelsup indicates the offset. + * + * If istoplevel is true, the Var is at the top level of a SELECT's + * targetlist, which means we need special treatment of whole-row Vars. + * Instead of the normal "tab.*", we'll print "tab.*::typename", which is a + * dirty hack to prevent "tab.*" from being expanded into multiple columns. + * (The parser will strip the useless coercion, so no inefficiency is added in + * dump and reload.) We used to print just "tab" in such cases, but that is + * ambiguous and will yield the wrong result if "tab" is also a plain column + * name in the query. + * + * Returns the attname of the Var, or NULL if the Var has no attname (because + * it is a whole-row Var or a subplan output reference). + */ +static char * +get_variable(Var *var, int levelsup, bool istoplevel, deparse_context *context) +{ + StringInfo buf = context->buf; + RangeTblEntry *rte; + AttrNumber attnum; + int netlevelsup; + deparse_namespace *dpns; + deparse_columns *colinfo; + char *refname; + char *attname; + + /* Find appropriate nesting depth */ + netlevelsup = var->varlevelsup + levelsup; + if (netlevelsup >= list_length(context->namespaces)) + elog(ERROR, "bogus varlevelsup: %d offset %d", + var->varlevelsup, levelsup); + dpns = (deparse_namespace *) list_nth(context->namespaces, + netlevelsup); + + /* + * Try to find the relevant RTE in this rtable. In a plan tree, it's + * likely that varno is OUTER_VAR or INNER_VAR, in which case we must dig + * down into the subplans, or INDEX_VAR, which is resolved similarly. Also + * find the aliases previously assigned for this RTE. + */ + if (var->varno >= 1 && var->varno <= list_length(dpns->rtable)) + { + rte = rt_fetch(var->varno, dpns->rtable); + refname = (char *) list_nth(dpns->rtable_names, var->varno - 1); + colinfo = deparse_columns_fetch(var->varno, dpns); + attnum = var->varattno; + } + else + { + resolve_special_varno((Node *) var, context, NULL, + get_special_variable); + return NULL; + } + + /* + * The planner will sometimes emit Vars referencing resjunk elements of a + * subquery's target list (this is currently only possible if it chooses + * to generate a "physical tlist" for a SubqueryScan or CteScan node). + * Although we prefer to print subquery-referencing Vars using the + * subquery's alias, that's not possible for resjunk items since they have + * no alias. So in that case, drill down to the subplan and print the + * contents of the referenced tlist item. This works because in a plan + * tree, such Vars can only occur in a SubqueryScan or CteScan node, and + * we'll have set dpns->inner_planstate to reference the child plan node. + */ + if ((rte->rtekind == RTE_SUBQUERY || rte->rtekind == RTE_CTE) && + attnum > list_length(rte->eref->colnames) && + dpns->inner_planstate) + { + TargetEntry *tle; + deparse_namespace save_dpns; + + tle = get_tle_by_resno(dpns->inner_tlist, var->varattno); + if (!tle) + elog(ERROR, "invalid attnum %d for relation \"%s\"", + var->varattno, rte->eref->aliasname); + + Assert(netlevelsup == 0); + push_child_plan(dpns, dpns->inner_planstate, &save_dpns); + + /* + * Force parentheses because our caller probably assumed a Var is a + * simple expression. + */ + if (!IsA(tle->expr, Var)) + appendStringInfoChar(buf, '('); + get_rule_expr((Node *) tle->expr, context, true); + if (!IsA(tle->expr, Var)) + appendStringInfoChar(buf, ')'); + + pop_child_plan(dpns, &save_dpns); + return NULL; + } + + /* + * If it's an unnamed join, look at the expansion of the alias variable. + * If it's a simple reference to one of the input vars, then recursively + * print the name of that var instead. When it's not a simple reference, + * we have to just print the unqualified join column name. (This can only + * happen with "dangerous" merged columns in a JOIN USING; we took pains + * previously to make the unqualified column name unique in such cases.) + * + * This wouldn't work in decompiling plan trees, because we don't store + * joinaliasvars lists after planning; but a plan tree should never + * contain a join alias variable. + */ + if (rte->rtekind == RTE_JOIN && rte->alias == NULL) + { + if (rte->joinaliasvars == NIL) + elog(ERROR, "cannot decompile join alias var in plan tree"); + if (attnum > 0) + { + Var *aliasvar; + + aliasvar = (Var *) list_nth(rte->joinaliasvars, attnum - 1); + /* we intentionally don't strip implicit coercions here */ + if (aliasvar && IsA(aliasvar, Var)) + { + return get_variable(aliasvar, var->varlevelsup + levelsup, + istoplevel, context); + } + } + + /* + * Unnamed join has no refname. (Note: since it's unnamed, there is + * no way the user could have referenced it to create a whole-row Var + * for it. So we don't have to cover that case below.) + */ + Assert(refname == NULL); + } + + if (attnum == InvalidAttrNumber) + attname = NULL; + else if (attnum > 0) + { + /* Get column name to use from the colinfo struct */ + if (attnum > colinfo->num_cols) + elog(ERROR, "invalid attnum %d for relation \"%s\"", + attnum, rte->eref->aliasname); + attname = colinfo->colnames[attnum - 1]; + if (attname == NULL) /* dropped column? */ + elog(ERROR, "invalid attnum %d for relation \"%s\"", + attnum, rte->eref->aliasname); + } + else if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) + { + /* System column on a Citus shard */ + attname = get_attname(rte->relid, attnum, false); + } + else + { + /* System column - name is fixed, get it from the catalog */ + attname = get_rte_attribute_name(rte, attnum); + } + + if (refname && (context->varprefix || attname == NULL)) + { + appendStringInfoString(buf, quote_identifier(refname)); + appendStringInfoChar(buf, '.'); + } + if (attname) + appendStringInfoString(buf, quote_identifier(attname)); + else + { + appendStringInfoChar(buf, '*'); + + if (istoplevel) + { + if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) + { + /* use rel.*::shard_name instead of rel.*::table_name */ + appendStringInfo(buf, "::%s", + generate_rte_shard_name(rte)); + } + else + { + appendStringInfo(buf, "::%s", + format_type_with_typemod(var->vartype, + var->vartypmod)); + } + } + } + + return attname; +} + +/* + * Deparse a Var which references OUTER_VAR, INNER_VAR, or INDEX_VAR. This + * routine is actually a callback for get_special_varno, which handles finding + * the correct TargetEntry. We get the expression contained in that + * TargetEntry and just need to deparse it, a job we can throw back on + * get_rule_expr. + */ +static void +get_special_variable(Node *node, deparse_context *context, void *private) +{ + StringInfo buf = context->buf; + + /* + * Force parentheses because our caller probably assumed a Var is a simple + * expression. + */ + if (!IsA(node, Var)) + appendStringInfoChar(buf, '('); + get_rule_expr(node, context, true); + if (!IsA(node, Var)) + appendStringInfoChar(buf, ')'); +} + +/* + * Chase through plan references to special varnos (OUTER_VAR, INNER_VAR, + * INDEX_VAR) until we find a real Var or some kind of non-Var node; then, + * invoke the callback provided. + */ +static void +resolve_special_varno(Node *node, deparse_context *context, void *private, + void (*callback) (Node *, deparse_context *, void *)) +{ + Var *var; + deparse_namespace *dpns; + + /* If it's not a Var, invoke the callback. */ + if (!IsA(node, Var)) + { + callback(node, context, private); + return; + } + + /* Find appropriate nesting depth */ + var = (Var *) node; + dpns = (deparse_namespace *) list_nth(context->namespaces, + var->varlevelsup); + + /* + * It's a special RTE, so recurse. + */ + if (var->varno == OUTER_VAR && dpns->outer_tlist) + { + TargetEntry *tle; + deparse_namespace save_dpns; + + tle = get_tle_by_resno(dpns->outer_tlist, var->varattno); + if (!tle) + elog(ERROR, "bogus varattno for OUTER_VAR var: %d", var->varattno); + + push_child_plan(dpns, dpns->outer_planstate, &save_dpns); + resolve_special_varno((Node *) tle->expr, context, private, callback); + pop_child_plan(dpns, &save_dpns); + return; + } + else if (var->varno == INNER_VAR && dpns->inner_tlist) + { + TargetEntry *tle; + deparse_namespace save_dpns; + + tle = get_tle_by_resno(dpns->inner_tlist, var->varattno); + if (!tle) + elog(ERROR, "bogus varattno for INNER_VAR var: %d", var->varattno); + + push_child_plan(dpns, dpns->inner_planstate, &save_dpns); + resolve_special_varno((Node *) tle->expr, context, private, callback); + pop_child_plan(dpns, &save_dpns); + return; + } + else if (var->varno == INDEX_VAR && dpns->index_tlist) + { + TargetEntry *tle; + + tle = get_tle_by_resno(dpns->index_tlist, var->varattno); + if (!tle) + elog(ERROR, "bogus varattno for INDEX_VAR var: %d", var->varattno); + + resolve_special_varno((Node *) tle->expr, context, private, callback); + return; + } + else if (var->varno < 1 || var->varno > list_length(dpns->rtable)) + elog(ERROR, "bogus varno: %d", var->varno); + + /* Not special. Just invoke the callback. */ + callback(node, context, private); +} + +/* + * Get the name of a field of an expression of composite type. The + * expression is usually a Var, but we handle other cases too. + * + * levelsup is an extra offset to interpret the Var's varlevelsup correctly. + * + * This is fairly straightforward when the expression has a named composite + * type; we need only look up the type in the catalogs. However, the type + * could also be RECORD. Since no actual table or view column is allowed to + * have type RECORD, a Var of type RECORD must refer to a JOIN or FUNCTION RTE + * or to a subquery output. We drill down to find the ultimate defining + * expression and attempt to infer the field name from it. We ereport if we + * can't determine the name. + * + * Similarly, a PARAM of type RECORD has to refer to some expression of + * a determinable composite type. + */ +static const char * +get_name_for_var_field(Var *var, int fieldno, + int levelsup, deparse_context *context) +{ + RangeTblEntry *rte; + AttrNumber attnum; + int netlevelsup; + deparse_namespace *dpns; + TupleDesc tupleDesc; + Node *expr; + + /* + * If it's a RowExpr that was expanded from a whole-row Var, use the + * column names attached to it. + */ + if (IsA(var, RowExpr)) + { + RowExpr *r = (RowExpr *) var; + + if (fieldno > 0 && fieldno <= list_length(r->colnames)) + return strVal(list_nth(r->colnames, fieldno - 1)); + } + + /* + * If it's a Param of type RECORD, try to find what the Param refers to. + */ + if (IsA(var, Param)) + { + Param *param = (Param *) var; + ListCell *ancestor_cell; + + expr = find_param_referent(param, context, &dpns, &ancestor_cell); + if (expr) + { + /* Found a match, so recurse to decipher the field name */ + deparse_namespace save_dpns; + const char *result; + + push_ancestor_plan(dpns, ancestor_cell, &save_dpns); + result = get_name_for_var_field((Var *) expr, fieldno, + 0, context); + pop_ancestor_plan(dpns, &save_dpns); + return result; + } + } + + /* + * If it's a Var of type RECORD, we have to find what the Var refers to; + * if not, we can use get_expr_result_tupdesc(). + */ + if (!IsA(var, Var) || + var->vartype != RECORDOID) + { + tupleDesc = get_expr_result_tupdesc((Node *) var, false); + /* Got the tupdesc, so we can extract the field name */ + Assert(fieldno >= 1 && fieldno <= tupleDesc->natts); + return NameStr(TupleDescAttr(tupleDesc, fieldno - 1)->attname); + } + + /* Find appropriate nesting depth */ + netlevelsup = var->varlevelsup + levelsup; + if (netlevelsup >= list_length(context->namespaces)) + elog(ERROR, "bogus varlevelsup: %d offset %d", + var->varlevelsup, levelsup); + dpns = (deparse_namespace *) list_nth(context->namespaces, + netlevelsup); + + /* + * Try to find the relevant RTE in this rtable. In a plan tree, it's + * likely that varno is OUTER_VAR or INNER_VAR, in which case we must dig + * down into the subplans, or INDEX_VAR, which is resolved similarly. + */ + if (var->varno >= 1 && var->varno <= list_length(dpns->rtable)) + { + rte = rt_fetch(var->varno, dpns->rtable); + attnum = var->varattno; + } + else if (var->varno == OUTER_VAR && dpns->outer_tlist) + { + TargetEntry *tle; + deparse_namespace save_dpns; + const char *result; + + tle = get_tle_by_resno(dpns->outer_tlist, var->varattno); + if (!tle) + elog(ERROR, "bogus varattno for OUTER_VAR var: %d", var->varattno); + + Assert(netlevelsup == 0); + push_child_plan(dpns, dpns->outer_planstate, &save_dpns); + + result = get_name_for_var_field((Var *) tle->expr, fieldno, + levelsup, context); + + pop_child_plan(dpns, &save_dpns); + return result; + } + else if (var->varno == INNER_VAR && dpns->inner_tlist) + { + TargetEntry *tle; + deparse_namespace save_dpns; + const char *result; + + tle = get_tle_by_resno(dpns->inner_tlist, var->varattno); + if (!tle) + elog(ERROR, "bogus varattno for INNER_VAR var: %d", var->varattno); + + Assert(netlevelsup == 0); + push_child_plan(dpns, dpns->inner_planstate, &save_dpns); + + result = get_name_for_var_field((Var *) tle->expr, fieldno, + levelsup, context); + + pop_child_plan(dpns, &save_dpns); + return result; + } + else if (var->varno == INDEX_VAR && dpns->index_tlist) + { + TargetEntry *tle; + const char *result; + + tle = get_tle_by_resno(dpns->index_tlist, var->varattno); + if (!tle) + elog(ERROR, "bogus varattno for INDEX_VAR var: %d", var->varattno); + + Assert(netlevelsup == 0); + + result = get_name_for_var_field((Var *) tle->expr, fieldno, + levelsup, context); + + return result; + } + else + { + elog(ERROR, "bogus varno: %d", var->varno); + return NULL; /* keep compiler quiet */ + } + + if (attnum == InvalidAttrNumber) + { + /* Var is whole-row reference to RTE, so select the right field */ + return get_rte_attribute_name(rte, fieldno); + } + + /* + * This part has essentially the same logic as the parser's + * expandRecordVariable() function, but we are dealing with a different + * representation of the input context, and we only need one field name + * not a TupleDesc. Also, we need special cases for finding subquery and + * CTE subplans when deparsing Plan trees. + */ + expr = (Node *) var; /* default if we can't drill down */ + + switch (rte->rtekind) + { + case RTE_RELATION: + case RTE_VALUES: + case RTE_NAMEDTUPLESTORE: + case RTE_RESULT: + + /* + * This case should not occur: a column of a table or values list + * shouldn't have type RECORD. Fall through and fail (most + * likely) at the bottom. + */ + break; + case RTE_SUBQUERY: + /* Subselect-in-FROM: examine sub-select's output expr */ + { + if (rte->subquery) + { + TargetEntry *ste = get_tle_by_resno(rte->subquery->targetList, + attnum); + + if (ste == NULL || ste->resjunk) + elog(ERROR, "subquery %s does not have attribute %d", + rte->eref->aliasname, attnum); + expr = (Node *) ste->expr; + if (IsA(expr, Var)) + { + /* + * Recurse into the sub-select to see what its Var + * refers to. We have to build an additional level of + * namespace to keep in step with varlevelsup in the + * subselect. + */ + deparse_namespace mydpns; + const char *result; + + set_deparse_for_query(&mydpns, rte->subquery, + context->namespaces); + + context->namespaces = lcons(&mydpns, + context->namespaces); + + result = get_name_for_var_field((Var *) expr, fieldno, + 0, context); + + context->namespaces = + list_delete_first(context->namespaces); + + return result; + } + /* else fall through to inspect the expression */ + } + else + { + /* + * We're deparsing a Plan tree so we don't have complete + * RTE entries (in particular, rte->subquery is NULL). But + * the only place we'd see a Var directly referencing a + * SUBQUERY RTE is in a SubqueryScan plan node, and we can + * look into the child plan's tlist instead. + */ + TargetEntry *tle; + deparse_namespace save_dpns; + const char *result; + + if (!dpns->inner_planstate) + elog(ERROR, "failed to find plan for subquery %s", + rte->eref->aliasname); + tle = get_tle_by_resno(dpns->inner_tlist, attnum); + if (!tle) + elog(ERROR, "bogus varattno for subquery var: %d", + attnum); + Assert(netlevelsup == 0); + push_child_plan(dpns, dpns->inner_planstate, &save_dpns); + + result = get_name_for_var_field((Var *) tle->expr, fieldno, + levelsup, context); + + pop_child_plan(dpns, &save_dpns); + return result; + } + } + break; + case RTE_JOIN: + /* Join RTE --- recursively inspect the alias variable */ + if (rte->joinaliasvars == NIL) + elog(ERROR, "cannot decompile join alias var in plan tree"); + Assert(attnum > 0 && attnum <= list_length(rte->joinaliasvars)); + expr = (Node *) list_nth(rte->joinaliasvars, attnum - 1); + Assert(expr != NULL); + /* we intentionally don't strip implicit coercions here */ + if (IsA(expr, Var)) + return get_name_for_var_field((Var *) expr, fieldno, + var->varlevelsup + levelsup, + context); + /* else fall through to inspect the expression */ + break; + case RTE_FUNCTION: + case RTE_TABLEFUNC: + + /* + * We couldn't get here unless a function is declared with one of + * its result columns as RECORD, which is not allowed. + */ + break; + case RTE_CTE: + /* CTE reference: examine subquery's output expr */ + { + CommonTableExpr *cte = NULL; + Index ctelevelsup; + ListCell *lc; + + /* + * Try to find the referenced CTE using the namespace stack. + */ + ctelevelsup = rte->ctelevelsup + netlevelsup; + if (ctelevelsup >= list_length(context->namespaces)) + lc = NULL; + else + { + deparse_namespace *ctedpns; + + ctedpns = (deparse_namespace *) + list_nth(context->namespaces, ctelevelsup); + foreach(lc, ctedpns->ctes) + { + cte = (CommonTableExpr *) lfirst(lc); + if (strcmp(cte->ctename, rte->ctename) == 0) + break; + } + } + if (lc != NULL) + { + Query *ctequery = (Query *) cte->ctequery; + TargetEntry *ste = get_tle_by_resno(GetCTETargetList(cte), + attnum); + + if (ste == NULL || ste->resjunk) + elog(ERROR, "subquery %s does not have attribute %d", + rte->eref->aliasname, attnum); + expr = (Node *) ste->expr; + if (IsA(expr, Var)) + { + /* + * Recurse into the CTE to see what its Var refers to. + * We have to build an additional level of namespace + * to keep in step with varlevelsup in the CTE. + * Furthermore it could be an outer CTE, so we may + * have to delete some levels of namespace. + */ + List *save_nslist = context->namespaces; + List *new_nslist; + deparse_namespace mydpns; + const char *result; + + set_deparse_for_query(&mydpns, ctequery, + context->namespaces); + + new_nslist = list_copy_tail(context->namespaces, + ctelevelsup); + context->namespaces = lcons(&mydpns, new_nslist); + + result = get_name_for_var_field((Var *) expr, fieldno, + 0, context); + + context->namespaces = save_nslist; + + return result; + } + /* else fall through to inspect the expression */ + } + else + { + /* + * We're deparsing a Plan tree so we don't have a CTE + * list. But the only place we'd see a Var directly + * referencing a CTE RTE is in a CteScan plan node, and we + * can look into the subplan's tlist instead. + */ + TargetEntry *tle; + deparse_namespace save_dpns; + const char *result; + + if (!dpns->inner_planstate) + elog(ERROR, "failed to find plan for CTE %s", + rte->eref->aliasname); + tle = get_tle_by_resno(dpns->inner_tlist, attnum); + if (!tle) + elog(ERROR, "bogus varattno for subquery var: %d", + attnum); + Assert(netlevelsup == 0); + push_child_plan(dpns, dpns->inner_planstate, &save_dpns); + + result = get_name_for_var_field((Var *) tle->expr, fieldno, + levelsup, context); + + pop_child_plan(dpns, &save_dpns); + return result; + } + } + break; + } + + /* + * We now have an expression we can't expand any more, so see if + * get_expr_result_tupdesc() can do anything with it. + */ + tupleDesc = get_expr_result_tupdesc(expr, false); + /* Got the tupdesc, so we can extract the field name */ + Assert(fieldno >= 1 && fieldno <= tupleDesc->natts); + return NameStr(TupleDescAttr(tupleDesc, fieldno - 1)->attname); +} + +/* + * Try to find the referenced expression for a PARAM_EXEC Param that might + * reference a parameter supplied by an upper NestLoop or SubPlan plan node. + * + * If successful, return the expression and set *dpns_p and *ancestor_cell_p + * appropriately for calling push_ancestor_plan(). If no referent can be + * found, return NULL. + */ +static Node * +find_param_referent(Param *param, deparse_context *context, + deparse_namespace **dpns_p, ListCell **ancestor_cell_p) +{ + /* Initialize output parameters to prevent compiler warnings */ + *dpns_p = NULL; + *ancestor_cell_p = NULL; + + /* + * If it's a PARAM_EXEC parameter, look for a matching NestLoopParam or + * SubPlan argument. This will necessarily be in some ancestor of the + * current expression's PlanState. + */ + if (param->paramkind == PARAM_EXEC) + { + deparse_namespace *dpns; + PlanState *child_ps; + bool in_same_plan_level; + ListCell *lc; + + dpns = (deparse_namespace *) linitial(context->namespaces); + child_ps = dpns->planstate; + in_same_plan_level = true; + + foreach(lc, dpns->ancestors) + { + PlanState *ps = (PlanState *) lfirst(lc); + ListCell *lc2; + + /* + * NestLoops transmit params to their inner child only; also, once + * we've crawled up out of a subplan, this couldn't possibly be + * the right match. + */ + if (IsA(ps, NestLoopState) && + child_ps == innerPlanState(ps) && + in_same_plan_level) + { + NestLoop *nl = (NestLoop *) ps->plan; + + foreach(lc2, nl->nestParams) + { + NestLoopParam *nlp = (NestLoopParam *) lfirst(lc2); + + if (nlp->paramno == param->paramid) + { + /* Found a match, so return it */ + *dpns_p = dpns; + *ancestor_cell_p = lc; + return (Node *) nlp->paramval; + } + } + } + + /* + * Check to see if we're crawling up from a subplan. + */ + foreach(lc2, ps->subPlan) + { + SubPlanState *sstate = (SubPlanState *) lfirst(lc2); + SubPlan *subplan = sstate->subplan; + ListCell *lc3; + ListCell *lc4; + + if (child_ps != sstate->planstate) + continue; + + /* Matched subplan, so check its arguments */ + forboth(lc3, subplan->parParam, lc4, subplan->args) + { + int paramid = lfirst_int(lc3); + Node *arg = (Node *) lfirst(lc4); + + if (paramid == param->paramid) + { + /* Found a match, so return it */ + *dpns_p = dpns; + *ancestor_cell_p = lc; + return arg; + } + } + + /* Keep looking, but we are emerging from a subplan. */ + in_same_plan_level = false; + break; + } + + /* + * Likewise check to see if we're emerging from an initplan. + * Initplans never have any parParams, so no need to search that + * list, but we need to know if we should reset + * in_same_plan_level. + */ + foreach(lc2, ps->initPlan) + { + SubPlanState *sstate = (SubPlanState *) lfirst(lc2); + + if (child_ps != sstate->planstate) + continue; + + /* No parameters to be had here. */ + Assert(sstate->subplan->parParam == NIL); + + /* Keep looking, but we are emerging from an initplan. */ + in_same_plan_level = false; + break; + } + + /* No luck, crawl up to next ancestor */ + child_ps = ps; + } + } + + /* No referent found */ + return NULL; +} + +/* + * Display a Param appropriately. + */ +static void +get_parameter(Param *param, deparse_context *context) +{ + Node *expr; + deparse_namespace *dpns; + ListCell *ancestor_cell; + + /* + * If it's a PARAM_EXEC parameter, try to locate the expression from which + * the parameter was computed. Note that failing to find a referent isn't + * an error, since the Param might well be a subplan output rather than an + * input. + */ + expr = find_param_referent(param, context, &dpns, &ancestor_cell); + if (expr) + { + /* Found a match, so print it */ + deparse_namespace save_dpns; + bool save_varprefix; + bool need_paren; + + /* Switch attention to the ancestor plan node */ + push_ancestor_plan(dpns, ancestor_cell, &save_dpns); + + /* + * Force prefixing of Vars, since they won't belong to the relation + * being scanned in the original plan node. + */ + save_varprefix = context->varprefix; + context->varprefix = true; + + /* + * A Param's expansion is typically a Var, Aggref, or upper-level + * Param, which wouldn't need extra parentheses. Otherwise, insert + * parens to ensure the expression looks atomic. + */ + need_paren = !(IsA(expr, Var) || + IsA(expr, Aggref) || + IsA(expr, Param)); + if (need_paren) + appendStringInfoChar(context->buf, '('); + + get_rule_expr(expr, context, false); + + if (need_paren) + appendStringInfoChar(context->buf, ')'); + + context->varprefix = save_varprefix; + + pop_ancestor_plan(dpns, &save_dpns); + + return; + } + + /* + * Not PARAM_EXEC, or couldn't find referent: for base types just print $N. + * For composite types, add cast to the parameter to ease remote node detect + * the type. + */ + if (param->paramtype >= FirstNormalObjectId) + { + char *typeName = format_type_with_typemod(param->paramtype, param->paramtypmod); + + appendStringInfo(context->buf, "$%d::%s", param->paramid, typeName); + } + else + { + appendStringInfo(context->buf, "$%d", param->paramid); + } +} + +/* + * get_simple_binary_op_name + * + * helper function for isSimpleNode + * will return single char binary operator name, or NULL if it's not + */ +static const char * +get_simple_binary_op_name(OpExpr *expr) +{ + List *args = expr->args; + + if (list_length(args) == 2) + { + /* binary operator */ + Node *arg1 = (Node *) linitial(args); + Node *arg2 = (Node *) lsecond(args); + const char *op; + + op = generate_operator_name(expr->opno, exprType(arg1), exprType(arg2)); + if (strlen(op) == 1) + return op; + } + return NULL; +} + + +/* + * isSimpleNode - check if given node is simple (doesn't need parenthesizing) + * + * true : simple in the context of parent node's type + * false : not simple + */ +static bool +isSimpleNode(Node *node, Node *parentNode, int prettyFlags) +{ + if (!node) + return false; + + switch (nodeTag(node)) + { + case T_Var: + case T_Const: + case T_Param: + case T_CoerceToDomainValue: + case T_SetToDefault: + case T_CurrentOfExpr: + /* single words: always simple */ + return true; + + case T_SubscriptingRef: + case T_ArrayExpr: + case T_RowExpr: + case T_CoalesceExpr: + case T_MinMaxExpr: + case T_SQLValueFunction: + case T_XmlExpr: + case T_NextValueExpr: + case T_NullIfExpr: + case T_Aggref: + case T_WindowFunc: + case T_FuncExpr: + /* function-like: name(..) or name[..] */ + return true; + + /* CASE keywords act as parentheses */ + case T_CaseExpr: + return true; + + case T_FieldSelect: + + /* + * appears simple since . has top precedence, unless parent is + * T_FieldSelect itself! + */ + return (IsA(parentNode, FieldSelect) ? false : true); + + case T_FieldStore: + + /* + * treat like FieldSelect (probably doesn't matter) + */ + return (IsA(parentNode, FieldStore) ? false : true); + + case T_CoerceToDomain: + /* maybe simple, check args */ + return isSimpleNode((Node *) ((CoerceToDomain *) node)->arg, + node, prettyFlags); + case T_RelabelType: + return isSimpleNode((Node *) ((RelabelType *) node)->arg, + node, prettyFlags); + case T_CoerceViaIO: + return isSimpleNode((Node *) ((CoerceViaIO *) node)->arg, + node, prettyFlags); + case T_ArrayCoerceExpr: + return isSimpleNode((Node *) ((ArrayCoerceExpr *) node)->arg, + node, prettyFlags); + case T_ConvertRowtypeExpr: + return isSimpleNode((Node *) ((ConvertRowtypeExpr *) node)->arg, + node, prettyFlags); + + case T_OpExpr: + { + /* depends on parent node type; needs further checking */ + if (prettyFlags & PRETTYFLAG_PAREN && IsA(parentNode, OpExpr)) + { + const char *op; + const char *parentOp; + bool is_lopriop; + bool is_hipriop; + bool is_lopriparent; + bool is_hipriparent; + + op = get_simple_binary_op_name((OpExpr *) node); + if (!op) + return false; + + /* We know only the basic operators + - and * / % */ + is_lopriop = (strchr("+-", *op) != NULL); + is_hipriop = (strchr("*/%", *op) != NULL); + if (!(is_lopriop || is_hipriop)) + return false; + + parentOp = get_simple_binary_op_name((OpExpr *) parentNode); + if (!parentOp) + return false; + + is_lopriparent = (strchr("+-", *parentOp) != NULL); + is_hipriparent = (strchr("*/%", *parentOp) != NULL); + if (!(is_lopriparent || is_hipriparent)) + return false; + + if (is_hipriop && is_lopriparent) + return true; /* op binds tighter than parent */ + + if (is_lopriop && is_hipriparent) + return false; + + /* + * Operators are same priority --- can skip parens only if + * we have (a - b) - c, not a - (b - c). + */ + if (node == (Node *) linitial(((OpExpr *) parentNode)->args)) + return true; + + return false; + } + /* else do the same stuff as for T_SubLink et al. */ + } + /* FALLTHROUGH */ + + case T_SubLink: + case T_NullTest: + case T_BooleanTest: + case T_DistinctExpr: + switch (nodeTag(parentNode)) + { + case T_FuncExpr: + { + /* special handling for casts */ + CoercionForm type = ((FuncExpr *) parentNode)->funcformat; + + if (type == COERCE_EXPLICIT_CAST || + type == COERCE_IMPLICIT_CAST) + return false; + return true; /* own parentheses */ + } + case T_BoolExpr: /* lower precedence */ + case T_SubscriptingRef: /* other separators */ + case T_ArrayExpr: /* other separators */ + case T_RowExpr: /* other separators */ + case T_CoalesceExpr: /* own parentheses */ + case T_MinMaxExpr: /* own parentheses */ + case T_XmlExpr: /* own parentheses */ + case T_NullIfExpr: /* other separators */ + case T_Aggref: /* own parentheses */ + case T_WindowFunc: /* own parentheses */ + case T_CaseExpr: /* other separators */ + return true; + default: + return false; + } + + case T_BoolExpr: + switch (nodeTag(parentNode)) + { + case T_BoolExpr: + if (prettyFlags & PRETTYFLAG_PAREN) + { + BoolExprType type; + BoolExprType parentType; + + type = ((BoolExpr *) node)->boolop; + parentType = ((BoolExpr *) parentNode)->boolop; + switch (type) + { + case NOT_EXPR: + case AND_EXPR: + if (parentType == AND_EXPR || parentType == OR_EXPR) + return true; + break; + case OR_EXPR: + if (parentType == OR_EXPR) + return true; + break; + } + } + return false; + case T_FuncExpr: + { + /* special handling for casts */ + CoercionForm type = ((FuncExpr *) parentNode)->funcformat; + + if (type == COERCE_EXPLICIT_CAST || + type == COERCE_IMPLICIT_CAST) + return false; + return true; /* own parentheses */ + } + case T_SubscriptingRef: /* other separators */ + case T_ArrayExpr: /* other separators */ + case T_RowExpr: /* other separators */ + case T_CoalesceExpr: /* own parentheses */ + case T_MinMaxExpr: /* own parentheses */ + case T_XmlExpr: /* own parentheses */ + case T_NullIfExpr: /* other separators */ + case T_Aggref: /* own parentheses */ + case T_WindowFunc: /* own parentheses */ + case T_CaseExpr: /* other separators */ + return true; + default: + return false; + } + + default: + break; + } + /* those we don't know: in dubio complexo */ + return false; +} + + +/* + * appendContextKeyword - append a keyword to buffer + * + * If prettyPrint is enabled, perform a line break, and adjust indentation. + * Otherwise, just append the keyword. + */ +static void +appendContextKeyword(deparse_context *context, const char *str, + int indentBefore, int indentAfter, int indentPlus) +{ + StringInfo buf = context->buf; + + if (PRETTY_INDENT(context)) + { + int indentAmount; + + context->indentLevel += indentBefore; + + /* remove any trailing spaces currently in the buffer ... */ + removeStringInfoSpaces(buf); + /* ... then add a newline and some spaces */ + appendStringInfoChar(buf, '\n'); + + if (context->indentLevel < PRETTYINDENT_LIMIT) + indentAmount = Max(context->indentLevel, 0) + indentPlus; + else + { + /* + * If we're indented more than PRETTYINDENT_LIMIT characters, try + * to conserve horizontal space by reducing the per-level + * indentation. For best results the scale factor here should + * divide all the indent amounts that get added to indentLevel + * (PRETTYINDENT_STD, etc). It's important that the indentation + * not grow unboundedly, else deeply-nested trees use O(N^2) + * whitespace; so we also wrap modulo PRETTYINDENT_LIMIT. + */ + indentAmount = PRETTYINDENT_LIMIT + + (context->indentLevel - PRETTYINDENT_LIMIT) / + (PRETTYINDENT_STD / 2); + indentAmount %= PRETTYINDENT_LIMIT; + /* scale/wrap logic affects indentLevel, but not indentPlus */ + indentAmount += indentPlus; + } + appendStringInfoSpaces(buf, indentAmount); + + appendStringInfoString(buf, str); + + context->indentLevel += indentAfter; + if (context->indentLevel < 0) + context->indentLevel = 0; + } + else + appendStringInfoString(buf, str); +} + +/* + * removeStringInfoSpaces - delete trailing spaces from a buffer. + * + * Possibly this should move to stringinfo.c at some point. + */ +static void +removeStringInfoSpaces(StringInfo str) +{ + while (str->len > 0 && str->data[str->len - 1] == ' ') + str->data[--(str->len)] = '\0'; +} + + +/* + * get_rule_expr_paren - deparse expr using get_rule_expr, + * embracing the string with parentheses if necessary for prettyPrint. + * + * Never embrace if prettyFlags=0, because it's done in the calling node. + * + * Any node that does *not* embrace its argument node by sql syntax (with + * parentheses, non-operator keywords like CASE/WHEN/ON, or comma etc) should + * use get_rule_expr_paren instead of get_rule_expr so parentheses can be + * added. + */ +static void +get_rule_expr_paren(Node *node, deparse_context *context, + bool showimplicit, Node *parentNode) +{ + bool need_paren; + + need_paren = PRETTY_PAREN(context) && + !isSimpleNode(node, parentNode, context->prettyFlags); + + if (need_paren) + appendStringInfoChar(context->buf, '('); + + get_rule_expr(node, context, showimplicit); + + if (need_paren) + appendStringInfoChar(context->buf, ')'); +} + + +/* ---------- + * get_rule_expr - Parse back an expression + * + * Note: showimplicit determines whether we display any implicit cast that + * is present at the top of the expression tree. It is a passed argument, + * not a field of the context struct, because we change the value as we + * recurse down into the expression. In general we suppress implicit casts + * when the result type is known with certainty (eg, the arguments of an + * OR must be boolean). We display implicit casts for arguments of functions + * and operators, since this is needed to be certain that the same function + * or operator will be chosen when the expression is re-parsed. + * ---------- + */ +static void +get_rule_expr(Node *node, deparse_context *context, + bool showimplicit) +{ + StringInfo buf = context->buf; + + if (node == NULL) + return; + + /* Guard against excessively long or deeply-nested queries */ + CHECK_FOR_INTERRUPTS(); + check_stack_depth(); + + /* + * Each level of get_rule_expr must emit an indivisible term + * (parenthesized if necessary) to ensure result is reparsed into the same + * expression tree. The only exception is that when the input is a List, + * we emit the component items comma-separated with no surrounding + * decoration; this is convenient for most callers. + */ + switch (nodeTag(node)) + { + case T_Var: + (void) get_variable((Var *) node, 0, false, context); + break; + + case T_Const: + get_const_expr((Const *) node, context, 0); + break; + + case T_Param: + get_parameter((Param *) node, context); + break; + + case T_Aggref: + get_agg_expr((Aggref *) node, context, (Aggref *) node); + break; + + case T_GroupingFunc: + { + GroupingFunc *gexpr = (GroupingFunc *) node; + + appendStringInfoString(buf, "GROUPING("); + get_rule_expr((Node *) gexpr->args, context, true); + appendStringInfoChar(buf, ')'); + } + break; + + case T_WindowFunc: + get_windowfunc_expr((WindowFunc *) node, context); + break; + + case T_SubscriptingRef: + { + SubscriptingRef *sbsref = (SubscriptingRef *) node; + bool need_parens; + + /* + * If the argument is a CaseTestExpr, we must be inside a + * FieldStore, ie, we are assigning to an element of an array + * within a composite column. Since we already punted on + * displaying the FieldStore's target information, just punt + * here too, and display only the assignment source + * expression. + */ + if (IsA(sbsref->refexpr, CaseTestExpr)) + { + Assert(sbsref->refassgnexpr); + get_rule_expr((Node *) sbsref->refassgnexpr, + context, showimplicit); + break; + } + + /* + * Parenthesize the argument unless it's a simple Var or a + * FieldSelect. (In particular, if it's another + * SubscriptingRef, we *must* parenthesize to avoid + * confusion.) + */ + need_parens = !IsA(sbsref->refexpr, Var) && + !IsA(sbsref->refexpr, FieldSelect); + if (need_parens) + appendStringInfoChar(buf, '('); + get_rule_expr((Node *) sbsref->refexpr, context, showimplicit); + if (need_parens) + appendStringInfoChar(buf, ')'); + + /* + * If there's a refassgnexpr, we want to print the node in the + * format "container[subscripts] := refassgnexpr". This is + * not legal SQL, so decompilation of INSERT or UPDATE + * statements should always use processIndirection as part of + * the statement-level syntax. We should only see this when + * EXPLAIN tries to print the targetlist of a plan resulting + * from such a statement. + */ + if (sbsref->refassgnexpr) + { + Node *refassgnexpr; + + /* + * Use processIndirection to print this node's subscripts + * as well as any additional field selections or + * subscripting in immediate descendants. It returns the + * RHS expr that is actually being "assigned". + */ + refassgnexpr = processIndirection(node, context); + appendStringInfoString(buf, " := "); + get_rule_expr(refassgnexpr, context, showimplicit); + } + else + { + /* Just an ordinary container fetch, so print subscripts */ + printSubscripts(sbsref, context); + } + } + break; + + case T_FuncExpr: + get_func_expr((FuncExpr *) node, context, showimplicit); + break; + + case T_NamedArgExpr: + { + NamedArgExpr *na = (NamedArgExpr *) node; + + appendStringInfo(buf, "%s => ", quote_identifier(na->name)); + get_rule_expr((Node *) na->arg, context, showimplicit); + } + break; + + case T_OpExpr: + get_oper_expr((OpExpr *) node, context); + break; + + case T_DistinctExpr: + { + DistinctExpr *expr = (DistinctExpr *) node; + List *args = expr->args; + Node *arg1 = (Node *) linitial(args); + Node *arg2 = (Node *) lsecond(args); + + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren(arg1, context, true, node); + appendStringInfoString(buf, " IS DISTINCT FROM "); + get_rule_expr_paren(arg2, context, true, node); + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + } + break; + + case T_NullIfExpr: + { + NullIfExpr *nullifexpr = (NullIfExpr *) node; + + appendStringInfoString(buf, "NULLIF("); + get_rule_expr((Node *) nullifexpr->args, context, true); + appendStringInfoChar(buf, ')'); + } + break; + + case T_ScalarArrayOpExpr: + { + ScalarArrayOpExpr *expr = (ScalarArrayOpExpr *) node; + List *args = expr->args; + Node *arg1 = (Node *) linitial(args); + Node *arg2 = (Node *) lsecond(args); + + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren(arg1, context, true, node); + appendStringInfo(buf, " %s %s (", + generate_operator_name(expr->opno, + exprType(arg1), + get_base_element_type(exprType(arg2))), + expr->useOr ? "ANY" : "ALL"); + get_rule_expr_paren(arg2, context, true, node); + + /* + * There's inherent ambiguity in "x op ANY/ALL (y)" when y is + * a bare sub-SELECT. Since we're here, the sub-SELECT must + * be meant as a scalar sub-SELECT yielding an array value to + * be used in ScalarArrayOpExpr; but the grammar will + * preferentially interpret such a construct as an ANY/ALL + * SubLink. To prevent misparsing the output that way, insert + * a dummy coercion (which will be stripped by parse analysis, + * so no inefficiency is added in dump and reload). This is + * indeed most likely what the user wrote to get the construct + * accepted in the first place. + */ + if (IsA(arg2, SubLink) && + ((SubLink *) arg2)->subLinkType == EXPR_SUBLINK) + appendStringInfo(buf, "::%s", + format_type_with_typemod(exprType(arg2), + exprTypmod(arg2))); + appendStringInfoChar(buf, ')'); + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + } + break; + + case T_BoolExpr: + { + BoolExpr *expr = (BoolExpr *) node; + Node *first_arg = linitial(expr->args); + ListCell *arg = list_second_cell(expr->args); + + switch (expr->boolop) + { + case AND_EXPR: + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren(first_arg, context, + false, node); + while (arg) + { + appendStringInfoString(buf, " AND "); + get_rule_expr_paren((Node *) lfirst(arg), context, + false, node); + arg = lnext(expr->args, arg); + } + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + break; + + case OR_EXPR: + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren(first_arg, context, + false, node); + while (arg) + { + appendStringInfoString(buf, " OR "); + get_rule_expr_paren((Node *) lfirst(arg), context, + false, node); + arg = lnext(expr->args, arg); + } + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + break; + + case NOT_EXPR: + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + appendStringInfoString(buf, "NOT "); + get_rule_expr_paren(first_arg, context, + false, node); + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + break; + + default: + elog(ERROR, "unrecognized boolop: %d", + (int) expr->boolop); + } + } + break; + + case T_SubLink: + get_sublink_expr((SubLink *) node, context); + break; + + case T_SubPlan: + { + SubPlan *subplan = (SubPlan *) node; + + /* + * We cannot see an already-planned subplan in rule deparsing, + * only while EXPLAINing a query plan. We don't try to + * reconstruct the original SQL, just reference the subplan + * that appears elsewhere in EXPLAIN's result. + */ + if (subplan->useHashTable) + appendStringInfo(buf, "(hashed %s)", subplan->plan_name); + else + appendStringInfo(buf, "(%s)", subplan->plan_name); + } + break; + + case T_AlternativeSubPlan: + { + AlternativeSubPlan *asplan = (AlternativeSubPlan *) node; + ListCell *lc; + + /* As above, this can only happen during EXPLAIN */ + appendStringInfoString(buf, "(alternatives: "); + foreach(lc, asplan->subplans) + { + SubPlan *splan = lfirst_node(SubPlan, lc); + + if (splan->useHashTable) + appendStringInfo(buf, "hashed %s", splan->plan_name); + else + appendStringInfoString(buf, splan->plan_name); + if (lnext(asplan->subplans, lc)) + appendStringInfoString(buf, " or "); + } + appendStringInfoChar(buf, ')'); + } + break; + + case T_FieldSelect: + { + FieldSelect *fselect = (FieldSelect *) node; + Node *arg = (Node *) fselect->arg; + int fno = fselect->fieldnum; + const char *fieldname; + bool need_parens; + + /* + * Parenthesize the argument unless it's an SubscriptingRef or + * another FieldSelect. Note in particular that it would be + * WRONG to not parenthesize a Var argument; simplicity is not + * the issue here, having the right number of names is. + */ + need_parens = !IsA(arg, SubscriptingRef) && + !IsA(arg, FieldSelect); + if (need_parens) + appendStringInfoChar(buf, '('); + get_rule_expr(arg, context, true); + if (need_parens) + appendStringInfoChar(buf, ')'); + + /* + * Get and print the field name. + */ + fieldname = get_name_for_var_field((Var *) arg, fno, + 0, context); + appendStringInfo(buf, ".%s", quote_identifier(fieldname)); + } + break; + + case T_FieldStore: + { + FieldStore *fstore = (FieldStore *) node; + bool need_parens; + + /* + * There is no good way to represent a FieldStore as real SQL, + * so decompilation of INSERT or UPDATE statements should + * always use processIndirection as part of the + * statement-level syntax. We should only get here when + * EXPLAIN tries to print the targetlist of a plan resulting + * from such a statement. The plan case is even harder than + * ordinary rules would be, because the planner tries to + * collapse multiple assignments to the same field or subfield + * into one FieldStore; so we can see a list of target fields + * not just one, and the arguments could be FieldStores + * themselves. We don't bother to try to print the target + * field names; we just print the source arguments, with a + * ROW() around them if there's more than one. This isn't + * terribly complete, but it's probably good enough for + * EXPLAIN's purposes; especially since anything more would be + * either hopelessly confusing or an even poorer + * representation of what the plan is actually doing. + */ + need_parens = (list_length(fstore->newvals) != 1); + if (need_parens) + appendStringInfoString(buf, "ROW("); + get_rule_expr((Node *) fstore->newvals, context, showimplicit); + if (need_parens) + appendStringInfoChar(buf, ')'); + } + break; + + case T_RelabelType: + { + RelabelType *relabel = (RelabelType *) node; + Node *arg = (Node *) relabel->arg; + + if (relabel->relabelformat == COERCE_IMPLICIT_CAST && + !showimplicit) + { + /* don't show the implicit cast */ + get_rule_expr_paren(arg, context, false, node); + } + else + { + get_coercion_expr(arg, context, + relabel->resulttype, + relabel->resulttypmod, + node); + } + } + break; + + case T_CoerceViaIO: + { + CoerceViaIO *iocoerce = (CoerceViaIO *) node; + Node *arg = (Node *) iocoerce->arg; + + if (iocoerce->coerceformat == COERCE_IMPLICIT_CAST && + !showimplicit) + { + /* don't show the implicit cast */ + get_rule_expr_paren(arg, context, false, node); + } + else + { + get_coercion_expr(arg, context, + iocoerce->resulttype, + -1, + node); + } + } + break; + + case T_ArrayCoerceExpr: + { + ArrayCoerceExpr *acoerce = (ArrayCoerceExpr *) node; + Node *arg = (Node *) acoerce->arg; + + if (acoerce->coerceformat == COERCE_IMPLICIT_CAST && + !showimplicit) + { + /* don't show the implicit cast */ + get_rule_expr_paren(arg, context, false, node); + } + else + { + get_coercion_expr(arg, context, + acoerce->resulttype, + acoerce->resulttypmod, + node); + } + } + break; + + case T_ConvertRowtypeExpr: + { + ConvertRowtypeExpr *convert = (ConvertRowtypeExpr *) node; + Node *arg = (Node *) convert->arg; + + if (convert->convertformat == COERCE_IMPLICIT_CAST && + !showimplicit) + { + /* don't show the implicit cast */ + get_rule_expr_paren(arg, context, false, node); + } + else + { + get_coercion_expr(arg, context, + convert->resulttype, -1, + node); + } + } + break; + + case T_CollateExpr: + { + CollateExpr *collate = (CollateExpr *) node; + Node *arg = (Node *) collate->arg; + + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren(arg, context, showimplicit, node); + appendStringInfo(buf, " COLLATE %s", + generate_collation_name(collate->collOid)); + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + } + break; + + case T_CaseExpr: + { + CaseExpr *caseexpr = (CaseExpr *) node; + ListCell *temp; + + appendContextKeyword(context, "CASE", + 0, PRETTYINDENT_VAR, 0); + if (caseexpr->arg) + { + appendStringInfoChar(buf, ' '); + get_rule_expr((Node *) caseexpr->arg, context, true); + } + foreach(temp, caseexpr->args) + { + CaseWhen *when = (CaseWhen *) lfirst(temp); + Node *w = (Node *) when->expr; + + if (caseexpr->arg) + { + /* + * The parser should have produced WHEN clauses of the + * form "CaseTestExpr = RHS", possibly with an + * implicit coercion inserted above the CaseTestExpr. + * For accurate decompilation of rules it's essential + * that we show just the RHS. However in an + * expression that's been through the optimizer, the + * WHEN clause could be almost anything (since the + * equality operator could have been expanded into an + * inline function). If we don't recognize the form + * of the WHEN clause, just punt and display it as-is. + */ + if (IsA(w, OpExpr)) + { + List *args = ((OpExpr *) w)->args; + + if (list_length(args) == 2 && + IsA(strip_implicit_coercions(linitial(args)), + CaseTestExpr)) + w = (Node *) lsecond(args); + } + } + + if (!PRETTY_INDENT(context)) + appendStringInfoChar(buf, ' '); + appendContextKeyword(context, "WHEN ", + 0, 0, 0); + get_rule_expr(w, context, false); + appendStringInfoString(buf, " THEN "); + get_rule_expr((Node *) when->result, context, true); + } + if (!PRETTY_INDENT(context)) + appendStringInfoChar(buf, ' '); + appendContextKeyword(context, "ELSE ", + 0, 0, 0); + get_rule_expr((Node *) caseexpr->defresult, context, true); + if (!PRETTY_INDENT(context)) + appendStringInfoChar(buf, ' '); + appendContextKeyword(context, "END", + -PRETTYINDENT_VAR, 0, 0); + } + break; + + case T_CaseTestExpr: + { + /* + * Normally we should never get here, since for expressions + * that can contain this node type we attempt to avoid + * recursing to it. But in an optimized expression we might + * be unable to avoid that (see comments for CaseExpr). If we + * do see one, print it as CASE_TEST_EXPR. + */ + appendStringInfoString(buf, "CASE_TEST_EXPR"); + } + break; + + case T_ArrayExpr: + { + ArrayExpr *arrayexpr = (ArrayExpr *) node; + + appendStringInfoString(buf, "ARRAY["); + get_rule_expr((Node *) arrayexpr->elements, context, true); + appendStringInfoChar(buf, ']'); + + /* + * If the array isn't empty, we assume its elements are + * coerced to the desired type. If it's empty, though, we + * need an explicit coercion to the array type. + */ + if (arrayexpr->elements == NIL) + appendStringInfo(buf, "::%s", + format_type_with_typemod(arrayexpr->array_typeid, -1)); + } + break; + + case T_RowExpr: + { + RowExpr *rowexpr = (RowExpr *) node; + TupleDesc tupdesc = NULL; + ListCell *arg; + int i; + char *sep; + + /* + * If it's a named type and not RECORD, we may have to skip + * dropped columns and/or claim there are NULLs for added + * columns. + */ + if (rowexpr->row_typeid != RECORDOID) + { + tupdesc = lookup_rowtype_tupdesc(rowexpr->row_typeid, -1); + Assert(list_length(rowexpr->args) <= tupdesc->natts); + } + + /* + * SQL99 allows "ROW" to be omitted when there is more than + * one column, but for simplicity we always print it. + */ + appendStringInfoString(buf, "ROW("); + sep = ""; + i = 0; + foreach(arg, rowexpr->args) + { + Node *e = (Node *) lfirst(arg); + + if (tupdesc == NULL || + !TupleDescAttr(tupdesc, i)->attisdropped) + { + appendStringInfoString(buf, sep); + /* Whole-row Vars need special treatment here */ + get_rule_expr_toplevel(e, context, true); + sep = ", "; + } + i++; + } + if (tupdesc != NULL) + { + while (i < tupdesc->natts) + { + if (!TupleDescAttr(tupdesc, i)->attisdropped) + { + appendStringInfoString(buf, sep); + appendStringInfoString(buf, "NULL"); + sep = ", "; + } + i++; + } + + ReleaseTupleDesc(tupdesc); + } + appendStringInfoChar(buf, ')'); + if (rowexpr->row_format == COERCE_EXPLICIT_CAST) + appendStringInfo(buf, "::%s", + format_type_with_typemod(rowexpr->row_typeid, -1)); + } + break; + + case T_RowCompareExpr: + { + RowCompareExpr *rcexpr = (RowCompareExpr *) node; + ListCell *arg; + char *sep; + + /* + * SQL99 allows "ROW" to be omitted when there is more than + * one column, but for simplicity we always print it. + */ + appendStringInfoString(buf, "(ROW("); + sep = ""; + foreach(arg, rcexpr->largs) + { + Node *e = (Node *) lfirst(arg); + + appendStringInfoString(buf, sep); + get_rule_expr(e, context, true); + sep = ", "; + } + + /* + * We assume that the name of the first-column operator will + * do for all the rest too. This is definitely open to + * failure, eg if some but not all operators were renamed + * since the construct was parsed, but there seems no way to + * be perfect. + */ + appendStringInfo(buf, ") %s ROW(", + generate_operator_name(linitial_oid(rcexpr->opnos), + exprType(linitial(rcexpr->largs)), + exprType(linitial(rcexpr->rargs)))); + sep = ""; + foreach(arg, rcexpr->rargs) + { + Node *e = (Node *) lfirst(arg); + + appendStringInfoString(buf, sep); + get_rule_expr(e, context, true); + sep = ", "; + } + appendStringInfoString(buf, "))"); + } + break; + + case T_CoalesceExpr: + { + CoalesceExpr *coalesceexpr = (CoalesceExpr *) node; + + appendStringInfoString(buf, "COALESCE("); + get_rule_expr((Node *) coalesceexpr->args, context, true); + appendStringInfoChar(buf, ')'); + } + break; + + case T_MinMaxExpr: + { + MinMaxExpr *minmaxexpr = (MinMaxExpr *) node; + + switch (minmaxexpr->op) + { + case IS_GREATEST: + appendStringInfoString(buf, "GREATEST("); + break; + case IS_LEAST: + appendStringInfoString(buf, "LEAST("); + break; + } + get_rule_expr((Node *) minmaxexpr->args, context, true); + appendStringInfoChar(buf, ')'); + } + break; + + case T_SQLValueFunction: + { + SQLValueFunction *svf = (SQLValueFunction *) node; + + /* + * Note: this code knows that typmod for time, timestamp, and + * timestamptz just prints as integer. + */ + switch (svf->op) + { + case SVFOP_CURRENT_DATE: + appendStringInfoString(buf, "CURRENT_DATE"); + break; + case SVFOP_CURRENT_TIME: + appendStringInfoString(buf, "CURRENT_TIME"); + break; + case SVFOP_CURRENT_TIME_N: + appendStringInfo(buf, "CURRENT_TIME(%d)", svf->typmod); + break; + case SVFOP_CURRENT_TIMESTAMP: + appendStringInfoString(buf, "CURRENT_TIMESTAMP"); + break; + case SVFOP_CURRENT_TIMESTAMP_N: + appendStringInfo(buf, "CURRENT_TIMESTAMP(%d)", + svf->typmod); + break; + case SVFOP_LOCALTIME: + appendStringInfoString(buf, "LOCALTIME"); + break; + case SVFOP_LOCALTIME_N: + appendStringInfo(buf, "LOCALTIME(%d)", svf->typmod); + break; + case SVFOP_LOCALTIMESTAMP: + appendStringInfoString(buf, "LOCALTIMESTAMP"); + break; + case SVFOP_LOCALTIMESTAMP_N: + appendStringInfo(buf, "LOCALTIMESTAMP(%d)", + svf->typmod); + break; + case SVFOP_CURRENT_ROLE: + appendStringInfoString(buf, "CURRENT_ROLE"); + break; + case SVFOP_CURRENT_USER: + appendStringInfoString(buf, "CURRENT_USER"); + break; + case SVFOP_USER: + appendStringInfoString(buf, "USER"); + break; + case SVFOP_SESSION_USER: + appendStringInfoString(buf, "SESSION_USER"); + break; + case SVFOP_CURRENT_CATALOG: + appendStringInfoString(buf, "CURRENT_CATALOG"); + break; + case SVFOP_CURRENT_SCHEMA: + appendStringInfoString(buf, "CURRENT_SCHEMA"); + break; + } + } + break; + + case T_XmlExpr: + { + XmlExpr *xexpr = (XmlExpr *) node; + bool needcomma = false; + ListCell *arg; + ListCell *narg; + Const *con; + + switch (xexpr->op) + { + case IS_XMLCONCAT: + appendStringInfoString(buf, "XMLCONCAT("); + break; + case IS_XMLELEMENT: + appendStringInfoString(buf, "XMLELEMENT("); + break; + case IS_XMLFOREST: + appendStringInfoString(buf, "XMLFOREST("); + break; + case IS_XMLPARSE: + appendStringInfoString(buf, "XMLPARSE("); + break; + case IS_XMLPI: + appendStringInfoString(buf, "XMLPI("); + break; + case IS_XMLROOT: + appendStringInfoString(buf, "XMLROOT("); + break; + case IS_XMLSERIALIZE: + appendStringInfoString(buf, "XMLSERIALIZE("); + break; + case IS_DOCUMENT: + break; + } + if (xexpr->op == IS_XMLPARSE || xexpr->op == IS_XMLSERIALIZE) + { + if (xexpr->xmloption == XMLOPTION_DOCUMENT) + appendStringInfoString(buf, "DOCUMENT "); + else + appendStringInfoString(buf, "CONTENT "); + } + if (xexpr->name) + { + appendStringInfo(buf, "NAME %s", + quote_identifier(map_xml_name_to_sql_identifier(xexpr->name))); + needcomma = true; + } + if (xexpr->named_args) + { + if (xexpr->op != IS_XMLFOREST) + { + if (needcomma) + appendStringInfoString(buf, ", "); + appendStringInfoString(buf, "XMLATTRIBUTES("); + needcomma = false; + } + forboth(arg, xexpr->named_args, narg, xexpr->arg_names) + { + Node *e = (Node *) lfirst(arg); + char *argname = strVal(lfirst(narg)); + + if (needcomma) + appendStringInfoString(buf, ", "); + get_rule_expr((Node *) e, context, true); + appendStringInfo(buf, " AS %s", + quote_identifier(map_xml_name_to_sql_identifier(argname))); + needcomma = true; + } + if (xexpr->op != IS_XMLFOREST) + appendStringInfoChar(buf, ')'); + } + if (xexpr->args) + { + if (needcomma) + appendStringInfoString(buf, ", "); + switch (xexpr->op) + { + case IS_XMLCONCAT: + case IS_XMLELEMENT: + case IS_XMLFOREST: + case IS_XMLPI: + case IS_XMLSERIALIZE: + /* no extra decoration needed */ + get_rule_expr((Node *) xexpr->args, context, true); + break; + case IS_XMLPARSE: + Assert(list_length(xexpr->args) == 2); + + get_rule_expr((Node *) linitial(xexpr->args), + context, true); + + con = lsecond_node(Const, xexpr->args); + Assert(!con->constisnull); + if (DatumGetBool(con->constvalue)) + appendStringInfoString(buf, + " PRESERVE WHITESPACE"); + else + appendStringInfoString(buf, + " STRIP WHITESPACE"); + break; + case IS_XMLROOT: + Assert(list_length(xexpr->args) == 3); + + get_rule_expr((Node *) linitial(xexpr->args), + context, true); + + appendStringInfoString(buf, ", VERSION "); + con = (Const *) lsecond(xexpr->args); + if (IsA(con, Const) && + con->constisnull) + appendStringInfoString(buf, "NO VALUE"); + else + get_rule_expr((Node *) con, context, false); + + con = lthird_node(Const, xexpr->args); + if (con->constisnull) + /* suppress STANDALONE NO VALUE */ ; + else + { + switch (DatumGetInt32(con->constvalue)) + { + case XML_STANDALONE_YES: + appendStringInfoString(buf, + ", STANDALONE YES"); + break; + case XML_STANDALONE_NO: + appendStringInfoString(buf, + ", STANDALONE NO"); + break; + case XML_STANDALONE_NO_VALUE: + appendStringInfoString(buf, + ", STANDALONE NO VALUE"); + break; + default: + break; + } + } + break; + case IS_DOCUMENT: + get_rule_expr_paren((Node *) xexpr->args, context, false, node); + break; + } + + } + if (xexpr->op == IS_XMLSERIALIZE) + appendStringInfo(buf, " AS %s", + format_type_with_typemod(xexpr->type, + xexpr->typmod)); + if (xexpr->op == IS_DOCUMENT) + appendStringInfoString(buf, " IS DOCUMENT"); + else + appendStringInfoChar(buf, ')'); + } + break; + + case T_NullTest: + { + NullTest *ntest = (NullTest *) node; + + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren((Node *) ntest->arg, context, true, node); + + /* + * For scalar inputs, we prefer to print as IS [NOT] NULL, + * which is shorter and traditional. If it's a rowtype input + * but we're applying a scalar test, must print IS [NOT] + * DISTINCT FROM NULL to be semantically correct. + */ + if (ntest->argisrow || + !type_is_rowtype(exprType((Node *) ntest->arg))) + { + switch (ntest->nulltesttype) + { + case IS_NULL: + appendStringInfoString(buf, " IS NULL"); + break; + case IS_NOT_NULL: + appendStringInfoString(buf, " IS NOT NULL"); + break; + default: + elog(ERROR, "unrecognized nulltesttype: %d", + (int) ntest->nulltesttype); + } + } + else + { + switch (ntest->nulltesttype) + { + case IS_NULL: + appendStringInfoString(buf, " IS NOT DISTINCT FROM NULL"); + break; + case IS_NOT_NULL: + appendStringInfoString(buf, " IS DISTINCT FROM NULL"); + break; + default: + elog(ERROR, "unrecognized nulltesttype: %d", + (int) ntest->nulltesttype); + } + } + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + } + break; + + case T_BooleanTest: + { + BooleanTest *btest = (BooleanTest *) node; + + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren((Node *) btest->arg, context, false, node); + switch (btest->booltesttype) + { + case IS_TRUE: + appendStringInfoString(buf, " IS TRUE"); + break; + case IS_NOT_TRUE: + appendStringInfoString(buf, " IS NOT TRUE"); + break; + case IS_FALSE: + appendStringInfoString(buf, " IS FALSE"); + break; + case IS_NOT_FALSE: + appendStringInfoString(buf, " IS NOT FALSE"); + break; + case IS_UNKNOWN: + appendStringInfoString(buf, " IS UNKNOWN"); + break; + case IS_NOT_UNKNOWN: + appendStringInfoString(buf, " IS NOT UNKNOWN"); + break; + default: + elog(ERROR, "unrecognized booltesttype: %d", + (int) btest->booltesttype); + } + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + } + break; + + case T_CoerceToDomain: + { + CoerceToDomain *ctest = (CoerceToDomain *) node; + Node *arg = (Node *) ctest->arg; + + if (ctest->coercionformat == COERCE_IMPLICIT_CAST && + !showimplicit) + { + /* don't show the implicit cast */ + get_rule_expr(arg, context, false); + } + else + { + get_coercion_expr(arg, context, + ctest->resulttype, + ctest->resulttypmod, + node); + } + } + break; + + case T_CoerceToDomainValue: + appendStringInfoString(buf, "VALUE"); + break; + + case T_SetToDefault: + appendStringInfoString(buf, "DEFAULT"); + break; + + case T_CurrentOfExpr: + { + CurrentOfExpr *cexpr = (CurrentOfExpr *) node; + + if (cexpr->cursor_name) + appendStringInfo(buf, "CURRENT OF %s", + quote_identifier(cexpr->cursor_name)); + else + appendStringInfo(buf, "CURRENT OF $%d", + cexpr->cursor_param); + } + break; + + case T_NextValueExpr: + { + NextValueExpr *nvexpr = (NextValueExpr *) node; + + /* + * This isn't exactly nextval(), but that seems close enough + * for EXPLAIN's purposes. + */ + appendStringInfoString(buf, "nextval("); + simple_quote_literal(buf, + generate_relation_name(nvexpr->seqid, + NIL)); + appendStringInfoChar(buf, ')'); + } + break; + + case T_InferenceElem: + { + InferenceElem *iexpr = (InferenceElem *) node; + bool save_varprefix; + bool need_parens; + + /* + * InferenceElem can only refer to target relation, so a + * prefix is not useful, and indeed would cause parse errors. + */ + save_varprefix = context->varprefix; + context->varprefix = false; + + /* + * Parenthesize the element unless it's a simple Var or a bare + * function call. Follows pg_get_indexdef_worker(). + */ + need_parens = !IsA(iexpr->expr, Var); + if (IsA(iexpr->expr, FuncExpr) && + ((FuncExpr *) iexpr->expr)->funcformat == + COERCE_EXPLICIT_CALL) + need_parens = false; + + if (need_parens) + appendStringInfoChar(buf, '('); + get_rule_expr((Node *) iexpr->expr, + context, false); + if (need_parens) + appendStringInfoChar(buf, ')'); + + context->varprefix = save_varprefix; + + if (iexpr->infercollid) + appendStringInfo(buf, " COLLATE %s", + generate_collation_name(iexpr->infercollid)); + + /* Add the operator class name, if not default */ + if (iexpr->inferopclass) + { + Oid inferopclass = iexpr->inferopclass; + Oid inferopcinputtype = get_opclass_input_type(iexpr->inferopclass); + + get_opclass_name(inferopclass, inferopcinputtype, buf); + } + } + break; + + case T_PartitionBoundSpec: + { + PartitionBoundSpec *spec = (PartitionBoundSpec *) node; + ListCell *cell; + char *sep; + + if (spec->is_default) + { + appendStringInfoString(buf, "DEFAULT"); + break; + } + + switch (spec->strategy) + { + case PARTITION_STRATEGY_HASH: + Assert(spec->modulus > 0 && spec->remainder >= 0); + Assert(spec->modulus > spec->remainder); + + appendStringInfoString(buf, "FOR VALUES"); + appendStringInfo(buf, " WITH (modulus %d, remainder %d)", + spec->modulus, spec->remainder); + break; + + case PARTITION_STRATEGY_LIST: + Assert(spec->listdatums != NIL); + + appendStringInfoString(buf, "FOR VALUES IN ("); + sep = ""; + foreach(cell, spec->listdatums) + { + Const *val = castNode(Const, lfirst(cell)); + + appendStringInfoString(buf, sep); + get_const_expr(val, context, -1); + sep = ", "; + } + + appendStringInfoChar(buf, ')'); + break; + + case PARTITION_STRATEGY_RANGE: + Assert(spec->lowerdatums != NIL && + spec->upperdatums != NIL && + list_length(spec->lowerdatums) == + list_length(spec->upperdatums)); + + appendStringInfo(buf, "FOR VALUES FROM %s TO %s", + get_range_partbound_string(spec->lowerdatums), + get_range_partbound_string(spec->upperdatums)); + break; + + default: + elog(ERROR, "unrecognized partition strategy: %d", + (int) spec->strategy); + break; + } + } + break; + + case T_List: + { + char *sep; + ListCell *l; + + sep = ""; + foreach(l, (List *) node) + { + appendStringInfoString(buf, sep); + get_rule_expr((Node *) lfirst(l), context, showimplicit); + sep = ", "; + } + } + break; + + case T_TableFunc: + get_tablefunc((TableFunc *) node, context, showimplicit); + break; + + default: + elog(ERROR, "unrecognized node type: %d", (int) nodeTag(node)); + break; + } +} + +/* + * get_rule_expr_toplevel - Parse back a toplevel expression + * + * Same as get_rule_expr(), except that if the expr is just a Var, we pass + * istoplevel = true not false to get_variable(). This causes whole-row Vars + * to get printed with decoration that will prevent expansion of "*". + * We need to use this in contexts such as ROW() and VALUES(), where the + * parser would expand "foo.*" appearing at top level. (In principle we'd + * use this in get_target_list() too, but that has additional worries about + * whether to print AS, so it needs to invoke get_variable() directly anyway.) + */ +static void +get_rule_expr_toplevel(Node *node, deparse_context *context, + bool showimplicit) +{ + if (node && IsA(node, Var)) + (void) get_variable((Var *) node, 0, true, context); + else + get_rule_expr(node, context, showimplicit); +} + +/* + * get_rule_expr_funccall - Parse back a function-call expression + * + * Same as get_rule_expr(), except that we guarantee that the output will + * look like a function call, or like one of the things the grammar treats as + * equivalent to a function call (see the func_expr_windowless production). + * This is needed in places where the grammar uses func_expr_windowless and + * you can't substitute a parenthesized a_expr. If what we have isn't going + * to look like a function call, wrap it in a dummy CAST() expression, which + * will satisfy the grammar --- and, indeed, is likely what the user wrote to + * produce such a thing. + */ +static void +get_rule_expr_funccall(Node *node, deparse_context *context, + bool showimplicit) +{ + if (looks_like_function(node)) + get_rule_expr(node, context, showimplicit); + else + { + StringInfo buf = context->buf; + + appendStringInfoString(buf, "CAST("); + /* no point in showing any top-level implicit cast */ + get_rule_expr(node, context, false); + appendStringInfo(buf, " AS %s)", + format_type_with_typemod(exprType(node), + exprTypmod(node))); + } +} + +/* + * Helper function to identify node types that satisfy func_expr_windowless. + * If in doubt, "false" is always a safe answer. + */ +static bool +looks_like_function(Node *node) +{ + if (node == NULL) + return false; /* probably shouldn't happen */ + switch (nodeTag(node)) + { + case T_FuncExpr: + /* OK, unless it's going to deparse as a cast */ + return (((FuncExpr *) node)->funcformat == COERCE_EXPLICIT_CALL); + case T_NullIfExpr: + case T_CoalesceExpr: + case T_MinMaxExpr: + case T_SQLValueFunction: + case T_XmlExpr: + /* these are all accepted by func_expr_common_subexpr */ + return true; + default: + break; + } + return false; +} + + +/* + * get_oper_expr - Parse back an OpExpr node + */ +static void +get_oper_expr(OpExpr *expr, deparse_context *context) +{ + StringInfo buf = context->buf; + Oid opno = expr->opno; + List *args = expr->args; + + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + if (list_length(args) == 2) + { + /* binary operator */ + Node *arg1 = (Node *) linitial(args); + Node *arg2 = (Node *) lsecond(args); + + get_rule_expr_paren(arg1, context, true, (Node *) expr); + appendStringInfo(buf, " %s ", + generate_operator_name(opno, + exprType(arg1), + exprType(arg2))); + get_rule_expr_paren(arg2, context, true, (Node *) expr); + } + else + { + /* unary operator --- but which side? */ + Node *arg = (Node *) linitial(args); + HeapTuple tp; + Form_pg_operator optup; + + tp = SearchSysCache1(OPEROID, ObjectIdGetDatum(opno)); + if (!HeapTupleIsValid(tp)) + elog(ERROR, "cache lookup failed for operator %u", opno); + optup = (Form_pg_operator) GETSTRUCT(tp); + switch (optup->oprkind) + { + case 'l': + appendStringInfo(buf, "%s ", + generate_operator_name(opno, + InvalidOid, + exprType(arg))); + get_rule_expr_paren(arg, context, true, (Node *) expr); + break; + case 'r': + get_rule_expr_paren(arg, context, true, (Node *) expr); + appendStringInfo(buf, " %s", + generate_operator_name(opno, + exprType(arg), + InvalidOid)); + break; + default: + elog(ERROR, "bogus oprkind: %d", optup->oprkind); + } + ReleaseSysCache(tp); + } + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); +} + +/* + * get_func_expr - Parse back a FuncExpr node + */ +static void +get_func_expr(FuncExpr *expr, deparse_context *context, + bool showimplicit) +{ + StringInfo buf = context->buf; + Oid funcoid = expr->funcid; + Oid argtypes[FUNC_MAX_ARGS]; + int nargs; + List *argnames; + bool use_variadic; + ListCell *l; + + /* + * If the function call came from an implicit coercion, then just show the + * first argument --- unless caller wants to see implicit coercions. + */ + if (expr->funcformat == COERCE_IMPLICIT_CAST && !showimplicit) + { + get_rule_expr_paren((Node *) linitial(expr->args), context, + false, (Node *) expr); + return; + } + + /* + * If the function call came from a cast, then show the first argument + * plus an explicit cast operation. + */ + if (expr->funcformat == COERCE_EXPLICIT_CAST || + expr->funcformat == COERCE_IMPLICIT_CAST) + { + Node *arg = linitial(expr->args); + Oid rettype = expr->funcresulttype; + int32 coercedTypmod; + + /* Get the typmod if this is a length-coercion function */ + (void) exprIsLengthCoercion((Node *) expr, &coercedTypmod); + + get_coercion_expr(arg, context, + rettype, coercedTypmod, + (Node *) expr); + + return; + } + + /* + * Normal function: display as proname(args). First we need to extract + * the argument datatypes. + */ + if (list_length(expr->args) > FUNC_MAX_ARGS) + ereport(ERROR, + (errcode(ERRCODE_TOO_MANY_ARGUMENTS), + errmsg("too many arguments"))); + nargs = 0; + argnames = NIL; + foreach(l, expr->args) + { + Node *arg = (Node *) lfirst(l); + + if (IsA(arg, NamedArgExpr)) + argnames = lappend(argnames, ((NamedArgExpr *) arg)->name); + argtypes[nargs] = exprType(arg); + nargs++; + } + + appendStringInfo(buf, "%s(", + generate_function_name(funcoid, nargs, + argnames, argtypes, + expr->funcvariadic, + &use_variadic, + context->special_exprkind)); + nargs = 0; + foreach(l, expr->args) + { + if (nargs++ > 0) + appendStringInfoString(buf, ", "); + if (use_variadic && lnext(expr->args, l) == NULL) + appendStringInfoString(buf, "VARIADIC "); + get_rule_expr((Node *) lfirst(l), context, true); + } + appendStringInfoChar(buf, ')'); +} + +/* + * get_agg_expr - Parse back an Aggref node + */ +static void +get_agg_expr(Aggref *aggref, deparse_context *context, + Aggref *original_aggref) +{ + StringInfo buf = context->buf; + Oid argtypes[FUNC_MAX_ARGS]; + int nargs; + bool use_variadic; + + /* + * For a combining aggregate, we look up and deparse the corresponding + * partial aggregate instead. This is necessary because our input + * argument list has been replaced; the new argument list always has just + * one element, which will point to a partial Aggref that supplies us with + * transition states to combine. + */ + if (DO_AGGSPLIT_COMBINE(aggref->aggsplit)) + { + TargetEntry *tle = linitial_node(TargetEntry, aggref->args); + + Assert(list_length(aggref->args) == 1); + resolve_special_varno((Node *) tle->expr, context, original_aggref, + get_agg_combine_expr); + return; + } + + /* + * Mark as PARTIAL, if appropriate. We look to the original aggref so as + * to avoid printing this when recursing from the code just above. + */ + if (DO_AGGSPLIT_SKIPFINAL(original_aggref->aggsplit)) + appendStringInfoString(buf, "PARTIAL "); + + /* Extract the argument types as seen by the parser */ + nargs = get_aggregate_argtypes(aggref, argtypes); + + /* Print the aggregate name, schema-qualified if needed */ + appendStringInfo(buf, "%s(%s", + generate_function_name(aggref->aggfnoid, nargs, + NIL, argtypes, + aggref->aggvariadic, + &use_variadic, + context->special_exprkind), + (aggref->aggdistinct != NIL) ? "DISTINCT " : ""); + + if (AGGKIND_IS_ORDERED_SET(aggref->aggkind)) + { + /* + * Ordered-set aggregates do not use "*" syntax. Also, we needn't + * worry about inserting VARIADIC. So we can just dump the direct + * args as-is. + */ + Assert(!aggref->aggvariadic); + get_rule_expr((Node *) aggref->aggdirectargs, context, true); + Assert(aggref->aggorder != NIL); + appendStringInfoString(buf, ") WITHIN GROUP (ORDER BY "); + get_rule_orderby(aggref->aggorder, aggref->args, false, context); + } + else + { + /* aggstar can be set only in zero-argument aggregates */ + if (aggref->aggstar) + appendStringInfoChar(buf, '*'); + else + { + ListCell *l; + int i; + + i = 0; + foreach(l, aggref->args) + { + TargetEntry *tle = (TargetEntry *) lfirst(l); + Node *arg = (Node *) tle->expr; + + Assert(!IsA(arg, NamedArgExpr)); + if (tle->resjunk) + continue; + if (i++ > 0) + appendStringInfoString(buf, ", "); + if (use_variadic && i == nargs) + appendStringInfoString(buf, "VARIADIC "); + get_rule_expr(arg, context, true); + } + } + + if (aggref->aggorder != NIL) + { + appendStringInfoString(buf, " ORDER BY "); + get_rule_orderby(aggref->aggorder, aggref->args, false, context); + } + } + + if (aggref->aggfilter != NULL) + { + appendStringInfoString(buf, ") FILTER (WHERE "); + get_rule_expr((Node *) aggref->aggfilter, context, false); + } + + appendStringInfoChar(buf, ')'); +} + +/* + * This is a helper function for get_agg_expr(). It's used when we deparse + * a combining Aggref; resolve_special_varno locates the corresponding partial + * Aggref and then calls this. + */ +static void +get_agg_combine_expr(Node *node, deparse_context *context, void *private) +{ + Aggref *aggref; + Aggref *original_aggref = private; + + if (!IsA(node, Aggref)) + elog(ERROR, "combining Aggref does not point to an Aggref"); + + aggref = (Aggref *) node; + get_agg_expr(aggref, context, original_aggref); +} + +/* + * get_windowfunc_expr - Parse back a WindowFunc node + */ +static void +get_windowfunc_expr(WindowFunc *wfunc, deparse_context *context) +{ + StringInfo buf = context->buf; + Oid argtypes[FUNC_MAX_ARGS]; + int nargs; + List *argnames; + ListCell *l; + + if (list_length(wfunc->args) > FUNC_MAX_ARGS) + ereport(ERROR, + (errcode(ERRCODE_TOO_MANY_ARGUMENTS), + errmsg("too many arguments"))); + nargs = 0; + argnames = NIL; + foreach(l, wfunc->args) + { + Node *arg = (Node *) lfirst(l); + + if (IsA(arg, NamedArgExpr)) + argnames = lappend(argnames, ((NamedArgExpr *) arg)->name); + argtypes[nargs] = exprType(arg); + nargs++; + } + + appendStringInfo(buf, "%s(", + generate_function_name(wfunc->winfnoid, nargs, + argnames, argtypes, + false, NULL, + context->special_exprkind)); + /* winstar can be set only in zero-argument aggregates */ + if (wfunc->winstar) + appendStringInfoChar(buf, '*'); + else + get_rule_expr((Node *) wfunc->args, context, true); + + if (wfunc->aggfilter != NULL) + { + appendStringInfoString(buf, ") FILTER (WHERE "); + get_rule_expr((Node *) wfunc->aggfilter, context, false); + } + + appendStringInfoString(buf, ") OVER "); + + foreach(l, context->windowClause) + { + WindowClause *wc = (WindowClause *) lfirst(l); + + if (wc->winref == wfunc->winref) + { + if (wc->name) + appendStringInfoString(buf, quote_identifier(wc->name)); + else + get_rule_windowspec(wc, context->windowTList, context); + break; + } + } + if (l == NULL) + { + if (context->windowClause) + elog(ERROR, "could not find window clause for winref %u", + wfunc->winref); + + /* + * In EXPLAIN, we don't have window context information available, so + * we have to settle for this: + */ + appendStringInfoString(buf, "(?)"); + } +} + +/* ---------- + * get_coercion_expr + * + * Make a string representation of a value coerced to a specific type + * ---------- + */ +static void +get_coercion_expr(Node *arg, deparse_context *context, + Oid resulttype, int32 resulttypmod, + Node *parentNode) +{ + StringInfo buf = context->buf; + + /* + * Since parse_coerce.c doesn't immediately collapse application of + * length-coercion functions to constants, what we'll typically see in + * such cases is a Const with typmod -1 and a length-coercion function + * right above it. Avoid generating redundant output. However, beware of + * suppressing casts when the user actually wrote something like + * 'foo'::text::char(3). + * + * Note: it might seem that we are missing the possibility of needing to + * print a COLLATE clause for such a Const. However, a Const could only + * have nondefault collation in a post-constant-folding tree, in which the + * length coercion would have been folded too. See also the special + * handling of CollateExpr in coerce_to_target_type(): any collation + * marking will be above the coercion node, not below it. + */ + if (arg && IsA(arg, Const) && + ((Const *) arg)->consttype == resulttype && + ((Const *) arg)->consttypmod == -1) + { + /* Show the constant without normal ::typename decoration */ + get_const_expr((Const *) arg, context, -1); + } + else + { + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren(arg, context, false, parentNode); + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + } + appendStringInfo(buf, "::%s", + format_type_with_typemod(resulttype, resulttypmod)); +} + +/* ---------- + * get_const_expr + * + * Make a string representation of a Const + * + * showtype can be -1 to never show "::typename" decoration, or +1 to always + * show it, or 0 to show it only if the constant wouldn't be assumed to be + * the right type by default. + * + * If the Const's collation isn't default for its type, show that too. + * We mustn't do this when showtype is -1 (since that means the caller will + * print "::typename", and we can't put a COLLATE clause in between). It's + * caller's responsibility that collation isn't missed in such cases. + * ---------- + */ +static void +get_const_expr(Const *constval, deparse_context *context, int showtype) +{ + StringInfo buf = context->buf; + Oid typoutput; + bool typIsVarlena; + char *extval; + bool needlabel = false; + + if (constval->constisnull) + { + /* + * Always label the type of a NULL constant to prevent misdecisions + * about type when reparsing. + */ + appendStringInfoString(buf, "NULL"); + if (showtype >= 0) + { + appendStringInfo(buf, "::%s", + format_type_with_typemod(constval->consttype, + constval->consttypmod)); + get_const_collation(constval, context); + } + return; + } + + getTypeOutputInfo(constval->consttype, + &typoutput, &typIsVarlena); + + extval = OidOutputFunctionCall(typoutput, constval->constvalue); + + switch (constval->consttype) + { + case INT4OID: + + /* + * INT4 can be printed without any decoration, unless it is + * negative; in that case print it as '-nnn'::integer to ensure + * that the output will re-parse as a constant, not as a constant + * plus operator. In most cases we could get away with printing + * (-nnn) instead, because of the way that gram.y handles negative + * literals; but that doesn't work for INT_MIN, and it doesn't + * seem that much prettier anyway. + */ + if (extval[0] != '-') + appendStringInfoString(buf, extval); + else + { + appendStringInfo(buf, "'%s'", extval); + needlabel = true; /* we must attach a cast */ + } + break; + + case NUMERICOID: + + /* + * NUMERIC can be printed without quotes if it looks like a float + * constant (not an integer, and not Infinity or NaN) and doesn't + * have a leading sign (for the same reason as for INT4). + */ + if (isdigit((unsigned char) extval[0]) && + strcspn(extval, "eE.") != strlen(extval)) + { + appendStringInfoString(buf, extval); + } + else + { + appendStringInfo(buf, "'%s'", extval); + needlabel = true; /* we must attach a cast */ + } + break; + + case BITOID: + case VARBITOID: + appendStringInfo(buf, "B'%s'", extval); + break; + + case BOOLOID: + if (strcmp(extval, "t") == 0) + appendStringInfoString(buf, "true"); + else + appendStringInfoString(buf, "false"); + break; + + default: + simple_quote_literal(buf, extval); + break; + } + + pfree(extval); + + if (showtype < 0) + return; + + /* + * For showtype == 0, append ::typename unless the constant will be + * implicitly typed as the right type when it is read in. + * + * XXX this code has to be kept in sync with the behavior of the parser, + * especially make_const. + */ + switch (constval->consttype) + { + case BOOLOID: + case UNKNOWNOID: + /* These types can be left unlabeled */ + needlabel = false; + break; + case INT4OID: + /* We determined above whether a label is needed */ + break; + case NUMERICOID: + + /* + * Float-looking constants will be typed as numeric, which we + * checked above; but if there's a nondefault typmod we need to + * show it. + */ + needlabel |= (constval->consttypmod >= 0); + break; + default: + needlabel = true; + break; + } + if (needlabel || showtype > 0) + appendStringInfo(buf, "::%s", + format_type_with_typemod(constval->consttype, + constval->consttypmod)); + + get_const_collation(constval, context); +} + +/* + * helper for get_const_expr: append COLLATE if needed + */ +static void +get_const_collation(Const *constval, deparse_context *context) +{ + StringInfo buf = context->buf; + + if (OidIsValid(constval->constcollid)) + { + Oid typcollation = get_typcollation(constval->consttype); + + if (constval->constcollid != typcollation) + { + appendStringInfo(buf, " COLLATE %s", + generate_collation_name(constval->constcollid)); + } + } +} + +/* + * simple_quote_literal - Format a string as a SQL literal, append to buf + */ +static void +simple_quote_literal(StringInfo buf, const char *val) +{ + const char *valptr; + + /* + * We form the string literal according to the prevailing setting of + * standard_conforming_strings; we never use E''. User is responsible for + * making sure result is used correctly. + */ + appendStringInfoChar(buf, '\''); + for (valptr = val; *valptr; valptr++) + { + char ch = *valptr; + + if (SQL_STR_DOUBLE(ch, !standard_conforming_strings)) + appendStringInfoChar(buf, ch); + appendStringInfoChar(buf, ch); + } + appendStringInfoChar(buf, '\''); +} + + +/* ---------- + * get_sublink_expr - Parse back a sublink + * ---------- + */ +static void +get_sublink_expr(SubLink *sublink, deparse_context *context) +{ + StringInfo buf = context->buf; + Query *query = (Query *) (sublink->subselect); + char *opname = NULL; + bool need_paren; + + if (sublink->subLinkType == ARRAY_SUBLINK) + appendStringInfoString(buf, "ARRAY("); + else + appendStringInfoChar(buf, '('); + + /* + * Note that we print the name of only the first operator, when there are + * multiple combining operators. This is an approximation that could go + * wrong in various scenarios (operators in different schemas, renamed + * operators, etc) but there is not a whole lot we can do about it, since + * the syntax allows only one operator to be shown. + */ + if (sublink->testexpr) + { + if (IsA(sublink->testexpr, OpExpr)) + { + /* single combining operator */ + OpExpr *opexpr = (OpExpr *) sublink->testexpr; + + get_rule_expr(linitial(opexpr->args), context, true); + opname = generate_operator_name(opexpr->opno, + exprType(linitial(opexpr->args)), + exprType(lsecond(opexpr->args))); + } + else if (IsA(sublink->testexpr, BoolExpr)) + { + /* multiple combining operators, = or <> cases */ + char *sep; + ListCell *l; + + appendStringInfoChar(buf, '('); + sep = ""; + foreach(l, ((BoolExpr *) sublink->testexpr)->args) + { + OpExpr *opexpr = lfirst_node(OpExpr, l); + + appendStringInfoString(buf, sep); + get_rule_expr(linitial(opexpr->args), context, true); + if (!opname) + opname = generate_operator_name(opexpr->opno, + exprType(linitial(opexpr->args)), + exprType(lsecond(opexpr->args))); + sep = ", "; + } + appendStringInfoChar(buf, ')'); + } + else if (IsA(sublink->testexpr, RowCompareExpr)) + { + /* multiple combining operators, < <= > >= cases */ + RowCompareExpr *rcexpr = (RowCompareExpr *) sublink->testexpr; + + appendStringInfoChar(buf, '('); + get_rule_expr((Node *) rcexpr->largs, context, true); + opname = generate_operator_name(linitial_oid(rcexpr->opnos), + exprType(linitial(rcexpr->largs)), + exprType(linitial(rcexpr->rargs))); + appendStringInfoChar(buf, ')'); + } + else + elog(ERROR, "unrecognized testexpr type: %d", + (int) nodeTag(sublink->testexpr)); + } + + need_paren = true; + + switch (sublink->subLinkType) + { + case EXISTS_SUBLINK: + appendStringInfoString(buf, "EXISTS "); + break; + + case ANY_SUBLINK: + if (strcmp(opname, "=") == 0) /* Represent = ANY as IN */ + appendStringInfoString(buf, " IN "); + else + appendStringInfo(buf, " %s ANY ", opname); + break; + + case ALL_SUBLINK: + appendStringInfo(buf, " %s ALL ", opname); + break; + + case ROWCOMPARE_SUBLINK: + appendStringInfo(buf, " %s ", opname); + break; + + case EXPR_SUBLINK: + case MULTIEXPR_SUBLINK: + case ARRAY_SUBLINK: + need_paren = false; + break; + + case CTE_SUBLINK: /* shouldn't occur in a SubLink */ + default: + elog(ERROR, "unrecognized sublink type: %d", + (int) sublink->subLinkType); + break; + } + + if (need_paren) + appendStringInfoChar(buf, '('); + + get_query_def(query, buf, context->namespaces, NULL, + context->prettyFlags, context->wrapColumn, + context->indentLevel); + + if (need_paren) + appendStringInfoString(buf, "))"); + else + appendStringInfoChar(buf, ')'); +} + + +/* ---------- + * get_tablefunc - Parse back a table function + * ---------- + */ +static void +get_tablefunc(TableFunc *tf, deparse_context *context, bool showimplicit) +{ + StringInfo buf = context->buf; + + /* XMLTABLE is the only existing implementation. */ + + appendStringInfoString(buf, "XMLTABLE("); + + if (tf->ns_uris != NIL) + { + ListCell *lc1, + *lc2; + bool first = true; + + appendStringInfoString(buf, "XMLNAMESPACES ("); + forboth(lc1, tf->ns_uris, lc2, tf->ns_names) + { + Node *expr = (Node *) lfirst(lc1); + char *name = strVal(lfirst(lc2)); + + if (!first) + appendStringInfoString(buf, ", "); + else + first = false; + + if (name != NULL) + { + get_rule_expr(expr, context, showimplicit); + appendStringInfo(buf, " AS %s", name); + } + else + { + appendStringInfoString(buf, "DEFAULT "); + get_rule_expr(expr, context, showimplicit); + } + } + appendStringInfoString(buf, "), "); + } + + appendStringInfoChar(buf, '('); + get_rule_expr((Node *) tf->rowexpr, context, showimplicit); + appendStringInfoString(buf, ") PASSING ("); + get_rule_expr((Node *) tf->docexpr, context, showimplicit); + appendStringInfoChar(buf, ')'); + + if (tf->colexprs != NIL) + { + ListCell *l1; + ListCell *l2; + ListCell *l3; + ListCell *l4; + ListCell *l5; + int colnum = 0; + + appendStringInfoString(buf, " COLUMNS "); + forfive(l1, tf->colnames, l2, tf->coltypes, l3, tf->coltypmods, + l4, tf->colexprs, l5, tf->coldefexprs) + { + char *colname = strVal(lfirst(l1)); + Oid typid = lfirst_oid(l2); + int32 typmod = lfirst_int(l3); + Node *colexpr = (Node *) lfirst(l4); + Node *coldefexpr = (Node *) lfirst(l5); + bool ordinality = (tf->ordinalitycol == colnum); + bool notnull = bms_is_member(colnum, tf->notnulls); + + if (colnum > 0) + appendStringInfoString(buf, ", "); + colnum++; + + appendStringInfo(buf, "%s %s", quote_identifier(colname), + ordinality ? "FOR ORDINALITY" : + format_type_with_typemod(typid, typmod)); + if (ordinality) + continue; + + if (coldefexpr != NULL) + { + appendStringInfoString(buf, " DEFAULT ("); + get_rule_expr((Node *) coldefexpr, context, showimplicit); + appendStringInfoChar(buf, ')'); + } + if (colexpr != NULL) + { + appendStringInfoString(buf, " PATH ("); + get_rule_expr((Node *) colexpr, context, showimplicit); + appendStringInfoChar(buf, ')'); + } + if (notnull) + appendStringInfoString(buf, " NOT NULL"); + } + } + + appendStringInfoChar(buf, ')'); +} + +/* ---------- + * get_from_clause - Parse back a FROM clause + * + * "prefix" is the keyword that denotes the start of the list of FROM + * elements. It is FROM when used to parse back SELECT and UPDATE, but + * is USING when parsing back DELETE. + * ---------- + */ +static void +get_from_clause(Query *query, const char *prefix, deparse_context *context) +{ + StringInfo buf = context->buf; + bool first = true; + ListCell *l; + + /* + * We use the query's jointree as a guide to what to print. However, we + * must ignore auto-added RTEs that are marked not inFromCl. (These can + * only appear at the top level of the jointree, so it's sufficient to + * check here.) This check also ensures we ignore the rule pseudo-RTEs + * for NEW and OLD. + */ + foreach(l, query->jointree->fromlist) + { + Node *jtnode = (Node *) lfirst(l); + + if (IsA(jtnode, RangeTblRef)) + { + int varno = ((RangeTblRef *) jtnode)->rtindex; + RangeTblEntry *rte = rt_fetch(varno, query->rtable); + + if (!rte->inFromCl) + continue; + } + + if (first) + { + appendContextKeyword(context, prefix, + -PRETTYINDENT_STD, PRETTYINDENT_STD, 2); + first = false; + + get_from_clause_item(jtnode, query, context); + } + else + { + StringInfoData itembuf; + + appendStringInfoString(buf, ", "); + + /* + * Put the new FROM item's text into itembuf so we can decide + * after we've got it whether or not it needs to go on a new line. + */ + initStringInfo(&itembuf); + context->buf = &itembuf; + + get_from_clause_item(jtnode, query, context); + + /* Restore context's output buffer */ + context->buf = buf; + + /* Consider line-wrapping if enabled */ + if (PRETTY_INDENT(context) && context->wrapColumn >= 0) + { + /* Does the new item start with a new line? */ + if (itembuf.len > 0 && itembuf.data[0] == '\n') + { + /* If so, we shouldn't add anything */ + /* instead, remove any trailing spaces currently in buf */ + removeStringInfoSpaces(buf); + } + else + { + char *trailing_nl; + + /* Locate the start of the current line in the buffer */ + trailing_nl = strrchr(buf->data, '\n'); + if (trailing_nl == NULL) + trailing_nl = buf->data; + else + trailing_nl++; + + /* + * Add a newline, plus some indentation, if the new item + * would cause an overflow. + */ + if (strlen(trailing_nl) + itembuf.len > context->wrapColumn) + appendContextKeyword(context, "", -PRETTYINDENT_STD, + PRETTYINDENT_STD, + PRETTYINDENT_VAR); + } + } + + /* Add the new item */ + appendStringInfoString(buf, itembuf.data); + + /* clean up */ + pfree(itembuf.data); + } + } +} + +static void +get_from_clause_item(Node *jtnode, Query *query, deparse_context *context) +{ + StringInfo buf = context->buf; + deparse_namespace *dpns = (deparse_namespace *) linitial(context->namespaces); + + if (IsA(jtnode, RangeTblRef)) + { + int varno = ((RangeTblRef *) jtnode)->rtindex; + RangeTblEntry *rte = rt_fetch(varno, query->rtable); + char *refname = get_rtable_name(varno, context); + deparse_columns *colinfo = deparse_columns_fetch(varno, dpns); + RangeTblFunction *rtfunc1 = NULL; + bool printalias; + CitusRTEKind rteKind = GetRangeTblKind(rte); + + if (rte->lateral) + appendStringInfoString(buf, "LATERAL "); + + /* Print the FROM item proper */ + switch (rte->rtekind) + { + case RTE_RELATION: + /* Normal relation RTE */ + appendStringInfo(buf, "%s%s", + only_marker(rte), + generate_relation_or_shard_name(rte->relid, + context->distrelid, + context->shardid, + context->namespaces)); + break; + case RTE_SUBQUERY: + /* Subquery RTE */ + appendStringInfoChar(buf, '('); + get_query_def(rte->subquery, buf, context->namespaces, NULL, + context->prettyFlags, context->wrapColumn, + context->indentLevel); + appendStringInfoChar(buf, ')'); + break; + case RTE_FUNCTION: + /* if it's a shard, do differently */ + if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) + { + char *fragmentSchemaName = NULL; + char *fragmentTableName = NULL; + + ExtractRangeTblExtraData(rte, NULL, &fragmentSchemaName, &fragmentTableName, NULL); + + /* use schema and table name from the remote alias */ + appendStringInfoString(buf, + generate_fragment_name(fragmentSchemaName, + fragmentTableName)); + break; + } + + /* Function RTE */ + rtfunc1 = (RangeTblFunction *) linitial(rte->functions); + + /* + * Omit ROWS FROM() syntax for just one function, unless it + * has both a coldeflist and WITH ORDINALITY. If it has both, + * we must use ROWS FROM() syntax to avoid ambiguity about + * whether the coldeflist includes the ordinality column. + */ + if (list_length(rte->functions) == 1 && + (rtfunc1->funccolnames == NIL || !rte->funcordinality)) + { + get_rule_expr_funccall(rtfunc1->funcexpr, context, true); + /* we'll print the coldeflist below, if it has one */ + } + else + { + bool all_unnest; + ListCell *lc; + + /* + * If all the function calls in the list are to unnest, + * and none need a coldeflist, then collapse the list back + * down to UNNEST(args). (If we had more than one + * built-in unnest function, this would get more + * difficult.) + * + * XXX This is pretty ugly, since it makes not-terribly- + * future-proof assumptions about what the parser would do + * with the output; but the alternative is to emit our + * nonstandard ROWS FROM() notation for what might have + * been a perfectly spec-compliant multi-argument + * UNNEST(). + */ + all_unnest = true; + foreach(lc, rte->functions) + { + RangeTblFunction *rtfunc = (RangeTblFunction *) lfirst(lc); + + if (!IsA(rtfunc->funcexpr, FuncExpr) || + ((FuncExpr *) rtfunc->funcexpr)->funcid != F_ARRAY_UNNEST || + rtfunc->funccolnames != NIL) + { + all_unnest = false; + break; + } + } + + if (all_unnest) + { + List *allargs = NIL; + + foreach(lc, rte->functions) + { + RangeTblFunction *rtfunc = (RangeTblFunction *) lfirst(lc); + List *args = ((FuncExpr *) rtfunc->funcexpr)->args; + + allargs = list_concat(allargs, list_copy(args)); + } + + appendStringInfoString(buf, "UNNEST("); + get_rule_expr((Node *) allargs, context, true); + appendStringInfoChar(buf, ')'); + } + else + { + int funcno = 0; + + appendStringInfoString(buf, "ROWS FROM("); + foreach(lc, rte->functions) + { + RangeTblFunction *rtfunc = (RangeTblFunction *) lfirst(lc); + + if (funcno > 0) + appendStringInfoString(buf, ", "); + get_rule_expr_funccall(rtfunc->funcexpr, context, true); + if (rtfunc->funccolnames != NIL) + { + /* Reconstruct the column definition list */ + appendStringInfoString(buf, " AS "); + get_from_clause_coldeflist(rtfunc, + NULL, + context); + } + funcno++; + } + appendStringInfoChar(buf, ')'); + } + /* prevent printing duplicate coldeflist below */ + rtfunc1 = NULL; + } + if (rte->funcordinality) + appendStringInfoString(buf, " WITH ORDINALITY"); + break; + case RTE_TABLEFUNC: + get_tablefunc(rte->tablefunc, context, true); + break; + case RTE_VALUES: + /* Values list RTE */ + appendStringInfoChar(buf, '('); + get_values_def(rte->values_lists, context); + appendStringInfoChar(buf, ')'); + break; + case RTE_CTE: + appendStringInfoString(buf, quote_identifier(rte->ctename)); + break; + default: + elog(ERROR, "unrecognized RTE kind: %d", (int) rte->rtekind); + break; + } + + /* Print the relation alias, if needed */ + printalias = false; + if (rte->alias != NULL) + { + /* Always print alias if user provided one */ + printalias = true; + } + else if (colinfo->printaliases) + { + /* Always print alias if we need to print column aliases */ + printalias = true; + } + else if (rte->rtekind == RTE_RELATION) + { + /* + * No need to print alias if it's same as relation name (this + * would normally be the case, but not if set_rtable_names had to + * resolve a conflict). + */ + if (strcmp(refname, get_relation_name(rte->relid)) != 0) + printalias = true; + } + else if (rte->rtekind == RTE_FUNCTION) + { + /* + * For a function RTE, always print alias. This covers possible + * renaming of the function and/or instability of the + * FigureColname rules for things that aren't simple functions. + * Note we'd need to force it anyway for the columndef list case. + */ + printalias = true; + } + else if (rte->rtekind == RTE_VALUES) + { + /* Alias is syntactically required for VALUES */ + printalias = true; + } + else if (rte->rtekind == RTE_CTE) + { + /* + * No need to print alias if it's same as CTE name (this would + * normally be the case, but not if set_rtable_names had to + * resolve a conflict). + */ + if (strcmp(refname, rte->ctename) != 0) + printalias = true; + } + else if (rte->rtekind == RTE_SUBQUERY) + { + /* subquery requires alias too */ + printalias = true; + } + if (printalias) + appendStringInfo(buf, " %s", quote_identifier(refname)); + + /* Print the column definitions or aliases, if needed */ + if (rtfunc1 && rtfunc1->funccolnames != NIL) + { + /* Reconstruct the columndef list, which is also the aliases */ + get_from_clause_coldeflist(rtfunc1, colinfo, context); + } + else if (GetRangeTblKind(rte) != CITUS_RTE_SHARD) + { + /* Else print column aliases as needed */ + get_column_alias_list(colinfo, context); + } + /* check if column's are given aliases in distributed tables */ + else if (colinfo->parentUsing != NIL) + { + Assert(colinfo->printaliases); + get_column_alias_list(colinfo, context); + } + + /* Tablesample clause must go after any alias */ + if ((rteKind == CITUS_RTE_RELATION || rteKind == CITUS_RTE_SHARD) && + rte->tablesample) + { + get_tablesample_def(rte->tablesample, context); + } + } + else if (IsA(jtnode, JoinExpr)) + { + JoinExpr *j = (JoinExpr *) jtnode; + deparse_columns *colinfo = deparse_columns_fetch(j->rtindex, dpns); + bool need_paren_on_right; + + need_paren_on_right = PRETTY_PAREN(context) && + !IsA(j->rarg, RangeTblRef) && + !(IsA(j->rarg, JoinExpr) &&((JoinExpr *) j->rarg)->alias != NULL); + + if (!PRETTY_PAREN(context) || j->alias != NULL) + appendStringInfoChar(buf, '('); + + get_from_clause_item(j->larg, query, context); + + switch (j->jointype) + { + case JOIN_INNER: + if (j->quals) + appendContextKeyword(context, " JOIN ", + -PRETTYINDENT_STD, + PRETTYINDENT_STD, + PRETTYINDENT_JOIN); + else + appendContextKeyword(context, " CROSS JOIN ", + -PRETTYINDENT_STD, + PRETTYINDENT_STD, + PRETTYINDENT_JOIN); + break; + case JOIN_LEFT: + appendContextKeyword(context, " LEFT JOIN ", + -PRETTYINDENT_STD, + PRETTYINDENT_STD, + PRETTYINDENT_JOIN); + break; + case JOIN_FULL: + appendContextKeyword(context, " FULL JOIN ", + -PRETTYINDENT_STD, + PRETTYINDENT_STD, + PRETTYINDENT_JOIN); + break; + case JOIN_RIGHT: + appendContextKeyword(context, " RIGHT JOIN ", + -PRETTYINDENT_STD, + PRETTYINDENT_STD, + PRETTYINDENT_JOIN); + break; + default: + elog(ERROR, "unrecognized join type: %d", + (int) j->jointype); + } + + if (need_paren_on_right) + appendStringInfoChar(buf, '('); + get_from_clause_item(j->rarg, query, context); + if (need_paren_on_right) + appendStringInfoChar(buf, ')'); + + if (j->usingClause) + { + ListCell *lc; + bool first = true; + + appendStringInfoString(buf, " USING ("); + /* Use the assigned names, not what's in usingClause */ + foreach(lc, colinfo->usingNames) + { + char *colname = (char *) lfirst(lc); + + if (first) + first = false; + else + appendStringInfoString(buf, ", "); + appendStringInfoString(buf, quote_identifier(colname)); + } + appendStringInfoChar(buf, ')'); + } + else if (j->quals) + { + appendStringInfoString(buf, " ON "); + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr(j->quals, context, false); + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + } + else if (j->jointype != JOIN_INNER) + { + /* If we didn't say CROSS JOIN above, we must provide an ON */ + appendStringInfoString(buf, " ON TRUE"); + } + + if (!PRETTY_PAREN(context) || j->alias != NULL) + appendStringInfoChar(buf, ')'); + + /* Yes, it's correct to put alias after the right paren ... */ + if (j->alias != NULL) + { + /* + * Note that it's correct to emit an alias clause if and only if + * there was one originally. Otherwise we'd be converting a named + * join to unnamed or vice versa, which creates semantic + * subtleties we don't want. However, we might print a different + * alias name than was there originally. + */ + appendStringInfo(buf, " %s", + quote_identifier(get_rtable_name(j->rtindex, + context))); + get_column_alias_list(colinfo, context); + } + } + else + elog(ERROR, "unrecognized node type: %d", + (int) nodeTag(jtnode)); +} + +/* + * get_column_alias_list - print column alias list for an RTE + * + * Caller must already have printed the relation's alias name. + */ +static void +get_column_alias_list(deparse_columns *colinfo, deparse_context *context) +{ + StringInfo buf = context->buf; + int i; + bool first = true; + + /* Don't print aliases if not needed */ + if (!colinfo->printaliases) + return; + + for (i = 0; i < colinfo->num_new_cols; i++) + { + char *colname = colinfo->new_colnames[i]; + + if (first) + { + appendStringInfoChar(buf, '('); + first = false; + } + else + appendStringInfoString(buf, ", "); + appendStringInfoString(buf, quote_identifier(colname)); + } + if (!first) + appendStringInfoChar(buf, ')'); +} + +/* + * get_from_clause_coldeflist - reproduce FROM clause coldeflist + * + * When printing a top-level coldeflist (which is syntactically also the + * relation's column alias list), use column names from colinfo. But when + * printing a coldeflist embedded inside ROWS FROM(), we prefer to use the + * original coldeflist's names, which are available in rtfunc->funccolnames. + * Pass NULL for colinfo to select the latter behavior. + * + * The coldeflist is appended immediately (no space) to buf. Caller is + * responsible for ensuring that an alias or AS is present before it. + */ +static void +get_from_clause_coldeflist(RangeTblFunction *rtfunc, + deparse_columns *colinfo, + deparse_context *context) +{ + StringInfo buf = context->buf; + ListCell *l1; + ListCell *l2; + ListCell *l3; + ListCell *l4; + int i; + + appendStringInfoChar(buf, '('); + + i = 0; + forfour(l1, rtfunc->funccoltypes, + l2, rtfunc->funccoltypmods, + l3, rtfunc->funccolcollations, + l4, rtfunc->funccolnames) + { + Oid atttypid = lfirst_oid(l1); + int32 atttypmod = lfirst_int(l2); + Oid attcollation = lfirst_oid(l3); + char *attname; + + if (colinfo) + attname = colinfo->colnames[i]; + else + attname = strVal(lfirst(l4)); + + Assert(attname); /* shouldn't be any dropped columns here */ + + if (i > 0) + appendStringInfoString(buf, ", "); + appendStringInfo(buf, "%s %s", + quote_identifier(attname), + format_type_with_typemod(atttypid, atttypmod)); + if (OidIsValid(attcollation) && + attcollation != get_typcollation(atttypid)) + appendStringInfo(buf, " COLLATE %s", + generate_collation_name(attcollation)); + + i++; + } + + appendStringInfoChar(buf, ')'); +} + +/* + * get_tablesample_def - print a TableSampleClause + */ +static void +get_tablesample_def(TableSampleClause *tablesample, deparse_context *context) +{ + StringInfo buf = context->buf; + Oid argtypes[1]; + int nargs; + ListCell *l; + + /* + * We should qualify the handler's function name if it wouldn't be + * resolved by lookup in the current search path. + */ + argtypes[0] = INTERNALOID; + appendStringInfo(buf, " TABLESAMPLE %s (", + generate_function_name(tablesample->tsmhandler, 1, + NIL, argtypes, + false, NULL, EXPR_KIND_NONE)); + + nargs = 0; + foreach(l, tablesample->args) + { + if (nargs++ > 0) + appendStringInfoString(buf, ", "); + get_rule_expr((Node *) lfirst(l), context, false); + } + appendStringInfoChar(buf, ')'); + + if (tablesample->repeatable != NULL) + { + appendStringInfoString(buf, " REPEATABLE ("); + get_rule_expr((Node *) tablesample->repeatable, context, false); + appendStringInfoChar(buf, ')'); + } +} + +/* + * get_opclass_name - fetch name of an index operator class + * + * The opclass name is appended (after a space) to buf. + * + * Output is suppressed if the opclass is the default for the given + * actual_datatype. (If you don't want this behavior, just pass + * InvalidOid for actual_datatype.) + */ +static void +get_opclass_name(Oid opclass, Oid actual_datatype, + StringInfo buf) +{ + HeapTuple ht_opc; + Form_pg_opclass opcrec; + char *opcname; + char *nspname; + + ht_opc = SearchSysCache1(CLAOID, ObjectIdGetDatum(opclass)); + if (!HeapTupleIsValid(ht_opc)) + elog(ERROR, "cache lookup failed for opclass %u", opclass); + opcrec = (Form_pg_opclass) GETSTRUCT(ht_opc); + + if (!OidIsValid(actual_datatype) || + GetDefaultOpClass(actual_datatype, opcrec->opcmethod) != opclass) + { + /* Okay, we need the opclass name. Do we need to qualify it? */ + opcname = NameStr(opcrec->opcname); + if (OpclassIsVisible(opclass)) + appendStringInfo(buf, " %s", quote_identifier(opcname)); + else + { + nspname = get_namespace_name(opcrec->opcnamespace); + appendStringInfo(buf, " %s.%s", + quote_identifier(nspname), + quote_identifier(opcname)); + } + } + ReleaseSysCache(ht_opc); +} + +/* + * processIndirection - take care of array and subfield assignment + * + * We strip any top-level FieldStore or assignment SubscriptingRef nodes that + * appear in the input, printing them as decoration for the base column + * name (which we assume the caller just printed). We might also need to + * strip CoerceToDomain nodes, but only ones that appear above assignment + * nodes. + * + * Returns the subexpression that's to be assigned. + */ +static Node * +processIndirection(Node *node, deparse_context *context) +{ + StringInfo buf = context->buf; + CoerceToDomain *cdomain = NULL; + + for (;;) + { + if (node == NULL) + break; + if (IsA(node, FieldStore)) + { + FieldStore *fstore = (FieldStore *) node; + Oid typrelid; + char *fieldname; + + /* lookup tuple type */ + typrelid = get_typ_typrelid(fstore->resulttype); + if (!OidIsValid(typrelid)) + elog(ERROR, "argument type %s of FieldStore is not a tuple type", + format_type_be(fstore->resulttype)); + + /* + * Print the field name. There should only be one target field in + * stored rules. There could be more than that in executable + * target lists, but this function cannot be used for that case. + */ + Assert(list_length(fstore->fieldnums) == 1); + fieldname = get_attname(typrelid, + linitial_int(fstore->fieldnums), false); + appendStringInfo(buf, ".%s", quote_identifier(fieldname)); + + /* + * We ignore arg since it should be an uninteresting reference to + * the target column or subcolumn. + */ + node = (Node *) linitial(fstore->newvals); + } + else if (IsA(node, SubscriptingRef)) + { + SubscriptingRef *sbsref = (SubscriptingRef *) node; + + if (sbsref->refassgnexpr == NULL) + break; + printSubscripts(sbsref, context); + + /* + * We ignore refexpr since it should be an uninteresting reference + * to the target column or subcolumn. + */ + node = (Node *) sbsref->refassgnexpr; + } + else if (IsA(node, CoerceToDomain)) + { + cdomain = (CoerceToDomain *) node; + /* If it's an explicit domain coercion, we're done */ + if (cdomain->coercionformat != COERCE_IMPLICIT_CAST) + break; + /* Tentatively descend past the CoerceToDomain */ + node = (Node *) cdomain->arg; + } + else + break; + } + + /* + * If we descended past a CoerceToDomain whose argument turned out not to + * be a FieldStore or array assignment, back up to the CoerceToDomain. + * (This is not enough to be fully correct if there are nested implicit + * CoerceToDomains, but such cases shouldn't ever occur.) + */ + if (cdomain && node == (Node *) cdomain->arg) + node = (Node *) cdomain; + + return node; +} + +static void +printSubscripts(SubscriptingRef *sbsref, deparse_context *context) +{ + StringInfo buf = context->buf; + ListCell *lowlist_item; + ListCell *uplist_item; + + lowlist_item = list_head(sbsref->reflowerindexpr); /* could be NULL */ + foreach(uplist_item, sbsref->refupperindexpr) + { + appendStringInfoChar(buf, '['); + if (lowlist_item) + { + /* If subexpression is NULL, get_rule_expr prints nothing */ + get_rule_expr((Node *) lfirst(lowlist_item), context, false); + appendStringInfoChar(buf, ':'); + lowlist_item = lnext(sbsref->refupperindexpr, lowlist_item); + } + /* If subexpression is NULL, get_rule_expr prints nothing */ + get_rule_expr((Node *) lfirst(uplist_item), context, false); + appendStringInfoChar(buf, ']'); + } +} + +/* + * get_relation_name + * Get the unqualified name of a relation specified by OID + * + * This differs from the underlying get_rel_name() function in that it will + * throw error instead of silently returning NULL if the OID is bad. + */ +static char * +get_relation_name(Oid relid) +{ + char *relname = get_rel_name(relid); + + if (!relname) + elog(ERROR, "cache lookup failed for relation %u", relid); + return relname; +} + +/* + * generate_relation_or_shard_name + * Compute the name to display for a relation or shard + * + * If the provided relid is equal to the provided distrelid, this function + * returns a shard-extended relation name; otherwise, it falls through to a + * simple generate_relation_name call. + */ +static char * +generate_relation_or_shard_name(Oid relid, Oid distrelid, int64 shardid, + List *namespaces) +{ + char *relname = NULL; + + if (relid == distrelid) + { + relname = get_relation_name(relid); + + if (shardid > 0) + { + Oid schemaOid = get_rel_namespace(relid); + char *schemaName = get_namespace_name(schemaOid); + + AppendShardIdToName(&relname, shardid); + + relname = quote_qualified_identifier(schemaName, relname); + } + } + else + { + relname = generate_relation_name(relid, namespaces); + } + + return relname; +} + +/* + * generate_relation_name + * Compute the name to display for a relation specified by OID + * + * The result includes all necessary quoting and schema-prefixing. + * + * If namespaces isn't NIL, it must be a list of deparse_namespace nodes. + * We will forcibly qualify the relation name if it equals any CTE name + * visible in the namespace list. + */ +char * +generate_relation_name(Oid relid, List *namespaces) +{ + HeapTuple tp; + Form_pg_class reltup; + bool need_qual; + ListCell *nslist; + char *relname; + char *nspname; + char *result; + + tp = SearchSysCache1(RELOID, ObjectIdGetDatum(relid)); + if (!HeapTupleIsValid(tp)) + elog(ERROR, "cache lookup failed for relation %u", relid); + reltup = (Form_pg_class) GETSTRUCT(tp); + relname = NameStr(reltup->relname); + + /* Check for conflicting CTE name */ + need_qual = false; + foreach(nslist, namespaces) + { + deparse_namespace *dpns = (deparse_namespace *) lfirst(nslist); + ListCell *ctlist; + + foreach(ctlist, dpns->ctes) + { + CommonTableExpr *cte = (CommonTableExpr *) lfirst(ctlist); + + if (strcmp(cte->ctename, relname) == 0) + { + need_qual = true; + break; + } + } + if (need_qual) + break; + } + + /* Otherwise, qualify the name if not visible in search path */ + if (!need_qual) + need_qual = !RelationIsVisible(relid); + + if (need_qual) + nspname = get_namespace_name(reltup->relnamespace); + else + nspname = NULL; + + result = quote_qualified_identifier(nspname, relname); + + ReleaseSysCache(tp); + + return result; +} + + +/* + * generate_rte_shard_name returns the qualified name of the shard given a + * CITUS_RTE_SHARD range table entry. + */ +static char * +generate_rte_shard_name(RangeTblEntry *rangeTableEntry) +{ + char *shardSchemaName = NULL; + char *shardTableName = NULL; + + Assert(GetRangeTblKind(rangeTableEntry) == CITUS_RTE_SHARD); + + ExtractRangeTblExtraData(rangeTableEntry, NULL, &shardSchemaName, &shardTableName, + NULL); + + return generate_fragment_name(shardSchemaName, shardTableName); +} + + +/* + * generate_fragment_name + * Compute the name to display for a shard or merged table + * + * The result includes all necessary quoting and schema-prefixing. The schema + * name can be NULL for regular shards. For merged tables, they are always + * declared within a job-specific schema, and therefore can't have null schema + * names. + */ +static char * +generate_fragment_name(char *schemaName, char *tableName) +{ + StringInfo fragmentNameString = makeStringInfo(); + + if (schemaName != NULL) + { + appendStringInfo(fragmentNameString, "%s.%s", quote_identifier(schemaName), + quote_identifier(tableName)); + } + else + { + appendStringInfoString(fragmentNameString, quote_identifier(tableName)); + } + + return fragmentNameString->data; +} + +/* + * generate_function_name + * Compute the name to display for a function specified by OID, + * given that it is being called with the specified actual arg names and + * types. (Those matter because of ambiguous-function resolution rules.) + * + * If we're dealing with a potentially variadic function (in practice, this + * means a FuncExpr or Aggref, not some other way of calling a function), then + * has_variadic must specify whether variadic arguments have been merged, + * and *use_variadic_p will be set to indicate whether to print VARIADIC in + * the output. For non-FuncExpr cases, has_variadic should be false and + * use_variadic_p can be NULL. + * + * The result includes all necessary quoting and schema-prefixing. + */ +static char * +generate_function_name(Oid funcid, int nargs, List *argnames, Oid *argtypes, + bool has_variadic, bool *use_variadic_p, + ParseExprKind special_exprkind) +{ + char *result; + HeapTuple proctup; + Form_pg_proc procform; + char *proname; + bool use_variadic; + char *nspname; + FuncDetailCode p_result; + Oid p_funcid; + Oid p_rettype; + bool p_retset; + int p_nvargs; + Oid p_vatype; + Oid *p_true_typeids; + bool force_qualify = false; + + proctup = SearchSysCache1(PROCOID, ObjectIdGetDatum(funcid)); + if (!HeapTupleIsValid(proctup)) + elog(ERROR, "cache lookup failed for function %u", funcid); + procform = (Form_pg_proc) GETSTRUCT(proctup); + proname = NameStr(procform->proname); + + /* + * Due to parser hacks to avoid needing to reserve CUBE, we need to force + * qualification in some special cases. + */ + if (special_exprkind == EXPR_KIND_GROUP_BY) + { + if (strcmp(proname, "cube") == 0 || strcmp(proname, "rollup") == 0) + force_qualify = true; + } + + /* + * Determine whether VARIADIC should be printed. We must do this first + * since it affects the lookup rules in func_get_detail(). + * + * Currently, we always print VARIADIC if the function has a merged + * variadic-array argument. Note that this is always the case for + * functions taking a VARIADIC argument type other than VARIADIC ANY. + * + * In principle, if VARIADIC wasn't originally specified and the array + * actual argument is deconstructable, we could print the array elements + * separately and not print VARIADIC, thus more nearly reproducing the + * original input. For the moment that seems like too much complication + * for the benefit, and anyway we do not know whether VARIADIC was + * originally specified if it's a non-ANY type. + */ + if (use_variadic_p) + { + /* Parser should not have set funcvariadic unless fn is variadic */ + Assert(!has_variadic || OidIsValid(procform->provariadic)); + use_variadic = has_variadic; + *use_variadic_p = use_variadic; + } + else + { + Assert(!has_variadic); + use_variadic = false; + } + + /* + * The idea here is to schema-qualify only if the parser would fail to + * resolve the correct function given the unqualified func name with the + * specified argtypes and VARIADIC flag. But if we already decided to + * force qualification, then we can skip the lookup and pretend we didn't + * find it. + */ + if (!force_qualify) + p_result = func_get_detail(list_make1(makeString(proname)), + NIL, argnames, nargs, argtypes, + !use_variadic, true, + &p_funcid, &p_rettype, + &p_retset, &p_nvargs, &p_vatype, + &p_true_typeids, NULL); + else + { + p_result = FUNCDETAIL_NOTFOUND; + p_funcid = InvalidOid; + } + + if ((p_result == FUNCDETAIL_NORMAL || + p_result == FUNCDETAIL_AGGREGATE || + p_result == FUNCDETAIL_WINDOWFUNC) && + p_funcid == funcid) + nspname = NULL; + else + nspname = get_namespace_name(procform->pronamespace); + + result = quote_qualified_identifier(nspname, proname); + + ReleaseSysCache(proctup); + + return result; +} + +/* + * generate_operator_name + * Compute the name to display for an operator specified by OID, + * given that it is being called with the specified actual arg types. + * (Arg types matter because of ambiguous-operator resolution rules. + * Pass InvalidOid for unused arg of a unary operator.) + * + * The result includes all necessary quoting and schema-prefixing, + * plus the OPERATOR() decoration needed to use a qualified operator name + * in an expression. + */ +char * +generate_operator_name(Oid operid, Oid arg1, Oid arg2) +{ + StringInfoData buf; + HeapTuple opertup; + Form_pg_operator operform; + char *oprname; + char *nspname; + + initStringInfo(&buf); + + opertup = SearchSysCache1(OPEROID, ObjectIdGetDatum(operid)); + if (!HeapTupleIsValid(opertup)) + elog(ERROR, "cache lookup failed for operator %u", operid); + operform = (Form_pg_operator) GETSTRUCT(opertup); + oprname = NameStr(operform->oprname); + + /* + * Unlike generate_operator_name() in postgres/src/backend/utils/adt/ruleutils.c, + * we don't check if the operator is in current namespace or not. This is + * because this check is costly when the operator is not in current namespace. + */ + nspname = get_namespace_name(operform->oprnamespace); + Assert(nspname != NULL); + appendStringInfo(&buf, "OPERATOR(%s.", quote_identifier(nspname)); + appendStringInfoString(&buf, oprname); + appendStringInfoChar(&buf, ')'); + + ReleaseSysCache(opertup); + + return buf.data; +} + +/* + * get_one_range_partition_bound_string + * A C string representation of one range partition bound + */ +char * +get_range_partbound_string(List *bound_datums) +{ + deparse_context context; + StringInfo buf = makeStringInfo(); + ListCell *cell; + char *sep; + + memset(&context, 0, sizeof(deparse_context)); + context.buf = buf; + + appendStringInfoString(buf, "("); + sep = ""; + foreach(cell, bound_datums) + { + PartitionRangeDatum *datum = + castNode(PartitionRangeDatum, lfirst(cell)); + + appendStringInfoString(buf, sep); + if (datum->kind == PARTITION_RANGE_DATUM_MINVALUE) + appendStringInfoString(buf, "MINVALUE"); + else if (datum->kind == PARTITION_RANGE_DATUM_MAXVALUE) + appendStringInfoString(buf, "MAXVALUE"); + else + { + Const *val = castNode(Const, datum->value); + + get_const_expr(val, &context, -1); + } + sep = ", "; + } + appendStringInfoChar(buf, ')'); + + return buf->data; +} + +#endif /* (PG_VERSION_NUM >= PG_VERSION_12) && (PG_VERSION_NUM < PG_VERSION_13) */ diff --git a/src/include/distributed/pg_version_constants.h b/src/include/distributed/pg_version_constants.h index 03da2d5af..68b8ffb72 100644 --- a/src/include/distributed/pg_version_constants.h +++ b/src/include/distributed/pg_version_constants.h @@ -14,5 +14,6 @@ #define PG_VERSION_11 110000 #define PG_VERSION_12 120000 #define PG_VERSION_13 130000 +#define PG_VERSION_14 140000 #endif /* PG_VERSION_CONSTANTS */ From 8ce8683ac44d7f636d09c0cc31ae116ccd0091f2 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Thu, 28 May 2020 12:26:31 +0300 Subject: [PATCH 03/52] Update ruleutils_13.c with postgres ruleutils Some manual updates are done for ruleutils_13 based on the difference between pg12 ruleutils and pg13 ruleutils. --- .../distributed/deparser/ruleutils_13.c | 326 ++++++++++++------ 1 file changed, 215 insertions(+), 111 deletions(-) diff --git a/src/backend/distributed/deparser/ruleutils_13.c b/src/backend/distributed/deparser/ruleutils_13.c index 3ca2da67b..18ef36b84 100644 --- a/src/backend/distributed/deparser/ruleutils_13.c +++ b/src/backend/distributed/deparser/ruleutils_13.c @@ -132,7 +132,8 @@ typedef struct int64 shardid; /* a distributed table's shardid, if positive */ ParseExprKind special_exprkind; /* set only for exprkinds needing special * handling */ - Bitmapset *appendparents; + Bitmapset *appendparents; /* if not null, map child Vars of these relids + * back to the parent rel */ } deparse_context; /* @@ -167,20 +168,26 @@ typedef struct List *rtable; /* List of RangeTblEntry nodes */ List *rtable_names; /* Parallel list of names for RTEs */ List *rtable_columns; /* Parallel list of deparse_columns structs */ + List *subplans; /* List of Plan trees for SubPlans */ List *ctes; /* List of CommonTableExpr nodes */ + AppendRelInfo **appendrels; /* Array of AppendRelInfo nodes, or NULL */ /* Workspace for column alias assignment: */ bool unique_using; /* Are we making USING names globally unique */ List *using_names; /* List of assigned names for USING columns */ /* Remaining fields are used only when deparsing a Plan tree: */ - PlanState *planstate; /* immediate parent of current expression */ + Plan *plan; /* immediate parent of current expression */ List *ancestors; /* ancestors of planstate */ - PlanState *outer_planstate; /* outer subplan state, or NULL if none */ - PlanState *inner_planstate; /* inner subplan state, or NULL if none */ + Plan *outer_plan; /* outer subnode, or NULL if none */ + Plan *inner_plan; /* inner subnode, or NULL if none */ List *outer_tlist; /* referent for OUTER_VAR Vars */ List *inner_tlist; /* referent for INNER_VAR Vars */ List *index_tlist; /* referent for INDEX_VAR Vars */ } deparse_namespace; +/* Callback signature for resolve_special_varno() */ +typedef void (*rsv_callback) (Node *node, deparse_context *context, + void *callback_arg); + /* * Per-relation data about column alias names. * @@ -333,8 +340,8 @@ static void identify_join_columns(JoinExpr *j, RangeTblEntry *jrte, static void flatten_join_using_qual(Node *qual, List **leftvars, List **rightvars); static char *get_rtable_name(int rtindex, deparse_context *context); -static void set_deparse_planstate(deparse_namespace *dpns, PlanState *ps); -static void push_child_plan(deparse_namespace *dpns, PlanState *ps, +static void set_deparse_plan(deparse_namespace *dpns, Plan *plan); +static void push_child_plan(deparse_namespace *dpns, Plan *plan, deparse_namespace *save_dpns); static void pop_child_plan(deparse_namespace *dpns, deparse_namespace *save_dpns); @@ -380,10 +387,9 @@ static void get_rule_windowspec(WindowClause *wc, List *targetList, static char *get_variable(Var *var, int levelsup, bool istoplevel, deparse_context *context); static void get_special_variable(Node *node, deparse_context *context, - void *private); + void *callback_arg); static void resolve_special_varno(Node *node, deparse_context *context, - void *private, - void (*callback) (Node *, deparse_context *, void *)); + rsv_callback callback, void *callback_arg); static Node *find_param_referent(Param *param, deparse_context *context, deparse_namespace **dpns_p, ListCell **ancestor_cell_p); static void get_parameter(Param *param, deparse_context *context); @@ -405,7 +411,7 @@ static void get_func_expr(FuncExpr *expr, deparse_context *context, static void get_agg_expr(Aggref *aggref, deparse_context *context, Aggref *original_aggref); static void get_agg_combine_expr(Node *node, deparse_context *context, - void *private); + void *callback_arg); static void get_windowfunc_expr(WindowFunc *wfunc, deparse_context *context); static void get_coercion_expr(Node *arg, deparse_context *context, Oid resulttype, int32 resulttypmod, @@ -672,7 +678,9 @@ set_deparse_for_query(deparse_namespace *dpns, Query *query, /* Initialize *dpns and fill rtable/ctes links */ memset(dpns, 0, sizeof(deparse_namespace)); dpns->rtable = query->rtable; + dpns->subplans = NIL; dpns->ctes = query->cteList; + dpns->appendrels = NULL; /* Assign a unique relation alias to each RTE */ set_rtable_names(dpns, parent_namespaces, NULL); @@ -1710,19 +1718,19 @@ get_rtable_name(int rtindex, deparse_context *context) } /* - * set_deparse_planstate: set up deparse_namespace to parse subexpressions - * of a given PlanState node + * set_deparse_plan: set up deparse_namespace to parse subexpressions + * of a given Plan node * - * This sets the planstate, outer_planstate, inner_planstate, outer_tlist, + * This sets the plan, outer_planstate, inner_planstate, outer_tlist, * inner_tlist, and index_tlist fields. Caller is responsible for adjusting * the ancestors list if necessary. Note that the rtable and ctes fields do * not need to change when shifting attention to different plan nodes in a * single plan tree. */ static void -set_deparse_planstate(deparse_namespace *dpns, PlanState *ps) +set_deparse_plan(deparse_namespace *dpns, Plan *plan) { - dpns->planstate = ps; + dpns->plan = plan; /* * We special-case Append and MergeAppend to pretend that the first child @@ -1732,17 +1740,17 @@ set_deparse_planstate(deparse_namespace *dpns, PlanState *ps) * first child plan is the OUTER referent; this is to support RETURNING * lists containing references to non-target relations. */ - if (IsA(ps, AppendState)) - dpns->outer_planstate = ((AppendState *) ps)->appendplans[0]; - else if (IsA(ps, MergeAppendState)) - dpns->outer_planstate = ((MergeAppendState *) ps)->mergeplans[0]; - else if (IsA(ps, ModifyTableState)) - dpns->outer_planstate = ((ModifyTableState *) ps)->mt_plans[0]; + if (IsA(plan, Append)) + dpns->outer_plan = linitial(((Append *) plan)->appendplans); + else if (IsA(plan, MergeAppend)) + dpns->outer_plan = linitial(((MergeAppend *) plan)->mergeplans); + else if (IsA(plan, ModifyTable)) + dpns->outer_plan = linitial(((ModifyTable *) plan)->plans); else - dpns->outer_planstate = outerPlanState(ps); + dpns->outer_plan = outerPlan(plan); - if (dpns->outer_planstate) - dpns->outer_tlist = dpns->outer_planstate->plan->targetlist; + if (dpns->outer_plan) + dpns->outer_tlist = dpns->outer_plan->targetlist; else dpns->outer_tlist = NIL; @@ -1755,29 +1763,30 @@ set_deparse_planstate(deparse_namespace *dpns, PlanState *ps) * to reuse OUTER, it's used for RETURNING in some modify table cases, * although not INSERT .. CONFLICT). */ - if (IsA(ps, SubqueryScanState)) - dpns->inner_planstate = ((SubqueryScanState *) ps)->subplan; - else if (IsA(ps, CteScanState)) - dpns->inner_planstate = ((CteScanState *) ps)->cteplanstate; - else if (IsA(ps, ModifyTableState)) - dpns->inner_planstate = ps; + if (IsA(plan, SubqueryScan)) + dpns->inner_plan = ((SubqueryScan *) plan)->subplan; + else if (IsA(plan, CteScan)) + dpns->inner_plan = list_nth(dpns->subplans, + ((CteScan *) plan)->ctePlanId - 1); + else if (IsA(plan, ModifyTable)) + dpns->inner_plan = plan; else - dpns->inner_planstate = innerPlanState(ps); + dpns->inner_plan = innerPlan(plan); - if (IsA(ps, ModifyTableState)) - dpns->inner_tlist = ((ModifyTableState *) ps)->mt_excludedtlist; - else if (dpns->inner_planstate) - dpns->inner_tlist = dpns->inner_planstate->plan->targetlist; + if (IsA(plan, ModifyTable)) + dpns->inner_tlist = ((ModifyTable *) plan)->exclRelTlist; + else if (dpns->inner_plan) + dpns->inner_tlist = dpns->inner_plan->targetlist; else dpns->inner_tlist = NIL; /* Set up referent for INDEX_VAR Vars, if needed */ - if (IsA(ps->plan, IndexOnlyScan)) - dpns->index_tlist = ((IndexOnlyScan *) ps->plan)->indextlist; - else if (IsA(ps->plan, ForeignScan)) - dpns->index_tlist = ((ForeignScan *) ps->plan)->fdw_scan_tlist; - else if (IsA(ps->plan, CustomScan)) - dpns->index_tlist = ((CustomScan *) ps->plan)->custom_scan_tlist; + if (IsA(plan, IndexOnlyScan)) + dpns->index_tlist = ((IndexOnlyScan *) plan)->indextlist; + else if (IsA(plan, ForeignScan)) + dpns->index_tlist = ((ForeignScan *) plan)->fdw_scan_tlist; + else if (IsA(plan, CustomScan)) + dpns->index_tlist = ((CustomScan *) plan)->custom_scan_tlist; else dpns->index_tlist = NIL; } @@ -1795,17 +1804,17 @@ set_deparse_planstate(deparse_namespace *dpns, PlanState *ps) * previous state for pop_child_plan. */ static void -push_child_plan(deparse_namespace *dpns, PlanState *ps, +push_child_plan(deparse_namespace *dpns, Plan *plan, deparse_namespace *save_dpns) { /* Save state for restoration later */ *save_dpns = *dpns; /* Link current plan node into ancestors list */ - dpns->ancestors = lcons(dpns->planstate, dpns->ancestors); + dpns->ancestors = lcons(dpns->plan, dpns->ancestors); /* Set attention on selected child */ - set_deparse_planstate(dpns, ps); + set_deparse_plan(dpns, plan); } /* @@ -1960,6 +1969,7 @@ get_query_def_extended(Query *query, StringInfo buf, List *parentnamespace, context.wrapColumn = wrapColumn; context.indentLevel = startIndent; context.special_exprkind = EXPR_KIND_NONE; + context.appendparents = NULL; context.distrelid = distrelid; context.shardid = shardid; @@ -2246,10 +2256,11 @@ get_select_query_def(Query *query, deparse_context *context, * if so, return the VALUES RTE. Otherwise return NULL. */ static RangeTblEntry * -get_simple_values_rte(Query *query) +get_simple_values_rte(Query *query, TupleDesc resultDesc) { RangeTblEntry *result = NULL; ListCell *lc; + int colno; /* * We want to return true even if the Query also contains OLD or NEW rule @@ -2286,14 +2297,24 @@ get_simple_values_rte(Query *query) if (list_length(query->targetList) != list_length(result->eref->colnames)) return NULL; /* this probably cannot happen */ + colno = 0; forboth(lc, query->targetList, lcn, result->eref->colnames) { TargetEntry *tle = (TargetEntry *) lfirst(lc); char *cname = strVal(lfirst(lcn)); + char *colname; if (tle->resjunk) return NULL; /* this probably cannot happen */ - if (tle->resname == NULL || strcmp(tle->resname, cname) != 0) + /* compute name that get_target_list would use for column */ + colno++; + if (resultDesc && colno <= resultDesc->natts) + colname = NameStr(TupleDescAttr(resultDesc, colno - 1)->attname); + else + colname = tle->resname; + + /* does it match the VALUES RTE? */ + if (colname == NULL || strcmp(colname, cname) != 0) return NULL; /* column name has been changed */ } } @@ -2321,7 +2342,7 @@ get_basic_select_query(Query *query, deparse_context *context, * VALUES part. This reverses what transformValuesClause() did at parse * time. */ - values_rte = get_simple_values_rte(query); + values_rte = get_simple_values_rte(query, resultDesc); if (values_rte) { get_values_def(values_rte->values_lists, context); @@ -3644,15 +3665,62 @@ get_variable(Var *var, int levelsup, bool istoplevel, deparse_context *context) */ if (var->varno >= 1 && var->varno <= list_length(dpns->rtable)) { - rte = rt_fetch(var->varno, dpns->rtable); - refname = (char *) list_nth(dpns->rtable_names, var->varno - 1); - colinfo = deparse_columns_fetch(var->varno, dpns); - attnum = var->varattno; + Index varno = var->varno; + AttrNumber varattno = var->varattno; + /* + * We might have been asked to map child Vars to some parent relation. + */ + if (context->appendparents && dpns->appendrels) + { + + Index pvarno = varno; + AttrNumber pvarattno = varattno; + AppendRelInfo *appinfo = dpns->appendrels[pvarno]; + bool found = false; + + /* Only map up to inheritance parents, not UNION ALL appendrels */ + while (appinfo && + rt_fetch(appinfo->parent_relid, + dpns->rtable)->rtekind == RTE_RELATION) + { + found = false; + if (pvarattno > 0) /* system columns stay as-is */ + { + if (pvarattno > appinfo->num_child_cols) + break; /* safety check */ + pvarattno = appinfo->parent_colnos[pvarattno - 1]; + if (pvarattno == 0) + break; /* Var is local to child */ + } + + pvarno = appinfo->parent_relid; + found = true; + + /* If the parent is itself a child, continue up. */ + Assert(pvarno > 0 && pvarno <= list_length(dpns->rtable)); + appinfo = dpns->appendrels[pvarno]; + } + + /* + * If we found an ancestral rel, and that rel is included in + * appendparents, print that column not the original one. + */ + if (found && bms_is_member(pvarno, context->appendparents)) + { + varno = pvarno; + varattno = pvarattno; + } + } + + rte = rt_fetch(varno, dpns->rtable); + refname = (char *) list_nth(dpns->rtable_names, varno - 1); + colinfo = deparse_columns_fetch(varno, dpns); + attnum = varattno; } else { - resolve_special_varno((Node *) var, context, NULL, - get_special_variable); + resolve_special_varno((Node *) var, context, get_special_variable, + NULL); return NULL; } @@ -3665,22 +3733,22 @@ get_variable(Var *var, int levelsup, bool istoplevel, deparse_context *context) * no alias. So in that case, drill down to the subplan and print the * contents of the referenced tlist item. This works because in a plan * tree, such Vars can only occur in a SubqueryScan or CteScan node, and - * we'll have set dpns->inner_planstate to reference the child plan node. + * we'll have set dpns->inner_plan to reference the child plan node. */ if ((rte->rtekind == RTE_SUBQUERY || rte->rtekind == RTE_CTE) && attnum > list_length(rte->eref->colnames) && - dpns->inner_planstate) + dpns->inner_plan) { TargetEntry *tle; deparse_namespace save_dpns; - tle = get_tle_by_resno(dpns->inner_tlist, var->varattno); + tle = get_tle_by_resno(dpns->inner_tlist, attnum); if (!tle) elog(ERROR, "invalid attnum %d for relation \"%s\"", - var->varattno, rte->eref->aliasname); + attnum, rte->eref->aliasname); Assert(netlevelsup == 0); - push_child_plan(dpns, dpns->inner_planstate, &save_dpns); + push_child_plan(dpns, dpns->inner_plan, &save_dpns); /* * Force parentheses because our caller probably assumed a Var is a @@ -3796,13 +3864,13 @@ get_variable(Var *var, int levelsup, bool istoplevel, deparse_context *context) * get_rule_expr. */ static void -get_special_variable(Node *node, deparse_context *context, void *private) +get_special_variable(Node *node, deparse_context *context, void *callback_arg) { StringInfo buf = context->buf; /* - * Force parentheses because our caller probably assumed a Var is a simple - * expression. + * For a non-Var referent, force parentheses because our caller probably + * assumed a Var is a simple expression. */ if (!IsA(node, Var)) appendStringInfoChar(buf, '('); @@ -3817,16 +3885,18 @@ get_special_variable(Node *node, deparse_context *context, void *private) * invoke the callback provided. */ static void -resolve_special_varno(Node *node, deparse_context *context, void *private, - void (*callback) (Node *, deparse_context *, void *)) +resolve_special_varno(Node *node, deparse_context *context, rsv_callback callback, void *callback_arg) { Var *var; deparse_namespace *dpns; + /* This function is recursive, so let's be paranoid. */ + check_stack_depth(); + /* If it's not a Var, invoke the callback. */ if (!IsA(node, Var)) { - callback(node, context, private); + (*callback) (node, context, callback_arg); return; } @@ -3842,14 +3912,30 @@ resolve_special_varno(Node *node, deparse_context *context, void *private, { TargetEntry *tle; deparse_namespace save_dpns; + Bitmapset *save_appendparents; tle = get_tle_by_resno(dpns->outer_tlist, var->varattno); if (!tle) elog(ERROR, "bogus varattno for OUTER_VAR var: %d", var->varattno); - push_child_plan(dpns, dpns->outer_planstate, &save_dpns); - resolve_special_varno((Node *) tle->expr, context, private, callback); + /* If we're descending to the first child of an Append or MergeAppend, + * update appendparents. This will affect deparsing of all Vars + * appearing within the eventually-resolved subexpression. + */ + save_appendparents = context->appendparents; + + if (IsA(dpns->plan, Append)) + context->appendparents = bms_union(context->appendparents, + ((Append *) dpns->plan)->apprelids); + else if (IsA(dpns->plan, MergeAppend)) + context->appendparents = bms_union(context->appendparents, + ((MergeAppend *) dpns->plan)->apprelids); + + push_child_plan(dpns, dpns->outer_plan, &save_dpns); + resolve_special_varno((Node *) tle->expr, context, + callback, callback_arg); pop_child_plan(dpns, &save_dpns); + context->appendparents = save_appendparents; return; } else if (var->varno == INNER_VAR && dpns->inner_tlist) @@ -3861,8 +3947,8 @@ resolve_special_varno(Node *node, deparse_context *context, void *private, if (!tle) elog(ERROR, "bogus varattno for INNER_VAR var: %d", var->varattno); - push_child_plan(dpns, dpns->inner_planstate, &save_dpns); - resolve_special_varno((Node *) tle->expr, context, private, callback); + push_child_plan(dpns, dpns->inner_plan, &save_dpns); + resolve_special_varno((Node *) tle->expr, context, callback, callback_arg); pop_child_plan(dpns, &save_dpns); return; } @@ -3874,14 +3960,14 @@ resolve_special_varno(Node *node, deparse_context *context, void *private, if (!tle) elog(ERROR, "bogus varattno for INDEX_VAR var: %d", var->varattno); - resolve_special_varno((Node *) tle->expr, context, private, callback); + resolve_special_varno((Node *) tle->expr, context, callback, callback_arg); return; } else if (var->varno < 1 || var->varno > list_length(dpns->rtable)) elog(ERROR, "bogus varno: %d", var->varno); /* Not special. Just invoke the callback. */ - callback(node, context, private); + (*callback) (node, context, callback_arg); } /* @@ -3989,7 +4075,7 @@ get_name_for_var_field(Var *var, int fieldno, elog(ERROR, "bogus varattno for OUTER_VAR var: %d", var->varattno); Assert(netlevelsup == 0); - push_child_plan(dpns, dpns->outer_planstate, &save_dpns); + push_child_plan(dpns, dpns->outer_plan, &save_dpns); result = get_name_for_var_field((Var *) tle->expr, fieldno, levelsup, context); @@ -4008,7 +4094,7 @@ get_name_for_var_field(Var *var, int fieldno, elog(ERROR, "bogus varattno for INNER_VAR var: %d", var->varattno); Assert(netlevelsup == 0); - push_child_plan(dpns, dpns->inner_planstate, &save_dpns); + push_child_plan(dpns, dpns->inner_plan, &save_dpns); result = get_name_for_var_field((Var *) tle->expr, fieldno, levelsup, context); @@ -4118,7 +4204,7 @@ get_name_for_var_field(Var *var, int fieldno, deparse_namespace save_dpns; const char *result; - if (!dpns->inner_planstate) + if (!dpns->inner_plan) elog(ERROR, "failed to find plan for subquery %s", rte->eref->aliasname); tle = get_tle_by_resno(dpns->inner_tlist, attnum); @@ -4126,7 +4212,7 @@ get_name_for_var_field(Var *var, int fieldno, elog(ERROR, "bogus varattno for subquery var: %d", attnum); Assert(netlevelsup == 0); - push_child_plan(dpns, dpns->inner_planstate, &save_dpns); + push_child_plan(dpns, dpns->inner_plan, &save_dpns); result = get_name_for_var_field((Var *) tle->expr, fieldno, levelsup, context); @@ -4236,7 +4322,7 @@ get_name_for_var_field(Var *var, int fieldno, deparse_namespace save_dpns; const char *result; - if (!dpns->inner_planstate) + if (!dpns->inner_plan) elog(ERROR, "failed to find plan for CTE %s", rte->eref->aliasname); tle = get_tle_by_resno(dpns->inner_tlist, attnum); @@ -4244,7 +4330,7 @@ get_name_for_var_field(Var *var, int fieldno, elog(ERROR, "bogus varattno for subquery var: %d", attnum); Assert(netlevelsup == 0); - push_child_plan(dpns, dpns->inner_planstate, &save_dpns); + push_child_plan(dpns, dpns->inner_plan, &save_dpns); result = get_name_for_var_field((Var *) tle->expr, fieldno, levelsup, context); @@ -4285,22 +4371,22 @@ find_param_referent(Param *param, deparse_context *context, /* * If it's a PARAM_EXEC parameter, look for a matching NestLoopParam or * SubPlan argument. This will necessarily be in some ancestor of the - * current expression's PlanState. + * current expression's Plan. */ if (param->paramkind == PARAM_EXEC) { deparse_namespace *dpns; - PlanState *child_ps; + Plan *child_plan; bool in_same_plan_level; ListCell *lc; dpns = (deparse_namespace *) linitial(context->namespaces); - child_ps = dpns->planstate; + child_plan = dpns->plan; in_same_plan_level = true; foreach(lc, dpns->ancestors) { - PlanState *ps = (PlanState *) lfirst(lc); + Node *ancestor = (Node *) lfirst(lc); ListCell *lc2; /* @@ -4308,11 +4394,11 @@ find_param_referent(Param *param, deparse_context *context, * we've crawled up out of a subplan, this couldn't possibly be * the right match. */ - if (IsA(ps, NestLoopState) && - child_ps == innerPlanState(ps) && + if (IsA(ancestor, NestLoop) && + child_plan == innerPlan(ancestor) && in_same_plan_level) { - NestLoop *nl = (NestLoop *) ps->plan; + NestLoop *nl = (NestLoop *) ancestor; foreach(lc2, nl->nestParams) { @@ -4331,16 +4417,12 @@ find_param_referent(Param *param, deparse_context *context, /* * Check to see if we're crawling up from a subplan. */ - foreach(lc2, ps->subPlan) + if(IsA(ancestor, SubPlan)) { - SubPlanState *sstate = (SubPlanState *) lfirst(lc2); - SubPlan *subplan = sstate->subplan; + SubPlan *subplan = (SubPlan *) ancestor; ListCell *lc3; ListCell *lc4; - if (child_ps != sstate->planstate) - continue; - /* Matched subplan, so check its arguments */ forboth(lc3, subplan->parParam, lc4, subplan->args) { @@ -4349,41 +4431,61 @@ find_param_referent(Param *param, deparse_context *context, if (paramid == param->paramid) { - /* Found a match, so return it */ - *dpns_p = dpns; - *ancestor_cell_p = lc; - return arg; + /* + * Found a match, so return it. But, since Vars in + * the arg are to be evaluated in the surrounding + * context, we have to point to the next ancestor item + * that is *not* a SubPlan. + */ + ListCell *rest; + + for_each_cell(rest, dpns->ancestors, + lnext(dpns->ancestors, lc)) + { + Node *ancestor2 = (Node *) lfirst(rest); + + if (!IsA(ancestor2, SubPlan)) + { + *dpns_p = dpns; + *ancestor_cell_p = rest; + return arg; + } + } + elog(ERROR, "SubPlan cannot be outermost ancestor"); } } - /* Keep looking, but we are emerging from a subplan. */ + /* We have emerged from a subplan. */ in_same_plan_level = false; - break; + + /* SubPlan isn't a kind of Plan, so skip the rest */ + continue; } /* - * Likewise check to see if we're emerging from an initplan. - * Initplans never have any parParams, so no need to search that - * list, but we need to know if we should reset + * Check to see if we're emerging from an initplan of the current + * ancestor plan. Initplans never have any parParams, so no need + * to search that list, but we need to know if we should reset * in_same_plan_level. */ - foreach(lc2, ps->initPlan) + foreach(lc2, ((Plan *) ancestor)->initPlan) { - SubPlanState *sstate = (SubPlanState *) lfirst(lc2); + SubPlan *subplan = castNode(SubPlan, lfirst(lc2)); - if (child_ps != sstate->planstate) + if (child_plan != (Plan *) list_nth(dpns->subplans, + subplan->plan_id - 1)) continue; /* No parameters to be had here. */ - Assert(sstate->subplan->parParam == NIL); + Assert(subplan->parParam == NIL); - /* Keep looking, but we are emerging from an initplan. */ + /* We have emerged from an initplan. */ in_same_plan_level = false; break; } /* No luck, crawl up to next ancestor */ - child_ps = ps; + child_plan = (Plan *) ancestor; } } @@ -6247,11 +6349,13 @@ get_agg_expr(Aggref *aggref, deparse_context *context, */ if (DO_AGGSPLIT_COMBINE(aggref->aggsplit)) { - TargetEntry *tle = linitial_node(TargetEntry, aggref->args); + TargetEntry *tle; + Assert(list_length(aggref->args) == 1); - resolve_special_varno((Node *) tle->expr, context, original_aggref, - get_agg_combine_expr); + tle = linitial_node(TargetEntry, aggref->args); + resolve_special_varno((Node *) tle->expr, context, + get_agg_combine_expr, original_aggref); return; } @@ -6336,10 +6440,10 @@ get_agg_expr(Aggref *aggref, deparse_context *context, * Aggref and then calls this. */ static void -get_agg_combine_expr(Node *node, deparse_context *context, void *private) +get_agg_combine_expr(Node *node, deparse_context *context, void *callback_arg) { Aggref *aggref; - Aggref *original_aggref = private; + Aggref *original_aggref = callback_arg; if (!IsA(node, Aggref)) elog(ERROR, "combining Aggref does not point to an Aggref"); @@ -7102,7 +7206,7 @@ get_from_clause_item(Node *jtnode, Query *query, deparse_context *context) RangeTblFunction *rtfunc = (RangeTblFunction *) lfirst(lc); List *args = ((FuncExpr *) rtfunc->funcexpr)->args; - allargs = list_concat(allargs, list_copy(args)); + allargs = list_concat(allargs, args); } appendStringInfoString(buf, "UNNEST("); @@ -7628,7 +7732,7 @@ printSubscripts(SubscriptingRef *sbsref, deparse_context *context) /* If subexpression is NULL, get_rule_expr prints nothing */ get_rule_expr((Node *) lfirst(lowlist_item), context, false); appendStringInfoChar(buf, ':'); - lowlist_item = lnext(sbsref->refupperindexpr, lowlist_item); + lowlist_item = lnext(sbsref->reflowerindexpr, lowlist_item); } /* If subexpression is NULL, get_rule_expr prints nothing */ get_rule_expr((Node *) lfirst(uplist_item), context, false); From 0819b796318fdd88749cbe095f40beb12a03b633 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Mon, 18 May 2020 11:50:22 +0300 Subject: [PATCH 04/52] introduce list compat macros Pass the list to lnext API lnext API now expects the list as well. The commit on Postgres that introduced the change: 1cff1b95ab6ddae32faa3efe0d95a820dbfdc164 lnext_compat and list_delete_cell_compat macros are introduced so that we can use these macros in the codebase without having to use #if directives in the codebase. Related commit on postgres: 1cff1b95ab6ddae32faa3efe0d95a820dbfdc164 Command to search in postgres: git log --all --grep="list_delete_cell" add ListCellAndListWrapper When iterating a list in separate function calls, we need both the list and the current cell starting from PG13, therefore ListCellAndListWrapper is added to store both as a wrapper. Use ListCellAndListWrapper in foreign key test udfs As we iterate a list in these udfs using a functionContext, we need to use the wrapper to be able to access both the list and the current cell. --- src/backend/distributed/commands/multi_copy.c | 10 +++-- .../distributed/operations/node_protocol.c | 15 +++++--- .../planner/intermediate_result_pruning.c | 10 +++-- .../test/foreign_key_relationship_query.c | 38 +++++++++++-------- src/include/distributed/listutils.h | 18 +++++++-- src/include/distributed/version_compat.h | 7 ++++ 6 files changed, 65 insertions(+), 33 deletions(-) diff --git a/src/backend/distributed/commands/multi_copy.c b/src/backend/distributed/commands/multi_copy.c index f8580a6b9..e1399e4e9 100644 --- a/src/backend/distributed/commands/multi_copy.c +++ b/src/backend/distributed/commands/multi_copy.c @@ -753,18 +753,20 @@ static List * RemoveOptionFromList(List *optionList, char *optionName) { ListCell *optionCell = NULL; + #if PG_VERSION_NUM < PG_VERSION_13 ListCell *previousCell = NULL; - + #endif foreach(optionCell, optionList) { DefElem *option = (DefElem *) lfirst(optionCell); if (strncmp(option->defname, optionName, NAMEDATALEN) == 0) { - return list_delete_cell(optionList, optionCell, previousCell); + return list_delete_cell_compat(optionList, optionCell, previousCell); } - + #if PG_VERSION_NUM < PG_VERSION_13 previousCell = optionCell; + #endif } return optionList; @@ -1423,7 +1425,7 @@ ColumnCoercionPaths(TupleDesc destTupleDescriptor, TupleDesc inputTupleDescripto ConversionPathForTypes(inputTupleType, destTupleType, &coercePaths[columnIndex]); - currentColumnName = lnext(currentColumnName); + currentColumnName = lnext_compat(columnNameList, currentColumnName); if (currentColumnName == NULL) { diff --git a/src/backend/distributed/operations/node_protocol.c b/src/backend/distributed/operations/node_protocol.c index 552c439ca..e3afea866 100644 --- a/src/backend/distributed/operations/node_protocol.c +++ b/src/backend/distributed/operations/node_protocol.c @@ -75,6 +75,7 @@ int NextPlacementId = 0; static List * GetTableReplicaIdentityCommand(Oid relationId); static Datum WorkerNodeGetDatum(WorkerNode *workerNode, TupleDesc tupleDescriptor); + /* exports for SQL callable functions */ PG_FUNCTION_INFO_V1(master_get_table_metadata); PG_FUNCTION_INFO_V1(master_get_table_ddl_events); @@ -221,8 +222,10 @@ master_get_table_ddl_events(PG_FUNCTION_ARGS) /* allocate DDL statements, and then save position in DDL statements */ List *tableDDLEventList = GetTableDDLEvents(relationId, includeSequenceDefaults); tableDDLEventCell = list_head(tableDDLEventList); - - functionContext->user_fctx = tableDDLEventCell; + ListCellAndListWrapper* wrapper = palloc0(sizeof(ListCellAndListWrapper)); + wrapper->list = tableDDLEventList; + wrapper->listCell = tableDDLEventCell; + functionContext->user_fctx = wrapper; MemoryContextSwitchTo(oldContext); } @@ -235,13 +238,13 @@ master_get_table_ddl_events(PG_FUNCTION_ARGS) */ functionContext = SRF_PERCALL_SETUP(); - tableDDLEventCell = (ListCell *) functionContext->user_fctx; - if (tableDDLEventCell != NULL) + ListCellAndListWrapper* wrapper = (ListCellAndListWrapper *) functionContext->user_fctx; + if (wrapper->listCell != NULL) { - char *ddlStatement = (char *) lfirst(tableDDLEventCell); + char *ddlStatement = (char *) lfirst(wrapper->listCell); text *ddlStatementText = cstring_to_text(ddlStatement); - functionContext->user_fctx = lnext(tableDDLEventCell); + wrapper->listCell = lnext_compat(wrapper->list, wrapper->listCell); SRF_RETURN_NEXT(functionContext, PointerGetDatum(ddlStatementText)); } diff --git a/src/backend/distributed/planner/intermediate_result_pruning.c b/src/backend/distributed/planner/intermediate_result_pruning.c index 2156e3469..4b7aa5a87 100644 --- a/src/backend/distributed/planner/intermediate_result_pruning.c +++ b/src/backend/distributed/planner/intermediate_result_pruning.c @@ -370,18 +370,20 @@ RemoveLocalNodeFromWorkerList(List *workerNodeList) int32 localGroupId = GetLocalGroupId(); ListCell *workerNodeCell = NULL; + #if PG_VERSION_NUM < PG_VERSION_13 ListCell *prev = NULL; + #endif foreach(workerNodeCell, workerNodeList) { WorkerNode *workerNode = (WorkerNode *) lfirst(workerNodeCell); if (workerNode->groupId == localGroupId) { - return list_delete_cell(workerNodeList, workerNodeCell, prev); + return list_delete_cell_compat(workerNodeList, workerNodeCell, prev); } - - prev = workerNodeCell; } - + #if PG_VERSION_NUM < PG_VERSION_13 + prev = workerNodeCell; + #endif return workerNodeList; } diff --git a/src/backend/distributed/test/foreign_key_relationship_query.c b/src/backend/distributed/test/foreign_key_relationship_query.c index d2cb207d4..d93938fa5 100644 --- a/src/backend/distributed/test/foreign_key_relationship_query.c +++ b/src/backend/distributed/test/foreign_key_relationship_query.c @@ -14,8 +14,9 @@ #include "fmgr.h" #include "funcapi.h" +#include "distributed/listutils.h" #include "distributed/metadata_cache.h" - +#include "distributed/version_compat.h" /* these functions are only exported in the regression tests */ PG_FUNCTION_INFO_V1(get_referencing_relation_id_list); @@ -47,10 +48,12 @@ get_referencing_relation_id_list(PG_FUNCTION_ARGS) MemoryContextSwitchTo(functionContext->multi_call_memory_ctx); List *refList = list_copy( cacheEntry->referencingRelationsViaForeignKey); - MemoryContextSwitchTo(oldContext); - + ListCellAndListWrapper* wrapper = palloc0(sizeof(ListCellAndListWrapper)); foreignRelationCell = list_head(refList); - functionContext->user_fctx = foreignRelationCell; + wrapper->list = refList; + wrapper->listCell = foreignRelationCell; + functionContext->user_fctx = wrapper; + MemoryContextSwitchTo(oldContext); } /* @@ -61,12 +64,12 @@ get_referencing_relation_id_list(PG_FUNCTION_ARGS) */ functionContext = SRF_PERCALL_SETUP(); - foreignRelationCell = (ListCell *) functionContext->user_fctx; - if (foreignRelationCell != NULL) + ListCellAndListWrapper* wrapper = (ListCellAndListWrapper *) functionContext->user_fctx; + if (wrapper->listCell != NULL) { - Oid refId = lfirst_oid(foreignRelationCell); + Oid refId = lfirst_oid(wrapper->listCell); - functionContext->user_fctx = lnext(foreignRelationCell); + wrapper->listCell = lnext_compat(wrapper->list, wrapper->listCell); SRF_RETURN_NEXT(functionContext, PointerGetDatum(refId)); } @@ -102,10 +105,12 @@ get_referenced_relation_id_list(PG_FUNCTION_ARGS) MemoryContext oldContext = MemoryContextSwitchTo(functionContext->multi_call_memory_ctx); List *refList = list_copy(cacheEntry->referencedRelationsViaForeignKey); - MemoryContextSwitchTo(oldContext); - foreignRelationCell = list_head(refList); - functionContext->user_fctx = foreignRelationCell; + ListCellAndListWrapper* wrapper = palloc0(sizeof(ListCellAndListWrapper)); + wrapper->list = refList; + wrapper->listCell = foreignRelationCell; + functionContext->user_fctx = wrapper; + MemoryContextSwitchTo(oldContext); } /* @@ -116,12 +121,13 @@ get_referenced_relation_id_list(PG_FUNCTION_ARGS) */ functionContext = SRF_PERCALL_SETUP(); - foreignRelationCell = (ListCell *) functionContext->user_fctx; - if (foreignRelationCell != NULL) - { - Oid refId = lfirst_oid(foreignRelationCell); + ListCellAndListWrapper* wrapper = (ListCellAndListWrapper *) functionContext->user_fctx; - functionContext->user_fctx = lnext(foreignRelationCell); + if (wrapper->listCell != NULL) + { + Oid refId = lfirst_oid(wrapper->listCell); + + wrapper->listCell = lnext_compat(wrapper->list, wrapper->listCell); SRF_RETURN_NEXT(functionContext, PointerGetDatum(refId)); } diff --git a/src/include/distributed/listutils.h b/src/include/distributed/listutils.h index 26819f00a..e37addbe1 100644 --- a/src/include/distributed/listutils.h +++ b/src/include/distributed/listutils.h @@ -18,8 +18,20 @@ #include "nodes/pg_list.h" #include "utils/array.h" #include "utils/hsearch.h" +#include "distributed/version_compat.h" +/* + * ListCellAndListWrapper stores a list and list cell. + * This struct is used for functionContext. When iterating a list + * in separate function calls, we need both the list and the current cell. + * Therefore this wrapper stores both of them. + */ +typedef struct ListCellAndListWrapper { + List *list; + ListCell *listCell; +} ListCellAndListWrapper; + /* * foreach_ptr - * a convenience macro which loops through a pointer list without needing a @@ -39,7 +51,7 @@ for (ListCell *(var ## CellDoNotUse) = list_head(l); \ (var ## CellDoNotUse) != NULL && \ (((var) = lfirst(var ## CellDoNotUse)) || true); \ - var ## CellDoNotUse = lnext(var ## CellDoNotUse)) + var ## CellDoNotUse = lnext_compat(l, var ## CellDoNotUse)) /* @@ -52,7 +64,7 @@ for (ListCell *(var ## CellDoNotUse) = list_head(l); \ (var ## CellDoNotUse) != NULL && \ (((var) = lfirst_int(var ## CellDoNotUse)) || true); \ - var ## CellDoNotUse = lnext(var ## CellDoNotUse)) + var ## CellDoNotUse = lnext_compat(l, var ## CellDoNotUse)) /* @@ -65,7 +77,7 @@ for (ListCell *(var ## CellDoNotUse) = list_head(l); \ (var ## CellDoNotUse) != NULL && \ (((var) = lfirst_oid(var ## CellDoNotUse)) || true); \ - var ## CellDoNotUse = lnext(var ## CellDoNotUse)) + var ## CellDoNotUse = lnext_compat(l, var ## CellDoNotUse)) /* utility functions declaration shared within this module */ diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index fdee6498b..a1fc4dfcc 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -24,6 +24,13 @@ #include "optimizer/optimizer.h" #endif +#if PG_VERSION_NUM >= PG_VERSION_13 +#define lnext_compat(l, r) lnext(l, r) +#define list_delete_cell_compat(l,c,p) list_delete_cell(l,c) +#else /* pre PG13 */ +#define lnext_compat(l, r) lnext(r) +#define list_delete_cell_compat(l,c,p) list_delete_cell(l,c,p) +#endif #if PG_VERSION_NUM >= PG_VERSION_12 #define MakeSingleTupleTableSlotCompat MakeSingleTupleTableSlot From bf831d2e59325cd507ef48c5efef72f6e3f0cf12 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Mon, 18 May 2020 12:19:29 +0300 Subject: [PATCH 05/52] Use table_openXXX methods in the codebase With PG13 heap_* (heap_open, heap_close etc) are replaced with table_* (table_open, table_close etc). It is better to use the new table access methods in the codebase and define the macros for the previous versions as we can easily remove the macro without having to change the codebase when we drop the support for the old version. Commits that introduced this change on Postgres: f25968c49697db673f6cd2a07b3f7626779f1827 e0c4ec07284db817e1f8d9adfb3fffc952252db0 4b21acf522d751ba5b6679df391d5121b6c4a35f Command to see relevant commits on Postgres side: git log --all --grep="heap_open" --- .../commands/create_distributed_table.c | 12 ++--- .../distributed/commands/foreign_constraint.c | 8 +-- src/backend/distributed/commands/function.c | 9 ++-- src/backend/distributed/commands/index.c | 16 +++--- .../distributed/commands/local_multi_copy.c | 5 +- src/backend/distributed/commands/multi_copy.c | 28 +++++----- src/backend/distributed/commands/role.c | 9 ++-- src/backend/distributed/commands/schema.c | 7 +-- src/backend/distributed/commands/trigger.c | 4 +- src/backend/distributed/commands/type.c | 5 +- .../distributed/deparser/citus_ruleutils.c | 8 +-- .../executor/insert_select_executor.c | 1 + src/backend/distributed/metadata/dependency.c | 9 ++-- src/backend/distributed/metadata/distobject.c | 11 ++-- .../distributed/metadata/metadata_cache.c | 52 +++++++++---------- .../distributed/metadata/metadata_sync.c | 4 +- .../distributed/metadata/metadata_utility.c | 42 +++++++-------- .../distributed/metadata/node_metadata.c | 35 +++++++------ .../distributed/operations/node_protocol.c | 5 +- .../distributed/planner/deparse_shard_query.c | 4 +- .../planner/insert_select_planner.c | 4 +- .../planner/multi_logical_optimizer.c | 4 +- .../distributed/planner/tdigest_extension.c | 4 +- .../transaction/transaction_recovery.c | 8 +-- .../distributed/utils/colocation_utils.c | 35 +++++++------ .../utils/foreign_key_relationship.c | 4 +- .../utils/multi_partitioning_utils.c | 21 ++++---- .../distributed/utils/statistics_collection.c | 4 +- .../worker/worker_merge_protocol.c | 4 +- src/include/distributed/version_compat.h | 4 ++ 30 files changed, 191 insertions(+), 175 deletions(-) diff --git a/src/backend/distributed/commands/create_distributed_table.c b/src/backend/distributed/commands/create_distributed_table.c index ce20f3fd4..8533676bf 100644 --- a/src/backend/distributed/commands/create_distributed_table.c +++ b/src/backend/distributed/commands/create_distributed_table.c @@ -571,7 +571,7 @@ ColocationIdForNewTable(Oid relationId, Var *distributionColumn, */ Assert(distributionMethod == DISTRIBUTE_BY_HASH); - Relation pgDistColocation = heap_open(DistColocationRelationId(), ExclusiveLock); + Relation pgDistColocation = table_open(DistColocationRelationId(), ExclusiveLock); Oid distributionColumnType = distributionColumn->vartype; Oid distributionColumnCollation = get_typcollation(distributionColumnType); @@ -618,12 +618,12 @@ ColocationIdForNewTable(Oid relationId, Var *distributionColumn, if (createdColocationGroup) { /* keep the exclusive lock */ - heap_close(pgDistColocation, NoLock); + table_close(pgDistColocation, NoLock); } else { /* release the exclusive lock */ - heap_close(pgDistColocation, ExclusiveLock); + table_close(pgDistColocation, ExclusiveLock); } } @@ -1266,7 +1266,7 @@ static void CopyLocalDataIntoShards(Oid distributedRelationId) { /* take an ExclusiveLock to block all operations except SELECT */ - Relation distributedRelation = heap_open(distributedRelationId, ExclusiveLock); + Relation distributedRelation = table_open(distributedRelationId, ExclusiveLock); /* * Skip copying from partitioned tables, we will copy the data from @@ -1274,7 +1274,7 @@ CopyLocalDataIntoShards(Oid distributedRelationId) */ if (PartitionedTable(distributedRelationId)) { - heap_close(distributedRelation, NoLock); + table_close(distributedRelation, NoLock); return; } @@ -1330,7 +1330,7 @@ CopyLocalDataIntoShards(Oid distributedRelationId) /* free memory and close the relation */ ExecDropSingleTupleTableSlot(slot); FreeExecutorState(estate); - heap_close(distributedRelation, NoLock); + table_close(distributedRelation, NoLock); PopActiveSnapshot(); } diff --git a/src/backend/distributed/commands/foreign_constraint.c b/src/backend/distributed/commands/foreign_constraint.c index 90cf3c234..b7aea3576 100644 --- a/src/backend/distributed/commands/foreign_constraint.c +++ b/src/backend/distributed/commands/foreign_constraint.c @@ -388,7 +388,7 @@ ColumnAppearsInForeignKeyToReferenceTable(char *columnName, Oid relationId) int scanKeyCount = 1; bool foreignKeyToReferenceTableIncludesGivenColumn = false; - Relation pgConstraint = heap_open(ConstraintRelationId, AccessShareLock); + Relation pgConstraint = table_open(ConstraintRelationId, AccessShareLock); ScanKeyInit(&scanKey[0], Anum_pg_constraint_contype, BTEqualStrategyNumber, F_CHAREQ, CharGetDatum(CONSTRAINT_FOREIGN)); @@ -446,7 +446,7 @@ ColumnAppearsInForeignKeyToReferenceTable(char *columnName, Oid relationId) /* clean up scan and close system catalog */ systable_endscan(scanDescriptor); - heap_close(pgConstraint, NoLock); + table_close(pgConstraint, NoLock); return foreignKeyToReferenceTableIncludesGivenColumn; } @@ -720,7 +720,7 @@ GetForeignKeyOids(Oid relationId, int flags) ScanKeyData scanKey[1]; int scanKeyCount = 1; - Relation pgConstraint = heap_open(ConstraintRelationId, AccessShareLock); + Relation pgConstraint = table_open(ConstraintRelationId, AccessShareLock); ScanKeyInit(&scanKey[0], pgConstraintTargetAttrNumber, BTEqualStrategyNumber, F_OIDEQ, relationId); SysScanDesc scanDescriptor = systable_beginscan(pgConstraint, indexOid, useIndex, @@ -770,7 +770,7 @@ GetForeignKeyOids(Oid relationId, int flags) * on pg_constraint to make sure that caller will process valid foreign key * constraints through the transaction. */ - heap_close(pgConstraint, NoLock); + table_close(pgConstraint, NoLock); return foreignKeyOids; } diff --git a/src/backend/distributed/commands/function.c b/src/backend/distributed/commands/function.c index 1164114a1..c27d8f671 100644 --- a/src/backend/distributed/commands/function.c +++ b/src/backend/distributed/commands/function.c @@ -47,6 +47,7 @@ #include "distributed/multi_executor.h" #include "distributed/namespace_utils.h" #include "distributed/relation_access_tracking.h" +#include "distributed/version_compat.h" #include "distributed/worker_create_or_replace.h" #include "distributed/worker_transaction.h" #include "nodes/makefuncs.h" @@ -352,7 +353,7 @@ GetFunctionColocationId(Oid functionOid, char *colocateWithTableName, Oid distributionArgumentOid) { int colocationId = INVALID_COLOCATION_ID; - Relation pgDistColocation = heap_open(DistColocationRelationId(), ShareLock); + Relation pgDistColocation = table_open(DistColocationRelationId(), ShareLock); if (pg_strncasecmp(colocateWithTableName, "default", NAMEDATALEN) == 0) { @@ -400,7 +401,7 @@ GetFunctionColocationId(Oid functionOid, char *colocateWithTableName, } /* keep the lock */ - heap_close(pgDistColocation, NoLock); + table_close(pgDistColocation, NoLock); return colocationId; } @@ -489,7 +490,7 @@ UpdateFunctionDistributionInfo(const ObjectAddress *distAddress, bool isnull[Natts_pg_dist_object]; bool replace[Natts_pg_dist_object]; - Relation pgDistObjectRel = heap_open(DistObjectRelationId(), RowExclusiveLock); + Relation pgDistObjectRel = table_open(DistObjectRelationId(), RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistObjectRel); /* scan pg_dist_object for classid = $1 AND objid = $2 AND objsubid = $3 via index */ @@ -549,7 +550,7 @@ UpdateFunctionDistributionInfo(const ObjectAddress *distAddress, systable_endscan(scanDescriptor); - heap_close(pgDistObjectRel, NoLock); + table_close(pgDistObjectRel, NoLock); } diff --git a/src/backend/distributed/commands/index.c b/src/backend/distributed/commands/index.c index d7af058c9..e6cf29f72 100644 --- a/src/backend/distributed/commands/index.c +++ b/src/backend/distributed/commands/index.c @@ -138,7 +138,7 @@ PreprocessIndexStmt(Node *node, const char *createIndexCommand) * checked permissions, and will only fail when executing the actual * index statements. */ - Relation relation = heap_openrv(createIndexStatement->relation, lockmode); + Relation relation = table_openrv(createIndexStatement->relation, lockmode); Oid relationId = RelationGetRelid(relation); bool isCitusRelation = IsCitusTable(relationId); @@ -160,7 +160,7 @@ PreprocessIndexStmt(Node *node, const char *createIndexCommand) relationContext, namespaceName); } - heap_close(relation, NoLock); + table_close(relation, NoLock); if (isCitusRelation) { @@ -246,7 +246,7 @@ PreprocessReindexStmt(Node *node, const char *reindexCommand) RangeVarGetRelidExtended(reindexStatement->relation, lockmode, 0, RangeVarCallbackOwnsTable, NULL); - relation = heap_openrv(reindexStatement->relation, NoLock); + relation = table_openrv(reindexStatement->relation, NoLock); relationId = RelationGetRelid(relation); } @@ -275,7 +275,7 @@ PreprocessReindexStmt(Node *node, const char *reindexCommand) } else { - heap_close(relation, NoLock); + table_close(relation, NoLock); } if (isCitusRelation) @@ -426,13 +426,13 @@ PostprocessIndexStmt(Node *node, const char *queryString) StartTransactionCommand(); /* get the affected relation and index */ - Relation relation = heap_openrv(indexStmt->relation, ShareUpdateExclusiveLock); + Relation relation = table_openrv(indexStmt->relation, ShareUpdateExclusiveLock); Oid indexRelationId = get_relname_relid(indexStmt->idxname, schemaId); Relation indexRelation = index_open(indexRelationId, RowExclusiveLock); /* close relations but retain locks */ - heap_close(relation, NoLock); + table_close(relation, NoLock); index_close(indexRelation, NoLock); /* mark index as invalid, in-place (cannot be rolled back) */ @@ -443,7 +443,7 @@ PostprocessIndexStmt(Node *node, const char *queryString) StartTransactionCommand(); /* now, update index's validity in a way that can roll back */ - Relation pg_index = heap_open(IndexRelationId, RowExclusiveLock); + Relation pg_index = table_open(IndexRelationId, RowExclusiveLock); HeapTuple indexTuple = SearchSysCacheCopy1(INDEXRELID, ObjectIdGetDatum( indexRelationId)); @@ -457,7 +457,7 @@ PostprocessIndexStmt(Node *node, const char *queryString) /* clean up; index now marked valid, but ROLLBACK will mark invalid */ heap_freetuple(indexTuple); - heap_close(pg_index, RowExclusiveLock); + table_close(pg_index, RowExclusiveLock); return NIL; } diff --git a/src/backend/distributed/commands/local_multi_copy.c b/src/backend/distributed/commands/local_multi_copy.c index 0f6f619e6..e90426d0f 100644 --- a/src/backend/distributed/commands/local_multi_copy.c +++ b/src/backend/distributed/commands/local_multi_copy.c @@ -34,6 +34,7 @@ #include "distributed/local_executor.h" #include "distributed/local_multi_copy.h" #include "distributed/shard_utils.h" +#include "distributed/version_compat.h" static int ReadFromLocalBufferCallback(void *outBuf, int minRead, int maxRead); static void AddSlotToBuffer(TupleTableSlot *slot, CitusCopyDestReceiver *copyDest, @@ -160,7 +161,7 @@ DoLocalCopy(StringInfo buffer, Oid relationId, int64 shardId, CopyStmt *copyStat LocalCopyBuffer = buffer; Oid shardOid = GetTableLocalShardOid(relationId, shardId); - Relation shard = heap_open(shardOid, RowExclusiveLock); + Relation shard = table_open(shardOid, RowExclusiveLock); ParseState *pState = make_parsestate(NULL); /* p_rtable of pState is set so that we can check constraints. */ @@ -172,7 +173,7 @@ DoLocalCopy(StringInfo buffer, Oid relationId, int64 shardId, CopyStmt *copyStat CopyFrom(cstate); EndCopyFrom(cstate); - heap_close(shard, NoLock); + table_close(shard, NoLock); free_parsestate(pState); } diff --git a/src/backend/distributed/commands/multi_copy.c b/src/backend/distributed/commands/multi_copy.c index e1399e4e9..a8af26a8e 100644 --- a/src/backend/distributed/commands/multi_copy.c +++ b/src/backend/distributed/commands/multi_copy.c @@ -410,7 +410,7 @@ CopyToExistingShards(CopyStmt *copyStatement, char *completionTag) ErrorContextCallback errorCallback; /* allocate column values and nulls arrays */ - Relation distributedRelation = heap_open(tableId, RowExclusiveLock); + Relation distributedRelation = table_open(tableId, RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(distributedRelation); uint32 columnCount = tupleDescriptor->natts; Datum *columnValues = palloc0(columnCount * sizeof(Datum)); @@ -545,7 +545,7 @@ CopyToExistingShards(CopyStmt *copyStatement, char *completionTag) ExecDropSingleTupleTableSlot(tupleTableSlot); FreeExecutorState(executorState); - heap_close(distributedRelation, NoLock); + table_close(distributedRelation, NoLock); /* mark failed placements as inactive */ MarkFailedShardPlacements(); @@ -568,7 +568,7 @@ static void CopyToNewShards(CopyStmt *copyStatement, char *completionTag, Oid relationId) { /* allocate column values and nulls arrays */ - Relation distributedRelation = heap_open(relationId, RowExclusiveLock); + Relation distributedRelation = table_open(relationId, RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(distributedRelation); uint32 columnCount = tupleDescriptor->natts; Datum *columnValues = palloc0(columnCount * sizeof(Datum)); @@ -732,7 +732,7 @@ CopyToNewShards(CopyStmt *copyStatement, char *completionTag, Oid relationId) } EndCopyFrom(copyState); - heap_close(distributedRelation, NoLock); + table_close(distributedRelation, NoLock); /* check for cancellation one last time before returning */ CHECK_FOR_INTERRUPTS(); @@ -2138,7 +2138,7 @@ CitusCopyDestReceiverStartup(DestReceiver *dest, int operation, const char *nullPrintCharacter = "\\N"; /* look up table properties */ - Relation distributedRelation = heap_open(tableId, RowExclusiveLock); + Relation distributedRelation = table_open(tableId, RowExclusiveLock); CitusTableCacheEntry *cacheEntry = GetCitusTableCacheEntry(tableId); partitionMethod = cacheEntry->partitionMethod; @@ -2626,7 +2626,7 @@ CitusCopyDestReceiverShutdown(DestReceiver *destReceiver) } PG_END_TRY(); - heap_close(distributedRelation, NoLock); + table_close(distributedRelation, NoLock); } @@ -2801,7 +2801,7 @@ ProcessCopyStmt(CopyStmt *copyStatement, char *completionTag, const char *queryS bool isFrom = copyStatement->is_from; /* consider using RangeVarGetRelidExtended to check perms before locking */ - Relation copiedRelation = heap_openrv(copyStatement->relation, + Relation copiedRelation = table_openrv(copyStatement->relation, isFrom ? RowExclusiveLock : AccessShareLock); @@ -2816,7 +2816,7 @@ ProcessCopyStmt(CopyStmt *copyStatement, char *completionTag, const char *queryS schemaName = MemoryContextStrdup(relationContext, schemaName); copyStatement->relation->schemaname = schemaName; - heap_close(copiedRelation, NoLock); + table_close(copiedRelation, NoLock); if (isCitusRelation) { @@ -2875,7 +2875,7 @@ CitusCopySelect(CopyStmt *copyStatement) SelectStmt *selectStmt = makeNode(SelectStmt); selectStmt->fromClause = list_make1(copyObject(copyStatement->relation)); - Relation distributedRelation = heap_openrv(copyStatement->relation, AccessShareLock); + Relation distributedRelation = table_openrv(copyStatement->relation, AccessShareLock); TupleDesc tupleDescriptor = RelationGetDescr(distributedRelation); List *targetList = NIL; @@ -2905,7 +2905,7 @@ CitusCopySelect(CopyStmt *copyStatement) targetList = lappend(targetList, selectTarget); } - heap_close(distributedRelation, NoLock); + table_close(distributedRelation, NoLock); selectStmt->targetList = targetList; return selectStmt; @@ -2922,7 +2922,7 @@ CitusCopyTo(CopyStmt *copyStatement, char *completionTag) ListCell *shardIntervalCell = NULL; int64 tuplesSent = 0; - Relation distributedRelation = heap_openrv(copyStatement->relation, AccessShareLock); + Relation distributedRelation = table_openrv(copyStatement->relation, AccessShareLock); Oid relationId = RelationGetRelid(distributedRelation); TupleDesc tupleDescriptor = RelationGetDescr(distributedRelation); @@ -3004,7 +3004,7 @@ CitusCopyTo(CopyStmt *copyStatement, char *completionTag) SendCopyEnd(copyOutState); - heap_close(distributedRelation, AccessShareLock); + table_close(distributedRelation, AccessShareLock); if (completionTag != NULL) { @@ -3079,7 +3079,7 @@ CheckCopyPermissions(CopyStmt *copyStatement) List *attnums; ListCell *cur; - rel = heap_openrv(copyStatement->relation, + rel = table_openrv(copyStatement->relation, is_from ? RowExclusiveLock : AccessShareLock); range_table = CreateRangeTable(rel, required_access); @@ -3105,7 +3105,7 @@ CheckCopyPermissions(CopyStmt *copyStatement) /* TODO: Perform RLS checks once supported */ - heap_close(rel, NoLock); + table_close(rel, NoLock); /* *INDENT-ON* */ } diff --git a/src/backend/distributed/commands/role.c b/src/backend/distributed/commands/role.c index bf0114620..6f655b934 100644 --- a/src/backend/distributed/commands/role.c +++ b/src/backend/distributed/commands/role.c @@ -33,6 +33,7 @@ #include "distributed/coordinator_protocol.h" #include "distributed/metadata/distobject.h" #include "distributed/metadata_sync.h" +#include "distributed/version_compat.h" #include "distributed/worker_transaction.h" #include "miscadmin.h" #include "nodes/makefuncs.h" @@ -315,7 +316,7 @@ CreateCreateOrAlterRoleCommand(const char *roleName, static const char * ExtractEncryptedPassword(Oid roleOid) { - Relation pgAuthId = heap_open(AuthIdRelationId, AccessShareLock); + Relation pgAuthId = table_open(AuthIdRelationId, AccessShareLock); TupleDesc pgAuthIdDescription = RelationGetDescr(pgAuthId); HeapTuple tuple = SearchSysCache1(AUTHOID, roleOid); bool isNull = true; @@ -328,7 +329,7 @@ ExtractEncryptedPassword(Oid roleOid) Datum passwordDatum = heap_getattr(tuple, Anum_pg_authid_rolpassword, pgAuthIdDescription, &isNull); - heap_close(pgAuthId, AccessShareLock); + table_close(pgAuthId, AccessShareLock); ReleaseSysCache(tuple); if (isNull) @@ -527,7 +528,7 @@ GenerateCreateOrAlterRoleCommand(Oid roleOid) List * GenerateAlterRoleSetCommandForRole(Oid roleid) { - Relation DbRoleSetting = heap_open(DbRoleSettingRelationId, AccessShareLock); + Relation DbRoleSetting = table_open(DbRoleSettingRelationId, AccessShareLock); TupleDesc DbRoleSettingDescription = RelationGetDescr(DbRoleSetting); HeapTuple tuple = NULL; List *commands = NIL; @@ -561,7 +562,7 @@ GenerateAlterRoleSetCommandForRole(Oid roleid) } heap_endscan(scan); - heap_close(DbRoleSetting, AccessShareLock); + table_close(DbRoleSetting, AccessShareLock); return commands; } diff --git a/src/backend/distributed/commands/schema.c b/src/backend/distributed/commands/schema.c index 3c1cd6388..c8d772ae0 100644 --- a/src/backend/distributed/commands/schema.c +++ b/src/backend/distributed/commands/schema.c @@ -30,6 +30,7 @@ #include "distributed/resource_lock.h" #include #include +#include "distributed/version_compat.h" #include "nodes/parsenodes.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" @@ -71,7 +72,7 @@ PreprocessDropSchemaStmt(Node *node, const char *queryString) continue; } - pgClass = heap_open(RelationRelationId, AccessShareLock); + pgClass = table_open(RelationRelationId, AccessShareLock); ScanKeyInit(&scanKey[0], Anum_pg_class_relnamespace, BTEqualStrategyNumber, F_OIDEQ, namespaceOid); @@ -105,7 +106,7 @@ PreprocessDropSchemaStmt(Node *node, const char *queryString) MarkInvalidateForeignKeyGraph(); systable_endscan(scanDescriptor); - heap_close(pgClass, NoLock); + table_close(pgClass, NoLock); return NIL; } @@ -113,7 +114,7 @@ PreprocessDropSchemaStmt(Node *node, const char *queryString) } systable_endscan(scanDescriptor); - heap_close(pgClass, NoLock); + table_close(pgClass, NoLock); } return NIL; diff --git a/src/backend/distributed/commands/trigger.c b/src/backend/distributed/commands/trigger.c index 098afc8a7..6051fbc2d 100644 --- a/src/backend/distributed/commands/trigger.c +++ b/src/backend/distributed/commands/trigger.c @@ -70,7 +70,7 @@ GetExplicitTriggerIdList(Oid relationId) { List *triggerIdList = NIL; - Relation pgTrigger = heap_open(TriggerRelationId, AccessShareLock); + Relation pgTrigger = table_open(TriggerRelationId, AccessShareLock); int scanKeyCount = 1; ScanKeyData scanKey[1]; @@ -103,7 +103,7 @@ GetExplicitTriggerIdList(Oid relationId) } systable_endscan(scanDescriptor); - heap_close(pgTrigger, NoLock); + table_close(pgTrigger, NoLock); return triggerIdList; } diff --git a/src/backend/distributed/commands/type.c b/src/backend/distributed/commands/type.c index 5d8b341e1..09042130f 100644 --- a/src/backend/distributed/commands/type.c +++ b/src/backend/distributed/commands/type.c @@ -64,6 +64,7 @@ #include "distributed/remote_commands.h" #include "distributed/transaction_management.h" #include "distributed/worker_create_or_replace.h" +#include "distributed/version_compat.h" #include "distributed/worker_manager.h" #include "distributed/worker_transaction.h" #include "miscadmin.h" @@ -791,7 +792,7 @@ EnumValsList(Oid typeOid) BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(typeOid)); - Relation enum_rel = heap_open(EnumRelationId, AccessShareLock); + Relation enum_rel = table_open(EnumRelationId, AccessShareLock); SysScanDesc enum_scan = systable_beginscan(enum_rel, EnumTypIdSortOrderIndexId, true, NULL, @@ -805,7 +806,7 @@ EnumValsList(Oid typeOid) } systable_endscan(enum_scan); - heap_close(enum_rel, AccessShareLock); + table_close(enum_rel, AccessShareLock); return vals; } diff --git a/src/backend/distributed/deparser/citus_ruleutils.c b/src/backend/distributed/deparser/citus_ruleutils.c index 3bef041e8..9f5b0f905 100644 --- a/src/backend/distributed/deparser/citus_ruleutils.c +++ b/src/backend/distributed/deparser/citus_ruleutils.c @@ -128,7 +128,7 @@ get_extension_schema(Oid ext_oid) HeapTuple tuple; ScanKeyData entry[1]; - rel = heap_open(ExtensionRelationId, AccessShareLock); + rel = table_open(ExtensionRelationId, AccessShareLock); ScanKeyInit(&entry[0], #if PG_VERSION_NUM >= PG_VERSION_12 @@ -152,7 +152,7 @@ get_extension_schema(Oid ext_oid) systable_endscan(scandesc); - heap_close(rel, AccessShareLock); + table_close(rel, AccessShareLock); return result; /* *INDENT-ON* */ @@ -1174,7 +1174,7 @@ pg_get_replica_identity_command(Oid tableRelationId) { StringInfo buf = makeStringInfo(); - Relation relation = heap_open(tableRelationId, AccessShareLock); + Relation relation = table_open(tableRelationId, AccessShareLock); char replicaIdentity = relation->rd_rel->relreplident; @@ -1202,7 +1202,7 @@ pg_get_replica_identity_command(Oid tableRelationId) relationName); } - heap_close(relation, AccessShareLock); + table_close(relation, AccessShareLock); return (buf->len > 0) ? buf->data : NULL; } diff --git a/src/backend/distributed/executor/insert_select_executor.c b/src/backend/distributed/executor/insert_select_executor.c index 3d4dbf3b0..87cdbad10 100644 --- a/src/backend/distributed/executor/insert_select_executor.c +++ b/src/backend/distributed/executor/insert_select_executor.c @@ -34,6 +34,7 @@ #include "distributed/shardinterval_utils.h" #include "distributed/subplan_execution.h" #include "distributed/transaction_management.h" +#include "distributed/version_compat.h" #include "executor/executor.h" #include "nodes/execnodes.h" #include "nodes/makefuncs.h" diff --git a/src/backend/distributed/metadata/dependency.c b/src/backend/distributed/metadata/dependency.c index f08ef494c..d87e93cfd 100644 --- a/src/backend/distributed/metadata/dependency.c +++ b/src/backend/distributed/metadata/dependency.c @@ -25,6 +25,7 @@ #include "distributed/metadata/dependency.h" #include "distributed/metadata/distobject.h" #include "distributed/metadata_cache.h" +#include "distributed/version_compat.h" #include "miscadmin.h" #include "utils/fmgroids.h" #include "utils/hsearch.h" @@ -304,7 +305,7 @@ DependencyDefinitionFromPgDepend(ObjectAddress target) /* * iterate the actual pg_depend catalog */ - Relation depRel = heap_open(DependRelationId, AccessShareLock); + Relation depRel = table_open(DependRelationId, AccessShareLock); /* scan pg_depend for classid = $1 AND objid = $2 using pg_depend_depender_index */ ScanKeyInit(&key[0], Anum_pg_depend_classid, BTEqualStrategyNumber, F_OIDEQ, @@ -346,7 +347,7 @@ DependencyDefinitionFromPgShDepend(ObjectAddress target) /* * iterate the actual pg_shdepend catalog */ - Relation shdepRel = heap_open(SharedDependRelationId, AccessShareLock); + Relation shdepRel = table_open(SharedDependRelationId, AccessShareLock); /* * Scan pg_shdepend for dbid = $1 AND classid = $2 AND objid = $3 using @@ -621,7 +622,7 @@ IsObjectAddressOwnedByExtension(const ObjectAddress *target, HeapTuple depTup = NULL; bool result = false; - Relation depRel = heap_open(DependRelationId, AccessShareLock); + Relation depRel = table_open(DependRelationId, AccessShareLock); /* scan pg_depend for classid = $1 AND objid = $2 using pg_depend_depender_index */ ScanKeyInit(&key[0], Anum_pg_depend_classid, BTEqualStrategyNumber, F_OIDEQ, @@ -647,7 +648,7 @@ IsObjectAddressOwnedByExtension(const ObjectAddress *target, } systable_endscan(depScan); - heap_close(depRel, AccessShareLock); + table_close(depRel, AccessShareLock); return result; } diff --git a/src/backend/distributed/metadata/distobject.c b/src/backend/distributed/metadata/distobject.c index 052032fd0..86823c62e 100644 --- a/src/backend/distributed/metadata/distobject.c +++ b/src/backend/distributed/metadata/distobject.c @@ -31,6 +31,7 @@ #include "distributed/metadata/distobject.h" #include "distributed/metadata/pg_dist_object.h" #include "distributed/metadata_cache.h" +#include "distributed/version_compat.h" #include "executor/spi.h" #include "nodes/makefuncs.h" #include "nodes/pg_list.h" @@ -103,7 +104,7 @@ ObjectExists(const ObjectAddress *address) if (is_objectclass_supported(address->classId)) { HeapTuple objtup; - Relation catalog = heap_open(address->classId, AccessShareLock); + Relation catalog = table_open(address->classId, AccessShareLock); #if PG_VERSION_NUM >= PG_VERSION_12 objtup = get_catalog_object_by_oid(catalog, get_object_attnum_oid( @@ -111,7 +112,7 @@ ObjectExists(const ObjectAddress *address) #else objtup = get_catalog_object_by_oid(catalog, address->objectId); #endif - heap_close(catalog, AccessShareLock); + table_close(catalog, AccessShareLock); if (objtup != NULL) { return true; @@ -257,7 +258,7 @@ IsObjectDistributed(const ObjectAddress *address) ScanKeyData key[3]; bool result = false; - Relation pgDistObjectRel = heap_open(DistObjectRelationId(), AccessShareLock); + Relation pgDistObjectRel = table_open(DistObjectRelationId(), AccessShareLock); /* scan pg_dist_object for classid = $1 AND objid = $2 AND objsubid = $3 via index */ ScanKeyInit(&key[0], Anum_pg_dist_object_classid, BTEqualStrategyNumber, F_OIDEQ, @@ -295,7 +296,7 @@ ClusterHasDistributedFunctionWithDistArgument(void) HeapTuple pgDistObjectTup = NULL; - Relation pgDistObjectRel = heap_open(DistObjectRelationId(), AccessShareLock); + Relation pgDistObjectRel = table_open(DistObjectRelationId(), AccessShareLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistObjectRel); @@ -340,7 +341,7 @@ GetDistributedObjectAddressList(void) HeapTuple pgDistObjectTup = NULL; List *objectAddressList = NIL; - Relation pgDistObjectRel = heap_open(DistObjectRelationId(), AccessShareLock); + Relation pgDistObjectRel = table_open(DistObjectRelationId(), AccessShareLock); SysScanDesc pgDistObjectScan = systable_beginscan(pgDistObjectRel, InvalidOid, false, NULL, 0, NULL); diff --git a/src/backend/distributed/metadata/metadata_cache.c b/src/backend/distributed/metadata/metadata_cache.c index 7178467bb..4a541e8ea 100644 --- a/src/backend/distributed/metadata/metadata_cache.c +++ b/src/backend/distributed/metadata/metadata_cache.c @@ -323,7 +323,7 @@ IsCitusTableViaCatalog(Oid relationId) ScanKeyData scanKey[1]; bool indexOK = true; - Relation pgDistPartition = heap_open(DistPartitionRelationId(), AccessShareLock); + Relation pgDistPartition = table_open(DistPartitionRelationId(), AccessShareLock); ScanKeyInit(&scanKey[0], Anum_pg_dist_partition_logicalrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relationId)); @@ -334,7 +334,7 @@ IsCitusTableViaCatalog(Oid relationId) HeapTuple partitionTuple = systable_getnext(scanDescriptor); systable_endscan(scanDescriptor); - heap_close(pgDistPartition, AccessShareLock); + table_close(pgDistPartition, AccessShareLock); return HeapTupleIsValid(partitionTuple); } @@ -1006,7 +1006,7 @@ LookupDistObjectCacheEntry(Oid classid, Oid objid, int32 objsubid) cacheEntry->key.objid = objid; cacheEntry->key.objsubid = objsubid; - Relation pgDistObjectRel = heap_open(DistObjectRelationId(), AccessShareLock); + Relation pgDistObjectRel = table_open(DistObjectRelationId(), AccessShareLock); TupleDesc pgDistObjectTupleDesc = RelationGetDescr(pgDistObjectRel); ScanKeyInit(&pgDistObjectKey[0], Anum_pg_dist_object_classid, @@ -1059,14 +1059,14 @@ LookupDistObjectCacheEntry(Oid classid, Oid objid, int32 objsubid) static CitusTableCacheEntry * BuildCitusTableCacheEntry(Oid relationId) { - Relation pgDistPartition = heap_open(DistPartitionRelationId(), AccessShareLock); + Relation pgDistPartition = table_open(DistPartitionRelationId(), AccessShareLock); HeapTuple distPartitionTuple = LookupDistPartitionTuple(pgDistPartition, relationId); if (distPartitionTuple == NULL) { /* not a distributed table, done */ - heap_close(pgDistPartition, NoLock); + table_close(pgDistPartition, NoLock); return NULL; } @@ -1166,7 +1166,7 @@ BuildCitusTableCacheEntry(Oid relationId) MemoryContextSwitchTo(oldContext); - heap_close(pgDistPartition, NoLock); + table_close(pgDistPartition, NoLock); cacheEntry->isValid = true; @@ -1201,7 +1201,7 @@ BuildCachedShardList(CitusTableCacheEntry *cacheEntry) int shardIntervalArrayLength = list_length(distShardTupleList); if (shardIntervalArrayLength > 0) { - Relation distShardRelation = heap_open(DistShardRelationId(), AccessShareLock); + Relation distShardRelation = table_open(DistShardRelationId(), AccessShareLock); TupleDesc distShardTupleDesc = RelationGetDescr(distShardRelation); int arrayIndex = 0; @@ -1236,7 +1236,7 @@ BuildCachedShardList(CitusTableCacheEntry *cacheEntry) arrayIndex++; } - heap_close(distShardRelation, AccessShareLock); + table_close(distShardRelation, AccessShareLock); } /* look up value comparison function */ @@ -1847,7 +1847,7 @@ InstalledExtensionVersion(void) InitializeCaches(); - Relation relation = heap_open(ExtensionRelationId, AccessShareLock); + Relation relation = table_open(ExtensionRelationId, AccessShareLock); ScanKeyInit(&entry[0], Anum_pg_extension_extname, BTEqualStrategyNumber, F_NAMEEQ, CStringGetDatum("citus")); @@ -1889,7 +1889,7 @@ InstalledExtensionVersion(void) systable_endscan(scandesc); - heap_close(relation, AccessShareLock); + table_close(relation, AccessShareLock); return installedExtensionVersion; } @@ -2400,7 +2400,7 @@ CitusExtensionOwner(void) return MetadataCache.extensionOwner; } - Relation relation = heap_open(ExtensionRelationId, AccessShareLock); + Relation relation = table_open(ExtensionRelationId, AccessShareLock); ScanKeyInit(&entry[0], Anum_pg_extension_extname, @@ -2440,7 +2440,7 @@ CitusExtensionOwner(void) systable_endscan(scandesc); - heap_close(relation, AccessShareLock); + table_close(relation, AccessShareLock); return MetadataCache.extensionOwner; } @@ -3228,7 +3228,7 @@ GetLocalGroupId(void) return 0; } - Relation pgDistLocalGroupId = heap_open(localGroupTableOid, AccessShareLock); + Relation pgDistLocalGroupId = table_open(localGroupTableOid, AccessShareLock); SysScanDesc scanDescriptor = systable_beginscan(pgDistLocalGroupId, InvalidOid, false, @@ -3260,7 +3260,7 @@ GetLocalGroupId(void) } systable_endscan(scanDescriptor); - heap_close(pgDistLocalGroupId, AccessShareLock); + table_close(pgDistLocalGroupId, AccessShareLock); return groupId; } @@ -3671,7 +3671,7 @@ DistTableOidList(void) int scanKeyCount = 0; List *distTableOidList = NIL; - Relation pgDistPartition = heap_open(DistPartitionRelationId(), AccessShareLock); + Relation pgDistPartition = table_open(DistPartitionRelationId(), AccessShareLock); SysScanDesc scanDescriptor = systable_beginscan(pgDistPartition, InvalidOid, false, @@ -3693,7 +3693,7 @@ DistTableOidList(void) } systable_endscan(scanDescriptor); - heap_close(pgDistPartition, AccessShareLock); + table_close(pgDistPartition, AccessShareLock); return distTableOidList; } @@ -3713,7 +3713,7 @@ ReferenceTableOidList() int scanKeyCount = 0; List *referenceTableOidList = NIL; - Relation pgDistPartition = heap_open(DistPartitionRelationId(), AccessShareLock); + Relation pgDistPartition = table_open(DistPartitionRelationId(), AccessShareLock); SysScanDesc scanDescriptor = systable_beginscan(pgDistPartition, InvalidOid, false, @@ -3742,7 +3742,7 @@ ReferenceTableOidList() } systable_endscan(scanDescriptor); - heap_close(pgDistPartition, AccessShareLock); + table_close(pgDistPartition, AccessShareLock); return referenceTableOidList; } @@ -3856,7 +3856,7 @@ LookupDistShardTuples(Oid relationId) List *distShardTupleList = NIL; ScanKeyData scanKey[1]; - Relation pgDistShard = heap_open(DistShardRelationId(), AccessShareLock); + Relation pgDistShard = table_open(DistShardRelationId(), AccessShareLock); /* copy scankey to local copy, it will be modified during the scan */ scanKey[0] = DistShardScanKey[0]; @@ -3878,7 +3878,7 @@ LookupDistShardTuples(Oid relationId) } systable_endscan(scanDescriptor); - heap_close(pgDistShard, AccessShareLock); + table_close(pgDistShard, AccessShareLock); return distShardTupleList; } @@ -3896,7 +3896,7 @@ LookupShardRelationFromCatalog(int64 shardId, bool missingOk) ScanKeyData scanKey[1]; int scanKeyCount = 1; Form_pg_dist_shard shardForm = NULL; - Relation pgDistShard = heap_open(DistShardRelationId(), AccessShareLock); + Relation pgDistShard = table_open(DistShardRelationId(), AccessShareLock); Oid relationId = InvalidOid; ScanKeyInit(&scanKey[0], Anum_pg_dist_shard_shardid, @@ -3924,7 +3924,7 @@ LookupShardRelationFromCatalog(int64 shardId, bool missingOk) } systable_endscan(scanDescriptor); - heap_close(pgDistShard, NoLock); + table_close(pgDistShard, NoLock); return relationId; } @@ -4206,7 +4206,7 @@ CitusInvalidateRelcacheByShardId(int64 shardId) ScanKeyData scanKey[1]; int scanKeyCount = 1; Form_pg_dist_shard shardForm = NULL; - Relation pgDistShard = heap_open(DistShardRelationId(), AccessShareLock); + Relation pgDistShard = table_open(DistShardRelationId(), AccessShareLock); /* * Load shard, to find the associated relation id. Can't use @@ -4249,7 +4249,7 @@ CitusInvalidateRelcacheByShardId(int64 shardId) } systable_endscan(scanDescriptor); - heap_close(pgDistShard, NoLock); + table_close(pgDistShard, NoLock); /* bump command counter, to force invalidation to take effect */ CommandCounterIncrement(); @@ -4274,7 +4274,7 @@ DistNodeMetadata(void) ereport(ERROR, (errmsg("pg_dist_node_metadata was not found"))); } - Relation pgDistNodeMetadata = heap_open(metadataTableOid, AccessShareLock); + Relation pgDistNodeMetadata = table_open(metadataTableOid, AccessShareLock); SysScanDesc scanDescriptor = systable_beginscan(pgDistNodeMetadata, InvalidOid, false, NULL, scanKeyCount, scanKey); @@ -4295,7 +4295,7 @@ DistNodeMetadata(void) } systable_endscan(scanDescriptor); - heap_close(pgDistNodeMetadata, AccessShareLock); + table_close(pgDistNodeMetadata, AccessShareLock); return metadata; } diff --git a/src/backend/distributed/metadata/metadata_sync.c b/src/backend/distributed/metadata/metadata_sync.c index 8b7c84be2..8736264d4 100644 --- a/src/backend/distributed/metadata/metadata_sync.c +++ b/src/backend/distributed/metadata/metadata_sync.c @@ -989,7 +989,7 @@ UpdateDistNodeBoolAttr(const char *nodeName, int32 nodePort, int attrNum, bool v bool isnull[Natts_pg_dist_node]; bool replace[Natts_pg_dist_node]; - Relation pgDistNode = heap_open(DistNodeRelationId(), RowExclusiveLock); + Relation pgDistNode = table_open(DistNodeRelationId(), RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistNode); ScanKeyInit(&scanKey[0], Anum_pg_dist_node_nodename, @@ -1022,7 +1022,7 @@ UpdateDistNodeBoolAttr(const char *nodeName, int32 nodePort, int attrNum, bool v CommandCounterIncrement(); systable_endscan(scanDescriptor); - heap_close(pgDistNode, NoLock); + table_close(pgDistNode, NoLock); } diff --git a/src/backend/distributed/metadata/metadata_utility.c b/src/backend/distributed/metadata/metadata_utility.c index 135f1434f..b3655853b 100644 --- a/src/backend/distributed/metadata/metadata_utility.c +++ b/src/backend/distributed/metadata/metadata_utility.c @@ -190,7 +190,7 @@ DistributedTableSize(Oid relationId, char *sizeQuery) totalRelationSize += relationSizeOnNode; } - heap_close(relation, AccessShareLock); + table_close(relation, AccessShareLock); return totalRelationSize; } @@ -633,7 +633,7 @@ NodeGroupHasShardPlacements(int32 groupId, bool onlyConsiderActivePlacements) ScanKeyData scanKey[2]; - Relation pgPlacement = heap_open(DistPlacementRelationId(), + Relation pgPlacement = table_open(DistPlacementRelationId(), AccessShareLock); ScanKeyInit(&scanKey[0], Anum_pg_dist_placement_groupid, @@ -654,7 +654,7 @@ NodeGroupHasShardPlacements(int32 groupId, bool onlyConsiderActivePlacements) bool hasActivePlacements = HeapTupleIsValid(heapTuple); systable_endscan(scanDescriptor); - heap_close(pgPlacement, NoLock); + table_close(pgPlacement, NoLock); return hasActivePlacements; } @@ -731,7 +731,7 @@ BuildShardPlacementList(ShardInterval *shardInterval) int scanKeyCount = 1; bool indexOK = true; - Relation pgPlacement = heap_open(DistPlacementRelationId(), AccessShareLock); + Relation pgPlacement = table_open(DistPlacementRelationId(), AccessShareLock); ScanKeyInit(&scanKey[0], Anum_pg_dist_placement_shardid, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(shardId)); @@ -755,7 +755,7 @@ BuildShardPlacementList(ShardInterval *shardInterval) } systable_endscan(scanDescriptor); - heap_close(pgPlacement, NoLock); + table_close(pgPlacement, NoLock); return shardPlacementList; } @@ -774,7 +774,7 @@ AllShardPlacementsOnNodeGroup(int32 groupId) int scanKeyCount = 1; bool indexOK = true; - Relation pgPlacement = heap_open(DistPlacementRelationId(), AccessShareLock); + Relation pgPlacement = table_open(DistPlacementRelationId(), AccessShareLock); ScanKeyInit(&scanKey[0], Anum_pg_dist_placement_groupid, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(groupId)); @@ -798,7 +798,7 @@ AllShardPlacementsOnNodeGroup(int32 groupId) } systable_endscan(scanDescriptor); - heap_close(pgPlacement, NoLock); + table_close(pgPlacement, NoLock); return shardPlacementList; } @@ -879,7 +879,7 @@ InsertShardRow(Oid relationId, uint64 shardId, char storageType, } /* open shard relation and insert new tuple */ - Relation pgDistShard = heap_open(DistShardRelationId(), RowExclusiveLock); + Relation pgDistShard = table_open(DistShardRelationId(), RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistShard); HeapTuple heapTuple = heap_form_tuple(tupleDescriptor, values, isNulls); @@ -890,7 +890,7 @@ InsertShardRow(Oid relationId, uint64 shardId, char storageType, CitusInvalidateRelcacheByRelid(relationId); CommandCounterIncrement(); - heap_close(pgDistShard, NoLock); + table_close(pgDistShard, NoLock); } @@ -923,7 +923,7 @@ InsertShardPlacementRow(uint64 shardId, uint64 placementId, values[Anum_pg_dist_placement_groupid - 1] = Int32GetDatum(groupId); /* open shard placement relation and insert new tuple */ - Relation pgDistPlacement = heap_open(DistPlacementRelationId(), RowExclusiveLock); + Relation pgDistPlacement = table_open(DistPlacementRelationId(), RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistPlacement); HeapTuple heapTuple = heap_form_tuple(tupleDescriptor, values, isNulls); @@ -933,7 +933,7 @@ InsertShardPlacementRow(uint64 shardId, uint64 placementId, CitusInvalidateRelcacheByShardId(shardId); CommandCounterIncrement(); - heap_close(pgDistPlacement, NoLock); + table_close(pgDistPlacement, NoLock); return placementId; } @@ -953,7 +953,7 @@ InsertIntoPgDistPartition(Oid relationId, char distributionMethod, bool newNulls[Natts_pg_dist_partition]; /* open system catalog and insert new tuple */ - Relation pgDistPartition = heap_open(DistPartitionRelationId(), RowExclusiveLock); + Relation pgDistPartition = table_open(DistPartitionRelationId(), RowExclusiveLock); /* form new tuple for pg_dist_partition */ memset(newValues, 0, sizeof(newValues)); @@ -991,7 +991,7 @@ InsertIntoPgDistPartition(Oid relationId, char distributionMethod, RecordDistributedRelationDependencies(relationId); CommandCounterIncrement(); - heap_close(pgDistPartition, NoLock); + table_close(pgDistPartition, NoLock); } @@ -1038,7 +1038,7 @@ DeletePartitionRow(Oid distributedRelationId) ScanKeyData scanKey[1]; int scanKeyCount = 1; - Relation pgDistPartition = heap_open(DistPartitionRelationId(), RowExclusiveLock); + Relation pgDistPartition = table_open(DistPartitionRelationId(), RowExclusiveLock); ScanKeyInit(&scanKey[0], Anum_pg_dist_partition_logicalrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(distributedRelationId)); @@ -1064,7 +1064,7 @@ DeletePartitionRow(Oid distributedRelationId) /* increment the counter so that next command can see the row */ CommandCounterIncrement(); - heap_close(pgDistPartition, NoLock); + table_close(pgDistPartition, NoLock); } @@ -1079,7 +1079,7 @@ DeleteShardRow(uint64 shardId) int scanKeyCount = 1; bool indexOK = true; - Relation pgDistShard = heap_open(DistShardRelationId(), RowExclusiveLock); + Relation pgDistShard = table_open(DistShardRelationId(), RowExclusiveLock); ScanKeyInit(&scanKey[0], Anum_pg_dist_shard_shardid, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(shardId)); @@ -1106,7 +1106,7 @@ DeleteShardRow(uint64 shardId) CitusInvalidateRelcacheByRelid(distributedRelationId); CommandCounterIncrement(); - heap_close(pgDistShard, NoLock); + table_close(pgDistShard, NoLock); } @@ -1122,7 +1122,7 @@ DeleteShardPlacementRow(uint64 placementId) bool indexOK = true; bool isNull = false; - Relation pgDistPlacement = heap_open(DistPlacementRelationId(), RowExclusiveLock); + Relation pgDistPlacement = table_open(DistPlacementRelationId(), RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistPlacement); ScanKeyInit(&scanKey[0], Anum_pg_dist_placement_placementid, @@ -1154,7 +1154,7 @@ DeleteShardPlacementRow(uint64 placementId) CitusInvalidateRelcacheByShardId(shardId); CommandCounterIncrement(); - heap_close(pgDistPlacement, NoLock); + table_close(pgDistPlacement, NoLock); } @@ -1251,7 +1251,7 @@ UpdateShardPlacementState(uint64 placementId, char shardState) bool replace[Natts_pg_dist_placement]; bool colIsNull = false; - Relation pgDistPlacement = heap_open(DistPlacementRelationId(), RowExclusiveLock); + Relation pgDistPlacement = table_open(DistPlacementRelationId(), RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistPlacement); ScanKeyInit(&scanKey[0], Anum_pg_dist_placement_placementid, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(placementId)); @@ -1288,7 +1288,7 @@ UpdateShardPlacementState(uint64 placementId, char shardState) CommandCounterIncrement(); systable_endscan(scanDescriptor); - heap_close(pgDistPlacement, NoLock); + table_close(pgDistPlacement, NoLock); } diff --git a/src/backend/distributed/metadata/node_metadata.c b/src/backend/distributed/metadata/node_metadata.c index ed50eabda..f8ed93978 100644 --- a/src/backend/distributed/metadata/node_metadata.c +++ b/src/backend/distributed/metadata/node_metadata.c @@ -41,6 +41,7 @@ #include "distributed/resource_lock.h" #include "distributed/shardinterval_utils.h" #include "distributed/shared_connection_stats.h" +#include "distributed/version_compat.h" #include "distributed/worker_manager.h" #include "distributed/worker_transaction.h" #include "lib/stringinfo.h" @@ -798,7 +799,7 @@ UpdateNodeLocation(int32 nodeId, char *newNodeName, int32 newNodePort) bool isnull[Natts_pg_dist_node]; bool replace[Natts_pg_dist_node]; - Relation pgDistNode = heap_open(DistNodeRelationId(), RowExclusiveLock); + Relation pgDistNode = table_open(DistNodeRelationId(), RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistNode); ScanKeyInit(&scanKey[0], Anum_pg_dist_node_nodeid, @@ -834,7 +835,7 @@ UpdateNodeLocation(int32 nodeId, char *newNodeName, int32 newNodePort) CommandCounterIncrement(); systable_endscan(scanDescriptor); - heap_close(pgDistNode, NoLock); + table_close(pgDistNode, NoLock); } @@ -978,7 +979,7 @@ FindWorkerNodeAnyCluster(const char *nodeName, int32 nodePort) { WorkerNode *workerNode = NULL; - Relation pgDistNode = heap_open(DistNodeRelationId(), AccessShareLock); + Relation pgDistNode = table_open(DistNodeRelationId(), AccessShareLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistNode); HeapTuple heapTuple = GetNodeTuple(nodeName, nodePort); @@ -987,7 +988,7 @@ FindWorkerNodeAnyCluster(const char *nodeName, int32 nodePort) workerNode = TupleToWorkerNode(tupleDescriptor, heapTuple); } - heap_close(pgDistNode, NoLock); + table_close(pgDistNode, NoLock); return workerNode; } @@ -1007,7 +1008,7 @@ ReadDistNode(bool includeNodesFromOtherClusters) int scanKeyCount = 0; List *workerNodeList = NIL; - Relation pgDistNode = heap_open(DistNodeRelationId(), AccessShareLock); + Relation pgDistNode = table_open(DistNodeRelationId(), AccessShareLock); SysScanDesc scanDescriptor = systable_beginscan(pgDistNode, InvalidOid, false, @@ -1031,7 +1032,7 @@ ReadDistNode(bool includeNodesFromOtherClusters) } systable_endscan(scanDescriptor); - heap_close(pgDistNode, NoLock); + table_close(pgDistNode, NoLock); return workerNodeList; } @@ -1208,7 +1209,7 @@ AddNodeMetadata(char *nodeName, int32 nodePort, static WorkerNode * SetWorkerColumn(WorkerNode *workerNode, int columnIndex, Datum value) { - Relation pgDistNode = heap_open(DistNodeRelationId(), RowExclusiveLock); + Relation pgDistNode = table_open(DistNodeRelationId(), RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistNode); HeapTuple heapTuple = GetNodeTuple(workerNode->workerName, workerNode->workerPort); @@ -1261,7 +1262,7 @@ SetWorkerColumn(WorkerNode *workerNode, int columnIndex, Datum value) WorkerNode *newWorkerNode = TupleToWorkerNode(tupleDescriptor, heapTuple); - heap_close(pgDistNode, NoLock); + table_close(pgDistNode, NoLock); /* we also update the column at worker nodes */ SendCommandToWorkersWithMetadata(metadataSyncCommand); @@ -1305,7 +1306,7 @@ SetNodeState(char *nodeName, int nodePort, bool isActive) static HeapTuple GetNodeTuple(const char *nodeName, int32 nodePort) { - Relation pgDistNode = heap_open(DistNodeRelationId(), AccessShareLock); + Relation pgDistNode = table_open(DistNodeRelationId(), AccessShareLock); const int scanKeyCount = 2; const bool indexOK = false; @@ -1326,7 +1327,7 @@ GetNodeTuple(const char *nodeName, int32 nodePort) } systable_endscan(scanDescriptor); - heap_close(pgDistNode, NoLock); + table_close(pgDistNode, NoLock); return nodeTuple; } @@ -1448,7 +1449,7 @@ InsertNodeRow(int nodeid, char *nodeName, int32 nodePort, NodeMetadata *nodeMeta values[Anum_pg_dist_node_shouldhaveshards - 1] = BoolGetDatum( nodeMetadata->shouldHaveShards); - Relation pgDistNode = heap_open(DistNodeRelationId(), RowExclusiveLock); + Relation pgDistNode = table_open(DistNodeRelationId(), RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistNode); HeapTuple heapTuple = heap_form_tuple(tupleDescriptor, values, isNulls); @@ -1461,7 +1462,7 @@ InsertNodeRow(int nodeid, char *nodeName, int32 nodePort, NodeMetadata *nodeMeta CommandCounterIncrement(); /* close relation */ - heap_close(pgDistNode, NoLock); + table_close(pgDistNode, NoLock); } @@ -1475,7 +1476,7 @@ DeleteNodeRow(char *nodeName, int32 nodePort) bool indexOK = false; ScanKeyData scanKey[2]; - Relation pgDistNode = heap_open(DistNodeRelationId(), RowExclusiveLock); + Relation pgDistNode = table_open(DistNodeRelationId(), RowExclusiveLock); /* * simple_heap_delete() expects that the caller has at least an @@ -1510,8 +1511,8 @@ DeleteNodeRow(char *nodeName, int32 nodePort) /* increment the counter so that next command won't see the row */ CommandCounterIncrement(); - heap_close(replicaIndex, AccessShareLock); - heap_close(pgDistNode, NoLock); + table_close(replicaIndex, AccessShareLock); + table_close(pgDistNode, NoLock); } @@ -1628,7 +1629,7 @@ UnsetMetadataSyncedForAll(void) * pg_dist_node in different orders. To protect against deadlock, we * get an exclusive lock here. */ - Relation relation = heap_open(DistNodeRelationId(), ExclusiveLock); + Relation relation = table_open(DistNodeRelationId(), ExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(relation); ScanKeyInit(&scanKey[0], Anum_pg_dist_node_hasmetadata, BTEqualStrategyNumber, F_BOOLEQ, BoolGetDatum(true)); @@ -1676,7 +1677,7 @@ UnsetMetadataSyncedForAll(void) systable_endscan(scanDescriptor); CatalogCloseIndexes(indstate); - heap_close(relation, NoLock); + table_close(relation, NoLock); return updatedAtLeastOne; } diff --git a/src/backend/distributed/operations/node_protocol.c b/src/backend/distributed/operations/node_protocol.c index e3afea866..d5fca40cc 100644 --- a/src/backend/distributed/operations/node_protocol.c +++ b/src/backend/distributed/operations/node_protocol.c @@ -48,6 +48,7 @@ #include "distributed/metadata_sync.h" #include "distributed/namespace_utils.h" #include "distributed/pg_dist_shard.h" +#include "distributed/version_compat.h" #include "distributed/worker_manager.h" #include "foreign/foreign.h" #include "lib/stringinfo.h" @@ -648,7 +649,7 @@ GetTableIndexAndConstraintCommands(Oid relationId) PushOverrideEmptySearchPath(CurrentMemoryContext); /* open system catalog and scan all indexes that belong to this table */ - Relation pgIndex = heap_open(IndexRelationId, AccessShareLock); + Relation pgIndex = table_open(IndexRelationId, AccessShareLock); ScanKeyInit(&scanKey[0], Anum_pg_index_indrelid, BTEqualStrategyNumber, F_OIDEQ, relationId); @@ -696,7 +697,7 @@ GetTableIndexAndConstraintCommands(Oid relationId) /* clean up scan and close system catalog */ systable_endscan(scanDescriptor); - heap_close(pgIndex, AccessShareLock); + table_close(pgIndex, AccessShareLock); /* revert back to original search_path */ PopOverrideSearchPath(); diff --git a/src/backend/distributed/planner/deparse_shard_query.c b/src/backend/distributed/planner/deparse_shard_query.c index d568ff3c7..d96108471 100644 --- a/src/backend/distributed/planner/deparse_shard_query.c +++ b/src/backend/distributed/planner/deparse_shard_query.c @@ -358,7 +358,7 @@ UpdateRelationsToLocalShardTables(Node *node, List *relationShardList) static void ConvertRteToSubqueryWithEmptyResult(RangeTblEntry *rte) { - Relation relation = heap_open(rte->relid, NoLock); + Relation relation = table_open(rte->relid, NoLock); TupleDesc tupleDescriptor = RelationGetDescr(relation); int columnCount = tupleDescriptor->natts; List *targetList = NIL; @@ -388,7 +388,7 @@ ConvertRteToSubqueryWithEmptyResult(RangeTblEntry *rte) targetList = lappend(targetList, targetEntry); } - heap_close(relation, NoLock); + table_close(relation, NoLock); FromExpr *joinTree = makeNode(FromExpr); joinTree->quals = makeBoolConst(false, false); diff --git a/src/backend/distributed/planner/insert_select_planner.c b/src/backend/distributed/planner/insert_select_planner.c index e92f0b043..aad31a2f0 100644 --- a/src/backend/distributed/planner/insert_select_planner.c +++ b/src/backend/distributed/planner/insert_select_planner.c @@ -1494,7 +1494,7 @@ AddInsertSelectCasts(List *insertTargetList, List *selectTargetList, */ Assert(list_length(insertTargetList) <= list_length(selectTargetList)); - Relation distributedRelation = heap_open(targetRelationId, RowExclusiveLock); + Relation distributedRelation = table_open(targetRelationId, RowExclusiveLock); TupleDesc destTupleDescriptor = RelationGetDescr(distributedRelation); int targetEntryIndex = 0; @@ -1579,7 +1579,7 @@ AddInsertSelectCasts(List *insertTargetList, List *selectTargetList, selectTargetEntry->resno = entryResNo++; } - heap_close(distributedRelation, NoLock); + table_close(distributedRelation, NoLock); return selectTargetList; } diff --git a/src/backend/distributed/planner/multi_logical_optimizer.c b/src/backend/distributed/planner/multi_logical_optimizer.c index be33e29ed..4d0fbc514 100644 --- a/src/backend/distributed/planner/multi_logical_optimizer.c +++ b/src/backend/distributed/planner/multi_logical_optimizer.c @@ -3564,7 +3564,7 @@ AggregateFunctionOid(const char *functionName, Oid inputType) ScanKeyData scanKey[1]; int scanKeyCount = 1; - Relation procRelation = heap_open(ProcedureRelationId, AccessShareLock); + Relation procRelation = table_open(ProcedureRelationId, AccessShareLock); ScanKeyInit(&scanKey[0], Anum_pg_proc_proname, BTEqualStrategyNumber, F_NAMEEQ, CStringGetDatum(functionName)); @@ -3605,7 +3605,7 @@ AggregateFunctionOid(const char *functionName, Oid inputType) } systable_endscan(scanDescriptor); - heap_close(procRelation, AccessShareLock); + table_close(procRelation, AccessShareLock); return functionOid; } diff --git a/src/backend/distributed/planner/tdigest_extension.c b/src/backend/distributed/planner/tdigest_extension.c index 158da9ee5..5b67c16cf 100644 --- a/src/backend/distributed/planner/tdigest_extension.c +++ b/src/backend/distributed/planner/tdigest_extension.c @@ -32,7 +32,7 @@ TDigestExtensionSchema() Form_pg_extension extensionForm = NULL; Oid tdigestExtensionSchema = InvalidOid; - Relation relation = heap_open(ExtensionRelationId, AccessShareLock); + Relation relation = table_open(ExtensionRelationId, AccessShareLock); ScanKeyInit(&entry[0], Anum_pg_extension_extname, @@ -57,7 +57,7 @@ TDigestExtensionSchema() systable_endscan(scandesc); - heap_close(relation, AccessShareLock); + table_close(relation, AccessShareLock); return tdigestExtensionSchema; } diff --git a/src/backend/distributed/transaction/transaction_recovery.c b/src/backend/distributed/transaction/transaction_recovery.c index 0e9942f5a..f868efffb 100644 --- a/src/backend/distributed/transaction/transaction_recovery.c +++ b/src/backend/distributed/transaction/transaction_recovery.c @@ -95,7 +95,7 @@ LogTransactionRecord(int32 groupId, char *transactionName) values[Anum_pg_dist_transaction_gid - 1] = CStringGetTextDatum(transactionName); /* open transaction relation and insert new tuple */ - Relation pgDistTransaction = heap_open(DistTransactionRelationId(), RowExclusiveLock); + Relation pgDistTransaction = table_open(DistTransactionRelationId(), RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistTransaction); HeapTuple heapTuple = heap_form_tuple(tupleDescriptor, values, isNulls); @@ -105,7 +105,7 @@ LogTransactionRecord(int32 groupId, char *transactionName) CommandCounterIncrement(); /* close relation and invalidate previous cache entry */ - heap_close(pgDistTransaction, NoLock); + table_close(pgDistTransaction, NoLock); } @@ -171,7 +171,7 @@ RecoverWorkerTransactions(WorkerNode *workerNode) MemoryContext oldContext = MemoryContextSwitchTo(localContext); /* take table lock first to avoid running concurrently */ - Relation pgDistTransaction = heap_open(DistTransactionRelationId(), + Relation pgDistTransaction = table_open(DistTransactionRelationId(), ShareUpdateExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistTransaction); @@ -344,7 +344,7 @@ RecoverWorkerTransactions(WorkerNode *workerNode) } systable_endscan(scanDescriptor); - heap_close(pgDistTransaction, NoLock); + table_close(pgDistTransaction, NoLock); if (!recoveryFailed) { diff --git a/src/backend/distributed/utils/colocation_utils.c b/src/backend/distributed/utils/colocation_utils.c index 7190f1716..dcaa70f9d 100644 --- a/src/backend/distributed/utils/colocation_utils.c +++ b/src/backend/distributed/utils/colocation_utils.c @@ -29,6 +29,7 @@ #include "distributed/pg_dist_colocation.h" #include "distributed/resource_lock.h" #include "distributed/shardinterval_utils.h" +#include "distributed/version_compat.h" #include "distributed/worker_protocol.h" #include "distributed/worker_transaction.h" #include "storage/lmgr.h" @@ -154,7 +155,7 @@ BreakColocation(Oid sourceRelationId) * can be sure that there will no modifications on the colocation table * until this transaction is committed. */ - Relation pgDistColocation = heap_open(DistColocationRelationId(), ExclusiveLock); + Relation pgDistColocation = table_open(DistColocationRelationId(), ExclusiveLock); uint32 newColocationId = GetNextColocationId(); UpdateRelationColocationGroup(sourceRelationId, newColocationId); @@ -162,7 +163,7 @@ BreakColocation(Oid sourceRelationId) /* if there is not any remaining table in the colocation group, delete it */ DeleteColocationGroupIfNoTablesBelong(sourceRelationId); - heap_close(pgDistColocation, NoLock); + table_close(pgDistColocation, NoLock); } @@ -248,7 +249,7 @@ MarkTablesColocated(Oid sourceRelationId, Oid targetRelationId) * can be sure that there will no modifications on the colocation table * until this transaction is committed. */ - Relation pgDistColocation = heap_open(DistColocationRelationId(), ExclusiveLock); + Relation pgDistColocation = table_open(DistColocationRelationId(), ExclusiveLock); /* check if shard placements are colocated */ ErrorIfShardPlacementsNotColocated(sourceRelationId, targetRelationId); @@ -271,7 +272,7 @@ MarkTablesColocated(Oid sourceRelationId, Oid targetRelationId) /* if there is not any remaining table in the colocation group, delete it */ DeleteColocationGroupIfNoTablesBelong(targetColocationId); - heap_close(pgDistColocation, NoLock); + table_close(pgDistColocation, NoLock); } @@ -514,7 +515,7 @@ ColocationId(int shardCount, int replicationFactor, Oid distributionColumnType, ScanKeyData scanKey[4]; bool indexOK = true; - Relation pgDistColocation = heap_open(DistColocationRelationId(), AccessShareLock); + Relation pgDistColocation = table_open(DistColocationRelationId(), AccessShareLock); /* set scan arguments */ ScanKeyInit(&scanKey[0], Anum_pg_dist_colocation_distributioncolumntype, @@ -541,7 +542,7 @@ ColocationId(int shardCount, int replicationFactor, Oid distributionColumnType, } systable_endscan(scanDescriptor); - heap_close(pgDistColocation, AccessShareLock); + table_close(pgDistColocation, AccessShareLock); return colocationId; } @@ -574,7 +575,7 @@ CreateColocationGroup(int shardCount, int replicationFactor, Oid distributionCol ObjectIdGetDatum(distributionColumnCollation); /* open colocation relation and insert the new tuple */ - Relation pgDistColocation = heap_open(DistColocationRelationId(), RowExclusiveLock); + Relation pgDistColocation = table_open(DistColocationRelationId(), RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistColocation); HeapTuple heapTuple = heap_form_tuple(tupleDescriptor, values, isNulls); @@ -583,7 +584,7 @@ CreateColocationGroup(int shardCount, int replicationFactor, Oid distributionCol /* increment the counter so that next command can see the row */ CommandCounterIncrement(); - heap_close(pgDistColocation, RowExclusiveLock); + table_close(pgDistColocation, RowExclusiveLock); return colocationId; } @@ -716,7 +717,7 @@ UpdateRelationColocationGroup(Oid distributedRelationId, uint32 colocationId) bool isNull[Natts_pg_dist_partition]; bool replace[Natts_pg_dist_partition]; - Relation pgDistPartition = heap_open(DistPartitionRelationId(), RowExclusiveLock); + Relation pgDistPartition = table_open(DistPartitionRelationId(), RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistPartition); ScanKeyInit(&scanKey[0], Anum_pg_dist_partition_logicalrelid, @@ -753,7 +754,7 @@ UpdateRelationColocationGroup(Oid distributedRelationId, uint32 colocationId) CommandCounterIncrement(); systable_endscan(scanDescriptor); - heap_close(pgDistPartition, NoLock); + table_close(pgDistPartition, NoLock); bool shouldSyncMetadata = ShouldSyncTableMetadata(distributedRelationId); if (shouldSyncMetadata) @@ -882,7 +883,7 @@ ColocationGroupTableList(Oid colocationId) ScanKeyInit(&scanKey[0], Anum_pg_dist_partition_colocationid, BTEqualStrategyNumber, F_INT4EQ, ObjectIdGetDatum(colocationId)); - Relation pgDistPartition = heap_open(DistPartitionRelationId(), AccessShareLock); + Relation pgDistPartition = table_open(DistPartitionRelationId(), AccessShareLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistPartition); SysScanDesc scanDescriptor = systable_beginscan(pgDistPartition, DistPartitionColocationidIndexId(), @@ -901,7 +902,7 @@ ColocationGroupTableList(Oid colocationId) } systable_endscan(scanDescriptor); - heap_close(pgDistPartition, AccessShareLock); + table_close(pgDistPartition, AccessShareLock); return colocatedTableList; } @@ -997,7 +998,7 @@ ColocatedTableId(Oid colocationId) ScanKeyInit(&scanKey[0], Anum_pg_dist_partition_colocationid, BTEqualStrategyNumber, F_INT4EQ, ObjectIdGetDatum(colocationId)); - Relation pgDistPartition = heap_open(DistPartitionRelationId(), AccessShareLock); + Relation pgDistPartition = table_open(DistPartitionRelationId(), AccessShareLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistPartition); SysScanDesc scanDescriptor = systable_beginscan(pgDistPartition, DistPartitionColocationidIndexId(), @@ -1034,7 +1035,7 @@ ColocatedTableId(Oid colocationId) } systable_endscan(scanDescriptor); - heap_close(pgDistPartition, AccessShareLock); + table_close(pgDistPartition, AccessShareLock); return colocatedTableId; } @@ -1085,7 +1086,7 @@ DeleteColocationGroup(uint32 colocationId) ScanKeyData scanKey[1]; bool indexOK = false; - Relation pgDistColocation = heap_open(DistColocationRelationId(), RowExclusiveLock); + Relation pgDistColocation = table_open(DistColocationRelationId(), RowExclusiveLock); ScanKeyInit(&scanKey[0], Anum_pg_dist_colocation_colocationid, BTEqualStrategyNumber, F_INT4EQ, UInt32GetDatum(colocationId)); @@ -1108,9 +1109,9 @@ DeleteColocationGroup(uint32 colocationId) CitusInvalidateRelcacheByRelid(DistColocationRelationId()); CommandCounterIncrement(); - heap_close(replicaIndex, AccessShareLock); + table_close(replicaIndex, AccessShareLock); } systable_endscan(scanDescriptor); - heap_close(pgDistColocation, RowExclusiveLock); + table_close(pgDistColocation, RowExclusiveLock); } diff --git a/src/backend/distributed/utils/foreign_key_relationship.c b/src/backend/distributed/utils/foreign_key_relationship.c index d530be06b..352335ab0 100644 --- a/src/backend/distributed/utils/foreign_key_relationship.c +++ b/src/backend/distributed/utils/foreign_key_relationship.c @@ -296,7 +296,7 @@ PopulateAdjacencyLists(void) Oid prevReferencedOid = InvalidOid; List *frelEdgeList = NIL; - Relation pgConstraint = heap_open(ConstraintRelationId, AccessShareLock); + Relation pgConstraint = table_open(ConstraintRelationId, AccessShareLock); ScanKeyInit(&scanKey[0], Anum_pg_constraint_contype, BTEqualStrategyNumber, F_CHAREQ, CharGetDatum(CONSTRAINT_FOREIGN)); @@ -345,7 +345,7 @@ PopulateAdjacencyLists(void) } systable_endscan(scanDescriptor); - heap_close(pgConstraint, AccessShareLock); + table_close(pgConstraint, AccessShareLock); } diff --git a/src/backend/distributed/utils/multi_partitioning_utils.c b/src/backend/distributed/utils/multi_partitioning_utils.c index da4b8df86..8833ae3ae 100644 --- a/src/backend/distributed/utils/multi_partitioning_utils.c +++ b/src/backend/distributed/utils/multi_partitioning_utils.c @@ -21,6 +21,7 @@ #include "distributed/coordinator_protocol.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/shardinterval_utils.h" +#include "distributed/version_compat.h" #include "lib/stringinfo.h" #include "nodes/pg_list.h" #include "pgstat.h" @@ -58,7 +59,7 @@ PartitionedTable(Oid relationId) } /* keep the lock */ - heap_close(rel, NoLock); + table_close(rel, NoLock); return partitionedTable; } @@ -87,7 +88,7 @@ PartitionedTableNoLock(Oid relationId) } /* keep the lock */ - heap_close(rel, NoLock); + table_close(rel, NoLock); return partitionedTable; } @@ -110,7 +111,7 @@ PartitionTable(Oid relationId) bool partitionTable = rel->rd_rel->relispartition; /* keep the lock */ - heap_close(rel, NoLock); + table_close(rel, NoLock); return partitionTable; } @@ -135,7 +136,7 @@ PartitionTableNoLock(Oid relationId) bool partitionTable = rel->rd_rel->relispartition; /* keep the lock */ - heap_close(rel, NoLock); + table_close(rel, NoLock); return partitionTable; } @@ -185,7 +186,7 @@ IsChildTable(Oid relationId) HeapTuple inheritsTuple = NULL; bool tableInherits = false; - Relation pgInherits = heap_open(InheritsRelationId, AccessShareLock); + Relation pgInherits = table_open(InheritsRelationId, AccessShareLock); ScanKeyInit(&key[0], Anum_pg_inherits_inhrelid, BTEqualStrategyNumber, F_OIDEQ, @@ -207,7 +208,7 @@ IsChildTable(Oid relationId) } systable_endscan(scan); - heap_close(pgInherits, AccessShareLock); + table_close(pgInherits, AccessShareLock); if (tableInherits && PartitionTable(relationId)) { @@ -229,7 +230,7 @@ IsParentTable(Oid relationId) ScanKeyData key[1]; bool tableInherited = false; - Relation pgInherits = heap_open(InheritsRelationId, AccessShareLock); + Relation pgInherits = table_open(InheritsRelationId, AccessShareLock); ScanKeyInit(&key[0], Anum_pg_inherits_inhparent, BTEqualStrategyNumber, F_OIDEQ, @@ -243,7 +244,7 @@ IsParentTable(Oid relationId) tableInherited = true; } systable_endscan(scan); - heap_close(pgInherits, AccessShareLock); + table_close(pgInherits, AccessShareLock); if (tableInherited && PartitionedTable(relationId)) { @@ -277,7 +278,7 @@ PartitionParentOid(Oid partitionOid) List * PartitionList(Oid parentRelationId) { - Relation rel = heap_open(parentRelationId, AccessShareLock); + Relation rel = table_open(parentRelationId, AccessShareLock); List *partitionList = NIL; @@ -298,7 +299,7 @@ PartitionList(Oid parentRelationId) } /* keep the lock */ - heap_close(rel, NoLock); + table_close(rel, NoLock); return partitionList; } diff --git a/src/backend/distributed/utils/statistics_collection.c b/src/backend/distributed/utils/statistics_collection.c index d1430abd1..f5ce2faff 100644 --- a/src/backend/distributed/utils/statistics_collection.c +++ b/src/backend/distributed/utils/statistics_collection.c @@ -197,14 +197,14 @@ DistributedTablesSize(List *distTableOids) if (PartitionMethod(relationId) == DISTRIBUTE_BY_HASH && !SingleReplicatedTable(relationId)) { - heap_close(relation, AccessShareLock); + table_close(relation, AccessShareLock); continue; } Datum tableSizeDatum = DirectFunctionCall1(citus_table_size, ObjectIdGetDatum(relationId)); totalSize += DatumGetInt64(tableSizeDatum); - heap_close(relation, AccessShareLock); + table_close(relation, AccessShareLock); } return totalSize; diff --git a/src/backend/distributed/worker/worker_merge_protocol.c b/src/backend/distributed/worker/worker_merge_protocol.c index 1b97337f7..b32204aa6 100644 --- a/src/backend/distributed/worker/worker_merge_protocol.c +++ b/src/backend/distributed/worker/worker_merge_protocol.c @@ -278,7 +278,7 @@ worker_cleanup_job_schema_cache(PG_FUNCTION_ARGS) CheckCitusVersion(ERROR); - pgNamespace = heap_open(NamespaceRelationId, AccessExclusiveLock); + pgNamespace = table_open(NamespaceRelationId, AccessExclusiveLock); #if PG_VERSION_NUM >= PG_VERSION_12 scanDescriptor = table_beginscan_catalog(pgNamespace, scanKeyCount, scanKey); #else @@ -304,7 +304,7 @@ worker_cleanup_job_schema_cache(PG_FUNCTION_ARGS) } heap_endscan(scanDescriptor); - heap_close(pgNamespace, AccessExclusiveLock); + table_close(pgNamespace, AccessExclusiveLock); PG_RETURN_VOID(); } diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index a1fc4dfcc..1004349a2 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -94,6 +94,10 @@ FileCompatFromFileStart(File fileDesc) #else /* pre PG12 */ +#define table_open(r, l) heap_open(r, l) +#define table_openrv(r, l) heap_openrv(r, l) +#define table_openrv_extended(r, l, m) heap_openrv_extended(r, l, m) +#define table_close(r, l) heap_close(r, l) #define QTW_EXAMINE_RTES_BEFORE QTW_EXAMINE_RTES #define MakeSingleTupleTableSlotCompat(tupleDesc, tts_opts) \ MakeSingleTupleTableSlot(tupleDesc) From 62879ee8c12f0efc4fa1f6c989d9e044337835dc Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Mon, 18 May 2020 16:48:26 +0300 Subject: [PATCH 06/52] introduce planner_compat and pg_plan_query_compat macros As the new planner and pg_plan_query_compat methods expect the query string as well, macros are defined to be compatible in different versions of postgres. Relevant commit on Postgres: 6aba63ef3e606db71beb596210dd95fa73c44ce2 Command on Postgres: git log --all --grep="pg_plan_query" --- src/backend/distributed/executor/local_executor.c | 5 +++-- src/backend/distributed/executor/multi_executor.c | 3 ++- .../distributed/executor/partitioned_intermediate_results.c | 3 ++- src/backend/distributed/planner/insert_select_planner.c | 2 +- src/backend/distributed/planner/local_plan_cache.c | 3 ++- src/backend/distributed/planner/multi_explain.c | 2 +- src/backend/distributed/planner/recursive_planning.c | 2 +- .../distributed/test/distributed_intermediate_results.c | 5 +++-- src/include/distributed/deparser.h | 1 + src/include/distributed/version_compat.h | 4 ++++ 10 files changed, 20 insertions(+), 10 deletions(-) diff --git a/src/backend/distributed/executor/local_executor.c b/src/backend/distributed/executor/local_executor.c index 0b590e629..d7b5d24b9 100644 --- a/src/backend/distributed/executor/local_executor.c +++ b/src/backend/distributed/executor/local_executor.c @@ -94,6 +94,7 @@ #include "distributed/relation_access_tracking.h" #include "distributed/remote_commands.h" /* to access LogRemoteCommands */ #include "distributed/transaction_management.h" +#include "distributed/version_compat.h" #include "distributed/worker_protocol.h" #include "executor/tstoreReceiver.h" #include "executor/tuptable.h" @@ -293,7 +294,7 @@ ExecuteLocalTaskListExtended(List *taskList, * implemented. So, let planner to call distributed_planner() which * eventually calls standard_planner(). */ - localPlan = planner(shardQuery, cursorOptions, paramListInfo); + localPlan = planner_compat(shardQuery, NULL, cursorOptions, paramListInfo); } char *shardQueryString = NULL; @@ -333,7 +334,7 @@ LocallyPlanAndExecuteMultipleQueries(List *queryStrings, TupleDestination *tuple 0); int cursorOptions = 0; ParamListInfo paramListInfo = NULL; - PlannedStmt *localPlan = planner(shardQuery, cursorOptions, paramListInfo); + PlannedStmt *localPlan = planner_compat(shardQuery, NULL, cursorOptions, paramListInfo); totalProcessedRows += ExecuteLocalTaskPlan(localPlan, queryString, tupleDest, task, paramListInfo); diff --git a/src/backend/distributed/executor/multi_executor.c b/src/backend/distributed/executor/multi_executor.c index c4657ccf5..7c3395404 100644 --- a/src/backend/distributed/executor/multi_executor.c +++ b/src/backend/distributed/executor/multi_executor.c @@ -32,6 +32,7 @@ #include "distributed/multi_server_executor.h" #include "distributed/resource_lock.h" #include "distributed/transaction_management.h" +#include "distributed/version_compat.h" #include "distributed/worker_shard_visibility.h" #include "distributed/worker_protocol.h" #include "executor/execdebug.h" @@ -604,7 +605,7 @@ ExecuteQueryIntoDestReceiver(Query *query, ParamListInfo params, DestReceiver *d } /* plan the subquery, this may be another distributed query */ - PlannedStmt *queryPlan = pg_plan_query(query, cursorOptions, params); + PlannedStmt *queryPlan = pg_plan_query_compat(query, NULL, cursorOptions, params); ExecutePlanIntoDestReceiver(queryPlan, params, dest); } diff --git a/src/backend/distributed/executor/partitioned_intermediate_results.c b/src/backend/distributed/executor/partitioned_intermediate_results.c index 781bc0823..b08028040 100644 --- a/src/backend/distributed/executor/partitioned_intermediate_results.c +++ b/src/backend/distributed/executor/partitioned_intermediate_results.c @@ -26,6 +26,7 @@ #include "distributed/pg_dist_shard.h" #include "distributed/remote_commands.h" #include "distributed/tuplestore.h" +#include "distributed/version_compat.h" #include "distributed/worker_protocol.h" #include "nodes/makefuncs.h" #include "nodes/primnodes.h" @@ -258,7 +259,7 @@ StartPortalForQueryExecution(const char *queryString) Query *query = ParseQueryString(queryString, NULL, 0); int cursorOptions = CURSOR_OPT_PARALLEL_OK; - PlannedStmt *queryPlan = pg_plan_query(query, cursorOptions, NULL); + PlannedStmt *queryPlan = pg_plan_query_compat(query, NULL, cursorOptions, NULL); Portal portal = CreateNewPortal(); diff --git a/src/backend/distributed/planner/insert_select_planner.c b/src/backend/distributed/planner/insert_select_planner.c index aad31a2f0..67037916b 100644 --- a/src/backend/distributed/planner/insert_select_planner.c +++ b/src/backend/distributed/planner/insert_select_planner.c @@ -1388,7 +1388,7 @@ CreateNonPushableInsertSelectPlan(uint64 planId, Query *parse, ParamListInfo bou /* plan the subquery, this may be another distributed query */ int cursorOptions = CURSOR_OPT_PARALLEL_OK; - PlannedStmt *selectPlan = pg_plan_query(selectQueryCopy, cursorOptions, + PlannedStmt *selectPlan = pg_plan_query_compat(selectQueryCopy, NULL, cursorOptions, boundParams); bool repartitioned = IsRedistributablePlan(selectPlan->planTree) && diff --git a/src/backend/distributed/planner/local_plan_cache.c b/src/backend/distributed/planner/local_plan_cache.c index 881693e64..380cb0952 100644 --- a/src/backend/distributed/planner/local_plan_cache.c +++ b/src/backend/distributed/planner/local_plan_cache.c @@ -17,6 +17,7 @@ #include "distributed/deparse_shard_query.h" #include "distributed/citus_ruleutils.h" #include "distributed/metadata_cache.h" +#include "distributed/version_compat.h" #if PG_VERSION_NUM >= PG_VERSION_12 #include "optimizer/optimizer.h" #else @@ -89,7 +90,7 @@ CacheLocalPlanForShardQuery(Task *task, DistributedPlan *originalDistributedPlan LockRelationOid(rangeTableEntry->relid, lockMode); LocalPlannedStatement *localPlannedStatement = CitusMakeNode(LocalPlannedStatement); - localPlan = planner(shardQuery, 0, NULL); + localPlan = planner_compat(shardQuery, NULL, 0, NULL); localPlannedStatement->localPlan = localPlan; localPlannedStatement->shardId = task->anchorShardId; localPlannedStatement->localGroupId = GetLocalGroupId(); diff --git a/src/backend/distributed/planner/multi_explain.c b/src/backend/distributed/planner/multi_explain.c index 13082b2ca..9b0cc1e2f 100644 --- a/src/backend/distributed/planner/multi_explain.c +++ b/src/backend/distributed/planner/multi_explain.c @@ -1457,7 +1457,7 @@ ExplainOneQuery(Query *query, int cursorOptions, INSTR_TIME_SET_CURRENT(planstart); /* plan the query */ - PlannedStmt *plan = pg_plan_query(query, cursorOptions, params); + PlannedStmt *plan = pg_plan_query_compat(query, NULL, cursorOptions, params); INSTR_TIME_SET_CURRENT(planduration); INSTR_TIME_SUBTRACT(planduration, planstart); diff --git a/src/backend/distributed/planner/recursive_planning.c b/src/backend/distributed/planner/recursive_planning.c index 71f396fce..d1c07adb1 100644 --- a/src/backend/distributed/planner/recursive_planning.c +++ b/src/backend/distributed/planner/recursive_planning.c @@ -1174,7 +1174,7 @@ CreateDistributedSubPlan(uint32 subPlanId, Query *subPlanQuery) } DistributedSubPlan *subPlan = CitusMakeNode(DistributedSubPlan); - subPlan->plan = planner(subPlanQuery, cursorOptions, NULL); + subPlan->plan = planner_compat(subPlanQuery, NULL, cursorOptions, NULL); subPlan->subPlanId = subPlanId; return subPlan; diff --git a/src/backend/distributed/test/distributed_intermediate_results.c b/src/backend/distributed/test/distributed_intermediate_results.c index 7a095504a..474403d3e 100644 --- a/src/backend/distributed/test/distributed_intermediate_results.c +++ b/src/backend/distributed/test/distributed_intermediate_results.c @@ -27,6 +27,7 @@ #include "distributed/remote_commands.h" #include "distributed/tuplestore.h" #include "distributed/listutils.h" +#include "distributed/version_compat.h" #include "tcop/tcopprot.h" PG_FUNCTION_INFO_V1(partition_task_list_results); @@ -49,7 +50,7 @@ partition_task_list_results(PG_FUNCTION_ARGS) bool binaryFormat = PG_GETARG_BOOL(3); Query *parsedQuery = ParseQueryString(queryString, NULL, 0); - PlannedStmt *queryPlan = pg_plan_query(parsedQuery, + PlannedStmt *queryPlan = pg_plan_query_compat(parsedQuery, queryString, CURSOR_OPT_PARALLEL_OK, NULL); if (!IsCitusCustomScan(queryPlan->planTree)) @@ -122,7 +123,7 @@ redistribute_task_list_results(PG_FUNCTION_ARGS) bool binaryFormat = PG_GETARG_BOOL(3); Query *parsedQuery = ParseQueryString(queryString, NULL, 0); - PlannedStmt *queryPlan = pg_plan_query(parsedQuery, + PlannedStmt *queryPlan = pg_plan_query_compat(parsedQuery, queryString, CURSOR_OPT_PARALLEL_OK, NULL); if (!IsCitusCustomScan(queryPlan->planTree)) diff --git a/src/include/distributed/deparser.h b/src/include/distributed/deparser.h index 527d806bb..590391f31 100644 --- a/src/include/distributed/deparser.h +++ b/src/include/distributed/deparser.h @@ -18,6 +18,7 @@ #include "nodes/nodes.h" #include "nodes/parsenodes.h" #include "catalog/objectaddress.h" +#include "lib/stringinfo.h" /* forward declarations for format_collate.c */ /* Control flags for FormatCollateExtended, compatible with format_type_extended */ diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index 1004349a2..ca343a652 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -27,9 +27,13 @@ #if PG_VERSION_NUM >= PG_VERSION_13 #define lnext_compat(l, r) lnext(l, r) #define list_delete_cell_compat(l,c,p) list_delete_cell(l,c) +#define pg_plan_query_compat(p,q,c,b) pg_plan_query(p,q,c,b) +#define planner_compat(p,q,c,b) planner(p,q,c,b) #else /* pre PG13 */ #define lnext_compat(l, r) lnext(r) #define list_delete_cell_compat(l,c,p) list_delete_cell(l,c,p) +#define pg_plan_query_compat(p,q,c,b) pg_plan_query(p,c,b) +#define planner_compat(p,q,c,b) planner(p,c,b) #endif #if PG_VERSION_NUM >= PG_VERSION_12 From 00e738600705638e1551d7bb5d9b3f7f9933bc50 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Mon, 18 May 2020 17:08:54 +0300 Subject: [PATCH 07/52] introduce PortalDefineQuerySelectCompat PortalDefineQuery doesn't accept char* for command tag anymore with PG >= 13. We are currently only using it with Select, therefore a Portal define query compat for select is created. Commit on PG side: 2f9661311b83dc481fc19f6e3bda015392010a40 --- src/backend/distributed/executor/multi_executor.c | 3 +-- .../executor/partitioned_intermediate_results.c | 2 +- src/include/distributed/version_compat.h | 8 +++++++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/backend/distributed/executor/multi_executor.c b/src/backend/distributed/executor/multi_executor.c index 7c3395404..92720e13e 100644 --- a/src/backend/distributed/executor/multi_executor.c +++ b/src/backend/distributed/executor/multi_executor.c @@ -628,10 +628,9 @@ ExecutePlanIntoDestReceiver(PlannedStmt *queryPlan, ParamListInfo params, /* don't display the portal in pg_cursors, it is for internal use only */ portal->visible = false; - PortalDefineQuery(portal, + PortalDefineQuerySelectCompat(portal, NULL, "", - "SELECT", list_make1(queryPlan), NULL); diff --git a/src/backend/distributed/executor/partitioned_intermediate_results.c b/src/backend/distributed/executor/partitioned_intermediate_results.c index b08028040..21722dbfc 100644 --- a/src/backend/distributed/executor/partitioned_intermediate_results.c +++ b/src/backend/distributed/executor/partitioned_intermediate_results.c @@ -266,7 +266,7 @@ StartPortalForQueryExecution(const char *queryString) /* don't display the portal in pg_cursors, it is for internal use only */ portal->visible = false; - PortalDefineQuery(portal, NULL, queryString, "SELECT", list_make1(queryPlan), NULL); + PortalDefineQuerySelectCompat(portal, NULL, queryString, list_make1(queryPlan), NULL); int eflags = 0; PortalStart(portal, NULL, eflags, GetActiveSnapshot()); diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index ca343a652..8fe66e709 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -24,16 +24,22 @@ #include "optimizer/optimizer.h" #endif +#if (PG_VERSION_NUM >= PG_VERSION_13) +#include "tcop/tcopprot.h" +#endif + #if PG_VERSION_NUM >= PG_VERSION_13 #define lnext_compat(l, r) lnext(l, r) #define list_delete_cell_compat(l,c,p) list_delete_cell(l,c) #define pg_plan_query_compat(p,q,c,b) pg_plan_query(p,q,c,b) -#define planner_compat(p,q,c,b) planner(p,q,c,b) +#define planner_compat(p,q,c,b) planner(p,q,c,b) +#define PortalDefineQuerySelectCompat(a,b,c,e,f) PortalDefineQuery(a,b,c,CMDTAG_SELECT,e,f) #else /* pre PG13 */ #define lnext_compat(l, r) lnext(r) #define list_delete_cell_compat(l,c,p) list_delete_cell(l,c,p) #define pg_plan_query_compat(p,q,c,b) pg_plan_query(p,c,b) #define planner_compat(p,q,c,b) planner(p,c,b) +#define PortalDefineQuerySelectCompat(a,b,c,e,f) PortalDefineQuery(a,b,c,"SELECT",e,f) #endif #if PG_VERSION_NUM >= PG_VERSION_12 From 01632c56a04447385c7d9a241142543eee54b5d0 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Mon, 18 May 2020 16:21:15 +0300 Subject: [PATCH 08/52] Change utils/hashutils.h to common/hashfn.h for PG >= 13 Commit on postgres side: 05d8449e73694585b59f8b03aaa087f04cc4679a Command on postgres side: git log --all --grep="hashutils" include common/hashfn.h for pg >= 13 tag_hash was moved from hsearch.h to hashutils.h then to hashfn.h Commits on Postgres side: 9341c783cc42ffae5860c86bdc713bd47d734ffd --- .../connection/locally_reserved_shared_connections.c | 8 +++++++- src/backend/distributed/connection/placement_connection.c | 5 +++++ .../distributed/connection/shared_connection_stats.c | 8 +++++++- src/backend/distributed/metadata/metadata_cache.c | 3 +++ .../distributed/planner/intermediate_result_pruning.c | 3 +++ .../distributed/transaction/relation_access_tracking.c | 5 +++++ src/backend/distributed/utils/foreign_key_relationship.c | 3 +++ src/backend/distributed/utils/maintenanced.c | 5 +++++ src/backend/distributed/utils/task_execution_utils.c | 6 ++++++ 9 files changed, 44 insertions(+), 2 deletions(-) diff --git a/src/backend/distributed/connection/locally_reserved_shared_connections.c b/src/backend/distributed/connection/locally_reserved_shared_connections.c index a4dc337cd..a4bd95f4c 100644 --- a/src/backend/distributed/connection/locally_reserved_shared_connections.c +++ b/src/backend/distributed/connection/locally_reserved_shared_connections.c @@ -33,6 +33,8 @@ #include "postgres.h" +#include "distributed/pg_version_constants.h" + #include "miscadmin.h" #include "access/hash.h" @@ -45,8 +47,12 @@ #include "distributed/shared_connection_stats.h" #include "distributed/tuplestore.h" #include "distributed/worker_manager.h" -#include "utils/hashutils.h" #include "utils/builtins.h" +#if PG_VERSION_NUM < PG_VERSION_13 +#include "utils/hashutils.h" +#else +#include "common/hashfn.h" +#endif #define RESERVED_CONNECTION_COLUMNS 4 diff --git a/src/backend/distributed/connection/placement_connection.c b/src/backend/distributed/connection/placement_connection.c index abaf334d9..5de19f9cd 100644 --- a/src/backend/distributed/connection/placement_connection.c +++ b/src/backend/distributed/connection/placement_connection.c @@ -11,6 +11,8 @@ #include "postgres.h" +#include "distributed/pg_version_constants.h" + #include "access/hash.h" #include "distributed/colocation_utils.h" #include "distributed/connection_management.h" @@ -24,6 +26,9 @@ #include "distributed/placement_connection.h" #include "distributed/relation_access_tracking.h" #include "utils/hsearch.h" +#if PG_VERSION_NUM >= PG_VERSION_13 +#include "common/hashfn.h" +#endif #include "utils/memutils.h" diff --git a/src/backend/distributed/connection/shared_connection_stats.c b/src/backend/distributed/connection/shared_connection_stats.c index 65a5536c4..1c82f3889 100644 --- a/src/backend/distributed/connection/shared_connection_stats.c +++ b/src/backend/distributed/connection/shared_connection_stats.c @@ -13,6 +13,8 @@ #include "postgres.h" #include "pgstat.h" +#include "distributed/pg_version_constants.h" + #include "libpq-fe.h" #include "miscadmin.h" @@ -33,8 +35,12 @@ #include "distributed/time_constants.h" #include "distributed/tuplestore.h" #include "utils/builtins.h" -#include "utils/hashutils.h" +#if PG_VERSION_NUM < PG_VERSION_13 #include "utils/hsearch.h" +#include "utils/hashutils.h" +#else +#include "common/hashfn.h" +#endif #include "storage/ipc.h" diff --git a/src/backend/distributed/metadata/metadata_cache.c b/src/backend/distributed/metadata/metadata_cache.c index 4a541e8ea..030f12083 100644 --- a/src/backend/distributed/metadata/metadata_cache.c +++ b/src/backend/distributed/metadata/metadata_cache.c @@ -65,6 +65,9 @@ #include "utils/datum.h" #include "utils/elog.h" #include "utils/hsearch.h" +#if PG_VERSION_NUM >= PG_VERSION_13 +#include "common/hashfn.h" +#endif #include "utils/inval.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" diff --git a/src/backend/distributed/planner/intermediate_result_pruning.c b/src/backend/distributed/planner/intermediate_result_pruning.c index 4b7aa5a87..1aa7fe7fd 100644 --- a/src/backend/distributed/planner/intermediate_result_pruning.c +++ b/src/backend/distributed/planner/intermediate_result_pruning.c @@ -21,6 +21,9 @@ #include "distributed/query_utils.h" #include "distributed/worker_manager.h" #include "utils/builtins.h" +#if PG_VERSION_NUM >= PG_VERSION_13 +#include "common/hashfn.h" +#endif /* controlled via GUC, used mostly for testing */ bool LogIntermediateResults = false; diff --git a/src/backend/distributed/transaction/relation_access_tracking.c b/src/backend/distributed/transaction/relation_access_tracking.c index 261257906..c2af6917b 100644 --- a/src/backend/distributed/transaction/relation_access_tracking.c +++ b/src/backend/distributed/transaction/relation_access_tracking.c @@ -15,6 +15,8 @@ */ #include "postgres.h" +#include "distributed/pg_version_constants.h" + #include "miscadmin.h" #include "access/xact.h" @@ -27,6 +29,9 @@ #include "distributed/metadata_cache.h" #include "distributed/relation_access_tracking.h" #include "utils/hsearch.h" +#if PG_VERSION_NUM >= PG_VERSION_13 +#include "common/hashfn.h" +#endif #include "utils/lsyscache.h" diff --git a/src/backend/distributed/utils/foreign_key_relationship.c b/src/backend/distributed/utils/foreign_key_relationship.c index 352335ab0..a116a5479 100644 --- a/src/backend/distributed/utils/foreign_key_relationship.c +++ b/src/backend/distributed/utils/foreign_key_relationship.c @@ -31,6 +31,9 @@ #include "storage/lockdefs.h" #include "utils/fmgroids.h" #include "utils/hsearch.h" +#if PG_VERSION_NUM >= PG_VERSION_13 +#include "common/hashfn.h" +#endif #include "utils/memutils.h" diff --git a/src/backend/distributed/utils/maintenanced.c b/src/backend/distributed/utils/maintenanced.c index f3a3edc40..77927ce16 100644 --- a/src/backend/distributed/utils/maintenanced.c +++ b/src/backend/distributed/utils/maintenanced.c @@ -16,6 +16,8 @@ #include "postgres.h" +#include "distributed/pg_version_constants.h" + #include #include "miscadmin.h" @@ -49,6 +51,9 @@ #include "storage/lmgr.h" #include "storage/lwlock.h" #include "tcop/tcopprot.h" +#if PG_VERSION_NUM >= PG_VERSION_13 +#include "common/hashfn.h" +#endif #include "utils/memutils.h" #include "utils/lsyscache.h" diff --git a/src/backend/distributed/utils/task_execution_utils.c b/src/backend/distributed/utils/task_execution_utils.c index 92b0c3ea2..902b483fd 100644 --- a/src/backend/distributed/utils/task_execution_utils.c +++ b/src/backend/distributed/utils/task_execution_utils.c @@ -6,6 +6,12 @@ #include #include +#include "distributed/pg_version_constants.h" + +#if PG_VERSION_NUM >= PG_VERSION_13 +#include "common/hashfn.h" +#endif + #include "commands/dbcommands.h" #include "distributed/citus_custom_scan.h" #include "distributed/citus_nodes.h" From 991f49efc96a78b2f007e84101e59b3a6308c584 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Mon, 18 May 2020 19:14:07 +0300 Subject: [PATCH 09/52] introduce getOwnedSequencesCompat macro Commit on Postgres side: 19781729f789f3c6b2540e02b96f8aa500460322 --- src/backend/distributed/metadata/metadata_sync.c | 2 +- src/include/distributed/version_compat.h | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/backend/distributed/metadata/metadata_sync.c b/src/backend/distributed/metadata/metadata_sync.c index 8736264d4..86a9666a3 100644 --- a/src/backend/distributed/metadata/metadata_sync.c +++ b/src/backend/distributed/metadata/metadata_sync.c @@ -1038,7 +1038,7 @@ List * SequenceDDLCommandsForTable(Oid relationId) { List *sequenceDDLList = NIL; - List *ownedSequences = getOwnedSequences(relationId, InvalidAttrNumber); + List *ownedSequences = getOwnedSequencesCompat(relationId, InvalidAttrNumber); char *ownerName = TableOwner(relationId); Oid sequenceOid = InvalidOid; diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index 8fe66e709..b3f9848f4 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -34,12 +34,14 @@ #define pg_plan_query_compat(p,q,c,b) pg_plan_query(p,q,c,b) #define planner_compat(p,q,c,b) planner(p,q,c,b) #define PortalDefineQuerySelectCompat(a,b,c,e,f) PortalDefineQuery(a,b,c,CMDTAG_SELECT,e,f) +#define getOwnedSequencesCompat(a,b) getOwnedSequences(a) #else /* pre PG13 */ #define lnext_compat(l, r) lnext(r) #define list_delete_cell_compat(l,c,p) list_delete_cell(l,c,p) #define pg_plan_query_compat(p,q,c,b) pg_plan_query(p,c,b) #define planner_compat(p,q,c,b) planner(p,c,b) #define PortalDefineQuerySelectCompat(a,b,c,e,f) PortalDefineQuery(a,b,c,"SELECT",e,f) +#define getOwnedSequencesCompat(a,b) getOwnedSequences(a,b) #endif #if PG_VERSION_NUM >= PG_VERSION_12 From 6314eba5df88733c31b57d5ecffbc447cdf1097e Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Mon, 18 May 2020 19:23:41 +0300 Subject: [PATCH 10/52] introduce standard_planner_compat standard_planner now takes the query string as a parameter as well with pg >= 13. Commit on Postgres Side: 66888f7424f7d6c7cea2c26e181054d1455d4e7a --- src/backend/distributed/planner/combine_query_planner.c | 2 +- src/backend/distributed/planner/distributed_planner.c | 5 +++-- src/include/distributed/version_compat.h | 2 ++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/backend/distributed/planner/combine_query_planner.c b/src/backend/distributed/planner/combine_query_planner.c index 9202a5d93..0db7d4b6f 100644 --- a/src/backend/distributed/planner/combine_query_planner.c +++ b/src/backend/distributed/planner/combine_query_planner.c @@ -295,7 +295,7 @@ BuildSelectStatementViaStdPlanner(Query *combineQuery, List *remoteScanTargetLis ReplaceCitusExtraDataContainer = true; ReplaceCitusExtraDataContainerWithCustomScan = remoteScan; - standardStmt = standard_planner(combineQuery, 0, NULL); + standardStmt = standard_planner_compat(combineQuery, NULL, 0, NULL); ReplaceCitusExtraDataContainer = false; ReplaceCitusExtraDataContainerWithCustomScan = NULL; diff --git a/src/backend/distributed/planner/distributed_planner.c b/src/backend/distributed/planner/distributed_planner.c index 8ae4fe6ab..738b3b37e 100644 --- a/src/backend/distributed/planner/distributed_planner.c +++ b/src/backend/distributed/planner/distributed_planner.c @@ -217,7 +217,8 @@ distributed_planner(Query *parse, int cursorOptions, ParamListInfo boundParams) * restriction information per table and parse tree transformations made by * postgres' planner. */ - planContext.plan = standard_planner(planContext.query, + planContext.plan = standard_planner_compat(planContext.query, + NULL, planContext.cursorOptions, planContext.boundParams); if (needsDistributedPlanning) @@ -1047,7 +1048,7 @@ CreateDistributedPlan(uint64 planId, Query *originalQuery, Query *query, ParamLi * being contiguous. */ - standard_planner(newQuery, 0, boundParams); + standard_planner_compat(newQuery, NULL, 0, boundParams); /* overwrite the old transformed query with the new transformed query */ *query = *newQuery; diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index b3f9848f4..be38c468c 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -33,6 +33,7 @@ #define list_delete_cell_compat(l,c,p) list_delete_cell(l,c) #define pg_plan_query_compat(p,q,c,b) pg_plan_query(p,q,c,b) #define planner_compat(p,q,c,b) planner(p,q,c,b) +#define standard_planner_compat(a,b,c,d) standard_planner(a,b,c,d) #define PortalDefineQuerySelectCompat(a,b,c,e,f) PortalDefineQuery(a,b,c,CMDTAG_SELECT,e,f) #define getOwnedSequencesCompat(a,b) getOwnedSequences(a) #else /* pre PG13 */ @@ -40,6 +41,7 @@ #define list_delete_cell_compat(l,c,p) list_delete_cell(l,c,p) #define pg_plan_query_compat(p,q,c,b) pg_plan_query(p,c,b) #define planner_compat(p,q,c,b) planner(p,c,b) +#define standard_planner_compat(a,b,c,d) standard_planner(a,c,d) #define PortalDefineQuerySelectCompat(a,b,c,e,f) PortalDefineQuery(a,b,c,"SELECT",e,f) #define getOwnedSequencesCompat(a,b) getOwnedSequences(a,b) #endif From 688ab16bbaf4c7dd88ddda92f29ff01a934785e8 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Wed, 20 May 2020 11:48:09 +0300 Subject: [PATCH 11/52] Introduce ExplainOnePlanCompat Since ExplainOnePlan expects BufferUsage as well with PG >= 13, ExplainOnePlanCompat is added. Commit on Postgres side: ed7a5095716ee498ecc406e1b8d5ab92c7662d10 --- src/backend/distributed/planner/multi_explain.c | 6 +++--- src/include/distributed/version_compat.h | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/backend/distributed/planner/multi_explain.c b/src/backend/distributed/planner/multi_explain.c index 9b0cc1e2f..b134078b5 100644 --- a/src/backend/distributed/planner/multi_explain.c +++ b/src/backend/distributed/planner/multi_explain.c @@ -313,7 +313,7 @@ ExplainSubPlans(DistributedPlan *distributedPlan, ExplainState *es) INSTR_TIME_SET_ZERO(planduration); - ExplainOnePlan(plan, into, es, queryString, params, NULL, &planduration); + ExplainOnePlanCompat(plan, into, es, queryString, params, NULL, &planduration, NULL); if (es->format == EXPLAIN_FORMAT_TEXT) { @@ -1463,8 +1463,8 @@ ExplainOneQuery(Query *query, int cursorOptions, INSTR_TIME_SUBTRACT(planduration, planstart); /* run it (if needed) and produce output */ - ExplainOnePlan(plan, into, es, queryString, params, queryEnv, - &planduration); + ExplainOnePlanCompat(plan, into, es, queryString, params, queryEnv, + &planduration, NULL); } } diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index be38c468c..cc5357ad0 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -36,6 +36,7 @@ #define standard_planner_compat(a,b,c,d) standard_planner(a,b,c,d) #define PortalDefineQuerySelectCompat(a,b,c,e,f) PortalDefineQuery(a,b,c,CMDTAG_SELECT,e,f) #define getOwnedSequencesCompat(a,b) getOwnedSequences(a) +#define ExplainOnePlanCompat(a,b,c,d,e,f,g,h) ExplainOnePlan(a,b,c,d,e,f,g,h) #else /* pre PG13 */ #define lnext_compat(l, r) lnext(r) #define list_delete_cell_compat(l,c,p) list_delete_cell(l,c,p) @@ -44,6 +45,7 @@ #define standard_planner_compat(a,b,c,d) standard_planner(a,c,d) #define PortalDefineQuerySelectCompat(a,b,c,e,f) PortalDefineQuery(a,b,c,"SELECT",e,f) #define getOwnedSequencesCompat(a,b) getOwnedSequences(a,b) +#define ExplainOnePlanCompat(a,b,c,d,e,f,g,h) ExplainOnePlan(a,b,c,d,e,f,g) #endif #if PG_VERSION_NUM >= PG_VERSION_12 From ab85a8129df0d23ba1cb5685de3bdb92e306a952 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Wed, 20 May 2020 12:10:07 +0300 Subject: [PATCH 12/52] map varoattno and varnoold fields in Var With PG13 varoattno and varnoold fields were renamed as varattnosyn and varnosyn. A macro is defined for these. Commit on Postgres side: 9ce77d75c5ab094637cc4a446296dc3be6e3c221 Command on Postgres side: git log --all --grep="varoattno" --- src/include/distributed/version_compat.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index cc5357ad0..53acbf786 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -37,6 +37,8 @@ #define PortalDefineQuerySelectCompat(a,b,c,e,f) PortalDefineQuery(a,b,c,CMDTAG_SELECT,e,f) #define getOwnedSequencesCompat(a,b) getOwnedSequences(a) #define ExplainOnePlanCompat(a,b,c,d,e,f,g,h) ExplainOnePlan(a,b,c,d,e,f,g,h) +#define varoattno varattnosyn +#define varnoold varnosyn #else /* pre PG13 */ #define lnext_compat(l, r) lnext(r) #define list_delete_cell_compat(l,c,p) list_delete_cell(l,c,p) From 4ed30a08241292112c73389ba2eb38ca43b0cc56 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Wed, 20 May 2020 12:25:25 +0300 Subject: [PATCH 13/52] create Set_ptr_value Since PG13 changed the list, a listcell doesn't contain data anymore. Therefore Set_ptr_value macro is created, so that depending on the version it will either use cell->data.ptr_value or cell->ptr_value. Commit on Postgres side: 1cff1b95ab6ddae32faa3efe0d95a820dbfdc164 --- src/backend/distributed/planner/multi_physical_planner.c | 2 +- src/backend/distributed/planner/multi_router_planner.c | 3 +-- src/include/distributed/version_compat.h | 2 ++ 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/backend/distributed/planner/multi_physical_planner.c b/src/backend/distributed/planner/multi_physical_planner.c index afdb65c31..38fd3ed61 100644 --- a/src/backend/distributed/planner/multi_physical_planner.c +++ b/src/backend/distributed/planner/multi_physical_planner.c @@ -5141,7 +5141,7 @@ GreedyAssignTask(WorkerNode *workerNode, List *taskList, List *activeShardPlacem rotatePlacementListBy = replicaIndex; /* overwrite task list to signal that this task is assigned */ - taskCell->data.ptr_value = NULL; + Set_ptr_value(taskCell, NULL); break; } } diff --git a/src/backend/distributed/planner/multi_router_planner.c b/src/backend/distributed/planner/multi_router_planner.c index 6fd2b3159..75493e91a 100644 --- a/src/backend/distributed/planner/multi_router_planner.c +++ b/src/backend/distributed/planner/multi_router_planner.c @@ -2976,8 +2976,7 @@ NormalizeMultiRowInsertTargetList(Query *query) expandedValuesList = lappend(expandedValuesList, targetExpr); } - - valuesListCell->data.ptr_value = (void *) expandedValuesList; + Set_ptr_value(valuesListCell, (void *) expandedValuesList); } /* reset coltypes, coltypmods, colcollations and rebuild them below */ diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index 53acbf786..6eaeaabcd 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -39,6 +39,7 @@ #define ExplainOnePlanCompat(a,b,c,d,e,f,g,h) ExplainOnePlan(a,b,c,d,e,f,g,h) #define varoattno varattnosyn #define varnoold varnosyn +#define Set_ptr_value(a,b) a->ptr_value = b #else /* pre PG13 */ #define lnext_compat(l, r) lnext(r) #define list_delete_cell_compat(l,c,p) list_delete_cell(l,c,p) @@ -48,6 +49,7 @@ #define PortalDefineQuerySelectCompat(a,b,c,e,f) PortalDefineQuery(a,b,c,"SELECT",e,f) #define getOwnedSequencesCompat(a,b) getOwnedSequences(a,b) #define ExplainOnePlanCompat(a,b,c,d,e,f,g,h) ExplainOnePlan(a,b,c,d,e,f,g) +#define Set_ptr_value(a,b) a->data.ptr_value = b #endif #if PG_VERSION_NUM >= PG_VERSION_12 From 1a7ccac6efa19d1b6e1c019898d95da3fed6e12a Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Wed, 20 May 2020 12:56:49 +0300 Subject: [PATCH 14/52] Add RangeTableEntryFromNSItem macro addRangeTableEntryXXX methods return a ParseNamespaceItem with pg >= 13. RangeTableEntryFromNSItem macro is added so that we return the range table entry from the ParseNamespaceItem in pg>=13 and for pg < 13 rte would already be returned with addRangeTableEntryXXX methods. Commit on Postgres side: 5815696bc66b3092f6361f53e0394909647042c8 --- src/backend/distributed/executor/insert_select_executor.c | 5 ++--- src/include/distributed/version_compat.h | 2 ++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/backend/distributed/executor/insert_select_executor.c b/src/backend/distributed/executor/insert_select_executor.c index 87cdbad10..c85bbf958 100644 --- a/src/backend/distributed/executor/insert_select_executor.c +++ b/src/backend/distributed/executor/insert_select_executor.c @@ -346,9 +346,8 @@ WrapSubquery(Query *subquery) /* create range table entries */ Alias *selectAlias = makeAlias("citus_insert_select_subquery", NIL); - RangeTblEntry *newRangeTableEntry = addRangeTableEntryForSubquery(pstate, subquery, - selectAlias, false, - true); + RangeTblEntry *newRangeTableEntry = RangeTableEntryFromNSItem(addRangeTableEntryForSubquery( + pstate, subquery, selectAlias, false, true)); outerQuery->rtable = list_make1(newRangeTableEntry); /* set the FROM expression to the subquery */ diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index 6eaeaabcd..80e67e06b 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -40,6 +40,7 @@ #define varoattno varattnosyn #define varnoold varnosyn #define Set_ptr_value(a,b) a->ptr_value = b +#define RangeTableEntryFromNSItem(a) a->p_rte #else /* pre PG13 */ #define lnext_compat(l, r) lnext(r) #define list_delete_cell_compat(l,c,p) list_delete_cell(l,c,p) @@ -50,6 +51,7 @@ #define getOwnedSequencesCompat(a,b) getOwnedSequences(a,b) #define ExplainOnePlanCompat(a,b,c,d,e,f,g,h) ExplainOnePlan(a,b,c,d,e,f,g) #define Set_ptr_value(a,b) a->data.ptr_value = b +#define RangeTableEntryFromNSItem(a) a #endif #if PG_VERSION_NUM >= PG_VERSION_12 From 9f1ec792b33a2e4fb131d7b2eefbbf17911d4e74 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Wed, 27 May 2020 11:10:48 +0300 Subject: [PATCH 15/52] add queryString to distributed_planner distributed_planner now takes query string as a parameter. related commit on PG side: 6aba63ef3e606db71beb596210dd95fa73c44ce2 --- src/backend/distributed/planner/distributed_planner.c | 7 ++++++- src/include/distributed/distributed_planner.h | 9 +++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/backend/distributed/planner/distributed_planner.c b/src/backend/distributed/planner/distributed_planner.c index 738b3b37e..4c22b4f19 100644 --- a/src/backend/distributed/planner/distributed_planner.c +++ b/src/backend/distributed/planner/distributed_planner.c @@ -127,7 +127,12 @@ static PlannedStmt * PlanDistributedStmt(DistributedPlanningContext *planContext /* Distributed planner hook */ PlannedStmt * -distributed_planner(Query *parse, int cursorOptions, ParamListInfo boundParams) +distributed_planner(Query *parse, + #if PG_VERSION_NUM >= PG_VERSION_13 + const char *query_string, + #endif + int cursorOptions, + ParamListInfo boundParams) { bool needsDistributedPlanning = false; bool fastPathRouterQuery = false; diff --git a/src/include/distributed/distributed_planner.h b/src/include/distributed/distributed_planner.h index 581eeb948..9a71b939c 100644 --- a/src/include/distributed/distributed_planner.h +++ b/src/include/distributed/distributed_planner.h @@ -183,8 +183,13 @@ typedef struct CitusCustomScanPath } CitusCustomScanPath; -extern PlannedStmt * distributed_planner(Query *parse, int cursorOptions, - ParamListInfo boundParams); +extern PlannedStmt * distributed_planner( + Query *parse, + #if PG_VERSION_NUM >= PG_VERSION_13 + const char *query_string, + #endif + int cursorOptions, + ParamListInfo boundParams); extern List * ExtractRangeTableEntryList(Query *query); extern List * ExtractReferenceTableRTEList(List *rteList); extern bool NeedsDistributedPlanning(Query *query); From 38aaf1faba08fb3126e7d8c88e98fbb72a06a5d1 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Wed, 27 May 2020 12:29:59 +0300 Subject: [PATCH 16/52] use QueryCompletion struct Postgres introduced QueryCompletion struct. Hence a compat utility is added to finish query completion for older versions and pg >= 13. The commit on Postgres side: 2f9661311b83dc481fc19f6e3bda015392010a40 --- src/backend/distributed/commands/multi_copy.c | 38 +++++++++++-------- .../distributed/commands/utility_hook.c | 9 ++++- src/include/distributed/commands/multi_copy.h | 3 +- .../distributed/commands/utility_hook.h | 9 ++++- src/include/distributed/version_compat.h | 2 + 5 files changed, 41 insertions(+), 20 deletions(-) diff --git a/src/backend/distributed/commands/multi_copy.c b/src/backend/distributed/commands/multi_copy.c index a8af26a8e..19df51152 100644 --- a/src/backend/distributed/commands/multi_copy.c +++ b/src/backend/distributed/commands/multi_copy.c @@ -93,6 +93,7 @@ #include "distributed/hash_helpers.h" #include "executor/executor.h" #include "foreign/foreign.h" +#include "tcop/cmdtag.h" #include "libpq/libpq.h" #include "libpq/pqformat.h" #include "nodes/makefuncs.h" @@ -211,8 +212,8 @@ typedef struct ShardConnections /* Local functions forward declarations */ -static void CopyToExistingShards(CopyStmt *copyStatement, char *completionTag); -static void CopyToNewShards(CopyStmt *copyStatement, char *completionTag, Oid relationId); +static void CopyToExistingShards(CopyStmt *copyStatement, QueryCompletionCompat *completionTag); +static void CopyToNewShards(CopyStmt *copyStatement, QueryCompletionCompat *completionTag, Oid relationId); static void OpenCopyConnectionsForNewShards(CopyStmt *copyStatement, ShardConnections *shardConnections, bool stopOnFailure, @@ -244,7 +245,7 @@ static FmgrInfo * TypeOutputFunctions(uint32 columnCount, Oid *typeIdArray, bool binaryFormat); static List * CopyGetAttnums(TupleDesc tupDesc, Relation rel, List *attnamelist); static bool CopyStatementHasFormat(CopyStmt *copyStatement, char *formatName); -static void CitusCopyFrom(CopyStmt *copyStatement, char *completionTag); +static void CitusCopyFrom(CopyStmt *copyStatement, QueryCompletionCompat *completionTag); static HTAB * CreateConnectionStateHash(MemoryContext memoryContext); static HTAB * CreateShardStateHash(MemoryContext memoryContext); static CopyConnectionState * GetConnectionState(HTAB *connectionStateHash, @@ -277,7 +278,7 @@ static void UnclaimCopyConnections(List *connectionStateList); static void ShutdownCopyConnectionState(CopyConnectionState *connectionState, CitusCopyDestReceiver *copyDest); static SelectStmt * CitusCopySelect(CopyStmt *copyStatement); -static void CitusCopyTo(CopyStmt *copyStatement, char *completionTag); +static void CitusCopyTo(CopyStmt *copyStatement, QueryCompletionCompat *completionTag); static int64 ForwardCopyDataFromConnection(CopyOutState copyOutState, MultiConnection *connection); @@ -313,6 +314,7 @@ static bool CitusCopyDestReceiverReceive(TupleTableSlot *slot, static void CitusCopyDestReceiverShutdown(DestReceiver *destReceiver); static void CitusCopyDestReceiverDestroy(DestReceiver *destReceiver); static bool ContainsLocalPlacement(int64 shardId); +static void CompleteCopyQueryTagCompat(QueryCompletionCompat* completionTag, uint64 processedRowCount); static void FinishLocalCopy(CitusCopyDestReceiver *copyDest); static void CloneCopyOutStateForLocalCopy(CopyOutState from, CopyOutState to); static bool ShouldExecuteCopyLocally(bool isIntermediateResult); @@ -329,7 +331,7 @@ PG_FUNCTION_INFO_V1(citus_text_send_as_jsonb); * and the partition method of the distributed table. */ static void -CitusCopyFrom(CopyStmt *copyStatement, char *completionTag) +CitusCopyFrom(CopyStmt *copyStatement, QueryCompletionCompat *completionTag) { UseCoordinatedTransaction(); @@ -385,7 +387,7 @@ CitusCopyFrom(CopyStmt *copyStatement, char *completionTag) * rows. */ static void -CopyToExistingShards(CopyStmt *copyStatement, char *completionTag) +CopyToExistingShards(CopyStmt *copyStatement, QueryCompletionCompat *completionTag) { Oid tableId = RangeVarGetRelid(copyStatement->relation, NoLock, false); @@ -554,8 +556,7 @@ CopyToExistingShards(CopyStmt *copyStatement, char *completionTag) if (completionTag != NULL) { - SafeSnprintf(completionTag, COMPLETION_TAG_BUFSIZE, - "COPY " UINT64_FORMAT, processedRowCount); + CompleteCopyQueryTagCompat(completionTag, processedRowCount); } } @@ -565,7 +566,7 @@ CopyToExistingShards(CopyStmt *copyStatement, char *completionTag) * tables where we create new shards into which to copy rows. */ static void -CopyToNewShards(CopyStmt *copyStatement, char *completionTag, Oid relationId) +CopyToNewShards(CopyStmt *copyStatement, QueryCompletionCompat *completionTag, Oid relationId) { /* allocate column values and nulls arrays */ Relation distributedRelation = table_open(relationId, RowExclusiveLock); @@ -739,11 +740,19 @@ CopyToNewShards(CopyStmt *copyStatement, char *completionTag, Oid relationId) if (completionTag != NULL) { - SafeSnprintf(completionTag, COMPLETION_TAG_BUFSIZE, - "COPY " UINT64_FORMAT, processedRowCount); + CompleteCopyQueryTagCompat(completionTag, processedRowCount); } } +static void CompleteCopyQueryTagCompat(QueryCompletionCompat* completionTag, uint64 processedRowCount) { + #if PG_VERSION_NUM >= PG_VERSION_13 + SetQueryCompletion(completionTag, CMDTAG_COPY, processedRowCount); + #else + SafeSnprintf(completionTag, COMPLETION_TAG_BUFSIZE, + "COPY " UINT64_FORMAT, processedRowCount); + #endif +} + /* * RemoveOptionFromList removes an option from a list of options in a @@ -2769,7 +2778,7 @@ CopyStatementHasFormat(CopyStmt *copyStatement, char *formatName) * further processing is needed. */ Node * -ProcessCopyStmt(CopyStmt *copyStatement, char *completionTag, const char *queryString) +ProcessCopyStmt(CopyStmt *copyStatement, QueryCompletionCompat *completionTag, const char *queryString) { /* * Handle special COPY "resultid" FROM STDIN WITH (format result) commands @@ -2917,7 +2926,7 @@ CitusCopySelect(CopyStmt *copyStatement) * table dump. */ static void -CitusCopyTo(CopyStmt *copyStatement, char *completionTag) +CitusCopyTo(CopyStmt *copyStatement, QueryCompletionCompat *completionTag) { ListCell *shardIntervalCell = NULL; int64 tuplesSent = 0; @@ -3008,8 +3017,7 @@ CitusCopyTo(CopyStmt *copyStatement, char *completionTag) if (completionTag != NULL) { - SafeSnprintf(completionTag, COMPLETION_TAG_BUFSIZE, "COPY " UINT64_FORMAT, - tuplesSent); + CompleteCopyQueryTagCompat(completionTag, tuplesSent); } } diff --git a/src/backend/distributed/commands/utility_hook.c b/src/backend/distributed/commands/utility_hook.c index 682c08240..af80c8b31 100644 --- a/src/backend/distributed/commands/utility_hook.c +++ b/src/backend/distributed/commands/utility_hook.c @@ -25,6 +25,8 @@ *------------------------------------------------------------------------- */ +#include "distributed/pg_version_constants.h" + #include "postgres.h" #include "miscadmin.h" @@ -88,7 +90,9 @@ static bool IsDropSchemaOrDB(Node *parsetree); */ void CitusProcessUtility(Node *node, const char *queryString, ProcessUtilityContext context, - ParamListInfo params, DestReceiver *dest, char *completionTag) + ParamListInfo params, DestReceiver *dest, + QueryCompletionCompat *completionTag + ) { PlannedStmt *plannedStmt = makeNode(PlannedStmt); plannedStmt->commandType = CMD_UTILITY; @@ -115,7 +119,8 @@ multi_ProcessUtility(PlannedStmt *pstmt, ParamListInfo params, struct QueryEnvironment *queryEnv, DestReceiver *dest, - char *completionTag) + QueryCompletionCompat *completionTag + ) { Node *parsetree = pstmt->utilityStmt; List *ddlJobs = NIL; diff --git a/src/include/distributed/commands/multi_copy.h b/src/include/distributed/commands/multi_copy.h index b76037706..e840b73b6 100644 --- a/src/include/distributed/commands/multi_copy.h +++ b/src/include/distributed/commands/multi_copy.h @@ -15,6 +15,7 @@ #include "distributed/metadata_utility.h" #include "distributed/metadata_cache.h" +#include "distributed/version_compat.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_coerce.h" @@ -155,7 +156,7 @@ extern void AppendCopyBinaryHeaders(CopyOutState headerOutputState); extern void AppendCopyBinaryFooters(CopyOutState footerOutputState); extern void EndRemoteCopy(int64 shardId, List *connectionList); extern List * CreateRangeTable(Relation rel, AclMode requiredAccess); -extern Node * ProcessCopyStmt(CopyStmt *copyStatement, char *completionTag, +extern Node * ProcessCopyStmt(CopyStmt *copyStatement, QueryCompletionCompat *completionTag, const char *queryString); extern void CheckCopyPermissions(CopyStmt *copyStatement); extern bool IsCopyResultStmt(CopyStmt *copyStatement); diff --git a/src/include/distributed/commands/utility_hook.h b/src/include/distributed/commands/utility_hook.h index 56ea1e48c..c9543d8b1 100644 --- a/src/include/distributed/commands/utility_hook.h +++ b/src/include/distributed/commands/utility_hook.h @@ -10,6 +10,8 @@ #ifndef MULTI_UTILITY_H #define MULTI_UTILITY_H +#include "distributed/pg_version_constants.h" + #include "postgres.h" #include "utils/relcache.h" @@ -51,10 +53,13 @@ typedef struct DDLJob extern void multi_ProcessUtility(PlannedStmt *pstmt, const char *queryString, ProcessUtilityContext context, ParamListInfo params, struct QueryEnvironment *queryEnv, DestReceiver *dest, - char *completionTag); + QueryCompletionCompat *completionTag + ); extern void CitusProcessUtility(Node *node, const char *queryString, ProcessUtilityContext context, ParamListInfo params, - DestReceiver *dest, char *completionTag); + DestReceiver *dest, + QueryCompletionCompat * completionTag + ); extern void MarkInvalidateForeignKeyGraph(void); extern void InvalidateForeignKeyGraphForDDL(void); extern List * DDLTaskList(Oid relationId, const char *commandString); diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index 80e67e06b..9c2fbfb91 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -41,6 +41,7 @@ #define varnoold varnosyn #define Set_ptr_value(a,b) a->ptr_value = b #define RangeTableEntryFromNSItem(a) a->p_rte +#define QueryCompletionCompat QueryCompletion #else /* pre PG13 */ #define lnext_compat(l, r) lnext(r) #define list_delete_cell_compat(l,c,p) list_delete_cell(l,c,p) @@ -52,6 +53,7 @@ #define ExplainOnePlanCompat(a,b,c,d,e,f,g,h) ExplainOnePlan(a,b,c,d,e,f,g) #define Set_ptr_value(a,b) a->data.ptr_value = b #define RangeTableEntryFromNSItem(a) a +#define QueryCompletionCompat char #endif #if PG_VERSION_NUM >= PG_VERSION_12 From 135af848599012985c4522ba337a6a90abbee3e6 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Fri, 29 May 2020 10:47:13 +0300 Subject: [PATCH 17/52] Update ruleutils for join related changes of postgres Postgres changed some join related fields and therefore they also changed ruleutils, this commit applies those changes to our copy of ruleutils. Related commit on postgres side: 9ce77d75c5ab094637cc4a446296dc3be6e3c221 --- .../distributed/deparser/ruleutils_13.c | 247 ++++++------------ 1 file changed, 84 insertions(+), 163 deletions(-) diff --git a/src/backend/distributed/deparser/ruleutils_13.c b/src/backend/distributed/deparser/ruleutils_13.c index 18ef36b84..2919f64ae 100644 --- a/src/backend/distributed/deparser/ruleutils_13.c +++ b/src/backend/distributed/deparser/ruleutils_13.c @@ -337,8 +337,6 @@ static char *make_colname_unique(char *colname, deparse_namespace *dpns, static void expand_colnames_array_to(deparse_columns *colinfo, int n); static void identify_join_columns(JoinExpr *j, RangeTblEntry *jrte, deparse_columns *colinfo); -static void flatten_join_using_qual(Node *qual, - List **leftvars, List **rightvars); static char *get_rtable_name(int rtindex, deparse_context *context); static void set_deparse_plan(deparse_namespace *dpns, Plan *plan); static void push_child_plan(deparse_namespace *dpns, Plan *plan, @@ -775,13 +773,13 @@ has_dangerous_join_using(deparse_namespace *dpns, Node *jtnode) * dangerous situation and must pick unique aliases. */ RangeTblEntry *jrte = rt_fetch(j->rtindex, dpns->rtable); - ListCell *lc; - foreach(lc, jrte->joinaliasvars) + /* We need only examine the merged columns */ + for (int i = 0; i < jrte->joinmergedcols; i++) { - Var *aliasvar = (Var *) lfirst(lc); + Node *aliasvar = list_nth(jrte->joinaliasvars, i); - if (aliasvar != NULL && !IsA(aliasvar, Var)) + if (!IsA(aliasvar, Var)) return true; } } @@ -1190,12 +1188,8 @@ set_join_column_names(deparse_namespace *dpns, RangeTblEntry *rte, char *colname = colinfo->colnames[i]; char *real_colname; - /* Ignore dropped column (only possible for non-merged column) */ - if (colinfo->leftattnos[i] == 0 && colinfo->rightattnos[i] == 0) - { - Assert(colname == NULL); - continue; - } + /* Join column must refer to at least one input column */ + Assert(colinfo->leftattnos[i] != 0 || colinfo->rightattnos[i] != 0); /* Get the child column name */ if (colinfo->leftattnos[i] > 0) @@ -1207,7 +1201,12 @@ set_join_column_names(deparse_namespace *dpns, RangeTblEntry *rte, /* We're joining system columns --- use eref name */ real_colname = strVal(list_nth(rte->eref->colnames, i)); } - Assert(real_colname != NULL); + /* If child col has been dropped, no need to assign a join colname */ + if (real_colname == NULL) + { + colinfo->colnames[i] = NULL; + continue; + } /* In an unnamed join, just report child column names as-is */ if (rte->alias == NULL) @@ -1532,7 +1531,8 @@ identify_join_columns(JoinExpr *j, RangeTblEntry *jrte, deparse_columns *colinfo) { int numjoincols; - int i; + int jcolno; + int rcolno; ListCell *lc; /* Extract left/right child RT indexes */ @@ -1561,146 +1561,32 @@ identify_join_columns(JoinExpr *j, RangeTblEntry *jrte, colinfo->leftattnos = (int *) palloc0(numjoincols * sizeof(int)); colinfo->rightattnos = (int *) palloc0(numjoincols * sizeof(int)); - /* Scan the joinaliasvars list to identify simple column references */ - i = 0; - foreach(lc, jrte->joinaliasvars) - { - Var *aliasvar = (Var *) lfirst(lc); - - /* get rid of any implicit coercion above the Var */ - aliasvar = (Var *) strip_implicit_coercions((Node *) aliasvar); - - if (aliasvar == NULL) - { - /* It's a dropped column; nothing to do here */ - } - else if (IsA(aliasvar, Var)) - { - Assert(aliasvar->varlevelsup == 0); - Assert(aliasvar->varattno != 0); - if (aliasvar->varno == colinfo->leftrti) - colinfo->leftattnos[i] = aliasvar->varattno; - else if (aliasvar->varno == colinfo->rightrti) - colinfo->rightattnos[i] = aliasvar->varattno; - else - elog(ERROR, "unexpected varno %d in JOIN RTE", - aliasvar->varno); - } - else if (IsA(aliasvar, CoalesceExpr)) - { - /* - * It's a merged column in FULL JOIN USING. Ignore it for now and - * let the code below identify the merged columns. - */ - } - else - elog(ERROR, "unrecognized node type in join alias vars: %d", - (int) nodeTag(aliasvar)); - - i++; - } - - /* - * If there's a USING clause, deconstruct the join quals to identify the - * merged columns. This is a tad painful but if we cannot rely on the - * column names, there is no other representation of which columns were - * joined by USING. (Unless the join type is FULL, we can't tell from the - * joinaliasvars list which columns are merged.) Note: we assume that the - * merged columns are the first output column(s) of the join. + /* + * Deconstruct RTE's joinleftcols/joinrightcols into desired format. + * Recall that the column(s) merged due to USING are the first column(s) + * of the join output. We need not do anything special while scanning + * joinleftcols, but while scanning joinrightcols we must distinguish + * merged from unmerged columns. */ - if (j->usingClause) + jcolno = 0; + foreach(lc, jrte->joinleftcols) { - List *leftvars = NIL; - List *rightvars = NIL; - ListCell *lc2; + int leftattno = lfirst_int(lc); - /* Extract left- and right-side Vars from the qual expression */ - flatten_join_using_qual(j->quals, &leftvars, &rightvars); - Assert(list_length(leftvars) == list_length(j->usingClause)); - Assert(list_length(rightvars) == list_length(j->usingClause)); - - /* Mark the output columns accordingly */ - i = 0; - forboth(lc, leftvars, lc2, rightvars) - { - Var *leftvar = (Var *) lfirst(lc); - Var *rightvar = (Var *) lfirst(lc2); - - Assert(leftvar->varlevelsup == 0); - Assert(leftvar->varattno != 0); - if (leftvar->varno != colinfo->leftrti) - elog(ERROR, "unexpected varno %d in JOIN USING qual", - leftvar->varno); - colinfo->leftattnos[i] = leftvar->varattno; - - Assert(rightvar->varlevelsup == 0); - Assert(rightvar->varattno != 0); - if (rightvar->varno != colinfo->rightrti) - elog(ERROR, "unexpected varno %d in JOIN USING qual", - rightvar->varno); - colinfo->rightattnos[i] = rightvar->varattno; - - i++; - } + colinfo->leftattnos[jcolno++] = leftattno; } -} - -/* - * flatten_join_using_qual: extract Vars being joined from a JOIN/USING qual - * - * We assume that transformJoinUsingClause won't have produced anything except - * AND nodes, equality operator nodes, and possibly implicit coercions, and - * that the AND node inputs match left-to-right with the original USING list. - * - * Caller must initialize the result lists to NIL. - */ -static void -flatten_join_using_qual(Node *qual, List **leftvars, List **rightvars) -{ - if (IsA(qual, BoolExpr)) + rcolno = 0; + foreach(lc, jrte->joinrightcols) { - /* Handle AND nodes by recursion */ - BoolExpr *b = (BoolExpr *) qual; - ListCell *lc; + int rightattno = lfirst_int(lc); - Assert(b->boolop == AND_EXPR); - foreach(lc, b->args) - { - flatten_join_using_qual((Node *) lfirst(lc), - leftvars, rightvars); - } - } - else if (IsA(qual, OpExpr)) - { - /* Otherwise we should have an equality operator */ - OpExpr *op = (OpExpr *) qual; - Var *var; - - if (list_length(op->args) != 2) - elog(ERROR, "unexpected unary operator in JOIN/USING qual"); - /* Arguments should be Vars with perhaps implicit coercions */ - var = (Var *) strip_implicit_coercions((Node *) linitial(op->args)); - if (!IsA(var, Var)) - elog(ERROR, "unexpected node type in JOIN/USING qual: %d", - (int) nodeTag(var)); - *leftvars = lappend(*leftvars, var); - var = (Var *) strip_implicit_coercions((Node *) lsecond(op->args)); - if (!IsA(var, Var)) - elog(ERROR, "unexpected node type in JOIN/USING qual: %d", - (int) nodeTag(var)); - *rightvars = lappend(*rightvars, var); - } - else - { - /* Perhaps we have an implicit coercion to boolean? */ - Node *q = strip_implicit_coercions(qual); - - if (q != qual) - flatten_join_using_qual(q, leftvars, rightvars); + if (rcolno < jrte->joinmergedcols) /* merged column? */ + colinfo->rightattnos[rcolno] = rightattno; else - elog(ERROR, "unexpected node type in JOIN/USING qual: %d", - (int) nodeTag(qual)); + colinfo->rightattnos[jcolno++] = rightattno; + rcolno++; } + Assert(jcolno == numjoincols); } /* @@ -3643,6 +3529,8 @@ get_variable(Var *var, int levelsup, bool istoplevel, deparse_context *context) StringInfo buf = context->buf; RangeTblEntry *rte; AttrNumber attnum; + Index varno; + AttrNumber varattno; int netlevelsup; deparse_namespace *dpns; deparse_columns *colinfo; @@ -3656,6 +3544,24 @@ get_variable(Var *var, int levelsup, bool istoplevel, deparse_context *context) var->varlevelsup, levelsup); dpns = (deparse_namespace *) list_nth(context->namespaces, netlevelsup); + /* + * If we have a syntactic referent for the Var, and we're working from a + * parse tree, prefer to use the syntactic referent. Otherwise, fall back + * on the semantic referent. (Forcing use of the semantic referent when + * printing plan trees is a design choice that's perhaps more motivated by + * backwards compatibility than anything else. But it does have the + * advantage of making plans more explicit.) + */ + if (var->varnosyn > 0 && dpns->plan == NULL) + { + varno = var->varnosyn; + varattno = var->varattnosyn; + } + else + { + varno = var->varno; + varattno = var->varattno; + } /* * Try to find the relevant RTE in this rtable. In a plan tree, it's @@ -3663,10 +3569,9 @@ get_variable(Var *var, int levelsup, bool istoplevel, deparse_context *context) * down into the subplans, or INDEX_VAR, which is resolved similarly. Also * find the aliases previously assigned for this RTE. */ - if (var->varno >= 1 && var->varno <= list_length(dpns->rtable)) + if (varno >= 1 && varno <= list_length(dpns->rtable)) { - Index varno = var->varno; - AttrNumber varattno = var->varattno; + /* * We might have been asked to map child Vars to some parent relation. */ @@ -3995,6 +3900,8 @@ get_name_for_var_field(Var *var, int fieldno, AttrNumber attnum; int netlevelsup; deparse_namespace *dpns; + Index varno; + AttrNumber varattno; TupleDesc tupleDesc; Node *expr; @@ -4053,26 +3960,40 @@ get_name_for_var_field(Var *var, int fieldno, var->varlevelsup, levelsup); dpns = (deparse_namespace *) list_nth(context->namespaces, netlevelsup); - + /* + * If we have a syntactic referent for the Var, and we're working from a + * parse tree, prefer to use the syntactic referent. Otherwise, fall back + * on the semantic referent. (See comments in get_variable().) + */ + if (var->varnosyn > 0 && dpns->plan == NULL) + { + varno = var->varnosyn; + varattno = var->varattnosyn; + } + else + { + varno = var->varno; + varattno = var->varattno; + } /* * Try to find the relevant RTE in this rtable. In a plan tree, it's * likely that varno is OUTER_VAR or INNER_VAR, in which case we must dig * down into the subplans, or INDEX_VAR, which is resolved similarly. */ - if (var->varno >= 1 && var->varno <= list_length(dpns->rtable)) + if (varno >= 1 && varno <= list_length(dpns->rtable)) { - rte = rt_fetch(var->varno, dpns->rtable); - attnum = var->varattno; + rte = rt_fetch(varno, dpns->rtable); + attnum = varattno; } - else if (var->varno == OUTER_VAR && dpns->outer_tlist) + else if (varno == OUTER_VAR && dpns->outer_tlist) { TargetEntry *tle; deparse_namespace save_dpns; const char *result; - tle = get_tle_by_resno(dpns->outer_tlist, var->varattno); + tle = get_tle_by_resno(dpns->outer_tlist, varattno); if (!tle) - elog(ERROR, "bogus varattno for OUTER_VAR var: %d", var->varattno); + elog(ERROR, "bogus varattno for OUTER_VAR var: %d", varattno); Assert(netlevelsup == 0); push_child_plan(dpns, dpns->outer_plan, &save_dpns); @@ -4083,15 +4004,15 @@ get_name_for_var_field(Var *var, int fieldno, pop_child_plan(dpns, &save_dpns); return result; } - else if (var->varno == INNER_VAR && dpns->inner_tlist) + else if (varno == INNER_VAR && dpns->inner_tlist) { TargetEntry *tle; deparse_namespace save_dpns; const char *result; - tle = get_tle_by_resno(dpns->inner_tlist, var->varattno); + tle = get_tle_by_resno(dpns->inner_tlist, varattno); if (!tle) - elog(ERROR, "bogus varattno for INNER_VAR var: %d", var->varattno); + elog(ERROR, "bogus varattno for INNER_VAR var: %d", varattno); Assert(netlevelsup == 0); push_child_plan(dpns, dpns->inner_plan, &save_dpns); @@ -4102,14 +4023,14 @@ get_name_for_var_field(Var *var, int fieldno, pop_child_plan(dpns, &save_dpns); return result; } - else if (var->varno == INDEX_VAR && dpns->index_tlist) + else if (varno == INDEX_VAR && dpns->index_tlist) { TargetEntry *tle; const char *result; - tle = get_tle_by_resno(dpns->index_tlist, var->varattno); + tle = get_tle_by_resno(dpns->index_tlist, varattno); if (!tle) - elog(ERROR, "bogus varattno for INDEX_VAR var: %d", var->varattno); + elog(ERROR, "bogus varattno for INDEX_VAR var: %d", varattno); Assert(netlevelsup == 0); @@ -4120,7 +4041,7 @@ get_name_for_var_field(Var *var, int fieldno, } else { - elog(ERROR, "bogus varno: %d", var->varno); + elog(ERROR, "bogus varno: %d", varno); return NULL; /* keep compiler quiet */ } From bc20920252726dda9a70cabfaf480747ea700931 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Fri, 29 May 2020 22:22:17 +0300 Subject: [PATCH 18/52] introduce SetJoinRelatedColumnsCompat PG13 uses joinmergedcols, joinleftcols and joinrightcols for finding join order now. There relevant fields are set on citus side. Postgres side commit: 9ce77d75c5ab094637cc4a446296dc3be6e3c221 --- .../planner/multi_physical_planner.c | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/backend/distributed/planner/multi_physical_planner.c b/src/backend/distributed/planner/multi_physical_planner.c index 38fd3ed61..c82239db8 100644 --- a/src/backend/distributed/planner/multi_physical_planner.c +++ b/src/backend/distributed/planner/multi_physical_planner.c @@ -130,6 +130,8 @@ static List * QuerySelectClauseList(MultiNode *multiNode); static List * QueryFromList(List *rangeTableList); static Node * QueryJoinTree(MultiNode *multiNode, List *dependentJobList, List **rangeTableList); +static void SetJoinRelatedColumnsCompat(RangeTblEntry *rangeTableEntry, int joinMergedCols, + List *leftColumnVars, List *rightColumnVars); static RangeTblEntry * JoinRangeTableEntry(JoinExpr *joinExpr, List *dependentJobList, List *rangeTableList); static int ExtractRangeTableId(Node *node); @@ -1260,9 +1262,33 @@ JoinRangeTableEntry(JoinExpr *joinExpr, List *dependentJobList, List *rangeTable rangeTableEntry->eref->colnames = joinedColumnNames; rangeTableEntry->joinaliasvars = joinedColumnVars; + SetJoinRelatedColumnsCompat(rangeTableEntry, list_length(joinExpr->usingClause), + leftColumnVars, rightColumnVars); + return rangeTableEntry; } +static void SetJoinRelatedColumnsCompat(RangeTblEntry *rangeTableEntry, int joinMergedCols, + List *leftColumnVars, List *rightColumnVars) { + #if PG_VERSION_NUM >= PG_VERSION_13 + + rangeTableEntry->joinmergedcols = joinMergedCols; + + Var* var = NULL; + List* joinleftcols = NIL; + foreach_ptr(var, leftColumnVars) { + joinleftcols = lappend_int(joinleftcols, var->varno); + } + + List* joinrightcols = NIL; + foreach_ptr(var, rightColumnVars) { + joinrightcols = lappend_int(joinrightcols, var->varno); + } + + rangeTableEntry->joinleftcols = joinleftcols; + rangeTableEntry->joinrightcols = joinrightcols; + #endif +} /* * ExtractRangeTableId gets the range table id from a node that could From 3cc7717e64a48b5f07d324b6ca38f03af4f755ac Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Mon, 1 Jun 2020 11:17:45 +0300 Subject: [PATCH 19/52] Fill new join fields for PG>=13 For joins 3 new fields are added, joinleftcols, joinrightcols, and joinmergedcols. We are not interested in joinmergedcols because we always expand the column used in joins. There joinmergedcols is always 0 in our case. For filling joinleftcols and joinrightcols we basically construct the lists with sequences so either list is of the form: [1 2 3 4 .... n] Ruleutils is not completed synced with postgres ruleutils and the most important part is identify_join_columns function change, which now uses joinleftcols and joinrightcols. Commit on postgres side: 9ce77d75c5ab094637cc4a446296dc3be6e3c221 A useful email thread: https://www.postgresql.org/message-id/flat/7115.1577986646%40sss.pgh.pa.us#0ae1d66feeb400013fbaa67a7cccd6ca --- .../distributed/deparser/ruleutils_13.c | 34 ++++++------- .../planner/multi_physical_planner.c | 50 +++++++++---------- 2 files changed, 41 insertions(+), 43 deletions(-) diff --git a/src/backend/distributed/deparser/ruleutils_13.c b/src/backend/distributed/deparser/ruleutils_13.c index 2919f64ae..bdd9e7f8e 100644 --- a/src/backend/distributed/deparser/ruleutils_13.c +++ b/src/backend/distributed/deparser/ruleutils_13.c @@ -1561,7 +1561,7 @@ identify_join_columns(JoinExpr *j, RangeTblEntry *jrte, colinfo->leftattnos = (int *) palloc0(numjoincols * sizeof(int)); colinfo->rightattnos = (int *) palloc0(numjoincols * sizeof(int)); - /* + /* * Deconstruct RTE's joinleftcols/joinrightcols into desired format. * Recall that the column(s) merged due to USING are the first column(s) * of the join output. We need not do anything special while scanning @@ -3552,16 +3552,16 @@ get_variable(Var *var, int levelsup, bool istoplevel, deparse_context *context) * backwards compatibility than anything else. But it does have the * advantage of making plans more explicit.) */ - if (var->varnosyn > 0 && dpns->plan == NULL) - { - varno = var->varnosyn; - varattno = var->varattnosyn; - } - else - { + // if (var->varnosyn > 0 && dpns->plan == NULL) + // { + // varno = var->varnosyn; + // varattno = var->varattnosyn; + // } + // else + // { varno = var->varno; varattno = var->varattno; - } + // } /* * Try to find the relevant RTE in this rtable. In a plan tree, it's @@ -3965,16 +3965,16 @@ get_name_for_var_field(Var *var, int fieldno, * parse tree, prefer to use the syntactic referent. Otherwise, fall back * on the semantic referent. (See comments in get_variable().) */ - if (var->varnosyn > 0 && dpns->plan == NULL) - { - varno = var->varnosyn; - varattno = var->varattnosyn; - } - else - { + // if (var->varnosyn > 0 && dpns->plan == NULL) + // { + // varno = var->varnosyn; + // varattno = var->varattnosyn; + // } + // else + // { varno = var->varno; varattno = var->varattno; - } + // } /* * Try to find the relevant RTE in this rtable. In a plan tree, it's * likely that varno is OUTER_VAR or INNER_VAR, in which case we must dig diff --git a/src/backend/distributed/planner/multi_physical_planner.c b/src/backend/distributed/planner/multi_physical_planner.c index c82239db8..874b69bba 100644 --- a/src/backend/distributed/planner/multi_physical_planner.c +++ b/src/backend/distributed/planner/multi_physical_planner.c @@ -130,8 +130,8 @@ static List * QuerySelectClauseList(MultiNode *multiNode); static List * QueryFromList(List *rangeTableList); static Node * QueryJoinTree(MultiNode *multiNode, List *dependentJobList, List **rangeTableList); -static void SetJoinRelatedColumnsCompat(RangeTblEntry *rangeTableEntry, int joinMergedCols, - List *leftColumnVars, List *rightColumnVars); +static void SetJoinRelatedColumnsCompat(RangeTblEntry *rangeTableEntry, + List *l_colnames, List *r_colnames, List* leftColVars, List* rightColVars); static RangeTblEntry * JoinRangeTableEntry(JoinExpr *joinExpr, List *dependentJobList, List *rangeTableList); static int ExtractRangeTableId(Node *node); @@ -1077,11 +1077,6 @@ QueryJoinTree(MultiNode *multiNode, List *dependentJobList, List **rangeTableLis joinExpr->jointype = JOIN_LEFT; } - RangeTblEntry *rangeTableEntry = JoinRangeTableEntry(joinExpr, - dependentJobList, - *rangeTableList); - *rangeTableList = lappend(*rangeTableList, rangeTableEntry); - /* fix the column attributes in ON (...) clauses */ List *columnList = pull_var_clause_default((Node *) joinNode->joinClauseList); foreach(columnCell, columnList) @@ -1097,6 +1092,11 @@ QueryJoinTree(MultiNode *multiNode, List *dependentJobList, List **rangeTableLis /* make AND clauses explicit after fixing them */ joinExpr->quals = (Node *) make_ands_explicit(joinNode->joinClauseList); + RangeTblEntry *rangeTableEntry = JoinRangeTableEntry(joinExpr, + dependentJobList, + *rangeTableList); + *rangeTableList = lappend(*rangeTableList, rangeTableEntry); + return (Node *) joinExpr; } @@ -1230,10 +1230,10 @@ static RangeTblEntry * JoinRangeTableEntry(JoinExpr *joinExpr, List *dependentJobList, List *rangeTableList) { RangeTblEntry *rangeTableEntry = makeNode(RangeTblEntry); - List *joinedColumnNames = NIL; - List *joinedColumnVars = NIL; List *leftColumnNames = NIL; List *leftColumnVars = NIL; + List *joinedColumnNames = NIL; + List *joinedColumnVars = NIL; int leftRangeTableId = ExtractRangeTableId(joinExpr->larg); RangeTblEntry *leftRTE = rt_fetch(leftRangeTableId, rangeTableList); List *rightColumnNames = NIL; @@ -1253,40 +1253,38 @@ JoinRangeTableEntry(JoinExpr *joinExpr, List *dependentJobList, List *rangeTable &leftColumnNames, &leftColumnVars); ExtractColumns(rightRTE, rightRangeTableId, dependentJobList, &rightColumnNames, &rightColumnVars); - joinedColumnNames = list_concat(joinedColumnNames, leftColumnNames); - joinedColumnVars = list_concat(joinedColumnVars, leftColumnVars); joinedColumnNames = list_concat(joinedColumnNames, rightColumnNames); + joinedColumnVars = list_concat(joinedColumnVars, leftColumnVars); joinedColumnVars = list_concat(joinedColumnVars, rightColumnVars); rangeTableEntry->eref->colnames = joinedColumnNames; rangeTableEntry->joinaliasvars = joinedColumnVars; - SetJoinRelatedColumnsCompat(rangeTableEntry, list_length(joinExpr->usingClause), - leftColumnVars, rightColumnVars); + SetJoinRelatedColumnsCompat(rangeTableEntry, + leftColumnNames, rightColumnNames, leftColumnVars, rightColumnVars); return rangeTableEntry; } -static void SetJoinRelatedColumnsCompat(RangeTblEntry *rangeTableEntry, int joinMergedCols, - List *leftColumnVars, List *rightColumnVars) { +static void SetJoinRelatedColumnsCompat(RangeTblEntry *rangeTableEntry, + List *leftColumnNames, List *rightColumnNames, List* leftColumnVars, List* rightColumnVars) { + #if PG_VERSION_NUM >= PG_VERSION_13 - rangeTableEntry->joinmergedcols = joinMergedCols; - + /* We don't have any merged columns so set it to 0 */ + rangeTableEntry->joinmergedcols = 0; Var* var = NULL; - List* joinleftcols = NIL; + int varId = 1; foreach_ptr(var, leftColumnVars) { - joinleftcols = lappend_int(joinleftcols, var->varno); - } - - List* joinrightcols = NIL; + rangeTableEntry->joinleftcols = lappend_int(rangeTableEntry->joinleftcols, varId); + varId++; + } + varId = 1; foreach_ptr(var, rightColumnVars) { - joinrightcols = lappend_int(joinrightcols, var->varno); + rangeTableEntry->joinrightcols = lappend_int(rangeTableEntry->joinrightcols, varId); + varId++; } - - rangeTableEntry->joinleftcols = joinleftcols; - rangeTableEntry->joinrightcols = joinrightcols; #endif } From 70f27c10e5cdc485a8cae93e94f3e459b1450707 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Wed, 3 Jun 2020 20:25:33 +0300 Subject: [PATCH 20/52] Add some normalization rules for tests The not-null constraint message changed with pg13 slightly hence a normalization rule is added for that, which converts it to pg < 13 output. Commit on postgres: 05f18c6b6b6e4b44302ee20a042cedc664532aa2 An extra debug message is added related to indexes on postgres, these are safe to be ignored, so we can delete them from tests. Commit on Postgres side: 612a1ab76724aa1514b6509269342649f8cab375 varnoold is renamed as varnosyn and varoattno is renamed as varattnosyn so in the output we normalize the values as the old ones to simply pass the tests. --- src/test/regress/bin/normalize.sed | 10 ++++++++++ src/test/regress/expected/local_shard_copy.out | 3 --- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/test/regress/bin/normalize.sed b/src/test/regress/bin/normalize.sed index 4d5a5f611..80dfe2122 100644 --- a/src/test/regress/bin/normalize.sed +++ b/src/test/regress/bin/normalize.sed @@ -86,6 +86,16 @@ s/_ref_id_id_fkey_/_ref_id_fkey_/g s/fk_test_2_col1_col2_fkey/fk_test_2_col1_fkey/g s/_id_other_column_ref_fkey/_id_fkey/g +# pg13 changes +s/of relation ".*" violates not-null constraint/violates not-null constraint/g +s/varnosyn/varnoold/g +s/varattnosyn/varoattno/g +/DEBUG: index ".*" can safely use deduplication.*$/d +/DEBUG: index ".*" cannot use deduplication.*$/d +/DEBUG: building index ".*" on table ".*" serially.*$/d + + + # intermediate_results s/(ERROR.*)pgsql_job_cache\/([0-9]+_[0-9]+_[0-9]+)\/(.*).data/\1pgsql_job_cache\/xx_x_xxx\/\3.data/g diff --git a/src/test/regress/expected/local_shard_copy.out b/src/test/regress/expected/local_shard_copy.out index 92c3f8ee8..8fdba05df 100644 --- a/src/test/regress/expected/local_shard_copy.out +++ b/src/test/regress/expected/local_shard_copy.out @@ -14,7 +14,6 @@ SET citus.shard_replication_factor TO 1; SET citus.replication_model TO 'streaming'; CREATE TABLE reference_table (key int PRIMARY KEY); DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "reference_table_pkey" for table "reference_table" -DEBUG: building index "reference_table_pkey" on table "reference_table" serially SELECT create_reference_table('reference_table'); create_reference_table --------------------------------------------------------------------- @@ -23,7 +22,6 @@ SELECT create_reference_table('reference_table'); CREATE TABLE distributed_table (key int PRIMARY KEY, age bigint CHECK (age >= 10)); DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "distributed_table_pkey" for table "distributed_table" -DEBUG: building index "distributed_table_pkey" on table "distributed_table" serially SELECT create_distributed_table('distributed_table','key'); create_distributed_table --------------------------------------------------------------------- @@ -38,7 +36,6 @@ DEBUG: distributed INSERT ... SELECT can only select from distributed tables DEBUG: Collecting INSERT ... SELECT results on coordinator CREATE TABLE local_table (key int PRIMARY KEY); DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "local_table_pkey" for table "local_table" -DEBUG: building index "local_table_pkey" on table "local_table" serially INSERT INTO local_table SELECT * from generate_series(1, 10); -- partitioned table CREATE TABLE collections_list ( From 0f6c21d41829add72ae90571aa820d27585b672a Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Wed, 3 Jun 2020 20:41:56 +0300 Subject: [PATCH 21/52] sort result in ch_bench_having_mx test --- src/test/regress/expected/ch_bench_having_mx.out | 6 +++--- src/test/regress/sql/ch_bench_having_mx.sql | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/test/regress/expected/ch_bench_having_mx.out b/src/test/regress/expected/ch_bench_having_mx.out index 0ba516539..1509376e6 100644 --- a/src/test/regress/expected/ch_bench_having_mx.out +++ b/src/test/regress/expected/ch_bench_having_mx.out @@ -345,11 +345,11 @@ having sum(s_order_cnt) > where mod((s_w_id * s_i_id),10000) = s_suppkey and s_nationkey = n_nationkey and n_name = 'GERMANY') -order by ordercount desc; +order by s_i_id, ordercount desc; s_i_id | ordercount --------------------------------------------------------------------- - 33 | 1 1 | 1 + 33 | 1 (2 rows) insert into stock VALUES @@ -366,7 +366,7 @@ having sum(s_order_cnt) > where mod((s_w_id * s_i_id),10000) = s_suppkey and s_nationkey = n_nationkey and n_name = 'GERMANY') -order by ordercount desc; +order by s_i_id, ordercount desc; s_i_id | ordercount --------------------------------------------------------------------- 1 | 100001 diff --git a/src/test/regress/sql/ch_bench_having_mx.sql b/src/test/regress/sql/ch_bench_having_mx.sql index cdd0bf8d6..419107f63 100644 --- a/src/test/regress/sql/ch_bench_having_mx.sql +++ b/src/test/regress/sql/ch_bench_having_mx.sql @@ -173,7 +173,7 @@ having sum(s_order_cnt) > where mod((s_w_id * s_i_id),10000) = s_suppkey and s_nationkey = n_nationkey and n_name = 'GERMANY') -order by ordercount desc; +order by s_i_id, ordercount desc; insert into stock VALUES (10033, 1, 1, 1, 100000, 1, '', '','','','','','','','','',''); @@ -190,7 +190,7 @@ having sum(s_order_cnt) > where mod((s_w_id * s_i_id),10000) = s_suppkey and s_nationkey = n_nationkey and n_name = 'GERMANY') -order by ordercount desc; +order by s_i_id, ordercount desc; \c - - - :master_port BEGIN; From 80d2bc2317def9232899030ccb2f88e87bb5dbbc Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Thu, 4 Jun 2020 10:00:25 +0300 Subject: [PATCH 22/52] normalize some output and sort test result --- src/test/regress/bin/normalize.sed | 5 +++- src/test/regress/expected/ch_bench_having.out | 6 ++--- .../foreign_key_restriction_enforcement.out | 24 ------------------- .../expected/insert_select_repartition.out | 1 - .../expected/intermediate_result_pruning.out | 4 ---- src/test/regress/sql/ch_bench_having.sql | 4 ++-- .../regress/sql/multi_mx_create_table.sql | 4 ++-- 7 files changed, 11 insertions(+), 37 deletions(-) diff --git a/src/test/regress/bin/normalize.sed b/src/test/regress/bin/normalize.sed index 80dfe2122..cd1866ea9 100644 --- a/src/test/regress/bin/normalize.sed +++ b/src/test/regress/bin/normalize.sed @@ -93,7 +93,10 @@ s/varattnosyn/varoattno/g /DEBUG: index ".*" can safely use deduplication.*$/d /DEBUG: index ".*" cannot use deduplication.*$/d /DEBUG: building index ".*" on table ".*" serially.*$/d - +s/partition ".*" would be violated by some row/partition would be violated by some row/g +/.*Peak Memory Usage:.*$/d +s/of relation ".*" contains null values/contains null values/g +s/of relation "t1" is violated by some row/is violated by some row/g # intermediate_results diff --git a/src/test/regress/expected/ch_bench_having.out b/src/test/regress/expected/ch_bench_having.out index 6c713a06e..29feb0305 100644 --- a/src/test/regress/expected/ch_bench_having.out +++ b/src/test/regress/expected/ch_bench_having.out @@ -335,11 +335,11 @@ having sum(s_order_cnt) > where mod((s_w_id * s_i_id),10000) = s_suppkey and s_nationkey = n_nationkey and n_name = 'GERMANY') -order by ordercount desc; +order by s_i_id, ordercount desc; s_i_id | ordercount --------------------------------------------------------------------- - 33 | 1 1 | 1 + 33 | 1 (2 rows) insert into stock VALUES @@ -356,7 +356,7 @@ having sum(s_order_cnt) > where mod((s_w_id * s_i_id),10000) = s_suppkey and s_nationkey = n_nationkey and n_name = 'GERMANY') -order by ordercount desc; +order by s_i_id, ordercount desc; s_i_id | ordercount --------------------------------------------------------------------- 1 | 100001 diff --git a/src/test/regress/expected/foreign_key_restriction_enforcement.out b/src/test/regress/expected/foreign_key_restriction_enforcement.out index 0fa3ad4eb..d7b147370 100644 --- a/src/test/regress/expected/foreign_key_restriction_enforcement.out +++ b/src/test/regress/expected/foreign_key_restriction_enforcement.out @@ -192,7 +192,6 @@ BEGIN; ALTER TABLE on_update_fkey_table ALTER COLUMN value_1 SET DATA TYPE bigint; DEBUG: rewriting table "on_update_fkey_table" -DEBUG: building index "on_update_fkey_table_pkey" on table "on_update_fkey_table" serially DEBUG: validating foreign key constraint "fkey" ROLLBACK; BEGIN; @@ -204,7 +203,6 @@ BEGIN; ALTER TABLE on_update_fkey_table ALTER COLUMN value_1 SET DATA TYPE bigint; DEBUG: rewriting table "on_update_fkey_table" -DEBUG: building index "on_update_fkey_table_pkey" on table "on_update_fkey_table" serially DEBUG: validating foreign key constraint "fkey" ROLLBACK; -- case 1.6: SELECT to a reference table is followed by an unrelated DDL @@ -488,7 +486,6 @@ DEBUG: switching to sequential query execution mode DETAIL: Reference table "reference_table" is modified, which might lead to data inconsistencies or distributed deadlocks via parallel accesses to hash distributed tables due to foreign keys. Any parallel modification to those hash distributed tables in the same transaction can only be executed in sequential query execution mode ALTER TABLE on_update_fkey_table ALTER COLUMN value_1 SET DATA TYPE bigint; DEBUG: rewriting table "on_update_fkey_table" -DEBUG: building index "on_update_fkey_table_pkey" on table "on_update_fkey_table" serially DEBUG: validating foreign key constraint "fkey" ROLLBACK; BEGIN; @@ -497,7 +494,6 @@ DEBUG: switching to sequential query execution mode DETAIL: Reference table "transitive_reference_table" is modified, which might lead to data inconsistencies or distributed deadlocks via parallel accesses to hash distributed tables due to foreign keys. Any parallel modification to those hash distributed tables in the same transaction can only be executed in sequential query execution mode ALTER TABLE on_update_fkey_table ALTER COLUMN value_1 SET DATA TYPE bigint; DEBUG: rewriting table "on_update_fkey_table" -DEBUG: building index "on_update_fkey_table_pkey" on table "on_update_fkey_table" serially DEBUG: validating foreign key constraint "fkey" ROLLBACK; -- case 2.6: UPDATE to a reference table is followed by an unrelated DDL @@ -534,14 +530,12 @@ BEGIN; DEBUG: switching to sequential query execution mode DETAIL: Reference table "reference_table" is modified, which might lead to data inconsistencies or distributed deadlocks via parallel accesses to hash distributed tables due to foreign keys. Any parallel modification to those hash distributed tables in the same transaction can only be executed in sequential query execution mode TRUNCATE on_update_fkey_table; -DEBUG: building index "on_update_fkey_table_pkey" on table "on_update_fkey_table" serially ROLLBACK; BEGIN; UPDATE transitive_reference_table SET id = 101 WHERE id = 99; DEBUG: switching to sequential query execution mode DETAIL: Reference table "transitive_reference_table" is modified, which might lead to data inconsistencies or distributed deadlocks via parallel accesses to hash distributed tables due to foreign keys. Any parallel modification to those hash distributed tables in the same transaction can only be executed in sequential query execution mode TRUNCATE on_update_fkey_table; -DEBUG: building index "on_update_fkey_table_pkey" on table "on_update_fkey_table" serially ROLLBACK; -- case 3.1: an unrelated DDL to a reference table is followed by a real-time SELECT BEGIN; @@ -621,38 +615,30 @@ ROLLBACK; BEGIN; ALTER TABLE reference_table ALTER COLUMN id SET DATA TYPE smallint; DEBUG: rewriting table "reference_table" -DEBUG: building index "reference_table_pkey" on table "reference_table" serially DEBUG: validating foreign key constraint "fkey" CREATE INDEX fkey_test_index_1 ON on_update_fkey_table(value_1); -DEBUG: building index "fkey_test_index_1" on table "on_update_fkey_table" serially ROLLBACK; BEGIN; ALTER TABLE transitive_reference_table ALTER COLUMN id SET DATA TYPE smallint; DEBUG: rewriting table "transitive_reference_table" -DEBUG: building index "transitive_reference_table_pkey" on table "transitive_reference_table" serially DEBUG: validating foreign key constraint "fkey" CREATE INDEX fkey_test_index_1 ON on_update_fkey_table(value_1); -DEBUG: building index "fkey_test_index_1" on table "on_update_fkey_table" serially ROLLBACK; -- case 4.6: DDL to reference table followed by a DDL to dist table, both touching fkey columns BEGIN; ALTER TABLE reference_table ALTER COLUMN id SET DATA TYPE smallint; DEBUG: rewriting table "reference_table" -DEBUG: building index "reference_table_pkey" on table "reference_table" serially DEBUG: validating foreign key constraint "fkey" ALTER TABLE on_update_fkey_table ALTER COLUMN value_1 SET DATA TYPE smallint; DEBUG: rewriting table "on_update_fkey_table" -DEBUG: building index "on_update_fkey_table_pkey" on table "on_update_fkey_table" serially DEBUG: validating foreign key constraint "fkey" ROLLBACK; BEGIN; ALTER TABLE transitive_reference_table ALTER COLUMN id SET DATA TYPE smallint; DEBUG: rewriting table "transitive_reference_table" -DEBUG: building index "transitive_reference_table_pkey" on table "transitive_reference_table" serially DEBUG: validating foreign key constraint "fkey" ALTER TABLE on_update_fkey_table ALTER COLUMN value_1 SET DATA TYPE smallint; DEBUG: rewriting table "on_update_fkey_table" -DEBUG: building index "on_update_fkey_table_pkey" on table "on_update_fkey_table" serially DEBUG: validating foreign key constraint "fkey" ROLLBACK; -- case 3.7: DDL to a reference table is followed by COPY @@ -674,31 +660,25 @@ BEGIN; DEBUG: switching to sequential query execution mode DETAIL: Reference table "reference_table" is modified, which might lead to data inconsistencies or distributed deadlocks via parallel accesses to hash distributed tables due to foreign keys. Any parallel modification to those hash distributed tables in the same transaction can only be executed in sequential query execution mode TRUNCATE on_update_fkey_table; -DEBUG: building index "on_update_fkey_table_pkey" on table "on_update_fkey_table" serially ROLLBACK; BEGIN; ALTER TABLE transitive_reference_table ADD COLUMN X int; DEBUG: switching to sequential query execution mode DETAIL: Reference table "transitive_reference_table" is modified, which might lead to data inconsistencies or distributed deadlocks via parallel accesses to hash distributed tables due to foreign keys. Any parallel modification to those hash distributed tables in the same transaction can only be executed in sequential query execution mode TRUNCATE on_update_fkey_table; -DEBUG: building index "on_update_fkey_table_pkey" on table "on_update_fkey_table" serially ROLLBACK; -- case 3.9: DDL to a reference table is followed by TRUNCATE BEGIN; ALTER TABLE reference_table ALTER COLUMN id SET DATA TYPE smallint; DEBUG: rewriting table "reference_table" -DEBUG: building index "reference_table_pkey" on table "reference_table" serially DEBUG: validating foreign key constraint "fkey" TRUNCATE on_update_fkey_table; -DEBUG: building index "on_update_fkey_table_pkey" on table "on_update_fkey_table" serially ROLLBACK; BEGIN; ALTER TABLE transitive_reference_table ALTER COLUMN id SET DATA TYPE smallint; DEBUG: rewriting table "transitive_reference_table" -DEBUG: building index "transitive_reference_table_pkey" on table "transitive_reference_table" serially DEBUG: validating foreign key constraint "fkey" TRUNCATE on_update_fkey_table; -DEBUG: building index "on_update_fkey_table_pkey" on table "on_update_fkey_table" serially ROLLBACK; --------------------------------------------------------------------- --- Now, start testing the other way araound @@ -790,7 +770,6 @@ BEGIN; ALTER TABLE reference_table ALTER COLUMN id SET DATA TYPE smallint; DEBUG: rewriting table "reference_table" -DEBUG: building index "reference_table_pkey" on table "reference_table" serially DEBUG: validating foreign key constraint "fkey" ERROR: cannot execute DDL on reference table "reference_table" because there was a parallel SELECT access to distributed table "on_update_fkey_table" in the same transaction HINT: Try re-running the transaction with "SET LOCAL citus.multi_shard_modify_mode TO 'sequential';" @@ -804,7 +783,6 @@ BEGIN; ALTER TABLE transitive_reference_table ALTER COLUMN id SET DATA TYPE smallint; DEBUG: rewriting table "transitive_reference_table" -DEBUG: building index "transitive_reference_table_pkey" on table "transitive_reference_table" serially DEBUG: validating foreign key constraint "fkey" ERROR: cannot execute DDL on reference table "transitive_reference_table" because there was a parallel SELECT access to distributed table "on_update_fkey_table" in the same transaction HINT: Try re-running the transaction with "SET LOCAL citus.multi_shard_modify_mode TO 'sequential';" @@ -1402,7 +1380,6 @@ SET client_min_messages TO DEBUG1; -- set the mode to sequential for the next operations CREATE TABLE reference_table(id int PRIMARY KEY); DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "reference_table_pkey" for table "reference_table" -DEBUG: building index "reference_table_pkey" on table "reference_table" serially SELECT create_reference_table('reference_table'); create_reference_table --------------------------------------------------------------------- @@ -1411,7 +1388,6 @@ SELECT create_reference_table('reference_table'); CREATE TABLE distributed_table(id int PRIMARY KEY, value_1 int); DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "distributed_table_pkey" for table "distributed_table" -DEBUG: building index "distributed_table_pkey" on table "distributed_table" serially SELECT create_distributed_table('distributed_table', 'id'); create_distributed_table --------------------------------------------------------------------- diff --git a/src/test/regress/expected/insert_select_repartition.out b/src/test/regress/expected/insert_select_repartition.out index 858d7a05d..8845e7001 100644 --- a/src/test/regress/expected/insert_select_repartition.out +++ b/src/test/regress/expected/insert_select_repartition.out @@ -274,7 +274,6 @@ TRUNCATE target_table; SET citus.log_remote_commands TO true; SET client_min_messages TO DEBUG; CREATE TABLE results AS SELECT max(-a), array_agg(mapped_key) FROM source_table GROUP BY a; DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: building index "pg_toast_xxxxx_index" on table "pg_toast_xxxxx" serially NOTICE: issuing SELECT max((OPERATOR(pg_catalog.-) a)) AS max, array_agg(mapped_key) AS array_agg, a AS worker_column_3 FROM insert_select_repartition.source_table_4213601 source_table WHERE true GROUP BY a NOTICE: issuing SELECT max((OPERATOR(pg_catalog.-) a)) AS max, array_agg(mapped_key) AS array_agg, a AS worker_column_3 FROM insert_select_repartition.source_table_4213602 source_table WHERE true GROUP BY a NOTICE: issuing SELECT max((OPERATOR(pg_catalog.-) a)) AS max, array_agg(mapped_key) AS array_agg, a AS worker_column_3 FROM insert_select_repartition.source_table_4213603 source_table WHERE true GROUP BY a diff --git a/src/test/regress/expected/intermediate_result_pruning.out b/src/test/regress/expected/intermediate_result_pruning.out index c265a7c9b..dbf6acc02 100644 --- a/src/test/regress/expected/intermediate_result_pruning.out +++ b/src/test/regress/expected/intermediate_result_pruning.out @@ -935,13 +935,9 @@ DEBUG: Subplan XXX_1 will be sent to localhost:xxxxx -- test case for issue #3556 CREATE TABLE accounts (id text PRIMARY KEY); -DEBUG: building index "pg_toast_xxxxx_index" on table "pg_toast_xxxxx" serially DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "accounts_pkey" for table "accounts" -DEBUG: building index "accounts_pkey" on table "accounts" serially CREATE TABLE stats (account_id text PRIMARY KEY, spent int); -DEBUG: building index "pg_toast_xxxxx_index" on table "pg_toast_xxxxx" serially DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "stats_pkey" for table "stats" -DEBUG: building index "stats_pkey" on table "stats" serially SELECT create_distributed_table('accounts', 'id', colocate_with => 'none'); create_distributed_table --------------------------------------------------------------------- diff --git a/src/test/regress/sql/ch_bench_having.sql b/src/test/regress/sql/ch_bench_having.sql index 890e6a3a2..b6996cbe3 100644 --- a/src/test/regress/sql/ch_bench_having.sql +++ b/src/test/regress/sql/ch_bench_having.sql @@ -159,7 +159,7 @@ having sum(s_order_cnt) > where mod((s_w_id * s_i_id),10000) = s_suppkey and s_nationkey = n_nationkey and n_name = 'GERMANY') -order by ordercount desc; +order by s_i_id, ordercount desc; insert into stock VALUES (10033, 1, 1, 1, 100000, 1, '', '','','','','','','','','',''); @@ -176,7 +176,7 @@ having sum(s_order_cnt) > where mod((s_w_id * s_i_id),10000) = s_suppkey and s_nationkey = n_nationkey and n_name = 'GERMANY') -order by ordercount desc; +order by s_i_id, ordercount desc; BEGIN; SET LOCAL client_min_messages TO WARNING; diff --git a/src/test/regress/sql/multi_mx_create_table.sql b/src/test/regress/sql/multi_mx_create_table.sql index da16122f2..cb5316f91 100644 --- a/src/test/regress/sql/multi_mx_create_table.sql +++ b/src/test/regress/sql/multi_mx_create_table.sql @@ -261,10 +261,10 @@ CREATE TABLE lineitem_mx ( l_shipmode char(10) not null, l_comment varchar(44) not null, PRIMARY KEY(l_orderkey, l_linenumber) ); - -SET citus.shard_count TO 16; SELECT create_distributed_table('lineitem_mx', 'l_orderkey'); +-- SET citus.shard_count TO 16; + CREATE INDEX lineitem_mx_time_index ON lineitem_mx (l_shipdate); CREATE TABLE orders_mx ( From de82d0ff79d9f3af94692fc7f21a1d55a0acff69 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Sat, 6 Jun 2020 12:55:21 +0300 Subject: [PATCH 23/52] add output for pg13 for propagate extension commands CREATE EXTENSION FROM is not supported anymore with postgres 13. An alternative output is added for pg13 where we basically error for that statement. --- .../expected/propagate_extension_commands.out | 13 +- .../propagate_extension_commands_1.out | 444 ++++++++++++++++++ 2 files changed, 451 insertions(+), 6 deletions(-) create mode 100644 src/test/regress/expected/propagate_extension_commands_1.out diff --git a/src/test/regress/expected/propagate_extension_commands.out b/src/test/regress/expected/propagate_extension_commands.out index 36f7254c9..969a52b97 100644 --- a/src/test/regress/expected/propagate_extension_commands.out +++ b/src/test/regress/expected/propagate_extension_commands.out @@ -226,16 +226,17 @@ $$); (1 row) CREATE EXTENSION dict_int FROM unpackaged; +ERROR: CREATE EXTENSION ... FROM is no longer supported SELECT run_command_on_workers($$SELECT count(extnamespace) FROM pg_extension WHERE extname = 'dict_int'$$); run_command_on_workers --------------------------------------------------------------------- - (localhost,57637,t,1) + (localhost,57637,t,0) (1 row) SELECT run_command_on_workers($$SELECT extversion FROM pg_extension WHERE extname = 'dict_int'$$); run_command_on_workers --------------------------------------------------------------------- - (localhost,57637,t,1.0) + (localhost,57637,t,"") (1 row) -- and add the other node @@ -264,15 +265,15 @@ SELECT run_command_on_workers($$SELECT extversion FROM pg_extension WHERE extnam SELECT run_command_on_workers($$SELECT count(extnamespace) FROM pg_extension WHERE extname = 'dict_int'$$); run_command_on_workers --------------------------------------------------------------------- - (localhost,57637,t,1) - (localhost,57638,t,1) + (localhost,57637,t,0) + (localhost,57638,t,0) (2 rows) SELECT run_command_on_workers($$SELECT extversion FROM pg_extension WHERE extname = 'dict_int'$$); run_command_on_workers --------------------------------------------------------------------- - (localhost,57637,t,1.0) - (localhost,57638,t,1.0) + (localhost,57637,t,"") + (localhost,57638,t,"") (2 rows) -- and similarly check for the reference table diff --git a/src/test/regress/expected/propagate_extension_commands_1.out b/src/test/regress/expected/propagate_extension_commands_1.out new file mode 100644 index 000000000..36f7254c9 --- /dev/null +++ b/src/test/regress/expected/propagate_extension_commands_1.out @@ -0,0 +1,444 @@ +CREATE SCHEMA "extension'test"; +-- use a schema name with escape character +SET search_path TO "extension'test"; +SET client_min_messages TO WARNING; +-- create an extension on the given search_path +-- the extension is on contrib, so should be avaliable for the regression tests +CREATE EXTENSION seg; +-- make sure that both the schema and the extension is distributed +SELECT count(*) FROM citus.pg_dist_object WHERE objid = (SELECT oid FROM pg_extension WHERE extname = 'seg'); + count +--------------------------------------------------------------------- + 1 +(1 row) + +SELECT count(*) FROM citus.pg_dist_object WHERE objid = (SELECT oid FROM pg_namespace WHERE nspname = 'extension''test'); + count +--------------------------------------------------------------------- + 1 +(1 row) + +CREATE TABLE test_table (key int, value seg); +SELECT create_distributed_table('test_table', 'key'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- make sure that the table is also distributed now +SELECT count(*) from pg_dist_partition where logicalrelid='extension''test.test_table'::regclass; + count +--------------------------------------------------------------------- + 1 +(1 row) + +CREATE TYPE two_segs AS (seg_1 seg, seg_2 seg); +-- verify that the type that depends on the extension is also marked as distributed +SELECT count(*) FROM citus.pg_dist_object WHERE objid = (SELECT oid FROM pg_type WHERE typname = 'two_segs' AND typnamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'extension''test')); + count +--------------------------------------------------------------------- + 1 +(1 row) + +-- now try to run CREATE EXTENSION within a transction block, all should work fine +BEGIN; + CREATE EXTENSION isn WITH SCHEMA public; + -- now, try create a reference table relying on the data types + -- this should not succeed as we do not distribute extension commands within transaction blocks + CREATE TABLE dist_table (key int, value public.issn); + SELECT create_distributed_table('dist_table', 'key'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + + -- we can even run queries (sequentially) over the distributed table + SELECT * FROM dist_table; + key | value +--------------------------------------------------------------------- +(0 rows) + + INSERT INTO dist_table VALUES (1, public.issn('1436-4522')); + INSERT INTO dist_table SELECT * FROM dist_table RETURNING *; + key | value +--------------------------------------------------------------------- + 1 | 1436-4522 +(1 row) + +COMMIT; +-- make sure that the extension is distributed even if we run create extension in a transaction block +SELECT count(*) FROM citus.pg_dist_object WHERE objid = (SELECT oid FROM pg_extension WHERE extname = 'isn'); + count +--------------------------------------------------------------------- + 1 +(1 row) + +SELECT run_command_on_workers($$SELECT count(*) FROM pg_extension WHERE extname = 'isn'$$); + run_command_on_workers +--------------------------------------------------------------------- + (localhost,57637,t,1) + (localhost,57638,t,1) +(2 rows) + +CREATE TABLE ref_table (a public.issn); +-- now, create a reference table relying on the data types +SELECT create_reference_table('ref_table'); + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +-- now, drop the extension, recreate it with an older version and update it to latest version +DROP EXTENSION isn CASCADE; +CREATE EXTENSION isn WITH VERSION "1.1"; +-- before updating the version, ensure the current version +SELECT run_command_on_workers($$SELECT extversion FROM pg_extension WHERE extname = 'isn'$$); + run_command_on_workers +--------------------------------------------------------------------- + (localhost,57637,t,1.1) + (localhost,57638,t,1.1) +(2 rows) + +-- now, update to a newer version +ALTER EXTENSION isn UPDATE TO '1.2'; +-- show that ALTER EXTENSION is propagated +SELECT run_command_on_workers($$SELECT extversion FROM pg_extension WHERE extname = 'isn'$$); + run_command_on_workers +--------------------------------------------------------------------- + (localhost,57637,t,1.2) + (localhost,57638,t,1.2) +(2 rows) + +-- before changing the schema, ensure the current schmea +SELECT run_command_on_workers($$SELECT nspname from pg_namespace where oid=(SELECT extnamespace FROM pg_extension WHERE extname = 'isn')$$); + run_command_on_workers +--------------------------------------------------------------------- + (localhost,57637,t,extension'test) + (localhost,57638,t,extension'test) +(2 rows) + +-- now change the schema +ALTER EXTENSION isn SET SCHEMA public; +-- switch back to public schema as we set extension's schema to public +SET search_path TO public; +-- make sure that the extension is distributed +SELECT count(*) FROM citus.pg_dist_object WHERE objid = (SELECT oid FROM pg_extension WHERE extname = 'isn'); + count +--------------------------------------------------------------------- + 1 +(1 row) + +-- show that the ALTER EXTENSION command is propagated +SELECT run_command_on_workers($$SELECT nspname from pg_namespace where oid=(SELECT extnamespace FROM pg_extension WHERE extname = 'isn')$$); + run_command_on_workers +--------------------------------------------------------------------- + (localhost,57637,t,public) + (localhost,57638,t,public) +(2 rows) + +-- drop the extension finally +DROP EXTENSION isn CASCADE; +-- now make sure that the reference tables depending on an extension can be succesfully created. +-- we should also ensure that we replicate this reference table (and hence the extension) +-- to new nodes after calling master_activate_node. +-- now, first drop seg and existing objects before next test +DROP EXTENSION seg CASCADE; +-- but as we have only 2 ports in postgresql tests, let's remove one of the nodes first +-- before remove, first remove the existing relations (due to the other tests) +DROP SCHEMA "extension'test" CASCADE; +SELECT 1 from master_remove_node('localhost', :worker_2_port); + ?column? +--------------------------------------------------------------------- + 1 +(1 row) + +-- then create the extension +CREATE EXTENSION seg; +-- show that the extension is created on existing worker +SELECT run_command_on_workers($$SELECT count(extnamespace) FROM pg_extension WHERE extname = 'seg'$$); + run_command_on_workers +--------------------------------------------------------------------- + (localhost,57637,t,1) +(1 row) + +SELECT run_command_on_workers($$SELECT extversion FROM pg_extension WHERE extname = 'seg'$$); + run_command_on_workers +--------------------------------------------------------------------- + (localhost,57637,t,1.3) +(1 row) + +-- now create the reference table +CREATE TABLE ref_table_2 (x seg); +SELECT create_reference_table('ref_table_2'); + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +-- we also add an old style extension from before extensions which we upgrade to an extension +-- by exercising it before the add node we verify it will create the extension (without upgrading) +-- it on the new worker as well. For this we use the dict_int extension which is in contrib, +-- supports FROM unpackaged, and is relatively small +-- create objects for dict_int manually so we can upgrade from unpacked +CREATE FUNCTION dintdict_init(internal) RETURNS internal AS 'dict_int.so' LANGUAGE C STRICT; +CREATE FUNCTION dintdict_lexize(internal, internal, internal, internal) RETURNS internal AS 'dict_int.so' LANGUAGE C STRICT; +CREATE TEXT SEARCH TEMPLATE intdict_template (LEXIZE = dintdict_lexize, INIT = dintdict_init ); +CREATE TEXT SEARCH DICTIONARY intdict (TEMPLATE = intdict_template); +COMMENT ON TEXT SEARCH DICTIONARY intdict IS 'dictionary for integers'; +SELECT run_command_on_workers($$ +CREATE FUNCTION dintdict_init(internal) RETURNS internal AS 'dict_int.so' LANGUAGE C STRICT; +$$); + run_command_on_workers +--------------------------------------------------------------------- + (localhost,57637,t,"CREATE FUNCTION") +(1 row) + +SELECT run_command_on_workers($$ +CREATE FUNCTION dintdict_lexize(internal, internal, internal, internal) RETURNS internal AS 'dict_int.so' LANGUAGE C STRICT; +$$); + run_command_on_workers +--------------------------------------------------------------------- + (localhost,57637,t,"CREATE FUNCTION") +(1 row) + +SELECT run_command_on_workers($$ +CREATE TEXT SEARCH TEMPLATE intdict_template (LEXIZE = dintdict_lexize, INIT = dintdict_init ); +$$); + run_command_on_workers +--------------------------------------------------------------------- + (localhost,57637,t,"CREATE TEXT SEARCH TEMPLATE") +(1 row) + +SELECT run_command_on_workers($$ +CREATE TEXT SEARCH DICTIONARY intdict (TEMPLATE = intdict_template); +$$); + run_command_on_workers +--------------------------------------------------------------------- + (localhost,57637,t,"CREATE TEXT SEARCH DICTIONARY") +(1 row) + +SELECT run_command_on_workers($$ +COMMENT ON TEXT SEARCH DICTIONARY intdict IS 'dictionary for integers'; +$$); + run_command_on_workers +--------------------------------------------------------------------- + (localhost,57637,t,COMMENT) +(1 row) + +CREATE EXTENSION dict_int FROM unpackaged; +SELECT run_command_on_workers($$SELECT count(extnamespace) FROM pg_extension WHERE extname = 'dict_int'$$); + run_command_on_workers +--------------------------------------------------------------------- + (localhost,57637,t,1) +(1 row) + +SELECT run_command_on_workers($$SELECT extversion FROM pg_extension WHERE extname = 'dict_int'$$); + run_command_on_workers +--------------------------------------------------------------------- + (localhost,57637,t,1.0) +(1 row) + +-- and add the other node +SELECT 1 from master_add_node('localhost', :worker_2_port); + ?column? +--------------------------------------------------------------------- + 1 +(1 row) + +-- show that the extension is created on both existing and new node +SELECT run_command_on_workers($$SELECT count(extnamespace) FROM pg_extension WHERE extname = 'seg'$$); + run_command_on_workers +--------------------------------------------------------------------- + (localhost,57637,t,1) + (localhost,57638,t,1) +(2 rows) + +SELECT run_command_on_workers($$SELECT extversion FROM pg_extension WHERE extname = 'seg'$$); + run_command_on_workers +--------------------------------------------------------------------- + (localhost,57637,t,1.3) + (localhost,57638,t,1.3) +(2 rows) + +-- check for the unpackaged extension to be created correctly +SELECT run_command_on_workers($$SELECT count(extnamespace) FROM pg_extension WHERE extname = 'dict_int'$$); + run_command_on_workers +--------------------------------------------------------------------- + (localhost,57637,t,1) + (localhost,57638,t,1) +(2 rows) + +SELECT run_command_on_workers($$SELECT extversion FROM pg_extension WHERE extname = 'dict_int'$$); + run_command_on_workers +--------------------------------------------------------------------- + (localhost,57637,t,1.0) + (localhost,57638,t,1.0) +(2 rows) + +-- and similarly check for the reference table +select count(*) from pg_dist_partition where partmethod='n' and logicalrelid='ref_table_2'::regclass; + count +--------------------------------------------------------------------- + 1 +(1 row) + +SELECT count(*) FROM pg_dist_shard WHERE logicalrelid='ref_table_2'::regclass; + count +--------------------------------------------------------------------- + 1 +(1 row) + +DROP TABLE ref_table_2; +-- now test create extension in another transaction block but rollback this time +BEGIN; + CREATE EXTENSION isn WITH VERSION '1.1' SCHEMA public; +ROLLBACK; +-- at the end of the transaction block, we did not create isn extension in coordinator or worker nodes as we rollback'ed +-- make sure that the extension is not distributed +SELECT count(*) FROM citus.pg_dist_object WHERE objid = (SELECT oid FROM pg_extension WHERE extname = 'isn'); + count +--------------------------------------------------------------------- + 0 +(1 row) + +-- and the extension does not exist on workers +SELECT run_command_on_workers($$SELECT count(*) FROM pg_extension WHERE extname = 'isn'$$); + run_command_on_workers +--------------------------------------------------------------------- + (localhost,57637,t,0) + (localhost,57638,t,0) +(2 rows) + +-- give a notice for the following commands saying that it is not +-- propagated to the workers. the user should run it manually on the workers +CREATE TABLE t1 (A int); +CREATE VIEW v1 AS select * from t1; +ALTER EXTENSION seg ADD VIEW v1; +ALTER EXTENSION seg DROP VIEW v1; +DROP VIEW v1; +DROP TABLE t1; +-- drop multiple extensions at the same time +CREATE EXTENSION isn WITH VERSION '1.1' SCHEMA public; +-- let's create another extension locally +set citus.enable_ddl_propagation to 'off'; +CREATE EXTENSION pg_buffercache; +set citus.enable_ddl_propagation to 'on'; +DROP EXTENSION pg_buffercache, isn CASCADE; +SELECT count(*) FROM pg_extension WHERE extname IN ('pg_buffercache', 'isn'); + count +--------------------------------------------------------------------- + 0 +(1 row) + +-- drop extension should just work +DROP EXTENSION seg CASCADE; +SELECT count(*) FROM citus.pg_dist_object WHERE objid = (SELECT oid FROM pg_extension WHERE extname = 'seg'); + count +--------------------------------------------------------------------- + 0 +(1 row) + +SELECT run_command_on_workers($$SELECT count(*) FROM pg_extension WHERE extname = 'seg'$$); + run_command_on_workers +--------------------------------------------------------------------- + (localhost,57637,t,0) + (localhost,57638,t,0) +(2 rows) + +-- make sure that the extension is not avaliable anymore as a distributed object +SELECT count(*) FROM citus.pg_dist_object WHERE objid = (SELECT oid FROM pg_extension WHERE extname IN ('seg', 'isn')); + count +--------------------------------------------------------------------- + 0 +(1 row) + +CREATE SCHEMA "extension'test"; +SET search_path TO "extension'test"; +-- check restriction for sequential execution +-- enable it and see that create command errors but continues its execution by changing citus.multi_shard_modify_mode TO 'off +BEGIN; + CREATE TABLE some_random_table (a int); + SELECT create_distributed_table('some_random_table', 'a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + + CREATE EXTENSION seg; + CREATE TABLE some_random_table_2 (a int, b seg); + SELECT create_distributed_table('some_random_table_2', 'a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +ROLLBACK; +-- show that the CREATE EXTENSION command propagated even if the transaction +-- block is rollbacked, that's a shortcoming of dependency creation logic +SELECT run_command_on_workers($$SELECT extversion FROM pg_extension WHERE extname = 'seg'$$); + run_command_on_workers +--------------------------------------------------------------------- + (localhost,57637,t,1.3) + (localhost,57638,t,1.3) +(2 rows) + +-- drop the schema and all the objects +DROP SCHEMA "extension'test" CASCADE; +-- recreate for the next tests +CREATE SCHEMA "extension'test"; +-- use a schema name with escape character +SET search_path TO "extension'test"; +-- remove the node, we'll add back again +SELECT 1 from master_remove_node('localhost', :worker_2_port); + ?column? +--------------------------------------------------------------------- + 1 +(1 row) + +-- now, create a type that depends on another type, which +-- finally depends on an extension +BEGIN; + SET citus.shard_replication_factor TO 1; + CREATE EXTENSION seg; + CREATE EXTENSION isn; + CREATE TYPE test_type AS (a int, b seg); + CREATE TYPE test_type_2 AS (a int, b test_type); + CREATE TABLE t2 (a int, b test_type_2, c issn); + SELECT create_distributed_table('t2', 'a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + + CREATE TYPE test_type_3 AS (a int, b test_type, c issn); + CREATE TABLE t3 (a int, b test_type_3); + SELECT create_reference_table('t3'); + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +COMMIT; +-- add the node back +SELECT 1 from master_add_node('localhost', :worker_2_port); + ?column? +--------------------------------------------------------------------- + 1 +(1 row) + +-- make sure that both extensions are created on both nodes +SELECT count(*) FROM citus.pg_dist_object WHERE objid IN (SELECT oid FROM pg_extension WHERE extname IN ('seg', 'isn')); + count +--------------------------------------------------------------------- + 2 +(1 row) + +SELECT run_command_on_workers($$SELECT count(*) FROM pg_extension WHERE extname IN ('seg', 'isn')$$); + run_command_on_workers +--------------------------------------------------------------------- + (localhost,57637,t,2) + (localhost,57638,t,2) +(2 rows) + +-- drop the schema and all the objects +DROP SCHEMA "extension'test" CASCADE; From 6ad708642ed1987dd5328eabaa72951cd8e4fb35 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Mon, 8 Jun 2020 17:16:18 +0300 Subject: [PATCH 24/52] Fix rte index with pg >=13 Rte index is increased by range table index offset in pg >= 13. The offset is removed with the pg >= 13. Currently pushdown for union all is disabled because translatedVars is set to nil on postgres side, and we were using translatedVars to figure out if partition key has the same index in both sides of union all. This should be fixed. Commit on postgres side: 6ef77cf46e81f45716ec981cb08781d426181378 fix union all pushdown logic for pg13 Before pg 13, there was a field, translatedVars, and we were using that to understand if the partition key has the same index on both sides of the union all. With pg13 there is a parent_colnos field in appendRelInfo and we can use that to get the attribute numbers(varattnos) in union all vars. We make use of parent_colnos instead of translatedVars in pg >=13. --- .../relation_restriction_equivalence.c | 77 +++++++++++++------ 1 file changed, 54 insertions(+), 23 deletions(-) diff --git a/src/backend/distributed/planner/relation_restriction_equivalence.c b/src/backend/distributed/planner/relation_restriction_equivalence.c index b00bc6287..a322f4ccf 100644 --- a/src/backend/distributed/planner/relation_restriction_equivalence.c +++ b/src/backend/distributed/planner/relation_restriction_equivalence.c @@ -73,8 +73,8 @@ typedef struct AttributeEquivalenceClassMember static bool ContextContainsLocalRelation(RelationRestrictionContext *restrictionContext); -static Var * FindTranslatedVar(List *appendRelList, Oid relationOid, - Index relationRteIndex, Index *partitionKeyIndex); +static Var * FindUnionAllVar(PlannerInfo *root, List *appendRelList, Oid relationOid, + Index relationRteIndex, Index *partitionKeyIndex); static bool ContainsMultipleDistributedRelations(PlannerRestrictionContext * plannerRestrictionContext); static List * GenerateAttributeEquivalencesForRelationRestrictions( @@ -149,6 +149,7 @@ static JoinRestrictionContext * FilterJoinRestrictionContext( static bool RangeTableArrayContainsAnyRTEIdentities(RangeTblEntry **rangeTableEntries, int rangeTableArrayLength, Relids queryRteIdentities); +static int RangeTableOffsetCompat(PlannerInfo *root, AppendRelInfo *appendRelInfo); static Relids QueryRteIdentities(Query *queryTree); static bool JoinRestrictionListExistsInContext(JoinRestriction *joinRestrictionInput, JoinRestrictionContext * @@ -275,10 +276,10 @@ SafeToPushdownUnionSubquery(PlannerRestrictionContext *plannerRestrictionContext */ if (appendRelList != NULL) { - varToBeAdded = FindTranslatedVar(appendRelList, - relationRestriction->relationId, - relationRestriction->index, - &partitionKeyIndex); + varToBeAdded = FindUnionAllVar(relationPlannerRoot, appendRelList, + relationRestriction->relationId, + relationRestriction->index, + &partitionKeyIndex); /* union does not have partition key in the target list */ if (partitionKeyIndex == 0) @@ -370,22 +371,16 @@ SafeToPushdownUnionSubquery(PlannerRestrictionContext *plannerRestrictionContext /* - * FindTranslatedVar iterates on the appendRelList and tries to find a translated - * child var identified by the relation id and the relation rte index. - * - * Note that postgres translates UNION ALL target list elements into translated_vars - * list on the corresponding AppendRelInfo struct. For details, see the related - * structs. - * - * The function returns NULL if it cannot find a translated var. + * FindUnionAllVar finds the variable used in union all for the side that has + * relationRteIndex as its index and the same varattno as the partition key of + * the given relation with relationOid. */ static Var * -FindTranslatedVar(List *appendRelList, Oid relationOid, Index relationRteIndex, - Index *partitionKeyIndex) +FindUnionAllVar(PlannerInfo *root, List *appendRelList, Oid relationOid, + Index relationRteIndex, Index *partitionKeyIndex) { ListCell *appendRelCell = NULL; AppendRelInfo *targetAppendRelInfo = NULL; - ListCell *translatedVarCell = NULL; AttrNumber childAttrNumber = 0; *partitionKeyIndex = 0; @@ -395,25 +390,41 @@ FindTranslatedVar(List *appendRelList, Oid relationOid, Index relationRteIndex, { AppendRelInfo *appendRelInfo = (AppendRelInfo *) lfirst(appendRelCell); + + int rtoffset = RangeTableOffsetCompat(root, appendRelInfo); + /* * We're only interested in the child rel that is equal to the * relation we're investigating. */ - if (appendRelInfo->child_relid == relationRteIndex) + if (appendRelInfo->child_relid - rtoffset == relationRteIndex) { targetAppendRelInfo = appendRelInfo; break; } } - /* we couldn't find the necessary append rel info */ - if (targetAppendRelInfo == NULL) + if (!targetAppendRelInfo) { return NULL; } Var *relationPartitionKey = ForceDistPartitionKey(relationOid); + #if PG_VERSION_NUM >= PG_VERSION_13 + for (; childAttrNumber < targetAppendRelInfo->num_child_cols; childAttrNumber++) + { + int curAttNo = targetAppendRelInfo->parent_colnos[childAttrNumber]; + if (curAttNo == relationPartitionKey->varattno) + { + *partitionKeyIndex = (childAttrNumber + 1); + int rtoffset = RangeTableOffsetCompat(root, targetAppendRelInfo); + relationPartitionKey->varno = targetAppendRelInfo->child_relid - rtoffset; + return relationPartitionKey; + } + } + #else + ListCell *translatedVarCell; List *translaterVars = targetAppendRelInfo->translated_vars; foreach(translatedVarCell, translaterVars) { @@ -435,7 +446,7 @@ FindTranslatedVar(List *appendRelList, Oid relationOid, Index relationRteIndex, return targetVar; } } - + #endif return NULL; } @@ -1346,15 +1357,35 @@ AddUnionAllSetOperationsToAttributeEquivalenceClass(AttributeEquivalenceClass ** { continue; } - + int rtoffset = RangeTableOffsetCompat(root, appendRelInfo); /* set the varno accordingly for this specific child */ - varToBeAdded->varno = appendRelInfo->child_relid; + varToBeAdded->varno = appendRelInfo->child_relid - rtoffset; AddToAttributeEquivalenceClass(attributeEquivalenceClass, root, varToBeAdded); } } +/* + * RangeTableOffsetCompat returns the range table offset(in glob->finalrtable) for the appendRelInfo. + * For PG < 13 this is a no op. + */ +static int RangeTableOffsetCompat(PlannerInfo *root, AppendRelInfo *appendRelInfo) { + #if PG_VERSION_NUM >= PG_VERSION_13 + int i = 1; + for (i = 1 ; i < root->simple_rel_array_size; i++) { + RangeTblEntry* rte = root->simple_rte_array[i]; + if (rte->inh) { + break; + } + } + int indexInRtable = (i - 1); + return appendRelInfo->parent_relid - 1 - (indexInRtable); + #else + return 0; + #endif +} + /* * AddUnionSetOperationsToAttributeEquivalenceClass recursively iterates on all the From 17388e2e919bd607c21d811ea43cdbfbc386c83b Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Mon, 8 Jun 2020 22:38:53 +0300 Subject: [PATCH 25/52] update some tests --- src/test/regress/expected/cte_inline.out | 4 +- src/test/regress/expected/cte_inline_0.out | 400 +++-- src/test/regress/expected/cte_inline_1.out | 1443 +++++++++++++++ .../multi_complex_count_distinct_1.out | 1128 ++++++++++++ .../expected/multi_select_distinct.out | 20 +- .../expected/multi_select_distinct_1.out | 1569 +++++++++++++++++ 6 files changed, 4412 insertions(+), 152 deletions(-) create mode 100644 src/test/regress/expected/cte_inline_1.out create mode 100644 src/test/regress/expected/multi_complex_count_distinct_1.out create mode 100644 src/test/regress/expected/multi_select_distinct_1.out diff --git a/src/test/regress/expected/cte_inline.out b/src/test/regress/expected/cte_inline.out index 4daaaa45c..98cd7343d 100644 --- a/src/test/regress/expected/cte_inline.out +++ b/src/test/regress/expected/cte_inline.out @@ -15,7 +15,7 @@ SHOW server_version \gset SELECT substring(:'server_version', '\d+')::int; substring --------------------------------------------------------------------- - 12 + 13 (1 row) SET client_min_messages TO DEBUG; @@ -747,8 +747,8 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: (SELECT intermediate DEBUG: Creating router plan key | value | other_value | ?column? --------------------------------------------------------------------- - 9 | test29 | {"f1": 29, "f2": 522, "f3": "test29"} | 1 9 | test9 | {"f1": 9, "f2": 162, "f3": "test9"} | 1 + 9 | test29 | {"f1": 29, "f2": 522, "f3": "test29"} | 1 9 | test19 | {"f1": 19, "f2": 342, "f3": "test19"} | 1 (3 rows) diff --git a/src/test/regress/expected/cte_inline_0.out b/src/test/regress/expected/cte_inline_0.out index 66efffa01..cd57658a7 100644 --- a/src/test/regress/expected/cte_inline_0.out +++ b/src/test/regress/expected/cte_inline_0.out @@ -15,7 +15,7 @@ SHOW server_version \gset SELECT substring(:'server_version', '\d+')::int; substring --------------------------------------------------------------------- - 11 + 12 (1 row) SET client_min_messages TO DEBUG; @@ -48,7 +48,19 @@ SELECT FROM cte_1 ORDER BY 2 DESC LIMIT 1; -ERROR: syntax error at or near "NOT" +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, other_value, (SELECT 1) FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 ORDER BY value DESC LIMIT 1 +DEBUG: Creating router plan +DEBUG: Plan is router executable + key | value | other_value | ?column? +--------------------------------------------------------------------- + 9 | test99 | {"f1": 99, "f2": 1782, "f3": "test99"} | 1 +(1 row) + -- the cte can be inlined because the unsupported -- part of the query (subquery in WHERE clause) -- doesn't access the cte @@ -292,7 +304,9 @@ FROM WHERE key = 1; DEBUG: CTE a is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Creating router plan +DEBUG: Plan is router executable +DETAIL: distribution column value: 1 count --------------------------------------------------------------------- 10 @@ -306,7 +320,15 @@ FROM a WHERE key = 1; -ERROR: syntax error at or near "NOT" +DEBUG: CTE a is going to be inlined via distributed planning +DEBUG: Creating router plan +DEBUG: Plan is router executable +DETAIL: distribution column value: 1 + count +--------------------------------------------------------------------- + 10 +(1 row) + -- using MATERIALIZED should cause inlining not to happen WITH a AS MATERIALIZED (SELECT * FROM test_table) SELECT @@ -315,7 +337,17 @@ FROM a WHERE key = 1; -ERROR: syntax error at or near "MATERIALIZED" +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE a: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) a WHERE (key OPERATOR(pg_catalog.=) 1) +DEBUG: Creating router plan +DEBUG: Plan is router executable + count +--------------------------------------------------------------------- + 10 +(1 row) + -- EXPLAIN should show the difference between materialized an not materialized EXPLAIN (COSTS OFF) WITH a AS (SELECT * FROM test_table) SELECT @@ -325,19 +357,20 @@ FROM WHERE key = 1; DEBUG: CTE a is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries - QUERY PLAN +DEBUG: Creating router plan +DEBUG: Plan is router executable +DETAIL: distribution column value: 1 + QUERY PLAN --------------------------------------------------------------------- - Aggregate - -> Custom Scan (Citus Adaptive) - Task Count: 4 - Tasks Shown: One of 4 - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Aggregate - -> Seq Scan on test_table_1960000 test_table - Filter: (key = 1) -(9 rows) + Custom Scan (Citus Adaptive) + Task Count: 1 + Tasks Shown: All + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Aggregate + -> Seq Scan on test_table_1960000 test_table + Filter: (key = 1) +(8 rows) EXPLAIN (COSTS OFF) WITH a AS MATERIALIZED (SELECT * FROM test_table) SELECT @@ -346,7 +379,31 @@ FROM a WHERE key = 1; -ERROR: syntax error at or near "MATERIALIZED" +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE a: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) a WHERE (key OPERATOR(pg_catalog.=) 1) +DEBUG: Creating router plan +DEBUG: Plan is router executable + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (Citus Adaptive) + -> Distributed Subplan XXX_1 + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on test_table_1960000 test_table + Task Count: 1 + Tasks Shown: All + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Aggregate + -> Function Scan on read_intermediate_result intermediate_result + Filter: (key = 1) +(15 rows) + -- citus should not inline the CTE because it is used multiple times WITH cte_1 AS (SELECT * FROM test_table) SELECT @@ -375,7 +432,25 @@ FROM JOIN cte_1 as second_entry USING (key); -ERROR: syntax error at or near "NOT" +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: join prunable for intervals [-2147483648,-1073741825] and [-1073741824,-1] +DEBUG: join prunable for intervals [-2147483648,-1073741825] and [0,1073741823] +DEBUG: join prunable for intervals [-2147483648,-1073741825] and [1073741824,2147483647] +DEBUG: join prunable for intervals [-1073741824,-1] and [-2147483648,-1073741825] +DEBUG: join prunable for intervals [-1073741824,-1] and [0,1073741823] +DEBUG: join prunable for intervals [-1073741824,-1] and [1073741824,2147483647] +DEBUG: join prunable for intervals [0,1073741823] and [-2147483648,-1073741825] +DEBUG: join prunable for intervals [0,1073741823] and [-1073741824,-1] +DEBUG: join prunable for intervals [0,1073741823] and [1073741824,2147483647] +DEBUG: join prunable for intervals [1073741824,2147483647] and [-2147483648,-1073741825] +DEBUG: join prunable for intervals [1073741824,2147483647] and [-1073741824,-1] +DEBUG: join prunable for intervals [1073741824,2147483647] and [0,1073741823] + count +--------------------------------------------------------------------- + 1021 +(1 row) + -- EXPLAIN should show the differences between MATERIALIZED and NOT MATERIALIZED EXPLAIN (COSTS OFF) WITH cte_1 AS (SELECT * FROM test_table) SELECT @@ -423,7 +498,36 @@ FROM JOIN cte_1 as second_entry USING (key); -ERROR: syntax error at or near "NOT" +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: join prunable for intervals [-2147483648,-1073741825] and [-1073741824,-1] +DEBUG: join prunable for intervals [-2147483648,-1073741825] and [0,1073741823] +DEBUG: join prunable for intervals [-2147483648,-1073741825] and [1073741824,2147483647] +DEBUG: join prunable for intervals [-1073741824,-1] and [-2147483648,-1073741825] +DEBUG: join prunable for intervals [-1073741824,-1] and [0,1073741823] +DEBUG: join prunable for intervals [-1073741824,-1] and [1073741824,2147483647] +DEBUG: join prunable for intervals [0,1073741823] and [-2147483648,-1073741825] +DEBUG: join prunable for intervals [0,1073741823] and [-1073741824,-1] +DEBUG: join prunable for intervals [0,1073741823] and [1073741824,2147483647] +DEBUG: join prunable for intervals [1073741824,2147483647] and [-2147483648,-1073741825] +DEBUG: join prunable for intervals [1073741824,2147483647] and [-1073741824,-1] +DEBUG: join prunable for intervals [1073741824,2147483647] and [0,1073741823] + QUERY PLAN +--------------------------------------------------------------------- + Aggregate + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Aggregate + -> Hash Join + Hash Cond: (test_table.key = test_table_1.key) + -> Seq Scan on test_table_1960000 test_table + -> Hash + -> Seq Scan on test_table_1960000 test_table_1 +(12 rows) + -- ctes with volatile functions are not -- inlined WITH cte_1 AS (SELECT *, random() FROM test_table) @@ -448,7 +552,17 @@ SELECT count(*) FROM cte_1; -ERROR: syntax error at or near "NOT" +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value, random() AS random FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value, intermediate_result.random FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb, random double precision)) cte_1 +DEBUG: Creating router plan +DEBUG: Plan is router executable + count +--------------------------------------------------------------------- + 101 +(1 row) + -- cte_1 should be able to inlined even if -- it is used one level below WITH cte_1 AS (SELECT * FROM test_table) @@ -504,11 +618,11 @@ SELECT count(*) LEFT JOIN test_table u2 ON u2.key = bar.key) AS foo ON TRUE; DEBUG: CTE bar is going to be inlined via distributed planning DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: skipping recursive planning for the subquery since it contains references to outer queries -DEBUG: skipping recursive planning for the subquery since it contains references to outer queries -DEBUG: skipping recursive planning for the subquery since it contains references to outer queries -DEBUG: Router planner cannot handle multi-shard select queries -ERROR: CTEs that refer to other subqueries are not supported in multi-shard queries + count +--------------------------------------------------------------------- + 10331 +(1 row) + -- inlined CTE contains a reference to outer query -- should be fine (even if the recursive planning fails -- to recursively plan the query) @@ -581,7 +695,9 @@ DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Creating router plan +DEBUG: Plan is router executable +DETAIL: distribution column value: 1 count | key --------------------------------------------------------------------- (0 rows) @@ -695,17 +811,11 @@ FROM cte LEFT JOIN test_table USING (key); DEBUG: CTE cte is going to be inlined via distributed planning DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte LEFT JOIN cte_inline.test_table USING (key)) -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE cte: SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte LEFT JOIN cte_inline.test_table USING (key)) -DEBUG: Router planner cannot handle multi-shard select queries -ERROR: cannot pushdown the subquery -DETAIL: Complex subqueries and CTEs cannot be in the outer part of the outer join + count +--------------------------------------------------------------------- + 1021 +(1 row) + -- the CTEs are very simple, so postgres -- can pull-up the subqueries after inlining -- the CTEs, and the query that we send to workers @@ -719,10 +829,18 @@ FROM DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: CTE cte_2 is going to be inlined via distributed planning DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT key FROM cte_inline.test_table -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT test_table.key FROM cte_inline.test_table) cte_1 JOIN (SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) cte_2 USING (key)) -DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: join prunable for intervals [-2147483648,-1073741825] and [-1073741824,-1] +DEBUG: join prunable for intervals [-2147483648,-1073741825] and [0,1073741823] +DEBUG: join prunable for intervals [-2147483648,-1073741825] and [1073741824,2147483647] +DEBUG: join prunable for intervals [-1073741824,-1] and [-2147483648,-1073741825] +DEBUG: join prunable for intervals [-1073741824,-1] and [0,1073741823] +DEBUG: join prunable for intervals [-1073741824,-1] and [1073741824,2147483647] +DEBUG: join prunable for intervals [0,1073741823] and [-2147483648,-1073741825] +DEBUG: join prunable for intervals [0,1073741823] and [-1073741824,-1] +DEBUG: join prunable for intervals [0,1073741823] and [1073741824,2147483647] +DEBUG: join prunable for intervals [1073741824,2147483647] and [-2147483648,-1073741825] +DEBUG: join prunable for intervals [1073741824,2147483647] and [-1073741824,-1] +DEBUG: join prunable for intervals [1073741824,2147483647] and [0,1073741823] count --------------------------------------------------------------------- 1021 @@ -778,7 +896,12 @@ DEBUG: Creating router plan -- we de still don't support it WITH cte_1 AS NOT MATERIALIZED (SELECT * FROM test_table) DELETE FROM test_table WHERE key NOT IN (SELECT key FROM cte_1); -ERROR: syntax error at or near "NOT" +DEBUG: complex joins are only supported when all distributed tables are co-located and joined on their distribution columns +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: DELETE FROM cte_inline.test_table WHERE (NOT (key OPERATOR(pg_catalog.=) ANY (SELECT cte_1.key FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1))) +DEBUG: Creating router plan +DEBUG: Plan is router executable -- we don't inline CTEs if they are modifying CTEs WITH cte_1 AS (DELETE FROM test_table WHERE key % 3 = 1 RETURNING key) SELECT * FROM cte_1 ORDER BY 1 DESC LIMIT 3; @@ -797,7 +920,18 @@ DEBUG: Creating router plan -- NOT MATERIALIZED should not affect modifying CTEs WITH cte_1 AS NOT MATERIALIZED (DELETE FROM test_table WHERE key % 3 = 0 RETURNING key) SELECT count(*) FROM cte_1; -ERROR: syntax error at or near "NOT" +DEBUG: data-modifying statements are not supported in the WITH clauses of distributed queries +DEBUG: generating subplan XXX_1 for CTE cte_1: DELETE FROM cte_inline.test_table WHERE ((key OPERATOR(pg_catalog.%) 3) OPERATOR(pg_catalog.=) 0) RETURNING key +DEBUG: Creating router plan +DEBUG: Plan is router executable +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) cte_1 +DEBUG: Creating router plan +DEBUG: Plan is router executable + count +--------------------------------------------------------------------- + 164 +(1 row) + -- cte with column aliases SELECT * FROM test_table, (WITH cte_1 (x,y) AS (SELECT * FROM test_table), @@ -813,13 +947,13 @@ DEBUG: Router planner cannot handle multi-shard select queries DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT test_table.key, test_table.value, test_table.other_value, bar.z, bar.y, bar.key, bar.t, bar.m, bar.cte_2_key FROM cte_inline.test_table, (SELECT cte_2.z, cte_2.y, cte_2.key, cte_3.t, cte_3.m, cte_3.cte_2_key FROM (SELECT intermediate_result.value AS z, intermediate_result.other_value AS y, intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text, other_value jsonb, key integer)) cte_2, (SELECT cte_2_1.z AS t, cte_2_1.y AS m, cte_2_1.key AS cte_2_key FROM (SELECT intermediate_result.value AS z, intermediate_result.other_value AS y, intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text, other_value jsonb, key integer)) cte_2_1) cte_3) bar ORDER BY test_table.value, test_table.other_value, bar.z, bar.y, bar.t, bar.m, bar.cte_2_key LIMIT 5 DEBUG: Router planner cannot handle multi-shard select queries DEBUG: push down of limit count: 5 - key | value | other_value | z | y | key | t | m | cte_2_key + key | value | other_value | z | y | key | t | m | cte_2_key --------------------------------------------------------------------- - 0 | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | 0 | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | 0 - 0 | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | 0 | test0 | | 0 - 0 | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | 0 | test0 | | 0 - 0 | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | 0 | test0 | | 0 - 0 | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | 0 | test10 | {"f1": 10, "f2": 180, "f3": "test10"} | 0 + 2 | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | 2 | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | 2 + 2 | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | 2 | test12 | | 2 + 2 | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | 2 | test12 | | 2 + 2 | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | 2 | test12 | | 2 + 2 | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | 2 | test15 | {"f1": 15, "f2": 270, "f3": "test15"} | 5 (5 rows) -- cte used in HAVING subquery just works fine @@ -843,12 +977,10 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, count(*) DEBUG: Router planner cannot handle multi-shard select queries key | count --------------------------------------------------------------------- - 0 | 44 - 9 | 40 8 | 40 - 6 | 40 5 | 40 -(5 rows) + 2 | 40 +(3 rows) -- cte used in ORDER BY just works fine -- even if it is inlined @@ -869,9 +1001,9 @@ DEBUG: Router planner cannot handle multi-shard select queries DEBUG: push down of limit count: 3 key --------------------------------------------------------------------- - 9 - 9 - 9 + 8 + 8 + 8 (3 rows) PREPARE inlined_cte_without_params AS @@ -904,7 +1036,7 @@ DEBUG: Router planner cannot handle multi-shard select queries DEBUG: push down of limit count: 3 count --------------------------------------------------------------------- - 44 + 40 40 40 (3 rows) @@ -912,7 +1044,7 @@ DEBUG: push down of limit count: 3 EXECUTE inlined_cte_without_params; count --------------------------------------------------------------------- - 44 + 40 40 40 (3 rows) @@ -920,7 +1052,7 @@ EXECUTE inlined_cte_without_params; EXECUTE inlined_cte_without_params; count --------------------------------------------------------------------- - 44 + 40 40 40 (3 rows) @@ -928,7 +1060,7 @@ EXECUTE inlined_cte_without_params; EXECUTE inlined_cte_without_params; count --------------------------------------------------------------------- - 44 + 40 40 40 (3 rows) @@ -936,7 +1068,7 @@ EXECUTE inlined_cte_without_params; EXECUTE inlined_cte_without_params; count --------------------------------------------------------------------- - 44 + 40 40 40 (3 rows) @@ -944,7 +1076,7 @@ EXECUTE inlined_cte_without_params; EXECUTE inlined_cte_without_params; count --------------------------------------------------------------------- - 44 + 40 40 40 (3 rows) @@ -959,49 +1091,49 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, o DEBUG: Creating router plan key | value | other_value | ?column? --------------------------------------------------------------------- - 9 | test99 | | 1 - 9 | test99 | | 1 - 9 | test99 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 (3 rows) EXECUTE non_inlined_cte_without_params; key | value | other_value | ?column? --------------------------------------------------------------------- - 9 | test99 | | 1 - 9 | test99 | | 1 - 9 | test99 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 (3 rows) EXECUTE non_inlined_cte_without_params; key | value | other_value | ?column? --------------------------------------------------------------------- - 9 | test99 | | 1 - 9 | test99 | | 1 - 9 | test99 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 (3 rows) EXECUTE non_inlined_cte_without_params; key | value | other_value | ?column? --------------------------------------------------------------------- - 9 | test99 | | 1 - 9 | test99 | | 1 - 9 | test99 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 (3 rows) EXECUTE non_inlined_cte_without_params; key | value | other_value | ?column? --------------------------------------------------------------------- - 9 | test99 | | 1 - 9 | test99 | | 1 - 9 | test99 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 (3 rows) EXECUTE non_inlined_cte_without_params; key | value | other_value | ?column? --------------------------------------------------------------------- - 9 | test99 | | 1 - 9 | test99 | | 1 - 9 | test99 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 (3 rows) EXECUTE inlined_cte_has_parameter_on_non_dist_key('test1'); @@ -1027,8 +1159,7 @@ DEBUG: Router planner cannot handle multi-shard select queries DEBUG: push down of limit count: 3 count --------------------------------------------------------------------- - 4 -(1 row) +(0 rows) EXECUTE inlined_cte_has_parameter_on_non_dist_key('test4'); DEBUG: CTE cte_1 is going to be inlined via distributed planning @@ -1055,8 +1186,7 @@ DEBUG: Router planner cannot handle multi-shard select queries DEBUG: push down of limit count: 3 count --------------------------------------------------------------------- - 4 -(1 row) +(0 rows) EXECUTE inlined_cte_has_parameter_on_dist_key(1); DEBUG: CTE cte_1 is going to be inlined via distributed planning @@ -1077,8 +1207,7 @@ DEBUG: push down of limit count: 3 --------------------------------------------------------------------- 40 40 - 40 -(3 rows) +(2 rows) EXECUTE inlined_cte_has_parameter_on_dist_key(3); DEBUG: CTE cte_1 is going to be inlined via distributed planning @@ -1088,8 +1217,7 @@ DEBUG: push down of limit count: 3 --------------------------------------------------------------------- 40 40 - 40 -(3 rows) +(2 rows) EXECUTE inlined_cte_has_parameter_on_dist_key(4); DEBUG: CTE cte_1 is going to be inlined via distributed planning @@ -1099,8 +1227,7 @@ DEBUG: push down of limit count: 3 --------------------------------------------------------------------- 40 40 - 40 -(3 rows) +(2 rows) EXECUTE inlined_cte_has_parameter_on_dist_key(5); DEBUG: CTE cte_1 is going to be inlined via distributed planning @@ -1109,9 +1236,7 @@ DEBUG: push down of limit count: 3 count --------------------------------------------------------------------- 40 - 40 - 40 -(3 rows) +(1 row) EXECUTE inlined_cte_has_parameter_on_dist_key(6); DEBUG: CTE cte_1 is going to be inlined via distributed planning @@ -1122,8 +1247,7 @@ DEBUG: push down of limit count: 3 count --------------------------------------------------------------------- 40 - 40 -(2 rows) +(1 row) EXECUTE non_inlined_cte_has_parameter_on_dist_key(1); DEBUG: CTE cte_1 is going to be inlined via distributed planning @@ -1135,9 +1259,9 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, o DEBUG: Creating router plan key | value | other_value | ?column? --------------------------------------------------------------------- - 9 | test99 | | 1 - 9 | test99 | | 1 - 9 | test99 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 (3 rows) EXECUTE non_inlined_cte_has_parameter_on_dist_key(2); @@ -1150,9 +1274,9 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, o DEBUG: Creating router plan key | value | other_value | ?column? --------------------------------------------------------------------- - 9 | test99 | | 1 - 9 | test99 | | 1 - 9 | test99 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 (3 rows) EXECUTE non_inlined_cte_has_parameter_on_dist_key(3); @@ -1165,9 +1289,9 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, o DEBUG: Creating router plan key | value | other_value | ?column? --------------------------------------------------------------------- - 9 | test99 | | 1 - 9 | test99 | | 1 - 9 | test99 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 (3 rows) EXECUTE non_inlined_cte_has_parameter_on_dist_key(4); @@ -1180,9 +1304,9 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, o DEBUG: Creating router plan key | value | other_value | ?column? --------------------------------------------------------------------- - 9 | test99 | | 1 - 9 | test99 | | 1 - 9 | test99 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 (3 rows) EXECUTE non_inlined_cte_has_parameter_on_dist_key(5); @@ -1195,9 +1319,9 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, o DEBUG: Creating router plan key | value | other_value | ?column? --------------------------------------------------------------------- - 9 | test99 | | 1 - 9 | test99 | | 1 - 9 | test99 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 (3 rows) EXECUTE non_inlined_cte_has_parameter_on_dist_key(6); @@ -1212,49 +1336,49 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, o DEBUG: Creating router plan key | value | other_value | ?column? --------------------------------------------------------------------- - 9 | test99 | | 1 - 9 | test99 | | 1 - 9 | test99 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 (3 rows) EXECUTE retry_planning(1); DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: Router planner cannot handle multi-shard select queries - json_object_agg + json_object_agg --------------------------------------------------------------------- - { "2" : "test12", "2" : "test2", "2" : "test22", "2" : "test32", "2" : "test42", "2" : "test52", "2" : "test62", "2" : "test72", "2" : "test82", "2" : "test92", "3" : "test13", "3" : "test23", "3" : "test3", "3" : "test33", "3" : "test43", "3" : "test53", "3" : "test63", "3" : "test73", "3" : "test83", "3" : "test93", "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "6" : "test16", "6" : "test26", "6" : "test36", "6" : "test46", "6" : "test56", "6" : "test6", "6" : "test66", "6" : "test76", "6" : "test86", "6" : "test96", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } + { "2" : "test12", "2" : "test2", "2" : "test22", "2" : "test32", "2" : "test42", "2" : "test52", "2" : "test62", "2" : "test72", "2" : "test82", "2" : "test92", "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98" } (1 row) EXECUTE retry_planning(2); DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: Router planner cannot handle multi-shard select queries - json_object_agg + json_object_agg --------------------------------------------------------------------- - { "3" : "test13", "3" : "test23", "3" : "test3", "3" : "test33", "3" : "test43", "3" : "test53", "3" : "test63", "3" : "test73", "3" : "test83", "3" : "test93", "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "6" : "test16", "6" : "test26", "6" : "test36", "6" : "test46", "6" : "test56", "6" : "test6", "6" : "test66", "6" : "test76", "6" : "test86", "6" : "test96", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } + { "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98" } (1 row) EXECUTE retry_planning(3); DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: Router planner cannot handle multi-shard select queries - json_object_agg + json_object_agg --------------------------------------------------------------------- - { "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "6" : "test16", "6" : "test26", "6" : "test36", "6" : "test46", "6" : "test56", "6" : "test6", "6" : "test66", "6" : "test76", "6" : "test86", "6" : "test96", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } + { "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98" } (1 row) EXECUTE retry_planning(4); DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: Router planner cannot handle multi-shard select queries - json_object_agg + json_object_agg --------------------------------------------------------------------- - { "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "6" : "test16", "6" : "test26", "6" : "test36", "6" : "test46", "6" : "test56", "6" : "test6", "6" : "test66", "6" : "test76", "6" : "test86", "6" : "test96", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } + { "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98" } (1 row) EXECUTE retry_planning(5); DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: Router planner cannot handle multi-shard select queries - json_object_agg + json_object_agg --------------------------------------------------------------------- - { "6" : "test16", "6" : "test26", "6" : "test36", "6" : "test46", "6" : "test56", "6" : "test6", "6" : "test66", "6" : "test76", "6" : "test86", "6" : "test96", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } + { "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98" } (1 row) EXECUTE retry_planning(6); @@ -1262,9 +1386,9 @@ DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: Router planner cannot handle multi-shard select queries DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: Router planner cannot handle multi-shard select queries - json_object_agg + json_object_agg --------------------------------------------------------------------- - { "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } + { "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98" } (1 row) -- this test can only work if the CTE is recursively @@ -1286,7 +1410,7 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS c DEBUG: Creating router plan count --------------------------------------------------------------------- - 11536 + 4800 (1 row) -- this becomes a non-colocated subquery join @@ -1304,7 +1428,7 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS c DEBUG: Router planner cannot handle multi-shard select queries count --------------------------------------------------------------------- - 1136 + 480 (1 row) -- cte a has to be recursively planned because of OFFSET 0 @@ -1328,7 +1452,7 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT min(a.key) AS DEBUG: Creating router plan min --------------------------------------------------------------------- - 0 + 2 (1 row) -- after both CTEs are inlined, this becomes non-colocated subquery join @@ -1343,11 +1467,11 @@ DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FRO DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT cte_1.key, cte_1.value, cte_1.other_value, cte_2.key, cte_2.value, cte_2.other_value FROM ((SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) cte_1 JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_2 ON ((cte_1.value OPERATOR(pg_catalog.>) cte_2.value))) ORDER BY cte_1.key, cte_1.value, cte_1.other_value, cte_2.key, cte_2.value, cte_2.other_value DESC LIMIT 3 DEBUG: Router planner cannot handle multi-shard select queries DEBUG: push down of limit count: 3 - key | value | other_value | key | value | other_value + key | value | other_value | key | value | other_value --------------------------------------------------------------------- - 0 | test10 | {"f1": 10, "f2": 180, "f3": "test10"} | 0 | test0 | - 0 | test10 | {"f1": 10, "f2": 180, "f3": "test10"} | 0 | test0 | - 0 | test10 | {"f1": 10, "f2": 180, "f3": "test10"} | 0 | test0 | + 2 | test2 | {"f1": 2, "f2": 36, "f3": "test2"} | 2 | test12 | + 2 | test2 | {"f1": 2, "f2": 36, "f3": "test2"} | 2 | test12 | + 2 | test2 | {"f1": 2, "f2": 36, "f3": "test2"} | 2 | test12 | (3 rows) -- full join is only supported when both sides are @@ -1371,9 +1495,9 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT value FROM (( DEBUG: Creating router plan value --------------------------------------------------------------------- - test99 - test99 - test99 + test98 + test98 + test98 (3 rows) -- an unsupported agg. for multi-shard queries @@ -1382,9 +1506,9 @@ WITH cte_1 AS (SELECT * FROM test_table WHERE key > 1) SELECT json_object_agg(DISTINCT key, value) FROM cte_1; DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: Router planner cannot handle multi-shard select queries - json_object_agg + json_object_agg --------------------------------------------------------------------- - { "2" : "test12", "2" : "test2", "2" : "test22", "2" : "test32", "2" : "test42", "2" : "test52", "2" : "test62", "2" : "test72", "2" : "test82", "2" : "test92", "3" : "test13", "3" : "test23", "3" : "test3", "3" : "test33", "3" : "test43", "3" : "test53", "3" : "test63", "3" : "test73", "3" : "test83", "3" : "test93", "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "6" : "test16", "6" : "test26", "6" : "test36", "6" : "test46", "6" : "test56", "6" : "test6", "6" : "test66", "6" : "test76", "6" : "test86", "6" : "test96", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } + { "2" : "test12", "2" : "test2", "2" : "test22", "2" : "test32", "2" : "test42", "2" : "test52", "2" : "test62", "2" : "test72", "2" : "test82", "2" : "test92", "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98" } (1 row) -- both cte_1 and cte_2 are going to be inlined. diff --git a/src/test/regress/expected/cte_inline_1.out b/src/test/regress/expected/cte_inline_1.out new file mode 100644 index 000000000..f5d011bd0 --- /dev/null +++ b/src/test/regress/expected/cte_inline_1.out @@ -0,0 +1,1443 @@ +CREATE SCHEMA cte_inline; +SET search_path TO cte_inline; +SET citus.next_shard_id TO 1960000; +CREATE TABLE test_table (key int, value text, other_value jsonb); +SELECT create_distributed_table ('test_table', 'key'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO test_table SELECT i % 10, 'test' || i, row_to_json(row(i, i*18, 'test' || i)) FROM generate_series (0, 100) i; +-- server version because CTE inlining might produce +-- different debug messages in PG 11 vs PG 12 +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int; + substring +--------------------------------------------------------------------- + 11 +(1 row) + +SET client_min_messages TO DEBUG; +-- Citus should not inline this CTE because otherwise it cannot +-- plan the query +WITH cte_1 AS (SELECT * FROM test_table) +SELECT + *, (SELECT 1) +FROM + cte_1 +ORDER BY 1 DESC LIMIT 3; +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, other_value, (SELECT 1) FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 ORDER BY key DESC LIMIT 3 +DEBUG: Creating router plan +DEBUG: Plan is router executable + key | value | other_value | ?column? +--------------------------------------------------------------------- + 9 | test9 | {"f1": 9, "f2": 162, "f3": "test9"} | 1 + 9 | test19 | {"f1": 19, "f2": 342, "f3": "test19"} | 1 + 9 | test29 | {"f1": 29, "f2": 522, "f3": "test29"} | 1 +(3 rows) + +-- Should still not be inlined even if NOT MATERIALIZED is passed +WITH cte_1 AS NOT MATERIALIZED (SELECT * FROM test_table) +SELECT + *, (SELECT 1) +FROM + cte_1 +ORDER BY 2 DESC LIMIT 1; +ERROR: syntax error at or near "NOT" +-- the cte can be inlined because the unsupported +-- part of the query (subquery in WHERE clause) +-- doesn't access the cte +WITH cte_1 AS (SELECT * FROM test_table) +SELECT + count(*) +FROM + cte_1 +WHERE + key IN ( + SELECT + (SELECT 1) + FROM + test_table WHERE key = 1 + ); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Creating router plan +DEBUG: Plan is router executable +DETAIL: distribution column value: 1 +DEBUG: generating subplan XXX_1 for subquery SELECT (SELECT 1) FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.=) 1) +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) cte_1 WHERE (key OPERATOR(pg_catalog.=) ANY (SELECT intermediate_result."?column?" FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result("?column?" integer))) +DEBUG: Router planner cannot handle multi-shard select queries + count +--------------------------------------------------------------------- + 10 +(1 row) + +-- a similar query as the above, and this time the planning +-- fails, but it fails because the subquery in WHERE clause +-- cannot be planned by Citus +WITH cte_1 AS (SELECT * FROM test_table) +SELECT + count(*) +FROM + cte_1 +WHERE + key IN ( + SELECT + key + FROM + test_table + FOR UPDATE + ); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: SELECT FOR UPDATE with table replication factor > 1 not supported for non-reference tables. +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: SELECT FOR UPDATE with table replication factor > 1 not supported for non-reference tables. +ERROR: could not run distributed query with FOR UPDATE/SHARE commands +HINT: Consider using an equality filter on the distributed table's partition column. +-- Citus does the inlining, the planning fails +-- and retries without inlining, which works +-- fine later via recursive planning +WITH cte_1 AS + (SELECT * + FROM test_table) +SELECT *, (SELECT 1) +FROM + (SELECT * + FROM cte_1) AS foo +ORDER BY 2 DESC LIMIT 1; +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, other_value, (SELECT 1) FROM (SELECT cte_1.key, cte_1.value, cte_1.other_value FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1) foo ORDER BY value DESC LIMIT 1 +DEBUG: Creating router plan +DEBUG: Plan is router executable + key | value | other_value | ?column? +--------------------------------------------------------------------- + 9 | test99 | {"f1": 99, "f2": 1782, "f3": "test99"} | 1 +(1 row) + +-- a little more complicated query tree +-- Citus does the inlining, the planning fails +-- and retries without inlining, which works +WITH top_cte AS + (SELECT * + FROM test_table) +SELECT count(*) +FROM top_cte, + (WITH cte_1 AS + (SELECT * + FROM test_table) SELECT *, (SELECT 1) + FROM + (SELECT * + FROM cte_1) AS foo) AS bar; +DEBUG: CTE top_cte is going to be inlined via distributed planning +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE top_cte: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_2 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Creating router plan +DEBUG: Plan is router executable +DEBUG: generating subplan XXX_3 for subquery SELECT key, value, other_value, (SELECT 1) FROM (SELECT cte_1.key, cte_1.value, cte_1.other_value FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1) foo +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) top_cte, (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value, intermediate_result."?column?" FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb, "?column?" integer)) bar +DEBUG: Creating router plan +DEBUG: Plan is router executable + count +--------------------------------------------------------------------- + 10201 +(1 row) + +-- CTE is used inside a subquery in WHERE clause +-- the query wouldn't work by inlining, so Citus +-- retries again via recursive planning, which +-- works fine +WITH cte_1 AS + (SELECT * + FROM test_table) +SELECT count(*) +FROM test_table +WHERE KEY IN + (SELECT (SELECT 1) + FROM + (SELECT *, + random() + FROM + (SELECT * + FROM cte_1) AS foo) AS bar); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Creating router plan +DEBUG: Plan is router executable +DEBUG: generating subplan XXX_2 for subquery SELECT (SELECT 1) FROM (SELECT foo.key, foo.value, foo.other_value, random() AS random FROM (SELECT cte_1.key, cte_1.value, cte_1.other_value FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1) foo) bar +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.=) ANY (SELECT intermediate_result."?column?" FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result("?column?" integer))) +DEBUG: Router planner cannot handle multi-shard select queries + count +--------------------------------------------------------------------- + 10 +(1 row) + +-- cte_1 is used inside another CTE, but still +-- doesn't work when inlined because it is finally +-- used in an unsupported query +-- but still works fine because recursive planning +-- kicks in +WITH cte_1 AS + (SELECT * + FROM test_table) +SELECT (SELECT 1) AS KEY FROM ( + WITH cte_2 AS (SELECT *, random() + FROM (SELECT *,random() FROM cte_1) as foo) +SELECT *, random() FROM cte_2) as bar ORDER BY 1 DESC LIMIT 3; +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_2: SELECT key, value, other_value, random, random() AS random FROM (SELECT cte_1.key, cte_1.value, cte_1.other_value, random() AS random FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) cte_1) foo +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (SELECT 1) AS key FROM (SELECT cte_2.key, cte_2.value, cte_2.other_value, cte_2.random, cte_2.random_1 AS random, random() AS random FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value, intermediate_result.random, intermediate_result.random_1 AS random FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb, random double precision, random_1 double precision)) cte_2(key, value, other_value, random, random_1)) bar(key, value, other_value, random, random_1, random_2) ORDER BY (SELECT 1) DESC LIMIT 3 +DEBUG: Creating router plan +DEBUG: Plan is router executable + key +--------------------------------------------------------------------- + 1 + 1 + 1 +(3 rows) + +-- in this example, cte_2 can be inlined, because it is not used +-- on any query that Citus cannot plan. However, cte_1 should not be +-- inlined, because it is used with a subquery in target list +WITH cte_1 AS (SELECT * FROM test_table), + cte_2 AS (select * from test_table) +SELECT + count(*) +FROM + (SELECT *, (SELECT 1) FROM cte_1) as foo + JOIN + cte_2 + ON (true); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: CTE cte_2 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT cte_1.key, cte_1.value, cte_1.other_value, (SELECT 1) FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) cte_1) foo JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_2 ON (true)) +DEBUG: Router planner cannot handle multi-shard select queries + count +--------------------------------------------------------------------- + 10201 +(1 row) + +-- unreferenced CTEs are just ignored +-- by Citus/Postgres +WITH a AS (SELECT * FROM test_table) +SELECT + *, row_number() OVER () +FROM + test_table +WHERE + key = 1 +ORDER BY 3 DESC +LIMIT 5; +DEBUG: Creating router plan +DEBUG: Plan is router executable +DETAIL: distribution column value: 1 + key | value | other_value | row_number +--------------------------------------------------------------------- + 1 | test91 | {"f1": 91, "f2": 1638, "f3": "test91"} | 10 + 1 | test81 | {"f1": 81, "f2": 1458, "f3": "test81"} | 9 + 1 | test71 | {"f1": 71, "f2": 1278, "f3": "test71"} | 8 + 1 | test61 | {"f1": 61, "f2": 1098, "f3": "test61"} | 7 + 1 | test51 | {"f1": 51, "f2": 918, "f3": "test51"} | 6 +(5 rows) + +-- router queries are affected by the distributed +-- cte inlining +WITH a AS (SELECT * FROM test_table WHERE key = 1) +SELECT + *, (SELECT 1) +FROM + a +WHERE + key = 1 +ORDER BY 1 DESC +LIMIT 5; +DEBUG: CTE a is going to be inlined via distributed planning +DEBUG: Creating router plan +DEBUG: Plan is router executable +DETAIL: distribution column value: 1 + key | value | other_value | ?column? +--------------------------------------------------------------------- + 1 | test1 | {"f1": 1, "f2": 18, "f3": "test1"} | 1 + 1 | test11 | {"f1": 11, "f2": 198, "f3": "test11"} | 1 + 1 | test21 | {"f1": 21, "f2": 378, "f3": "test21"} | 1 + 1 | test31 | {"f1": 31, "f2": 558, "f3": "test31"} | 1 + 1 | test41 | {"f1": 41, "f2": 738, "f3": "test41"} | 1 +(5 rows) + +-- non router queries are affected by the distributed +-- cte inlining as well +WITH a AS (SELECT * FROM test_table) +SELECT + count(*) +FROM + a +WHERE + key = 1; +DEBUG: CTE a is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries + count +--------------------------------------------------------------------- + 10 +(1 row) + +-- explicitely using NOT MATERIALIZED should result in the same +WITH a AS NOT MATERIALIZED (SELECT * FROM test_table) +SELECT + count(*) +FROM + a +WHERE + key = 1; +ERROR: syntax error at or near "NOT" +-- using MATERIALIZED should cause inlining not to happen +WITH a AS MATERIALIZED (SELECT * FROM test_table) +SELECT + count(*) +FROM + a +WHERE + key = 1; +ERROR: syntax error at or near "MATERIALIZED" +-- EXPLAIN should show the difference between materialized an not materialized +EXPLAIN (COSTS OFF) WITH a AS (SELECT * FROM test_table) +SELECT + count(*) +FROM + a +WHERE + key = 1; +DEBUG: CTE a is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries + QUERY PLAN +--------------------------------------------------------------------- + Aggregate + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Aggregate + -> Seq Scan on test_table_1960000 test_table + Filter: (key = 1) +(9 rows) + +EXPLAIN (COSTS OFF) WITH a AS MATERIALIZED (SELECT * FROM test_table) +SELECT + count(*) +FROM + a +WHERE + key = 1; +ERROR: syntax error at or near "MATERIALIZED" +-- citus should not inline the CTE because it is used multiple times +WITH cte_1 AS (SELECT * FROM test_table) +SELECT + count(*) +FROM + cte_1 as first_entry + JOIN + cte_1 as second_entry + USING (key); +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) first_entry JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) second_entry USING (key)) +DEBUG: Creating router plan +DEBUG: Plan is router executable + count +--------------------------------------------------------------------- + 1021 +(1 row) + +-- NOT MATERIALIZED should cause the query to be inlined twice +WITH cte_1 AS NOT MATERIALIZED (SELECT * FROM test_table) +SELECT + count(*) +FROM + cte_1 as first_entry + JOIN + cte_1 as second_entry + USING (key); +ERROR: syntax error at or near "NOT" +-- EXPLAIN should show the differences between MATERIALIZED and NOT MATERIALIZED +EXPLAIN (COSTS OFF) WITH cte_1 AS (SELECT * FROM test_table) +SELECT + count(*) +FROM + cte_1 as first_entry + JOIN + cte_1 as second_entry + USING (key); +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) first_entry JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) second_entry USING (key)) +DEBUG: Creating router plan +DEBUG: Plan is router executable + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (Citus Adaptive) + -> Distributed Subplan XXX_1 + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on test_table_1960000 test_table + Task Count: 1 + Tasks Shown: All + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (intermediate_result.key = intermediate_result_1.key) + -> Sort + Sort Key: intermediate_result.key + -> Function Scan on read_intermediate_result intermediate_result + -> Sort + Sort Key: intermediate_result_1.key + -> Function Scan on read_intermediate_result intermediate_result_1 +(21 rows) + +EXPLAIN (COSTS OFF) WITH cte_1 AS NOT MATERIALIZED (SELECT * FROM test_table) +SELECT + count(*) +FROM + cte_1 as first_entry + JOIN + cte_1 as second_entry + USING (key); +ERROR: syntax error at or near "NOT" +-- ctes with volatile functions are not +-- inlined +WITH cte_1 AS (SELECT *, random() FROM test_table) +SELECT + key, value +FROM + cte_1 +ORDER BY 2 DESC LIMIT 1; +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value, random() AS random FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value, intermediate_result.random FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb, random double precision)) cte_1 ORDER BY value DESC LIMIT 1 +DEBUG: Creating router plan +DEBUG: Plan is router executable + key | value +--------------------------------------------------------------------- + 9 | test99 +(1 row) + +-- even with NOT MATERIALIZED volatile functions should not be inlined +WITH cte_1 AS NOT MATERIALIZED (SELECT *, random() FROM test_table) +SELECT + count(*) +FROM + cte_1; +ERROR: syntax error at or near "NOT" +-- cte_1 should be able to inlined even if +-- it is used one level below +WITH cte_1 AS (SELECT * FROM test_table) +SELECT + count(*) +FROM +( + WITH ct2 AS (SELECT * FROM cte_1) + SELECT * FROM ct2 +) as foo; +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: CTE ct2 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries + count +--------------------------------------------------------------------- + 101 +(1 row) + +-- a similar query, but there is also +-- one more cte, which relies on the previous +-- CTE +WITH cte_1 AS (SELECT * FROM test_table) +SELECT + count(DISTINCT key) +FROM +( + WITH cte_2 AS (SELECT * FROM cte_1), + cte_3 AS (SELECT * FROM cte_2) + SELECT * FROM cte_3 +) as foo; +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: CTE cte_2 is going to be inlined via distributed planning +DEBUG: CTE cte_3 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries + count +--------------------------------------------------------------------- + 10 +(1 row) + +-- inlined CTE contains a reference to outer query +-- should be fine (because we pushdown the whole query) +SELECT count(*) + FROM + (SELECT * + FROM test_table) AS test_table_cte + JOIN LATERAL + (WITH bar AS (SELECT * + FROM test_table + WHERE key = test_table_cte.key) + SELECT * + FROM + bar + LEFT JOIN test_table u2 ON u2.key = bar.key) AS foo ON TRUE; +DEBUG: CTE bar is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: skipping recursive planning for the subquery since it contains references to outer queries +DEBUG: skipping recursive planning for the subquery since it contains references to outer queries +DEBUG: skipping recursive planning for the subquery since it contains references to outer queries +DEBUG: Router planner cannot handle multi-shard select queries +ERROR: CTEs that refer to other subqueries are not supported in multi-shard queries +-- inlined CTE contains a reference to outer query +-- should be fine (even if the recursive planning fails +-- to recursively plan the query) +SELECT count(*) + FROM + (SELECT * + FROM test_table) AS test_table_cte + JOIN LATERAL + (WITH bar AS (SELECT * + FROM test_table + WHERE key = test_table_cte.key) + SELECT * + FROM + bar + LEFT JOIN test_table u2 ON u2.key = bar.value::int) AS foo ON TRUE; +DEBUG: CTE bar is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: skipping recursive planning for the subquery since it contains references to outer queries +DEBUG: skipping recursive planning for the subquery since it contains references to outer queries +DEBUG: skipping recursive planning for the subquery since it contains references to outer queries +DEBUG: Router planner cannot handle multi-shard select queries +ERROR: CTEs that refer to other subqueries are not supported in multi-shard queries +-- inlined CTE can recursively planned later, that's the decision +-- recursive planning makes +-- LIMIT 5 in cte2 triggers recusrive planning, after cte inlining +WITH cte_1 AS (SELECT * FROM test_table) +SELECT + * +FROM +( + WITH ct2 AS (SELECT * FROM cte_1 ORDER BY 1, 2, 3 LIMIT 5) + SELECT * FROM ct2 +) as foo ORDER BY 1 DESC, 2 DESC, 3 DESC LIMIT 5; +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: CTE ct2 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: push down of limit count: 5 +DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) cte_1 ORDER BY key, value, other_value LIMIT 5 +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, other_value FROM (SELECT ct2.key, ct2.value, ct2.other_value FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) ct2) foo ORDER BY key DESC, value DESC, other_value DESC LIMIT 5 +DEBUG: Creating router plan +DEBUG: Plan is router executable + key | value | other_value +--------------------------------------------------------------------- + 0 | test30 | {"f1": 30, "f2": 540, "f3": "test30"} + 0 | test20 | {"f1": 20, "f2": 360, "f3": "test20"} + 0 | test100 | {"f1": 100, "f2": 1800, "f3": "test100"} + 0 | test10 | {"f1": 10, "f2": 180, "f3": "test10"} + 0 | test0 | {"f1": 0, "f2": 0, "f3": "test0"} +(5 rows) + +-- all nested CTEs can be inlinied +WITH cte_1 AS ( + WITH cte_1 AS ( + WITH cte_1 AS ( + WITH cte_1 AS ( + WITH cte_1 AS ( + WITH cte_1 AS ( + WITH cte_1 AS (SELECT count(*), key FROM test_table GROUP BY key) + SELECT * FROM cte_1) + SELECT * FROM cte_1 WHERE key = 1) + SELECT * FROM cte_1 WHERE key = 2) + SELECT * FROM cte_1 WHERE key = 3) + SELECT * FROM cte_1 WHERE key = 4) + SELECT * FROM cte_1 WHERE key = 5) +SELECT * FROM cte_1 WHERE key = 6; +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries + count | key +--------------------------------------------------------------------- +(0 rows) + +-- ctes can be inlined even if they are used +-- in set operations +WITH cte_1 AS (SELECT * FROM test_table), + cte_2 AS (SELECT * FROM test_table) +SELECT count(*) FROM ( +(SELECT * FROM cte_1 EXCEPT SELECT * FROM test_table) +UNION +(SELECT * FROM cte_2)) as foo; +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: CTE cte_2 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) cte_1 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_2 for subquery SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_3 for subquery SELECT key, value, other_value FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) cte_2 +DEBUG: Creating router plan +DEBUG: Plan is router executable +DEBUG: generating subplan XXX_4 for subquery (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb) EXCEPT SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) UNION SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb) +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_4'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) foo +DEBUG: Creating router plan +DEBUG: Plan is router executable + count +--------------------------------------------------------------------- + 101 +(1 row) + +-- cte_1 is going to be inlined even inside another set operation +WITH cte_1 AS (SELECT * FROM test_table), + cte_2 AS (SELECT * FROM test_table ORDER BY 1 DESC LIMIT 3) +(SELECT *, (SELECT 1) FROM cte_1 EXCEPT SELECT *, 1 FROM test_table) +UNION +(SELECT *, 1 FROM cte_2); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: CTE cte_2 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: push down of limit count: 3 +DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM cte_inline.test_table ORDER BY key DESC LIMIT 3 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_2 for CTE cte_2: SELECT key, value, other_value FROM cte_inline.test_table ORDER BY key DESC LIMIT 3 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: push down of limit count: 3 +DEBUG: Creating router plan +DEBUG: Plan is router executable +DEBUG: generating subplan XXX_3 for subquery SELECT key, value, other_value, (SELECT 1) FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_4 for subquery SELECT key, value, other_value, 1 FROM cte_inline.test_table +DEBUG: Plan XXX query after replacing subqueries and CTEs: (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value, intermediate_result."?column?" FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb, "?column?" integer) EXCEPT SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value, intermediate_result."?column?" FROM read_intermediate_result('XXX_4'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb, "?column?" integer)) UNION SELECT cte_2.key, cte_2.value, cte_2.other_value, 1 FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_2 +DEBUG: Creating router plan +DEBUG: Plan is router executable + key | value | other_value | ?column? +--------------------------------------------------------------------- + 9 | test29 | {"f1": 29, "f2": 522, "f3": "test29"} | 1 + 9 | test9 | {"f1": 9, "f2": 162, "f3": "test9"} | 1 + 9 | test19 | {"f1": 19, "f2": 342, "f3": "test19"} | 1 +(3 rows) + +-- cte_1 is safe to inline, even if because after inlining +-- it'd be in a query tree where there is a query that is +-- not supported by Citus unless recursively planned +-- cte_2 is on another queryTree, should be fine +WITH cte_1 AS (SELECT * FROM test_table), + cte_2 AS (SELECT * FROM test_table) +(SELECT *, (SELECT key FROM cte_1) FROM test_table) +UNION +(SELECT *, 1 FROM cte_2); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: CTE cte_2 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_2 for CTE cte_2: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +ERROR: could not run distributed query with subquery outside the FROM, WHERE and HAVING clauses +HINT: Consider using an equality filter on the distributed table's partition column. +-- after inlining CTEs, the query becomes +-- subquery pushdown with set operations +WITH cte_1 AS (SELECT * FROM test_table), + cte_2 AS (SELECT * FROM test_table) +SELECT max(key) FROM +( + SELECT * FROM cte_1 + UNION + SELECT * FROM cte_2 +) as bar; +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: CTE cte_2 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries + max +--------------------------------------------------------------------- + 9 +(1 row) + +-- cte LEFT JOIN subquery should only work +-- when CTE is inlined, as Citus currently +-- doesn't know how to handle intermediate +-- results in the outer parts of outer +-- queries +WITH cte AS (SELECT * FROM test_table) +SELECT + count(*) +FROM + cte LEFT JOIN test_table USING (key); +DEBUG: CTE cte is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte LEFT JOIN cte_inline.test_table USING (key)) +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte LEFT JOIN cte_inline.test_table USING (key)) +DEBUG: Router planner cannot handle multi-shard select queries +ERROR: cannot pushdown the subquery +DETAIL: Complex subqueries and CTEs cannot be in the outer part of the outer join +-- the CTEs are very simple, so postgres +-- can pull-up the subqueries after inlining +-- the CTEs, and the query that we send to workers +-- becomes a join between two tables +WITH cte_1 AS (SELECT key FROM test_table), + cte_2 AS (SELECT key FROM test_table) +SELECT + count(*) +FROM + cte_1 JOIN cte_2 USING (key); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: CTE cte_2 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for subquery SELECT key FROM cte_inline.test_table +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT test_table.key FROM cte_inline.test_table) cte_1 JOIN (SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) cte_2 USING (key)) +DEBUG: Router planner cannot handle multi-shard select queries + count +--------------------------------------------------------------------- + 1021 +(1 row) + +-- the following query is kind of interesting +-- During INSERT .. SELECT via coordinator, +-- Citus moves the CTEs into SELECT part, and plans/execute +-- the SELECT separately. Thus, fist_table_cte can be inlined +-- by Citus -- but not by Postgres +WITH fist_table_cte AS + (SELECT * FROM test_table) +INSERT INTO test_table + (key, value) + SELECT + key, value + FROM + fist_table_cte; +DEBUG: distributed INSERT ... SELECT can only select from distributed tables +DEBUG: CTE fist_table_cte is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: performing repartitioned INSERT ... SELECT +DEBUG: partitioning SELECT query by column index 0 with name 'key' +DEBUG: distributed statement: INSERT INTO cte_inline.test_table_1960000 AS citus_table_alias (key, value) SELECT key, value FROM read_intermediate_results('{repartitioned_results_xxxxx_from_1960000_to_0}'::text[], 'binary'::citus_copy_format) intermediate_result(key integer, value text) +DEBUG: distributed statement: INSERT INTO cte_inline.test_table_1960001 AS citus_table_alias (key, value) SELECT key, value FROM read_intermediate_results('{repartitioned_results_xxxxx_from_1960001_to_1}'::text[], 'binary'::citus_copy_format) intermediate_result(key integer, value text) +DEBUG: distributed statement: INSERT INTO cte_inline.test_table_1960002 AS citus_table_alias (key, value) SELECT key, value FROM read_intermediate_results('{repartitioned_results_xxxxx_from_1960002_to_2}'::text[], 'binary'::citus_copy_format) intermediate_result(key integer, value text) +DEBUG: distributed statement: INSERT INTO cte_inline.test_table_1960003 AS citus_table_alias (key, value) SELECT key, value FROM read_intermediate_results('{repartitioned_results_xxxxx_from_1960003_to_3}'::text[], 'binary'::citus_copy_format) intermediate_result(key integer, value text) +-- the following INSERT..SELECT is even more interesting +-- the CTE becomes pushdownable +INSERT INTO test_table +WITH fist_table_cte AS + (SELECT * FROM test_table) + SELECT + key, value + FROM + fist_table_cte; +DEBUG: CTE fist_table_cte is going to be inlined via distributed planning +DEBUG: distributed statement: INSERT INTO cte_inline.test_table_1960000 AS citus_table_alias (key, value) SELECT key, value FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table_1960000 test_table) fist_table_cte WHERE ((worker_hash(key) OPERATOR(pg_catalog.>=) '-2147483648'::integer) AND (worker_hash(key) OPERATOR(pg_catalog.<=) '-1073741825'::integer)) +DEBUG: distributed statement: INSERT INTO cte_inline.test_table_1960001 AS citus_table_alias (key, value) SELECT key, value FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table_1960001 test_table) fist_table_cte WHERE ((worker_hash(key) OPERATOR(pg_catalog.>=) '-1073741824'::integer) AND (worker_hash(key) OPERATOR(pg_catalog.<=) '-1'::integer)) +DEBUG: distributed statement: INSERT INTO cte_inline.test_table_1960002 AS citus_table_alias (key, value) SELECT key, value FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table_1960002 test_table) fist_table_cte WHERE ((worker_hash(key) OPERATOR(pg_catalog.>=) 0) AND (worker_hash(key) OPERATOR(pg_catalog.<=) 1073741823)) +DEBUG: distributed statement: INSERT INTO cte_inline.test_table_1960003 AS citus_table_alias (key, value) SELECT key, value FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table_1960003 test_table) fist_table_cte WHERE ((worker_hash(key) OPERATOR(pg_catalog.>=) 1073741824) AND (worker_hash(key) OPERATOR(pg_catalog.<=) 2147483647)) +DEBUG: Plan is router executable +-- update/delete/modifying ctes +-- we don't support any cte inlining in modifications +-- queries and modifying CTEs +WITH cte_1 AS (SELECT * FROM test_table) + DELETE FROM test_table WHERE key NOT IN (SELECT key FROM cte_1); +DEBUG: complex joins are only supported when all distributed tables are co-located and joined on their distribution columns +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: DELETE FROM cte_inline.test_table WHERE (NOT (key OPERATOR(pg_catalog.=) ANY (SELECT cte_1.key FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1))) +DEBUG: Creating router plan +DEBUG: Plan is router executable +-- NOT MATERIALIZED should not CTEs that are used in a modifying query, because +-- we de still don't support it +WITH cte_1 AS NOT MATERIALIZED (SELECT * FROM test_table) + DELETE FROM test_table WHERE key NOT IN (SELECT key FROM cte_1); +ERROR: syntax error at or near "NOT" +-- we don't inline CTEs if they are modifying CTEs +WITH cte_1 AS (DELETE FROM test_table WHERE key % 3 = 1 RETURNING key) +SELECT * FROM cte_1 ORDER BY 1 DESC LIMIT 3; +DEBUG: data-modifying statements are not supported in the WITH clauses of distributed queries +DEBUG: generating subplan XXX_1 for CTE cte_1: DELETE FROM cte_inline.test_table WHERE ((key OPERATOR(pg_catalog.%) 3) OPERATOR(pg_catalog.=) 1) RETURNING key +DEBUG: Creating router plan +DEBUG: Plan is router executable +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key FROM (SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) cte_1 ORDER BY key DESC LIMIT 3 +DEBUG: Creating router plan +DEBUG: Plan is router executable + key +--------------------------------------------------------------------- + 7 + 7 + 7 +(3 rows) + +-- NOT MATERIALIZED should not affect modifying CTEs +WITH cte_1 AS NOT MATERIALIZED (DELETE FROM test_table WHERE key % 3 = 0 RETURNING key) +SELECT count(*) FROM cte_1; +ERROR: syntax error at or near "NOT" +-- cte with column aliases +SELECT * FROM test_table, +(WITH cte_1 (x,y) AS (SELECT * FROM test_table), + cte_2 (z,y) AS (SELECT value, other_value, key FROM test_table), + cte_3 (t,m) AS (SELECT z, y, key as cte_2_key FROM cte_2) + SELECT * FROM cte_2, cte_3) as bar +ORDER BY value, other_value, z, y, t, m, cte_2_key +LIMIT 5; +DEBUG: CTE cte_3 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_2: SELECT value, other_value, key FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT test_table.key, test_table.value, test_table.other_value, bar.z, bar.y, bar.key, bar.t, bar.m, bar.cte_2_key FROM cte_inline.test_table, (SELECT cte_2.z, cte_2.y, cte_2.key, cte_3.t, cte_3.m, cte_3.cte_2_key FROM (SELECT intermediate_result.value AS z, intermediate_result.other_value AS y, intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text, other_value jsonb, key integer)) cte_2, (SELECT cte_2_1.z AS t, cte_2_1.y AS m, cte_2_1.key AS cte_2_key FROM (SELECT intermediate_result.value AS z, intermediate_result.other_value AS y, intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text, other_value jsonb, key integer)) cte_2_1) cte_3) bar ORDER BY test_table.value, test_table.other_value, bar.z, bar.y, bar.t, bar.m, bar.cte_2_key LIMIT 5 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: push down of limit count: 5 + key | value | other_value | z | y | key | t | m | cte_2_key +--------------------------------------------------------------------- + 0 | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | 0 | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | 0 + 0 | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | 0 | test0 | | 0 + 0 | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | 0 | test0 | | 0 + 0 | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | 0 | test0 | | 0 + 0 | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | 0 | test10 | {"f1": 10, "f2": 180, "f3": "test10"} | 0 +(5 rows) + +-- cte used in HAVING subquery just works fine +-- even if it is inlined +WITH cte_1 AS (SELECT max(key) as max FROM test_table) +SELECT + key, count(*) +FROM + test_table +GROUP BY + key +HAVING + (count(*) > (SELECT max FROM cte_1)) +ORDER BY 2 DESC, 1 DESC +LIMIT 5; +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for subquery SELECT max(key) AS max FROM cte_inline.test_table +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, count(*) AS count FROM cte_inline.test_table GROUP BY key HAVING (count(*) OPERATOR(pg_catalog.>) (SELECT cte_1.max FROM (SELECT intermediate_result.max FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(max integer)) cte_1)) ORDER BY (count(*)) DESC, key DESC LIMIT 5 +DEBUG: Router planner cannot handle multi-shard select queries + key | count +--------------------------------------------------------------------- + 0 | 44 + 9 | 40 + 8 | 40 + 6 | 40 + 5 | 40 +(5 rows) + +-- cte used in ORDER BY just works fine +-- even if it is inlined +WITH cte_1 AS (SELECT max(key) as max FROM test_table) +SELECT + key +FROM + test_table JOIN cte_1 ON (key = max) +ORDER BY + cte_1.max +LIMIT 3; +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for subquery SELECT max(key) AS max FROM cte_inline.test_table +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT test_table.key FROM (cte_inline.test_table JOIN (SELECT intermediate_result.max FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(max integer)) cte_1 ON ((test_table.key OPERATOR(pg_catalog.=) cte_1.max))) ORDER BY cte_1.max LIMIT 3 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: push down of limit count: 3 + key +--------------------------------------------------------------------- + 9 + 9 + 9 +(3 rows) + +PREPARE inlined_cte_without_params AS + WITH cte_1 AS (SELECT count(*) FROM test_table GROUP BY key) + SELECT * FROM cte_1 ORDER BY 1 DESC LIMIT 3; +PREPARE non_inlined_cte_without_params AS + WITH cte_1 AS (SELECT * FROM test_table) + SELECT + *, (SELECT 1) + FROM + cte_1 ORDER BY 1 DESC, 2 DESC, 3 DESC LIMIT 3; +PREPARE inlined_cte_has_parameter_on_non_dist_key(text) AS + WITH cte_1 AS (SELECT count(*) FROM test_table WHERE value = $1 GROUP BY key) + SELECT * FROM cte_1 ORDER BY 1 DESC LIMIT 3; +PREPARE inlined_cte_has_parameter_on_dist_key(int) AS + WITH cte_1 AS (SELECT count(*) FROM test_table WHERE key > $1 GROUP BY key) + SELECT * FROM cte_1 ORDER BY 1 DESC LIMIT 3; +PREPARE non_inlined_cte_has_parameter_on_dist_key(int) AS + WITH cte_1 AS (SELECT * FROM test_table where key > $1) + SELECT + *, (SELECT 1) + FROM + cte_1 ORDER BY 1 DESC, 2 DESC, 3 DESC LIMIT 3; +PREPARE retry_planning(int) AS + WITH cte_1 AS (SELECT * FROM test_table WHERE key > $1) + SELECT json_object_agg(DISTINCT key, value) FROM cte_1 ORDER BY max(key), min(value) DESC LIMIT 3; +EXECUTE inlined_cte_without_params; +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: push down of limit count: 3 + count +--------------------------------------------------------------------- + 44 + 40 + 40 +(3 rows) + +EXECUTE inlined_cte_without_params; + count +--------------------------------------------------------------------- + 44 + 40 + 40 +(3 rows) + +EXECUTE inlined_cte_without_params; + count +--------------------------------------------------------------------- + 44 + 40 + 40 +(3 rows) + +EXECUTE inlined_cte_without_params; + count +--------------------------------------------------------------------- + 44 + 40 + 40 +(3 rows) + +EXECUTE inlined_cte_without_params; + count +--------------------------------------------------------------------- + 44 + 40 + 40 +(3 rows) + +EXECUTE inlined_cte_without_params; + count +--------------------------------------------------------------------- + 44 + 40 + 40 +(3 rows) + +EXECUTE non_inlined_cte_without_params; +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, other_value, (SELECT 1) FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 ORDER BY key DESC, value DESC, other_value DESC LIMIT 3 +DEBUG: Creating router plan +DEBUG: Plan is router executable + key | value | other_value | ?column? +--------------------------------------------------------------------- + 9 | test99 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 +(3 rows) + +EXECUTE non_inlined_cte_without_params; + key | value | other_value | ?column? +--------------------------------------------------------------------- + 9 | test99 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 +(3 rows) + +EXECUTE non_inlined_cte_without_params; + key | value | other_value | ?column? +--------------------------------------------------------------------- + 9 | test99 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 +(3 rows) + +EXECUTE non_inlined_cte_without_params; + key | value | other_value | ?column? +--------------------------------------------------------------------- + 9 | test99 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 +(3 rows) + +EXECUTE non_inlined_cte_without_params; + key | value | other_value | ?column? +--------------------------------------------------------------------- + 9 | test99 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 +(3 rows) + +EXECUTE non_inlined_cte_without_params; + key | value | other_value | ?column? +--------------------------------------------------------------------- + 9 | test99 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 +(3 rows) + +EXECUTE inlined_cte_has_parameter_on_non_dist_key('test1'); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: push down of limit count: 3 + count +--------------------------------------------------------------------- +(0 rows) + +EXECUTE inlined_cte_has_parameter_on_non_dist_key('test2'); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: push down of limit count: 3 + count +--------------------------------------------------------------------- + 4 +(1 row) + +EXECUTE inlined_cte_has_parameter_on_non_dist_key('test3'); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: push down of limit count: 3 + count +--------------------------------------------------------------------- + 4 +(1 row) + +EXECUTE inlined_cte_has_parameter_on_non_dist_key('test4'); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: push down of limit count: 3 + count +--------------------------------------------------------------------- +(0 rows) + +EXECUTE inlined_cte_has_parameter_on_non_dist_key('test5'); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: push down of limit count: 3 + count +--------------------------------------------------------------------- + 4 +(1 row) + +EXECUTE inlined_cte_has_parameter_on_non_dist_key('test6'); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: push down of limit count: 3 + count +--------------------------------------------------------------------- + 4 +(1 row) + +EXECUTE inlined_cte_has_parameter_on_dist_key(1); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: push down of limit count: 3 + count +--------------------------------------------------------------------- + 40 + 40 + 40 +(3 rows) + +EXECUTE inlined_cte_has_parameter_on_dist_key(2); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: push down of limit count: 3 + count +--------------------------------------------------------------------- + 40 + 40 + 40 +(3 rows) + +EXECUTE inlined_cte_has_parameter_on_dist_key(3); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: push down of limit count: 3 + count +--------------------------------------------------------------------- + 40 + 40 + 40 +(3 rows) + +EXECUTE inlined_cte_has_parameter_on_dist_key(4); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: push down of limit count: 3 + count +--------------------------------------------------------------------- + 40 + 40 + 40 +(3 rows) + +EXECUTE inlined_cte_has_parameter_on_dist_key(5); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: push down of limit count: 3 + count +--------------------------------------------------------------------- + 40 + 40 + 40 +(3 rows) + +EXECUTE inlined_cte_has_parameter_on_dist_key(6); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: push down of limit count: 3 + count +--------------------------------------------------------------------- + 40 + 40 +(2 rows) + +EXECUTE non_inlined_cte_has_parameter_on_dist_key(1); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.>) 1) +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, other_value, (SELECT 1) FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 ORDER BY key DESC, value DESC, other_value DESC LIMIT 3 +DEBUG: Creating router plan +DEBUG: Plan is router executable + key | value | other_value | ?column? +--------------------------------------------------------------------- + 9 | test99 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 +(3 rows) + +EXECUTE non_inlined_cte_has_parameter_on_dist_key(2); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.>) 2) +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, other_value, (SELECT 1) FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 ORDER BY key DESC, value DESC, other_value DESC LIMIT 3 +DEBUG: Creating router plan +DEBUG: Plan is router executable + key | value | other_value | ?column? +--------------------------------------------------------------------- + 9 | test99 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 +(3 rows) + +EXECUTE non_inlined_cte_has_parameter_on_dist_key(3); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.>) 3) +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, other_value, (SELECT 1) FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 ORDER BY key DESC, value DESC, other_value DESC LIMIT 3 +DEBUG: Creating router plan +DEBUG: Plan is router executable + key | value | other_value | ?column? +--------------------------------------------------------------------- + 9 | test99 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 +(3 rows) + +EXECUTE non_inlined_cte_has_parameter_on_dist_key(4); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.>) 4) +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, other_value, (SELECT 1) FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 ORDER BY key DESC, value DESC, other_value DESC LIMIT 3 +DEBUG: Creating router plan +DEBUG: Plan is router executable + key | value | other_value | ?column? +--------------------------------------------------------------------- + 9 | test99 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 +(3 rows) + +EXECUTE non_inlined_cte_has_parameter_on_dist_key(5); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.>) 5) +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, other_value, (SELECT 1) FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 ORDER BY key DESC, value DESC, other_value DESC LIMIT 3 +DEBUG: Creating router plan +DEBUG: Plan is router executable + key | value | other_value | ?column? +--------------------------------------------------------------------- + 9 | test99 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 +(3 rows) + +EXECUTE non_inlined_cte_has_parameter_on_dist_key(6); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.>) 6) +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, other_value, (SELECT 1) FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 ORDER BY key DESC, value DESC, other_value DESC LIMIT 3 +DEBUG: Creating router plan +DEBUG: Plan is router executable + key | value | other_value | ?column? +--------------------------------------------------------------------- + 9 | test99 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 +(3 rows) + +EXECUTE retry_planning(1); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries + json_object_agg +--------------------------------------------------------------------- + { "2" : "test12", "2" : "test2", "2" : "test22", "2" : "test32", "2" : "test42", "2" : "test52", "2" : "test62", "2" : "test72", "2" : "test82", "2" : "test92", "3" : "test13", "3" : "test23", "3" : "test3", "3" : "test33", "3" : "test43", "3" : "test53", "3" : "test63", "3" : "test73", "3" : "test83", "3" : "test93", "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "6" : "test16", "6" : "test26", "6" : "test36", "6" : "test46", "6" : "test56", "6" : "test6", "6" : "test66", "6" : "test76", "6" : "test86", "6" : "test96", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } +(1 row) + +EXECUTE retry_planning(2); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries + json_object_agg +--------------------------------------------------------------------- + { "3" : "test13", "3" : "test23", "3" : "test3", "3" : "test33", "3" : "test43", "3" : "test53", "3" : "test63", "3" : "test73", "3" : "test83", "3" : "test93", "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "6" : "test16", "6" : "test26", "6" : "test36", "6" : "test46", "6" : "test56", "6" : "test6", "6" : "test66", "6" : "test76", "6" : "test86", "6" : "test96", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } +(1 row) + +EXECUTE retry_planning(3); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries + json_object_agg +--------------------------------------------------------------------- + { "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "6" : "test16", "6" : "test26", "6" : "test36", "6" : "test46", "6" : "test56", "6" : "test6", "6" : "test66", "6" : "test76", "6" : "test86", "6" : "test96", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } +(1 row) + +EXECUTE retry_planning(4); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries + json_object_agg +--------------------------------------------------------------------- + { "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "6" : "test16", "6" : "test26", "6" : "test36", "6" : "test46", "6" : "test56", "6" : "test6", "6" : "test66", "6" : "test76", "6" : "test86", "6" : "test96", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } +(1 row) + +EXECUTE retry_planning(5); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries + json_object_agg +--------------------------------------------------------------------- + { "6" : "test16", "6" : "test26", "6" : "test36", "6" : "test46", "6" : "test56", "6" : "test6", "6" : "test66", "6" : "test76", "6" : "test86", "6" : "test96", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } +(1 row) + +EXECUTE retry_planning(6); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries + json_object_agg +--------------------------------------------------------------------- + { "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } +(1 row) + +-- this test can only work if the CTE is recursively +-- planned +WITH b AS (SELECT * FROM test_table) +SELECT count(*) FROM (SELECT key as x FROM test_table OFFSET 0) as ref LEFT JOIN b ON (ref.x = b.key); +DEBUG: CTE b is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for subquery SELECT key AS x FROM cte_inline.test_table OFFSET 0 +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT intermediate_result.x FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(x integer)) ref LEFT JOIN (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) b ON ((ref.x OPERATOR(pg_catalog.=) b.key))) +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE b: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_2 for subquery SELECT key AS x FROM cte_inline.test_table OFFSET 0 +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT intermediate_result.x FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(x integer)) ref LEFT JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) b ON ((ref.x OPERATOR(pg_catalog.=) b.key))) +DEBUG: Creating router plan +DEBUG: Plan is router executable + count +--------------------------------------------------------------------- + 11536 +(1 row) + +-- this becomes a non-colocated subquery join +-- because after the CTEs are inlined the joins +-- become a non-colocated subquery join +WITH a AS (SELECT * FROM test_table), +b AS (SELECT * FROM test_table) +SELECT count(*) FROM a LEFT JOIN b ON (a.value = b.value); +DEBUG: CTE a is going to be inlined via distributed planning +DEBUG: CTE b is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) a LEFT JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) b ON ((a.value OPERATOR(pg_catalog.=) b.value))) +DEBUG: Router planner cannot handle multi-shard select queries + count +--------------------------------------------------------------------- + 1136 +(1 row) + +-- cte a has to be recursively planned because of OFFSET 0 +-- after that, cte b also requires recursive planning +WITH a AS (SELECT * FROM test_table OFFSET 0), +b AS (SELECT * FROM test_table) +SELECT min(a.key) FROM a LEFT JOIN b ON (a.value = b.value); +DEBUG: CTE a is going to be inlined via distributed planning +DEBUG: CTE b is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM cte_inline.test_table OFFSET 0 +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT min(a.key) AS min FROM ((SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) a LEFT JOIN (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) b ON ((a.value OPERATOR(pg_catalog.=) b.value))) +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE a: SELECT key, value, other_value FROM cte_inline.test_table OFFSET 0 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_2 for CTE b: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT min(a.key) AS min FROM ((SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) a LEFT JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) b ON ((a.value OPERATOR(pg_catalog.=) b.value))) +DEBUG: Creating router plan +DEBUG: Plan is router executable + min +--------------------------------------------------------------------- + 0 +(1 row) + +-- after both CTEs are inlined, this becomes non-colocated subquery join +WITH cte_1 AS (SELECT * FROM test_table), +cte_2 AS (SELECT * FROM test_table) +SELECT * FROM cte_1 JOIN cte_2 ON (cte_1.value > cte_2.value) ORDER BY 1,2,3,4,5,6 DESC LIMIT 3;; +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: CTE cte_2 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT cte_1.key, cte_1.value, cte_1.other_value, cte_2.key, cte_2.value, cte_2.other_value FROM ((SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) cte_1 JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_2 ON ((cte_1.value OPERATOR(pg_catalog.>) cte_2.value))) ORDER BY cte_1.key, cte_1.value, cte_1.other_value, cte_2.key, cte_2.value, cte_2.other_value DESC LIMIT 3 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: push down of limit count: 3 + key | value | other_value | key | value | other_value +--------------------------------------------------------------------- + 0 | test10 | {"f1": 10, "f2": 180, "f3": "test10"} | 0 | test0 | + 0 | test10 | {"f1": 10, "f2": 180, "f3": "test10"} | 0 | test0 | + 0 | test10 | {"f1": 10, "f2": 180, "f3": "test10"} | 0 | test0 | +(3 rows) + +-- full join is only supported when both sides are +-- recursively planned +WITH cte_1 AS (SELECT value FROM test_table WHERE key > 1), + cte_2 AS (SELECT value FROM test_table WHERE key > 3) +SELECT * FROM cte_1 FULL JOIN cte_2 USING (value) ORDER BY 1 DESC LIMIT 3;; +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: CTE cte_2 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for subquery SELECT value FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.>) 3) +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT value FROM ((SELECT test_table.value FROM cte_inline.test_table WHERE (test_table.key OPERATOR(pg_catalog.>) 1)) cte_1 FULL JOIN (SELECT intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text)) cte_2 USING (value)) ORDER BY value DESC LIMIT 3 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT value FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.>) 1) +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_2 for CTE cte_2: SELECT value FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.>) 3) +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT value FROM ((SELECT intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text)) cte_1 FULL JOIN (SELECT intermediate_result.value FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(value text)) cte_2 USING (value)) ORDER BY value DESC LIMIT 3 +DEBUG: Creating router plan +DEBUG: Plan is router executable + value +--------------------------------------------------------------------- + test99 + test99 + test99 +(3 rows) + +-- an unsupported agg. for multi-shard queries +-- so CTE has to be recursively planned +WITH cte_1 AS (SELECT * FROM test_table WHERE key > 1) +SELECT json_object_agg(DISTINCT key, value) FROM cte_1; +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries + json_object_agg +--------------------------------------------------------------------- + { "2" : "test12", "2" : "test2", "2" : "test22", "2" : "test32", "2" : "test42", "2" : "test52", "2" : "test62", "2" : "test72", "2" : "test82", "2" : "test92", "3" : "test13", "3" : "test23", "3" : "test3", "3" : "test33", "3" : "test43", "3" : "test53", "3" : "test63", "3" : "test73", "3" : "test83", "3" : "test93", "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "6" : "test16", "6" : "test26", "6" : "test36", "6" : "test46", "6" : "test56", "6" : "test6", "6" : "test66", "6" : "test76", "6" : "test86", "6" : "test96", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } +(1 row) + +-- both cte_1 and cte_2 are going to be inlined. +-- later, cte_2 is recursively planned since it doesn't have +-- GROUP BY but aggragate in a subquery. +-- this is an important example of being able to recursively plan +-- "some" of the CTEs +WITH cte_1 AS (SELECT value FROM test_table WHERE key > 1), + cte_2 AS (SELECT max(value) as value FROM test_table WHERE key > 3) +SELECT count(*) FROM cte_1 JOIN cte_2 USING (value); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: CTE cte_2 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for subquery SELECT max(value) AS value FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.>) 3) +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT test_table.value FROM cte_inline.test_table WHERE (test_table.key OPERATOR(pg_catalog.>) 1)) cte_1 JOIN (SELECT intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text)) cte_2 USING (value)) +DEBUG: Router planner cannot handle multi-shard select queries + count +--------------------------------------------------------------------- + 4 +(1 row) + +-- prevent DROP CASCADE to give notices +SET client_min_messages TO ERROR; +DROP SCHEMA cte_inline CASCADE; diff --git a/src/test/regress/expected/multi_complex_count_distinct_1.out b/src/test/regress/expected/multi_complex_count_distinct_1.out new file mode 100644 index 000000000..b14e9c7e8 --- /dev/null +++ b/src/test/regress/expected/multi_complex_count_distinct_1.out @@ -0,0 +1,1128 @@ +-- +-- COMPLEX_COUNT_DISTINCT +-- +SET citus.next_shard_id TO 240000; +SET citus.shard_count TO 8; +SET citus.shard_replication_factor TO 1; +SET citus.coordinator_aggregation_strategy TO 'disabled'; +CREATE TABLE lineitem_hash ( + l_orderkey bigint not null, + l_partkey integer not null, + l_suppkey integer not null, + l_linenumber integer not null, + l_quantity decimal(15, 2) not null, + l_extendedprice decimal(15, 2) not null, + l_discount decimal(15, 2) not null, + l_tax decimal(15, 2) not null, + l_returnflag char(1) not null, + l_linestatus char(1) not null, + l_shipdate date not null, + l_commitdate date not null, + l_receiptdate date not null, + l_shipinstruct char(25) not null, + l_shipmode char(10) not null, + l_comment varchar(44) not null, + PRIMARY KEY(l_orderkey, l_linenumber) ); +SELECT create_distributed_table('lineitem_hash', 'l_orderkey', 'hash'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +\copy lineitem_hash FROM '/home/talha/citus/src/test/regress/data/lineitem.1.data' with delimiter '|' +\copy lineitem_hash FROM '/home/talha/citus/src/test/regress/data/lineitem.2.data' with delimiter '|' +ANALYZE lineitem_hash; +SET citus.task_executor_type to "task-tracker"; +-- count(distinct) is supported on top level query if there +-- is a grouping on the partition key +SELECT + l_orderkey, count(DISTINCT l_partkey) + FROM lineitem_hash + GROUP BY l_orderkey + ORDER BY 2 DESC, 1 DESC + LIMIT 10; + l_orderkey | count +--------------------------------------------------------------------- + 14885 | 7 + 14884 | 7 + 14821 | 7 + 14790 | 7 + 14785 | 7 + 14755 | 7 + 14725 | 7 + 14694 | 7 + 14627 | 7 + 14624 | 7 +(10 rows) + +EXPLAIN (COSTS false, VERBOSE true) +SELECT + l_orderkey, count(DISTINCT l_partkey) + FROM lineitem_hash + GROUP BY l_orderkey + ORDER BY 2 DESC, 1 DESC + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + Output: remote_scan.l_orderkey, remote_scan.count + -> Sort + Output: remote_scan.l_orderkey, remote_scan.count + Sort Key: remote_scan.count DESC, remote_scan.l_orderkey DESC + -> Custom Scan (Citus Task-Tracker) + Output: remote_scan.l_orderkey, remote_scan.count + Task Count: 8 + Tasks Shown: One of 8 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Limit + Output: l_orderkey, (count(DISTINCT l_partkey)) + -> Sort + Output: l_orderkey, (count(DISTINCT l_partkey)) + Sort Key: (count(DISTINCT lineitem_hash.l_partkey)) DESC, lineitem_hash.l_orderkey DESC + -> GroupAggregate + Output: l_orderkey, count(DISTINCT l_partkey) + Group Key: lineitem_hash.l_orderkey + -> Index Scan Backward using lineitem_hash_pkey_240000 on public.lineitem_hash_240000 lineitem_hash + Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment +(21 rows) + +-- it is also supported if there is no grouping or grouping is on non-partition field +SELECT + count(DISTINCT l_partkey) + FROM lineitem_hash + ORDER BY 1 DESC + LIMIT 10; + count +--------------------------------------------------------------------- + 11661 +(1 row) + +EXPLAIN (COSTS false, VERBOSE true) +SELECT + count(DISTINCT l_partkey) + FROM lineitem_hash + ORDER BY 1 DESC + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + Output: (count(DISTINCT remote_scan.count)) + -> Sort + Output: (count(DISTINCT remote_scan.count)) + Sort Key: (count(DISTINCT remote_scan.count)) DESC + -> Aggregate + Output: count(DISTINCT remote_scan.count) + -> Custom Scan (Citus Task-Tracker) + Output: remote_scan.count + Task Count: 8 + Tasks Shown: One of 8 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: l_partkey + Group Key: lineitem_hash.l_partkey + -> Seq Scan on public.lineitem_hash_240000 lineitem_hash + Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment +(18 rows) + +SELECT + l_shipmode, count(DISTINCT l_partkey) + FROM lineitem_hash + GROUP BY l_shipmode + ORDER BY 2 DESC, 1 DESC + LIMIT 10; + l_shipmode | count +--------------------------------------------------------------------- + TRUCK | 1757 + MAIL | 1730 + AIR | 1702 + FOB | 1700 + RAIL | 1696 + SHIP | 1684 + REG AIR | 1676 +(7 rows) + +EXPLAIN (COSTS false, VERBOSE true) +SELECT + l_shipmode, count(DISTINCT l_partkey) + FROM lineitem_hash + GROUP BY l_shipmode + ORDER BY 2 DESC, 1 DESC + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + Output: remote_scan.l_shipmode, (count(DISTINCT remote_scan.count)) + -> Sort + Output: remote_scan.l_shipmode, (count(DISTINCT remote_scan.count)) + Sort Key: (count(DISTINCT remote_scan.count)) DESC, remote_scan.l_shipmode DESC + -> GroupAggregate + Output: remote_scan.l_shipmode, count(DISTINCT remote_scan.count) + Group Key: remote_scan.l_shipmode + -> Sort + Output: remote_scan.l_shipmode, remote_scan.count + Sort Key: remote_scan.l_shipmode DESC + -> Custom Scan (Citus Task-Tracker) + Output: remote_scan.l_shipmode, remote_scan.count + Task Count: 8 + Tasks Shown: One of 8 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: l_shipmode, l_partkey + Group Key: lineitem_hash.l_shipmode, lineitem_hash.l_partkey + -> Seq Scan on public.lineitem_hash_240000 lineitem_hash + Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment +(22 rows) + +-- mixed mode count distinct, grouped by partition column +SELECT + l_orderkey, count(distinct l_partkey), count(distinct l_shipmode) + FROM lineitem_hash + GROUP BY l_orderkey + ORDER BY 3 DESC, 2 DESC, 1 + LIMIT 10; + l_orderkey | count | count +--------------------------------------------------------------------- + 226 | 7 | 7 + 1316 | 7 | 7 + 1477 | 7 | 7 + 3555 | 7 | 7 + 12258 | 7 | 7 + 12835 | 7 | 7 + 768 | 7 | 6 + 1121 | 7 | 6 + 1153 | 7 | 6 + 1281 | 7 | 6 +(10 rows) + +EXPLAIN (COSTS false, VERBOSE true) +SELECT + l_orderkey, count(distinct l_partkey), count(distinct l_shipmode) + FROM lineitem_hash + GROUP BY l_orderkey + ORDER BY 3 DESC, 2 DESC, 1 + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + Output: remote_scan.l_orderkey, remote_scan.count, remote_scan.count_1 + -> Sort + Output: remote_scan.l_orderkey, remote_scan.count, remote_scan.count_1 + Sort Key: remote_scan.count_1 DESC, remote_scan.count DESC, remote_scan.l_orderkey + -> Custom Scan (Citus Task-Tracker) + Output: remote_scan.l_orderkey, remote_scan.count, remote_scan.count_1 + Task Count: 8 + Tasks Shown: One of 8 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Limit + Output: l_orderkey, (count(DISTINCT l_partkey)), (count(DISTINCT l_shipmode)) + -> Sort + Output: l_orderkey, (count(DISTINCT l_partkey)), (count(DISTINCT l_shipmode)) + Sort Key: (count(DISTINCT lineitem_hash.l_shipmode)) DESC, (count(DISTINCT lineitem_hash.l_partkey)) DESC, lineitem_hash.l_orderkey + -> GroupAggregate + Output: l_orderkey, count(DISTINCT l_partkey), count(DISTINCT l_shipmode) + Group Key: lineitem_hash.l_orderkey + -> Index Scan using lineitem_hash_pkey_240000 on public.lineitem_hash_240000 lineitem_hash + Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment +(21 rows) + +-- partition/non-partition column count distinct no grouping +SELECT + count(distinct l_orderkey), count(distinct l_partkey), count(distinct l_shipmode) + FROM lineitem_hash; + count | count | count +--------------------------------------------------------------------- + 2985 | 11661 | 7 +(1 row) + +EXPLAIN (COSTS false, VERBOSE true) +SELECT + count(distinct l_orderkey), count(distinct l_partkey), count(distinct l_shipmode) + FROM lineitem_hash; + QUERY PLAN +--------------------------------------------------------------------- + Aggregate + Output: count(DISTINCT remote_scan.count), count(DISTINCT remote_scan.count_1), count(DISTINCT remote_scan.count_2) + -> Custom Scan (Citus Task-Tracker) + Output: remote_scan.count, remote_scan.count_1, remote_scan.count_2 + Task Count: 8 + Tasks Shown: One of 8 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: l_orderkey, l_partkey, l_shipmode + Group Key: lineitem_hash.l_orderkey, lineitem_hash.l_partkey, lineitem_hash.l_shipmode + -> Seq Scan on public.lineitem_hash_240000 lineitem_hash + Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment +(13 rows) + +-- distinct/non-distinct on partition and non-partition columns +SELECT + count(distinct l_orderkey), count(l_orderkey), + count(distinct l_partkey), count(l_partkey), + count(distinct l_shipmode), count(l_shipmode) + FROM lineitem_hash; + count | count | count | count | count | count +--------------------------------------------------------------------- + 2985 | 12000 | 11661 | 12000 | 7 | 12000 +(1 row) + +-- mixed mode count distinct, grouped by non-partition column +SELECT + l_shipmode, count(distinct l_partkey), count(distinct l_orderkey) + FROM lineitem_hash + GROUP BY l_shipmode + ORDER BY 1, 2 DESC, 3 DESC; + l_shipmode | count | count +--------------------------------------------------------------------- + AIR | 1702 | 1327 + FOB | 1700 | 1276 + MAIL | 1730 | 1299 + RAIL | 1696 | 1265 + REG AIR | 1676 | 1275 + SHIP | 1684 | 1289 + TRUCK | 1757 | 1333 +(7 rows) + +-- mixed mode count distinct, grouped by non-partition column +-- having on partition column +SELECT + l_shipmode, count(distinct l_partkey), count(distinct l_orderkey) + FROM lineitem_hash + GROUP BY l_shipmode + HAVING count(distinct l_orderkey) > 1300 + ORDER BY 1, 2 DESC; + l_shipmode | count | count +--------------------------------------------------------------------- + AIR | 1702 | 1327 + TRUCK | 1757 | 1333 +(2 rows) + +-- same but having clause is not on target list +SELECT + l_shipmode, count(distinct l_partkey) + FROM lineitem_hash + GROUP BY l_shipmode + HAVING count(distinct l_orderkey) > 1300 + ORDER BY 1, 2 DESC; + l_shipmode | count +--------------------------------------------------------------------- + AIR | 1702 + TRUCK | 1757 +(2 rows) + +-- mixed mode count distinct, grouped by non-partition column +-- having on non-partition column +SELECT + l_shipmode, count(distinct l_partkey), count(distinct l_suppkey) + FROM lineitem_hash + GROUP BY l_shipmode + HAVING count(distinct l_suppkey) > 1550 + ORDER BY 1, 2 DESC; + l_shipmode | count | count +--------------------------------------------------------------------- + AIR | 1702 | 1564 + FOB | 1700 | 1571 + MAIL | 1730 | 1573 + RAIL | 1696 | 1581 + REG AIR | 1676 | 1557 + SHIP | 1684 | 1554 + TRUCK | 1757 | 1602 +(7 rows) + +-- same but having clause is not on target list +SELECT + l_shipmode, count(distinct l_partkey) + FROM lineitem_hash + GROUP BY l_shipmode + HAVING count(distinct l_suppkey) > 1550 + ORDER BY 1, 2 DESC; + l_shipmode | count +--------------------------------------------------------------------- + AIR | 1702 + FOB | 1700 + MAIL | 1730 + RAIL | 1696 + REG AIR | 1676 + SHIP | 1684 + TRUCK | 1757 +(7 rows) + +EXPLAIN (COSTS false, VERBOSE true) +SELECT + l_shipmode, count(distinct l_partkey) + FROM lineitem_hash + GROUP BY l_shipmode + HAVING count(distinct l_suppkey) > 1550 + ORDER BY 1, 2 DESC; + QUERY PLAN +--------------------------------------------------------------------- + Sort + Output: remote_scan.l_shipmode, (count(DISTINCT remote_scan.count)) + Sort Key: remote_scan.l_shipmode, (count(DISTINCT remote_scan.count)) DESC + -> GroupAggregate + Output: remote_scan.l_shipmode, count(DISTINCT remote_scan.count) + Group Key: remote_scan.l_shipmode + Filter: (count(DISTINCT remote_scan.worker_column_3) > 1550) + -> Sort + Output: remote_scan.l_shipmode, remote_scan.count, remote_scan.worker_column_3 + Sort Key: remote_scan.l_shipmode + -> Custom Scan (Citus Task-Tracker) + Output: remote_scan.l_shipmode, remote_scan.count, remote_scan.worker_column_3 + Task Count: 8 + Tasks Shown: One of 8 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: l_shipmode, l_partkey, l_suppkey + Group Key: lineitem_hash.l_shipmode, lineitem_hash.l_partkey, lineitem_hash.l_suppkey + -> Seq Scan on public.lineitem_hash_240000 lineitem_hash + Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment +(21 rows) + +-- count distinct is supported on single table subqueries +SELECT * + FROM ( + SELECT + l_orderkey, count(DISTINCT l_partkey) + FROM lineitem_hash + GROUP BY l_orderkey) sub + ORDER BY 2 DESC, 1 DESC + LIMIT 10; + l_orderkey | count +--------------------------------------------------------------------- + 14885 | 7 + 14884 | 7 + 14821 | 7 + 14790 | 7 + 14785 | 7 + 14755 | 7 + 14725 | 7 + 14694 | 7 + 14627 | 7 + 14624 | 7 +(10 rows) + +SELECT * + FROM ( + SELECT + l_partkey, count(DISTINCT l_orderkey) + FROM lineitem_hash + GROUP BY l_partkey) sub + ORDER BY 2 DESC, 1 DESC + LIMIT 10; + l_partkey | count +--------------------------------------------------------------------- + 199146 | 3 + 188804 | 3 + 177771 | 3 + 160895 | 3 + 149926 | 3 + 136884 | 3 + 87761 | 3 + 15283 | 3 + 6983 | 3 + 1927 | 3 +(10 rows) + +EXPLAIN (COSTS false, VERBOSE true) +SELECT * + FROM ( + SELECT + l_partkey, count(DISTINCT l_orderkey) + FROM lineitem_hash + GROUP BY l_partkey) sub + ORDER BY 2 DESC, 1 DESC + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + Output: remote_scan.l_partkey, remote_scan.count + -> Sort + Output: remote_scan.l_partkey, remote_scan.count + Sort Key: remote_scan.count DESC, remote_scan.l_partkey DESC + -> Custom Scan (Citus Task-Tracker) + Output: remote_scan.l_partkey, remote_scan.count + Task Count: 4 + Tasks Shown: None, not supported for re-partition queries + -> MapMergeJob + Map Task Count: 8 + Merge Task Count: 4 +(12 rows) + +-- count distinct with filters +SELECT + l_orderkey, + count(DISTINCT l_suppkey) FILTER (WHERE l_shipmode = 'AIR'), + count(DISTINCT l_suppkey) + FROM lineitem_hash + GROUP BY l_orderkey + ORDER BY 2 DESC, 3 DESC, 1 + LIMIT 10; + l_orderkey | count | count +--------------------------------------------------------------------- + 4964 | 4 | 7 + 12005 | 4 | 7 + 5409 | 4 | 6 + 164 | 3 | 7 + 322 | 3 | 7 + 871 | 3 | 7 + 1156 | 3 | 7 + 1574 | 3 | 7 + 2054 | 3 | 7 + 2309 | 3 | 7 +(10 rows) + +EXPLAIN (COSTS false, VERBOSE true) +SELECT + l_orderkey, + count(DISTINCT l_suppkey) FILTER (WHERE l_shipmode = 'AIR'), + count(DISTINCT l_suppkey) + FROM lineitem_hash + GROUP BY l_orderkey + ORDER BY 2 DESC, 3 DESC, 1 + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + Output: remote_scan.l_orderkey, remote_scan.count, remote_scan.count_1 + -> Sort + Output: remote_scan.l_orderkey, remote_scan.count, remote_scan.count_1 + Sort Key: remote_scan.count DESC, remote_scan.count_1 DESC, remote_scan.l_orderkey + -> Custom Scan (Citus Task-Tracker) + Output: remote_scan.l_orderkey, remote_scan.count, remote_scan.count_1 + Task Count: 8 + Tasks Shown: One of 8 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Limit + Output: l_orderkey, (count(DISTINCT l_suppkey) FILTER (WHERE (l_shipmode = 'AIR'::bpchar))), (count(DISTINCT l_suppkey)) + -> Sort + Output: l_orderkey, (count(DISTINCT l_suppkey) FILTER (WHERE (l_shipmode = 'AIR'::bpchar))), (count(DISTINCT l_suppkey)) + Sort Key: (count(DISTINCT lineitem_hash.l_suppkey) FILTER (WHERE (lineitem_hash.l_shipmode = 'AIR'::bpchar))) DESC, (count(DISTINCT lineitem_hash.l_suppkey)) DESC, lineitem_hash.l_orderkey + -> GroupAggregate + Output: l_orderkey, count(DISTINCT l_suppkey) FILTER (WHERE (l_shipmode = 'AIR'::bpchar)), count(DISTINCT l_suppkey) + Group Key: lineitem_hash.l_orderkey + -> Index Scan using lineitem_hash_pkey_240000 on public.lineitem_hash_240000 lineitem_hash + Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment +(21 rows) + +-- group by on non-partition column +SELECT + l_suppkey, count(DISTINCT l_partkey) FILTER (WHERE l_shipmode = 'AIR') + FROM lineitem_hash + GROUP BY l_suppkey + ORDER BY 2 DESC, 1 DESC + LIMIT 10; + l_suppkey | count +--------------------------------------------------------------------- + 7680 | 4 + 7703 | 3 + 7542 | 3 + 7072 | 3 + 6335 | 3 + 5873 | 3 + 1318 | 3 + 1042 | 3 + 160 | 3 + 9872 | 2 +(10 rows) + +-- explaining the same query fails +EXPLAIN (COSTS false, VERBOSE true) +SELECT + l_suppkey, count(DISTINCT l_partkey) FILTER (WHERE l_shipmode = 'AIR') + FROM lineitem_hash + GROUP BY l_suppkey + ORDER BY 2 DESC, 1 DESC + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + Output: remote_scan.l_suppkey, (count(DISTINCT remote_scan.count) FILTER (WHERE (remote_scan.count_1 = 'AIR'::bpchar))) + -> Sort + Output: remote_scan.l_suppkey, (count(DISTINCT remote_scan.count) FILTER (WHERE (remote_scan.count_1 = 'AIR'::bpchar))) + Sort Key: (count(DISTINCT remote_scan.count) FILTER (WHERE (remote_scan.count_1 = 'AIR'::bpchar))) DESC, remote_scan.l_suppkey DESC + -> GroupAggregate + Output: remote_scan.l_suppkey, count(DISTINCT remote_scan.count) FILTER (WHERE (remote_scan.count_1 = 'AIR'::bpchar)) + Group Key: remote_scan.l_suppkey + -> Sort + Output: remote_scan.l_suppkey, remote_scan.count, remote_scan.count_1 + Sort Key: remote_scan.l_suppkey DESC + -> Custom Scan (Citus Task-Tracker) + Output: remote_scan.l_suppkey, remote_scan.count, remote_scan.count_1 + Task Count: 8 + Tasks Shown: One of 8 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: l_suppkey, l_partkey, l_shipmode + Group Key: lineitem_hash.l_suppkey, lineitem_hash.l_partkey, lineitem_hash.l_shipmode + -> Seq Scan on public.lineitem_hash_240000 lineitem_hash + Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment +(22 rows) + +-- without group by, on partition column +SELECT + count(DISTINCT l_orderkey) FILTER (WHERE l_shipmode = 'AIR') + FROM lineitem_hash; + count +--------------------------------------------------------------------- + 1327 +(1 row) + +-- without group by, on non-partition column +SELECT + count(DISTINCT l_partkey) FILTER (WHERE l_shipmode = 'AIR') + FROM lineitem_hash; + count +--------------------------------------------------------------------- + 1702 +(1 row) + +SELECT + count(DISTINCT l_partkey) FILTER (WHERE l_shipmode = 'AIR'), + count(DISTINCT l_partkey), + count(DISTINCT l_shipdate) + FROM lineitem_hash; + count | count | count +--------------------------------------------------------------------- + 1702 | 11661 | 2470 +(1 row) + +-- filter column already exists in target list +SELECT * + FROM ( + SELECT + l_orderkey, count(DISTINCT l_partkey) FILTER (WHERE l_orderkey > 100) + FROM lineitem_hash + GROUP BY l_orderkey) sub + ORDER BY 2 DESC, 1 DESC + LIMIT 10; + l_orderkey | count +--------------------------------------------------------------------- + 14885 | 7 + 14884 | 7 + 14821 | 7 + 14790 | 7 + 14785 | 7 + 14755 | 7 + 14725 | 7 + 14694 | 7 + 14627 | 7 + 14624 | 7 +(10 rows) + +-- filter column does not exist in target list +SELECT * + FROM ( + SELECT + l_orderkey, count(DISTINCT l_partkey) FILTER (WHERE l_shipmode = 'AIR') + FROM lineitem_hash + GROUP BY l_orderkey) sub + ORDER BY 2 DESC, 1 DESC + LIMIT 10; + l_orderkey | count +--------------------------------------------------------------------- + 12005 | 4 + 5409 | 4 + 4964 | 4 + 14848 | 3 + 14496 | 3 + 13473 | 3 + 13122 | 3 + 12929 | 3 + 12645 | 3 + 12417 | 3 +(10 rows) + +-- case expr in count distinct is supported. +-- count orders partkeys if l_shipmode is air +SELECT * + FROM ( + SELECT + l_orderkey, count(DISTINCT CASE WHEN l_shipmode = 'AIR' THEN l_partkey ELSE NULL END) as count + FROM lineitem_hash + GROUP BY l_orderkey) sub + WHERE count > 0 + ORDER BY 2 DESC, 1 DESC + LIMIT 10; + l_orderkey | count +--------------------------------------------------------------------- + 12005 | 4 + 5409 | 4 + 4964 | 4 + 14848 | 3 + 14496 | 3 + 13473 | 3 + 13122 | 3 + 12929 | 3 + 12645 | 3 + 12417 | 3 +(10 rows) + +-- text like operator is also supported +SELECT * + FROM ( + SELECT + l_orderkey, count(DISTINCT CASE WHEN l_shipmode like '%A%' THEN l_partkey ELSE NULL END) as count + FROM lineitem_hash + GROUP BY l_orderkey) sub + WHERE count > 0 + ORDER BY 2 DESC, 1 DESC + LIMIT 10; + l_orderkey | count +--------------------------------------------------------------------- + 14275 | 7 + 14181 | 7 + 13605 | 7 + 12707 | 7 + 12384 | 7 + 11746 | 7 + 10727 | 7 + 10467 | 7 + 5636 | 7 + 4614 | 7 +(10 rows) + +-- count distinct is rejected if it does not reference any columns +SELECT * + FROM ( + SELECT + l_linenumber, count(DISTINCT 1) + FROM lineitem_hash + GROUP BY l_linenumber) sub + ORDER BY 2 DESC, 1 DESC + LIMIT 10; +ERROR: cannot compute aggregate (distinct) +DETAIL: aggregate (distinct) with no columns is unsupported +HINT: You can load the hll extension from contrib packages and enable distinct approximations. +-- count distinct is rejected if it does not reference any columns +SELECT * + FROM ( + SELECT + l_linenumber, count(DISTINCT (random() * 5)::int) + FROM lineitem_hash + GROUP BY l_linenumber) sub + ORDER BY 2 DESC, 1 DESC + LIMIT 10; +ERROR: cannot compute aggregate (distinct) +DETAIL: aggregate (distinct) with no columns is unsupported +HINT: You can load the hll extension from contrib packages and enable distinct approximations. +-- even non-const function calls are supported within count distinct +SELECT * + FROM ( + SELECT + l_orderkey, count(DISTINCT (random() * 5)::int = l_linenumber) + FROM lineitem_hash + GROUP BY l_orderkey) sub + ORDER BY 2 DESC, 1 DESC + LIMIT 0; + l_orderkey | count +--------------------------------------------------------------------- +(0 rows) + +-- multiple nested subquery +SELECT + total, + avg(avg_count) as total_avg_count + FROM ( + SELECT + number_sum, + count(DISTINCT l_suppkey) as total, + avg(total_count) avg_count + FROM ( + SELECT + l_suppkey, + sum(l_linenumber) as number_sum, + count(DISTINCT l_shipmode) as total_count + FROM + lineitem_hash + WHERE + l_partkey > 100 and + l_quantity > 2 and + l_orderkey < 10000 + GROUP BY + l_suppkey) as distributed_table + WHERE + number_sum >= 10 + GROUP BY + number_sum) as distributed_table_2 + GROUP BY + total + ORDER BY + total_avg_count DESC; + total | total_avg_count +--------------------------------------------------------------------- + 1 | 3.6000000000000000 + 6 | 2.8333333333333333 + 10 | 2.6000000000000000 + 27 | 2.5555555555555556 + 32 | 2.4687500000000000 + 77 | 2.1948051948051948 + 57 | 2.1754385964912281 +(7 rows) + +-- multiple cases query +SELECT * + FROM ( + SELECT + count(DISTINCT + CASE + WHEN l_shipmode = 'TRUCK' THEN l_partkey + WHEN l_shipmode = 'AIR' THEN l_quantity + WHEN l_shipmode = 'SHIP' THEN l_discount + ELSE l_suppkey + END) as count, + l_shipdate + FROM + lineitem_hash + GROUP BY + l_shipdate) sub + WHERE + count > 0 + ORDER BY + 1 DESC, 2 DESC + LIMIT 10; + count | l_shipdate +--------------------------------------------------------------------- + 14 | 07-30-1997 + 13 | 05-26-1998 + 13 | 08-08-1997 + 13 | 11-17-1995 + 13 | 01-09-1993 + 12 | 01-15-1998 + 12 | 10-15-1997 + 12 | 09-07-1997 + 12 | 06-02-1997 + 12 | 03-14-1997 +(10 rows) + +-- count DISTINCT expression +SELECT * + FROM ( + SELECT + l_quantity, count(DISTINCT ((l_orderkey / 1000) * 1000 )) as count + FROM + lineitem_hash + GROUP BY + l_quantity) sub + WHERE + count > 0 + ORDER BY + 2 DESC, 1 DESC + LIMIT 10; + l_quantity | count +--------------------------------------------------------------------- + 48.00 | 13 + 47.00 | 13 + 37.00 | 13 + 33.00 | 13 + 26.00 | 13 + 25.00 | 13 + 23.00 | 13 + 21.00 | 13 + 15.00 | 13 + 12.00 | 13 +(10 rows) + +-- count DISTINCT is part of an expression which includes another aggregate +SELECT * + FROM ( + SELECT + sum(((l_partkey * l_tax) / 100)) / + count(DISTINCT + CASE + WHEN l_shipmode = 'TRUCK' THEN l_partkey + ELSE l_suppkey + END) as avg, + l_shipmode + FROM + lineitem_hash + GROUP BY + l_shipmode) sub + ORDER BY + 1 DESC, 2 DESC + LIMIT 10; + avg | l_shipmode +--------------------------------------------------------------------- + 44.82904609027336300064 | MAIL + 44.80704536679536679537 | SHIP + 44.68891732736572890026 | AIR + 44.34106724470134874759 | REG AIR + 43.12739987269255251432 | FOB + 43.07299253636938646426 | RAIL + 40.50298377916903813318 | TRUCK +(7 rows) + +-- count DISTINCT CASE WHEN expression +SELECT * + FROM ( + SELECT + count(DISTINCT + CASE + WHEN l_shipmode = 'TRUCK' THEN l_linenumber + WHEN l_shipmode = 'AIR' THEN l_linenumber + 10 + ELSE 2 + END) as avg + FROM + lineitem_hash + GROUP BY l_shipdate) sub + ORDER BY 1 DESC + LIMIT 10; + avg +--------------------------------------------------------------------- + 7 + 6 + 6 + 6 + 6 + 6 + 6 + 6 + 5 + 5 +(10 rows) + +-- COUNT DISTINCT (c1, c2) +SELECT * + FROM + (SELECT + l_shipmode, + count(DISTINCT (l_shipdate, l_tax)) + FROM + lineitem_hash + GROUP BY + l_shipmode) t + ORDER BY + 2 DESC,1 DESC + LIMIT 10; + l_shipmode | count +--------------------------------------------------------------------- + TRUCK | 1689 + MAIL | 1683 + FOB | 1655 + AIR | 1650 + SHIP | 1644 + RAIL | 1636 + REG AIR | 1607 +(7 rows) + +-- distinct on non-var (type cast/field select) columns are also +-- supported if grouped on distribution column +-- random is added to prevent flattening by postgresql +SELECT + l_orderkey, count(a::int), count(distinct a::int) + FROM ( + SELECT l_orderkey, l_orderkey * 1.5 a, random() b + FROM lineitem_hash) sub + GROUP BY 1 + ORDER BY 1 DESC + LIMIT 5; + l_orderkey | count | count +--------------------------------------------------------------------- + 14947 | 2 | 1 + 14946 | 2 | 1 + 14945 | 6 | 1 + 14944 | 2 | 1 + 14919 | 1 | 1 +(5 rows) + +SELECT user_id, + count(sub.a::int), + count(DISTINCT sub.a::int), + count(DISTINCT (sub).a) +FROM + (SELECT user_id, + unnest(ARRAY[user_id * 1.5])a, + random() b + FROM users_table + ) sub +GROUP BY 1 +ORDER BY 1 DESC +LIMIT 5; + user_id | count | count | count +--------------------------------------------------------------------- + 6 | 11 | 1 | 1 + 5 | 27 | 1 | 1 + 4 | 24 | 1 | 1 + 3 | 18 | 1 | 1 + 2 | 19 | 1 | 1 +(5 rows) + +CREATE TYPE test_item AS +( + id INTEGER, + duration INTEGER +); +CREATE TABLE test_count_distinct_array (key int, value int , value_arr test_item[]); +SELECT create_distributed_table('test_count_distinct_array', 'key'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO test_count_distinct_array SELECT i, i, ARRAY[(i,i)::test_item] FROM generate_Series(0, 1000) i; +SELECT + key, + count(DISTINCT value), + count(DISTINCT (item)."id"), + count(DISTINCT (item)."id" * 3) +FROM + ( + SELECT key, unnest(value_arr) as item, value FROM test_count_distinct_array + ) as sub +GROUP BY 1 +ORDER BY 1 DESC +LIMIT 5; + key | count | count | count +--------------------------------------------------------------------- + 1000 | 1 | 1 | 1 + 999 | 1 | 1 | 1 + 998 | 1 | 1 | 1 + 997 | 1 | 1 | 1 + 996 | 1 | 1 | 1 +(5 rows) + +DROP TABLE test_count_distinct_array; +DROP TYPE test_item; +-- other distinct aggregate are not supported +SELECT * + FROM ( + SELECT + l_linenumber, sum(DISTINCT l_partkey) + FROM lineitem_hash + GROUP BY l_linenumber) sub + ORDER BY 2 DESC, 1 DESC + LIMIT 10; +ERROR: cannot compute aggregate (distinct) +DETAIL: Only count(distinct) aggregate is supported in subqueries +SELECT * + FROM ( + SELECT + l_linenumber, avg(DISTINCT l_partkey) + FROM lineitem_hash + GROUP BY l_linenumber) sub + ORDER BY 2 DESC, 1 DESC + LIMIT 10; +ERROR: cannot compute aggregate (distinct) +DETAIL: Only count(distinct) aggregate is supported in subqueries +-- whole row references, oid, and ctid are not supported in count distinct +-- test table does not have oid or ctid enabled, so tests for them are skipped +SELECT * + FROM ( + SELECT + l_linenumber, count(DISTINCT lineitem_hash) + FROM lineitem_hash + GROUP BY l_linenumber) sub + ORDER BY 2 DESC, 1 DESC + LIMIT 10; +ERROR: cannot compute count (distinct) +DETAIL: Non-column references are not supported yet +SELECT * + FROM ( + SELECT + l_linenumber, count(DISTINCT lineitem_hash.*) + FROM lineitem_hash + GROUP BY l_linenumber) sub + ORDER BY 2 DESC, 1 DESC + LIMIT 10; +ERROR: cannot compute count (distinct) +DETAIL: Non-column references are not supported yet +-- count distinct pushdown is enabled +SELECT * + FROM ( + SELECT + l_shipdate, + count(DISTINCT + CASE + WHEN l_shipmode = 'TRUCK' THEN l_partkey + ELSE NULL + END) as distinct_part, + extract(year from l_shipdate) as year + FROM + lineitem_hash + GROUP BY l_shipdate, year) sub + WHERE year = 1995 + ORDER BY 2 DESC, 1 + LIMIT 10; + l_shipdate | distinct_part | year +--------------------------------------------------------------------- + 11-29-1995 | 5 | 1995 + 03-24-1995 | 4 | 1995 + 09-18-1995 | 4 | 1995 + 01-17-1995 | 3 | 1995 + 04-02-1995 | 3 | 1995 + 05-23-1995 | 3 | 1995 + 08-11-1995 | 3 | 1995 + 09-27-1995 | 3 | 1995 + 10-27-1995 | 3 | 1995 + 10-30-1995 | 3 | 1995 +(10 rows) + +RESET citus.task_executor_type; +-- count distinct pushdown is enabled +SELECT * + FROM ( + SELECT + l_shipdate, + count(DISTINCT + CASE + WHEN l_shipmode = 'TRUCK' THEN l_partkey + ELSE NULL + END) as distinct_part, + extract(year from l_shipdate) as year + FROM + lineitem_hash + GROUP BY l_shipdate, year) sub + WHERE year = 1995 + ORDER BY 2 DESC, 1 + LIMIT 10; + l_shipdate | distinct_part | year +--------------------------------------------------------------------- + 11-29-1995 | 5 | 1995 + 03-24-1995 | 4 | 1995 + 09-18-1995 | 4 | 1995 + 01-17-1995 | 3 | 1995 + 04-02-1995 | 3 | 1995 + 05-23-1995 | 3 | 1995 + 08-11-1995 | 3 | 1995 + 09-27-1995 | 3 | 1995 + 10-27-1995 | 3 | 1995 + 10-30-1995 | 3 | 1995 +(10 rows) + +SELECT * + FROM ( + SELECT + l_shipdate, + count(DISTINCT + CASE + WHEN l_shipmode = 'TRUCK' THEN l_partkey + ELSE NULL + END) as distinct_part, + extract(year from l_shipdate) as year + FROM + lineitem_hash + GROUP BY l_shipdate) sub + WHERE year = 1995 + ORDER BY 2 DESC, 1 + LIMIT 10; + l_shipdate | distinct_part | year +--------------------------------------------------------------------- + 11-29-1995 | 5 | 1995 + 03-24-1995 | 4 | 1995 + 09-18-1995 | 4 | 1995 + 01-17-1995 | 3 | 1995 + 04-02-1995 | 3 | 1995 + 05-23-1995 | 3 | 1995 + 08-11-1995 | 3 | 1995 + 09-27-1995 | 3 | 1995 + 10-27-1995 | 3 | 1995 + 10-30-1995 | 3 | 1995 +(10 rows) + +DROP TABLE lineitem_hash; diff --git a/src/test/regress/expected/multi_select_distinct.out b/src/test/regress/expected/multi_select_distinct.out index 506ce215f..bfb189095 100644 --- a/src/test/regress/expected/multi_select_distinct.out +++ b/src/test/regress/expected/multi_select_distinct.out @@ -946,7 +946,7 @@ EXPLAIN (COSTS FALSE) GROUP BY l_orderkey ORDER BY 2 LIMIT 15; - QUERY PLAN + QUERY PLAN --------------------------------------------------------------------- Limit -> Sort @@ -958,12 +958,10 @@ EXPLAIN (COSTS FALSE) Tasks Shown: One of 4 -> Task Node: host=localhost port=xxxxx dbname=regression - -> GroupAggregate + -> HashAggregate Group Key: l_orderkey - -> Sort - Sort Key: l_orderkey - -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part -(15 rows) + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(13 rows) -- check the plan if the hash aggreate is disabled. SET enable_hashagg TO off; @@ -973,7 +971,7 @@ EXPLAIN (COSTS FALSE) GROUP BY l_orderkey ORDER BY 2 LIMIT 15; - QUERY PLAN + QUERY PLAN --------------------------------------------------------------------- Limit -> Unique @@ -984,12 +982,10 @@ EXPLAIN (COSTS FALSE) Tasks Shown: One of 4 -> Task Node: host=localhost port=xxxxx dbname=regression - -> GroupAggregate + -> HashAggregate Group Key: l_orderkey - -> Sort - Sort Key: l_orderkey - -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part -(14 rows) + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(12 rows) SET enable_hashagg TO on; -- distinct on non-partition column with aggregate diff --git a/src/test/regress/expected/multi_select_distinct_1.out b/src/test/regress/expected/multi_select_distinct_1.out new file mode 100644 index 000000000..506ce215f --- /dev/null +++ b/src/test/regress/expected/multi_select_distinct_1.out @@ -0,0 +1,1569 @@ +-- +-- MULTI_SELECT_DISTINCT +-- +-- Tests select distinct, and select distinct on features. +-- +ANALYZE lineitem_hash_part; +-- function calls are supported +SELECT DISTINCT l_orderkey, now() FROM lineitem_hash_part LIMIT 0; + l_orderkey | now +--------------------------------------------------------------------- +(0 rows) + +SELECT DISTINCT l_orderkey, avg(l_linenumber) +FROM lineitem_hash_part +GROUP BY l_orderkey +HAVING avg(l_linenumber) = (select avg(distinct l_linenumber)) +LIMIT 10; +ERROR: Subqueries in HAVING cannot refer to outer query +SELECT DISTINCT l_orderkey +FROM lineitem_hash_part +GROUP BY l_orderkey +HAVING (select avg(distinct l_linenumber) = l_orderkey) +LIMIT 10; +ERROR: Subqueries in HAVING cannot refer to outer query +SELECT DISTINCT l_partkey, 1 + (random() * 0)::int FROM lineitem_hash_part ORDER BY 1 DESC LIMIT 3; + l_partkey | ?column? +--------------------------------------------------------------------- + 199973 | 1 + 199946 | 1 + 199943 | 1 +(3 rows) + +-- const expressions are supported +SELECT DISTINCT l_orderkey, 1+1 FROM lineitem_hash_part ORDER BY 1 LIMIT 5; + l_orderkey | ?column? +--------------------------------------------------------------------- + 1 | 2 + 2 | 2 + 3 | 2 + 4 | 2 + 5 | 2 +(5 rows) + +-- non const expressions are also supported +SELECT DISTINCT l_orderkey, l_partkey + 1 FROM lineitem_hash_part ORDER BY 1, 2 LIMIT 5; + l_orderkey | ?column? +--------------------------------------------------------------------- + 1 | 2133 + 1 | 15636 + 1 | 24028 + 1 | 63701 + 1 | 67311 +(5 rows) + +-- column expressions are supported +SELECT DISTINCT l_orderkey, l_shipinstruct || l_shipmode FROM lineitem_hash_part ORDER BY 2 , 1 LIMIT 5; + l_orderkey | ?column? +--------------------------------------------------------------------- + 32 | COLLECT CODAIR + 39 | COLLECT CODAIR + 66 | COLLECT CODAIR + 70 | COLLECT CODAIR + 98 | COLLECT CODAIR +(5 rows) + +-- function calls with const input are supported +SELECT DISTINCT l_orderkey, strpos('AIR', 'A') FROM lineitem_hash_part ORDER BY 1,2 LIMIT 5; + l_orderkey | strpos +--------------------------------------------------------------------- + 1 | 1 + 2 | 1 + 3 | 1 + 4 | 1 + 5 | 1 +(5 rows) + +-- function calls with non-const input are supported +SELECT DISTINCT l_orderkey, strpos(l_shipmode, 'I') + FROM lineitem_hash_part + WHERE strpos(l_shipmode, 'I') > 1 + ORDER BY 2, 1 + LIMIT 5; + l_orderkey | strpos +--------------------------------------------------------------------- + 1 | 2 + 3 | 2 + 5 | 2 + 32 | 2 + 33 | 2 +(5 rows) + +-- row types are supported +SELECT DISTINCT (l_orderkey, l_partkey) AS pair FROM lineitem_hash_part ORDER BY 1 LIMIT 5; + pair +--------------------------------------------------------------------- + (1,2132) + (1,15635) + (1,24027) + (1,63700) + (1,67310) +(5 rows) + +-- distinct on partition column +-- verify counts match with respect to count(distinct) +CREATE TEMP TABLE temp_orderkeys AS SELECT DISTINCT l_orderkey FROM lineitem_hash_part; +SELECT COUNT(*) FROM temp_orderkeys; + count +--------------------------------------------------------------------- + 2985 +(1 row) + +SELECT COUNT(DISTINCT l_orderkey) FROM lineitem_hash_part; + count +--------------------------------------------------------------------- + 2985 +(1 row) + +SELECT DISTINCT l_orderkey FROM lineitem_hash_part WHERE l_orderkey < 500 and l_partkey < 5000 order by 1; + l_orderkey +--------------------------------------------------------------------- + 1 + 3 + 32 + 35 + 39 + 65 + 129 + 130 + 134 + 164 + 194 + 228 + 261 + 290 + 320 + 321 + 354 + 418 +(18 rows) + +-- distinct on non-partition column +SELECT DISTINCT l_partkey FROM lineitem_hash_part WHERE l_orderkey > 5 and l_orderkey < 20 order by 1; + l_partkey +--------------------------------------------------------------------- + 79251 + 94780 + 139636 + 145243 + 151894 + 157238 + 163073 + 182052 +(8 rows) + +SELECT DISTINCT l_shipmode FROM lineitem_hash_part ORDER BY 1 DESC; + l_shipmode +--------------------------------------------------------------------- + TRUCK + SHIP + REG AIR + RAIL + MAIL + FOB + AIR +(7 rows) + +-- distinct with multiple columns +SELECT DISTINCT l_orderkey, o_orderdate + FROM lineitem_hash_part JOIN orders_hash_part ON (l_orderkey = o_orderkey) + WHERE l_orderkey < 10 + ORDER BY l_orderkey; + l_orderkey | o_orderdate +--------------------------------------------------------------------- + 1 | 01-02-1996 + 2 | 12-01-1996 + 3 | 10-14-1993 + 4 | 10-11-1995 + 5 | 07-30-1994 + 6 | 02-21-1992 + 7 | 01-10-1996 +(7 rows) + +-- distinct on partition column with aggregate +-- this is the same as the one without distinct due to group by +SELECT DISTINCT l_orderkey, count(*) + FROM lineitem_hash_part + WHERE l_orderkey < 200 + GROUP BY 1 + HAVING count(*) > 5 + ORDER BY 2 DESC, 1; + l_orderkey | count +--------------------------------------------------------------------- + 7 | 7 + 68 | 7 + 129 | 7 + 164 | 7 + 194 | 7 + 1 | 6 + 3 | 6 + 32 | 6 + 35 | 6 + 39 | 6 + 67 | 6 + 69 | 6 + 70 | 6 + 71 | 6 + 134 | 6 + 135 | 6 + 163 | 6 + 192 | 6 + 197 | 6 +(19 rows) + +-- explain the query to see actual plan +EXPLAIN (COSTS FALSE) + SELECT DISTINCT l_orderkey, count(*) + FROM lineitem_hash_part + WHERE l_orderkey < 200 + GROUP BY 1 + HAVING count(*) > 5 + ORDER BY 2 DESC, 1; + QUERY PLAN +--------------------------------------------------------------------- + Sort + Sort Key: remote_scan.count DESC, remote_scan.l_orderkey + -> HashAggregate + Group Key: remote_scan.count, remote_scan.l_orderkey + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Group Key: l_orderkey + Filter: (count(*) > 5) + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part + Filter: (l_orderkey < 200) +(14 rows) + +-- check the plan if the hash aggreate is disabled +SET enable_hashagg TO off; +EXPLAIN (COSTS FALSE) + SELECT DISTINCT l_orderkey, count(*) + FROM lineitem_hash_part + WHERE l_orderkey < 200 + GROUP BY 1 + HAVING count(*) > 5 + ORDER BY 2 DESC, 1; + QUERY PLAN +--------------------------------------------------------------------- + Unique + -> Sort + Sort Key: remote_scan.count DESC, remote_scan.l_orderkey + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Group Key: l_orderkey + Filter: (count(*) > 5) + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part + Filter: (l_orderkey < 200) +(13 rows) + +SET enable_hashagg TO on; +-- distinct on aggregate of group by columns, we try to check whether we handle +-- queries which does not have any group by column in distinct columns properly. +SELECT DISTINCT count(*) + FROM lineitem_hash_part + GROUP BY l_suppkey, l_linenumber + ORDER BY 1; + count +--------------------------------------------------------------------- + 1 + 2 + 3 + 4 +(4 rows) + +-- explain the query to see actual plan. We expect to see Aggregate node having +-- group by key on count(*) column, since columns in the Group By doesn't guarantee +-- the uniqueness of the result. +EXPLAIN (COSTS FALSE) + SELECT DISTINCT count(*) + FROM lineitem_hash_part + GROUP BY l_suppkey, l_linenumber + ORDER BY 1; + QUERY PLAN +--------------------------------------------------------------------- + Unique + -> Sort + Sort Key: (COALESCE((pg_catalog.sum(remote_scan.count))::bigint, '0'::bigint)) + -> HashAggregate + Group Key: remote_scan.worker_column_2, remote_scan.worker_column_3 + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Group Key: l_suppkey, l_linenumber + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(13 rows) + +-- check the plan if the hash aggreate is disabled. We expect to see sort+unique +-- instead of aggregate plan node to handle distinct. +SET enable_hashagg TO off; +EXPLAIN (COSTS FALSE) + SELECT DISTINCT count(*) + FROM lineitem_hash_part + GROUP BY l_suppkey, l_linenumber + ORDER BY 1; + QUERY PLAN +--------------------------------------------------------------------- + Unique + -> Sort + Sort Key: (COALESCE((pg_catalog.sum(remote_scan.count))::bigint, '0'::bigint)) + -> GroupAggregate + Group Key: remote_scan.worker_column_2, remote_scan.worker_column_3 + -> Sort + Sort Key: remote_scan.worker_column_2, remote_scan.worker_column_3 + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Group Key: l_suppkey, l_linenumber + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(15 rows) + +SET enable_hashagg TO on; +-- Now we have only part of group clause columns in distinct, yet it is still not +-- enough to use Group By columns to guarantee uniqueness of result list. +SELECT DISTINCT l_suppkey, count(*) + FROM lineitem_hash_part + GROUP BY l_suppkey, l_linenumber + ORDER BY 1 + LIMIT 10; + l_suppkey | count +--------------------------------------------------------------------- + 1 | 1 + 2 | 1 + 3 | 1 + 4 | 1 + 5 | 1 + 7 | 1 + 10 | 1 + 12 | 1 + 13 | 1 + 14 | 1 +(10 rows) + +-- explain the query to see actual plan. Similar to the explain of the query above. +EXPLAIN (COSTS FALSE) + SELECT DISTINCT l_suppkey, count(*) + FROM lineitem_hash_part + GROUP BY l_suppkey, l_linenumber + ORDER BY 1 + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Unique + -> Sort + Sort Key: remote_scan.l_suppkey, (COALESCE((pg_catalog.sum(remote_scan.count))::bigint, '0'::bigint)) + -> HashAggregate + Group Key: remote_scan.l_suppkey, remote_scan.worker_column_3 + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Group Key: l_suppkey, l_linenumber + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(14 rows) + +-- check the plan if the hash aggreate is disabled. Similar to the explain of +-- the query above. +SET enable_hashagg TO off; +EXPLAIN (COSTS FALSE) + SELECT DISTINCT l_suppkey, count(*) + FROM lineitem_hash_part + GROUP BY l_suppkey, l_linenumber + ORDER BY 1 + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Unique + -> Sort + Sort Key: remote_scan.l_suppkey, (COALESCE((pg_catalog.sum(remote_scan.count))::bigint, '0'::bigint)) + -> GroupAggregate + Group Key: remote_scan.l_suppkey, remote_scan.worker_column_3 + -> Sort + Sort Key: remote_scan.l_suppkey, remote_scan.worker_column_3 + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Group Key: l_suppkey, l_linenumber + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(16 rows) + +SET enable_hashagg TO on; +-- Similar to the above query, not with count but avg. Only difference with the +-- above query is that, we create run two aggregate functions in workers. +SELECT DISTINCT l_suppkey, avg(l_partkey) + FROM lineitem_hash_part + GROUP BY l_suppkey, l_linenumber + ORDER BY 1,2 + LIMIT 10; + l_suppkey | avg +--------------------------------------------------------------------- + 1 | 190000.000000000000 + 2 | 172450.000000000000 + 3 | 112469.000000000000 + 3 | 134976.000000000000 + 4 | 112470.000000000000 + 4 | 142461.000000000000 + 5 | 182450.000000000000 + 7 | 137493.000000000000 + 10 | 150009.000000000000 + 12 | 17510.0000000000000000 +(10 rows) + +-- explain the query to see actual plan. Similar to the explain of the query above. +-- Only aggregate functions will be changed. +EXPLAIN (COSTS FALSE) + SELECT DISTINCT l_suppkey, avg(l_partkey) + FROM lineitem_hash_part + GROUP BY l_suppkey, l_linenumber + ORDER BY 1,2 + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Unique + -> Sort + Sort Key: remote_scan.l_suppkey, ((pg_catalog.sum(remote_scan.avg) / pg_catalog.sum(remote_scan.avg_1))) + -> HashAggregate + Group Key: remote_scan.l_suppkey, remote_scan.worker_column_4 + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Group Key: l_suppkey, l_linenumber + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(14 rows) + +-- check the plan if the hash aggreate is disabled. This explain errors out due +-- to a bug right now, expectation must be corrected after fixing it. +SET enable_hashagg TO off; +EXPLAIN (COSTS FALSE) + SELECT DISTINCT l_suppkey, avg(l_partkey) + FROM lineitem_hash_part + GROUP BY l_suppkey, l_linenumber + ORDER BY 1,2 + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Unique + -> Sort + Sort Key: remote_scan.l_suppkey, ((pg_catalog.sum(remote_scan.avg) / pg_catalog.sum(remote_scan.avg_1))) + -> GroupAggregate + Group Key: remote_scan.l_suppkey, remote_scan.worker_column_4 + -> Sort + Sort Key: remote_scan.l_suppkey, remote_scan.worker_column_4 + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Group Key: l_suppkey, l_linenumber + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(16 rows) + +SET enable_hashagg TO on; +-- Similar to the above query but with distinct on +SELECT DISTINCT ON (l_suppkey) avg(l_partkey) + FROM lineitem_hash_part + GROUP BY l_suppkey, l_linenumber + ORDER BY l_suppkey,1 + LIMIT 10; + avg +--------------------------------------------------------------------- + 190000.000000000000 + 172450.000000000000 + 112469.000000000000 + 112470.000000000000 + 182450.000000000000 + 137493.000000000000 + 150009.000000000000 + 17510.0000000000000000 + 87504.000000000000 + 77506.000000000000 +(10 rows) + +-- explain the query to see actual plan. We expect to see sort+unique to handle +-- distinct on. +EXPLAIN (COSTS FALSE) + SELECT DISTINCT ON (l_suppkey) avg(l_partkey) + FROM lineitem_hash_part + GROUP BY l_suppkey, l_linenumber + ORDER BY l_suppkey,1 + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Unique + -> Sort + Sort Key: remote_scan.worker_column_3, ((pg_catalog.sum(remote_scan.avg) / pg_catalog.sum(remote_scan.avg_1))) + -> HashAggregate + Group Key: remote_scan.worker_column_3, remote_scan.worker_column_4 + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Group Key: l_suppkey, l_linenumber + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(14 rows) + +-- check the plan if the hash aggreate is disabled. We expect to see sort+unique to +-- handle distinct on. +SET enable_hashagg TO off; +EXPLAIN (COSTS FALSE) + SELECT DISTINCT ON (l_suppkey) avg(l_partkey) + FROM lineitem_hash_part + GROUP BY l_suppkey, l_linenumber + ORDER BY l_suppkey,1 + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Unique + -> Sort + Sort Key: remote_scan.worker_column_3, ((pg_catalog.sum(remote_scan.avg) / pg_catalog.sum(remote_scan.avg_1))) + -> GroupAggregate + Group Key: remote_scan.worker_column_3, remote_scan.worker_column_4 + -> Sort + Sort Key: remote_scan.worker_column_3, remote_scan.worker_column_4 + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Group Key: l_suppkey, l_linenumber + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(16 rows) + +SET enable_hashagg TO on; +-- distinct with expression and aggregation +SELECT DISTINCT avg(ceil(l_partkey / 2)) + FROM lineitem_hash_part + GROUP BY l_suppkey, l_linenumber + ORDER BY 1 + LIMIT 10; + avg +--------------------------------------------------------------------- + 9 + 39 + 74 + 87 + 89 + 91 + 97 + 102 + 111 + 122 +(10 rows) + +-- explain the query to see actual plan +EXPLAIN (COSTS FALSE) + SELECT DISTINCT avg(ceil(l_partkey / 2)) + FROM lineitem_hash_part + GROUP BY l_suppkey, l_linenumber + ORDER BY 1 + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Unique + -> Sort + Sort Key: ((sum(remote_scan.avg) / (pg_catalog.sum(remote_scan.avg_1))::double precision)) + -> HashAggregate + Group Key: remote_scan.worker_column_3, remote_scan.worker_column_4 + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Group Key: l_suppkey, l_linenumber + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(14 rows) + +-- check the plan if the hash aggreate is disabled. This explain errors out due +-- to a bug right now, expectation must be corrected after fixing it. +SET enable_hashagg TO off; +EXPLAIN (COSTS FALSE) + SELECT DISTINCT avg(ceil(l_partkey / 2)) + FROM lineitem_hash_part + GROUP BY l_suppkey, l_linenumber + ORDER BY 1 + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Unique + -> Sort + Sort Key: ((sum(remote_scan.avg) / (pg_catalog.sum(remote_scan.avg_1))::double precision)) + -> GroupAggregate + Group Key: remote_scan.worker_column_3, remote_scan.worker_column_4 + -> Sort + Sort Key: remote_scan.worker_column_3, remote_scan.worker_column_4 + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Group Key: l_suppkey, l_linenumber + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(16 rows) + +SET enable_hashagg TO on; +-- expression among aggregations. +SELECT DISTINCT sum(l_suppkey) + count(l_partkey) AS dis + FROM lineitem_hash_part + GROUP BY l_suppkey, l_linenumber + ORDER BY 1 + LIMIT 10; + dis +--------------------------------------------------------------------- + 2 + 3 + 4 + 5 + 6 + 8 + 11 + 13 + 14 + 15 +(10 rows) + +-- explain the query to see actual plan +EXPLAIN (COSTS FALSE) + SELECT DISTINCT sum(l_suppkey) + count(l_partkey) AS dis + FROM lineitem_hash_part + GROUP BY l_suppkey, l_linenumber + ORDER BY 1 + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Unique + -> Sort + Sort Key: (((pg_catalog.sum(remote_scan.dis))::bigint + COALESCE((pg_catalog.sum(remote_scan.dis_1))::bigint, '0'::bigint))) + -> HashAggregate + Group Key: remote_scan.worker_column_3, remote_scan.worker_column_4 + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Group Key: l_suppkey, l_linenumber + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(14 rows) + +-- check the plan if the hash aggreate is disabled. This explain errors out due +-- to a bug right now, expectation must be corrected after fixing it. +SET enable_hashagg TO off; +EXPLAIN (COSTS FALSE) + SELECT DISTINCT sum(l_suppkey) + count(l_partkey) AS dis + FROM lineitem_hash_part + GROUP BY l_suppkey, l_linenumber + ORDER BY 1 + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Unique + -> Sort + Sort Key: (((pg_catalog.sum(remote_scan.dis))::bigint + COALESCE((pg_catalog.sum(remote_scan.dis_1))::bigint, '0'::bigint))) + -> GroupAggregate + Group Key: remote_scan.worker_column_3, remote_scan.worker_column_4 + -> Sort + Sort Key: remote_scan.worker_column_3, remote_scan.worker_column_4 + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Group Key: l_suppkey, l_linenumber + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(16 rows) + +SET enable_hashagg TO on; +-- distinct on all columns, note Group By columns guarantees uniqueness of the +-- result list. +SELECT DISTINCT * + FROM lineitem_hash_part + GROUP BY 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16 + ORDER BY 1,2 + LIMIT 10; + l_orderkey | l_partkey | l_suppkey | l_linenumber | l_quantity | l_extendedprice | l_discount | l_tax | l_returnflag | l_linestatus | l_shipdate | l_commitdate | l_receiptdate | l_shipinstruct | l_shipmode | l_comment +--------------------------------------------------------------------- + 1 | 2132 | 4633 | 4 | 28.00 | 28955.64 | 0.09 | 0.06 | N | O | 04-21-1996 | 03-30-1996 | 05-16-1996 | NONE | AIR | lites. fluffily even de + 1 | 15635 | 638 | 6 | 32.00 | 49620.16 | 0.07 | 0.02 | N | O | 01-30-1996 | 02-07-1996 | 02-03-1996 | DELIVER IN PERSON | MAIL | arefully slyly ex + 1 | 24027 | 1534 | 5 | 24.00 | 22824.48 | 0.10 | 0.04 | N | O | 03-30-1996 | 03-14-1996 | 04-01-1996 | NONE | FOB | pending foxes. slyly re + 1 | 63700 | 3701 | 3 | 8.00 | 13309.60 | 0.10 | 0.02 | N | O | 01-29-1996 | 03-05-1996 | 01-31-1996 | TAKE BACK RETURN | REG AIR | riously. regular, express dep + 1 | 67310 | 7311 | 2 | 36.00 | 45983.16 | 0.09 | 0.06 | N | O | 04-12-1996 | 02-28-1996 | 04-20-1996 | TAKE BACK RETURN | MAIL | ly final dependencies: slyly bold + 1 | 155190 | 7706 | 1 | 17.00 | 21168.23 | 0.04 | 0.02 | N | O | 03-13-1996 | 02-12-1996 | 03-22-1996 | DELIVER IN PERSON | TRUCK | egular courts above the + 2 | 106170 | 1191 | 1 | 38.00 | 44694.46 | 0.00 | 0.05 | N | O | 01-28-1997 | 01-14-1997 | 02-02-1997 | TAKE BACK RETURN | RAIL | ven requests. deposits breach a + 3 | 4297 | 1798 | 1 | 45.00 | 54058.05 | 0.06 | 0.00 | R | F | 02-02-1994 | 01-04-1994 | 02-23-1994 | NONE | AIR | ongside of the furiously brave acco + 3 | 19036 | 6540 | 2 | 49.00 | 46796.47 | 0.10 | 0.00 | R | F | 11-09-1993 | 12-20-1993 | 11-24-1993 | TAKE BACK RETURN | RAIL | unusual accounts. eve + 3 | 29380 | 1883 | 4 | 2.00 | 2618.76 | 0.01 | 0.06 | A | F | 12-04-1993 | 01-07-1994 | 01-01-1994 | NONE | TRUCK | y. fluffily pending d +(10 rows) + +-- explain the query to see actual plan. We expect to see only one aggregation +-- node since group by columns guarantees the uniqueness. +EXPLAIN (COSTS FALSE) + SELECT DISTINCT * + FROM lineitem_hash_part + GROUP BY 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16 + ORDER BY 1,2 + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Sort + Sort Key: remote_scan.l_orderkey, remote_scan.l_partkey + -> HashAggregate + Group Key: remote_scan.l_orderkey, remote_scan.l_partkey, remote_scan.l_suppkey, remote_scan.l_linenumber, remote_scan.l_quantity, remote_scan.l_extendedprice, remote_scan.l_discount, remote_scan.l_tax, remote_scan.l_returnflag, remote_scan.l_linestatus, remote_scan.l_shipdate, remote_scan.l_commitdate, remote_scan.l_receiptdate, remote_scan.l_shipinstruct, remote_scan.l_shipmode, remote_scan.l_comment + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Limit + -> Unique + -> Group + Group Key: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment + -> Sort + Sort Key: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(17 rows) + +-- check the plan if the hash aggreate is disabled. We expect to see only one +-- aggregation node since group by columns guarantees the uniqueness. +SET enable_hashagg TO off; +EXPLAIN (COSTS FALSE) + SELECT DISTINCT * + FROM lineitem_hash_part + GROUP BY 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16 + ORDER BY 1,2 + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Unique + -> Sort + Sort Key: remote_scan.l_orderkey, remote_scan.l_partkey, remote_scan.l_suppkey, remote_scan.l_linenumber, remote_scan.l_quantity, remote_scan.l_extendedprice, remote_scan.l_discount, remote_scan.l_tax, remote_scan.l_returnflag, remote_scan.l_linestatus, remote_scan.l_shipdate, remote_scan.l_commitdate, remote_scan.l_receiptdate, remote_scan.l_shipinstruct, remote_scan.l_shipmode, remote_scan.l_comment + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Limit + -> Unique + -> Group + Group Key: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment + -> Sort + Sort Key: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(16 rows) + +SET enable_hashagg TO on; +-- distinct on count distinct +SELECT DISTINCT count(DISTINCT l_partkey), count(DISTINCT l_shipmode) + FROM lineitem_hash_part + GROUP BY l_orderkey + ORDER BY 1,2; + count | count +--------------------------------------------------------------------- + 1 | 1 + 2 | 1 + 2 | 2 + 3 | 1 + 3 | 2 + 3 | 3 + 4 | 1 + 4 | 2 + 4 | 3 + 4 | 4 + 5 | 2 + 5 | 3 + 5 | 4 + 5 | 5 + 6 | 2 + 6 | 3 + 6 | 4 + 6 | 5 + 6 | 6 + 7 | 2 + 7 | 3 + 7 | 4 + 7 | 5 + 7 | 6 + 7 | 7 +(25 rows) + +-- explain the query to see actual plan. We expect to see aggregation plan for +-- the outer distinct. +EXPLAIN (COSTS FALSE) + SELECT DISTINCT count(DISTINCT l_partkey), count(DISTINCT l_shipmode) + FROM lineitem_hash_part + GROUP BY l_orderkey + ORDER BY 1,2; + QUERY PLAN +--------------------------------------------------------------------- + Sort + Sort Key: remote_scan.count, remote_scan.count_1 + -> HashAggregate + Group Key: remote_scan.count, remote_scan.count_1 + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> GroupAggregate + Group Key: l_orderkey + -> Sort + Sort Key: l_orderkey + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(14 rows) + +-- check the plan if the hash aggreate is disabled. We expect to see sort + unique +-- plans for the outer distinct. +SET enable_hashagg TO off; +EXPLAIN (COSTS FALSE) + SELECT DISTINCT count(DISTINCT l_partkey), count(DISTINCT l_shipmode) + FROM lineitem_hash_part + GROUP BY l_orderkey + ORDER BY 1,2; + QUERY PLAN +--------------------------------------------------------------------- + Unique + -> Sort + Sort Key: remote_scan.count, remote_scan.count_1 + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> GroupAggregate + Group Key: l_orderkey + -> Sort + Sort Key: l_orderkey + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(13 rows) + +SET enable_hashagg TO on; +-- distinct on aggregation with filter and expression +SELECT DISTINCT ceil(count(case when l_partkey > 100000 THEN 1 ELSE 0 END) / 2) AS count + FROM lineitem_hash_part + GROUP BY l_suppkey + ORDER BY 1; + count +--------------------------------------------------------------------- + 0 + 1 + 2 + 3 + 4 +(5 rows) + +-- explain the query to see actual plan +EXPLAIN (COSTS FALSE) + SELECT DISTINCT ceil(count(case when l_partkey > 100000 THEN 1 ELSE 0 END) / 2) AS count + FROM lineitem_hash_part + GROUP BY l_suppkey + ORDER BY 1; + QUERY PLAN +--------------------------------------------------------------------- + Unique + -> Sort + Sort Key: (ceil(((COALESCE((pg_catalog.sum(remote_scan.count))::bigint, '0'::bigint) / 2))::double precision)) + -> HashAggregate + Group Key: remote_scan.worker_column_2 + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Group Key: l_suppkey + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(13 rows) + +-- check the plan if the hash aggreate is disabled +SET enable_hashagg TO off; +EXPLAIN (COSTS FALSE) + SELECT DISTINCT ceil(count(case when l_partkey > 100000 THEN 1 ELSE 0 END) / 2) AS count + FROM lineitem_hash_part + GROUP BY l_suppkey + ORDER BY 1; + QUERY PLAN +--------------------------------------------------------------------- + Unique + -> Sort + Sort Key: (ceil(((COALESCE((pg_catalog.sum(remote_scan.count))::bigint, '0'::bigint) / 2))::double precision)) + -> GroupAggregate + Group Key: remote_scan.worker_column_2 + -> Sort + Sort Key: remote_scan.worker_column_2 + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Group Key: l_suppkey + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(15 rows) + +SET enable_hashagg TO on; +-- explain the query to see actual plan with array_agg aggregation. +EXPLAIN (COSTS FALSE) + SELECT DISTINCT array_agg(l_linenumber), array_length(array_agg(l_linenumber), 1) + FROM lineitem_hash_part + GROUP BY l_orderkey + ORDER BY 2 + LIMIT 15; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Sort + Sort Key: remote_scan.array_length + -> HashAggregate + Group Key: remote_scan.array_length, remote_scan.array_agg + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> GroupAggregate + Group Key: l_orderkey + -> Sort + Sort Key: l_orderkey + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(15 rows) + +-- check the plan if the hash aggreate is disabled. +SET enable_hashagg TO off; +EXPLAIN (COSTS FALSE) + SELECT DISTINCT array_agg(l_linenumber), array_length(array_agg(l_linenumber), 1) + FROM lineitem_hash_part + GROUP BY l_orderkey + ORDER BY 2 + LIMIT 15; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Unique + -> Sort + Sort Key: remote_scan.array_length, remote_scan.array_agg + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> GroupAggregate + Group Key: l_orderkey + -> Sort + Sort Key: l_orderkey + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(14 rows) + +SET enable_hashagg TO on; +-- distinct on non-partition column with aggregate +-- this is the same as non-distinct version due to group by +SELECT DISTINCT l_partkey, count(*) + FROM lineitem_hash_part + GROUP BY 1 + HAVING count(*) > 2 + ORDER BY 1; + l_partkey | count +--------------------------------------------------------------------- + 1051 | 3 + 1927 | 3 + 6983 | 3 + 15283 | 3 + 87761 | 3 + 136884 | 3 + 149926 | 3 + 160895 | 3 + 177771 | 3 + 188804 | 3 + 199146 | 3 +(11 rows) + +-- explain the query to see actual plan +EXPLAIN (COSTS FALSE) + SELECT DISTINCT l_partkey, count(*) + FROM lineitem_hash_part + GROUP BY 1 + HAVING count(*) > 2 + ORDER BY 1; + QUERY PLAN +--------------------------------------------------------------------- + Unique + -> Sort + Sort Key: remote_scan.l_partkey, (COALESCE((pg_catalog.sum(remote_scan.count))::bigint, '0'::bigint)) + -> HashAggregate + Group Key: remote_scan.l_partkey + Filter: (COALESCE((pg_catalog.sum(remote_scan.worker_column_3))::bigint, '0'::bigint) > 2) + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Group Key: l_partkey + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(14 rows) + +-- distinct on non-partition column and avg +SELECT DISTINCT l_partkey, avg(l_linenumber) + FROM lineitem_hash_part + WHERE l_partkey < 500 + GROUP BY 1 + HAVING avg(l_linenumber) > 2 + ORDER BY 1; + l_partkey | avg +--------------------------------------------------------------------- + 18 | 7.0000000000000000 + 79 | 6.0000000000000000 + 149 | 4.5000000000000000 + 175 | 5.0000000000000000 + 179 | 6.0000000000000000 + 182 | 3.0000000000000000 + 222 | 4.0000000000000000 + 278 | 3.0000000000000000 + 299 | 7.0000000000000000 + 308 | 7.0000000000000000 + 309 | 5.0000000000000000 + 321 | 3.0000000000000000 + 337 | 6.0000000000000000 + 364 | 3.0000000000000000 + 403 | 4.0000000000000000 +(15 rows) + +-- distinct on multiple non-partition columns +SELECT DISTINCT l_partkey, l_suppkey + FROM lineitem_hash_part + WHERE l_shipmode = 'AIR' AND l_orderkey < 100 + ORDER BY 1, 2; + l_partkey | l_suppkey +--------------------------------------------------------------------- + 2132 | 4633 + 4297 | 1798 + 37531 | 35 + 44161 | 6666 + 44706 | 4707 + 67831 | 5350 + 85811 | 8320 + 94368 | 6878 + 108338 | 849 + 108570 | 8571 + 137267 | 4807 + 137469 | 9983 + 173489 | 3490 + 196156 | 1195 + 197921 | 441 +(15 rows) + +EXPLAIN (COSTS FALSE) + SELECT DISTINCT l_partkey, l_suppkey + FROM lineitem_hash_part + WHERE l_shipmode = 'AIR' AND l_orderkey < 100 + ORDER BY 1, 2; + QUERY PLAN +--------------------------------------------------------------------- + Sort + Sort Key: remote_scan.l_partkey, remote_scan.l_suppkey + -> HashAggregate + Group Key: remote_scan.l_partkey, remote_scan.l_suppkey + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Unique + -> Sort + Sort Key: l_partkey, l_suppkey + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part + Filter: ((l_orderkey < 100) AND (l_shipmode = 'AIR'::bpchar)) +(14 rows) + +-- distinct on partition column +SELECT DISTINCT ON (l_orderkey) l_orderkey, l_partkey, l_suppkey + FROM lineitem_hash_part + WHERE l_orderkey < 35 + ORDER BY 1, 2, 3; + l_orderkey | l_partkey | l_suppkey +--------------------------------------------------------------------- + 1 | 2132 | 4633 + 2 | 106170 | 1191 + 3 | 4297 | 1798 + 4 | 88035 | 5560 + 5 | 37531 | 35 + 6 | 139636 | 2150 + 7 | 79251 | 1759 + 32 | 2743 | 7744 + 33 | 33918 | 3919 + 34 | 88362 | 871 +(10 rows) + +EXPLAIN (COSTS FALSE) + SELECT DISTINCT ON (l_orderkey) l_orderkey, l_partkey, l_suppkey + FROM lineitem_hash_part + WHERE l_orderkey < 35 + ORDER BY 1, 2, 3; + QUERY PLAN +--------------------------------------------------------------------- + Unique + -> Sort + Sort Key: remote_scan.l_orderkey, remote_scan.l_partkey, remote_scan.l_suppkey + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Unique + -> Sort + Sort Key: l_orderkey, l_partkey, l_suppkey + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part + Filter: (l_orderkey < 35) +(13 rows) + +-- distinct on non-partition column +-- note order by is required here +-- otherwise query results will be different since +-- distinct on clause is on non-partition column +SELECT DISTINCT ON (l_partkey) l_partkey, l_orderkey + FROM lineitem_hash_part + ORDER BY 1,2 + LIMIT 20; + l_partkey | l_orderkey +--------------------------------------------------------------------- + 18 | 12005 + 79 | 5121 + 91 | 2883 + 149 | 807 + 175 | 4102 + 179 | 2117 + 182 | 548 + 195 | 2528 + 204 | 10048 + 222 | 9413 + 245 | 9446 + 278 | 1287 + 299 | 1122 + 308 | 11137 + 309 | 2374 + 318 | 321 + 321 | 5984 + 337 | 10403 + 350 | 13698 + 358 | 4323 +(20 rows) + +EXPLAIN (COSTS FALSE) + SELECT DISTINCT ON (l_partkey) l_partkey, l_orderkey + FROM lineitem_hash_part + ORDER BY 1,2 + LIMIT 20; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Unique + -> Sort + Sort Key: remote_scan.l_partkey, remote_scan.l_orderkey + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Limit + -> Unique + -> Sort + Sort Key: l_partkey, l_orderkey + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(14 rows) + +-- distinct on with joins +-- each customer's first order key +SELECT DISTINCT ON (o_custkey) o_custkey, l_orderkey + FROM lineitem_hash_part JOIN orders_hash_part ON (l_orderkey = o_orderkey) + WHERE o_custkey < 15 + ORDER BY 1,2; + o_custkey | l_orderkey +--------------------------------------------------------------------- + 1 | 9154 + 2 | 10563 + 4 | 320 + 5 | 11682 + 7 | 10402 + 8 | 102 + 10 | 1602 + 11 | 12800 + 13 | 994 + 14 | 11011 +(10 rows) + +SELECT coordinator_plan($Q$ +EXPLAIN (COSTS FALSE) + SELECT DISTINCT ON (o_custkey) o_custkey, l_orderkey + FROM lineitem_hash_part JOIN orders_hash_part ON (l_orderkey = o_orderkey) + WHERE o_custkey < 15 + ORDER BY 1,2; +$Q$); + coordinator_plan +--------------------------------------------------------------------- + Unique + -> Sort + Sort Key: remote_scan.o_custkey, remote_scan.l_orderkey + -> Custom Scan (Citus Adaptive) + Task Count: 4 +(5 rows) + +-- explain without order by +-- notice master plan has order by on distinct on column +SELECT coordinator_plan($Q$ +EXPLAIN (COSTS FALSE) + SELECT DISTINCT ON (o_custkey) o_custkey, l_orderkey + FROM lineitem_hash_part JOIN orders_hash_part ON (l_orderkey = o_orderkey) + WHERE o_custkey < 15; +$Q$); + coordinator_plan +--------------------------------------------------------------------- + Unique + -> Sort + Sort Key: remote_scan.o_custkey + -> Custom Scan (Citus Adaptive) + Task Count: 4 +(5 rows) + +-- each customer's each order's first l_partkey +SELECT DISTINCT ON (o_custkey, l_orderkey) o_custkey, l_orderkey, l_linenumber, l_partkey + FROM lineitem_hash_part JOIN orders_hash_part ON (l_orderkey = o_orderkey) + WHERE o_custkey < 20 + ORDER BY 1,2,3; + o_custkey | l_orderkey | l_linenumber | l_partkey +--------------------------------------------------------------------- + 1 | 9154 | 1 | 86513 + 1 | 14656 | 1 | 59539 + 2 | 10563 | 1 | 147459 + 4 | 320 | 1 | 4415 + 4 | 739 | 1 | 84489 + 4 | 10688 | 1 | 45037 + 4 | 10788 | 1 | 50814 + 4 | 13728 | 1 | 86216 + 5 | 11682 | 1 | 31634 + 5 | 11746 | 1 | 180724 + 5 | 14308 | 1 | 157430 + 7 | 10402 | 1 | 53661 + 7 | 13031 | 1 | 112161 + 7 | 14145 | 1 | 138729 + 7 | 14404 | 1 | 143034 + 8 | 102 | 1 | 88914 + 8 | 164 | 1 | 91309 + 8 | 13601 | 1 | 40504 + 10 | 1602 | 1 | 182806 + 10 | 9862 | 1 | 86241 + 10 | 11431 | 1 | 62112 + 10 | 13124 | 1 | 29414 + 11 | 12800 | 1 | 152806 + 13 | 994 | 1 | 64486 + 13 | 1603 | 1 | 38191 + 13 | 4704 | 1 | 77934 + 13 | 9927 | 1 | 875 + 14 | 11011 | 1 | 172485 + 17 | 896 | 1 | 38675 + 17 | 5507 | 1 | 9600 + 19 | 353 | 1 | 119305 + 19 | 1504 | 1 | 81389 + 19 | 1669 | 1 | 78373 + 19 | 5893 | 1 | 133707 + 19 | 9954 | 1 | 92138 + 19 | 14885 | 1 | 36154 +(36 rows) + +-- explain without order by +SELECT coordinator_plan($Q$ +EXPLAIN (COSTS FALSE) + SELECT DISTINCT ON (o_custkey, l_orderkey) o_custkey, l_orderkey, l_linenumber, l_partkey + FROM lineitem_hash_part JOIN orders_hash_part ON (l_orderkey = o_orderkey) + WHERE o_custkey < 20; +$Q$); + coordinator_plan +--------------------------------------------------------------------- + Unique + -> Sort + Sort Key: remote_scan.o_custkey, remote_scan.l_orderkey + -> Custom Scan (Citus Adaptive) + Task Count: 4 +(5 rows) + +-- each customer's each order's last l_partkey +SELECT DISTINCT ON (o_custkey, l_orderkey) o_custkey, l_orderkey, l_linenumber, l_partkey + FROM lineitem_hash_part JOIN orders_hash_part ON (l_orderkey = o_orderkey) + WHERE o_custkey < 15 + ORDER BY 1,2,3 DESC; + o_custkey | l_orderkey | l_linenumber | l_partkey +--------------------------------------------------------------------- + 1 | 9154 | 7 | 173448 + 1 | 14656 | 1 | 59539 + 2 | 10563 | 4 | 110741 + 4 | 320 | 2 | 192158 + 4 | 739 | 5 | 187523 + 4 | 10688 | 2 | 132574 + 4 | 10788 | 4 | 196473 + 4 | 13728 | 3 | 12450 + 5 | 11682 | 3 | 177152 + 5 | 11746 | 7 | 193807 + 5 | 14308 | 3 | 140916 + 7 | 10402 | 2 | 64514 + 7 | 13031 | 6 | 7761 + 7 | 14145 | 6 | 130723 + 7 | 14404 | 7 | 35349 + 8 | 102 | 4 | 61158 + 8 | 164 | 7 | 3037 + 8 | 13601 | 5 | 12470 + 10 | 1602 | 1 | 182806 + 10 | 9862 | 5 | 135675 + 10 | 11431 | 7 | 8563 + 10 | 13124 | 3 | 67055 + 11 | 12800 | 5 | 179110 + 13 | 994 | 4 | 130471 + 13 | 1603 | 2 | 65209 + 13 | 4704 | 3 | 63081 + 13 | 9927 | 6 | 119356 + 14 | 11011 | 7 | 95939 +(28 rows) + +-- subqueries +SELECT DISTINCT l_orderkey, l_partkey + FROM ( + SELECT l_orderkey, l_partkey + FROM lineitem_hash_part + ) q + ORDER BY 1,2 + LIMIT 10; + l_orderkey | l_partkey +--------------------------------------------------------------------- + 1 | 2132 + 1 | 15635 + 1 | 24027 + 1 | 63700 + 1 | 67310 + 1 | 155190 + 2 | 106170 + 3 | 4297 + 3 | 19036 + 3 | 29380 +(10 rows) + +EXPLAIN (COSTS FALSE) + SELECT DISTINCT l_orderkey, l_partkey + FROM ( + SELECT l_orderkey, l_partkey + FROM lineitem_hash_part + ) q + ORDER BY 1,2 + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Sort + Sort Key: remote_scan.l_orderkey, remote_scan.l_partkey + -> HashAggregate + Group Key: remote_scan.l_orderkey, remote_scan.l_partkey + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Limit + -> Sort + Sort Key: l_orderkey, l_partkey + -> HashAggregate + Group Key: l_orderkey, l_partkey + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(16 rows) + +SELECT DISTINCT l_orderkey, cnt + FROM ( + SELECT l_orderkey, count(*) as cnt + FROM lineitem_hash_part + GROUP BY 1 + ) q + ORDER BY 1,2 + LIMIT 10; + l_orderkey | cnt +--------------------------------------------------------------------- + 1 | 6 + 2 | 1 + 3 | 6 + 4 | 1 + 5 | 3 + 6 | 1 + 7 | 7 + 32 | 6 + 33 | 4 + 34 | 3 +(10 rows) + +EXPLAIN (COSTS FALSE) + SELECT DISTINCT l_orderkey, cnt + FROM ( + SELECT l_orderkey, count(*) as cnt + FROM lineitem_hash_part + GROUP BY 1 + ) q + ORDER BY 1,2 + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Sort + Sort Key: remote_scan.l_orderkey, remote_scan.cnt + -> HashAggregate + Group Key: remote_scan.l_orderkey, remote_scan.cnt + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Limit + -> Sort + Sort Key: lineitem_hash_part.l_orderkey, (count(*)) + -> HashAggregate + Group Key: lineitem_hash_part.l_orderkey, count(*) + -> HashAggregate + Group Key: lineitem_hash_part.l_orderkey + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(18 rows) + +-- distinct on partition column +-- random() is added to inner query to prevent flattening +SELECT DISTINCT ON (l_orderkey) l_orderkey, l_partkey + FROM ( + SELECT l_orderkey, l_partkey, (random()*10)::int + 2 as r + FROM lineitem_hash_part + ) q + WHERE r > 1 + ORDER BY 1,2 + LIMIT 10; + l_orderkey | l_partkey +--------------------------------------------------------------------- + 1 | 2132 + 2 | 106170 + 3 | 4297 + 4 | 88035 + 5 | 37531 + 6 | 139636 + 7 | 79251 + 32 | 2743 + 33 | 33918 + 34 | 88362 +(10 rows) + +EXPLAIN (COSTS FALSE) + SELECT DISTINCT ON (l_orderkey) l_orderkey, l_partkey + FROM ( + SELECT l_orderkey, l_partkey, (random()*10)::int + 2 as r + FROM lineitem_hash_part + ) q + WHERE r > 1 + ORDER BY 1,2 + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Unique + -> Sort + Sort Key: remote_scan.l_orderkey, remote_scan.l_partkey + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Limit + -> Unique + -> Sort + Sort Key: q.l_orderkey, q.l_partkey + -> Subquery Scan on q + Filter: (q.r > 1) + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(16 rows) + +-- distinct on non-partition column +SELECT DISTINCT ON (l_partkey) l_orderkey, l_partkey + FROM ( + SELECT l_orderkey, l_partkey, (random()*10)::int + 2 as r + FROM lineitem_hash_part + ) q + WHERE r > 1 + ORDER BY 2,1 + LIMIT 10; + l_orderkey | l_partkey +--------------------------------------------------------------------- + 12005 | 18 + 5121 | 79 + 2883 | 91 + 807 | 149 + 4102 | 175 + 2117 | 179 + 548 | 182 + 2528 | 195 + 10048 | 204 + 9413 | 222 +(10 rows) + +EXPLAIN (COSTS FALSE) + SELECT DISTINCT ON (l_partkey) l_orderkey, l_partkey + FROM ( + SELECT l_orderkey, l_partkey, (random()*10)::int + 2 as r + FROM lineitem_hash_part + ) q + WHERE r > 1 + ORDER BY 2,1 + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Unique + -> Sort + Sort Key: remote_scan.l_partkey, remote_scan.l_orderkey + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Limit + -> Unique + -> Sort + Sort Key: q.l_partkey, q.l_orderkey + -> Subquery Scan on q + Filter: (q.r > 1) + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(16 rows) + From 76c7b3d1c6b21f1602a1bce4424f7e9ec3cb30b1 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Tue, 9 Jun 2020 12:08:41 +0300 Subject: [PATCH 26/52] Remove unused steps in isolation tests PG13 gives a warning for unused steps therefore we should remove the unused steps in isolation tests. --- ...dd_node_vs_reference_table_operations.spec | 5 -- .../spec/isolation_add_remove_node.spec | 15 ----- .../spec/isolation_append_copy_vs_all.spec | 2 - .../spec/isolation_copy_vs_all_on_mx.spec | 5 -- ...ation_create_table_vs_add_remove_node.spec | 10 ---- .../regress/spec/isolation_ddl_vs_all.spec | 7 --- .../regress/spec/isolation_delete_vs_all.spec | 2 - .../isolation_dis2ref_foreign_keys_on_mx.spec | 5 -- ...lation_distributed_deadlock_detection.spec | 60 ------------------- .../isolation_distributed_transaction_id.spec | 11 ---- .../regress/spec/isolation_dml_vs_repair.spec | 5 -- .../regress/spec/isolation_drop_vs_all.spec | 2 - ...ation_ensure_dependency_activate_node.spec | 10 ---- ...ation_get_distributed_wait_queries_mx.spec | 15 ----- .../spec/isolation_hash_copy_vs_all.spec | 2 - .../spec/isolation_insert_select_vs_all.spec | 2 - .../isolation_insert_select_vs_all_on_mx.spec | 5 -- .../spec/isolation_insert_vs_all_on_mx.spec | 5 -- .../spec/isolation_master_update_node.spec | 1 - ...isolation_modify_with_subquery_vs_dml.spec | 5 -- .../isolation_multi_shard_modify_vs_all.spec | 5 -- .../spec/isolation_multiuser_locking.spec | 10 ---- .../isolation_partitioned_copy_vs_all.spec | 8 --- .../spec/isolation_range_copy_vs_all.spec | 2 - .../isolation_ref2ref_foreign_keys_on_mx.spec | 10 ---- ...ref_update_delete_upsert_vs_all_on_mx.spec | 10 ---- .../spec/isolation_reference_copy_vs_all.spec | 6 -- .../spec/isolation_reference_on_mx.spec | 5 -- .../spec/isolation_select_vs_all_on_mx.spec | 5 -- .../spec/isolation_truncate_vs_all.spec | 2 - ...ion_update_delete_upsert_vs_all_on_mx.spec | 5 -- .../regress/spec/isolation_update_node.spec | 13 ---- .../isolation_update_node_lock_writes.spec | 5 -- .../regress/spec/isolation_upsert_vs_all.spec | 1 - .../spec/isolation_validate_vs_insert.spec | 1 - 35 files changed, 262 deletions(-) diff --git a/src/test/regress/spec/isolation_add_node_vs_reference_table_operations.spec b/src/test/regress/spec/isolation_add_node_vs_reference_table_operations.spec index 72517669a..fd4eac77f 100644 --- a/src/test/regress/spec/isolation_add_node_vs_reference_table_operations.spec +++ b/src/test/regress/spec/isolation_add_node_vs_reference_table_operations.spec @@ -44,11 +44,6 @@ step "s1-add-second-worker" SELECT 1 FROM master_add_node('localhost', 57638); } -step "s1-remove-second-worker" -{ - SELECT master_remove_node('localhost', 57638); -} - step "s1-drop-reference-table" { DROP TABLE test_reference_table; diff --git a/src/test/regress/spec/isolation_add_remove_node.spec b/src/test/regress/spec/isolation_add_remove_node.spec index 40a008749..e8bbe62ed 100644 --- a/src/test/regress/spec/isolation_add_remove_node.spec +++ b/src/test/regress/spec/isolation_add_remove_node.spec @@ -45,11 +45,6 @@ step "s1-remove-node-1" SELECT * FROM master_remove_node('localhost', 57637); } -step "s1-remove-node-2" -{ - SELECT * FROM master_remove_node('localhost', 57638); -} - step "s1-abort" { ABORT; @@ -67,11 +62,6 @@ step "s1-show-nodes" session "s2" -step "s2-begin" -{ - BEGIN; -} - step "s2-add-node-1" { SELECT 1 FROM master_add_node('localhost', 57637); @@ -102,11 +92,6 @@ step "s2-remove-node-2" SELECT * FROM master_remove_node('localhost', 57638); } -step "s2-commit" -{ - COMMIT; -} - // session 1 adds a node, session 2 removes it, should be ok permutation "s1-begin" "s1-add-node-1" "s2-remove-node-1" "s1-commit" "s1-show-nodes" // add a different node from 2 sessions, should be ok diff --git a/src/test/regress/spec/isolation_append_copy_vs_all.spec b/src/test/regress/spec/isolation_append_copy_vs_all.spec index 0c2bc1b12..e6c1a03fc 100644 --- a/src/test/regress/spec/isolation_append_copy_vs_all.spec +++ b/src/test/regress/spec/isolation_append_copy_vs_all.spec @@ -40,7 +40,6 @@ step "s1-ddl-drop-index" { DROP INDEX append_copy_index; } step "s1-ddl-add-column" { ALTER TABLE append_copy ADD new_column int DEFAULT 0; } step "s1-ddl-drop-column" { ALTER TABLE append_copy DROP new_column; } step "s1-ddl-rename-column" { ALTER TABLE append_copy RENAME data TO new_column; } -step "s1-ddl-unique-constraint" { ALTER TABLE append_copy ADD CONSTRAINT append_copy_unique UNIQUE(id); } step "s1-table-size" { SELECT citus_total_relation_size('append_copy'); } step "s1-master-apply-delete-command" { SELECT master_apply_delete_command('DELETE FROM append_copy WHERE id <= 4;'); } step "s1-master-drop-all-shards" { SELECT master_drop_all_shards('append_copy'::regclass, 'public', 'append_copy'); } @@ -54,7 +53,6 @@ step "s1-commit" { COMMIT; } // session 2 session "s2" step "s2-copy" { COPY append_copy FROM PROGRAM 'echo 5, f, 5 && echo 6, g, 6 && echo 7, h, 7 && echo 8, i, 8 && echo 9, j, 9' WITH CSV; } -step "s2-copy-additional-column" { COPY append_copy FROM PROGRAM 'echo 5, f, 5, 5 && echo 6, g, 6, 6 && echo 7, h, 7, 7 && echo 8, i, 8, 8 && echo 9, j, 9, 9' WITH CSV; } step "s2-router-select" { SELECT * FROM append_copy WHERE id = 1; } step "s2-real-time-select" { SELECT * FROM append_copy ORDER BY 1, 2; } step "s2-adaptive-select" diff --git a/src/test/regress/spec/isolation_copy_vs_all_on_mx.spec b/src/test/regress/spec/isolation_copy_vs_all_on_mx.spec index 53292f48a..92fb26e73 100644 --- a/src/test/regress/spec/isolation_copy_vs_all_on_mx.spec +++ b/src/test/regress/spec/isolation_copy_vs_all_on_mx.spec @@ -79,11 +79,6 @@ step "s2-select-for-update" SELECT run_commands_on_session_level_connection_to_node('SELECT * FROM copy_table WHERE id=5 FOR UPDATE'); } -step "s2-coordinator-create-index-concurrently" -{ - CREATE INDEX CONCURRENTLY copy_table_index ON copy_table(id); -} - step "s2-commit-worker" { SELECT run_commands_on_session_level_connection_to_node('COMMIT'); diff --git a/src/test/regress/spec/isolation_create_table_vs_add_remove_node.spec b/src/test/regress/spec/isolation_create_table_vs_add_remove_node.spec index bb03b843b..2f91ae1a7 100644 --- a/src/test/regress/spec/isolation_create_table_vs_add_remove_node.spec +++ b/src/test/regress/spec/isolation_create_table_vs_add_remove_node.spec @@ -38,16 +38,6 @@ step "s1-commit" COMMIT; } -step "s1-query-table" -{ - SELECT * FROM dist_table; -} - -step "s1-show-nodes" -{ - SELECT nodename, nodeport, isactive FROM pg_dist_node ORDER BY nodename, nodeport; -} - step "s1-show-placements" { SELECT diff --git a/src/test/regress/spec/isolation_ddl_vs_all.spec b/src/test/regress/spec/isolation_ddl_vs_all.spec index 84603ba2c..81a0501cf 100644 --- a/src/test/regress/spec/isolation_ddl_vs_all.spec +++ b/src/test/regress/spec/isolation_ddl_vs_all.spec @@ -26,16 +26,13 @@ session "s1" step "s1-initialize" { COPY ddl_hash FROM PROGRAM 'echo 0, a && echo 1, b && echo 2, c && echo 3, d && echo 4, e' WITH CSV; } step "s1-begin" { BEGIN; } step "s1-ddl-create-index" { CREATE INDEX ddl_hash_index ON ddl_hash(id); } -step "s1-ddl-drop-index" { DROP INDEX ddl_hash_index; } step "s1-ddl-add-column" { ALTER TABLE ddl_hash ADD new_column_1 int DEFAULT 0; } -step "s1-ddl-drop-column" { ALTER TABLE ddl_hash DROP new_column_2; } step "s1-ddl-rename-column" { ALTER TABLE ddl_hash RENAME data TO new_column; } step "s1-table-size" { SELECT citus_total_relation_size('ddl_hash'); } step "s1-master-modify-multiple-shards" { DELETE FROM ddl_hash; } step "s1-drop" { DROP TABLE ddl_hash; } step "s1-create-non-distributed-table" { CREATE TABLE ddl_hash(id integer, data text); COPY ddl_hash FROM PROGRAM 'echo 0, a && echo 1, b && echo 2, c && echo 3, d && echo 4, e' WITH CSV; } step "s1-distribute-table" { SELECT create_distributed_table('ddl_hash', 'id'); } -step "s1-select-count" { SELECT COUNT(*) FROM ddl_hash; } step "s1-show-indexes" { SELECT run_command_on_workers('SELECT COUNT(*) FROM pg_indexes WHERE tablename LIKE ''ddl_hash%'''); } step "s1-show-columns" { SELECT run_command_on_workers('SELECT column_name FROM information_schema.columns WHERE table_name LIKE ''ddl_hash%'' AND column_name = ''new_column'' ORDER BY 1 LIMIT 1'); } step "s1-commit" { COMMIT; } @@ -44,16 +41,12 @@ step "s1-commit" { COMMIT; } session "s2" step "s2-begin" { BEGIN; } step "s2-ddl-create-index" { CREATE INDEX ddl_hash_index ON ddl_hash(id); } -step "s2-ddl-drop-index" { DROP INDEX ddl_hash_index; } step "s2-ddl-create-index-concurrently" { CREATE INDEX CONCURRENTLY ddl_hash_index ON ddl_hash(id); } step "s2-ddl-add-column" { ALTER TABLE ddl_hash ADD new_column_2 int DEFAULT 0; } -step "s2-ddl-drop-column" { ALTER TABLE ddl_hash DROP new_column_1; } step "s2-ddl-rename-column" { ALTER TABLE ddl_hash RENAME data TO new_column; } step "s2-table-size" { SELECT citus_total_relation_size('ddl_hash'); } step "s2-master-modify-multiple-shards" { DELETE FROM ddl_hash; } -step "s2-create-non-distributed-table" { CREATE TABLE ddl_hash(id integer, data text); COPY ddl_hash FROM PROGRAM 'echo 0, a && echo 1, b && echo 2, c && echo 3, d && echo 4, e' WITH CSV; } step "s2-distribute-table" { SELECT create_distributed_table('ddl_hash', 'id'); } -step "s2-select" { SELECT * FROM ddl_hash ORDER BY 1, 2; } step "s2-commit" { COMMIT; } // permutations - DDL vs DDL diff --git a/src/test/regress/spec/isolation_delete_vs_all.spec b/src/test/regress/spec/isolation_delete_vs_all.spec index 422384c24..b93baf43b 100644 --- a/src/test/regress/spec/isolation_delete_vs_all.spec +++ b/src/test/regress/spec/isolation_delete_vs_all.spec @@ -54,9 +54,7 @@ step "s2-ddl-add-column" { ALTER TABLE delete_hash ADD new_column int DEFAULT 0; step "s2-ddl-drop-column" { ALTER TABLE delete_hash DROP new_column; } step "s2-ddl-rename-column" { ALTER TABLE delete_hash RENAME data TO new_column; } step "s2-table-size" { SELECT citus_total_relation_size('delete_hash'); } -step "s2-create-non-distributed-table" { CREATE TABLE delete_hash(id integer, data text); COPY delete_hash FROM PROGRAM 'echo 0, a && echo 1, b && echo 2, c && echo 3, d && echo 4, e' WITH CSV; } step "s2-distribute-table" { SELECT create_distributed_table('delete_hash', 'id'); } -step "s2-select" { SELECT * FROM delete_hash ORDER BY 1, 2; } step "s2-commit" { COMMIT; } // permutations - DELETE vs DELETE diff --git a/src/test/regress/spec/isolation_dis2ref_foreign_keys_on_mx.spec b/src/test/regress/spec/isolation_dis2ref_foreign_keys_on_mx.spec index 7bacf4fb5..eb312fae3 100644 --- a/src/test/regress/spec/isolation_dis2ref_foreign_keys_on_mx.spec +++ b/src/test/regress/spec/isolation_dis2ref_foreign_keys_on_mx.spec @@ -102,11 +102,6 @@ step "s2-select-for-udpate" SELECT run_commands_on_session_level_connection_to_node('SELECT * FROM dist_table WHERE id=1 FOR UPDATE'); } -step "s2-coordinator-create-index-concurrently" -{ - CREATE INDEX CONCURRENTLY dist_table_index ON dist_table(id); -} - step "s2-commit-worker" { SELECT run_commands_on_session_level_connection_to_node('COMMIT'); diff --git a/src/test/regress/spec/isolation_distributed_deadlock_detection.spec b/src/test/regress/spec/isolation_distributed_deadlock_detection.spec index f738bc45d..9676530bc 100644 --- a/src/test/regress/spec/isolation_distributed_deadlock_detection.spec +++ b/src/test/regress/spec/isolation_distributed_deadlock_detection.spec @@ -47,11 +47,6 @@ step "s1-update-2" UPDATE deadlock_detection_test SET some_val = 1 WHERE user_id = 2; } -step "s1-update-3" -{ - UPDATE deadlock_detection_test SET some_val = 1 WHERE user_id = 3; -} - step "s1-update-4" { UPDATE deadlock_detection_test SET some_val = 1 WHERE user_id = 4; @@ -234,11 +229,6 @@ step "s4-update-2" UPDATE deadlock_detection_test SET some_val = 4 WHERE user_id = 2; } -step "s4-update-3" -{ - UPDATE deadlock_detection_test SET some_val = 4 WHERE user_id = 3; -} - step "s4-update-4" { UPDATE deadlock_detection_test SET some_val = 4 WHERE user_id = 4; @@ -254,11 +244,6 @@ step "s4-update-6" UPDATE deadlock_detection_test SET some_val = 4 WHERE user_id = 6; } -step "s4-update-7" -{ - UPDATE deadlock_detection_test SET some_val = 4 WHERE user_id = 7; -} - step "s4-random-adv-lock" { SELECT pg_advisory_xact_lock(8765); @@ -281,21 +266,6 @@ step "s5-update-1" UPDATE deadlock_detection_test SET some_val = 5 WHERE user_id = 1; } -step "s5-update-2" -{ - UPDATE deadlock_detection_test SET some_val = 5 WHERE user_id = 2; -} - -step "s5-update-3" -{ - UPDATE deadlock_detection_test SET some_val = 5 WHERE user_id = 3; -} - -step "s5-update-4" -{ - UPDATE deadlock_detection_test SET some_val = 5 WHERE user_id = 4; -} - step "s5-update-5" { UPDATE deadlock_detection_test SET some_val = 5 WHERE user_id = 5; @@ -306,11 +276,6 @@ step "s5-update-6" UPDATE deadlock_detection_test SET some_val = 5 WHERE user_id = 6; } -step "s5-update-7" -{ - UPDATE deadlock_detection_test SET some_val = 5 WHERE user_id = 7; -} - step "s5-random-adv-lock" { SELECT pg_advisory_xact_lock(8765); @@ -328,26 +293,6 @@ step "s6-begin" BEGIN; } -step "s6-update-1" -{ - UPDATE deadlock_detection_test SET some_val = 6 WHERE user_id = 1; -} - -step "s6-update-2" -{ - UPDATE deadlock_detection_test SET some_val = 6 WHERE user_id = 2; -} - -step "s6-update-3" -{ - UPDATE deadlock_detection_test SET some_val = 6 WHERE user_id = 3; -} - -step "s6-update-4" -{ - UPDATE deadlock_detection_test SET some_val = 6 WHERE user_id = 4; -} - step "s6-update-5" { UPDATE deadlock_detection_test SET some_val = 6 WHERE user_id = 5; @@ -358,11 +303,6 @@ step "s6-update-6" UPDATE deadlock_detection_test SET some_val = 6 WHERE user_id = 6; } -step "s6-update-7" -{ - UPDATE deadlock_detection_test SET some_val = 6 WHERE user_id = 7; -} - step "s6-commit" { COMMIT; diff --git a/src/test/regress/spec/isolation_distributed_transaction_id.spec b/src/test/regress/spec/isolation_distributed_transaction_id.spec index 53d6dd828..7bf4c02c0 100644 --- a/src/test/regress/spec/isolation_distributed_transaction_id.spec +++ b/src/test/regress/spec/isolation_distributed_transaction_id.spec @@ -86,17 +86,6 @@ step "s2-commit" COMMIT; } -// print only the necessary parts to prevent concurrent runs to print different values -step "s2-get-first-worker-active-transactions" -{ - SELECT * FROM run_command_on_workers('SELECT row(initiator_node_identifier, transaction_number) - FROM - get_all_active_transactions(); - ') - WHERE nodeport = 57637; -; -} - step "s2-get-all-transactions" { SELECT initiator_node_identifier, transaction_number, transaction_stamp FROM get_current_transaction_id() ORDER BY 1,2,3; diff --git a/src/test/regress/spec/isolation_dml_vs_repair.spec b/src/test/regress/spec/isolation_dml_vs_repair.spec index 78a2a764c..46a93ed8d 100644 --- a/src/test/regress/spec/isolation_dml_vs_repair.spec +++ b/src/test/regress/spec/isolation_dml_vs_repair.spec @@ -68,11 +68,6 @@ step "s2-invalidate-57637" UPDATE pg_dist_shard_placement SET shardstate = '3' WHERE shardid = (SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'test_dml_vs_repair'::regclass) AND nodeport = 57637; } -step "s2-revalidate-57637" -{ - UPDATE pg_dist_shard_placement SET shardstate = '1' WHERE shardid = (SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'test_dml_vs_repair'::regclass) AND nodeport = 57637; -} - step "s2-invalidate-57638" { UPDATE pg_dist_shard_placement SET shardstate = '3' WHERE shardid = (SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'test_dml_vs_repair'::regclass) AND nodeport = 57638; diff --git a/src/test/regress/spec/isolation_drop_vs_all.spec b/src/test/regress/spec/isolation_drop_vs_all.spec index c23471a90..3aef01eee 100644 --- a/src/test/regress/spec/isolation_drop_vs_all.spec +++ b/src/test/regress/spec/isolation_drop_vs_all.spec @@ -50,9 +50,7 @@ step "s2-ddl-add-column" { ALTER TABLE drop_hash ADD new_column int DEFAULT 0; } step "s2-ddl-drop-column" { ALTER TABLE drop_hash DROP new_column; } step "s2-ddl-rename-column" { ALTER TABLE drop_hash RENAME data TO new_column; } step "s2-table-size" { SELECT citus_total_relation_size('drop_hash'); } -step "s2-create-non-distributed-table" { CREATE TABLE drop_hash(id integer, data text); COPY drop_hash FROM PROGRAM 'echo 0, a && echo 1, b && echo 2, c && echo 3, d && echo 4, e' WITH CSV; } step "s2-distribute-table" { SELECT create_distributed_table('drop_hash', 'id'); } -step "s2-select" { SELECT * FROM drop_hash ORDER BY 1, 2; } step "s2-commit" { COMMIT; } // permutations - DROP vs DROP diff --git a/src/test/regress/spec/isolation_ensure_dependency_activate_node.spec b/src/test/regress/spec/isolation_ensure_dependency_activate_node.spec index f36b1cbb5..539c71c5c 100644 --- a/src/test/regress/spec/isolation_ensure_dependency_activate_node.spec +++ b/src/test/regress/spec/isolation_ensure_dependency_activate_node.spec @@ -149,11 +149,6 @@ step "s2-print-distributed-objects" session "s3" -step "s3-public-schema" -{ - SET search_path TO public; -} - step "s3-use-schema" { SET search_path TO myschema; @@ -172,11 +167,6 @@ step "s3-wait-for-metadata-sync" SELECT public.wait_until_metadata_sync(5000); } -step "s3-listen-channel" -{ - LISTEN metadata_sync; -} - step "s3-create-schema2" { CREATE SCHEMA myschema2; diff --git a/src/test/regress/spec/isolation_get_distributed_wait_queries_mx.spec b/src/test/regress/spec/isolation_get_distributed_wait_queries_mx.spec index d49aa1f90..b43708ab1 100644 --- a/src/test/regress/spec/isolation_get_distributed_wait_queries_mx.spec +++ b/src/test/regress/spec/isolation_get_distributed_wait_queries_mx.spec @@ -100,11 +100,6 @@ step "s1-commit" session "s2" -step "s2-begin" -{ - COMMIT; -} - step "s2-start-session-level-connection" { SELECT start_session_level_connection_to_node('localhost', 57638); @@ -125,16 +120,6 @@ step "s2-update-ref-table" SELECT run_commands_on_session_level_connection_to_node('UPDATE ref_table SET value_1 = 12 WHERE user_id = 1'); } -step "s2-select-from-ref-table" -{ - SELECT run_commands_on_session_level_connection_to_node('SELECT count(*) FROM ref_table'); -} - -step "s2-delete-from-ref-table" -{ - SELECT run_commands_on_session_level_connection_to_node('DELETE FROM ref_table WHERE user_id = 2'); -} - step "s2-insert-into-ref-table" { SELECT run_commands_on_session_level_connection_to_node('INSERT INTO ref_table VALUES(8,81),(9,91)'); diff --git a/src/test/regress/spec/isolation_hash_copy_vs_all.spec b/src/test/regress/spec/isolation_hash_copy_vs_all.spec index f2d25b92a..419485c70 100644 --- a/src/test/regress/spec/isolation_hash_copy_vs_all.spec +++ b/src/test/regress/spec/isolation_hash_copy_vs_all.spec @@ -40,7 +40,6 @@ step "s1-ddl-drop-index" { DROP INDEX hash_copy_index; } step "s1-ddl-add-column" { ALTER TABLE hash_copy ADD new_column int DEFAULT 0; } step "s1-ddl-drop-column" { ALTER TABLE hash_copy DROP new_column; } step "s1-ddl-rename-column" { ALTER TABLE hash_copy RENAME data TO new_column; } -step "s1-ddl-unique-constraint" { ALTER TABLE hash_copy ADD CONSTRAINT hash_copy_unique UNIQUE(id); } step "s1-table-size" { SELECT citus_total_relation_size('hash_copy'); } step "s1-master-modify-multiple-shards" { DELETE FROM hash_copy; } step "s1-master-drop-all-shards" { SELECT master_drop_all_shards('hash_copy'::regclass, 'public', 'hash_copy'); } @@ -61,7 +60,6 @@ step "s1-recreate-with-replication-2" // session 2 session "s2" step "s2-copy" { COPY hash_copy FROM PROGRAM 'echo 5, f, 5 && echo 6, g, 6 && echo 7, h, 7 && echo 8, i, 8 && echo 9, j, 9' WITH CSV; } -step "s2-copy-additional-column" { COPY hash_copy FROM PROGRAM 'echo 5, f, 5, 5 && echo 6, g, 6, 6 && echo 7, h, 7, 7 && echo 8, i, 8, 8 && echo 9, j, 9, 9' WITH CSV; } step "s2-router-select" { SELECT * FROM hash_copy WHERE id = 1; } step "s2-real-time-select" { SELECT * FROM hash_copy ORDER BY 1, 2; } step "s2-adaptive-select" diff --git a/src/test/regress/spec/isolation_insert_select_vs_all.spec b/src/test/regress/spec/isolation_insert_select_vs_all.spec index b9cef7db1..ca28f461c 100644 --- a/src/test/regress/spec/isolation_insert_select_vs_all.spec +++ b/src/test/regress/spec/isolation_insert_select_vs_all.spec @@ -81,7 +81,6 @@ step "s2-ddl-rename-column-on-inserted" { ALTER TABLE insert_of_insert_select_ha step "s2-table-size-on-inserted" { SELECT citus_total_relation_size('insert_of_insert_select_hash'); } step "s2-master-modify-multiple-shards-on-inserted" { DELETE FROM insert_of_insert_select_hash; } step "s2-master-drop-all-shards-on-inserted" { SELECT master_drop_all_shards('insert_of_insert_select_hash'::regclass, 'public', 'insert_of_insert_select_hash'); } -step "s2-create-non-distributed-table-on-inserted" { CREATE TABLE insert_of_insert_select_hash(id integer, data text); } step "s2-distribute-table-on-inserted" { SELECT create_distributed_table('insert_of_insert_select_hash', 'id'); } step "s2-update-on-selected" { UPDATE select_of_insert_select_hash SET data = 'l' WHERE id = 4; } step "s2-delete-on-selected" { DELETE FROM select_of_insert_select_hash WHERE id = 4; } @@ -96,7 +95,6 @@ step "s2-ddl-rename-column-on-selected" { ALTER TABLE select_of_insert_select_ha step "s2-table-size-on-selected" { SELECT citus_total_relation_size('select_of_insert_select_hash'); } step "s2-master-modify-multiple-shards-on-selected" { DELETE FROM select_of_insert_select_hash; } step "s2-master-drop-all-shards-on-selected" { SELECT master_drop_all_shards('select_of_insert_select_hash'::regclass, 'public', 'select_of_insert_select_hash'); } -step "s2-create-non-distributed-table-on-selected" { CREATE TABLE select_of_insert_select_hash(id integer, data text); } step "s2-distribute-table-on-selected" { SELECT create_distributed_table('select_of_insert_select_hash', 'id'); } // permutations - INSERT/SELECT vs INSERT/SELECT diff --git a/src/test/regress/spec/isolation_insert_select_vs_all_on_mx.spec b/src/test/regress/spec/isolation_insert_select_vs_all_on_mx.spec index 4073ef027..82d8d8638 100644 --- a/src/test/regress/spec/isolation_insert_select_vs_all_on_mx.spec +++ b/src/test/regress/spec/isolation_insert_select_vs_all_on_mx.spec @@ -109,11 +109,6 @@ step "s2-select-for-update" SELECT run_commands_on_session_level_connection_to_node('SELECT * FROM dist_table WHERE id = 5 FOR UPDATE'); } -step "s2-coordinator-create-index-concurrently" -{ - CREATE INDEX CONCURRENTLY dist_table_index ON dist_table(id); -} - step "s2-commit-worker" { SELECT run_commands_on_session_level_connection_to_node('COMMIT'); diff --git a/src/test/regress/spec/isolation_insert_vs_all_on_mx.spec b/src/test/regress/spec/isolation_insert_vs_all_on_mx.spec index 88f2e0603..8f98289ad 100644 --- a/src/test/regress/spec/isolation_insert_vs_all_on_mx.spec +++ b/src/test/regress/spec/isolation_insert_vs_all_on_mx.spec @@ -110,11 +110,6 @@ step "s2-select-for-update" SELECT run_commands_on_session_level_connection_to_node('SELECT * FROM insert_table WHERE id = 6 FOR UPDATE'); } -step "s2-coordinator-create-index-concurrently" -{ - CREATE INDEX CONCURRENTLY insert_table_index ON insert_table(id); -} - step "s2-commit-worker" { SELECT run_commands_on_session_level_connection_to_node('COMMIT'); diff --git a/src/test/regress/spec/isolation_master_update_node.spec b/src/test/regress/spec/isolation_master_update_node.spec index a7fdf3652..5234db32b 100644 --- a/src/test/regress/spec/isolation_master_update_node.spec +++ b/src/test/regress/spec/isolation_master_update_node.spec @@ -18,7 +18,6 @@ teardown session "s1" step "s1-begin" { BEGIN; } step "s1-insert" { INSERT INTO t1 SELECT generate_series(1, 100); } -step "s1-verify-terminated" { -- verify the connection has been terminated } step "s1-abort" { ABORT; } session "s2" diff --git a/src/test/regress/spec/isolation_modify_with_subquery_vs_dml.spec b/src/test/regress/spec/isolation_modify_with_subquery_vs_dml.spec index d3e193bd3..903be0c1b 100644 --- a/src/test/regress/spec/isolation_modify_with_subquery_vs_dml.spec +++ b/src/test/regress/spec/isolation_modify_with_subquery_vs_dml.spec @@ -71,11 +71,6 @@ step "s2-modify_with_subquery_v1" UPDATE users_test_table SET value_2 = 5 FROM events_test_table WHERE users_test_table.user_id = events_test_table.user_id; } -step "s2-modify_with_subquery_v2" -{ - UPDATE users_test_table SET value_1 = 3 WHERE user_id IN (SELECT user_id FROM events_test_table); -} - step "s2-commit" { COMMIT; diff --git a/src/test/regress/spec/isolation_multi_shard_modify_vs_all.spec b/src/test/regress/spec/isolation_multi_shard_modify_vs_all.spec index 6424863a1..61ecd1a28 100644 --- a/src/test/regress/spec/isolation_multi_shard_modify_vs_all.spec +++ b/src/test/regress/spec/isolation_multi_shard_modify_vs_all.spec @@ -67,11 +67,6 @@ step "s1-update_value_1_of_1_or_3_to_5" UPDATE users_test_table SET value_1 = 5 WHERE user_id = 1 or user_id = 3; } -step "s1-update_value_1_of_1_or_3_to_7" -{ - UPDATE users_test_table SET value_1 = 7 WHERE user_id = 1 or user_id = 3; -} - step "s1-update_value_1_of_2_or_4_to_5" { UPDATE users_test_table SET value_1 = 5 WHERE user_id = 2 or user_id = 4; diff --git a/src/test/regress/spec/isolation_multiuser_locking.spec b/src/test/regress/spec/isolation_multiuser_locking.spec index af225ce61..3c5193e96 100644 --- a/src/test/regress/spec/isolation_multiuser_locking.spec +++ b/src/test/regress/spec/isolation_multiuser_locking.spec @@ -33,11 +33,6 @@ step "s1-grant" GRANT ALL ON test_table TO test_user_2; } -step "s1-setrole" -{ - SET ROLE test_user_1; -} - step "s1-begin" { BEGIN; @@ -76,11 +71,6 @@ step "s1-commit" session "s2" -step "s2-setrole" -{ - SET ROLE test_user_2; -} - step "s2-begin" { BEGIN; diff --git a/src/test/regress/spec/isolation_partitioned_copy_vs_all.spec b/src/test/regress/spec/isolation_partitioned_copy_vs_all.spec index bebddd15f..f893e5282 100644 --- a/src/test/regress/spec/isolation_partitioned_copy_vs_all.spec +++ b/src/test/regress/spec/isolation_partitioned_copy_vs_all.spec @@ -38,26 +38,21 @@ step "s1-update" { UPDATE partitioned_copy SET data = 'l' WHERE id = 0; } step "s1-delete" { DELETE FROM partitioned_copy WHERE id = 1; } step "s1-truncate" { TRUNCATE partitioned_copy; } step "s1-drop" { DROP TABLE partitioned_copy; } -step "s1-ddl-create-index" { CREATE INDEX partitioned_copy_index ON partitioned_copy(id); } -step "s1-ddl-drop-index" { DROP INDEX partitioned_copy_index; } step "s1-ddl-add-column" { ALTER TABLE partitioned_copy ADD new_column int DEFAULT 0; } step "s1-ddl-drop-column" { ALTER TABLE partitioned_copy DROP new_column; } step "s1-ddl-rename-column" { ALTER TABLE partitioned_copy RENAME data TO new_column; } -step "s1-ddl-unique-constraint" { ALTER TABLE partitioned_copy ADD CONSTRAINT partitioned_copy_unique UNIQUE(id); } step "s1-table-size" { SELECT citus_total_relation_size('partitioned_copy'); } step "s1-master-modify-multiple-shards" { DELETE FROM partitioned_copy; } step "s1-master-drop-all-shards" { SELECT master_drop_all_shards('partitioned_copy'::regclass, 'public', 'partitioned_copy'); } step "s1-create-non-distributed-table" { CREATE TABLE partitioned_copy(id integer, data text, int_data int); COPY partitioned_copy FROM PROGRAM 'echo 0, a, 0 && echo 1, b, 1 && echo 2, c, 2 && echo 3, d, 3 && echo 4, e, 4' WITH CSV; } step "s1-distribute-table" { SELECT create_distributed_table('partitioned_copy', 'id'); } step "s1-select-count" { SELECT COUNT(*) FROM partitioned_copy; } -step "s1-show-indexes" { SELECT run_command_on_workers('SELECT COUNT(*) FROM pg_indexes WHERE tablename LIKE ''partitioned_copy%'''); } step "s1-show-columns" { SELECT run_command_on_workers('SELECT column_name FROM information_schema.columns WHERE table_name LIKE ''partitioned_copy%'' AND column_name = ''new_column'' ORDER BY 1 LIMIT 1'); } step "s1-commit" { COMMIT; } // session 2 session "s2" step "s2-copy" { COPY partitioned_copy FROM PROGRAM 'echo 5, f, 5 && echo 6, g, 6 && echo 7, h, 7 && echo 8, i, 8 && echo 9, j, 9' WITH CSV; } -step "s2-copy-additional-column" { COPY partitioned_copy FROM PROGRAM 'echo 5, f, 5, 5 && echo 6, g, 6, 6 && echo 7, h, 7, 7 && echo 8, i, 8, 8 && echo 9, j, 9, 9' WITH CSV; } step "s2-router-select" { SELECT * FROM partitioned_copy WHERE id = 1; } step "s2-real-time-select" { SELECT * FROM partitioned_copy ORDER BY 1, 2; } step "s2-adaptive-select" @@ -71,9 +66,6 @@ step "s2-update" { UPDATE partitioned_copy SET data = 'l' WHERE id = 0; } step "s2-delete" { DELETE FROM partitioned_copy WHERE id = 1; } step "s2-truncate" { TRUNCATE partitioned_copy; } step "s2-drop" { DROP TABLE partitioned_copy; } -step "s2-ddl-create-index" { CREATE INDEX partitioned_copy_index ON partitioned_copy(id); } -step "s2-ddl-drop-index" { DROP INDEX partitioned_copy_index; } -step "s2-ddl-create-index-concurrently" { CREATE INDEX CONCURRENTLY partitioned_copy_index ON partitioned_copy(id); } step "s2-ddl-add-column" { ALTER TABLE partitioned_copy ADD new_column int DEFAULT 0; } step "s2-ddl-drop-column" { ALTER TABLE partitioned_copy DROP new_column; } step "s2-ddl-rename-column" { ALTER TABLE partitioned_copy RENAME data TO new_column; } diff --git a/src/test/regress/spec/isolation_range_copy_vs_all.spec b/src/test/regress/spec/isolation_range_copy_vs_all.spec index facc432f3..eb0451eeb 100644 --- a/src/test/regress/spec/isolation_range_copy_vs_all.spec +++ b/src/test/regress/spec/isolation_range_copy_vs_all.spec @@ -40,7 +40,6 @@ step "s1-ddl-drop-index" { DROP INDEX range_copy_index; } step "s1-ddl-add-column" { ALTER TABLE range_copy ADD new_column int DEFAULT 0; } step "s1-ddl-drop-column" { ALTER TABLE range_copy DROP new_column; } step "s1-ddl-rename-column" { ALTER TABLE range_copy RENAME data TO new_column; } -step "s1-ddl-unique-constraint" { ALTER TABLE range_copy ADD CONSTRAINT range_copy_unique UNIQUE(id); } step "s1-table-size" { SELECT citus_total_relation_size('range_copy'); } step "s1-master-modify-multiple-shards" { DELETE FROM range_copy; } step "s1-master-apply-delete-command" { SELECT master_apply_delete_command('DELETE FROM range_copy WHERE id <= 4;'); } @@ -56,7 +55,6 @@ step "s1-commit" { COMMIT; } // session 2 session "s2" step "s2-copy" { COPY range_copy FROM PROGRAM 'echo 5, f, 5 && echo 6, g, 6 && echo 7, h, 7 && echo 8, i, 8 && echo 9, j, 9' WITH CSV; } -step "s2-copy-additional-column" { COPY range_copy FROM PROGRAM 'echo 5, f, 5, 5 && echo 6, g, 6, 6 && echo 7, h, 7, 7 && echo 8, i, 8, 8 && echo 9, j, 9, 9' WITH CSV; } step "s2-router-select" { SELECT * FROM range_copy WHERE id = 1; } step "s2-real-time-select" { SELECT * FROM range_copy ORDER BY 1, 2; } step "s2-adaptive-select" diff --git a/src/test/regress/spec/isolation_ref2ref_foreign_keys_on_mx.spec b/src/test/regress/spec/isolation_ref2ref_foreign_keys_on_mx.spec index c59963188..2f7a0b74c 100644 --- a/src/test/regress/spec/isolation_ref2ref_foreign_keys_on_mx.spec +++ b/src/test/regress/spec/isolation_ref2ref_foreign_keys_on_mx.spec @@ -29,11 +29,6 @@ step "s1-start-session-level-connection" SELECT start_session_level_connection_to_node('localhost', 57637); } -step "s1-begin-on-worker" -{ - SELECT run_commands_on_session_level_connection_to_node('BEGIN'); -} - step "s1-view-locks" { SELECT * FROM master_run_on_worker( @@ -46,11 +41,6 @@ step "s1-view-locks" false); } -step "s1-rollback-worker" -{ - SELECT run_commands_on_session_level_connection_to_node('ROLLBACK'); -} - step "s1-stop-connection" { SELECT stop_session_level_connection_to_node(); diff --git a/src/test/regress/spec/isolation_ref_update_delete_upsert_vs_all_on_mx.spec b/src/test/regress/spec/isolation_ref_update_delete_upsert_vs_all_on_mx.spec index 952bf079c..8f7de5f6e 100644 --- a/src/test/regress/spec/isolation_ref_update_delete_upsert_vs_all_on_mx.spec +++ b/src/test/regress/spec/isolation_ref_update_delete_upsert_vs_all_on_mx.spec @@ -33,11 +33,6 @@ step "s1-begin-on-worker" SELECT run_commands_on_session_level_connection_to_node('BEGIN'); } -step "s1-update" -{ - SELECT run_commands_on_session_level_connection_to_node('UPDATE ref_table SET value=12 WHERE id=1 OR id=2'); -} - step "s1-delete" { SELECT run_commands_on_session_level_connection_to_node('DELETE FROM ref_table WHERE id=1 OR id=2'); @@ -93,11 +88,6 @@ step "s2-truncate" SELECT run_commands_on_session_level_connection_to_node('TRUNCATE ref_table'); } -step "s2-coordinator-create-index-concurrently" -{ - CREATE INDEX CONCURRENTLY ref_table_index ON ref_table(id); -} - step "s2-commit-worker" { SELECT run_commands_on_session_level_connection_to_node('COMMIT'); diff --git a/src/test/regress/spec/isolation_reference_copy_vs_all.spec b/src/test/regress/spec/isolation_reference_copy_vs_all.spec index aa9028831..1ee681b81 100644 --- a/src/test/regress/spec/isolation_reference_copy_vs_all.spec +++ b/src/test/regress/spec/isolation_reference_copy_vs_all.spec @@ -39,10 +39,8 @@ step "s1-ddl-drop-index" { DROP INDEX reference_copy_index; } step "s1-ddl-add-column" { ALTER TABLE reference_copy ADD new_column int DEFAULT 0; } step "s1-ddl-drop-column" { ALTER TABLE reference_copy DROP new_column; } step "s1-ddl-rename-column" { ALTER TABLE reference_copy RENAME data TO new_column; } -step "s1-ddl-unique-constraint" { ALTER TABLE reference_copy ADD CONSTRAINT reference_copy_unique UNIQUE(id); } step "s1-table-size" { SELECT citus_total_relation_size('reference_copy'); } step "s1-master-modify-multiple-shards" { DELETE FROM reference_copy; } -step "s1-master-apply-delete-command" { SELECT master_apply_delete_command('DELETE FROM reference_copy WHERE id <= 4;'); } step "s1-create-non-distributed-table" { CREATE TABLE reference_copy(id integer, data text, int_data int); COPY reference_copy FROM PROGRAM 'echo 0, a, 0 && echo 1, b, 1 && echo 2, c, 2 && echo 3, d, 3 && echo 4, e, 4' WITH CSV; } step "s1-distribute-table" { SELECT create_reference_table('reference_copy'); } step "s1-select-count" { SELECT COUNT(*) FROM reference_copy; } @@ -53,7 +51,6 @@ step "s1-commit" { COMMIT; } // session 2 session "s2" step "s2-copy" { COPY reference_copy FROM PROGRAM 'echo 5, f, 5 && echo 6, g, 6 && echo 7, h, 7 && echo 8, i, 8 && echo 9, j, 9' WITH CSV; } -step "s2-copy-additional-column" { COPY reference_copy FROM PROGRAM 'echo 5, f, 5, 5 && echo 6, g, 6, 6 && echo 7, h, 7, 7 && echo 8, i, 8, 8 && echo 9, j, 9, 9' WITH CSV; } step "s2-router-select" { SELECT * FROM reference_copy WHERE id = 1; } step "s2-real-time-select" { SELECT * FROM reference_copy ORDER BY 1, 2; } step "s2-adaptive-select" @@ -74,9 +71,6 @@ step "s2-ddl-drop-column" { ALTER TABLE reference_copy DROP new_column; } step "s2-ddl-rename-column" { ALTER TABLE reference_copy RENAME data TO new_column; } step "s2-table-size" { SELECT citus_total_relation_size('reference_copy'); } step "s2-master-modify-multiple-shards" { DELETE FROM reference_copy; } -step "s2-master-apply-delete-command" { SELECT master_apply_delete_command('DELETE FROM reference_copy WHERE id <= 4;'); } -step "s2-master-drop-all-shards" { SELECT master_drop_all_shards('reference_copy'::regclass, 'public', 'reference_copy'); } -step "s2-create-non-distributed-table" { CREATE TABLE reference_copy(id integer, data text, int_data int); COPY reference_copy FROM PROGRAM 'echo 0, a, 0 && echo 1, b, 1 && echo 2, c, 2 && echo 3, d, 3 && echo 4, e, 4' WITH CSV; } step "s2-distribute-table" { SELECT create_reference_table('reference_copy'); } // permutations - COPY vs COPY diff --git a/src/test/regress/spec/isolation_reference_on_mx.spec b/src/test/regress/spec/isolation_reference_on_mx.spec index 6b63e4162..787928b4f 100644 --- a/src/test/regress/spec/isolation_reference_on_mx.spec +++ b/src/test/regress/spec/isolation_reference_on_mx.spec @@ -101,11 +101,6 @@ step "s2-select-from-ref-table" SELECT run_commands_on_session_level_connection_to_node('SELECT count(*) FROM ref_table'); } -step "s2-delete-from-ref-table" -{ - SELECT run_commands_on_session_level_connection_to_node('DELETE FROM ref_table WHERE user_id = 2'); -} - step "s2-insert-into-ref-table" { SELECT run_commands_on_session_level_connection_to_node('INSERT INTO ref_table VALUES(8,81),(9,91)'); diff --git a/src/test/regress/spec/isolation_select_vs_all_on_mx.spec b/src/test/regress/spec/isolation_select_vs_all_on_mx.spec index c567d579f..5ce2d0cce 100644 --- a/src/test/regress/spec/isolation_select_vs_all_on_mx.spec +++ b/src/test/regress/spec/isolation_select_vs_all_on_mx.spec @@ -74,11 +74,6 @@ step "s2-select" SELECT run_commands_on_session_level_connection_to_node('SELECT * FROM select_table'); } -step "s2-insert" -{ - SELECT run_commands_on_session_level_connection_to_node('INSERT INTO select_table VALUES(6, 60)'); -} - step "s2-insert-select" { SELECT run_commands_on_session_level_connection_to_node('INSERT INTO select_table SELECT * FROM select_table'); diff --git a/src/test/regress/spec/isolation_truncate_vs_all.spec b/src/test/regress/spec/isolation_truncate_vs_all.spec index 901f8e6f8..b35fa3965 100644 --- a/src/test/regress/spec/isolation_truncate_vs_all.spec +++ b/src/test/regress/spec/isolation_truncate_vs_all.spec @@ -58,9 +58,7 @@ step "s2-table-size" { SELECT citus_total_relation_size('truncate_append'); } step "s2-master-modify-multiple-shards" { DELETE FROM truncate_append; } step "s2-master-apply-delete-command" { SELECT master_apply_delete_command('DELETE FROM truncate_append WHERE id <= 4;'); } step "s2-master-drop-all-shards" { SELECT master_drop_all_shards('truncate_append'::regclass, 'public', 'truncate_append'); } -step "s2-create-non-distributed-table" { CREATE TABLE truncate_append(id integer, data text); } step "s2-distribute-table" { SELECT create_distributed_table('truncate_append', 'id', 'append'); } -step "s2-select" { SELECT * FROM truncate_append ORDER BY 1, 2; } step "s2-commit" { COMMIT; } // permutations - TRUNCATE vs TRUNCATE diff --git a/src/test/regress/spec/isolation_update_delete_upsert_vs_all_on_mx.spec b/src/test/regress/spec/isolation_update_delete_upsert_vs_all_on_mx.spec index 5c473d2ed..b91b50390 100644 --- a/src/test/regress/spec/isolation_update_delete_upsert_vs_all_on_mx.spec +++ b/src/test/regress/spec/isolation_update_delete_upsert_vs_all_on_mx.spec @@ -83,11 +83,6 @@ step "s2-select-for-update" SELECT run_commands_on_session_level_connection_to_node('SELECT * FROM dist_table WHERE id=5 FOR UPDATE'); } -step "s2-coordinator-create-index-concurrently" -{ - CREATE INDEX CONCURRENTLY dist_table_index ON dist_table(id); -} - step "s2-commit-worker" { SELECT run_commands_on_session_level_connection_to_node('COMMIT'); diff --git a/src/test/regress/spec/isolation_update_node.spec b/src/test/regress/spec/isolation_update_node.spec index b48d5415c..feb21ffaf 100644 --- a/src/test/regress/spec/isolation_update_node.spec +++ b/src/test/regress/spec/isolation_update_node.spec @@ -27,14 +27,6 @@ step "s1-update-node-1" 58637); } -step "s1-update-node-2" -{ - SELECT 1 FROM master_update_node( - (select nodeid from pg_dist_node where nodeport = 57638), - 'localhost', - 58638); -} - step "s1-commit" { COMMIT; @@ -89,11 +81,6 @@ step "s2-abort" ABORT; } -step "s2-commit" -{ - COMMIT; -} - // session 1 updates node 1, session 2 updates node 2, should be ok permutation "s1-begin" "s1-update-node-1" "s2-update-node-2" "s1-commit" "s1-show-nodes" diff --git a/src/test/regress/spec/isolation_update_node_lock_writes.spec b/src/test/regress/spec/isolation_update_node_lock_writes.spec index 25d99b3e6..6915b6a46 100644 --- a/src/test/regress/spec/isolation_update_node_lock_writes.spec +++ b/src/test/regress/spec/isolation_update_node_lock_writes.spec @@ -36,11 +36,6 @@ step "s1-commit" COMMIT; } -step "s1-abort" -{ - ABORT; -} - session "s2" step "s2-begin" diff --git a/src/test/regress/spec/isolation_upsert_vs_all.spec b/src/test/regress/spec/isolation_upsert_vs_all.spec index a168ab79a..b9f0db869 100644 --- a/src/test/regress/spec/isolation_upsert_vs_all.spec +++ b/src/test/regress/spec/isolation_upsert_vs_all.spec @@ -38,7 +38,6 @@ step "s1-ddl-rename-column" { ALTER TABLE upsert_hash RENAME data TO new_column; step "s1-table-size" { SELECT citus_total_relation_size('upsert_hash'); } step "s1-master-modify-multiple-shards" { DELETE FROM upsert_hash; } step "s1-create-non-distributed-table" { CREATE TABLE upsert_hash(id integer PRIMARY KEY, data text); } -step "s1-distribute-table" { SELECT create_distributed_table('upsert_hash', 'id'); } step "s1-select-count" { SELECT COUNT(*) FROM upsert_hash; } step "s1-show-indexes" { SELECT run_command_on_workers('SELECT COUNT(*) FROM pg_indexes WHERE tablename LIKE ''upsert_hash%'''); } step "s1-show-columns" { SELECT run_command_on_workers('SELECT column_name FROM information_schema.columns WHERE table_name LIKE ''upsert_hash%'' AND column_name = ''new_column'' ORDER BY 1 LIMIT 1'); } diff --git a/src/test/regress/spec/isolation_validate_vs_insert.spec b/src/test/regress/spec/isolation_validate_vs_insert.spec index 05e9f81f5..e50ddb5eb 100644 --- a/src/test/regress/spec/isolation_validate_vs_insert.spec +++ b/src/test/regress/spec/isolation_validate_vs_insert.spec @@ -28,7 +28,6 @@ step "s1-commit" { COMMIT; } session "s2" step "s2-begin" { BEGIN; } step "s2-insert" { INSERT INTO constrained_table VALUES(10, 10); } -step "s2-insert-invalid" { INSERT INTO constrained_table VALUES(100, 100); } step "s2-select" { SELECT sum(int_data) FROM constrained_table; } step "s2-commit" { COMMIT; } From c5c9ec288f58cbc2ebc3234a12c6a6b78a34e984 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Wed, 10 Jun 2020 12:46:07 +0300 Subject: [PATCH 27/52] fix multi_mx_create_table test --- src/backend/distributed/commands/multi_copy.c | 2 ++ src/test/regress/sql/multi_mx_create_table.sql | 3 +-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/backend/distributed/commands/multi_copy.c b/src/backend/distributed/commands/multi_copy.c index 19df51152..650763739 100644 --- a/src/backend/distributed/commands/multi_copy.c +++ b/src/backend/distributed/commands/multi_copy.c @@ -93,7 +93,9 @@ #include "distributed/hash_helpers.h" #include "executor/executor.h" #include "foreign/foreign.h" +#if PG_VERSION_NUM >= PG_VERSION_13 #include "tcop/cmdtag.h" +#endif #include "libpq/libpq.h" #include "libpq/pqformat.h" #include "nodes/makefuncs.h" diff --git a/src/test/regress/sql/multi_mx_create_table.sql b/src/test/regress/sql/multi_mx_create_table.sql index cb5316f91..67ceab032 100644 --- a/src/test/regress/sql/multi_mx_create_table.sql +++ b/src/test/regress/sql/multi_mx_create_table.sql @@ -261,10 +261,9 @@ CREATE TABLE lineitem_mx ( l_shipmode char(10) not null, l_comment varchar(44) not null, PRIMARY KEY(l_orderkey, l_linenumber) ); +SET citus.shard_count TO 16; SELECT create_distributed_table('lineitem_mx', 'l_orderkey'); --- SET citus.shard_count TO 16; - CREATE INDEX lineitem_mx_time_index ON lineitem_mx (l_shipdate); CREATE TABLE orders_mx ( From 108a2972c238347b03c7b1357008e975b891343c Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Wed, 10 Jun 2020 23:14:00 +0300 Subject: [PATCH 28/52] Introduce a workaround for join aliases When there is a join alias, var->varnosync will point to the alias and var->varno will point to the table itself, but we need to use the alias when deparsing the query. Hence a workaround is introduced to solve this problem in ruleutils. Normally this case can be understood with dpns->plan == NULL check but in our case, dpns->plan is always NULL. We should sync our ruleutils at some point with postgres ruleutils. This could be a wrong solution as well but the tests pass. --- .../distributed/deparser/ruleutils_13.c | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/backend/distributed/deparser/ruleutils_13.c b/src/backend/distributed/deparser/ruleutils_13.c index bdd9e7f8e..08b502dba 100644 --- a/src/backend/distributed/deparser/ruleutils_13.c +++ b/src/backend/distributed/deparser/ruleutils_13.c @@ -16,6 +16,8 @@ */ #include "distributed/pg_version_constants.h" +#pragma GCC optimize ("O0") + #include "pg_config.h" #if (PG_VERSION_NUM >= PG_VERSION_13) && (PG_VERSION_NUM < PG_VERSION_14) @@ -3544,24 +3546,22 @@ get_variable(Var *var, int levelsup, bool istoplevel, deparse_context *context) var->varlevelsup, levelsup); dpns = (deparse_namespace *) list_nth(context->namespaces, netlevelsup); - /* - * If we have a syntactic referent for the Var, and we're working from a - * parse tree, prefer to use the syntactic referent. Otherwise, fall back - * on the semantic referent. (Forcing use of the semantic referent when - * printing plan trees is a design choice that's perhaps more motivated by - * backwards compatibility than anything else. But it does have the - * advantage of making plans more explicit.) - */ - // if (var->varnosyn > 0 && dpns->plan == NULL) - // { - // varno = var->varnosyn; - // varattno = var->varattnosyn; - // } - // else - // { - varno = var->varno; - varattno = var->varattno; - // } + + varno = var->varno; + varattno = var->varattno; + + + if (var->varnosyn > 0 && var->varnosyn <= list_length(dpns->rtable) && dpns->plan == NULL) { + rte = rt_fetch(var->varnosyn, dpns->rtable); + + // if the rte var->varnosync points to is not a regular table and it is a join + // then the correct relname will be found with var->varnosync and var->varattnosync + // TODO:: this is a workaround and it can be simplified. + if (rte->rtekind == RTE_JOIN && rte->relid == 0 && var->varnosyn != var->varno) { + varno = var->varnosyn; + varattno = var->varattnosyn; + } + } /* * Try to find the relevant RTE in this rtable. In a plan tree, it's From a34a1126ec24e6e7daa6f8e9b833e9bbbcd5b5fc Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Thu, 11 Jun 2020 09:38:34 +0300 Subject: [PATCH 29/52] add alternative output for pg13 in some tests --- .../multi_complex_count_distinct_1.out | 1128 ---------- .../regress/expected/multi_partitioning.out | 42 +- .../regress/expected/multi_partitioning_1.out | 1967 +++++++++++++++++ .../input/multi_complex_count_distinct.source | 8 - 4 files changed, 1988 insertions(+), 1157 deletions(-) delete mode 100644 src/test/regress/expected/multi_complex_count_distinct_1.out create mode 100644 src/test/regress/expected/multi_partitioning_1.out diff --git a/src/test/regress/expected/multi_complex_count_distinct_1.out b/src/test/regress/expected/multi_complex_count_distinct_1.out deleted file mode 100644 index b14e9c7e8..000000000 --- a/src/test/regress/expected/multi_complex_count_distinct_1.out +++ /dev/null @@ -1,1128 +0,0 @@ --- --- COMPLEX_COUNT_DISTINCT --- -SET citus.next_shard_id TO 240000; -SET citus.shard_count TO 8; -SET citus.shard_replication_factor TO 1; -SET citus.coordinator_aggregation_strategy TO 'disabled'; -CREATE TABLE lineitem_hash ( - l_orderkey bigint not null, - l_partkey integer not null, - l_suppkey integer not null, - l_linenumber integer not null, - l_quantity decimal(15, 2) not null, - l_extendedprice decimal(15, 2) not null, - l_discount decimal(15, 2) not null, - l_tax decimal(15, 2) not null, - l_returnflag char(1) not null, - l_linestatus char(1) not null, - l_shipdate date not null, - l_commitdate date not null, - l_receiptdate date not null, - l_shipinstruct char(25) not null, - l_shipmode char(10) not null, - l_comment varchar(44) not null, - PRIMARY KEY(l_orderkey, l_linenumber) ); -SELECT create_distributed_table('lineitem_hash', 'l_orderkey', 'hash'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -\copy lineitem_hash FROM '/home/talha/citus/src/test/regress/data/lineitem.1.data' with delimiter '|' -\copy lineitem_hash FROM '/home/talha/citus/src/test/regress/data/lineitem.2.data' with delimiter '|' -ANALYZE lineitem_hash; -SET citus.task_executor_type to "task-tracker"; --- count(distinct) is supported on top level query if there --- is a grouping on the partition key -SELECT - l_orderkey, count(DISTINCT l_partkey) - FROM lineitem_hash - GROUP BY l_orderkey - ORDER BY 2 DESC, 1 DESC - LIMIT 10; - l_orderkey | count ---------------------------------------------------------------------- - 14885 | 7 - 14884 | 7 - 14821 | 7 - 14790 | 7 - 14785 | 7 - 14755 | 7 - 14725 | 7 - 14694 | 7 - 14627 | 7 - 14624 | 7 -(10 rows) - -EXPLAIN (COSTS false, VERBOSE true) -SELECT - l_orderkey, count(DISTINCT l_partkey) - FROM lineitem_hash - GROUP BY l_orderkey - ORDER BY 2 DESC, 1 DESC - LIMIT 10; - QUERY PLAN ---------------------------------------------------------------------- - Limit - Output: remote_scan.l_orderkey, remote_scan.count - -> Sort - Output: remote_scan.l_orderkey, remote_scan.count - Sort Key: remote_scan.count DESC, remote_scan.l_orderkey DESC - -> Custom Scan (Citus Task-Tracker) - Output: remote_scan.l_orderkey, remote_scan.count - Task Count: 8 - Tasks Shown: One of 8 - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Limit - Output: l_orderkey, (count(DISTINCT l_partkey)) - -> Sort - Output: l_orderkey, (count(DISTINCT l_partkey)) - Sort Key: (count(DISTINCT lineitem_hash.l_partkey)) DESC, lineitem_hash.l_orderkey DESC - -> GroupAggregate - Output: l_orderkey, count(DISTINCT l_partkey) - Group Key: lineitem_hash.l_orderkey - -> Index Scan Backward using lineitem_hash_pkey_240000 on public.lineitem_hash_240000 lineitem_hash - Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment -(21 rows) - --- it is also supported if there is no grouping or grouping is on non-partition field -SELECT - count(DISTINCT l_partkey) - FROM lineitem_hash - ORDER BY 1 DESC - LIMIT 10; - count ---------------------------------------------------------------------- - 11661 -(1 row) - -EXPLAIN (COSTS false, VERBOSE true) -SELECT - count(DISTINCT l_partkey) - FROM lineitem_hash - ORDER BY 1 DESC - LIMIT 10; - QUERY PLAN ---------------------------------------------------------------------- - Limit - Output: (count(DISTINCT remote_scan.count)) - -> Sort - Output: (count(DISTINCT remote_scan.count)) - Sort Key: (count(DISTINCT remote_scan.count)) DESC - -> Aggregate - Output: count(DISTINCT remote_scan.count) - -> Custom Scan (Citus Task-Tracker) - Output: remote_scan.count - Task Count: 8 - Tasks Shown: One of 8 - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> HashAggregate - Output: l_partkey - Group Key: lineitem_hash.l_partkey - -> Seq Scan on public.lineitem_hash_240000 lineitem_hash - Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment -(18 rows) - -SELECT - l_shipmode, count(DISTINCT l_partkey) - FROM lineitem_hash - GROUP BY l_shipmode - ORDER BY 2 DESC, 1 DESC - LIMIT 10; - l_shipmode | count ---------------------------------------------------------------------- - TRUCK | 1757 - MAIL | 1730 - AIR | 1702 - FOB | 1700 - RAIL | 1696 - SHIP | 1684 - REG AIR | 1676 -(7 rows) - -EXPLAIN (COSTS false, VERBOSE true) -SELECT - l_shipmode, count(DISTINCT l_partkey) - FROM lineitem_hash - GROUP BY l_shipmode - ORDER BY 2 DESC, 1 DESC - LIMIT 10; - QUERY PLAN ---------------------------------------------------------------------- - Limit - Output: remote_scan.l_shipmode, (count(DISTINCT remote_scan.count)) - -> Sort - Output: remote_scan.l_shipmode, (count(DISTINCT remote_scan.count)) - Sort Key: (count(DISTINCT remote_scan.count)) DESC, remote_scan.l_shipmode DESC - -> GroupAggregate - Output: remote_scan.l_shipmode, count(DISTINCT remote_scan.count) - Group Key: remote_scan.l_shipmode - -> Sort - Output: remote_scan.l_shipmode, remote_scan.count - Sort Key: remote_scan.l_shipmode DESC - -> Custom Scan (Citus Task-Tracker) - Output: remote_scan.l_shipmode, remote_scan.count - Task Count: 8 - Tasks Shown: One of 8 - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> HashAggregate - Output: l_shipmode, l_partkey - Group Key: lineitem_hash.l_shipmode, lineitem_hash.l_partkey - -> Seq Scan on public.lineitem_hash_240000 lineitem_hash - Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment -(22 rows) - --- mixed mode count distinct, grouped by partition column -SELECT - l_orderkey, count(distinct l_partkey), count(distinct l_shipmode) - FROM lineitem_hash - GROUP BY l_orderkey - ORDER BY 3 DESC, 2 DESC, 1 - LIMIT 10; - l_orderkey | count | count ---------------------------------------------------------------------- - 226 | 7 | 7 - 1316 | 7 | 7 - 1477 | 7 | 7 - 3555 | 7 | 7 - 12258 | 7 | 7 - 12835 | 7 | 7 - 768 | 7 | 6 - 1121 | 7 | 6 - 1153 | 7 | 6 - 1281 | 7 | 6 -(10 rows) - -EXPLAIN (COSTS false, VERBOSE true) -SELECT - l_orderkey, count(distinct l_partkey), count(distinct l_shipmode) - FROM lineitem_hash - GROUP BY l_orderkey - ORDER BY 3 DESC, 2 DESC, 1 - LIMIT 10; - QUERY PLAN ---------------------------------------------------------------------- - Limit - Output: remote_scan.l_orderkey, remote_scan.count, remote_scan.count_1 - -> Sort - Output: remote_scan.l_orderkey, remote_scan.count, remote_scan.count_1 - Sort Key: remote_scan.count_1 DESC, remote_scan.count DESC, remote_scan.l_orderkey - -> Custom Scan (Citus Task-Tracker) - Output: remote_scan.l_orderkey, remote_scan.count, remote_scan.count_1 - Task Count: 8 - Tasks Shown: One of 8 - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Limit - Output: l_orderkey, (count(DISTINCT l_partkey)), (count(DISTINCT l_shipmode)) - -> Sort - Output: l_orderkey, (count(DISTINCT l_partkey)), (count(DISTINCT l_shipmode)) - Sort Key: (count(DISTINCT lineitem_hash.l_shipmode)) DESC, (count(DISTINCT lineitem_hash.l_partkey)) DESC, lineitem_hash.l_orderkey - -> GroupAggregate - Output: l_orderkey, count(DISTINCT l_partkey), count(DISTINCT l_shipmode) - Group Key: lineitem_hash.l_orderkey - -> Index Scan using lineitem_hash_pkey_240000 on public.lineitem_hash_240000 lineitem_hash - Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment -(21 rows) - --- partition/non-partition column count distinct no grouping -SELECT - count(distinct l_orderkey), count(distinct l_partkey), count(distinct l_shipmode) - FROM lineitem_hash; - count | count | count ---------------------------------------------------------------------- - 2985 | 11661 | 7 -(1 row) - -EXPLAIN (COSTS false, VERBOSE true) -SELECT - count(distinct l_orderkey), count(distinct l_partkey), count(distinct l_shipmode) - FROM lineitem_hash; - QUERY PLAN ---------------------------------------------------------------------- - Aggregate - Output: count(DISTINCT remote_scan.count), count(DISTINCT remote_scan.count_1), count(DISTINCT remote_scan.count_2) - -> Custom Scan (Citus Task-Tracker) - Output: remote_scan.count, remote_scan.count_1, remote_scan.count_2 - Task Count: 8 - Tasks Shown: One of 8 - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> HashAggregate - Output: l_orderkey, l_partkey, l_shipmode - Group Key: lineitem_hash.l_orderkey, lineitem_hash.l_partkey, lineitem_hash.l_shipmode - -> Seq Scan on public.lineitem_hash_240000 lineitem_hash - Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment -(13 rows) - --- distinct/non-distinct on partition and non-partition columns -SELECT - count(distinct l_orderkey), count(l_orderkey), - count(distinct l_partkey), count(l_partkey), - count(distinct l_shipmode), count(l_shipmode) - FROM lineitem_hash; - count | count | count | count | count | count ---------------------------------------------------------------------- - 2985 | 12000 | 11661 | 12000 | 7 | 12000 -(1 row) - --- mixed mode count distinct, grouped by non-partition column -SELECT - l_shipmode, count(distinct l_partkey), count(distinct l_orderkey) - FROM lineitem_hash - GROUP BY l_shipmode - ORDER BY 1, 2 DESC, 3 DESC; - l_shipmode | count | count ---------------------------------------------------------------------- - AIR | 1702 | 1327 - FOB | 1700 | 1276 - MAIL | 1730 | 1299 - RAIL | 1696 | 1265 - REG AIR | 1676 | 1275 - SHIP | 1684 | 1289 - TRUCK | 1757 | 1333 -(7 rows) - --- mixed mode count distinct, grouped by non-partition column --- having on partition column -SELECT - l_shipmode, count(distinct l_partkey), count(distinct l_orderkey) - FROM lineitem_hash - GROUP BY l_shipmode - HAVING count(distinct l_orderkey) > 1300 - ORDER BY 1, 2 DESC; - l_shipmode | count | count ---------------------------------------------------------------------- - AIR | 1702 | 1327 - TRUCK | 1757 | 1333 -(2 rows) - --- same but having clause is not on target list -SELECT - l_shipmode, count(distinct l_partkey) - FROM lineitem_hash - GROUP BY l_shipmode - HAVING count(distinct l_orderkey) > 1300 - ORDER BY 1, 2 DESC; - l_shipmode | count ---------------------------------------------------------------------- - AIR | 1702 - TRUCK | 1757 -(2 rows) - --- mixed mode count distinct, grouped by non-partition column --- having on non-partition column -SELECT - l_shipmode, count(distinct l_partkey), count(distinct l_suppkey) - FROM lineitem_hash - GROUP BY l_shipmode - HAVING count(distinct l_suppkey) > 1550 - ORDER BY 1, 2 DESC; - l_shipmode | count | count ---------------------------------------------------------------------- - AIR | 1702 | 1564 - FOB | 1700 | 1571 - MAIL | 1730 | 1573 - RAIL | 1696 | 1581 - REG AIR | 1676 | 1557 - SHIP | 1684 | 1554 - TRUCK | 1757 | 1602 -(7 rows) - --- same but having clause is not on target list -SELECT - l_shipmode, count(distinct l_partkey) - FROM lineitem_hash - GROUP BY l_shipmode - HAVING count(distinct l_suppkey) > 1550 - ORDER BY 1, 2 DESC; - l_shipmode | count ---------------------------------------------------------------------- - AIR | 1702 - FOB | 1700 - MAIL | 1730 - RAIL | 1696 - REG AIR | 1676 - SHIP | 1684 - TRUCK | 1757 -(7 rows) - -EXPLAIN (COSTS false, VERBOSE true) -SELECT - l_shipmode, count(distinct l_partkey) - FROM lineitem_hash - GROUP BY l_shipmode - HAVING count(distinct l_suppkey) > 1550 - ORDER BY 1, 2 DESC; - QUERY PLAN ---------------------------------------------------------------------- - Sort - Output: remote_scan.l_shipmode, (count(DISTINCT remote_scan.count)) - Sort Key: remote_scan.l_shipmode, (count(DISTINCT remote_scan.count)) DESC - -> GroupAggregate - Output: remote_scan.l_shipmode, count(DISTINCT remote_scan.count) - Group Key: remote_scan.l_shipmode - Filter: (count(DISTINCT remote_scan.worker_column_3) > 1550) - -> Sort - Output: remote_scan.l_shipmode, remote_scan.count, remote_scan.worker_column_3 - Sort Key: remote_scan.l_shipmode - -> Custom Scan (Citus Task-Tracker) - Output: remote_scan.l_shipmode, remote_scan.count, remote_scan.worker_column_3 - Task Count: 8 - Tasks Shown: One of 8 - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> HashAggregate - Output: l_shipmode, l_partkey, l_suppkey - Group Key: lineitem_hash.l_shipmode, lineitem_hash.l_partkey, lineitem_hash.l_suppkey - -> Seq Scan on public.lineitem_hash_240000 lineitem_hash - Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment -(21 rows) - --- count distinct is supported on single table subqueries -SELECT * - FROM ( - SELECT - l_orderkey, count(DISTINCT l_partkey) - FROM lineitem_hash - GROUP BY l_orderkey) sub - ORDER BY 2 DESC, 1 DESC - LIMIT 10; - l_orderkey | count ---------------------------------------------------------------------- - 14885 | 7 - 14884 | 7 - 14821 | 7 - 14790 | 7 - 14785 | 7 - 14755 | 7 - 14725 | 7 - 14694 | 7 - 14627 | 7 - 14624 | 7 -(10 rows) - -SELECT * - FROM ( - SELECT - l_partkey, count(DISTINCT l_orderkey) - FROM lineitem_hash - GROUP BY l_partkey) sub - ORDER BY 2 DESC, 1 DESC - LIMIT 10; - l_partkey | count ---------------------------------------------------------------------- - 199146 | 3 - 188804 | 3 - 177771 | 3 - 160895 | 3 - 149926 | 3 - 136884 | 3 - 87761 | 3 - 15283 | 3 - 6983 | 3 - 1927 | 3 -(10 rows) - -EXPLAIN (COSTS false, VERBOSE true) -SELECT * - FROM ( - SELECT - l_partkey, count(DISTINCT l_orderkey) - FROM lineitem_hash - GROUP BY l_partkey) sub - ORDER BY 2 DESC, 1 DESC - LIMIT 10; - QUERY PLAN ---------------------------------------------------------------------- - Limit - Output: remote_scan.l_partkey, remote_scan.count - -> Sort - Output: remote_scan.l_partkey, remote_scan.count - Sort Key: remote_scan.count DESC, remote_scan.l_partkey DESC - -> Custom Scan (Citus Task-Tracker) - Output: remote_scan.l_partkey, remote_scan.count - Task Count: 4 - Tasks Shown: None, not supported for re-partition queries - -> MapMergeJob - Map Task Count: 8 - Merge Task Count: 4 -(12 rows) - --- count distinct with filters -SELECT - l_orderkey, - count(DISTINCT l_suppkey) FILTER (WHERE l_shipmode = 'AIR'), - count(DISTINCT l_suppkey) - FROM lineitem_hash - GROUP BY l_orderkey - ORDER BY 2 DESC, 3 DESC, 1 - LIMIT 10; - l_orderkey | count | count ---------------------------------------------------------------------- - 4964 | 4 | 7 - 12005 | 4 | 7 - 5409 | 4 | 6 - 164 | 3 | 7 - 322 | 3 | 7 - 871 | 3 | 7 - 1156 | 3 | 7 - 1574 | 3 | 7 - 2054 | 3 | 7 - 2309 | 3 | 7 -(10 rows) - -EXPLAIN (COSTS false, VERBOSE true) -SELECT - l_orderkey, - count(DISTINCT l_suppkey) FILTER (WHERE l_shipmode = 'AIR'), - count(DISTINCT l_suppkey) - FROM lineitem_hash - GROUP BY l_orderkey - ORDER BY 2 DESC, 3 DESC, 1 - LIMIT 10; - QUERY PLAN ---------------------------------------------------------------------- - Limit - Output: remote_scan.l_orderkey, remote_scan.count, remote_scan.count_1 - -> Sort - Output: remote_scan.l_orderkey, remote_scan.count, remote_scan.count_1 - Sort Key: remote_scan.count DESC, remote_scan.count_1 DESC, remote_scan.l_orderkey - -> Custom Scan (Citus Task-Tracker) - Output: remote_scan.l_orderkey, remote_scan.count, remote_scan.count_1 - Task Count: 8 - Tasks Shown: One of 8 - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Limit - Output: l_orderkey, (count(DISTINCT l_suppkey) FILTER (WHERE (l_shipmode = 'AIR'::bpchar))), (count(DISTINCT l_suppkey)) - -> Sort - Output: l_orderkey, (count(DISTINCT l_suppkey) FILTER (WHERE (l_shipmode = 'AIR'::bpchar))), (count(DISTINCT l_suppkey)) - Sort Key: (count(DISTINCT lineitem_hash.l_suppkey) FILTER (WHERE (lineitem_hash.l_shipmode = 'AIR'::bpchar))) DESC, (count(DISTINCT lineitem_hash.l_suppkey)) DESC, lineitem_hash.l_orderkey - -> GroupAggregate - Output: l_orderkey, count(DISTINCT l_suppkey) FILTER (WHERE (l_shipmode = 'AIR'::bpchar)), count(DISTINCT l_suppkey) - Group Key: lineitem_hash.l_orderkey - -> Index Scan using lineitem_hash_pkey_240000 on public.lineitem_hash_240000 lineitem_hash - Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment -(21 rows) - --- group by on non-partition column -SELECT - l_suppkey, count(DISTINCT l_partkey) FILTER (WHERE l_shipmode = 'AIR') - FROM lineitem_hash - GROUP BY l_suppkey - ORDER BY 2 DESC, 1 DESC - LIMIT 10; - l_suppkey | count ---------------------------------------------------------------------- - 7680 | 4 - 7703 | 3 - 7542 | 3 - 7072 | 3 - 6335 | 3 - 5873 | 3 - 1318 | 3 - 1042 | 3 - 160 | 3 - 9872 | 2 -(10 rows) - --- explaining the same query fails -EXPLAIN (COSTS false, VERBOSE true) -SELECT - l_suppkey, count(DISTINCT l_partkey) FILTER (WHERE l_shipmode = 'AIR') - FROM lineitem_hash - GROUP BY l_suppkey - ORDER BY 2 DESC, 1 DESC - LIMIT 10; - QUERY PLAN ---------------------------------------------------------------------- - Limit - Output: remote_scan.l_suppkey, (count(DISTINCT remote_scan.count) FILTER (WHERE (remote_scan.count_1 = 'AIR'::bpchar))) - -> Sort - Output: remote_scan.l_suppkey, (count(DISTINCT remote_scan.count) FILTER (WHERE (remote_scan.count_1 = 'AIR'::bpchar))) - Sort Key: (count(DISTINCT remote_scan.count) FILTER (WHERE (remote_scan.count_1 = 'AIR'::bpchar))) DESC, remote_scan.l_suppkey DESC - -> GroupAggregate - Output: remote_scan.l_suppkey, count(DISTINCT remote_scan.count) FILTER (WHERE (remote_scan.count_1 = 'AIR'::bpchar)) - Group Key: remote_scan.l_suppkey - -> Sort - Output: remote_scan.l_suppkey, remote_scan.count, remote_scan.count_1 - Sort Key: remote_scan.l_suppkey DESC - -> Custom Scan (Citus Task-Tracker) - Output: remote_scan.l_suppkey, remote_scan.count, remote_scan.count_1 - Task Count: 8 - Tasks Shown: One of 8 - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> HashAggregate - Output: l_suppkey, l_partkey, l_shipmode - Group Key: lineitem_hash.l_suppkey, lineitem_hash.l_partkey, lineitem_hash.l_shipmode - -> Seq Scan on public.lineitem_hash_240000 lineitem_hash - Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment -(22 rows) - --- without group by, on partition column -SELECT - count(DISTINCT l_orderkey) FILTER (WHERE l_shipmode = 'AIR') - FROM lineitem_hash; - count ---------------------------------------------------------------------- - 1327 -(1 row) - --- without group by, on non-partition column -SELECT - count(DISTINCT l_partkey) FILTER (WHERE l_shipmode = 'AIR') - FROM lineitem_hash; - count ---------------------------------------------------------------------- - 1702 -(1 row) - -SELECT - count(DISTINCT l_partkey) FILTER (WHERE l_shipmode = 'AIR'), - count(DISTINCT l_partkey), - count(DISTINCT l_shipdate) - FROM lineitem_hash; - count | count | count ---------------------------------------------------------------------- - 1702 | 11661 | 2470 -(1 row) - --- filter column already exists in target list -SELECT * - FROM ( - SELECT - l_orderkey, count(DISTINCT l_partkey) FILTER (WHERE l_orderkey > 100) - FROM lineitem_hash - GROUP BY l_orderkey) sub - ORDER BY 2 DESC, 1 DESC - LIMIT 10; - l_orderkey | count ---------------------------------------------------------------------- - 14885 | 7 - 14884 | 7 - 14821 | 7 - 14790 | 7 - 14785 | 7 - 14755 | 7 - 14725 | 7 - 14694 | 7 - 14627 | 7 - 14624 | 7 -(10 rows) - --- filter column does not exist in target list -SELECT * - FROM ( - SELECT - l_orderkey, count(DISTINCT l_partkey) FILTER (WHERE l_shipmode = 'AIR') - FROM lineitem_hash - GROUP BY l_orderkey) sub - ORDER BY 2 DESC, 1 DESC - LIMIT 10; - l_orderkey | count ---------------------------------------------------------------------- - 12005 | 4 - 5409 | 4 - 4964 | 4 - 14848 | 3 - 14496 | 3 - 13473 | 3 - 13122 | 3 - 12929 | 3 - 12645 | 3 - 12417 | 3 -(10 rows) - --- case expr in count distinct is supported. --- count orders partkeys if l_shipmode is air -SELECT * - FROM ( - SELECT - l_orderkey, count(DISTINCT CASE WHEN l_shipmode = 'AIR' THEN l_partkey ELSE NULL END) as count - FROM lineitem_hash - GROUP BY l_orderkey) sub - WHERE count > 0 - ORDER BY 2 DESC, 1 DESC - LIMIT 10; - l_orderkey | count ---------------------------------------------------------------------- - 12005 | 4 - 5409 | 4 - 4964 | 4 - 14848 | 3 - 14496 | 3 - 13473 | 3 - 13122 | 3 - 12929 | 3 - 12645 | 3 - 12417 | 3 -(10 rows) - --- text like operator is also supported -SELECT * - FROM ( - SELECT - l_orderkey, count(DISTINCT CASE WHEN l_shipmode like '%A%' THEN l_partkey ELSE NULL END) as count - FROM lineitem_hash - GROUP BY l_orderkey) sub - WHERE count > 0 - ORDER BY 2 DESC, 1 DESC - LIMIT 10; - l_orderkey | count ---------------------------------------------------------------------- - 14275 | 7 - 14181 | 7 - 13605 | 7 - 12707 | 7 - 12384 | 7 - 11746 | 7 - 10727 | 7 - 10467 | 7 - 5636 | 7 - 4614 | 7 -(10 rows) - --- count distinct is rejected if it does not reference any columns -SELECT * - FROM ( - SELECT - l_linenumber, count(DISTINCT 1) - FROM lineitem_hash - GROUP BY l_linenumber) sub - ORDER BY 2 DESC, 1 DESC - LIMIT 10; -ERROR: cannot compute aggregate (distinct) -DETAIL: aggregate (distinct) with no columns is unsupported -HINT: You can load the hll extension from contrib packages and enable distinct approximations. --- count distinct is rejected if it does not reference any columns -SELECT * - FROM ( - SELECT - l_linenumber, count(DISTINCT (random() * 5)::int) - FROM lineitem_hash - GROUP BY l_linenumber) sub - ORDER BY 2 DESC, 1 DESC - LIMIT 10; -ERROR: cannot compute aggregate (distinct) -DETAIL: aggregate (distinct) with no columns is unsupported -HINT: You can load the hll extension from contrib packages and enable distinct approximations. --- even non-const function calls are supported within count distinct -SELECT * - FROM ( - SELECT - l_orderkey, count(DISTINCT (random() * 5)::int = l_linenumber) - FROM lineitem_hash - GROUP BY l_orderkey) sub - ORDER BY 2 DESC, 1 DESC - LIMIT 0; - l_orderkey | count ---------------------------------------------------------------------- -(0 rows) - --- multiple nested subquery -SELECT - total, - avg(avg_count) as total_avg_count - FROM ( - SELECT - number_sum, - count(DISTINCT l_suppkey) as total, - avg(total_count) avg_count - FROM ( - SELECT - l_suppkey, - sum(l_linenumber) as number_sum, - count(DISTINCT l_shipmode) as total_count - FROM - lineitem_hash - WHERE - l_partkey > 100 and - l_quantity > 2 and - l_orderkey < 10000 - GROUP BY - l_suppkey) as distributed_table - WHERE - number_sum >= 10 - GROUP BY - number_sum) as distributed_table_2 - GROUP BY - total - ORDER BY - total_avg_count DESC; - total | total_avg_count ---------------------------------------------------------------------- - 1 | 3.6000000000000000 - 6 | 2.8333333333333333 - 10 | 2.6000000000000000 - 27 | 2.5555555555555556 - 32 | 2.4687500000000000 - 77 | 2.1948051948051948 - 57 | 2.1754385964912281 -(7 rows) - --- multiple cases query -SELECT * - FROM ( - SELECT - count(DISTINCT - CASE - WHEN l_shipmode = 'TRUCK' THEN l_partkey - WHEN l_shipmode = 'AIR' THEN l_quantity - WHEN l_shipmode = 'SHIP' THEN l_discount - ELSE l_suppkey - END) as count, - l_shipdate - FROM - lineitem_hash - GROUP BY - l_shipdate) sub - WHERE - count > 0 - ORDER BY - 1 DESC, 2 DESC - LIMIT 10; - count | l_shipdate ---------------------------------------------------------------------- - 14 | 07-30-1997 - 13 | 05-26-1998 - 13 | 08-08-1997 - 13 | 11-17-1995 - 13 | 01-09-1993 - 12 | 01-15-1998 - 12 | 10-15-1997 - 12 | 09-07-1997 - 12 | 06-02-1997 - 12 | 03-14-1997 -(10 rows) - --- count DISTINCT expression -SELECT * - FROM ( - SELECT - l_quantity, count(DISTINCT ((l_orderkey / 1000) * 1000 )) as count - FROM - lineitem_hash - GROUP BY - l_quantity) sub - WHERE - count > 0 - ORDER BY - 2 DESC, 1 DESC - LIMIT 10; - l_quantity | count ---------------------------------------------------------------------- - 48.00 | 13 - 47.00 | 13 - 37.00 | 13 - 33.00 | 13 - 26.00 | 13 - 25.00 | 13 - 23.00 | 13 - 21.00 | 13 - 15.00 | 13 - 12.00 | 13 -(10 rows) - --- count DISTINCT is part of an expression which includes another aggregate -SELECT * - FROM ( - SELECT - sum(((l_partkey * l_tax) / 100)) / - count(DISTINCT - CASE - WHEN l_shipmode = 'TRUCK' THEN l_partkey - ELSE l_suppkey - END) as avg, - l_shipmode - FROM - lineitem_hash - GROUP BY - l_shipmode) sub - ORDER BY - 1 DESC, 2 DESC - LIMIT 10; - avg | l_shipmode ---------------------------------------------------------------------- - 44.82904609027336300064 | MAIL - 44.80704536679536679537 | SHIP - 44.68891732736572890026 | AIR - 44.34106724470134874759 | REG AIR - 43.12739987269255251432 | FOB - 43.07299253636938646426 | RAIL - 40.50298377916903813318 | TRUCK -(7 rows) - --- count DISTINCT CASE WHEN expression -SELECT * - FROM ( - SELECT - count(DISTINCT - CASE - WHEN l_shipmode = 'TRUCK' THEN l_linenumber - WHEN l_shipmode = 'AIR' THEN l_linenumber + 10 - ELSE 2 - END) as avg - FROM - lineitem_hash - GROUP BY l_shipdate) sub - ORDER BY 1 DESC - LIMIT 10; - avg ---------------------------------------------------------------------- - 7 - 6 - 6 - 6 - 6 - 6 - 6 - 6 - 5 - 5 -(10 rows) - --- COUNT DISTINCT (c1, c2) -SELECT * - FROM - (SELECT - l_shipmode, - count(DISTINCT (l_shipdate, l_tax)) - FROM - lineitem_hash - GROUP BY - l_shipmode) t - ORDER BY - 2 DESC,1 DESC - LIMIT 10; - l_shipmode | count ---------------------------------------------------------------------- - TRUCK | 1689 - MAIL | 1683 - FOB | 1655 - AIR | 1650 - SHIP | 1644 - RAIL | 1636 - REG AIR | 1607 -(7 rows) - --- distinct on non-var (type cast/field select) columns are also --- supported if grouped on distribution column --- random is added to prevent flattening by postgresql -SELECT - l_orderkey, count(a::int), count(distinct a::int) - FROM ( - SELECT l_orderkey, l_orderkey * 1.5 a, random() b - FROM lineitem_hash) sub - GROUP BY 1 - ORDER BY 1 DESC - LIMIT 5; - l_orderkey | count | count ---------------------------------------------------------------------- - 14947 | 2 | 1 - 14946 | 2 | 1 - 14945 | 6 | 1 - 14944 | 2 | 1 - 14919 | 1 | 1 -(5 rows) - -SELECT user_id, - count(sub.a::int), - count(DISTINCT sub.a::int), - count(DISTINCT (sub).a) -FROM - (SELECT user_id, - unnest(ARRAY[user_id * 1.5])a, - random() b - FROM users_table - ) sub -GROUP BY 1 -ORDER BY 1 DESC -LIMIT 5; - user_id | count | count | count ---------------------------------------------------------------------- - 6 | 11 | 1 | 1 - 5 | 27 | 1 | 1 - 4 | 24 | 1 | 1 - 3 | 18 | 1 | 1 - 2 | 19 | 1 | 1 -(5 rows) - -CREATE TYPE test_item AS -( - id INTEGER, - duration INTEGER -); -CREATE TABLE test_count_distinct_array (key int, value int , value_arr test_item[]); -SELECT create_distributed_table('test_count_distinct_array', 'key'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO test_count_distinct_array SELECT i, i, ARRAY[(i,i)::test_item] FROM generate_Series(0, 1000) i; -SELECT - key, - count(DISTINCT value), - count(DISTINCT (item)."id"), - count(DISTINCT (item)."id" * 3) -FROM - ( - SELECT key, unnest(value_arr) as item, value FROM test_count_distinct_array - ) as sub -GROUP BY 1 -ORDER BY 1 DESC -LIMIT 5; - key | count | count | count ---------------------------------------------------------------------- - 1000 | 1 | 1 | 1 - 999 | 1 | 1 | 1 - 998 | 1 | 1 | 1 - 997 | 1 | 1 | 1 - 996 | 1 | 1 | 1 -(5 rows) - -DROP TABLE test_count_distinct_array; -DROP TYPE test_item; --- other distinct aggregate are not supported -SELECT * - FROM ( - SELECT - l_linenumber, sum(DISTINCT l_partkey) - FROM lineitem_hash - GROUP BY l_linenumber) sub - ORDER BY 2 DESC, 1 DESC - LIMIT 10; -ERROR: cannot compute aggregate (distinct) -DETAIL: Only count(distinct) aggregate is supported in subqueries -SELECT * - FROM ( - SELECT - l_linenumber, avg(DISTINCT l_partkey) - FROM lineitem_hash - GROUP BY l_linenumber) sub - ORDER BY 2 DESC, 1 DESC - LIMIT 10; -ERROR: cannot compute aggregate (distinct) -DETAIL: Only count(distinct) aggregate is supported in subqueries --- whole row references, oid, and ctid are not supported in count distinct --- test table does not have oid or ctid enabled, so tests for them are skipped -SELECT * - FROM ( - SELECT - l_linenumber, count(DISTINCT lineitem_hash) - FROM lineitem_hash - GROUP BY l_linenumber) sub - ORDER BY 2 DESC, 1 DESC - LIMIT 10; -ERROR: cannot compute count (distinct) -DETAIL: Non-column references are not supported yet -SELECT * - FROM ( - SELECT - l_linenumber, count(DISTINCT lineitem_hash.*) - FROM lineitem_hash - GROUP BY l_linenumber) sub - ORDER BY 2 DESC, 1 DESC - LIMIT 10; -ERROR: cannot compute count (distinct) -DETAIL: Non-column references are not supported yet --- count distinct pushdown is enabled -SELECT * - FROM ( - SELECT - l_shipdate, - count(DISTINCT - CASE - WHEN l_shipmode = 'TRUCK' THEN l_partkey - ELSE NULL - END) as distinct_part, - extract(year from l_shipdate) as year - FROM - lineitem_hash - GROUP BY l_shipdate, year) sub - WHERE year = 1995 - ORDER BY 2 DESC, 1 - LIMIT 10; - l_shipdate | distinct_part | year ---------------------------------------------------------------------- - 11-29-1995 | 5 | 1995 - 03-24-1995 | 4 | 1995 - 09-18-1995 | 4 | 1995 - 01-17-1995 | 3 | 1995 - 04-02-1995 | 3 | 1995 - 05-23-1995 | 3 | 1995 - 08-11-1995 | 3 | 1995 - 09-27-1995 | 3 | 1995 - 10-27-1995 | 3 | 1995 - 10-30-1995 | 3 | 1995 -(10 rows) - -RESET citus.task_executor_type; --- count distinct pushdown is enabled -SELECT * - FROM ( - SELECT - l_shipdate, - count(DISTINCT - CASE - WHEN l_shipmode = 'TRUCK' THEN l_partkey - ELSE NULL - END) as distinct_part, - extract(year from l_shipdate) as year - FROM - lineitem_hash - GROUP BY l_shipdate, year) sub - WHERE year = 1995 - ORDER BY 2 DESC, 1 - LIMIT 10; - l_shipdate | distinct_part | year ---------------------------------------------------------------------- - 11-29-1995 | 5 | 1995 - 03-24-1995 | 4 | 1995 - 09-18-1995 | 4 | 1995 - 01-17-1995 | 3 | 1995 - 04-02-1995 | 3 | 1995 - 05-23-1995 | 3 | 1995 - 08-11-1995 | 3 | 1995 - 09-27-1995 | 3 | 1995 - 10-27-1995 | 3 | 1995 - 10-30-1995 | 3 | 1995 -(10 rows) - -SELECT * - FROM ( - SELECT - l_shipdate, - count(DISTINCT - CASE - WHEN l_shipmode = 'TRUCK' THEN l_partkey - ELSE NULL - END) as distinct_part, - extract(year from l_shipdate) as year - FROM - lineitem_hash - GROUP BY l_shipdate) sub - WHERE year = 1995 - ORDER BY 2 DESC, 1 - LIMIT 10; - l_shipdate | distinct_part | year ---------------------------------------------------------------------- - 11-29-1995 | 5 | 1995 - 03-24-1995 | 4 | 1995 - 09-18-1995 | 4 | 1995 - 01-17-1995 | 3 | 1995 - 04-02-1995 | 3 | 1995 - 05-23-1995 | 3 | 1995 - 08-11-1995 | 3 | 1995 - 09-27-1995 | 3 | 1995 - 10-27-1995 | 3 | 1995 - 10-30-1995 | 3 | 1995 -(10 rows) - -DROP TABLE lineitem_hash; diff --git a/src/test/regress/expected/multi_partitioning.out b/src/test/regress/expected/multi_partitioning.out index e85a4a2a9..66dbcfc94 100644 --- a/src/test/regress/expected/multi_partitioning.out +++ b/src/test/regress/expected/multi_partitioning.out @@ -1613,14 +1613,14 @@ SELECT * FROM partitioning_hash_test JOIN partitioning_hash_join_test USING (id, -> Hash Join Hash Cond: ((partitioning_hash_join_test.id = partitioning_hash_test.id) AND (partitioning_hash_join_test.subid = partitioning_hash_test.subid)) -> Append - -> Seq Scan on partitioning_hash_join_test_0_1660133 partitioning_hash_join_test - -> Seq Scan on partitioning_hash_join_test_1_1660137 partitioning_hash_join_test_1 - -> Seq Scan on partitioning_hash_join_test_2_1660141 partitioning_hash_join_test_2 + -> Seq Scan on partitioning_hash_join_test_0_1660133 partitioning_hash_join_test_1 + -> Seq Scan on partitioning_hash_join_test_1_1660137 partitioning_hash_join_test_2 + -> Seq Scan on partitioning_hash_join_test_2_1660141 partitioning_hash_join_test_3 -> Hash -> Append - -> Seq Scan on partitioning_hash_test_0_1660016 partitioning_hash_test - -> Seq Scan on partitioning_hash_test_1_1660020 partitioning_hash_test_1 - -> Seq Scan on partitioning_hash_test_2_1660032 partitioning_hash_test_2 + -> Seq Scan on partitioning_hash_test_0_1660016 partitioning_hash_test_1 + -> Seq Scan on partitioning_hash_test_1_1660020 partitioning_hash_test_2 + -> Seq Scan on partitioning_hash_test_2_1660032 partitioning_hash_test_3 (16 rows) -- set partition-wise join on and parallel to off @@ -1651,20 +1651,20 @@ SELECT * FROM partitioning_hash_test JOIN partitioning_hash_join_test USING (id, Node: host=localhost port=xxxxx dbname=regression -> Append -> Hash Join - Hash Cond: ((partitioning_hash_join_test.id = partitioning_hash_test.id) AND (partitioning_hash_join_test.subid = partitioning_hash_test.subid)) - -> Seq Scan on partitioning_hash_join_test_0_1660133 partitioning_hash_join_test + Hash Cond: ((partitioning_hash_join_test_1.id = partitioning_hash_test_1.id) AND (partitioning_hash_join_test_1.subid = partitioning_hash_test_1.subid)) + -> Seq Scan on partitioning_hash_join_test_0_1660133 partitioning_hash_join_test_1 -> Hash - -> Seq Scan on partitioning_hash_test_0_1660016 partitioning_hash_test + -> Seq Scan on partitioning_hash_test_0_1660016 partitioning_hash_test_1 -> Hash Join - Hash Cond: ((partitioning_hash_test_1.id = partitioning_hash_join_test_1.id) AND (partitioning_hash_test_1.subid = partitioning_hash_join_test_1.subid)) - -> Seq Scan on partitioning_hash_test_1_1660020 partitioning_hash_test_1 + Hash Cond: ((partitioning_hash_test_2.id = partitioning_hash_join_test_2.id) AND (partitioning_hash_test_2.subid = partitioning_hash_join_test_2.subid)) + -> Seq Scan on partitioning_hash_test_1_1660020 partitioning_hash_test_2 -> Hash - -> Seq Scan on partitioning_hash_join_test_1_1660137 partitioning_hash_join_test_1 + -> Seq Scan on partitioning_hash_join_test_1_1660137 partitioning_hash_join_test_2 -> Hash Join - Hash Cond: ((partitioning_hash_join_test_2.id = partitioning_hash_test_2.id) AND (partitioning_hash_join_test_2.subid = partitioning_hash_test_2.subid)) - -> Seq Scan on partitioning_hash_join_test_2_1660141 partitioning_hash_join_test_2 + Hash Cond: ((partitioning_hash_join_test_3.id = partitioning_hash_test_3.id) AND (partitioning_hash_join_test_3.subid = partitioning_hash_test_3.subid)) + -> Seq Scan on partitioning_hash_join_test_2_1660141 partitioning_hash_join_test_3 -> Hash - -> Seq Scan on partitioning_hash_test_2_1660032 partitioning_hash_test_2 + -> Seq Scan on partitioning_hash_test_2_1660032 partitioning_hash_test_3 (21 rows) -- note that partition-wise joins only work when partition key is in the join @@ -1682,14 +1682,14 @@ SELECT * FROM partitioning_hash_test JOIN partitioning_hash_join_test USING (id) -> Hash Join Hash Cond: (partitioning_hash_join_test.id = partitioning_hash_test.id) -> Append - -> Seq Scan on partitioning_hash_join_test_0_1660133 partitioning_hash_join_test - -> Seq Scan on partitioning_hash_join_test_1_1660137 partitioning_hash_join_test_1 - -> Seq Scan on partitioning_hash_join_test_2_1660141 partitioning_hash_join_test_2 + -> Seq Scan on partitioning_hash_join_test_0_1660133 partitioning_hash_join_test_1 + -> Seq Scan on partitioning_hash_join_test_1_1660137 partitioning_hash_join_test_2 + -> Seq Scan on partitioning_hash_join_test_2_1660141 partitioning_hash_join_test_3 -> Hash -> Append - -> Seq Scan on partitioning_hash_test_0_1660016 partitioning_hash_test - -> Seq Scan on partitioning_hash_test_1_1660020 partitioning_hash_test_1 - -> Seq Scan on partitioning_hash_test_2_1660032 partitioning_hash_test_2 + -> Seq Scan on partitioning_hash_test_0_1660016 partitioning_hash_test_1 + -> Seq Scan on partitioning_hash_test_1_1660020 partitioning_hash_test_2 + -> Seq Scan on partitioning_hash_test_2_1660032 partitioning_hash_test_3 (16 rows) -- reset partition-wise join diff --git a/src/test/regress/expected/multi_partitioning_1.out b/src/test/regress/expected/multi_partitioning_1.out new file mode 100644 index 000000000..18691286c --- /dev/null +++ b/src/test/regress/expected/multi_partitioning_1.out @@ -0,0 +1,1967 @@ +-- +-- Distributed Partitioned Table Tests +-- +SET citus.next_shard_id TO 1660000; +SET citus.shard_count TO 4; +SET citus.shard_replication_factor TO 1; +-- +-- Distributed Partitioned Table Creation Tests +-- +-- 1-) Distributing partitioned table +-- create partitioned table +CREATE TABLE partitioning_test(id int, time date) PARTITION BY RANGE (time); +CREATE TABLE partitioning_hash_test(id int, subid int) PARTITION BY HASH(subid); +-- create its partitions +CREATE TABLE partitioning_test_2009 PARTITION OF partitioning_test FOR VALUES FROM ('2009-01-01') TO ('2010-01-01'); +CREATE TABLE partitioning_test_2010 PARTITION OF partitioning_test FOR VALUES FROM ('2010-01-01') TO ('2011-01-01'); +CREATE TABLE partitioning_hash_test_0 PARTITION OF partitioning_hash_test FOR VALUES WITH (MODULUS 3, REMAINDER 0); +CREATE TABLE partitioning_hash_test_1 PARTITION OF partitioning_hash_test FOR VALUES WITH (MODULUS 3, REMAINDER 1); +-- load some data and distribute tables +INSERT INTO partitioning_test VALUES (1, '2009-06-06'); +INSERT INTO partitioning_test VALUES (2, '2010-07-07'); +INSERT INTO partitioning_test_2009 VALUES (3, '2009-09-09'); +INSERT INTO partitioning_test_2010 VALUES (4, '2010-03-03'); +INSERT INTO partitioning_hash_test VALUES (1, 2); +INSERT INTO partitioning_hash_test VALUES (2, 13); +INSERT INTO partitioning_hash_test VALUES (3, 7); +INSERT INTO partitioning_hash_test VALUES (4, 4); +-- distribute partitioned table +SELECT create_distributed_table('partitioning_test', 'id'); +NOTICE: Copying data from local table... +NOTICE: copying the data has completed +DETAIL: The local data in the table is no longer visible, but is still on disk. +HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.partitioning_test_2009$$) +NOTICE: Copying data from local table... +NOTICE: copying the data has completed +DETAIL: The local data in the table is no longer visible, but is still on disk. +HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.partitioning_test_2010$$) + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('partitioning_hash_test', 'id'); +NOTICE: Copying data from local table... +NOTICE: copying the data has completed +DETAIL: The local data in the table is no longer visible, but is still on disk. +HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.partitioning_hash_test_0$$) +NOTICE: Copying data from local table... +NOTICE: copying the data has completed +DETAIL: The local data in the table is no longer visible, but is still on disk. +HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.partitioning_hash_test_1$$) + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- see the data is loaded to shards +SELECT * FROM partitioning_test ORDER BY 1; + id | time +--------------------------------------------------------------------- + 1 | 06-06-2009 + 2 | 07-07-2010 + 3 | 09-09-2009 + 4 | 03-03-2010 +(4 rows) + +SELECT * FROM partitioning_hash_test ORDER BY 1; + id | subid +--------------------------------------------------------------------- + 1 | 2 + 2 | 13 + 3 | 7 + 4 | 4 +(4 rows) + +-- see partitioned table and its partitions are distributed +SELECT + logicalrelid +FROM + pg_dist_partition +WHERE + logicalrelid IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010') +ORDER BY 1; + logicalrelid +--------------------------------------------------------------------- + partitioning_test + partitioning_test_2009 + partitioning_test_2010 +(3 rows) + +SELECT + logicalrelid, count(*) +FROM pg_dist_shard + WHERE logicalrelid IN ('partitioning_test', 'partitioning_test_2009', 'partitioning_test_2010') +GROUP BY + logicalrelid +ORDER BY + 1,2; + logicalrelid | count +--------------------------------------------------------------------- + partitioning_test | 4 + partitioning_test_2009 | 4 + partitioning_test_2010 | 4 +(3 rows) + +SELECT + logicalrelid +FROM + pg_dist_partition +WHERE + logicalrelid IN ('partitioning_hash_test', 'partitioning_hash_test_0', 'partitioning_hash_test_1') +ORDER BY 1; + logicalrelid +--------------------------------------------------------------------- + partitioning_hash_test + partitioning_hash_test_0 + partitioning_hash_test_1 +(3 rows) + +SELECT + logicalrelid, count(*) +FROM pg_dist_shard + WHERE logicalrelid IN ('partitioning_hash_test', 'partitioning_hash_test_0', 'partitioning_hash_test_1') +GROUP BY + logicalrelid +ORDER BY + 1,2; + logicalrelid | count +--------------------------------------------------------------------- + partitioning_hash_test | 4 + partitioning_hash_test_0 | 4 + partitioning_hash_test_1 | 4 +(3 rows) + +-- 2-) Creating partition of a distributed table +CREATE TABLE partitioning_test_2011 PARTITION OF partitioning_test FOR VALUES FROM ('2011-01-01') TO ('2012-01-01'); +-- new partition is automatically distributed as well +SELECT + logicalrelid +FROM + pg_dist_partition +WHERE + logicalrelid IN ('partitioning_test', 'partitioning_test_2011') +ORDER BY 1; + logicalrelid +--------------------------------------------------------------------- + partitioning_test + partitioning_test_2011 +(2 rows) + +SELECT + logicalrelid, count(*) +FROM pg_dist_shard + WHERE logicalrelid IN ('partitioning_test', 'partitioning_test_2011') +GROUP BY + logicalrelid +ORDER BY + 1,2; + logicalrelid | count +--------------------------------------------------------------------- + partitioning_test | 4 + partitioning_test_2011 | 4 +(2 rows) + +-- 3-) Attaching non distributed table to a distributed table +CREATE TABLE partitioning_test_2012(id int, time date); +-- load some data +INSERT INTO partitioning_test_2012 VALUES (5, '2012-06-06'); +INSERT INTO partitioning_test_2012 VALUES (6, '2012-07-07'); +ALTER TABLE partitioning_test ATTACH PARTITION partitioning_test_2012 FOR VALUES FROM ('2012-01-01') TO ('2013-01-01'); +NOTICE: Copying data from local table... +NOTICE: copying the data has completed +DETAIL: The local data in the table is no longer visible, but is still on disk. +HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.partitioning_test_2012$$) +-- attached partition is distributed as well +SELECT + logicalrelid +FROM + pg_dist_partition +WHERE + logicalrelid IN ('partitioning_test', 'partitioning_test_2012') +ORDER BY 1; + logicalrelid +--------------------------------------------------------------------- + partitioning_test + partitioning_test_2012 +(2 rows) + +SELECT + logicalrelid, count(*) +FROM pg_dist_shard + WHERE logicalrelid IN ('partitioning_test', 'partitioning_test_2012') +GROUP BY + logicalrelid +ORDER BY + 1,2; + logicalrelid | count +--------------------------------------------------------------------- + partitioning_test | 4 + partitioning_test_2012 | 4 +(2 rows) + +-- try to insert a new data to hash partitioned table +-- no partition is defined for value 5 +INSERT INTO partitioning_hash_test VALUES (8, 5); +ERROR: no partition of relation "partitioning_hash_test_1660012" found for row +DETAIL: Partition key of the failing row contains (subid) = (5). +CONTEXT: while executing command on localhost:xxxxx +INSERT INTO partitioning_hash_test VALUES (9, 12); +ERROR: no partition of relation "partitioning_hash_test_1660015" found for row +DETAIL: Partition key of the failing row contains (subid) = (12). +CONTEXT: while executing command on localhost:xxxxx +CREATE TABLE partitioning_hash_test_2 (id int, subid int); +INSERT INTO partitioning_hash_test_2 VALUES (8, 5); +ALTER TABLE partitioning_hash_test ATTACH PARTITION partitioning_hash_test_2 FOR VALUES WITH (MODULUS 3, REMAINDER 2); +NOTICE: Copying data from local table... +NOTICE: copying the data has completed +DETAIL: The local data in the table is no longer visible, but is still on disk. +HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.partitioning_hash_test_2$$) +INSERT INTO partitioning_hash_test VALUES (9, 12); +-- see the data is loaded to shards +SELECT * FROM partitioning_test ORDER BY 1; + id | time +--------------------------------------------------------------------- + 1 | 06-06-2009 + 2 | 07-07-2010 + 3 | 09-09-2009 + 4 | 03-03-2010 + 5 | 06-06-2012 + 6 | 07-07-2012 +(6 rows) + +SELECT * FROM partitioning_hash_test ORDER BY 1; + id | subid +--------------------------------------------------------------------- + 1 | 2 + 2 | 13 + 3 | 7 + 4 | 4 + 8 | 5 + 9 | 12 +(6 rows) + +-- 4-) Attaching distributed table to distributed table +CREATE TABLE partitioning_test_2013(id int, time date); +SELECT create_distributed_table('partitioning_test_2013', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- load some data +INSERT INTO partitioning_test_2013 VALUES (7, '2013-06-06'); +INSERT INTO partitioning_test_2013 VALUES (8, '2013-07-07'); +ALTER TABLE partitioning_test ATTACH PARTITION partitioning_test_2013 FOR VALUES FROM ('2013-01-01') TO ('2014-01-01'); +-- see the data is loaded to shards +SELECT * FROM partitioning_test ORDER BY 1; + id | time +--------------------------------------------------------------------- + 1 | 06-06-2009 + 2 | 07-07-2010 + 3 | 09-09-2009 + 4 | 03-03-2010 + 5 | 06-06-2012 + 6 | 07-07-2012 + 7 | 06-06-2013 + 8 | 07-07-2013 +(8 rows) + +-- 5-) Failure cases while creating distributed partitioned tables +-- cannot distribute a partition if its parent is not distributed +CREATE TABLE partitioning_test_failure(id int, time date) PARTITION BY RANGE (time); +CREATE TABLE partitioning_test_failure_2009 PARTITION OF partitioning_test_failure FOR VALUES FROM ('2009-01-01') TO ('2010-01-01'); +SELECT create_distributed_table('partitioning_test_failure_2009', 'id'); +ERROR: cannot distribute relation "partitioning_test_failure_2009" which is partition of "partitioning_test_failure" +DETAIL: Citus does not support distributing partitions if their parent is not distributed table. +HINT: Distribute the partitioned table "partitioning_test_failure" instead. +-- only hash distributed tables can have partitions +SELECT create_distributed_table('partitioning_test_failure', 'id', 'append'); +ERROR: distributing partitioned tables in only supported for hash-distributed tables +SELECT create_distributed_table('partitioning_test_failure', 'id', 'range'); +ERROR: distributing partitioned tables in only supported for hash-distributed tables +SELECT create_reference_table('partitioning_test_failure'); +ERROR: distributing partitioned tables in only supported for hash-distributed tables +SET citus.shard_replication_factor TO 1; +-- non-distributed tables cannot have distributed partitions; +DROP TABLE partitioning_test_failure_2009; +CREATE TABLE partitioning_test_failure_2009(id int, time date); +SELECT create_distributed_table('partitioning_test_failure_2009', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +ALTER TABLE partitioning_test_failure ATTACH PARTITION partitioning_test_failure_2009 FOR VALUES FROM ('2009-01-01') TO ('2010-01-01'); +ERROR: non-distributed tables cannot have distributed partitions +HINT: Distribute the partitioned table "partitioning_test_failure_2009" instead +-- multi-level partitioning is not allowed +DROP TABLE partitioning_test_failure_2009; +CREATE TABLE partitioning_test_failure_2009 PARTITION OF partitioning_test_failure FOR VALUES FROM ('2009-01-01') TO ('2010-01-01') PARTITION BY RANGE (time); +SELECT create_distributed_table('partitioning_test_failure', 'id'); +ERROR: distributing multi-level partitioned tables is not supported +DETAIL: Relation "partitioning_test_failure_2009" is partitioned table itself and it is also partition of relation "partitioning_test_failure". +-- multi-level partitioning is not allowed in different order +DROP TABLE partitioning_test_failure_2009; +SELECT create_distributed_table('partitioning_test_failure', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +CREATE TABLE partitioning_test_failure_2009 PARTITION OF partitioning_test_failure FOR VALUES FROM ('2009-01-01') TO ('2010-01-01') PARTITION BY RANGE (time); +ERROR: distributing multi-level partitioned tables is not supported +DETAIL: Relation "partitioning_test_failure_2009" is partitioned table itself and it is also partition of relation "partitioning_test_failure". +-- +-- DMLs in distributed partitioned tables +-- +-- test COPY +-- COPY data to partitioned table +COPY partitioning_test FROM STDIN WITH CSV; +-- COPY data to partition directly +COPY partitioning_test_2009 FROM STDIN WITH CSV; +-- see the data is loaded to shards +SELECT * FROM partitioning_test WHERE id >= 9 ORDER BY 1; + id | time +--------------------------------------------------------------------- + 9 | 01-01-2009 + 10 | 01-01-2010 + 11 | 01-01-2011 + 12 | 01-01-2012 + 13 | 01-02-2009 + 14 | 01-03-2009 +(6 rows) + +-- test INSERT +-- INSERT INTO the partitioned table +INSERT INTO partitioning_test VALUES(15, '2009-02-01'); +INSERT INTO partitioning_test VALUES(16, '2010-02-01'); +INSERT INTO partitioning_test VALUES(17, '2011-02-01'); +INSERT INTO partitioning_test VALUES(18, '2012-02-01'); +-- INSERT INTO the partitions directly table +INSERT INTO partitioning_test VALUES(19, '2009-02-02'); +INSERT INTO partitioning_test VALUES(20, '2010-02-02'); +-- see the data is loaded to shards +SELECT * FROM partitioning_test WHERE id >= 15 ORDER BY 1; + id | time +--------------------------------------------------------------------- + 15 | 02-01-2009 + 16 | 02-01-2010 + 17 | 02-01-2011 + 18 | 02-01-2012 + 19 | 02-02-2009 + 20 | 02-02-2010 +(6 rows) + +-- test INSERT/SELECT +-- INSERT/SELECT from partition to partitioned table +INSERT INTO partitioning_test SELECT * FROM partitioning_test_2011; +-- INSERT/SELECT from partitioned table to partition +INSERT INTO partitioning_test_2012 SELECT * FROM partitioning_test WHERE time >= '2012-01-01' AND time < '2013-01-01'; +-- see the data is loaded to shards (rows in the given range should be duplicated) +SELECT * FROM partitioning_test WHERE time >= '2011-01-01' AND time < '2013-01-01' ORDER BY 1; + id | time +--------------------------------------------------------------------- + 5 | 06-06-2012 + 5 | 06-06-2012 + 6 | 07-07-2012 + 6 | 07-07-2012 + 11 | 01-01-2011 + 11 | 01-01-2011 + 12 | 01-01-2012 + 12 | 01-01-2012 + 17 | 02-01-2011 + 17 | 02-01-2011 + 18 | 02-01-2012 + 18 | 02-01-2012 +(12 rows) + +-- test UPDATE +-- UPDATE partitioned table +UPDATE partitioning_test SET time = '2013-07-07' WHERE id = 7; +-- UPDATE partition directly +UPDATE partitioning_test_2013 SET time = '2013-08-08' WHERE id = 8; +-- see the data is updated +SELECT * FROM partitioning_test WHERE id = 7 OR id = 8 ORDER BY 1; + id | time +--------------------------------------------------------------------- + 7 | 07-07-2013 + 8 | 08-08-2013 +(2 rows) + +-- UPDATE that tries to move a row to a non-existing partition (this should fail) +UPDATE partitioning_test SET time = '2020-07-07' WHERE id = 7; +ERROR: no partition of relation "partitioning_test_1660001" found for row +DETAIL: Partition key of the failing row contains ("time") = (2020-07-07). +CONTEXT: while executing command on localhost:xxxxx +-- UPDATE with subqueries on partitioned table +UPDATE + partitioning_test +SET + time = time + INTERVAL '1 day' +WHERE + id IN (SELECT id FROM partitioning_test WHERE id = 1); +-- UPDATE with subqueries on partition +UPDATE + partitioning_test_2009 +SET + time = time + INTERVAL '1 month' +WHERE + id IN (SELECT id FROM partitioning_test WHERE id = 2); +-- see the data is updated +SELECT * FROM partitioning_test WHERE id = 1 OR id = 2 ORDER BY 1; + id | time +--------------------------------------------------------------------- + 1 | 06-07-2009 + 2 | 07-07-2010 +(2 rows) + +-- test DELETE +-- DELETE from partitioned table +DELETE FROM partitioning_test WHERE id = 9; +-- DELETE from partition directly +DELETE FROM partitioning_test_2010 WHERE id = 10; +-- see the data is deleted +SELECT * FROM partitioning_test WHERE id = 9 OR id = 10 ORDER BY 1; + id | time +--------------------------------------------------------------------- +(0 rows) + +-- create default partition +CREATE TABLE partitioning_test_default PARTITION OF partitioning_test DEFAULT; +\d+ partitioning_test + Table "public.partitioning_test" + Column | Type | Collation | Nullable | Default | Storage | Stats target | Description +--------------------------------------------------------------------- + id | integer | | | | plain | | + time | date | | | | plain | | +Partition key: RANGE ("time") +Partitions: partitioning_test_2009 FOR VALUES FROM ('01-01-2009') TO ('01-01-2010'), + partitioning_test_2010 FOR VALUES FROM ('01-01-2010') TO ('01-01-2011'), + partitioning_test_2011 FOR VALUES FROM ('01-01-2011') TO ('01-01-2012'), + partitioning_test_2012 FOR VALUES FROM ('01-01-2012') TO ('01-01-2013'), + partitioning_test_2013 FOR VALUES FROM ('01-01-2013') TO ('01-01-2014'), + partitioning_test_default DEFAULT + +INSERT INTO partitioning_test VALUES(21, '2014-02-02'); +INSERT INTO partitioning_test VALUES(22, '2015-04-02'); +-- see they are inserted into default partition +SELECT * FROM partitioning_test WHERE id > 20 ORDER BY 1, 2; + id | time +--------------------------------------------------------------------- + 21 | 02-02-2014 + 22 | 04-02-2015 +(2 rows) + +SELECT * FROM partitioning_test_default ORDER BY 1, 2; + id | time +--------------------------------------------------------------------- + 21 | 02-02-2014 + 22 | 04-02-2015 +(2 rows) + +-- create a new partition (will fail) +CREATE TABLE partitioning_test_2014 PARTITION OF partitioning_test FOR VALUES FROM ('2014-01-01') TO ('2015-01-01'); +ERROR: updated partition constraint for default partition would be violated by some row +CONTEXT: while executing command on localhost:xxxxx +BEGIN; +ALTER TABLE partitioning_test DETACH PARTITION partitioning_test_default; +CREATE TABLE partitioning_test_2014 PARTITION OF partitioning_test FOR VALUES FROM ('2014-01-01') TO ('2015-01-01'); +INSERT INTO partitioning_test SELECT * FROM partitioning_test_default WHERE time >= '2014-01-01' AND time < '2015-01-01'; +DELETE FROM partitioning_test_default WHERE time >= '2014-01-01' AND time < '2015-01-01'; +ALTER TABLE partitioning_test ATTACH PARTITION partitioning_test_default DEFAULT; +END; +-- see data is in the table, but some moved out from default partition +SELECT * FROM partitioning_test WHERE id > 20 ORDER BY 1, 2; + id | time +--------------------------------------------------------------------- + 21 | 02-02-2014 + 22 | 04-02-2015 +(2 rows) + +SELECT * FROM partitioning_test_default ORDER BY 1, 2; + id | time +--------------------------------------------------------------------- + 22 | 04-02-2015 +(1 row) + +-- multi-shard UPDATE on partitioned table +UPDATE partitioning_test SET time = time + INTERVAL '1 day'; +-- see rows are UPDATED +SELECT * FROM partitioning_test ORDER BY 1; + id | time +--------------------------------------------------------------------- + 1 | 06-08-2009 + 2 | 07-08-2010 + 3 | 09-10-2009 + 4 | 03-04-2010 + 5 | 06-07-2012 + 5 | 06-07-2012 + 6 | 07-08-2012 + 6 | 07-08-2012 + 7 | 07-08-2013 + 8 | 08-09-2013 + 11 | 01-02-2011 + 11 | 01-02-2011 + 12 | 01-02-2012 + 12 | 01-02-2012 + 13 | 01-03-2009 + 14 | 01-04-2009 + 15 | 02-02-2009 + 16 | 02-02-2010 + 17 | 02-02-2011 + 17 | 02-02-2011 + 18 | 02-02-2012 + 18 | 02-02-2012 + 19 | 02-03-2009 + 20 | 02-03-2010 + 21 | 02-03-2014 + 22 | 04-03-2015 +(26 rows) + +-- multi-shard UPDATE on partition directly +UPDATE partitioning_test_2009 SET time = time + INTERVAL '1 day'; +-- see rows are UPDATED +SELECT * FROM partitioning_test_2009 ORDER BY 1; + id | time +--------------------------------------------------------------------- + 1 | 06-09-2009 + 3 | 09-11-2009 + 13 | 01-04-2009 + 14 | 01-05-2009 + 15 | 02-03-2009 + 19 | 02-04-2009 +(6 rows) + +-- test multi-shard UPDATE which fails in workers (updated value is outside of partition bounds) +UPDATE partitioning_test_2009 SET time = time + INTERVAL '6 month'; +ERROR: new row for relation "partitioning_test_2009_1660005" violates partition constraint +DETAIL: Failing row contains (3, 2010-03-11). +CONTEXT: while executing command on localhost:xxxxx +-- +-- DDL in distributed partitioned tables +-- +-- test CREATE INDEX +-- CREATE INDEX on partitioned table - this will error out +-- on earlier versions of postgres earlier than 11. +CREATE INDEX partitioning_index ON partitioning_test(id); +-- CREATE INDEX on partition +CREATE INDEX partitioning_2009_index ON partitioning_test_2009(id); +-- CREATE INDEX CONCURRENTLY on partition +CREATE INDEX CONCURRENTLY partitioned_2010_index ON partitioning_test_2010(id); +-- see index is created +SELECT tablename, indexname FROM pg_indexes WHERE tablename LIKE 'partitioning_test_%' ORDER BY indexname; + tablename | indexname +--------------------------------------------------------------------- + partitioning_test_2010 | partitioned_2010_index + partitioning_test_2009 | partitioning_2009_index + partitioning_test_2009 | partitioning_test_2009_id_idx + partitioning_test_2010 | partitioning_test_2010_id_idx + partitioning_test_2011 | partitioning_test_2011_id_idx + partitioning_test_2012 | partitioning_test_2012_id_idx + partitioning_test_2013 | partitioning_test_2013_id_idx + partitioning_test_2014 | partitioning_test_2014_id_idx + partitioning_test_default | partitioning_test_default_id_idx +(9 rows) + +-- test drop +-- indexes created on parent table can only be dropped on parent table +-- ie using the same index name +-- following will fail +DROP INDEX partitioning_test_2009_id_idx; +ERROR: cannot drop index partitioning_test_2009_id_idx because index partitioning_index requires it +HINT: You can drop index partitioning_index instead. +-- but dropping index on parent table will succeed +DROP INDEX partitioning_index; +-- this index was already created on partition table +DROP INDEX partitioning_2009_index; +-- test drop index on non-distributed, partitioned table +CREATE TABLE non_distributed_partitioned_table(a int, b int) PARTITION BY RANGE (a); +CREATE TABLE non_distributed_partitioned_table_1 PARTITION OF non_distributed_partitioned_table +FOR VALUES FROM (0) TO (10); +CREATE INDEX non_distributed_partitioned_table_index ON non_distributed_partitioned_table(a); +-- see index is created +SELECT tablename, indexname FROM pg_indexes WHERE tablename LIKE 'non_distributed_partitioned_table_%' ORDER BY indexname; + tablename | indexname +--------------------------------------------------------------------- + non_distributed_partitioned_table_1 | non_distributed_partitioned_table_1_a_idx +(1 row) + +-- drop the index and see it is dropped +DROP INDEX non_distributed_partitioned_table_index; +SELECT tablename, indexname FROM pg_indexes WHERE tablename LIKE 'non_distributed%' ORDER BY indexname; + tablename | indexname +--------------------------------------------------------------------- +(0 rows) + +-- test add COLUMN +-- add COLUMN to partitioned table +ALTER TABLE partitioning_test ADD new_column int; +-- add COLUMN to partition - this will error out +ALTER TABLE partitioning_test_2010 ADD new_column_2 int; +ERROR: cannot add column to a partition +-- see additional column is created +SELECT name, type FROM table_attrs WHERE relid = 'partitioning_test'::regclass ORDER BY 1; + name | type +--------------------------------------------------------------------- + id | integer + new_column | integer + time | date +(3 rows) + +SELECT name, type FROM table_attrs WHERE relid = 'partitioning_test_2010'::regclass ORDER BY 1; + name | type +--------------------------------------------------------------------- + id | integer + new_column | integer + time | date +(3 rows) + +-- test add PRIMARY KEY +-- add PRIMARY KEY to partitioned table - this will error out +ALTER TABLE partitioning_test ADD CONSTRAINT partitioning_primary PRIMARY KEY (id); +ERROR: insufficient columns in PRIMARY KEY constraint definition +DETAIL: PRIMARY KEY constraint on table "partitioning_test" lacks column "time" which is part of the partition key. +-- ADD PRIMARY KEY to partition +ALTER TABLE partitioning_test_2009 ADD CONSTRAINT partitioning_2009_primary PRIMARY KEY (id); +-- see PRIMARY KEY is created +SELECT + table_name, + constraint_name, + constraint_type +FROM + information_schema.table_constraints +WHERE + table_name = 'partitioning_test_2009' AND + constraint_name = 'partitioning_2009_primary'; + table_name | constraint_name | constraint_type +--------------------------------------------------------------------- + partitioning_test_2009 | partitioning_2009_primary | PRIMARY KEY +(1 row) + +-- however, you can add primary key if it contains both distribution and partition key +ALTER TABLE partitioning_hash_test ADD CONSTRAINT partitioning_hash_primary PRIMARY KEY (id, subid); +-- see PRIMARY KEY is created +SELECT + table_name, + constraint_name, + constraint_type +FROM + information_schema.table_constraints +WHERE + table_name LIKE 'partitioning_hash_test%' AND + constraint_type = 'PRIMARY KEY' +ORDER BY 1; + table_name | constraint_name | constraint_type +--------------------------------------------------------------------- + partitioning_hash_test | partitioning_hash_primary | PRIMARY KEY + partitioning_hash_test_0 | partitioning_hash_test_0_pkey | PRIMARY KEY + partitioning_hash_test_1 | partitioning_hash_test_1_pkey | PRIMARY KEY + partitioning_hash_test_2 | partitioning_hash_test_2_pkey | PRIMARY KEY +(4 rows) + +-- test ADD FOREIGN CONSTRAINT +-- add FOREIGN CONSTRAINT to partitioned table -- this will error out (it is a self reference) +ALTER TABLE partitioning_test ADD CONSTRAINT partitioning_foreign FOREIGN KEY (id) REFERENCES partitioning_test_2009 (id); +ERROR: cannot ALTER TABLE "partitioning_test_2009" because it is being used by active queries in this session +-- add FOREIGN CONSTRAINT to partition +INSERT INTO partitioning_test_2009 VALUES (5, '2009-06-06'); +INSERT INTO partitioning_test_2009 VALUES (6, '2009-07-07'); +INSERT INTO partitioning_test_2009 VALUES(12, '2009-02-01'); +INSERT INTO partitioning_test_2009 VALUES(18, '2009-02-01'); +ALTER TABLE partitioning_test_2012 ADD CONSTRAINT partitioning_2012_foreign FOREIGN KEY (id) REFERENCES partitioning_test_2009 (id) ON DELETE CASCADE; +-- see FOREIGN KEY is created +SELECT "Constraint" FROM table_fkeys WHERE relid = 'partitioning_test_2012'::regclass ORDER BY 1; + Constraint +--------------------------------------------------------------------- + partitioning_2012_foreign +(1 row) + +-- test ON DELETE CASCADE works +DELETE FROM partitioning_test_2009 WHERE id = 5; +-- see that element is deleted from both partitions +SELECT * FROM partitioning_test_2009 WHERE id = 5 ORDER BY 1; + id | time | new_column +--------------------------------------------------------------------- +(0 rows) + +SELECT * FROM partitioning_test_2012 WHERE id = 5 ORDER BY 1; + id | time | new_column +--------------------------------------------------------------------- +(0 rows) + +-- test DETACH partition +ALTER TABLE partitioning_test DETACH PARTITION partitioning_test_2009; +-- see DETACHed partitions content is not accessible from partitioning_test; +SELECT * FROM partitioning_test WHERE time >= '2009-01-01' AND time < '2010-01-01' ORDER BY 1; + id | time | new_column +--------------------------------------------------------------------- +(0 rows) + +-- delete from default partition +DELETE FROM partitioning_test WHERE time >= '2015-01-01'; +SELECT * FROM partitioning_test_default; + id | time | new_column +--------------------------------------------------------------------- +(0 rows) + +-- create a reference table for foreign key test +CREATE TABLE partitioning_test_reference(id int PRIMARY KEY, subid int); +INSERT INTO partitioning_test_reference SELECT a, a FROM generate_series(1, 50) a; +SELECT create_reference_table('partitioning_test_reference'); +NOTICE: Copying data from local table... +NOTICE: copying the data has completed +DETAIL: The local data in the table is no longer visible, but is still on disk. +HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.partitioning_test_reference$$) + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +ALTER TABLE partitioning_test ADD CONSTRAINT partitioning_reference_fkey FOREIGN KEY (id) REFERENCES partitioning_test_reference(id) ON DELETE CASCADE; +CREATE TABLE partitioning_test_foreign_key(id int PRIMARY KEY, value int); +SELECT create_distributed_table('partitioning_test_foreign_key', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO partitioning_test_foreign_key SELECT * FROM partitioning_test_reference; +ALTER TABLE partitioning_hash_test ADD CONSTRAINT partitioning_reference_fk_test FOREIGN KEY (id) REFERENCES partitioning_test_foreign_key(id) ON DELETE CASCADE; +-- check foreign keys on partitions +SELECT + table_name, constraint_name, constraint_type FROm information_schema.table_constraints +WHERE + table_name LIKE 'partitioning_hash_test%' AND + constraint_type = 'FOREIGN KEY' +ORDER BY + 1,2; + table_name | constraint_name | constraint_type +--------------------------------------------------------------------- + partitioning_hash_test | partitioning_reference_fk_test | FOREIGN KEY + partitioning_hash_test_0 | partitioning_reference_fk_test | FOREIGN KEY + partitioning_hash_test_1 | partitioning_reference_fk_test | FOREIGN KEY + partitioning_hash_test_2 | partitioning_reference_fk_test | FOREIGN KEY +(4 rows) + +-- check foreign keys on partition shards +-- there is some text ordering issue regarding table name +-- forcing integer sort by extracting shardid +CREATE TYPE foreign_key_details AS (table_name text, constraint_name text, constraint_type text); +SELECT right(table_name, 7)::int as shardid, * FROM ( + SELECT (json_populate_record(NULL::foreign_key_details, + json_array_elements_text(result::json)::json )).* + FROM run_command_on_workers($$ + SELECT + COALESCE(json_agg(row_to_json(q)), '[]'::json) + FROM ( + SELECT + table_name, constraint_name, constraint_type + FROM information_schema.table_constraints + WHERE + table_name LIKE 'partitioning_hash_test%' AND + constraint_type = 'FOREIGN KEY' + ORDER BY 1, 2, 3 + ) q + $$) ) w +ORDER BY 1, 2, 3, 4; + shardid | table_name | constraint_name | constraint_type +--------------------------------------------------------------------- + 1660012 | partitioning_hash_test_1660012 | partitioning_reference_fk_test_1660012 | FOREIGN KEY + 1660013 | partitioning_hash_test_1660013 | partitioning_reference_fk_test_1660013 | FOREIGN KEY + 1660014 | partitioning_hash_test_1660014 | partitioning_reference_fk_test_1660014 | FOREIGN KEY + 1660015 | partitioning_hash_test_1660015 | partitioning_reference_fk_test_1660015 | FOREIGN KEY + 1660016 | partitioning_hash_test_0_1660016 | partitioning_reference_fk_test_1660012 | FOREIGN KEY + 1660017 | partitioning_hash_test_0_1660017 | partitioning_reference_fk_test_1660013 | FOREIGN KEY + 1660018 | partitioning_hash_test_0_1660018 | partitioning_reference_fk_test_1660014 | FOREIGN KEY + 1660019 | partitioning_hash_test_0_1660019 | partitioning_reference_fk_test_1660015 | FOREIGN KEY + 1660020 | partitioning_hash_test_1_1660020 | partitioning_reference_fk_test_1660012 | FOREIGN KEY + 1660021 | partitioning_hash_test_1_1660021 | partitioning_reference_fk_test_1660013 | FOREIGN KEY + 1660022 | partitioning_hash_test_1_1660022 | partitioning_reference_fk_test_1660014 | FOREIGN KEY + 1660023 | partitioning_hash_test_1_1660023 | partitioning_reference_fk_test_1660015 | FOREIGN KEY + 1660032 | partitioning_hash_test_2_1660032 | partitioning_reference_fk_test_1660012 | FOREIGN KEY + 1660033 | partitioning_hash_test_2_1660033 | partitioning_reference_fk_test_1660013 | FOREIGN KEY + 1660034 | partitioning_hash_test_2_1660034 | partitioning_reference_fk_test_1660014 | FOREIGN KEY + 1660035 | partitioning_hash_test_2_1660035 | partitioning_reference_fk_test_1660015 | FOREIGN KEY +(16 rows) + +DROP TYPE foreign_key_details; +-- set replication factor back to 1 since it gots reset +-- after connection re-establishment +SET citus.shard_replication_factor TO 1; +SELECT * FROM partitioning_test WHERE id = 11 or id = 12; + id | time | new_column +--------------------------------------------------------------------- + 11 | 01-02-2011 | + 11 | 01-02-2011 | + 12 | 01-02-2012 | + 12 | 01-02-2012 | +(4 rows) + +DELETE FROM partitioning_test_reference WHERE id = 11 or id = 12; +SELECT * FROM partitioning_hash_test ORDER BY 1, 2; + id | subid +--------------------------------------------------------------------- + 1 | 2 + 2 | 13 + 3 | 7 + 4 | 4 + 8 | 5 + 9 | 12 +(6 rows) + +DELETE FROM partitioning_test_foreign_key WHERE id = 2 OR id = 9; +-- see data is deleted from referencing table +SELECT * FROM partitioning_test WHERE id = 11 or id = 12; + id | time | new_column +--------------------------------------------------------------------- +(0 rows) + +SELECT * FROM partitioning_hash_test ORDER BY 1, 2; + id | subid +--------------------------------------------------------------------- + 1 | 2 + 3 | 7 + 4 | 4 + 8 | 5 +(4 rows) + +-- +-- Transaction tests +-- +-- DDL in transaction +BEGIN; +ALTER TABLE partitioning_test ADD newer_column int; +-- see additional column is created +SELECT name, type FROM table_attrs WHERE relid = 'partitioning_test'::regclass ORDER BY 1; + name | type +--------------------------------------------------------------------- + id | integer + new_column | integer + newer_column | integer + time | date +(4 rows) + +ROLLBACK; +-- see rollback is successful +SELECT name, type FROM table_attrs WHERE relid = 'partitioning_test'::regclass ORDER BY 1; + name | type +--------------------------------------------------------------------- + id | integer + new_column | integer + time | date +(3 rows) + +-- COPY in transaction +BEGIN; +COPY partitioning_test FROM STDIN WITH CSV; +-- see the data is loaded to shards +SELECT * FROM partitioning_test WHERE id = 22 ORDER BY 1; + id | time | new_column +--------------------------------------------------------------------- + 22 | 01-01-2010 | 22 +(1 row) + +SELECT * FROM partitioning_test WHERE id = 23 ORDER BY 1; + id | time | new_column +--------------------------------------------------------------------- + 23 | 01-01-2011 | 23 +(1 row) + +SELECT * FROM partitioning_test WHERE id = 24 ORDER BY 1; + id | time | new_column +--------------------------------------------------------------------- + 24 | 01-01-2013 | 24 +(1 row) + +ROLLBACK; +-- see rollback is successful +SELECT * FROM partitioning_test WHERE id >= 22 ORDER BY 1; + id | time | new_column +--------------------------------------------------------------------- +(0 rows) + +-- DML in transaction +BEGIN; +-- INSERT in transaction +INSERT INTO partitioning_test VALUES(25, '2010-02-02'); +-- see the data is loaded to shards +SELECT * FROM partitioning_test WHERE id = 25 ORDER BY 1; + id | time | new_column +--------------------------------------------------------------------- + 25 | 02-02-2010 | +(1 row) + +-- INSERT/SELECT in transaction +INSERT INTO partitioning_test SELECT * FROM partitioning_test WHERE id = 25; +-- see the data is loaded to shards +SELECT * FROM partitioning_test WHERE id = 25 ORDER BY 1; + id | time | new_column +--------------------------------------------------------------------- + 25 | 02-02-2010 | + 25 | 02-02-2010 | +(2 rows) + +-- UPDATE in transaction +UPDATE partitioning_test SET time = '2010-10-10' WHERE id = 25; +-- see the data is updated +SELECT * FROM partitioning_test WHERE id = 25 ORDER BY 1; + id | time | new_column +--------------------------------------------------------------------- + 25 | 10-10-2010 | + 25 | 10-10-2010 | +(2 rows) + +-- perform operations on partition and partioned tables together +INSERT INTO partitioning_test VALUES(26, '2010-02-02', 26); +INSERT INTO partitioning_test_2010 VALUES(26, '2010-02-02', 26); +COPY partitioning_test FROM STDIN WITH CSV; +COPY partitioning_test_2010 FROM STDIN WITH CSV; +-- see the data is loaded to shards (we should see 4 rows with same content) +SELECT * FROM partitioning_test WHERE id = 26 ORDER BY 1; + id | time | new_column +--------------------------------------------------------------------- + 26 | 02-02-2010 | 26 + 26 | 02-02-2010 | 26 + 26 | 02-02-2010 | 26 + 26 | 02-02-2010 | 26 +(4 rows) + +ROLLBACK; +-- see rollback is successful +SELECT * FROM partitioning_test WHERE id = 26 ORDER BY 1; + id | time | new_column +--------------------------------------------------------------------- +(0 rows) + +-- DETACH and DROP in a transaction +BEGIN; +ALTER TABLE partitioning_test DETACH PARTITION partitioning_test_2011; +DROP TABLE partitioning_test_2011; +COMMIT; +-- see DROPed partitions content is not accessible +SELECT * FROM partitioning_test WHERE time >= '2011-01-01' AND time < '2012-01-01' ORDER BY 1; + id | time | new_column +--------------------------------------------------------------------- +(0 rows) + +-- +-- Misc tests +-- +-- test TRUNCATE +-- test TRUNCATE partition +TRUNCATE partitioning_test_2012; +-- see partition is TRUNCATEd +SELECT * FROM partitioning_test_2012 ORDER BY 1; + id | time | new_column +--------------------------------------------------------------------- +(0 rows) + +-- test TRUNCATE partitioned table +TRUNCATE partitioning_test; +-- see partitioned table is TRUNCATEd +SELECT * FROM partitioning_test ORDER BY 1; + id | time | new_column +--------------------------------------------------------------------- +(0 rows) + +-- test DROP +-- test DROP partition +INSERT INTO partitioning_test_2010 VALUES(27, '2010-02-01'); +DROP TABLE partitioning_test_2010; +-- see DROPped partitions content is not accessible from partitioning_test; +SELECT * FROM partitioning_test WHERE time >= '2010-01-01' AND time < '2011-01-01' ORDER BY 1; + id | time | new_column +--------------------------------------------------------------------- +(0 rows) + +-- test DROP partitioned table +DROP TABLE partitioning_test; +DROP TABLE partitioning_test_reference; +-- dropping the parent should CASCADE to the children as well +SELECT table_name FROM information_schema.tables WHERE table_name LIKE 'partitioning_test%' ORDER BY 1; + table_name +--------------------------------------------------------------------- + partitioning_test_2009 + partitioning_test_failure + partitioning_test_foreign_key +(3 rows) + +-- test distributing partitioned table colocated with non-partitioned table +CREATE TABLE partitioned_users_table (user_id int, time timestamp, value_1 int, value_2 int, value_3 float, value_4 bigint) PARTITION BY RANGE (time); +CREATE TABLE partitioned_events_table (user_id int, time timestamp, event_type int, value_2 int, value_3 float, value_4 bigint) PARTITION BY RANGE (time); +SELECT create_distributed_table('partitioned_users_table', 'user_id', colocate_with => 'users_table'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('partitioned_events_table', 'user_id', colocate_with => 'events_table'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- INSERT/SELECT from regular table to partitioned table +CREATE TABLE partitioned_users_table_2009 PARTITION OF partitioned_users_table FOR VALUES FROM ('2017-01-01') TO ('2018-01-01'); +CREATE TABLE partitioned_events_table_2009 PARTITION OF partitioned_events_table FOR VALUES FROM ('2017-01-01') TO ('2018-01-01'); +INSERT INTO partitioned_events_table SELECT * FROM events_table; +INSERT INTO partitioned_users_table_2009 SELECT * FROM users_table; +-- +-- Complex JOINs, subqueries, UNIONs etc... +-- +-- subquery with UNIONs on partitioned table +SELECT ("final_query"."event_types") as types, count(*) AS sumOfEventType +FROM + (SELECT *, random() + FROM + (SELECT + "t"."user_id", "t"."time", unnest("t"."collected_events") AS "event_types" + FROM + (SELECT + "t1"."user_id", min("t1"."time") AS "time", array_agg(("t1"."event") ORDER BY TIME ASC, event DESC) AS collected_events + FROM( + (SELECT + "events"."user_id", "events"."time", 0 AS event + FROM + partitioned_events_table as "events" + WHERE + event_type IN (1, 2) ) + UNION + (SELECT + "events"."user_id", "events"."time", 1 AS event + FROM + partitioned_events_table as "events" + WHERE + event_type IN (3, 4) ) + UNION + (SELECT + "events"."user_id", "events"."time", 2 AS event + FROM + partitioned_events_table as "events" + WHERE + event_type IN (5, 6) ) + UNION + (SELECT + "events"."user_id", "events"."time", 3 AS event + FROM + partitioned_events_table as "events" + WHERE + event_type IN (1, 6))) t1 + GROUP BY "t1"."user_id") AS t) "q" +) AS final_query +GROUP BY types +ORDER BY types; + types | sumofeventtype +--------------------------------------------------------------------- + 0 | 43 + 1 | 44 + 2 | 8 + 3 | 25 +(4 rows) + +-- UNION and JOIN on both partitioned and regular tables +SELECT ("final_query"."event_types") as types, count(*) AS sumOfEventType +FROM + (SELECT + *, random() + FROM + (SELECT + "t"."user_id", "t"."time", unnest("t"."collected_events") AS "event_types" + FROM + (SELECT + "t1"."user_id", min("t1"."time") AS "time", array_agg(("t1"."event") ORDER BY TIME ASC, event DESC) AS collected_events + FROM ( + (SELECT + * + FROM + (SELECT + "events"."time", 0 AS event, "events"."user_id" + FROM + partitioned_events_table as "events" + WHERE + event_type IN (1, 2)) events_subquery_1) + UNION + (SELECT * + FROM + ( + SELECT * FROM + ( + SELECT + max("events"."time"), + 0 AS event, + "events"."user_id" + FROM + events_table as "events", users_table as "users" + WHERE + events.user_id = users.user_id AND + event_type IN (1, 2) + GROUP BY "events"."user_id" + ) as events_subquery_5 + ) events_subquery_2) + UNION + (SELECT * + FROM + (SELECT + "events"."time", 2 AS event, "events"."user_id" + FROM + partitioned_events_table as "events" + WHERE + event_type IN (3, 4)) events_subquery_3) + UNION + (SELECT * + FROM + (SELECT + "events"."time", 3 AS event, "events"."user_id" + FROM + events_table as "events" + WHERE + event_type IN (5, 6)) events_subquery_4) + ) t1 + GROUP BY "t1"."user_id") AS t) "q" +INNER JOIN + (SELECT + "users"."user_id" + FROM + partitioned_users_table as "users" + WHERE + value_1 > 2 and value_1 < 5) AS t + ON (t.user_id = q.user_id)) as final_query +GROUP BY + types +ORDER BY + types; + types | sumofeventtype +--------------------------------------------------------------------- + 0 | 367 + 2 | 360 + 3 | 57 +(3 rows) + +-- test LIST partitioning +CREATE TABLE list_partitioned_events_table (user_id int, time date, event_type int, value_2 int, value_3 float, value_4 bigint) PARTITION BY LIST (time); +CREATE TABLE list_partitioned_events_table_2014_01_01_05 PARTITION OF list_partitioned_events_table FOR VALUES IN ('2017-11-21', '2017-11-22', '2017-11-23', '2017-11-24', '2017-11-25'); +CREATE TABLE list_partitioned_events_table_2014_01_06_10 PARTITION OF list_partitioned_events_table FOR VALUES IN ('2017-11-26', '2017-11-27', '2017-11-28', '2017-11-29', '2017-11-30'); +CREATE TABLE list_partitioned_events_table_2014_01_11_15 PARTITION OF list_partitioned_events_table FOR VALUES IN ('2017-12-01', '2017-12-02', '2017-12-03', '2017-12-04', '2017-12-05'); +-- test distributing partitioned table colocated with another partitioned table +SELECT create_distributed_table('list_partitioned_events_table', 'user_id', colocate_with => 'partitioned_events_table'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- INSERT/SELECT from partitioned table to partitioned table +INSERT INTO + list_partitioned_events_table +SELECT + user_id, + date_trunc('day', time) as time, + event_type, + value_2, + value_3, + value_4 +FROM + events_table +WHERE + time >= '2017-11-21' AND + time <= '2017-12-01'; +-- LEFT JOINs used with INNER JOINs on range partitioned table, list partitioned table and non-partitioned table +SELECT +count(*) AS cnt, "generated_group_field" + FROM + (SELECT + "eventQuery"."user_id", random(), generated_group_field + FROM + (SELECT + "multi_group_wrapper_1".*, generated_group_field, random() + FROM + (SELECT * + FROM + (SELECT + "list_partitioned_events_table"."time", "list_partitioned_events_table"."user_id" as event_user_id + FROM + list_partitioned_events_table as "list_partitioned_events_table" + WHERE + user_id > 2) "temp_data_queries" + INNER JOIN + (SELECT + "users"."user_id" + FROM + partitioned_users_table as "users" + WHERE + user_id > 2 and value_2 = 1) "user_filters_1" + ON ("temp_data_queries".event_user_id = "user_filters_1".user_id)) AS "multi_group_wrapper_1" + LEFT JOIN + (SELECT + "users"."user_id" AS "user_id", value_2 AS "generated_group_field" + FROM + partitioned_users_table as "users") "left_group_by_1" + ON ("left_group_by_1".user_id = "multi_group_wrapper_1".event_user_id)) "eventQuery") "pushedDownQuery" + GROUP BY + "generated_group_field" + ORDER BY + cnt DESC, generated_group_field ASC + LIMIT 10; + cnt | generated_group_field +--------------------------------------------------------------------- + 1851 | 1 + 1077 | 4 + 963 | 2 + 955 | 3 + 768 | 5 + 639 | 0 +(6 rows) + +-- +-- Additional partitioning features +-- +-- test multi column partitioning +CREATE TABLE multi_column_partitioning(c1 int, c2 int) PARTITION BY RANGE (c1, c2); +CREATE TABLE multi_column_partitioning_0_0_10_0 PARTITION OF multi_column_partitioning FOR VALUES FROM (0, 0) TO (10, 0); +SELECT create_distributed_table('multi_column_partitioning', 'c1'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- test INSERT to multi-column partitioned table +INSERT INTO multi_column_partitioning VALUES(1, 1); +INSERT INTO multi_column_partitioning_0_0_10_0 VALUES(5, -5); +-- test INSERT to multi-column partitioned table where no suitable partition exists +INSERT INTO multi_column_partitioning VALUES(10, 1); +ERROR: no partition of relation "multi_column_partitioning_1660101" found for row +DETAIL: Partition key of the failing row contains (c1, c2) = (10, 1). +CONTEXT: while executing command on localhost:xxxxx +-- test with MINVALUE/MAXVALUE +CREATE TABLE multi_column_partitioning_10_max_20_min PARTITION OF multi_column_partitioning FOR VALUES FROM (10, MAXVALUE) TO (20, MINVALUE); +-- test INSERT to partition with MINVALUE/MAXVALUE bounds +INSERT INTO multi_column_partitioning VALUES(11, -11); +INSERT INTO multi_column_partitioning_10_max_20_min VALUES(19, -19); +-- test INSERT to multi-column partitioned table where no suitable partition exists +INSERT INTO multi_column_partitioning VALUES(20, -20); +ERROR: no partition of relation "multi_column_partitioning_1660101" found for row +DETAIL: Partition key of the failing row contains (c1, c2) = (20, -20). +CONTEXT: while executing command on localhost:xxxxx +-- see data is loaded to multi-column partitioned table +SELECT * FROM multi_column_partitioning ORDER BY 1, 2; + c1 | c2 +--------------------------------------------------------------------- + 1 | 1 + 5 | -5 + 11 | -11 + 19 | -19 +(4 rows) + +-- +-- Tests for locks on partitioned tables +-- +CREATE TABLE partitioning_locks(id int, ref_id int, time date) PARTITION BY RANGE (time); +-- create its partitions +CREATE TABLE partitioning_locks_2009 PARTITION OF partitioning_locks FOR VALUES FROM ('2009-01-01') TO ('2010-01-01'); +CREATE TABLE partitioning_locks_2010 PARTITION OF partitioning_locks FOR VALUES FROM ('2010-01-01') TO ('2011-01-01'); +-- distribute partitioned table +SELECT create_distributed_table('partitioning_locks', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- test locks on router SELECT +BEGIN; +SELECT * FROM partitioning_locks WHERE id = 1 ORDER BY 1, 2; + id | ref_id | time +--------------------------------------------------------------------- +(0 rows) + +SELECT relation::regclass, locktype, mode FROM pg_locks WHERE relation::regclass::text LIKE 'partitioning_locks%' AND pid = pg_backend_pid() ORDER BY 1, 2, 3; + relation | locktype | mode +--------------------------------------------------------------------- + partitioning_locks | relation | AccessShareLock + partitioning_locks_2009 | relation | AccessShareLock + partitioning_locks_2010 | relation | AccessShareLock +(3 rows) + +COMMIT; +-- test locks on real-time SELECT +BEGIN; +SELECT * FROM partitioning_locks ORDER BY 1, 2; + id | ref_id | time +--------------------------------------------------------------------- +(0 rows) + +SELECT relation::regclass, locktype, mode FROM pg_locks WHERE relation::regclass::text LIKE 'partitioning_locks%' AND pid = pg_backend_pid() ORDER BY 1, 2, 3; + relation | locktype | mode +--------------------------------------------------------------------- + partitioning_locks | relation | AccessShareLock + partitioning_locks_2009 | relation | AccessShareLock + partitioning_locks_2010 | relation | AccessShareLock +(3 rows) + +COMMIT; +-- test locks on task-tracker SELECT +SET citus.task_executor_type TO 'task-tracker'; +BEGIN; +SELECT * FROM partitioning_locks AS pl1 JOIN partitioning_locks AS pl2 ON pl1.id = pl2.ref_id ORDER BY 1, 2; + id | ref_id | time | id | ref_id | time +--------------------------------------------------------------------- +(0 rows) + +SELECT relation::regclass, locktype, mode FROM pg_locks WHERE relation::regclass::text LIKE 'partitioning_locks%' AND pid = pg_backend_pid() ORDER BY 1, 2, 3; + relation | locktype | mode +--------------------------------------------------------------------- + partitioning_locks | relation | AccessShareLock + partitioning_locks_2009 | relation | AccessShareLock + partitioning_locks_2010 | relation | AccessShareLock +(3 rows) + +COMMIT; +RESET citus.task_executor_type; +-- test locks on INSERT +BEGIN; +INSERT INTO partitioning_locks VALUES(1, 1, '2009-01-01'); +SELECT relation::regclass, locktype, mode FROM pg_locks WHERE relation::regclass::text LIKE 'partitioning_locks%' AND pid = pg_backend_pid() ORDER BY 1, 2, 3; + relation | locktype | mode +--------------------------------------------------------------------- + partitioning_locks | relation | AccessShareLock + partitioning_locks | relation | RowExclusiveLock + partitioning_locks_2009 | relation | AccessShareLock + partitioning_locks_2009 | relation | RowExclusiveLock + partitioning_locks_2010 | relation | AccessShareLock + partitioning_locks_2010 | relation | RowExclusiveLock +(6 rows) + +COMMIT; +-- test locks on UPDATE +BEGIN; +UPDATE partitioning_locks SET time = '2009-02-01' WHERE id = 1; +SELECT relation::regclass, locktype, mode FROM pg_locks WHERE relation::regclass::text LIKE 'partitioning_locks%' AND pid = pg_backend_pid() ORDER BY 1, 2, 3; + relation | locktype | mode +--------------------------------------------------------------------- + partitioning_locks | relation | AccessShareLock + partitioning_locks | relation | RowExclusiveLock + partitioning_locks_2009 | relation | AccessShareLock + partitioning_locks_2009 | relation | RowExclusiveLock + partitioning_locks_2010 | relation | AccessShareLock + partitioning_locks_2010 | relation | RowExclusiveLock +(6 rows) + +COMMIT; +-- test locks on DELETE +BEGIN; +DELETE FROM partitioning_locks WHERE id = 1; +SELECT relation::regclass, locktype, mode FROM pg_locks WHERE relation::regclass::text LIKE 'partitioning_locks%' AND pid = pg_backend_pid() ORDER BY 1, 2, 3; + relation | locktype | mode +--------------------------------------------------------------------- + partitioning_locks | relation | AccessShareLock + partitioning_locks | relation | RowExclusiveLock + partitioning_locks_2009 | relation | AccessShareLock + partitioning_locks_2009 | relation | RowExclusiveLock + partitioning_locks_2010 | relation | AccessShareLock + partitioning_locks_2010 | relation | RowExclusiveLock +(6 rows) + +COMMIT; +-- test locks on INSERT/SELECT +CREATE TABLE partitioning_locks_for_select(id int, ref_id int, time date); +SELECT create_distributed_table('partitioning_locks_for_select', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +BEGIN; +INSERT INTO partitioning_locks SELECT * FROM partitioning_locks_for_select; +SELECT relation::regclass, locktype, mode FROM pg_locks WHERE relation::regclass::text LIKE 'partitioning_locks%' AND pid = pg_backend_pid() ORDER BY 1, 2, 3; + relation | locktype | mode +--------------------------------------------------------------------- + partitioning_locks | relation | AccessShareLock + partitioning_locks | relation | RowExclusiveLock + partitioning_locks_2009 | relation | AccessShareLock + partitioning_locks_2009 | relation | RowExclusiveLock + partitioning_locks_2010 | relation | AccessShareLock + partitioning_locks_2010 | relation | RowExclusiveLock + partitioning_locks_for_select | relation | AccessShareLock +(7 rows) + +COMMIT; +-- test locks on coordinator INSERT/SELECT +BEGIN; +INSERT INTO partitioning_locks SELECT * FROM partitioning_locks_for_select LIMIT 5; +SELECT relation::regclass, locktype, mode FROM pg_locks WHERE relation::regclass::text LIKE 'partitioning_locks%' AND pid = pg_backend_pid() ORDER BY 1, 2, 3; + relation | locktype | mode +--------------------------------------------------------------------- + partitioning_locks | relation | AccessShareLock + partitioning_locks | relation | RowExclusiveLock + partitioning_locks_2009 | relation | RowExclusiveLock + partitioning_locks_2010 | relation | RowExclusiveLock + partitioning_locks_for_select | relation | AccessShareLock +(5 rows) + +COMMIT; +-- test locks on multi-shard UPDATE +BEGIN; +UPDATE partitioning_locks SET time = '2009-03-01'; +SELECT relation::regclass, locktype, mode FROM pg_locks WHERE relation::regclass::text LIKE 'partitioning_locks%' AND pid = pg_backend_pid() ORDER BY 1, 2, 3; + relation | locktype | mode +--------------------------------------------------------------------- + partitioning_locks | relation | AccessShareLock + partitioning_locks | relation | RowExclusiveLock + partitioning_locks_2009 | relation | AccessShareLock + partitioning_locks_2009 | relation | RowExclusiveLock + partitioning_locks_2010 | relation | AccessShareLock + partitioning_locks_2010 | relation | RowExclusiveLock +(6 rows) + +COMMIT; +-- test locks on DDL +BEGIN; +ALTER TABLE partitioning_locks ADD COLUMN new_column int; +SELECT relation::regclass, locktype, mode FROM pg_locks WHERE relation::regclass::text LIKE 'partitioning_locks%' AND pid = pg_backend_pid() ORDER BY 1, 2, 3; + relation | locktype | mode +--------------------------------------------------------------------- + partitioning_locks | relation | AccessExclusiveLock + partitioning_locks | relation | AccessShareLock + partitioning_locks_2009 | relation | AccessExclusiveLock + partitioning_locks_2009 | relation | AccessShareLock + partitioning_locks_2010 | relation | AccessExclusiveLock + partitioning_locks_2010 | relation | AccessShareLock +(6 rows) + +COMMIT; +-- test locks on TRUNCATE +BEGIN; +TRUNCATE partitioning_locks; +SELECT relation::regclass, locktype, mode FROM pg_locks WHERE relation::regclass::text LIKE 'partitioning_locks%' AND pid = pg_backend_pid() ORDER BY 1, 2, 3; + relation | locktype | mode +--------------------------------------------------------------------- + partitioning_locks | relation | AccessExclusiveLock + partitioning_locks | relation | AccessShareLock + partitioning_locks_2009 | relation | AccessExclusiveLock + partitioning_locks_2009 | relation | AccessShareLock + partitioning_locks_2009 | relation | ShareLock + partitioning_locks_2010 | relation | AccessExclusiveLock + partitioning_locks_2010 | relation | AccessShareLock + partitioning_locks_2010 | relation | ShareLock +(8 rows) + +COMMIT; +CREATE VIEW lockinfo AS + SELECT + logicalrelid, + CASE + WHEN l.objsubid = 5 THEN 'shard' + WHEN l.objsubid = 4 THEN 'shard_metadata' + ELSE 'colocated_shards_metadata' + END AS locktype, + mode + FROM + pg_locks AS l JOIN (select row_number() over (partition by logicalrelid order by shardminvalue) -1 as shardintervalindex, * from pg_dist_shard) AS s + ON + (l.objsubid IN (4, 5) AND l.objid = s.shardid ) + OR (l.objsubid = 8 + AND l.objid IN (select colocationid from pg_dist_partition AS p where p.logicalrelid = s.logicalrelid) + AND l.classid = shardintervalindex + ) + WHERE + logicalrelid IN ('partitioning_locks', 'partitioning_locks_2009', 'partitioning_locks_2010') + AND pid = pg_backend_pid() + AND l.locktype = 'advisory' + ORDER BY + 1, 2, 3; +-- test shard resource locks with multi-shard UPDATE +BEGIN; +UPDATE partitioning_locks_2009 SET time = '2009-03-01'; +-- see the locks on parent table +SELECT * FROM lockinfo; + logicalrelid | locktype | mode +--------------------------------------------------------------------- + partitioning_locks | colocated_shards_metadata | ShareLock + partitioning_locks | colocated_shards_metadata | ShareLock + partitioning_locks | colocated_shards_metadata | ShareLock + partitioning_locks | colocated_shards_metadata | ShareLock + partitioning_locks | shard | ShareUpdateExclusiveLock + partitioning_locks | shard | ShareUpdateExclusiveLock + partitioning_locks | shard | ShareUpdateExclusiveLock + partitioning_locks | shard | ShareUpdateExclusiveLock + partitioning_locks_2009 | colocated_shards_metadata | ShareLock + partitioning_locks_2009 | colocated_shards_metadata | ShareLock + partitioning_locks_2009 | colocated_shards_metadata | ShareLock + partitioning_locks_2009 | colocated_shards_metadata | ShareLock + partitioning_locks_2009 | shard | ShareUpdateExclusiveLock + partitioning_locks_2009 | shard | ShareUpdateExclusiveLock + partitioning_locks_2009 | shard | ShareUpdateExclusiveLock + partitioning_locks_2009 | shard | ShareUpdateExclusiveLock + partitioning_locks_2010 | colocated_shards_metadata | ShareLock + partitioning_locks_2010 | colocated_shards_metadata | ShareLock + partitioning_locks_2010 | colocated_shards_metadata | ShareLock + partitioning_locks_2010 | colocated_shards_metadata | ShareLock +(20 rows) + +COMMIT; +-- test shard resource locks with TRUNCATE +BEGIN; +TRUNCATE partitioning_locks_2009; +-- see the locks on parent table +SELECT * FROM lockinfo; + logicalrelid | locktype | mode +--------------------------------------------------------------------- + partitioning_locks | colocated_shards_metadata | ShareLock + partitioning_locks | colocated_shards_metadata | ShareLock + partitioning_locks | colocated_shards_metadata | ShareLock + partitioning_locks | colocated_shards_metadata | ShareLock + partitioning_locks_2009 | colocated_shards_metadata | ShareLock + partitioning_locks_2009 | colocated_shards_metadata | ShareLock + partitioning_locks_2009 | colocated_shards_metadata | ShareLock + partitioning_locks_2009 | colocated_shards_metadata | ShareLock + partitioning_locks_2010 | colocated_shards_metadata | ShareLock + partitioning_locks_2010 | colocated_shards_metadata | ShareLock + partitioning_locks_2010 | colocated_shards_metadata | ShareLock + partitioning_locks_2010 | colocated_shards_metadata | ShareLock +(12 rows) + +COMMIT; +-- test shard resource locks with INSERT/SELECT +BEGIN; +INSERT INTO partitioning_locks_2009 SELECT * FROM partitioning_locks WHERE time >= '2009-01-01' AND time < '2010-01-01'; +-- see the locks on parent table +SELECT * FROM lockinfo; + logicalrelid | locktype | mode +--------------------------------------------------------------------- + partitioning_locks | colocated_shards_metadata | ShareLock + partitioning_locks | colocated_shards_metadata | ShareLock + partitioning_locks | colocated_shards_metadata | ShareLock + partitioning_locks | colocated_shards_metadata | ShareLock + partitioning_locks | shard | ShareUpdateExclusiveLock + partitioning_locks | shard | ShareUpdateExclusiveLock + partitioning_locks | shard | ShareUpdateExclusiveLock + partitioning_locks | shard | ShareUpdateExclusiveLock + partitioning_locks_2009 | colocated_shards_metadata | ShareLock + partitioning_locks_2009 | colocated_shards_metadata | ShareLock + partitioning_locks_2009 | colocated_shards_metadata | ShareLock + partitioning_locks_2009 | colocated_shards_metadata | ShareLock + partitioning_locks_2009 | shard | ShareUpdateExclusiveLock + partitioning_locks_2009 | shard | ShareUpdateExclusiveLock + partitioning_locks_2009 | shard | ShareUpdateExclusiveLock + partitioning_locks_2009 | shard | ShareUpdateExclusiveLock + partitioning_locks_2010 | colocated_shards_metadata | ShareLock + partitioning_locks_2010 | colocated_shards_metadata | ShareLock + partitioning_locks_2010 | colocated_shards_metadata | ShareLock + partitioning_locks_2010 | colocated_shards_metadata | ShareLock +(20 rows) + +COMMIT; +-- test partition-wise join +CREATE TABLE partitioning_hash_join_test(id int, subid int) PARTITION BY HASH(subid); +CREATE TABLE partitioning_hash_join_test_0 PARTITION OF partitioning_hash_join_test FOR VALUES WITH (MODULUS 3, REMAINDER 0); +CREATE TABLE partitioning_hash_join_test_1 PARTITION OF partitioning_hash_join_test FOR VALUES WITH (MODULUS 3, REMAINDER 1); +CREATE TABLE partitioning_hash_join_test_2 PARTITION OF partitioning_hash_join_test FOR VALUES WITH (MODULUS 3, REMAINDER 2); +SELECT create_distributed_table('partitioning_hash_join_test', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT success FROM run_command_on_workers('alter system set enable_mergejoin to off'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +SELECT success FROM run_command_on_workers('alter system set enable_nestloop to off'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +SELECT success FROM run_command_on_workers('alter system set enable_indexscan to off'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +SELECT success FROM run_command_on_workers('alter system set enable_indexonlyscan to off'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +SELECT success FROM run_command_on_workers('alter system set enable_partitionwise_join to off'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +SELECT success FROM run_command_on_workers('select pg_reload_conf()'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +EXPLAIN (COSTS OFF) +SELECT * FROM partitioning_hash_test JOIN partitioning_hash_join_test USING (id, subid); + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Hash Join + Hash Cond: ((partitioning_hash_join_test.id = partitioning_hash_test.id) AND (partitioning_hash_join_test.subid = partitioning_hash_test.subid)) + -> Append + -> Seq Scan on partitioning_hash_join_test_0_1660133 partitioning_hash_join_test + -> Seq Scan on partitioning_hash_join_test_1_1660137 partitioning_hash_join_test_1 + -> Seq Scan on partitioning_hash_join_test_2_1660141 partitioning_hash_join_test_2 + -> Hash + -> Append + -> Seq Scan on partitioning_hash_test_0_1660016 partitioning_hash_test + -> Seq Scan on partitioning_hash_test_1_1660020 partitioning_hash_test_1 + -> Seq Scan on partitioning_hash_test_2_1660032 partitioning_hash_test_2 +(16 rows) + +-- set partition-wise join on and parallel to off +SELECT success FROM run_command_on_workers('alter system set enable_partitionwise_join to on'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +SELECT success FROM run_command_on_workers('select pg_reload_conf()'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +SET enable_partitionwise_join TO on; +ANALYZE partitioning_hash_test, partitioning_hash_join_test; +EXPLAIN (COSTS OFF) +SELECT * FROM partitioning_hash_test JOIN partitioning_hash_join_test USING (id, subid); + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Append + -> Hash Join + Hash Cond: ((partitioning_hash_join_test.id = partitioning_hash_test.id) AND (partitioning_hash_join_test.subid = partitioning_hash_test.subid)) + -> Seq Scan on partitioning_hash_join_test_0_1660133 partitioning_hash_join_test + -> Hash + -> Seq Scan on partitioning_hash_test_0_1660016 partitioning_hash_test + -> Hash Join + Hash Cond: ((partitioning_hash_test_1.id = partitioning_hash_join_test_1.id) AND (partitioning_hash_test_1.subid = partitioning_hash_join_test_1.subid)) + -> Seq Scan on partitioning_hash_test_1_1660020 partitioning_hash_test_1 + -> Hash + -> Seq Scan on partitioning_hash_join_test_1_1660137 partitioning_hash_join_test_1 + -> Hash Join + Hash Cond: ((partitioning_hash_join_test_2.id = partitioning_hash_test_2.id) AND (partitioning_hash_join_test_2.subid = partitioning_hash_test_2.subid)) + -> Seq Scan on partitioning_hash_join_test_2_1660141 partitioning_hash_join_test_2 + -> Hash + -> Seq Scan on partitioning_hash_test_2_1660032 partitioning_hash_test_2 +(21 rows) + +-- note that partition-wise joins only work when partition key is in the join +-- following join does not have that, therefore join will not be pushed down to +-- partitions +EXPLAIN (COSTS OFF) +SELECT * FROM partitioning_hash_test JOIN partitioning_hash_join_test USING (id); + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Hash Join + Hash Cond: (partitioning_hash_join_test.id = partitioning_hash_test.id) + -> Append + -> Seq Scan on partitioning_hash_join_test_0_1660133 partitioning_hash_join_test + -> Seq Scan on partitioning_hash_join_test_1_1660137 partitioning_hash_join_test_1 + -> Seq Scan on partitioning_hash_join_test_2_1660141 partitioning_hash_join_test_2 + -> Hash + -> Append + -> Seq Scan on partitioning_hash_test_0_1660016 partitioning_hash_test + -> Seq Scan on partitioning_hash_test_1_1660020 partitioning_hash_test_1 + -> Seq Scan on partitioning_hash_test_2_1660032 partitioning_hash_test_2 +(16 rows) + +-- reset partition-wise join +SELECT success FROM run_command_on_workers('alter system reset enable_partitionwise_join'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +SELECT success FROM run_command_on_workers('alter system reset enable_mergejoin'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +SELECT success FROM run_command_on_workers('alter system reset enable_nestloop'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +SELECT success FROM run_command_on_workers('alter system reset enable_indexscan'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +SELECT success FROM run_command_on_workers('alter system reset enable_indexonlyscan'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +SELECT success FROM run_command_on_workers('select pg_reload_conf()'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +RESET enable_partitionwise_join; +DROP VIEW lockinfo; +DROP TABLE +IF EXISTS + partitioning_test_2009, + partitioned_events_table, + partitioned_users_table, + list_partitioned_events_table, + multi_column_partitioning, + partitioning_locks, + partitioning_locks_for_select; +-- make sure we can create a partitioned table with streaming replication +SET citus.replication_model TO 'streaming'; +CREATE TABLE partitioning_test(id int, time date) PARTITION BY RANGE (time); +CREATE TABLE partitioning_test_2009 PARTITION OF partitioning_test FOR VALUES FROM ('2009-01-01') TO ('2010-01-01'); +SELECT create_distributed_table('partitioning_test', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +DROP TABLE partitioning_test; +-- make sure we can attach partitions to a distributed table in a schema +CREATE SCHEMA partitioning_schema; +CREATE TABLE partitioning_schema."schema-test"(id int, time date) PARTITION BY RANGE (time); +SELECT create_distributed_table('partitioning_schema."schema-test"', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +CREATE TABLE partitioning_schema."schema-test_2009"(id int, time date); +ALTER TABLE partitioning_schema."schema-test" ATTACH PARTITION partitioning_schema."schema-test_2009" FOR VALUES FROM ('2009-01-01') TO ('2010-01-01'); +-- attached partition is distributed as well +SELECT + logicalrelid +FROM + pg_dist_partition +WHERE + logicalrelid IN ('partitioning_schema."schema-test"'::regclass, 'partitioning_schema."schema-test_2009"'::regclass) +ORDER BY 1; + logicalrelid +--------------------------------------------------------------------- + partitioning_schema."schema-test" + partitioning_schema."schema-test_2009" +(2 rows) + +SELECT + logicalrelid, count(*) +FROM + pg_dist_shard +WHERE + logicalrelid IN ('partitioning_schema."schema-test"'::regclass, 'partitioning_schema."schema-test_2009"'::regclass) +GROUP BY + logicalrelid +ORDER BY + 1,2; + logicalrelid | count +--------------------------------------------------------------------- + partitioning_schema."schema-test" | 4 + partitioning_schema."schema-test_2009" | 4 +(2 rows) + +DROP TABLE partitioning_schema."schema-test"; +-- make sure we can create partition of a distributed table in a schema +CREATE TABLE partitioning_schema."schema-test"(id int, time date) PARTITION BY RANGE (time); +SELECT create_distributed_table('partitioning_schema."schema-test"', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +CREATE TABLE partitioning_schema."schema-test_2009" PARTITION OF partitioning_schema."schema-test" FOR VALUES FROM ('2009-01-01') TO ('2010-01-01'); +-- newly created partition is distributed as well +SELECT + logicalrelid +FROM + pg_dist_partition +WHERE + logicalrelid IN ('partitioning_schema."schema-test"'::regclass, 'partitioning_schema."schema-test_2009"'::regclass) +ORDER BY 1; + logicalrelid +--------------------------------------------------------------------- + partitioning_schema."schema-test" + partitioning_schema."schema-test_2009" +(2 rows) + +SELECT + logicalrelid, count(*) +FROM + pg_dist_shard +WHERE + logicalrelid IN ('partitioning_schema."schema-test"'::regclass, 'partitioning_schema."schema-test_2009"'::regclass) +GROUP BY + logicalrelid +ORDER BY + 1,2; + logicalrelid | count +--------------------------------------------------------------------- + partitioning_schema."schema-test" | 4 + partitioning_schema."schema-test_2009" | 4 +(2 rows) + +DROP TABLE partitioning_schema."schema-test"; +-- make sure creating partitioned tables works while search_path is set +CREATE TABLE partitioning_schema."schema-test"(id int, time date) PARTITION BY RANGE (time); +SET search_path = partitioning_schema; +SELECT create_distributed_table('"schema-test"', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +CREATE TABLE partitioning_schema."schema-test_2009" PARTITION OF "schema-test" FOR VALUES FROM ('2009-01-01') TO ('2010-01-01'); +-- newly created partition is distributed as well +SELECT + logicalrelid +FROM + pg_dist_partition +WHERE + logicalrelid IN ('partitioning_schema."schema-test"'::regclass, 'partitioning_schema."schema-test_2009"'::regclass) +ORDER BY 1; + logicalrelid +--------------------------------------------------------------------- + "schema-test" + "schema-test_2009" +(2 rows) + +SELECT + logicalrelid, count(*) +FROM + pg_dist_shard +WHERE + logicalrelid IN ('partitioning_schema."schema-test"'::regclass, 'partitioning_schema."schema-test_2009"'::regclass) +GROUP BY + logicalrelid +ORDER BY + 1,2; + logicalrelid | count +--------------------------------------------------------------------- + "schema-test" | 4 + "schema-test_2009" | 4 +(2 rows) + +-- test we don't deadlock when attaching and detaching partitions from partitioned +-- tables with foreign keys +CREATE TABLE reference_table(id int PRIMARY KEY); +SELECT create_reference_table('reference_table'); + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +CREATE TABLE reference_table_2(id int PRIMARY KEY); +SELECT create_reference_table('reference_table_2'); + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +CREATE TABLE partitioning_test(id int, time date) PARTITION BY RANGE (time); +CREATE TABLE partitioning_test_2008 PARTITION OF partitioning_test FOR VALUES FROM ('2008-01-01') TO ('2009-01-01'); +CREATE TABLE partitioning_test_2009 (LIKE partitioning_test); +CREATE TABLE partitioning_test_2010 (LIKE partitioning_test); +CREATE TABLE partitioning_test_2011 (LIKE partitioning_test); +-- distributing partitioning_test will also distribute partitioning_test_2008 +SELECT create_distributed_table('partitioning_test', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('partitioning_test_2009', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('partitioning_test_2010', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('partitioning_test_2011', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +ALTER TABLE partitioning_test ADD CONSTRAINT partitioning_reference_fkey + FOREIGN KEY (id) REFERENCES reference_table(id) ON DELETE CASCADE; +ALTER TABLE partitioning_test_2009 ADD CONSTRAINT partitioning_reference_fkey_2009 + FOREIGN KEY (id) REFERENCES reference_table(id) ON DELETE CASCADE; +INSERT INTO partitioning_test_2010 VALUES (1, '2010-02-01'); +-- This should fail because of foreign key constraint violation +ALTER TABLE partitioning_test ATTACH PARTITION partitioning_test_2010 + FOR VALUES FROM ('2010-01-01') TO ('2011-01-01'); +ERROR: insert or update on table "partitioning_test_2010_1660191" violates foreign key constraint "partitioning_reference_fkey_1660179" +DETAIL: Key (id)=(X) is not present in table "reference_table_1660177". +CONTEXT: while executing command on localhost:xxxxx +-- Truncate, so attaching again won't fail +TRUNCATE partitioning_test_2010; +-- Attach a table which already has the same constraint +ALTER TABLE partitioning_test ATTACH PARTITION partitioning_test_2009 + FOR VALUES FROM ('2009-01-01') TO ('2010-01-01'); +-- Attach a table which doesn't have the constraint +ALTER TABLE partitioning_test ATTACH PARTITION partitioning_test_2010 + FOR VALUES FROM ('2010-01-01') TO ('2011-01-01'); +-- Attach a table which has a different constraint +ALTER TABLE partitioning_test ATTACH PARTITION partitioning_test_2011 + FOR VALUES FROM ('2011-01-01') TO ('2012-01-01'); +ALTER TABLE partitioning_test DETACH PARTITION partitioning_test_2008; +ALTER TABLE partitioning_test DETACH PARTITION partitioning_test_2009; +ALTER TABLE partitioning_test DETACH PARTITION partitioning_test_2010; +ALTER TABLE partitioning_test DETACH PARTITION partitioning_test_2011; +DROP TABLE partitioning_test, partitioning_test_2008, partitioning_test_2009, + partitioning_test_2010, partitioning_test_2011, + reference_table, reference_table_2; +DROP SCHEMA partitioning_schema CASCADE; +NOTICE: drop cascades to table "schema-test" +RESET SEARCH_PATH; +DROP TABLE IF EXISTS + partitioning_hash_test, + partitioning_hash_join_test, + partitioning_test_failure, + non_distributed_partitioned_table, + partitioning_test_foreign_key; diff --git a/src/test/regress/input/multi_complex_count_distinct.source b/src/test/regress/input/multi_complex_count_distinct.source index a30cb119c..13b25e0d3 100644 --- a/src/test/regress/input/multi_complex_count_distinct.source +++ b/src/test/regress/input/multi_complex_count_distinct.source @@ -154,14 +154,6 @@ SELECT HAVING count(distinct l_suppkey) > 1550 ORDER BY 1, 2 DESC; -EXPLAIN (COSTS false, VERBOSE true) -SELECT - l_shipmode, count(distinct l_partkey) - FROM lineitem_hash - GROUP BY l_shipmode - HAVING count(distinct l_suppkey) > 1550 - ORDER BY 1, 2 DESC; - -- count distinct is supported on single table subqueries SELECT * FROM ( From 6ff4e427067fb4db14eba9dca550a3a381c6cc78 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Thu, 11 Jun 2020 10:34:11 +0300 Subject: [PATCH 30/52] Add alternative output for multi_function_in_join With pg13, constants functions from "FROM" clause are replaced. This means that in citus side, we will see the constraints in restriction info, instead of the function call. For example: SELECT * FROM table1 JOIN add(3,5) sum ON (id = sum) ORDER BY id ASC; Assuming that the function `add` returns constant, it will be evaluated on postgres side. This means that this query will be routable because there will be only one shard after pruning with the restrictions. However before pg13, this would be multi shard query. And it would go into recursive planning, the function would be evaluated on the coordinator because it can be. This means that with pg13, users will need to distribute the function because when it is routable executable, it will currently also send the function call to the worker in the query. So the function should exist in the worker. It could be better to replace the constant in the query tree as well so that the query string sent to the worker has the constant value and therefore it doesn't need the function. However I feel like users would already have the function in workers if they have any multi shard query. Commit on Postgres side: 7266d0997dd2a0632da38a594c78e25ff21df67e --- .../expected/multi_function_in_join.out | 12 +- .../expected/multi_function_in_join_0.out | 250 ++++++++++++++++++ .../regress/sql/multi_function_in_join.sql | 3 + 3 files changed, 263 insertions(+), 2 deletions(-) create mode 100644 src/test/regress/expected/multi_function_in_join_0.out diff --git a/src/test/regress/expected/multi_function_in_join.out b/src/test/regress/expected/multi_function_in_join.out index ac0ecd748..1f253aad9 100644 --- a/src/test/regress/expected/multi_function_in_join.out +++ b/src/test/regress/expected/multi_function_in_join.out @@ -12,6 +12,8 @@ CREATE SCHEMA functions_in_joins; SET search_path TO 'functions_in_joins'; SET citus.next_shard_id TO 2500000; +SET citus.replication_model to 'streaming'; +SET citus.shard_replication_factor to 1; CREATE TABLE table1 (id int, data int); SELECT create_distributed_table('table1','id'); create_distributed_table @@ -38,9 +40,15 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT table1.id, ta CREATE FUNCTION add(integer, integer) RETURNS integer AS 'SELECT $1 + $2;' LANGUAGE SQL; +SELECT create_distributed_function('add(integer,integer)'); +DEBUG: switching to sequential query execution mode +DETAIL: A distributed function is created. To make sure subsequent commands see the type correctly we need to make sure to use only one connection for all future commands + create_distributed_function +--------------------------------------------------------------------- + +(1 row) + SELECT * FROM table1 JOIN add(3,5) sum ON (id = sum) ORDER BY id ASC; -DEBUG: generating subplan XXX_1 for subquery SELECT sum FROM functions_in_joins.add(3, 5) sum(sum) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT table1.id, table1.data, sum.sum FROM (functions_in_joins.table1 JOIN (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum integer)) sum ON ((table1.id OPERATOR(pg_catalog.=) sum.sum))) ORDER BY table1.id id | data | sum --------------------------------------------------------------------- 8 | 64 | 8 diff --git a/src/test/regress/expected/multi_function_in_join_0.out b/src/test/regress/expected/multi_function_in_join_0.out new file mode 100644 index 000000000..8498c4a13 --- /dev/null +++ b/src/test/regress/expected/multi_function_in_join_0.out @@ -0,0 +1,250 @@ +-- +-- multi function in join queries aims to test the function calls that are +-- used in joins. +-- +-- These functions are supposed to be executed on the worker and to ensure +-- that we wrap those functions inside (SELECT * FROM fnc()) sub queries. +-- +-- We do not yet support those functions that: +-- - have lateral joins +-- - have WITH ORDINALITY clause +-- - are user-defined and immutable +CREATE SCHEMA functions_in_joins; +SET search_path TO 'functions_in_joins'; +SET citus.next_shard_id TO 2500000; +SET citus.replication_model to 'streaming'; +SET citus.shard_replication_factor to 1; +CREATE TABLE table1 (id int, data int); +SELECT create_distributed_table('table1','id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO table1 +SELECT x, x*x +from generate_series(1, 100) as f (x); +-- Verbose messages for observing the subqueries that wrapped function calls +SET client_min_messages TO DEBUG1; +-- Check joins on a sequence +CREATE SEQUENCE numbers; +SELECT * FROM table1 JOIN nextval('numbers') n ON (id = n) ORDER BY id ASC; +DEBUG: generating subplan XXX_1 for subquery SELECT n FROM nextval('functions_in_joins.numbers'::regclass) n(n) +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT table1.id, table1.data, n.n FROM (functions_in_joins.table1 JOIN (SELECT intermediate_result.n FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(n bigint)) n ON ((table1.id OPERATOR(pg_catalog.=) n.n))) ORDER BY table1.id + id | data | n +--------------------------------------------------------------------- + 1 | 1 | 1 +(1 row) + +-- Check joins of a function that returns a single integer +CREATE FUNCTION add(integer, integer) RETURNS integer +AS 'SELECT $1 + $2;' +LANGUAGE SQL; +SELECT create_distributed_function('add(integer,integer)'); +DEBUG: switching to sequential query execution mode +DETAIL: A distributed function is created. To make sure subsequent commands see the type correctly we need to make sure to use only one connection for all future commands + create_distributed_function +--------------------------------------------------------------------- + +(1 row) + +SELECT * FROM table1 JOIN add(3,5) sum ON (id = sum) ORDER BY id ASC; +DEBUG: generating subplan XXX_1 for subquery SELECT sum FROM functions_in_joins.add(3, 5) sum(sum) +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT table1.id, table1.data, sum.sum FROM (functions_in_joins.table1 JOIN (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum integer)) sum ON ((table1.id OPERATOR(pg_catalog.=) sum.sum))) ORDER BY table1.id + id | data | sum +--------------------------------------------------------------------- + 8 | 64 | 8 +(1 row) + +-- Check join of plpgsql functions +-- a function returning a single integer +CREATE OR REPLACE FUNCTION increment(i integer) RETURNS integer AS $$ +BEGIN + RETURN i + 1; +END; +$$ LANGUAGE plpgsql; +SELECT * FROM table1 JOIN increment(2) val ON (id = val) ORDER BY id ASC; +DEBUG: generating subplan XXX_1 for subquery SELECT val FROM functions_in_joins.increment(2) val(val) +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT table1.id, table1.data, val.val FROM (functions_in_joins.table1 JOIN (SELECT intermediate_result.val FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(val integer)) val ON ((table1.id OPERATOR(pg_catalog.=) val.val))) ORDER BY table1.id + id | data | val +--------------------------------------------------------------------- + 3 | 9 | 3 +(1 row) + +-- a function that returns a set of integers +CREATE OR REPLACE FUNCTION next_k_integers(IN first_value INTEGER, + IN k INTEGER DEFAULT 3, + OUT result INTEGER) + RETURNS SETOF INTEGER AS $$ +BEGIN + RETURN QUERY SELECT x FROM generate_series(first_value, first_value+k-1) f(x); +END; +$$ LANGUAGE plpgsql; +SELECT * +FROM table1 JOIN next_k_integers(3,2) next_integers ON (id = next_integers.result) +ORDER BY id ASC; +DEBUG: generating subplan XXX_1 for subquery SELECT result FROM functions_in_joins.next_k_integers(3, 2) next_integers(result) +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT table1.id, table1.data, next_integers.result FROM (functions_in_joins.table1 JOIN (SELECT intermediate_result.result FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(result integer)) next_integers ON ((table1.id OPERATOR(pg_catalog.=) next_integers.result))) ORDER BY table1.id + id | data | result +--------------------------------------------------------------------- + 3 | 9 | 3 + 4 | 16 | 4 +(2 rows) + +-- a function returning set of records +CREATE FUNCTION get_set_of_records() RETURNS SETOF RECORD AS $cmd$ +SELECT x, x+1 FROM generate_series(0,4) f(x) +$cmd$ +LANGUAGE SQL; +SELECT * FROM table1 JOIN get_set_of_records() AS t2(x int, y int) ON (id = x) ORDER BY id ASC; +DEBUG: generating subplan XXX_1 for subquery SELECT x, y FROM functions_in_joins.get_set_of_records() t2(x integer, y integer) +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT table1.id, table1.data, t2.x, t2.y FROM (functions_in_joins.table1 JOIN (SELECT intermediate_result.x, intermediate_result.y FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(x integer, y integer)) t2 ON ((table1.id OPERATOR(pg_catalog.=) t2.x))) ORDER BY table1.id + id | data | x | y +--------------------------------------------------------------------- + 1 | 1 | 1 | 2 + 2 | 4 | 2 | 3 + 3 | 9 | 3 | 4 + 4 | 16 | 4 | 5 +(4 rows) + +-- a function returning table +CREATE FUNCTION dup(int) RETURNS TABLE(f1 int, f2 text) +AS $$ SELECT $1, CAST($1 AS text) || ' is text' $$ +LANGUAGE SQL; +SELECT f.* FROM table1 t JOIN dup(32) f ON (f1 = id); +DEBUG: generating subplan XXX_1 for subquery SELECT f1, f2 FROM functions_in_joins.dup(32) f(f1, f2) +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT f.f1, f.f2 FROM (functions_in_joins.table1 t JOIN (SELECT intermediate_result.f1, intermediate_result.f2 FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(f1 integer, f2 text)) f ON ((f.f1 OPERATOR(pg_catalog.=) t.id))) + f1 | f2 +--------------------------------------------------------------------- + 32 | 32 is text +(1 row) + +-- a stable function +CREATE OR REPLACE FUNCTION the_minimum_id() + RETURNS INTEGER STABLE AS 'SELECT min(id) FROM table1' LANGUAGE SQL; +SELECT * FROM table1 JOIN the_minimum_id() min_id ON (id = min_id); +DEBUG: generating subplan XXX_1 for subquery SELECT min_id FROM functions_in_joins.the_minimum_id() min_id(min_id) +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT table1.id, table1.data, min_id.min_id FROM (functions_in_joins.table1 JOIN (SELECT intermediate_result.min_id FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(min_id integer)) min_id ON ((table1.id OPERATOR(pg_catalog.=) min_id.min_id))) + id | data | min_id +--------------------------------------------------------------------- + 1 | 1 | 1 +(1 row) + +-- a built-in immutable function +SELECT * FROM table1 JOIN abs(100) as hundred ON (id = hundred) ORDER BY id ASC; + id | data | hundred +--------------------------------------------------------------------- + 100 | 10000 | 100 +(1 row) + +-- function joins inside a CTE +WITH next_row_to_process AS ( + SELECT * FROM table1 JOIN nextval('numbers') n ON (id = n) + ) +SELECT * +FROM table1, next_row_to_process +WHERE table1.data <= next_row_to_process.data +ORDER BY 1,2 ASC; +DEBUG: generating subplan XXX_1 for CTE next_row_to_process: SELECT table1.id, table1.data, n.n FROM (functions_in_joins.table1 JOIN nextval('functions_in_joins.numbers'::regclass) n(n) ON ((table1.id OPERATOR(pg_catalog.=) n.n))) +DEBUG: generating subplan XXX_1 for subquery SELECT n FROM nextval('functions_in_joins.numbers'::regclass) n(n) +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT table1.id, table1.data, n.n FROM (functions_in_joins.table1 JOIN (SELECT intermediate_result.n FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(n bigint)) n ON ((table1.id OPERATOR(pg_catalog.=) n.n))) +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT table1.id, table1.data, next_row_to_process.id, next_row_to_process.data, next_row_to_process.n FROM functions_in_joins.table1, (SELECT intermediate_result.id, intermediate_result.data, intermediate_result.n FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(id integer, data integer, n bigint)) next_row_to_process WHERE (table1.data OPERATOR(pg_catalog.<=) next_row_to_process.data) ORDER BY table1.id, table1.data + id | data | id | data | n +--------------------------------------------------------------------- + 1 | 1 | 2 | 4 | 2 + 2 | 4 | 2 | 4 | 2 +(2 rows) + +-- Multiple functions in an RTE +SELECT * FROM ROWS FROM (next_k_integers(5), next_k_integers(10)) AS f(a, b), + table1 WHERE id = a ORDER BY id ASC; +DEBUG: generating subplan XXX_1 for subquery SELECT a, b FROM ROWS FROM(functions_in_joins.next_k_integers(5), functions_in_joins.next_k_integers(10)) f(a, b) +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT f.a, f.b, table1.id, table1.data FROM (SELECT intermediate_result.a, intermediate_result.b FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(a integer, b integer)) f(a, b), functions_in_joins.table1 WHERE (table1.id OPERATOR(pg_catalog.=) f.a) ORDER BY table1.id + a | b | id | data +--------------------------------------------------------------------- + 5 | 10 | 5 | 25 + 6 | 11 | 6 | 36 + 7 | 12 | 7 | 49 +(3 rows) + +-- Custom Type returning function used in a join +RESET client_min_messages; +CREATE TYPE min_and_max AS ( + minimum INT, + maximum INT +); +SET client_min_messages TO DEBUG1; +CREATE OR REPLACE FUNCTION max_and_min () RETURNS + min_and_max AS $$ +DECLARE + result min_and_max%rowtype; +begin + select into result min(data) as minimum, max(data) as maximum from table1; + return result; +end; +$$ language plpgsql; +SELECT * FROM table1 JOIN max_and_min() m ON (m.maximum = data OR m.minimum = data) ORDER BY 1,2,3,4; +DEBUG: generating subplan XXX_1 for subquery SELECT minimum, maximum FROM functions_in_joins.max_and_min() m(minimum, maximum) +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT table1.id, table1.data, m.minimum, m.maximum FROM (functions_in_joins.table1 JOIN (SELECT intermediate_result.minimum, intermediate_result.maximum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(minimum integer, maximum integer)) m ON (((m.maximum OPERATOR(pg_catalog.=) table1.data) OR (m.minimum OPERATOR(pg_catalog.=) table1.data)))) ORDER BY table1.id, table1.data, m.minimum, m.maximum + id | data | minimum | maximum +--------------------------------------------------------------------- + 1 | 1 | 1 | 10000 + 100 | 10000 | 1 | 10000 +(2 rows) + +-- The following tests will fail as we do not support all joins on +-- all kinds of functions +-- In other words, we cannot recursively plan the functions and hence +-- the query fails on the workers +SET client_min_messages TO ERROR; +\set VERBOSITY terse +-- function joins in CTE results can create lateral joins that are not supported +-- we execute the query within a function to consolidate the error messages +-- between different executors +CREATE FUNCTION raise_failed_execution_func_join(query text) RETURNS void AS $$ +BEGIN + EXECUTE query; + EXCEPTION WHEN OTHERS THEN + IF SQLERRM LIKE 'failed to execute task%' THEN + RAISE 'Task failed to execute'; + ELSIF SQLERRM LIKE '%does not exist%' THEN + RAISE 'Task failed to execute'; + END IF; +END; +$$LANGUAGE plpgsql; +SELECT raise_failed_execution_func_join($$ + WITH one_row AS ( + SELECT * FROM table1 WHERE id=52 + ) + SELECT table1.id, table1.data + FROM one_row, table1, next_k_integers(one_row.id, 5) next_five_ids + WHERE table1.id = next_five_ids; +$$); +ERROR: Task failed to execute +-- a user-defined immutable function +CREATE OR REPLACE FUNCTION the_answer_to_life() + RETURNS INTEGER IMMUTABLE AS 'SELECT 42' LANGUAGE SQL; +SELECT raise_failed_execution_func_join($$ + SELECT * FROM table1 JOIN the_answer_to_life() the_answer ON (id = the_answer); +$$); +ERROR: Task failed to execute +SELECT raise_failed_execution_func_join($$ + SELECT * + FROM table1 + JOIN next_k_integers(10,5) WITH ORDINALITY next_integers + ON (id = next_integers.result); +$$); +ERROR: Task failed to execute +-- WITH ORDINALITY clause +SELECT raise_failed_execution_func_join($$ + SELECT * + FROM table1 + JOIN next_k_integers(10,5) WITH ORDINALITY next_integers + ON (id = next_integers.result) + ORDER BY id ASC; +$$); +ERROR: Task failed to execute +RESET client_min_messages; +DROP SCHEMA functions_in_joins CASCADE; +NOTICE: drop cascades to 12 other objects +SET search_path TO DEFAULT; diff --git a/src/test/regress/sql/multi_function_in_join.sql b/src/test/regress/sql/multi_function_in_join.sql index a1f9c9e87..2fb7fdf18 100644 --- a/src/test/regress/sql/multi_function_in_join.sql +++ b/src/test/regress/sql/multi_function_in_join.sql @@ -13,6 +13,8 @@ CREATE SCHEMA functions_in_joins; SET search_path TO 'functions_in_joins'; SET citus.next_shard_id TO 2500000; +SET citus.replication_model to 'streaming'; +SET citus.shard_replication_factor to 1; CREATE TABLE table1 (id int, data int); SELECT create_distributed_table('table1','id'); @@ -32,6 +34,7 @@ SELECT * FROM table1 JOIN nextval('numbers') n ON (id = n) ORDER BY id ASC; CREATE FUNCTION add(integer, integer) RETURNS integer AS 'SELECT $1 + $2;' LANGUAGE SQL; +SELECT create_distributed_function('add(integer,integer)'); SELECT * FROM table1 JOIN add(3,5) sum ON (id = sum) ORDER BY id ASC; -- Check join of plpgsql functions From ff7a563c57da2b437a251a3099051d7f07cea4ce Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Thu, 11 Jun 2020 11:36:52 +0300 Subject: [PATCH 31/52] decrease log level to debug1 to prevent flaky debug --- src/test/regress/expected/multi_foreign_key.out | 2 +- src/test/regress/sql/multi_foreign_key.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/regress/expected/multi_foreign_key.out b/src/test/regress/expected/multi_foreign_key.out index 25908d481..72209b585 100644 --- a/src/test/regress/expected/multi_foreign_key.out +++ b/src/test/regress/expected/multi_foreign_key.out @@ -435,7 +435,7 @@ SELECT create_distributed_table('referencing_table', 'ref_id', 'hash'); -- not skipping validation would result in a distributed query, which emits debug messages BEGIN; SET LOCAL citus.enable_ddl_propagation TO off; -SET LOCAL client_min_messages TO DEBUG2; +SET LOCAL client_min_messages TO DEBUG1; ALTER TABLE referencing_table ADD CONSTRAINT test_constraint FOREIGN KEY (ref_id) REFERENCES referenced_table (id); ABORT; -- test foreign constraint creation diff --git a/src/test/regress/sql/multi_foreign_key.sql b/src/test/regress/sql/multi_foreign_key.sql index 62086ef22..084947053 100644 --- a/src/test/regress/sql/multi_foreign_key.sql +++ b/src/test/regress/sql/multi_foreign_key.sql @@ -250,7 +250,7 @@ SELECT create_distributed_table('referencing_table', 'ref_id', 'hash'); -- not skipping validation would result in a distributed query, which emits debug messages BEGIN; SET LOCAL citus.enable_ddl_propagation TO off; -SET LOCAL client_min_messages TO DEBUG2; +SET LOCAL client_min_messages TO DEBUG1; ALTER TABLE referencing_table ADD CONSTRAINT test_constraint FOREIGN KEY (ref_id) REFERENCES referenced_table (id); ABORT; From 1112b254a73a48fe6d550350737a6adff8b3600b Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Thu, 11 Jun 2020 11:59:44 +0300 Subject: [PATCH 32/52] adapt recently added code for pg13 This commit mostly adds pg_get_triggerdef_command to our ruleutils_13. This doesn't add anything extra for ruleutils 13 so it is basically a copy of the change on ruleutils_12 --- src/backend/distributed/commands/multi_copy.c | 26 +- .../distributed/commands/utility_hook.c | 6 +- .../connection/placement_connection.c | 2 +- .../distributed/deparser/ruleutils_13.c | 313 +++++++++++++++++- .../executor/insert_select_executor.c | 6 +- .../distributed/executor/local_executor.c | 3 +- .../distributed/executor/multi_executor.c | 8 +- .../distributed/metadata/metadata_cache.c | 2 +- .../distributed/operations/node_protocol.c | 5 +- .../distributed/planner/distributed_planner.c | 12 +- .../distributed/planner/multi_explain.c | 12 +- .../planner/multi_physical_planner.c | 31 +- .../relation_restriction_equivalence.c | 14 +- .../distributed/planner/tdigest_extension.c | 1 + .../test/distributed_intermediate_results.c | 8 +- .../test/foreign_key_relationship_query.c | 12 +- .../transaction/relation_access_tracking.c | 2 +- .../utils/foreign_key_relationship.c | 2 +- src/backend/distributed/utils/maintenanced.c | 2 +- src/include/distributed/commands/multi_copy.h | 3 +- .../distributed/commands/utility_hook.h | 4 +- src/include/distributed/distributed_planner.h | 19 +- src/include/distributed/listutils.h | 3 +- src/include/distributed/version_compat.h | 38 ++- 24 files changed, 440 insertions(+), 94 deletions(-) diff --git a/src/backend/distributed/commands/multi_copy.c b/src/backend/distributed/commands/multi_copy.c index 650763739..c857e754f 100644 --- a/src/backend/distributed/commands/multi_copy.c +++ b/src/backend/distributed/commands/multi_copy.c @@ -214,8 +214,10 @@ typedef struct ShardConnections /* Local functions forward declarations */ -static void CopyToExistingShards(CopyStmt *copyStatement, QueryCompletionCompat *completionTag); -static void CopyToNewShards(CopyStmt *copyStatement, QueryCompletionCompat *completionTag, Oid relationId); +static void CopyToExistingShards(CopyStmt *copyStatement, + QueryCompletionCompat *completionTag); +static void CopyToNewShards(CopyStmt *copyStatement, QueryCompletionCompat *completionTag, + Oid relationId); static void OpenCopyConnectionsForNewShards(CopyStmt *copyStatement, ShardConnections *shardConnections, bool stopOnFailure, @@ -316,7 +318,8 @@ static bool CitusCopyDestReceiverReceive(TupleTableSlot *slot, static void CitusCopyDestReceiverShutdown(DestReceiver *destReceiver); static void CitusCopyDestReceiverDestroy(DestReceiver *destReceiver); static bool ContainsLocalPlacement(int64 shardId); -static void CompleteCopyQueryTagCompat(QueryCompletionCompat* completionTag, uint64 processedRowCount); +static void CompleteCopyQueryTagCompat(QueryCompletionCompat *completionTag, uint64 + processedRowCount); static void FinishLocalCopy(CitusCopyDestReceiver *copyDest); static void CloneCopyOutStateForLocalCopy(CopyOutState from, CopyOutState to); static bool ShouldExecuteCopyLocally(bool isIntermediateResult); @@ -568,7 +571,8 @@ CopyToExistingShards(CopyStmt *copyStatement, QueryCompletionCompat *completionT * tables where we create new shards into which to copy rows. */ static void -CopyToNewShards(CopyStmt *copyStatement, QueryCompletionCompat *completionTag, Oid relationId) +CopyToNewShards(CopyStmt *copyStatement, QueryCompletionCompat *completionTag, Oid + relationId) { /* allocate column values and nulls arrays */ Relation distributedRelation = table_open(relationId, RowExclusiveLock); @@ -746,12 +750,15 @@ CopyToNewShards(CopyStmt *copyStatement, QueryCompletionCompat *completionTag, O } } -static void CompleteCopyQueryTagCompat(QueryCompletionCompat* completionTag, uint64 processedRowCount) { + +static void +CompleteCopyQueryTagCompat(QueryCompletionCompat *completionTag, uint64 processedRowCount) +{ #if PG_VERSION_NUM >= PG_VERSION_13 - SetQueryCompletion(completionTag, CMDTAG_COPY, processedRowCount); + SetQueryCompletion(completionTag, CMDTAG_COPY, processedRowCount); #else - SafeSnprintf(completionTag, COMPLETION_TAG_BUFSIZE, - "COPY " UINT64_FORMAT, processedRowCount); + SafeSnprintf(completionTag, COMPLETION_TAG_BUFSIZE, + "COPY " UINT64_FORMAT, processedRowCount); #endif } @@ -2780,7 +2787,8 @@ CopyStatementHasFormat(CopyStmt *copyStatement, char *formatName) * further processing is needed. */ Node * -ProcessCopyStmt(CopyStmt *copyStatement, QueryCompletionCompat *completionTag, const char *queryString) +ProcessCopyStmt(CopyStmt *copyStatement, QueryCompletionCompat *completionTag, const + char *queryString) { /* * Handle special COPY "resultid" FROM STDIN WITH (format result) commands diff --git a/src/backend/distributed/commands/utility_hook.c b/src/backend/distributed/commands/utility_hook.c index af80c8b31..3257aa4af 100644 --- a/src/backend/distributed/commands/utility_hook.c +++ b/src/backend/distributed/commands/utility_hook.c @@ -91,8 +91,7 @@ static bool IsDropSchemaOrDB(Node *parsetree); void CitusProcessUtility(Node *node, const char *queryString, ProcessUtilityContext context, ParamListInfo params, DestReceiver *dest, - QueryCompletionCompat *completionTag - ) + QueryCompletionCompat *completionTag) { PlannedStmt *plannedStmt = makeNode(PlannedStmt); plannedStmt->commandType = CMD_UTILITY; @@ -119,8 +118,7 @@ multi_ProcessUtility(PlannedStmt *pstmt, ParamListInfo params, struct QueryEnvironment *queryEnv, DestReceiver *dest, - QueryCompletionCompat *completionTag - ) + QueryCompletionCompat *completionTag) { Node *parsetree = pstmt->utilityStmt; List *ddlJobs = NIL; diff --git a/src/backend/distributed/connection/placement_connection.c b/src/backend/distributed/connection/placement_connection.c index 5de19f9cd..6075fe8dc 100644 --- a/src/backend/distributed/connection/placement_connection.c +++ b/src/backend/distributed/connection/placement_connection.c @@ -26,7 +26,7 @@ #include "distributed/placement_connection.h" #include "distributed/relation_access_tracking.h" #include "utils/hsearch.h" -#if PG_VERSION_NUM >= PG_VERSION_13 +#if PG_VERSION_NUM >= PG_VERSION_13 #include "common/hashfn.h" #endif #include "utils/memutils.h" diff --git a/src/backend/distributed/deparser/ruleutils_13.c b/src/backend/distributed/deparser/ruleutils_13.c index 08b502dba..5e18bf500 100644 --- a/src/backend/distributed/deparser/ruleutils_13.c +++ b/src/backend/distributed/deparser/ruleutils_13.c @@ -105,6 +105,7 @@ /* Pretty flags */ #define PRETTYFLAG_PAREN 0x0001 #define PRETTYFLAG_INDENT 0x0002 +#define PRETTYFLAG_SCHEMA 0x0004 /* Default line length for pretty-print wrapping: 0 means wrap always */ #define WRAP_COLUMN_DEFAULT 0 @@ -112,6 +113,7 @@ /* macros to test if pretty action needed */ #define PRETTY_PAREN(context) ((context)->prettyFlags & PRETTYFLAG_PAREN) #define PRETTY_INDENT(context) ((context)->prettyFlags & PRETTYFLAG_INDENT) +#define PRETTY_SCHEMA(context) ((context)->prettyFlags & PRETTYFLAG_SCHEMA) /* ---------- @@ -135,7 +137,7 @@ typedef struct ParseExprKind special_exprkind; /* set only for exprkinds needing special * handling */ Bitmapset *appendparents; /* if not null, map child Vars of these relids - * back to the parent rel */ + * back to the parent rel */ } deparse_context; /* @@ -434,6 +436,8 @@ static void get_from_clause_coldeflist(RangeTblFunction *rtfunc, deparse_context *context); static void get_tablesample_def(TableSampleClause *tablesample, deparse_context *context); +static char *pg_get_triggerdef_worker(Oid trigid, bool pretty); +static void set_simple_column_names(deparse_namespace *dpns); static void get_opclass_name(Oid opclass, Oid actual_datatype, StringInfo buf); static Node *processIndirection(Node *node, deparse_context *context); @@ -2185,7 +2189,7 @@ get_simple_values_rte(Query *query, TupleDesc resultDesc) if (list_length(query->targetList) != list_length(result->eref->colnames)) return NULL; /* this probably cannot happen */ - colno = 0; + colno = 0; forboth(lc, query->targetList, lcn, result->eref->colnames) { TargetEntry *tle = (TargetEntry *) lfirst(lc); @@ -3554,7 +3558,7 @@ get_variable(Var *var, int levelsup, bool istoplevel, deparse_context *context) if (var->varnosyn > 0 && var->varnosyn <= list_length(dpns->rtable) && dpns->plan == NULL) { rte = rt_fetch(var->varnosyn, dpns->rtable); - // if the rte var->varnosync points to is not a regular table and it is a join + // if the rte var->varnosync points to is not a regular table and it is a join // then the correct relname will be found with var->varnosync and var->varattnosync // TODO:: this is a workaround and it can be simplified. if (rte->rtekind == RTE_JOIN && rte->relid == 0 && var->varnosyn != var->varno) { @@ -3577,7 +3581,7 @@ get_variable(Var *var, int levelsup, bool istoplevel, deparse_context *context) */ if (context->appendparents && dpns->appendrels) { - + Index pvarno = varno; AttrNumber pvarattno = varattno; AppendRelInfo *appinfo = dpns->appendrels[pvarno]; @@ -7508,6 +7512,307 @@ get_tablesample_def(TableSampleClause *tablesample, deparse_context *context) } } +char * +pg_get_triggerdef_command(Oid triggerId) +{ + Assert(OidIsValid(triggerId)); + + /* no need to have pretty SQL command */ + bool prettyOutput = false; + return pg_get_triggerdef_worker(triggerId, prettyOutput); +} + +static char * +pg_get_triggerdef_worker(Oid trigid, bool pretty) +{ + HeapTuple ht_trig; + Form_pg_trigger trigrec; + StringInfoData buf; + Relation tgrel; + ScanKeyData skey[1]; + SysScanDesc tgscan; + int findx = 0; + char *tgname; + char *tgoldtable; + char *tgnewtable; + Oid argtypes[1]; /* dummy */ + Datum value; + bool isnull; + + /* + * Fetch the pg_trigger tuple by the Oid of the trigger + */ + tgrel = table_open(TriggerRelationId, AccessShareLock); + + ScanKeyInit(&skey[0], + Anum_pg_trigger_oid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(trigid)); + + tgscan = systable_beginscan(tgrel, TriggerOidIndexId, true, + NULL, 1, skey); + + ht_trig = systable_getnext(tgscan); + + if (!HeapTupleIsValid(ht_trig)) + { + systable_endscan(tgscan); + table_close(tgrel, AccessShareLock); + return NULL; + } + + trigrec = (Form_pg_trigger) GETSTRUCT(ht_trig); + + /* + * Start the trigger definition. Note that the trigger's name should never + * be schema-qualified, but the trigger rel's name may be. + */ + initStringInfo(&buf); + + tgname = NameStr(trigrec->tgname); + appendStringInfo(&buf, "CREATE %sTRIGGER %s ", + OidIsValid(trigrec->tgconstraint) ? "CONSTRAINT " : "", + quote_identifier(tgname)); + + if (TRIGGER_FOR_BEFORE(trigrec->tgtype)) + appendStringInfoString(&buf, "BEFORE"); + else if (TRIGGER_FOR_AFTER(trigrec->tgtype)) + appendStringInfoString(&buf, "AFTER"); + else if (TRIGGER_FOR_INSTEAD(trigrec->tgtype)) + appendStringInfoString(&buf, "INSTEAD OF"); + else + elog(ERROR, "unexpected tgtype value: %d", trigrec->tgtype); + + if (TRIGGER_FOR_INSERT(trigrec->tgtype)) + { + appendStringInfoString(&buf, " INSERT"); + findx++; + } + if (TRIGGER_FOR_DELETE(trigrec->tgtype)) + { + if (findx > 0) + appendStringInfoString(&buf, " OR DELETE"); + else + appendStringInfoString(&buf, " DELETE"); + findx++; + } + if (TRIGGER_FOR_UPDATE(trigrec->tgtype)) + { + if (findx > 0) + appendStringInfoString(&buf, " OR UPDATE"); + else + appendStringInfoString(&buf, " UPDATE"); + findx++; + /* tgattr is first var-width field, so OK to access directly */ + if (trigrec->tgattr.dim1 > 0) + { + int i; + + appendStringInfoString(&buf, " OF "); + for (i = 0; i < trigrec->tgattr.dim1; i++) + { + char *attname; + + if (i > 0) + appendStringInfoString(&buf, ", "); + attname = get_attname(trigrec->tgrelid, + trigrec->tgattr.values[i], false); + appendStringInfoString(&buf, quote_identifier(attname)); + } + } + } + if (TRIGGER_FOR_TRUNCATE(trigrec->tgtype)) + { + if (findx > 0) + appendStringInfoString(&buf, " OR TRUNCATE"); + else + appendStringInfoString(&buf, " TRUNCATE"); + findx++; + } + + /* + * In non-pretty mode, always schema-qualify the target table name for + * safety. In pretty mode, schema-qualify only if not visible. + */ + appendStringInfo(&buf, " ON %s ", + pretty ? + generate_relation_name(trigrec->tgrelid, NIL) : + generate_qualified_relation_name(trigrec->tgrelid)); + + if (OidIsValid(trigrec->tgconstraint)) + { + if (OidIsValid(trigrec->tgconstrrelid)) + appendStringInfo(&buf, "FROM %s ", + generate_relation_name(trigrec->tgconstrrelid, NIL)); + if (!trigrec->tgdeferrable) + appendStringInfoString(&buf, "NOT "); + appendStringInfoString(&buf, "DEFERRABLE INITIALLY "); + if (trigrec->tginitdeferred) + appendStringInfoString(&buf, "DEFERRED "); + else + appendStringInfoString(&buf, "IMMEDIATE "); + } + + value = fastgetattr(ht_trig, Anum_pg_trigger_tgoldtable, + tgrel->rd_att, &isnull); + if (!isnull) + tgoldtable = NameStr(*DatumGetName(value)); + else + tgoldtable = NULL; + value = fastgetattr(ht_trig, Anum_pg_trigger_tgnewtable, + tgrel->rd_att, &isnull); + if (!isnull) + tgnewtable = NameStr(*DatumGetName(value)); + else + tgnewtable = NULL; + if (tgoldtable != NULL || tgnewtable != NULL) + { + appendStringInfoString(&buf, "REFERENCING "); + if (tgoldtable != NULL) + appendStringInfo(&buf, "OLD TABLE AS %s ", + quote_identifier(tgoldtable)); + if (tgnewtable != NULL) + appendStringInfo(&buf, "NEW TABLE AS %s ", + quote_identifier(tgnewtable)); + } + + if (TRIGGER_FOR_ROW(trigrec->tgtype)) + appendStringInfoString(&buf, "FOR EACH ROW "); + else + appendStringInfoString(&buf, "FOR EACH STATEMENT "); + + /* If the trigger has a WHEN qualification, add that */ + value = fastgetattr(ht_trig, Anum_pg_trigger_tgqual, + tgrel->rd_att, &isnull); + if (!isnull) + { + Node *qual; + char relkind; + deparse_context context; + deparse_namespace dpns; + RangeTblEntry *oldrte; + RangeTblEntry *newrte; + + appendStringInfoString(&buf, "WHEN ("); + + qual = stringToNode(TextDatumGetCString(value)); + + relkind = get_rel_relkind(trigrec->tgrelid); + + /* Build minimal OLD and NEW RTEs for the rel */ + oldrte = makeNode(RangeTblEntry); + oldrte->rtekind = RTE_RELATION; + oldrte->relid = trigrec->tgrelid; + oldrte->relkind = relkind; + oldrte->rellockmode = AccessShareLock; + oldrte->alias = makeAlias("old", NIL); + oldrte->eref = oldrte->alias; + oldrte->lateral = false; + oldrte->inh = false; + oldrte->inFromCl = true; + + newrte = makeNode(RangeTblEntry); + newrte->rtekind = RTE_RELATION; + newrte->relid = trigrec->tgrelid; + newrte->relkind = relkind; + newrte->rellockmode = AccessShareLock; + newrte->alias = makeAlias("new", NIL); + newrte->eref = newrte->alias; + newrte->lateral = false; + newrte->inh = false; + newrte->inFromCl = true; + + /* Build two-element rtable */ + memset(&dpns, 0, sizeof(dpns)); + dpns.rtable = list_make2(oldrte, newrte); + dpns.ctes = NIL; + set_rtable_names(&dpns, NIL, NULL); + set_simple_column_names(&dpns); + + /* Set up context with one-deep namespace stack */ + context.buf = &buf; + context.namespaces = list_make1(&dpns); + context.windowClause = NIL; + context.windowTList = NIL; + context.varprefix = true; + context.prettyFlags = pretty ? (PRETTYFLAG_PAREN | PRETTYFLAG_INDENT | PRETTYFLAG_SCHEMA) : PRETTYFLAG_INDENT; + context.wrapColumn = WRAP_COLUMN_DEFAULT; + context.indentLevel = PRETTYINDENT_STD; + context.special_exprkind = EXPR_KIND_NONE; + + get_rule_expr(qual, &context, false); + + appendStringInfoString(&buf, ") "); + } + + appendStringInfo(&buf, "EXECUTE FUNCTION %s(", + generate_function_name(trigrec->tgfoid, 0, + NIL, argtypes, + false, NULL, EXPR_KIND_NONE)); + + if (trigrec->tgnargs > 0) + { + char *p; + int i; + + value = fastgetattr(ht_trig, Anum_pg_trigger_tgargs, + tgrel->rd_att, &isnull); + if (isnull) + elog(ERROR, "tgargs is null for trigger %u", trigid); + p = (char *) VARDATA_ANY(DatumGetByteaPP(value)); + for (i = 0; i < trigrec->tgnargs; i++) + { + if (i > 0) + appendStringInfoString(&buf, ", "); + simple_quote_literal(&buf, p); + /* advance p to next string embedded in tgargs */ + while (*p) + p++; + p++; + } + } + + /* We deliberately do not put semi-colon at end */ + appendStringInfoChar(&buf, ')'); + + /* Clean up */ + systable_endscan(tgscan); + + table_close(tgrel, AccessShareLock); + + return buf.data; +} + +/* + * set_simple_column_names: fill in column aliases for non-query situations + * + * This handles EXPLAIN and cases where we only have relation RTEs. Without + * a join tree, we can't do anything smart about join RTEs, but we don't + * need to (note that EXPLAIN should never see join alias Vars anyway). + * If we do hit a join RTE we'll just process it like a non-table base RTE. + */ +static void +set_simple_column_names(deparse_namespace *dpns) +{ + ListCell *lc; + ListCell *lc2; + + /* Initialize dpns->rtable_columns to contain zeroed structs */ + dpns->rtable_columns = NIL; + while (list_length(dpns->rtable_columns) < list_length(dpns->rtable)) + dpns->rtable_columns = lappend(dpns->rtable_columns, + palloc0(sizeof(deparse_columns))); + + /* Assign unique column aliases within each RTE */ + forboth(lc, dpns->rtable, lc2, dpns->rtable_columns) + { + RangeTblEntry *rte = (RangeTblEntry *) lfirst(lc); + deparse_columns *colinfo = (deparse_columns *) lfirst(lc2); + + set_relation_column_names(dpns, rte, colinfo); + } +} + /* * get_opclass_name - fetch name of an index operator class * diff --git a/src/backend/distributed/executor/insert_select_executor.c b/src/backend/distributed/executor/insert_select_executor.c index c85bbf958..dba7755d8 100644 --- a/src/backend/distributed/executor/insert_select_executor.c +++ b/src/backend/distributed/executor/insert_select_executor.c @@ -346,8 +346,10 @@ WrapSubquery(Query *subquery) /* create range table entries */ Alias *selectAlias = makeAlias("citus_insert_select_subquery", NIL); - RangeTblEntry *newRangeTableEntry = RangeTableEntryFromNSItem(addRangeTableEntryForSubquery( - pstate, subquery, selectAlias, false, true)); + RangeTblEntry *newRangeTableEntry = RangeTableEntryFromNSItem( + addRangeTableEntryForSubquery( + pstate, subquery, + selectAlias, false, true)); outerQuery->rtable = list_make1(newRangeTableEntry); /* set the FROM expression to the subquery */ diff --git a/src/backend/distributed/executor/local_executor.c b/src/backend/distributed/executor/local_executor.c index d7b5d24b9..e15624776 100644 --- a/src/backend/distributed/executor/local_executor.c +++ b/src/backend/distributed/executor/local_executor.c @@ -334,7 +334,8 @@ LocallyPlanAndExecuteMultipleQueries(List *queryStrings, TupleDestination *tuple 0); int cursorOptions = 0; ParamListInfo paramListInfo = NULL; - PlannedStmt *localPlan = planner_compat(shardQuery, NULL, cursorOptions, paramListInfo); + PlannedStmt *localPlan = planner_compat(shardQuery, NULL, cursorOptions, + paramListInfo); totalProcessedRows += ExecuteLocalTaskPlan(localPlan, queryString, tupleDest, task, paramListInfo); diff --git a/src/backend/distributed/executor/multi_executor.c b/src/backend/distributed/executor/multi_executor.c index 92720e13e..d0c04e396 100644 --- a/src/backend/distributed/executor/multi_executor.c +++ b/src/backend/distributed/executor/multi_executor.c @@ -629,10 +629,10 @@ ExecutePlanIntoDestReceiver(PlannedStmt *queryPlan, ParamListInfo params, portal->visible = false; PortalDefineQuerySelectCompat(portal, - NULL, - "", - list_make1(queryPlan), - NULL); + NULL, + "", + list_make1(queryPlan), + NULL); PortalStart(portal, params, eflags, GetActiveSnapshot()); PortalRun(portal, count, false, true, dest, dest, NULL); diff --git a/src/backend/distributed/metadata/metadata_cache.c b/src/backend/distributed/metadata/metadata_cache.c index 030f12083..146675fa0 100644 --- a/src/backend/distributed/metadata/metadata_cache.c +++ b/src/backend/distributed/metadata/metadata_cache.c @@ -65,7 +65,7 @@ #include "utils/datum.h" #include "utils/elog.h" #include "utils/hsearch.h" -#if PG_VERSION_NUM >= PG_VERSION_13 +#if PG_VERSION_NUM >= PG_VERSION_13 #include "common/hashfn.h" #endif #include "utils/inval.h" diff --git a/src/backend/distributed/operations/node_protocol.c b/src/backend/distributed/operations/node_protocol.c index d5fca40cc..3d10b1303 100644 --- a/src/backend/distributed/operations/node_protocol.c +++ b/src/backend/distributed/operations/node_protocol.c @@ -223,7 +223,7 @@ master_get_table_ddl_events(PG_FUNCTION_ARGS) /* allocate DDL statements, and then save position in DDL statements */ List *tableDDLEventList = GetTableDDLEvents(relationId, includeSequenceDefaults); tableDDLEventCell = list_head(tableDDLEventList); - ListCellAndListWrapper* wrapper = palloc0(sizeof(ListCellAndListWrapper)); + ListCellAndListWrapper *wrapper = palloc0(sizeof(ListCellAndListWrapper)); wrapper->list = tableDDLEventList; wrapper->listCell = tableDDLEventCell; functionContext->user_fctx = wrapper; @@ -239,7 +239,8 @@ master_get_table_ddl_events(PG_FUNCTION_ARGS) */ functionContext = SRF_PERCALL_SETUP(); - ListCellAndListWrapper* wrapper = (ListCellAndListWrapper *) functionContext->user_fctx; + ListCellAndListWrapper *wrapper = + (ListCellAndListWrapper *) functionContext->user_fctx; if (wrapper->listCell != NULL) { char *ddlStatement = (char *) lfirst(wrapper->listCell); diff --git a/src/backend/distributed/planner/distributed_planner.c b/src/backend/distributed/planner/distributed_planner.c index 4c22b4f19..3a7f77a95 100644 --- a/src/backend/distributed/planner/distributed_planner.c +++ b/src/backend/distributed/planner/distributed_planner.c @@ -129,10 +129,10 @@ static PlannedStmt * PlanDistributedStmt(DistributedPlanningContext *planContext PlannedStmt * distributed_planner(Query *parse, #if PG_VERSION_NUM >= PG_VERSION_13 - const char *query_string, + const char *query_string, #endif - int cursorOptions, - ParamListInfo boundParams) + int cursorOptions, + ParamListInfo boundParams) { bool needsDistributedPlanning = false; bool fastPathRouterQuery = false; @@ -223,9 +223,9 @@ distributed_planner(Query *parse, * postgres' planner. */ planContext.plan = standard_planner_compat(planContext.query, - NULL, - planContext.cursorOptions, - planContext.boundParams); + NULL, + planContext.cursorOptions, + planContext.boundParams); if (needsDistributedPlanning) { result = PlanDistributedStmt(&planContext, rteIdCounter); diff --git a/src/backend/distributed/planner/multi_explain.c b/src/backend/distributed/planner/multi_explain.c index b134078b5..c89e40d1c 100644 --- a/src/backend/distributed/planner/multi_explain.c +++ b/src/backend/distributed/planner/multi_explain.c @@ -221,6 +221,7 @@ NonPushableInsertSelectExplainScan(CustomScanState *node, List *ancestors, bool repartition = distributedPlan->insertSelectMethod == INSERT_SELECT_REPARTITION; + if (es->analyze) { ereport(ERROR, (errmsg("EXPLAIN ANALYZE is currently not supported for INSERT " @@ -313,7 +314,8 @@ ExplainSubPlans(DistributedPlan *distributedPlan, ExplainState *es) INSTR_TIME_SET_ZERO(planduration); - ExplainOnePlanCompat(plan, into, es, queryString, params, NULL, &planduration, NULL); + ExplainOnePlanCompat(plan, into, es, queryString, params, NULL, &planduration, + NULL); if (es->format == EXPLAIN_FORMAT_TEXT) { @@ -961,7 +963,7 @@ worker_save_query_explain_analyze(PG_FUNCTION_ARGS) INSTR_TIME_SET_CURRENT(planStart); - PlannedStmt *plan = pg_plan_query(query, 0, NULL); + PlannedStmt *plan = pg_plan_query_compat(query, NULL, 0, NULL); INSTR_TIME_SET_CURRENT(planDuration); INSTR_TIME_SUBTRACT(planDuration, planStart); @@ -1126,14 +1128,14 @@ CitusExplainOneQuery(Query *query, int cursorOptions, IntoClause *into, INSTR_TIME_SET_CURRENT(planstart); /* plan the query */ - PlannedStmt *plan = pg_plan_query(query, cursorOptions, params); + PlannedStmt *plan = pg_plan_query_compat(query, NULL, cursorOptions, params); INSTR_TIME_SET_CURRENT(planduration); INSTR_TIME_SUBTRACT(planduration, planstart); /* run it (if needed) and produce output */ - ExplainOnePlan(plan, into, es, queryString, params, queryEnv, - &planduration); + ExplainOnePlanCompat(plan, into, es, queryString, params, queryEnv, + &planduration, NULL); } diff --git a/src/backend/distributed/planner/multi_physical_planner.c b/src/backend/distributed/planner/multi_physical_planner.c index 874b69bba..764d89e6a 100644 --- a/src/backend/distributed/planner/multi_physical_planner.c +++ b/src/backend/distributed/planner/multi_physical_planner.c @@ -131,7 +131,8 @@ static List * QueryFromList(List *rangeTableList); static Node * QueryJoinTree(MultiNode *multiNode, List *dependentJobList, List **rangeTableList); static void SetJoinRelatedColumnsCompat(RangeTblEntry *rangeTableEntry, - List *l_colnames, List *r_colnames, List* leftColVars, List* rightColVars); + List *l_colnames, List *r_colnames, + List *leftColVars, List *rightColVars); static RangeTblEntry * JoinRangeTableEntry(JoinExpr *joinExpr, List *dependentJobList, List *rangeTableList); static int ExtractRangeTableId(Node *node); @@ -1262,32 +1263,40 @@ JoinRangeTableEntry(JoinExpr *joinExpr, List *dependentJobList, List *rangeTable rangeTableEntry->joinaliasvars = joinedColumnVars; SetJoinRelatedColumnsCompat(rangeTableEntry, - leftColumnNames, rightColumnNames, leftColumnVars, rightColumnVars); + leftColumnNames, rightColumnNames, leftColumnVars, + rightColumnVars); return rangeTableEntry; } -static void SetJoinRelatedColumnsCompat(RangeTblEntry *rangeTableEntry, - List *leftColumnNames, List *rightColumnNames, List* leftColumnVars, List* rightColumnVars) { +static void +SetJoinRelatedColumnsCompat(RangeTblEntry *rangeTableEntry, + List *leftColumnNames, List *rightColumnNames, + List *leftColumnVars, List *rightColumnVars) +{ #if PG_VERSION_NUM >= PG_VERSION_13 /* We don't have any merged columns so set it to 0 */ - rangeTableEntry->joinmergedcols = 0; - Var* var = NULL; + rangeTableEntry->joinmergedcols = 0; + Var *var = NULL; int varId = 1; - foreach_ptr(var, leftColumnVars) { + foreach_ptr(var, leftColumnVars) + { rangeTableEntry->joinleftcols = lappend_int(rangeTableEntry->joinleftcols, varId); - varId++; - } + varId++; + } varId = 1; - foreach_ptr(var, rightColumnVars) { - rangeTableEntry->joinrightcols = lappend_int(rangeTableEntry->joinrightcols, varId); + foreach_ptr(var, rightColumnVars) + { + rangeTableEntry->joinrightcols = lappend_int(rangeTableEntry->joinrightcols, + varId); varId++; } #endif } + /* * ExtractRangeTableId gets the range table id from a node that could * either be a JoinExpr or RangeTblRef. diff --git a/src/backend/distributed/planner/relation_restriction_equivalence.c b/src/backend/distributed/planner/relation_restriction_equivalence.c index a322f4ccf..d12674be4 100644 --- a/src/backend/distributed/planner/relation_restriction_equivalence.c +++ b/src/backend/distributed/planner/relation_restriction_equivalence.c @@ -1358,6 +1358,7 @@ AddUnionAllSetOperationsToAttributeEquivalenceClass(AttributeEquivalenceClass ** continue; } int rtoffset = RangeTableOffsetCompat(root, appendRelInfo); + /* set the varno accordingly for this specific child */ varToBeAdded->varno = appendRelInfo->child_relid - rtoffset; @@ -1366,16 +1367,21 @@ AddUnionAllSetOperationsToAttributeEquivalenceClass(AttributeEquivalenceClass ** } } + /* * RangeTableOffsetCompat returns the range table offset(in glob->finalrtable) for the appendRelInfo. * For PG < 13 this is a no op. */ -static int RangeTableOffsetCompat(PlannerInfo *root, AppendRelInfo *appendRelInfo) { +static int +RangeTableOffsetCompat(PlannerInfo *root, AppendRelInfo *appendRelInfo) +{ #if PG_VERSION_NUM >= PG_VERSION_13 int i = 1; - for (i = 1 ; i < root->simple_rel_array_size; i++) { - RangeTblEntry* rte = root->simple_rte_array[i]; - if (rte->inh) { + for (i = 1; i < root->simple_rel_array_size; i++) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + if (rte->inh) + { break; } } diff --git a/src/backend/distributed/planner/tdigest_extension.c b/src/backend/distributed/planner/tdigest_extension.c index 5b67c16cf..123b170d4 100644 --- a/src/backend/distributed/planner/tdigest_extension.c +++ b/src/backend/distributed/planner/tdigest_extension.c @@ -14,6 +14,7 @@ #include "catalog/pg_type.h" #include "distributed/metadata_cache.h" #include "distributed/tdigest_extension.h" +#include "distributed/version_compat.h" #include "parser/parse_func.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" diff --git a/src/backend/distributed/test/distributed_intermediate_results.c b/src/backend/distributed/test/distributed_intermediate_results.c index 474403d3e..2ab999685 100644 --- a/src/backend/distributed/test/distributed_intermediate_results.c +++ b/src/backend/distributed/test/distributed_intermediate_results.c @@ -51,8 +51,8 @@ partition_task_list_results(PG_FUNCTION_ARGS) Query *parsedQuery = ParseQueryString(queryString, NULL, 0); PlannedStmt *queryPlan = pg_plan_query_compat(parsedQuery, queryString, - CURSOR_OPT_PARALLEL_OK, - NULL); + CURSOR_OPT_PARALLEL_OK, + NULL); if (!IsCitusCustomScan(queryPlan->planTree)) { ereport(ERROR, (errmsg("query must be distributed and shouldn't require " @@ -124,8 +124,8 @@ redistribute_task_list_results(PG_FUNCTION_ARGS) Query *parsedQuery = ParseQueryString(queryString, NULL, 0); PlannedStmt *queryPlan = pg_plan_query_compat(parsedQuery, queryString, - CURSOR_OPT_PARALLEL_OK, - NULL); + CURSOR_OPT_PARALLEL_OK, + NULL); if (!IsCitusCustomScan(queryPlan->planTree)) { ereport(ERROR, (errmsg("query must be distributed and shouldn't require " diff --git a/src/backend/distributed/test/foreign_key_relationship_query.c b/src/backend/distributed/test/foreign_key_relationship_query.c index d93938fa5..b1f1f7c8d 100644 --- a/src/backend/distributed/test/foreign_key_relationship_query.c +++ b/src/backend/distributed/test/foreign_key_relationship_query.c @@ -48,11 +48,11 @@ get_referencing_relation_id_list(PG_FUNCTION_ARGS) MemoryContextSwitchTo(functionContext->multi_call_memory_ctx); List *refList = list_copy( cacheEntry->referencingRelationsViaForeignKey); - ListCellAndListWrapper* wrapper = palloc0(sizeof(ListCellAndListWrapper)); + ListCellAndListWrapper *wrapper = palloc0(sizeof(ListCellAndListWrapper)); foreignRelationCell = list_head(refList); wrapper->list = refList; wrapper->listCell = foreignRelationCell; - functionContext->user_fctx = wrapper; + functionContext->user_fctx = wrapper; MemoryContextSwitchTo(oldContext); } @@ -64,7 +64,8 @@ get_referencing_relation_id_list(PG_FUNCTION_ARGS) */ functionContext = SRF_PERCALL_SETUP(); - ListCellAndListWrapper* wrapper = (ListCellAndListWrapper *) functionContext->user_fctx; + ListCellAndListWrapper *wrapper = + (ListCellAndListWrapper *) functionContext->user_fctx; if (wrapper->listCell != NULL) { Oid refId = lfirst_oid(wrapper->listCell); @@ -106,7 +107,7 @@ get_referenced_relation_id_list(PG_FUNCTION_ARGS) MemoryContextSwitchTo(functionContext->multi_call_memory_ctx); List *refList = list_copy(cacheEntry->referencedRelationsViaForeignKey); foreignRelationCell = list_head(refList); - ListCellAndListWrapper* wrapper = palloc0(sizeof(ListCellAndListWrapper)); + ListCellAndListWrapper *wrapper = palloc0(sizeof(ListCellAndListWrapper)); wrapper->list = refList; wrapper->listCell = foreignRelationCell; functionContext->user_fctx = wrapper; @@ -121,7 +122,8 @@ get_referenced_relation_id_list(PG_FUNCTION_ARGS) */ functionContext = SRF_PERCALL_SETUP(); - ListCellAndListWrapper* wrapper = (ListCellAndListWrapper *) functionContext->user_fctx; + ListCellAndListWrapper *wrapper = + (ListCellAndListWrapper *) functionContext->user_fctx; if (wrapper->listCell != NULL) { diff --git a/src/backend/distributed/transaction/relation_access_tracking.c b/src/backend/distributed/transaction/relation_access_tracking.c index c2af6917b..b5dafa135 100644 --- a/src/backend/distributed/transaction/relation_access_tracking.c +++ b/src/backend/distributed/transaction/relation_access_tracking.c @@ -29,7 +29,7 @@ #include "distributed/metadata_cache.h" #include "distributed/relation_access_tracking.h" #include "utils/hsearch.h" -#if PG_VERSION_NUM >= PG_VERSION_13 +#if PG_VERSION_NUM >= PG_VERSION_13 #include "common/hashfn.h" #endif #include "utils/lsyscache.h" diff --git a/src/backend/distributed/utils/foreign_key_relationship.c b/src/backend/distributed/utils/foreign_key_relationship.c index a116a5479..1dd61bcf5 100644 --- a/src/backend/distributed/utils/foreign_key_relationship.c +++ b/src/backend/distributed/utils/foreign_key_relationship.c @@ -31,7 +31,7 @@ #include "storage/lockdefs.h" #include "utils/fmgroids.h" #include "utils/hsearch.h" -#if PG_VERSION_NUM >= PG_VERSION_13 +#if PG_VERSION_NUM >= PG_VERSION_13 #include "common/hashfn.h" #endif #include "utils/memutils.h" diff --git a/src/backend/distributed/utils/maintenanced.c b/src/backend/distributed/utils/maintenanced.c index 77927ce16..4d6b79d3e 100644 --- a/src/backend/distributed/utils/maintenanced.c +++ b/src/backend/distributed/utils/maintenanced.c @@ -51,7 +51,7 @@ #include "storage/lmgr.h" #include "storage/lwlock.h" #include "tcop/tcopprot.h" -#if PG_VERSION_NUM >= PG_VERSION_13 +#if PG_VERSION_NUM >= PG_VERSION_13 #include "common/hashfn.h" #endif #include "utils/memutils.h" diff --git a/src/include/distributed/commands/multi_copy.h b/src/include/distributed/commands/multi_copy.h index e840b73b6..c1aed58ee 100644 --- a/src/include/distributed/commands/multi_copy.h +++ b/src/include/distributed/commands/multi_copy.h @@ -156,7 +156,8 @@ extern void AppendCopyBinaryHeaders(CopyOutState headerOutputState); extern void AppendCopyBinaryFooters(CopyOutState footerOutputState); extern void EndRemoteCopy(int64 shardId, List *connectionList); extern List * CreateRangeTable(Relation rel, AclMode requiredAccess); -extern Node * ProcessCopyStmt(CopyStmt *copyStatement, QueryCompletionCompat *completionTag, +extern Node * ProcessCopyStmt(CopyStmt *copyStatement, + QueryCompletionCompat *completionTag, const char *queryString); extern void CheckCopyPermissions(CopyStmt *copyStatement); extern bool IsCopyResultStmt(CopyStmt *copyStatement); diff --git a/src/include/distributed/commands/utility_hook.h b/src/include/distributed/commands/utility_hook.h index c9543d8b1..3b6893e27 100644 --- a/src/include/distributed/commands/utility_hook.h +++ b/src/include/distributed/commands/utility_hook.h @@ -54,11 +54,11 @@ extern void multi_ProcessUtility(PlannedStmt *pstmt, const char *queryString, ProcessUtilityContext context, ParamListInfo params, struct QueryEnvironment *queryEnv, DestReceiver *dest, QueryCompletionCompat *completionTag - ); + ); extern void CitusProcessUtility(Node *node, const char *queryString, ProcessUtilityContext context, ParamListInfo params, DestReceiver *dest, - QueryCompletionCompat * completionTag + QueryCompletionCompat *completionTag ); extern void MarkInvalidateForeignKeyGraph(void); extern void InvalidateForeignKeyGraphForDDL(void); diff --git a/src/include/distributed/distributed_planner.h b/src/include/distributed/distributed_planner.h index 9a71b939c..b385e8244 100644 --- a/src/include/distributed/distributed_planner.h +++ b/src/include/distributed/distributed_planner.h @@ -183,13 +183,18 @@ typedef struct CitusCustomScanPath } CitusCustomScanPath; -extern PlannedStmt * distributed_planner( - Query *parse, - #if PG_VERSION_NUM >= PG_VERSION_13 - const char *query_string, - #endif - int cursorOptions, - ParamListInfo boundParams); +#if PG_VERSION_NUM >= PG_VERSION_13 +extern PlannedStmt * distributed_planner(Query *parse, + const char *query_string, + int cursorOptions, + ParamListInfo boundParams); +#else +extern PlannedStmt * distributed_planner(Query *parse, + int cursorOptions, + ParamListInfo boundParams); +#endif + + extern List * ExtractRangeTableEntryList(Query *query); extern List * ExtractReferenceTableRTEList(List *rteList); extern bool NeedsDistributedPlanning(Query *query); diff --git a/src/include/distributed/listutils.h b/src/include/distributed/listutils.h index e37addbe1..fb37bf829 100644 --- a/src/include/distributed/listutils.h +++ b/src/include/distributed/listutils.h @@ -27,7 +27,8 @@ * in separate function calls, we need both the list and the current cell. * Therefore this wrapper stores both of them. */ -typedef struct ListCellAndListWrapper { +typedef struct ListCellAndListWrapper +{ List *list; ListCell *listCell; } ListCellAndListWrapper; diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index 9c2fbfb91..52d4bd5d9 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -30,30 +30,34 @@ #if PG_VERSION_NUM >= PG_VERSION_13 #define lnext_compat(l, r) lnext(l, r) -#define list_delete_cell_compat(l,c,p) list_delete_cell(l,c) -#define pg_plan_query_compat(p,q,c,b) pg_plan_query(p,q,c,b) -#define planner_compat(p,q,c,b) planner(p,q,c,b) -#define standard_planner_compat(a,b,c,d) standard_planner(a,b,c,d) -#define PortalDefineQuerySelectCompat(a,b,c,e,f) PortalDefineQuery(a,b,c,CMDTAG_SELECT,e,f) -#define getOwnedSequencesCompat(a,b) getOwnedSequences(a) -#define ExplainOnePlanCompat(a,b,c,d,e,f,g,h) ExplainOnePlan(a,b,c,d,e,f,g,h) +#define list_delete_cell_compat(l, c, p) list_delete_cell(l, c) +#define pg_plan_query_compat(p, q, c, b) pg_plan_query(p, q, c, b) +#define planner_compat(p, q, c, b) planner(p, q, c, b) +#define standard_planner_compat(a, b, c, d) standard_planner(a, b, c, d) +#define PortalDefineQuerySelectCompat(a, b, c, e, f) PortalDefineQuery(a, b, c, \ + CMDTAG_SELECT, e, \ + f) +#define getOwnedSequencesCompat(a, b) getOwnedSequences(a) +#define ExplainOnePlanCompat(a, b, c, d, e, f, g, h) ExplainOnePlan(a, b, c, d, e, f, g, \ + h) #define varoattno varattnosyn #define varnoold varnosyn -#define Set_ptr_value(a,b) a->ptr_value = b +#define Set_ptr_value(a, b) a->ptr_value = b #define RangeTableEntryFromNSItem(a) a->p_rte #define QueryCompletionCompat QueryCompletion #else /* pre PG13 */ #define lnext_compat(l, r) lnext(r) -#define list_delete_cell_compat(l,c,p) list_delete_cell(l,c,p) -#define pg_plan_query_compat(p,q,c,b) pg_plan_query(p,c,b) -#define planner_compat(p,q,c,b) planner(p,c,b) -#define standard_planner_compat(a,b,c,d) standard_planner(a,c,d) -#define PortalDefineQuerySelectCompat(a,b,c,e,f) PortalDefineQuery(a,b,c,"SELECT",e,f) -#define getOwnedSequencesCompat(a,b) getOwnedSequences(a,b) -#define ExplainOnePlanCompat(a,b,c,d,e,f,g,h) ExplainOnePlan(a,b,c,d,e,f,g) -#define Set_ptr_value(a,b) a->data.ptr_value = b +#define list_delete_cell_compat(l, c, p) list_delete_cell(l, c, p) +#define pg_plan_query_compat(p, q, c, b) pg_plan_query(p, c, b) +#define planner_compat(p, q, c, b) planner(p, c, b) +#define standard_planner_compat(a, b, c, d) standard_planner(a, c, d) +#define PortalDefineQuerySelectCompat(a, b, c, e, f) PortalDefineQuery(a, b, c, "SELECT", \ + e, f) +#define getOwnedSequencesCompat(a, b) getOwnedSequences(a, b) +#define ExplainOnePlanCompat(a, b, c, d, e, f, g, h) ExplainOnePlan(a, b, c, d, e, f, g) +#define Set_ptr_value(a, b) a->data.ptr_value = b #define RangeTableEntryFromNSItem(a) a -#define QueryCompletionCompat char +#define QueryCompletionCompat char #endif #if PG_VERSION_NUM >= PG_VERSION_12 From 157af140e4c50bd9e01570648c2ad419ac957f06 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Mon, 15 Jun 2020 12:46:02 +0300 Subject: [PATCH 33/52] ignore concurrent root page split debugs --- src/test/regress/bin/normalize.sed | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/regress/bin/normalize.sed b/src/test/regress/bin/normalize.sed index cd1866ea9..c982066cf 100644 --- a/src/test/regress/bin/normalize.sed +++ b/src/test/regress/bin/normalize.sed @@ -50,6 +50,9 @@ s/"(raw_events_second_user_id_value_1_key_|agg_events_user_id_value_1_agg_key_)[ # ignore could not consume warnings /WARNING: could not consume data from worker node/d +# ignore page split with pg13 +/DEBUG: concurrent ROOT page split/d + # ignore WAL warnings /DEBUG: .+creating and filling new WAL file/d From 1070828465da77e454b8dc6ef3954e2f20a608df Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Mon, 15 Jun 2020 13:25:35 +0300 Subject: [PATCH 34/52] update cte inline output for pg13 Make some macros in version_compat more robust Remove commented code in ruleutils Remove unnecessary variable assignments --- .gitattributes | 1 - .../distributed/deparser/ruleutils_13.c | 33 +- .../relation_restriction_equivalence.c | 2 +- src/include/distributed/version_compat.h | 8 +- src/test/regress/expected/cte_inline.out | 15 +- src/test/regress/expected/cte_inline_0.out | 411 ++--- src/test/regress/expected/cte_inline_1.out | 1443 ----------------- src/test/regress/sql/cte_inline.sql | 5 +- 8 files changed, 176 insertions(+), 1742 deletions(-) delete mode 100644 src/test/regress/expected/cte_inline_1.out diff --git a/.gitattributes b/.gitattributes index 4a012b4c1..86ee448b9 100644 --- a/.gitattributes +++ b/.gitattributes @@ -27,7 +27,6 @@ configure -whitespace # except these exceptions... src/backend/distributed/utils/citus_outfuncs.c -citus-style src/backend/distributed/utils/pg11_snprintf.c -citus-style -src/backend/distributed/deparser/ruleutils_10.c -citus-style src/backend/distributed/deparser/ruleutils_11.c -citus-style src/backend/distributed/deparser/ruleutils_12.c -citus-style src/backend/distributed/deparser/ruleutils_13.c -citus-style diff --git a/src/backend/distributed/deparser/ruleutils_13.c b/src/backend/distributed/deparser/ruleutils_13.c index 5e18bf500..e19a2d6a8 100644 --- a/src/backend/distributed/deparser/ruleutils_13.c +++ b/src/backend/distributed/deparser/ruleutils_13.c @@ -16,8 +16,6 @@ */ #include "distributed/pg_version_constants.h" -#pragma GCC optimize ("O0") - #include "pg_config.h" #if (PG_VERSION_NUM >= PG_VERSION_13) && (PG_VERSION_NUM < PG_VERSION_14) @@ -3964,21 +3962,22 @@ get_name_for_var_field(Var *var, int fieldno, var->varlevelsup, levelsup); dpns = (deparse_namespace *) list_nth(context->namespaces, netlevelsup); - /* - * If we have a syntactic referent for the Var, and we're working from a - * parse tree, prefer to use the syntactic referent. Otherwise, fall back - * on the semantic referent. (See comments in get_variable().) - */ - // if (var->varnosyn > 0 && dpns->plan == NULL) - // { - // varno = var->varnosyn; - // varattno = var->varattnosyn; - // } - // else - // { - varno = var->varno; - varattno = var->varattno; - // } + + varno = var->varno; + varattno = var->varattno; + + if (var->varnosyn > 0 && var->varnosyn <= list_length(dpns->rtable) && dpns->plan == NULL) { + rte = rt_fetch(var->varnosyn, dpns->rtable); + + // if the rte var->varnosync points to is not a regular table and it is a join + // then the correct relname will be found with var->varnosync and var->varattnosync + // TODO:: this is a workaround and it can be simplified. + if (rte->rtekind == RTE_JOIN && rte->relid == 0 && var->varnosyn != var->varno) { + varno = var->varnosyn; + varattno = var->varattnosyn; + } + } + /* * Try to find the relevant RTE in this rtable. In a plan tree, it's * likely that varno is OUTER_VAR or INNER_VAR, in which case we must dig diff --git a/src/backend/distributed/planner/relation_restriction_equivalence.c b/src/backend/distributed/planner/relation_restriction_equivalence.c index d12674be4..0d766fde0 100644 --- a/src/backend/distributed/planner/relation_restriction_equivalence.c +++ b/src/backend/distributed/planner/relation_restriction_equivalence.c @@ -1377,7 +1377,7 @@ RangeTableOffsetCompat(PlannerInfo *root, AppendRelInfo *appendRelInfo) { #if PG_VERSION_NUM >= PG_VERSION_13 int i = 1; - for (i = 1; i < root->simple_rel_array_size; i++) + for (; i < root->simple_rel_array_size; i++) { RangeTblEntry *rte = root->simple_rte_array[i]; if (rte->inh) diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index 52d4bd5d9..62849040a 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -42,8 +42,8 @@ h) #define varoattno varattnosyn #define varnoold varnosyn -#define Set_ptr_value(a, b) a->ptr_value = b -#define RangeTableEntryFromNSItem(a) a->p_rte +#define Set_ptr_value(a, b) ((a)->ptr_value = (b)) +#define RangeTableEntryFromNSItem(a) ((a)->p_rte) #define QueryCompletionCompat QueryCompletion #else /* pre PG13 */ #define lnext_compat(l, r) lnext(r) @@ -55,8 +55,8 @@ e, f) #define getOwnedSequencesCompat(a, b) getOwnedSequences(a, b) #define ExplainOnePlanCompat(a, b, c, d, e, f, g, h) ExplainOnePlan(a, b, c, d, e, f, g) -#define Set_ptr_value(a, b) a->data.ptr_value = b -#define RangeTableEntryFromNSItem(a) a +#define Set_ptr_value(a, b) ((a)->data.ptr_value = (b)) +#define RangeTableEntryFromNSItem(a) (a) #define QueryCompletionCompat char #endif #if PG_VERSION_NUM >= PG_VERSION_12 diff --git a/src/test/regress/expected/cte_inline.out b/src/test/regress/expected/cte_inline.out index 98cd7343d..b960c3ba6 100644 --- a/src/test/regress/expected/cte_inline.out +++ b/src/test/regress/expected/cte_inline.out @@ -12,10 +12,10 @@ INSERT INTO test_table SELECT i % 10, 'test' || i, row_to_json(row(i, i*18, 'tes -- server version because CTE inlining might produce -- different debug messages in PG 11 vs PG 12 SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int; - substring +SELECT substring(:'server_version', '\d+')::int >= 12; + ?column? --------------------------------------------------------------------- - 13 + t (1 row) SET client_min_messages TO DEBUG; @@ -725,7 +725,8 @@ WITH cte_1 AS (SELECT * FROM test_table), cte_2 AS (SELECT * FROM test_table ORDER BY 1 DESC LIMIT 3) (SELECT *, (SELECT 1) FROM cte_1 EXCEPT SELECT *, 1 FROM test_table) UNION -(SELECT *, 1 FROM cte_2); +(SELECT *, 1 FROM cte_2) +ORDER BY 1,2; DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: CTE cte_2 is going to be inlined via distributed planning DEBUG: Router planner cannot handle multi-shard select queries @@ -743,13 +744,13 @@ DEBUG: Creating router plan DEBUG: generating subplan XXX_3 for subquery SELECT key, value, other_value, (SELECT 1) FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 DEBUG: Router planner cannot handle multi-shard select queries DEBUG: generating subplan XXX_4 for subquery SELECT key, value, other_value, 1 FROM cte_inline.test_table -DEBUG: Plan XXX query after replacing subqueries and CTEs: (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value, intermediate_result."?column?" FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb, "?column?" integer) EXCEPT SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value, intermediate_result."?column?" FROM read_intermediate_result('XXX_4'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb, "?column?" integer)) UNION SELECT cte_2.key, cte_2.value, cte_2.other_value, 1 FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_2 +DEBUG: Plan XXX query after replacing subqueries and CTEs: (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value, intermediate_result."?column?" FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb, "?column?" integer) EXCEPT SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value, intermediate_result."?column?" FROM read_intermediate_result('XXX_4'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb, "?column?" integer)) UNION SELECT cte_2.key, cte_2.value, cte_2.other_value, 1 FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_2 ORDER BY 1, 2 DEBUG: Creating router plan key | value | other_value | ?column? --------------------------------------------------------------------- - 9 | test9 | {"f1": 9, "f2": 162, "f3": "test9"} | 1 - 9 | test29 | {"f1": 29, "f2": 522, "f3": "test29"} | 1 9 | test19 | {"f1": 19, "f2": 342, "f3": "test19"} | 1 + 9 | test29 | {"f1": 29, "f2": 522, "f3": "test29"} | 1 + 9 | test9 | {"f1": 9, "f2": 162, "f3": "test9"} | 1 (3 rows) -- cte_1 is safe to inline, even if because after inlining diff --git a/src/test/regress/expected/cte_inline_0.out b/src/test/regress/expected/cte_inline_0.out index cd57658a7..851a80d82 100644 --- a/src/test/regress/expected/cte_inline_0.out +++ b/src/test/regress/expected/cte_inline_0.out @@ -12,10 +12,10 @@ INSERT INTO test_table SELECT i % 10, 'test' || i, row_to_json(row(i, i*18, 'tes -- server version because CTE inlining might produce -- different debug messages in PG 11 vs PG 12 SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int; - substring +SELECT substring(:'server_version', '\d+')::int >= 12; + ?column? --------------------------------------------------------------------- - 12 + f (1 row) SET client_min_messages TO DEBUG; @@ -48,19 +48,7 @@ SELECT FROM cte_1 ORDER BY 2 DESC LIMIT 1; -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, other_value, (SELECT 1) FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 ORDER BY value DESC LIMIT 1 -DEBUG: Creating router plan -DEBUG: Plan is router executable - key | value | other_value | ?column? ---------------------------------------------------------------------- - 9 | test99 | {"f1": 99, "f2": 1782, "f3": "test99"} | 1 -(1 row) - +ERROR: syntax error at or near "NOT" -- the cte can be inlined because the unsupported -- part of the query (subquery in WHERE clause) -- doesn't access the cte @@ -304,9 +292,7 @@ FROM WHERE key = 1; DEBUG: CTE a is going to be inlined via distributed planning -DEBUG: Creating router plan -DEBUG: Plan is router executable -DETAIL: distribution column value: 1 +DEBUG: Router planner cannot handle multi-shard select queries count --------------------------------------------------------------------- 10 @@ -320,15 +306,7 @@ FROM a WHERE key = 1; -DEBUG: CTE a is going to be inlined via distributed planning -DEBUG: Creating router plan -DEBUG: Plan is router executable -DETAIL: distribution column value: 1 - count ---------------------------------------------------------------------- - 10 -(1 row) - +ERROR: syntax error at or near "NOT" -- using MATERIALIZED should cause inlining not to happen WITH a AS MATERIALIZED (SELECT * FROM test_table) SELECT @@ -337,17 +315,7 @@ FROM a WHERE key = 1; -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE a: SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) a WHERE (key OPERATOR(pg_catalog.=) 1) -DEBUG: Creating router plan -DEBUG: Plan is router executable - count ---------------------------------------------------------------------- - 10 -(1 row) - +ERROR: syntax error at or near "MATERIALIZED" -- EXPLAIN should show the difference between materialized an not materialized EXPLAIN (COSTS OFF) WITH a AS (SELECT * FROM test_table) SELECT @@ -357,20 +325,19 @@ FROM WHERE key = 1; DEBUG: CTE a is going to be inlined via distributed planning -DEBUG: Creating router plan -DEBUG: Plan is router executable -DETAIL: distribution column value: 1 - QUERY PLAN +DEBUG: Router planner cannot handle multi-shard select queries + QUERY PLAN --------------------------------------------------------------------- - Custom Scan (Citus Adaptive) - Task Count: 1 - Tasks Shown: All - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Aggregate - -> Seq Scan on test_table_1960000 test_table - Filter: (key = 1) -(8 rows) + Aggregate + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Aggregate + -> Seq Scan on test_table_1960000 test_table + Filter: (key = 1) +(9 rows) EXPLAIN (COSTS OFF) WITH a AS MATERIALIZED (SELECT * FROM test_table) SELECT @@ -379,31 +346,7 @@ FROM a WHERE key = 1; -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE a: SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) a WHERE (key OPERATOR(pg_catalog.=) 1) -DEBUG: Creating router plan -DEBUG: Plan is router executable - QUERY PLAN ---------------------------------------------------------------------- - Custom Scan (Citus Adaptive) - -> Distributed Subplan XXX_1 - -> Custom Scan (Citus Adaptive) - Task Count: 4 - Tasks Shown: One of 4 - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Seq Scan on test_table_1960000 test_table - Task Count: 1 - Tasks Shown: All - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Aggregate - -> Function Scan on read_intermediate_result intermediate_result - Filter: (key = 1) -(15 rows) - +ERROR: syntax error at or near "MATERIALIZED" -- citus should not inline the CTE because it is used multiple times WITH cte_1 AS (SELECT * FROM test_table) SELECT @@ -432,25 +375,7 @@ FROM JOIN cte_1 as second_entry USING (key); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: join prunable for intervals [-2147483648,-1073741825] and [-1073741824,-1] -DEBUG: join prunable for intervals [-2147483648,-1073741825] and [0,1073741823] -DEBUG: join prunable for intervals [-2147483648,-1073741825] and [1073741824,2147483647] -DEBUG: join prunable for intervals [-1073741824,-1] and [-2147483648,-1073741825] -DEBUG: join prunable for intervals [-1073741824,-1] and [0,1073741823] -DEBUG: join prunable for intervals [-1073741824,-1] and [1073741824,2147483647] -DEBUG: join prunable for intervals [0,1073741823] and [-2147483648,-1073741825] -DEBUG: join prunable for intervals [0,1073741823] and [-1073741824,-1] -DEBUG: join prunable for intervals [0,1073741823] and [1073741824,2147483647] -DEBUG: join prunable for intervals [1073741824,2147483647] and [-2147483648,-1073741825] -DEBUG: join prunable for intervals [1073741824,2147483647] and [-1073741824,-1] -DEBUG: join prunable for intervals [1073741824,2147483647] and [0,1073741823] - count ---------------------------------------------------------------------- - 1021 -(1 row) - +ERROR: syntax error at or near "NOT" -- EXPLAIN should show the differences between MATERIALIZED and NOT MATERIALIZED EXPLAIN (COSTS OFF) WITH cte_1 AS (SELECT * FROM test_table) SELECT @@ -498,36 +423,7 @@ FROM JOIN cte_1 as second_entry USING (key); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: join prunable for intervals [-2147483648,-1073741825] and [-1073741824,-1] -DEBUG: join prunable for intervals [-2147483648,-1073741825] and [0,1073741823] -DEBUG: join prunable for intervals [-2147483648,-1073741825] and [1073741824,2147483647] -DEBUG: join prunable for intervals [-1073741824,-1] and [-2147483648,-1073741825] -DEBUG: join prunable for intervals [-1073741824,-1] and [0,1073741823] -DEBUG: join prunable for intervals [-1073741824,-1] and [1073741824,2147483647] -DEBUG: join prunable for intervals [0,1073741823] and [-2147483648,-1073741825] -DEBUG: join prunable for intervals [0,1073741823] and [-1073741824,-1] -DEBUG: join prunable for intervals [0,1073741823] and [1073741824,2147483647] -DEBUG: join prunable for intervals [1073741824,2147483647] and [-2147483648,-1073741825] -DEBUG: join prunable for intervals [1073741824,2147483647] and [-1073741824,-1] -DEBUG: join prunable for intervals [1073741824,2147483647] and [0,1073741823] - QUERY PLAN ---------------------------------------------------------------------- - Aggregate - -> Custom Scan (Citus Adaptive) - Task Count: 4 - Tasks Shown: One of 4 - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Aggregate - -> Hash Join - Hash Cond: (test_table.key = test_table_1.key) - -> Seq Scan on test_table_1960000 test_table - -> Hash - -> Seq Scan on test_table_1960000 test_table_1 -(12 rows) - +ERROR: syntax error at or near "NOT" -- ctes with volatile functions are not -- inlined WITH cte_1 AS (SELECT *, random() FROM test_table) @@ -552,17 +448,7 @@ SELECT count(*) FROM cte_1; -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value, random() AS random FROM cte_inline.test_table -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value, intermediate_result.random FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb, random double precision)) cte_1 -DEBUG: Creating router plan -DEBUG: Plan is router executable - count ---------------------------------------------------------------------- - 101 -(1 row) - +ERROR: syntax error at or near "NOT" -- cte_1 should be able to inlined even if -- it is used one level below WITH cte_1 AS (SELECT * FROM test_table) @@ -618,11 +504,11 @@ SELECT count(*) LEFT JOIN test_table u2 ON u2.key = bar.key) AS foo ON TRUE; DEBUG: CTE bar is going to be inlined via distributed planning DEBUG: Router planner cannot handle multi-shard select queries - count ---------------------------------------------------------------------- - 10331 -(1 row) - +DEBUG: skipping recursive planning for the subquery since it contains references to outer queries +DEBUG: skipping recursive planning for the subquery since it contains references to outer queries +DEBUG: skipping recursive planning for the subquery since it contains references to outer queries +DEBUG: Router planner cannot handle multi-shard select queries +ERROR: CTEs that refer to other subqueries are not supported in multi-shard queries -- inlined CTE contains a reference to outer query -- should be fine (even if the recursive planning fails -- to recursively plan the query) @@ -695,9 +581,7 @@ DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Creating router plan -DEBUG: Plan is router executable -DETAIL: distribution column value: 1 +DEBUG: Router planner cannot handle multi-shard select queries count | key --------------------------------------------------------------------- (0 rows) @@ -733,7 +617,8 @@ WITH cte_1 AS (SELECT * FROM test_table), cte_2 AS (SELECT * FROM test_table ORDER BY 1 DESC LIMIT 3) (SELECT *, (SELECT 1) FROM cte_1 EXCEPT SELECT *, 1 FROM test_table) UNION -(SELECT *, 1 FROM cte_2); +(SELECT *, 1 FROM cte_2) +ORDER BY 1,2; DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: CTE cte_2 is going to be inlined via distributed planning DEBUG: Router planner cannot handle multi-shard select queries @@ -751,13 +636,13 @@ DEBUG: Creating router plan DEBUG: generating subplan XXX_3 for subquery SELECT key, value, other_value, (SELECT 1) FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 DEBUG: Router planner cannot handle multi-shard select queries DEBUG: generating subplan XXX_4 for subquery SELECT key, value, other_value, 1 FROM cte_inline.test_table -DEBUG: Plan XXX query after replacing subqueries and CTEs: (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value, intermediate_result."?column?" FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb, "?column?" integer) EXCEPT SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value, intermediate_result."?column?" FROM read_intermediate_result('XXX_4'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb, "?column?" integer)) UNION SELECT cte_2.key, cte_2.value, cte_2.other_value, 1 FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_2 +DEBUG: Plan XXX query after replacing subqueries and CTEs: (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value, intermediate_result."?column?" FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb, "?column?" integer) EXCEPT SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value, intermediate_result."?column?" FROM read_intermediate_result('XXX_4'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb, "?column?" integer)) UNION SELECT cte_2.key, cte_2.value, cte_2.other_value, 1 FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_2 ORDER BY 1, 2 DEBUG: Creating router plan key | value | other_value | ?column? --------------------------------------------------------------------- + 9 | test19 | {"f1": 19, "f2": 342, "f3": "test19"} | 1 9 | test29 | {"f1": 29, "f2": 522, "f3": "test29"} | 1 9 | test9 | {"f1": 9, "f2": 162, "f3": "test9"} | 1 - 9 | test19 | {"f1": 19, "f2": 342, "f3": "test19"} | 1 (3 rows) -- cte_1 is safe to inline, even if because after inlining @@ -811,11 +696,17 @@ FROM cte LEFT JOIN test_table USING (key); DEBUG: CTE cte is going to be inlined via distributed planning DEBUG: Router planner cannot handle multi-shard select queries - count ---------------------------------------------------------------------- - 1021 -(1 row) - +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte LEFT JOIN cte_inline.test_table USING (key)) +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte LEFT JOIN cte_inline.test_table USING (key)) +DEBUG: Router planner cannot handle multi-shard select queries +ERROR: cannot pushdown the subquery +DETAIL: Complex subqueries and CTEs cannot be in the outer part of the outer join -- the CTEs are very simple, so postgres -- can pull-up the subqueries after inlining -- the CTEs, and the query that we send to workers @@ -829,18 +720,10 @@ FROM DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: CTE cte_2 is going to be inlined via distributed planning DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: join prunable for intervals [-2147483648,-1073741825] and [-1073741824,-1] -DEBUG: join prunable for intervals [-2147483648,-1073741825] and [0,1073741823] -DEBUG: join prunable for intervals [-2147483648,-1073741825] and [1073741824,2147483647] -DEBUG: join prunable for intervals [-1073741824,-1] and [-2147483648,-1073741825] -DEBUG: join prunable for intervals [-1073741824,-1] and [0,1073741823] -DEBUG: join prunable for intervals [-1073741824,-1] and [1073741824,2147483647] -DEBUG: join prunable for intervals [0,1073741823] and [-2147483648,-1073741825] -DEBUG: join prunable for intervals [0,1073741823] and [-1073741824,-1] -DEBUG: join prunable for intervals [0,1073741823] and [1073741824,2147483647] -DEBUG: join prunable for intervals [1073741824,2147483647] and [-2147483648,-1073741825] -DEBUG: join prunable for intervals [1073741824,2147483647] and [-1073741824,-1] -DEBUG: join prunable for intervals [1073741824,2147483647] and [0,1073741823] +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for subquery SELECT key FROM cte_inline.test_table +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT test_table.key FROM cte_inline.test_table) cte_1 JOIN (SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) cte_2 USING (key)) +DEBUG: Router planner cannot handle multi-shard select queries count --------------------------------------------------------------------- 1021 @@ -896,12 +779,7 @@ DEBUG: Creating router plan -- we de still don't support it WITH cte_1 AS NOT MATERIALIZED (SELECT * FROM test_table) DELETE FROM test_table WHERE key NOT IN (SELECT key FROM cte_1); -DEBUG: complex joins are only supported when all distributed tables are co-located and joined on their distribution columns -DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Plan XXX query after replacing subqueries and CTEs: DELETE FROM cte_inline.test_table WHERE (NOT (key OPERATOR(pg_catalog.=) ANY (SELECT cte_1.key FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1))) -DEBUG: Creating router plan -DEBUG: Plan is router executable +ERROR: syntax error at or near "NOT" -- we don't inline CTEs if they are modifying CTEs WITH cte_1 AS (DELETE FROM test_table WHERE key % 3 = 1 RETURNING key) SELECT * FROM cte_1 ORDER BY 1 DESC LIMIT 3; @@ -920,18 +798,7 @@ DEBUG: Creating router plan -- NOT MATERIALIZED should not affect modifying CTEs WITH cte_1 AS NOT MATERIALIZED (DELETE FROM test_table WHERE key % 3 = 0 RETURNING key) SELECT count(*) FROM cte_1; -DEBUG: data-modifying statements are not supported in the WITH clauses of distributed queries -DEBUG: generating subplan XXX_1 for CTE cte_1: DELETE FROM cte_inline.test_table WHERE ((key OPERATOR(pg_catalog.%) 3) OPERATOR(pg_catalog.=) 0) RETURNING key -DEBUG: Creating router plan -DEBUG: Plan is router executable -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) cte_1 -DEBUG: Creating router plan -DEBUG: Plan is router executable - count ---------------------------------------------------------------------- - 164 -(1 row) - +ERROR: syntax error at or near "NOT" -- cte with column aliases SELECT * FROM test_table, (WITH cte_1 (x,y) AS (SELECT * FROM test_table), @@ -947,13 +814,13 @@ DEBUG: Router planner cannot handle multi-shard select queries DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT test_table.key, test_table.value, test_table.other_value, bar.z, bar.y, bar.key, bar.t, bar.m, bar.cte_2_key FROM cte_inline.test_table, (SELECT cte_2.z, cte_2.y, cte_2.key, cte_3.t, cte_3.m, cte_3.cte_2_key FROM (SELECT intermediate_result.value AS z, intermediate_result.other_value AS y, intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text, other_value jsonb, key integer)) cte_2, (SELECT cte_2_1.z AS t, cte_2_1.y AS m, cte_2_1.key AS cte_2_key FROM (SELECT intermediate_result.value AS z, intermediate_result.other_value AS y, intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text, other_value jsonb, key integer)) cte_2_1) cte_3) bar ORDER BY test_table.value, test_table.other_value, bar.z, bar.y, bar.t, bar.m, bar.cte_2_key LIMIT 5 DEBUG: Router planner cannot handle multi-shard select queries DEBUG: push down of limit count: 5 - key | value | other_value | z | y | key | t | m | cte_2_key + key | value | other_value | z | y | key | t | m | cte_2_key --------------------------------------------------------------------- - 2 | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | 2 | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | 2 - 2 | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | 2 | test12 | | 2 - 2 | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | 2 | test12 | | 2 - 2 | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | 2 | test12 | | 2 - 2 | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | 2 | test15 | {"f1": 15, "f2": 270, "f3": "test15"} | 5 + 0 | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | 0 | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | 0 + 0 | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | 0 | test0 | | 0 + 0 | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | 0 | test0 | | 0 + 0 | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | 0 | test0 | | 0 + 0 | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | 0 | test10 | {"f1": 10, "f2": 180, "f3": "test10"} | 0 (5 rows) -- cte used in HAVING subquery just works fine @@ -977,10 +844,12 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, count(*) DEBUG: Router planner cannot handle multi-shard select queries key | count --------------------------------------------------------------------- + 0 | 44 + 9 | 40 8 | 40 + 6 | 40 5 | 40 - 2 | 40 -(3 rows) +(5 rows) -- cte used in ORDER BY just works fine -- even if it is inlined @@ -1001,9 +870,9 @@ DEBUG: Router planner cannot handle multi-shard select queries DEBUG: push down of limit count: 3 key --------------------------------------------------------------------- - 8 - 8 - 8 + 9 + 9 + 9 (3 rows) PREPARE inlined_cte_without_params AS @@ -1036,7 +905,7 @@ DEBUG: Router planner cannot handle multi-shard select queries DEBUG: push down of limit count: 3 count --------------------------------------------------------------------- - 40 + 44 40 40 (3 rows) @@ -1044,7 +913,7 @@ DEBUG: push down of limit count: 3 EXECUTE inlined_cte_without_params; count --------------------------------------------------------------------- - 40 + 44 40 40 (3 rows) @@ -1052,7 +921,7 @@ EXECUTE inlined_cte_without_params; EXECUTE inlined_cte_without_params; count --------------------------------------------------------------------- - 40 + 44 40 40 (3 rows) @@ -1060,7 +929,7 @@ EXECUTE inlined_cte_without_params; EXECUTE inlined_cte_without_params; count --------------------------------------------------------------------- - 40 + 44 40 40 (3 rows) @@ -1068,7 +937,7 @@ EXECUTE inlined_cte_without_params; EXECUTE inlined_cte_without_params; count --------------------------------------------------------------------- - 40 + 44 40 40 (3 rows) @@ -1076,7 +945,7 @@ EXECUTE inlined_cte_without_params; EXECUTE inlined_cte_without_params; count --------------------------------------------------------------------- - 40 + 44 40 40 (3 rows) @@ -1091,49 +960,49 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, o DEBUG: Creating router plan key | value | other_value | ?column? --------------------------------------------------------------------- - 8 | test98 | | 1 - 8 | test98 | | 1 - 8 | test98 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 (3 rows) EXECUTE non_inlined_cte_without_params; key | value | other_value | ?column? --------------------------------------------------------------------- - 8 | test98 | | 1 - 8 | test98 | | 1 - 8 | test98 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 (3 rows) EXECUTE non_inlined_cte_without_params; key | value | other_value | ?column? --------------------------------------------------------------------- - 8 | test98 | | 1 - 8 | test98 | | 1 - 8 | test98 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 (3 rows) EXECUTE non_inlined_cte_without_params; key | value | other_value | ?column? --------------------------------------------------------------------- - 8 | test98 | | 1 - 8 | test98 | | 1 - 8 | test98 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 (3 rows) EXECUTE non_inlined_cte_without_params; key | value | other_value | ?column? --------------------------------------------------------------------- - 8 | test98 | | 1 - 8 | test98 | | 1 - 8 | test98 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 (3 rows) EXECUTE non_inlined_cte_without_params; key | value | other_value | ?column? --------------------------------------------------------------------- - 8 | test98 | | 1 - 8 | test98 | | 1 - 8 | test98 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 (3 rows) EXECUTE inlined_cte_has_parameter_on_non_dist_key('test1'); @@ -1159,7 +1028,8 @@ DEBUG: Router planner cannot handle multi-shard select queries DEBUG: push down of limit count: 3 count --------------------------------------------------------------------- -(0 rows) + 4 +(1 row) EXECUTE inlined_cte_has_parameter_on_non_dist_key('test4'); DEBUG: CTE cte_1 is going to be inlined via distributed planning @@ -1186,7 +1056,8 @@ DEBUG: Router planner cannot handle multi-shard select queries DEBUG: push down of limit count: 3 count --------------------------------------------------------------------- -(0 rows) + 4 +(1 row) EXECUTE inlined_cte_has_parameter_on_dist_key(1); DEBUG: CTE cte_1 is going to be inlined via distributed planning @@ -1207,7 +1078,8 @@ DEBUG: push down of limit count: 3 --------------------------------------------------------------------- 40 40 -(2 rows) + 40 +(3 rows) EXECUTE inlined_cte_has_parameter_on_dist_key(3); DEBUG: CTE cte_1 is going to be inlined via distributed planning @@ -1217,7 +1089,8 @@ DEBUG: push down of limit count: 3 --------------------------------------------------------------------- 40 40 -(2 rows) + 40 +(3 rows) EXECUTE inlined_cte_has_parameter_on_dist_key(4); DEBUG: CTE cte_1 is going to be inlined via distributed planning @@ -1227,7 +1100,8 @@ DEBUG: push down of limit count: 3 --------------------------------------------------------------------- 40 40 -(2 rows) + 40 +(3 rows) EXECUTE inlined_cte_has_parameter_on_dist_key(5); DEBUG: CTE cte_1 is going to be inlined via distributed planning @@ -1236,7 +1110,9 @@ DEBUG: push down of limit count: 3 count --------------------------------------------------------------------- 40 -(1 row) + 40 + 40 +(3 rows) EXECUTE inlined_cte_has_parameter_on_dist_key(6); DEBUG: CTE cte_1 is going to be inlined via distributed planning @@ -1247,7 +1123,8 @@ DEBUG: push down of limit count: 3 count --------------------------------------------------------------------- 40 -(1 row) + 40 +(2 rows) EXECUTE non_inlined_cte_has_parameter_on_dist_key(1); DEBUG: CTE cte_1 is going to be inlined via distributed planning @@ -1259,9 +1136,9 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, o DEBUG: Creating router plan key | value | other_value | ?column? --------------------------------------------------------------------- - 8 | test98 | | 1 - 8 | test98 | | 1 - 8 | test98 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 (3 rows) EXECUTE non_inlined_cte_has_parameter_on_dist_key(2); @@ -1274,9 +1151,9 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, o DEBUG: Creating router plan key | value | other_value | ?column? --------------------------------------------------------------------- - 8 | test98 | | 1 - 8 | test98 | | 1 - 8 | test98 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 (3 rows) EXECUTE non_inlined_cte_has_parameter_on_dist_key(3); @@ -1289,9 +1166,9 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, o DEBUG: Creating router plan key | value | other_value | ?column? --------------------------------------------------------------------- - 8 | test98 | | 1 - 8 | test98 | | 1 - 8 | test98 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 (3 rows) EXECUTE non_inlined_cte_has_parameter_on_dist_key(4); @@ -1304,9 +1181,9 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, o DEBUG: Creating router plan key | value | other_value | ?column? --------------------------------------------------------------------- - 8 | test98 | | 1 - 8 | test98 | | 1 - 8 | test98 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 (3 rows) EXECUTE non_inlined_cte_has_parameter_on_dist_key(5); @@ -1319,9 +1196,9 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, o DEBUG: Creating router plan key | value | other_value | ?column? --------------------------------------------------------------------- - 8 | test98 | | 1 - 8 | test98 | | 1 - 8 | test98 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 (3 rows) EXECUTE non_inlined_cte_has_parameter_on_dist_key(6); @@ -1336,49 +1213,49 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, o DEBUG: Creating router plan key | value | other_value | ?column? --------------------------------------------------------------------- - 8 | test98 | | 1 - 8 | test98 | | 1 - 8 | test98 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 (3 rows) EXECUTE retry_planning(1); DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: Router planner cannot handle multi-shard select queries - json_object_agg + json_object_agg --------------------------------------------------------------------- - { "2" : "test12", "2" : "test2", "2" : "test22", "2" : "test32", "2" : "test42", "2" : "test52", "2" : "test62", "2" : "test72", "2" : "test82", "2" : "test92", "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98" } + { "2" : "test12", "2" : "test2", "2" : "test22", "2" : "test32", "2" : "test42", "2" : "test52", "2" : "test62", "2" : "test72", "2" : "test82", "2" : "test92", "3" : "test13", "3" : "test23", "3" : "test3", "3" : "test33", "3" : "test43", "3" : "test53", "3" : "test63", "3" : "test73", "3" : "test83", "3" : "test93", "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "6" : "test16", "6" : "test26", "6" : "test36", "6" : "test46", "6" : "test56", "6" : "test6", "6" : "test66", "6" : "test76", "6" : "test86", "6" : "test96", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } (1 row) EXECUTE retry_planning(2); DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: Router planner cannot handle multi-shard select queries - json_object_agg + json_object_agg --------------------------------------------------------------------- - { "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98" } + { "3" : "test13", "3" : "test23", "3" : "test3", "3" : "test33", "3" : "test43", "3" : "test53", "3" : "test63", "3" : "test73", "3" : "test83", "3" : "test93", "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "6" : "test16", "6" : "test26", "6" : "test36", "6" : "test46", "6" : "test56", "6" : "test6", "6" : "test66", "6" : "test76", "6" : "test86", "6" : "test96", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } (1 row) EXECUTE retry_planning(3); DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: Router planner cannot handle multi-shard select queries - json_object_agg + json_object_agg --------------------------------------------------------------------- - { "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98" } + { "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "6" : "test16", "6" : "test26", "6" : "test36", "6" : "test46", "6" : "test56", "6" : "test6", "6" : "test66", "6" : "test76", "6" : "test86", "6" : "test96", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } (1 row) EXECUTE retry_planning(4); DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: Router planner cannot handle multi-shard select queries - json_object_agg + json_object_agg --------------------------------------------------------------------- - { "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98" } + { "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "6" : "test16", "6" : "test26", "6" : "test36", "6" : "test46", "6" : "test56", "6" : "test6", "6" : "test66", "6" : "test76", "6" : "test86", "6" : "test96", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } (1 row) EXECUTE retry_planning(5); DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: Router planner cannot handle multi-shard select queries - json_object_agg + json_object_agg --------------------------------------------------------------------- - { "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98" } + { "6" : "test16", "6" : "test26", "6" : "test36", "6" : "test46", "6" : "test56", "6" : "test6", "6" : "test66", "6" : "test76", "6" : "test86", "6" : "test96", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } (1 row) EXECUTE retry_planning(6); @@ -1386,9 +1263,9 @@ DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: Router planner cannot handle multi-shard select queries DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: Router planner cannot handle multi-shard select queries - json_object_agg + json_object_agg --------------------------------------------------------------------- - { "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98" } + { "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } (1 row) -- this test can only work if the CTE is recursively @@ -1410,7 +1287,7 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS c DEBUG: Creating router plan count --------------------------------------------------------------------- - 4800 + 11536 (1 row) -- this becomes a non-colocated subquery join @@ -1428,7 +1305,7 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS c DEBUG: Router planner cannot handle multi-shard select queries count --------------------------------------------------------------------- - 480 + 1136 (1 row) -- cte a has to be recursively planned because of OFFSET 0 @@ -1452,7 +1329,7 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT min(a.key) AS DEBUG: Creating router plan min --------------------------------------------------------------------- - 2 + 0 (1 row) -- after both CTEs are inlined, this becomes non-colocated subquery join @@ -1467,11 +1344,11 @@ DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FRO DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT cte_1.key, cte_1.value, cte_1.other_value, cte_2.key, cte_2.value, cte_2.other_value FROM ((SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) cte_1 JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_2 ON ((cte_1.value OPERATOR(pg_catalog.>) cte_2.value))) ORDER BY cte_1.key, cte_1.value, cte_1.other_value, cte_2.key, cte_2.value, cte_2.other_value DESC LIMIT 3 DEBUG: Router planner cannot handle multi-shard select queries DEBUG: push down of limit count: 3 - key | value | other_value | key | value | other_value + key | value | other_value | key | value | other_value --------------------------------------------------------------------- - 2 | test2 | {"f1": 2, "f2": 36, "f3": "test2"} | 2 | test12 | - 2 | test2 | {"f1": 2, "f2": 36, "f3": "test2"} | 2 | test12 | - 2 | test2 | {"f1": 2, "f2": 36, "f3": "test2"} | 2 | test12 | + 0 | test10 | {"f1": 10, "f2": 180, "f3": "test10"} | 0 | test0 | + 0 | test10 | {"f1": 10, "f2": 180, "f3": "test10"} | 0 | test0 | + 0 | test10 | {"f1": 10, "f2": 180, "f3": "test10"} | 0 | test0 | (3 rows) -- full join is only supported when both sides are @@ -1495,9 +1372,9 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT value FROM (( DEBUG: Creating router plan value --------------------------------------------------------------------- - test98 - test98 - test98 + test99 + test99 + test99 (3 rows) -- an unsupported agg. for multi-shard queries @@ -1506,9 +1383,9 @@ WITH cte_1 AS (SELECT * FROM test_table WHERE key > 1) SELECT json_object_agg(DISTINCT key, value) FROM cte_1; DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: Router planner cannot handle multi-shard select queries - json_object_agg + json_object_agg --------------------------------------------------------------------- - { "2" : "test12", "2" : "test2", "2" : "test22", "2" : "test32", "2" : "test42", "2" : "test52", "2" : "test62", "2" : "test72", "2" : "test82", "2" : "test92", "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98" } + { "2" : "test12", "2" : "test2", "2" : "test22", "2" : "test32", "2" : "test42", "2" : "test52", "2" : "test62", "2" : "test72", "2" : "test82", "2" : "test92", "3" : "test13", "3" : "test23", "3" : "test3", "3" : "test33", "3" : "test43", "3" : "test53", "3" : "test63", "3" : "test73", "3" : "test83", "3" : "test93", "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "6" : "test16", "6" : "test26", "6" : "test36", "6" : "test46", "6" : "test56", "6" : "test6", "6" : "test66", "6" : "test76", "6" : "test86", "6" : "test96", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } (1 row) -- both cte_1 and cte_2 are going to be inlined. diff --git a/src/test/regress/expected/cte_inline_1.out b/src/test/regress/expected/cte_inline_1.out deleted file mode 100644 index f5d011bd0..000000000 --- a/src/test/regress/expected/cte_inline_1.out +++ /dev/null @@ -1,1443 +0,0 @@ -CREATE SCHEMA cte_inline; -SET search_path TO cte_inline; -SET citus.next_shard_id TO 1960000; -CREATE TABLE test_table (key int, value text, other_value jsonb); -SELECT create_distributed_table ('test_table', 'key'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -INSERT INTO test_table SELECT i % 10, 'test' || i, row_to_json(row(i, i*18, 'test' || i)) FROM generate_series (0, 100) i; --- server version because CTE inlining might produce --- different debug messages in PG 11 vs PG 12 -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int; - substring ---------------------------------------------------------------------- - 11 -(1 row) - -SET client_min_messages TO DEBUG; --- Citus should not inline this CTE because otherwise it cannot --- plan the query -WITH cte_1 AS (SELECT * FROM test_table) -SELECT - *, (SELECT 1) -FROM - cte_1 -ORDER BY 1 DESC LIMIT 3; -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, other_value, (SELECT 1) FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 ORDER BY key DESC LIMIT 3 -DEBUG: Creating router plan -DEBUG: Plan is router executable - key | value | other_value | ?column? ---------------------------------------------------------------------- - 9 | test9 | {"f1": 9, "f2": 162, "f3": "test9"} | 1 - 9 | test19 | {"f1": 19, "f2": 342, "f3": "test19"} | 1 - 9 | test29 | {"f1": 29, "f2": 522, "f3": "test29"} | 1 -(3 rows) - --- Should still not be inlined even if NOT MATERIALIZED is passed -WITH cte_1 AS NOT MATERIALIZED (SELECT * FROM test_table) -SELECT - *, (SELECT 1) -FROM - cte_1 -ORDER BY 2 DESC LIMIT 1; -ERROR: syntax error at or near "NOT" --- the cte can be inlined because the unsupported --- part of the query (subquery in WHERE clause) --- doesn't access the cte -WITH cte_1 AS (SELECT * FROM test_table) -SELECT - count(*) -FROM - cte_1 -WHERE - key IN ( - SELECT - (SELECT 1) - FROM - test_table WHERE key = 1 - ); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Creating router plan -DEBUG: Plan is router executable -DETAIL: distribution column value: 1 -DEBUG: generating subplan XXX_1 for subquery SELECT (SELECT 1) FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.=) 1) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) cte_1 WHERE (key OPERATOR(pg_catalog.=) ANY (SELECT intermediate_result."?column?" FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result("?column?" integer))) -DEBUG: Router planner cannot handle multi-shard select queries - count ---------------------------------------------------------------------- - 10 -(1 row) - --- a similar query as the above, and this time the planning --- fails, but it fails because the subquery in WHERE clause --- cannot be planned by Citus -WITH cte_1 AS (SELECT * FROM test_table) -SELECT - count(*) -FROM - cte_1 -WHERE - key IN ( - SELECT - key - FROM - test_table - FOR UPDATE - ); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: SELECT FOR UPDATE with table replication factor > 1 not supported for non-reference tables. -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: SELECT FOR UPDATE with table replication factor > 1 not supported for non-reference tables. -ERROR: could not run distributed query with FOR UPDATE/SHARE commands -HINT: Consider using an equality filter on the distributed table's partition column. --- Citus does the inlining, the planning fails --- and retries without inlining, which works --- fine later via recursive planning -WITH cte_1 AS - (SELECT * - FROM test_table) -SELECT *, (SELECT 1) -FROM - (SELECT * - FROM cte_1) AS foo -ORDER BY 2 DESC LIMIT 1; -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, other_value, (SELECT 1) FROM (SELECT cte_1.key, cte_1.value, cte_1.other_value FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1) foo ORDER BY value DESC LIMIT 1 -DEBUG: Creating router plan -DEBUG: Plan is router executable - key | value | other_value | ?column? ---------------------------------------------------------------------- - 9 | test99 | {"f1": 99, "f2": 1782, "f3": "test99"} | 1 -(1 row) - --- a little more complicated query tree --- Citus does the inlining, the planning fails --- and retries without inlining, which works -WITH top_cte AS - (SELECT * - FROM test_table) -SELECT count(*) -FROM top_cte, - (WITH cte_1 AS - (SELECT * - FROM test_table) SELECT *, (SELECT 1) - FROM - (SELECT * - FROM cte_1) AS foo) AS bar; -DEBUG: CTE top_cte is going to be inlined via distributed planning -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE top_cte: SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_2 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Creating router plan -DEBUG: Plan is router executable -DEBUG: generating subplan XXX_3 for subquery SELECT key, value, other_value, (SELECT 1) FROM (SELECT cte_1.key, cte_1.value, cte_1.other_value FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1) foo -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) top_cte, (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value, intermediate_result."?column?" FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb, "?column?" integer)) bar -DEBUG: Creating router plan -DEBUG: Plan is router executable - count ---------------------------------------------------------------------- - 10201 -(1 row) - --- CTE is used inside a subquery in WHERE clause --- the query wouldn't work by inlining, so Citus --- retries again via recursive planning, which --- works fine -WITH cte_1 AS - (SELECT * - FROM test_table) -SELECT count(*) -FROM test_table -WHERE KEY IN - (SELECT (SELECT 1) - FROM - (SELECT *, - random() - FROM - (SELECT * - FROM cte_1) AS foo) AS bar); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Creating router plan -DEBUG: Plan is router executable -DEBUG: generating subplan XXX_2 for subquery SELECT (SELECT 1) FROM (SELECT foo.key, foo.value, foo.other_value, random() AS random FROM (SELECT cte_1.key, cte_1.value, cte_1.other_value FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1) foo) bar -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.=) ANY (SELECT intermediate_result."?column?" FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result("?column?" integer))) -DEBUG: Router planner cannot handle multi-shard select queries - count ---------------------------------------------------------------------- - 10 -(1 row) - --- cte_1 is used inside another CTE, but still --- doesn't work when inlined because it is finally --- used in an unsupported query --- but still works fine because recursive planning --- kicks in -WITH cte_1 AS - (SELECT * - FROM test_table) -SELECT (SELECT 1) AS KEY FROM ( - WITH cte_2 AS (SELECT *, random() - FROM (SELECT *,random() FROM cte_1) as foo) -SELECT *, random() FROM cte_2) as bar ORDER BY 1 DESC LIMIT 3; -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE cte_2: SELECT key, value, other_value, random, random() AS random FROM (SELECT cte_1.key, cte_1.value, cte_1.other_value, random() AS random FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) cte_1) foo -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (SELECT 1) AS key FROM (SELECT cte_2.key, cte_2.value, cte_2.other_value, cte_2.random, cte_2.random_1 AS random, random() AS random FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value, intermediate_result.random, intermediate_result.random_1 AS random FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb, random double precision, random_1 double precision)) cte_2(key, value, other_value, random, random_1)) bar(key, value, other_value, random, random_1, random_2) ORDER BY (SELECT 1) DESC LIMIT 3 -DEBUG: Creating router plan -DEBUG: Plan is router executable - key ---------------------------------------------------------------------- - 1 - 1 - 1 -(3 rows) - --- in this example, cte_2 can be inlined, because it is not used --- on any query that Citus cannot plan. However, cte_1 should not be --- inlined, because it is used with a subquery in target list -WITH cte_1 AS (SELECT * FROM test_table), - cte_2 AS (select * from test_table) -SELECT - count(*) -FROM - (SELECT *, (SELECT 1) FROM cte_1) as foo - JOIN - cte_2 - ON (true); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: CTE cte_2 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT cte_1.key, cte_1.value, cte_1.other_value, (SELECT 1) FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) cte_1) foo JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_2 ON (true)) -DEBUG: Router planner cannot handle multi-shard select queries - count ---------------------------------------------------------------------- - 10201 -(1 row) - --- unreferenced CTEs are just ignored --- by Citus/Postgres -WITH a AS (SELECT * FROM test_table) -SELECT - *, row_number() OVER () -FROM - test_table -WHERE - key = 1 -ORDER BY 3 DESC -LIMIT 5; -DEBUG: Creating router plan -DEBUG: Plan is router executable -DETAIL: distribution column value: 1 - key | value | other_value | row_number ---------------------------------------------------------------------- - 1 | test91 | {"f1": 91, "f2": 1638, "f3": "test91"} | 10 - 1 | test81 | {"f1": 81, "f2": 1458, "f3": "test81"} | 9 - 1 | test71 | {"f1": 71, "f2": 1278, "f3": "test71"} | 8 - 1 | test61 | {"f1": 61, "f2": 1098, "f3": "test61"} | 7 - 1 | test51 | {"f1": 51, "f2": 918, "f3": "test51"} | 6 -(5 rows) - --- router queries are affected by the distributed --- cte inlining -WITH a AS (SELECT * FROM test_table WHERE key = 1) -SELECT - *, (SELECT 1) -FROM - a -WHERE - key = 1 -ORDER BY 1 DESC -LIMIT 5; -DEBUG: CTE a is going to be inlined via distributed planning -DEBUG: Creating router plan -DEBUG: Plan is router executable -DETAIL: distribution column value: 1 - key | value | other_value | ?column? ---------------------------------------------------------------------- - 1 | test1 | {"f1": 1, "f2": 18, "f3": "test1"} | 1 - 1 | test11 | {"f1": 11, "f2": 198, "f3": "test11"} | 1 - 1 | test21 | {"f1": 21, "f2": 378, "f3": "test21"} | 1 - 1 | test31 | {"f1": 31, "f2": 558, "f3": "test31"} | 1 - 1 | test41 | {"f1": 41, "f2": 738, "f3": "test41"} | 1 -(5 rows) - --- non router queries are affected by the distributed --- cte inlining as well -WITH a AS (SELECT * FROM test_table) -SELECT - count(*) -FROM - a -WHERE - key = 1; -DEBUG: CTE a is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries - count ---------------------------------------------------------------------- - 10 -(1 row) - --- explicitely using NOT MATERIALIZED should result in the same -WITH a AS NOT MATERIALIZED (SELECT * FROM test_table) -SELECT - count(*) -FROM - a -WHERE - key = 1; -ERROR: syntax error at or near "NOT" --- using MATERIALIZED should cause inlining not to happen -WITH a AS MATERIALIZED (SELECT * FROM test_table) -SELECT - count(*) -FROM - a -WHERE - key = 1; -ERROR: syntax error at or near "MATERIALIZED" --- EXPLAIN should show the difference between materialized an not materialized -EXPLAIN (COSTS OFF) WITH a AS (SELECT * FROM test_table) -SELECT - count(*) -FROM - a -WHERE - key = 1; -DEBUG: CTE a is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries - QUERY PLAN ---------------------------------------------------------------------- - Aggregate - -> Custom Scan (Citus Adaptive) - Task Count: 4 - Tasks Shown: One of 4 - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Aggregate - -> Seq Scan on test_table_1960000 test_table - Filter: (key = 1) -(9 rows) - -EXPLAIN (COSTS OFF) WITH a AS MATERIALIZED (SELECT * FROM test_table) -SELECT - count(*) -FROM - a -WHERE - key = 1; -ERROR: syntax error at or near "MATERIALIZED" --- citus should not inline the CTE because it is used multiple times -WITH cte_1 AS (SELECT * FROM test_table) -SELECT - count(*) -FROM - cte_1 as first_entry - JOIN - cte_1 as second_entry - USING (key); -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) first_entry JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) second_entry USING (key)) -DEBUG: Creating router plan -DEBUG: Plan is router executable - count ---------------------------------------------------------------------- - 1021 -(1 row) - --- NOT MATERIALIZED should cause the query to be inlined twice -WITH cte_1 AS NOT MATERIALIZED (SELECT * FROM test_table) -SELECT - count(*) -FROM - cte_1 as first_entry - JOIN - cte_1 as second_entry - USING (key); -ERROR: syntax error at or near "NOT" --- EXPLAIN should show the differences between MATERIALIZED and NOT MATERIALIZED -EXPLAIN (COSTS OFF) WITH cte_1 AS (SELECT * FROM test_table) -SELECT - count(*) -FROM - cte_1 as first_entry - JOIN - cte_1 as second_entry - USING (key); -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) first_entry JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) second_entry USING (key)) -DEBUG: Creating router plan -DEBUG: Plan is router executable - QUERY PLAN ---------------------------------------------------------------------- - Custom Scan (Citus Adaptive) - -> Distributed Subplan XXX_1 - -> Custom Scan (Citus Adaptive) - Task Count: 4 - Tasks Shown: One of 4 - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Seq Scan on test_table_1960000 test_table - Task Count: 1 - Tasks Shown: All - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Aggregate - -> Merge Join - Merge Cond: (intermediate_result.key = intermediate_result_1.key) - -> Sort - Sort Key: intermediate_result.key - -> Function Scan on read_intermediate_result intermediate_result - -> Sort - Sort Key: intermediate_result_1.key - -> Function Scan on read_intermediate_result intermediate_result_1 -(21 rows) - -EXPLAIN (COSTS OFF) WITH cte_1 AS NOT MATERIALIZED (SELECT * FROM test_table) -SELECT - count(*) -FROM - cte_1 as first_entry - JOIN - cte_1 as second_entry - USING (key); -ERROR: syntax error at or near "NOT" --- ctes with volatile functions are not --- inlined -WITH cte_1 AS (SELECT *, random() FROM test_table) -SELECT - key, value -FROM - cte_1 -ORDER BY 2 DESC LIMIT 1; -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value, random() AS random FROM cte_inline.test_table -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value, intermediate_result.random FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb, random double precision)) cte_1 ORDER BY value DESC LIMIT 1 -DEBUG: Creating router plan -DEBUG: Plan is router executable - key | value ---------------------------------------------------------------------- - 9 | test99 -(1 row) - --- even with NOT MATERIALIZED volatile functions should not be inlined -WITH cte_1 AS NOT MATERIALIZED (SELECT *, random() FROM test_table) -SELECT - count(*) -FROM - cte_1; -ERROR: syntax error at or near "NOT" --- cte_1 should be able to inlined even if --- it is used one level below -WITH cte_1 AS (SELECT * FROM test_table) -SELECT - count(*) -FROM -( - WITH ct2 AS (SELECT * FROM cte_1) - SELECT * FROM ct2 -) as foo; -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: CTE ct2 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries - count ---------------------------------------------------------------------- - 101 -(1 row) - --- a similar query, but there is also --- one more cte, which relies on the previous --- CTE -WITH cte_1 AS (SELECT * FROM test_table) -SELECT - count(DISTINCT key) -FROM -( - WITH cte_2 AS (SELECT * FROM cte_1), - cte_3 AS (SELECT * FROM cte_2) - SELECT * FROM cte_3 -) as foo; -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: CTE cte_2 is going to be inlined via distributed planning -DEBUG: CTE cte_3 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries - count ---------------------------------------------------------------------- - 10 -(1 row) - --- inlined CTE contains a reference to outer query --- should be fine (because we pushdown the whole query) -SELECT count(*) - FROM - (SELECT * - FROM test_table) AS test_table_cte - JOIN LATERAL - (WITH bar AS (SELECT * - FROM test_table - WHERE key = test_table_cte.key) - SELECT * - FROM - bar - LEFT JOIN test_table u2 ON u2.key = bar.key) AS foo ON TRUE; -DEBUG: CTE bar is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: skipping recursive planning for the subquery since it contains references to outer queries -DEBUG: skipping recursive planning for the subquery since it contains references to outer queries -DEBUG: skipping recursive planning for the subquery since it contains references to outer queries -DEBUG: Router planner cannot handle multi-shard select queries -ERROR: CTEs that refer to other subqueries are not supported in multi-shard queries --- inlined CTE contains a reference to outer query --- should be fine (even if the recursive planning fails --- to recursively plan the query) -SELECT count(*) - FROM - (SELECT * - FROM test_table) AS test_table_cte - JOIN LATERAL - (WITH bar AS (SELECT * - FROM test_table - WHERE key = test_table_cte.key) - SELECT * - FROM - bar - LEFT JOIN test_table u2 ON u2.key = bar.value::int) AS foo ON TRUE; -DEBUG: CTE bar is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: skipping recursive planning for the subquery since it contains references to outer queries -DEBUG: skipping recursive planning for the subquery since it contains references to outer queries -DEBUG: skipping recursive planning for the subquery since it contains references to outer queries -DEBUG: Router planner cannot handle multi-shard select queries -ERROR: CTEs that refer to other subqueries are not supported in multi-shard queries --- inlined CTE can recursively planned later, that's the decision --- recursive planning makes --- LIMIT 5 in cte2 triggers recusrive planning, after cte inlining -WITH cte_1 AS (SELECT * FROM test_table) -SELECT - * -FROM -( - WITH ct2 AS (SELECT * FROM cte_1 ORDER BY 1, 2, 3 LIMIT 5) - SELECT * FROM ct2 -) as foo ORDER BY 1 DESC, 2 DESC, 3 DESC LIMIT 5; -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: CTE ct2 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 5 -DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) cte_1 ORDER BY key, value, other_value LIMIT 5 -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, other_value FROM (SELECT ct2.key, ct2.value, ct2.other_value FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) ct2) foo ORDER BY key DESC, value DESC, other_value DESC LIMIT 5 -DEBUG: Creating router plan -DEBUG: Plan is router executable - key | value | other_value ---------------------------------------------------------------------- - 0 | test30 | {"f1": 30, "f2": 540, "f3": "test30"} - 0 | test20 | {"f1": 20, "f2": 360, "f3": "test20"} - 0 | test100 | {"f1": 100, "f2": 1800, "f3": "test100"} - 0 | test10 | {"f1": 10, "f2": 180, "f3": "test10"} - 0 | test0 | {"f1": 0, "f2": 0, "f3": "test0"} -(5 rows) - --- all nested CTEs can be inlinied -WITH cte_1 AS ( - WITH cte_1 AS ( - WITH cte_1 AS ( - WITH cte_1 AS ( - WITH cte_1 AS ( - WITH cte_1 AS ( - WITH cte_1 AS (SELECT count(*), key FROM test_table GROUP BY key) - SELECT * FROM cte_1) - SELECT * FROM cte_1 WHERE key = 1) - SELECT * FROM cte_1 WHERE key = 2) - SELECT * FROM cte_1 WHERE key = 3) - SELECT * FROM cte_1 WHERE key = 4) - SELECT * FROM cte_1 WHERE key = 5) -SELECT * FROM cte_1 WHERE key = 6; -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries - count | key ---------------------------------------------------------------------- -(0 rows) - --- ctes can be inlined even if they are used --- in set operations -WITH cte_1 AS (SELECT * FROM test_table), - cte_2 AS (SELECT * FROM test_table) -SELECT count(*) FROM ( -(SELECT * FROM cte_1 EXCEPT SELECT * FROM test_table) -UNION -(SELECT * FROM cte_2)) as foo; -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: CTE cte_2 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) cte_1 -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_2 for subquery SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_3 for subquery SELECT key, value, other_value FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) cte_2 -DEBUG: Creating router plan -DEBUG: Plan is router executable -DEBUG: generating subplan XXX_4 for subquery (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb) EXCEPT SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) UNION SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_4'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) foo -DEBUG: Creating router plan -DEBUG: Plan is router executable - count ---------------------------------------------------------------------- - 101 -(1 row) - --- cte_1 is going to be inlined even inside another set operation -WITH cte_1 AS (SELECT * FROM test_table), - cte_2 AS (SELECT * FROM test_table ORDER BY 1 DESC LIMIT 3) -(SELECT *, (SELECT 1) FROM cte_1 EXCEPT SELECT *, 1 FROM test_table) -UNION -(SELECT *, 1 FROM cte_2); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: CTE cte_2 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 -DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM cte_inline.test_table ORDER BY key DESC LIMIT 3 -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_2 for CTE cte_2: SELECT key, value, other_value FROM cte_inline.test_table ORDER BY key DESC LIMIT 3 -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 -DEBUG: Creating router plan -DEBUG: Plan is router executable -DEBUG: generating subplan XXX_3 for subquery SELECT key, value, other_value, (SELECT 1) FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_4 for subquery SELECT key, value, other_value, 1 FROM cte_inline.test_table -DEBUG: Plan XXX query after replacing subqueries and CTEs: (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value, intermediate_result."?column?" FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb, "?column?" integer) EXCEPT SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value, intermediate_result."?column?" FROM read_intermediate_result('XXX_4'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb, "?column?" integer)) UNION SELECT cte_2.key, cte_2.value, cte_2.other_value, 1 FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_2 -DEBUG: Creating router plan -DEBUG: Plan is router executable - key | value | other_value | ?column? ---------------------------------------------------------------------- - 9 | test29 | {"f1": 29, "f2": 522, "f3": "test29"} | 1 - 9 | test9 | {"f1": 9, "f2": 162, "f3": "test9"} | 1 - 9 | test19 | {"f1": 19, "f2": 342, "f3": "test19"} | 1 -(3 rows) - --- cte_1 is safe to inline, even if because after inlining --- it'd be in a query tree where there is a query that is --- not supported by Citus unless recursively planned --- cte_2 is on another queryTree, should be fine -WITH cte_1 AS (SELECT * FROM test_table), - cte_2 AS (SELECT * FROM test_table) -(SELECT *, (SELECT key FROM cte_1) FROM test_table) -UNION -(SELECT *, 1 FROM cte_2); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: CTE cte_2 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_2 for CTE cte_2: SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -ERROR: could not run distributed query with subquery outside the FROM, WHERE and HAVING clauses -HINT: Consider using an equality filter on the distributed table's partition column. --- after inlining CTEs, the query becomes --- subquery pushdown with set operations -WITH cte_1 AS (SELECT * FROM test_table), - cte_2 AS (SELECT * FROM test_table) -SELECT max(key) FROM -( - SELECT * FROM cte_1 - UNION - SELECT * FROM cte_2 -) as bar; -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: CTE cte_2 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries - max ---------------------------------------------------------------------- - 9 -(1 row) - --- cte LEFT JOIN subquery should only work --- when CTE is inlined, as Citus currently --- doesn't know how to handle intermediate --- results in the outer parts of outer --- queries -WITH cte AS (SELECT * FROM test_table) -SELECT - count(*) -FROM - cte LEFT JOIN test_table USING (key); -DEBUG: CTE cte is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte LEFT JOIN cte_inline.test_table USING (key)) -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE cte: SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte LEFT JOIN cte_inline.test_table USING (key)) -DEBUG: Router planner cannot handle multi-shard select queries -ERROR: cannot pushdown the subquery -DETAIL: Complex subqueries and CTEs cannot be in the outer part of the outer join --- the CTEs are very simple, so postgres --- can pull-up the subqueries after inlining --- the CTEs, and the query that we send to workers --- becomes a join between two tables -WITH cte_1 AS (SELECT key FROM test_table), - cte_2 AS (SELECT key FROM test_table) -SELECT - count(*) -FROM - cte_1 JOIN cte_2 USING (key); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: CTE cte_2 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT key FROM cte_inline.test_table -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT test_table.key FROM cte_inline.test_table) cte_1 JOIN (SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) cte_2 USING (key)) -DEBUG: Router planner cannot handle multi-shard select queries - count ---------------------------------------------------------------------- - 1021 -(1 row) - --- the following query is kind of interesting --- During INSERT .. SELECT via coordinator, --- Citus moves the CTEs into SELECT part, and plans/execute --- the SELECT separately. Thus, fist_table_cte can be inlined --- by Citus -- but not by Postgres -WITH fist_table_cte AS - (SELECT * FROM test_table) -INSERT INTO test_table - (key, value) - SELECT - key, value - FROM - fist_table_cte; -DEBUG: distributed INSERT ... SELECT can only select from distributed tables -DEBUG: CTE fist_table_cte is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: performing repartitioned INSERT ... SELECT -DEBUG: partitioning SELECT query by column index 0 with name 'key' -DEBUG: distributed statement: INSERT INTO cte_inline.test_table_1960000 AS citus_table_alias (key, value) SELECT key, value FROM read_intermediate_results('{repartitioned_results_xxxxx_from_1960000_to_0}'::text[], 'binary'::citus_copy_format) intermediate_result(key integer, value text) -DEBUG: distributed statement: INSERT INTO cte_inline.test_table_1960001 AS citus_table_alias (key, value) SELECT key, value FROM read_intermediate_results('{repartitioned_results_xxxxx_from_1960001_to_1}'::text[], 'binary'::citus_copy_format) intermediate_result(key integer, value text) -DEBUG: distributed statement: INSERT INTO cte_inline.test_table_1960002 AS citus_table_alias (key, value) SELECT key, value FROM read_intermediate_results('{repartitioned_results_xxxxx_from_1960002_to_2}'::text[], 'binary'::citus_copy_format) intermediate_result(key integer, value text) -DEBUG: distributed statement: INSERT INTO cte_inline.test_table_1960003 AS citus_table_alias (key, value) SELECT key, value FROM read_intermediate_results('{repartitioned_results_xxxxx_from_1960003_to_3}'::text[], 'binary'::citus_copy_format) intermediate_result(key integer, value text) --- the following INSERT..SELECT is even more interesting --- the CTE becomes pushdownable -INSERT INTO test_table -WITH fist_table_cte AS - (SELECT * FROM test_table) - SELECT - key, value - FROM - fist_table_cte; -DEBUG: CTE fist_table_cte is going to be inlined via distributed planning -DEBUG: distributed statement: INSERT INTO cte_inline.test_table_1960000 AS citus_table_alias (key, value) SELECT key, value FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table_1960000 test_table) fist_table_cte WHERE ((worker_hash(key) OPERATOR(pg_catalog.>=) '-2147483648'::integer) AND (worker_hash(key) OPERATOR(pg_catalog.<=) '-1073741825'::integer)) -DEBUG: distributed statement: INSERT INTO cte_inline.test_table_1960001 AS citus_table_alias (key, value) SELECT key, value FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table_1960001 test_table) fist_table_cte WHERE ((worker_hash(key) OPERATOR(pg_catalog.>=) '-1073741824'::integer) AND (worker_hash(key) OPERATOR(pg_catalog.<=) '-1'::integer)) -DEBUG: distributed statement: INSERT INTO cte_inline.test_table_1960002 AS citus_table_alias (key, value) SELECT key, value FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table_1960002 test_table) fist_table_cte WHERE ((worker_hash(key) OPERATOR(pg_catalog.>=) 0) AND (worker_hash(key) OPERATOR(pg_catalog.<=) 1073741823)) -DEBUG: distributed statement: INSERT INTO cte_inline.test_table_1960003 AS citus_table_alias (key, value) SELECT key, value FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table_1960003 test_table) fist_table_cte WHERE ((worker_hash(key) OPERATOR(pg_catalog.>=) 1073741824) AND (worker_hash(key) OPERATOR(pg_catalog.<=) 2147483647)) -DEBUG: Plan is router executable --- update/delete/modifying ctes --- we don't support any cte inlining in modifications --- queries and modifying CTEs -WITH cte_1 AS (SELECT * FROM test_table) - DELETE FROM test_table WHERE key NOT IN (SELECT key FROM cte_1); -DEBUG: complex joins are only supported when all distributed tables are co-located and joined on their distribution columns -DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Plan XXX query after replacing subqueries and CTEs: DELETE FROM cte_inline.test_table WHERE (NOT (key OPERATOR(pg_catalog.=) ANY (SELECT cte_1.key FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1))) -DEBUG: Creating router plan -DEBUG: Plan is router executable --- NOT MATERIALIZED should not CTEs that are used in a modifying query, because --- we de still don't support it -WITH cte_1 AS NOT MATERIALIZED (SELECT * FROM test_table) - DELETE FROM test_table WHERE key NOT IN (SELECT key FROM cte_1); -ERROR: syntax error at or near "NOT" --- we don't inline CTEs if they are modifying CTEs -WITH cte_1 AS (DELETE FROM test_table WHERE key % 3 = 1 RETURNING key) -SELECT * FROM cte_1 ORDER BY 1 DESC LIMIT 3; -DEBUG: data-modifying statements are not supported in the WITH clauses of distributed queries -DEBUG: generating subplan XXX_1 for CTE cte_1: DELETE FROM cte_inline.test_table WHERE ((key OPERATOR(pg_catalog.%) 3) OPERATOR(pg_catalog.=) 1) RETURNING key -DEBUG: Creating router plan -DEBUG: Plan is router executable -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key FROM (SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) cte_1 ORDER BY key DESC LIMIT 3 -DEBUG: Creating router plan -DEBUG: Plan is router executable - key ---------------------------------------------------------------------- - 7 - 7 - 7 -(3 rows) - --- NOT MATERIALIZED should not affect modifying CTEs -WITH cte_1 AS NOT MATERIALIZED (DELETE FROM test_table WHERE key % 3 = 0 RETURNING key) -SELECT count(*) FROM cte_1; -ERROR: syntax error at or near "NOT" --- cte with column aliases -SELECT * FROM test_table, -(WITH cte_1 (x,y) AS (SELECT * FROM test_table), - cte_2 (z,y) AS (SELECT value, other_value, key FROM test_table), - cte_3 (t,m) AS (SELECT z, y, key as cte_2_key FROM cte_2) - SELECT * FROM cte_2, cte_3) as bar -ORDER BY value, other_value, z, y, t, m, cte_2_key -LIMIT 5; -DEBUG: CTE cte_3 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE cte_2: SELECT value, other_value, key FROM cte_inline.test_table -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT test_table.key, test_table.value, test_table.other_value, bar.z, bar.y, bar.key, bar.t, bar.m, bar.cte_2_key FROM cte_inline.test_table, (SELECT cte_2.z, cte_2.y, cte_2.key, cte_3.t, cte_3.m, cte_3.cte_2_key FROM (SELECT intermediate_result.value AS z, intermediate_result.other_value AS y, intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text, other_value jsonb, key integer)) cte_2, (SELECT cte_2_1.z AS t, cte_2_1.y AS m, cte_2_1.key AS cte_2_key FROM (SELECT intermediate_result.value AS z, intermediate_result.other_value AS y, intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text, other_value jsonb, key integer)) cte_2_1) cte_3) bar ORDER BY test_table.value, test_table.other_value, bar.z, bar.y, bar.t, bar.m, bar.cte_2_key LIMIT 5 -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 5 - key | value | other_value | z | y | key | t | m | cte_2_key ---------------------------------------------------------------------- - 0 | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | 0 | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | 0 - 0 | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | 0 | test0 | | 0 - 0 | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | 0 | test0 | | 0 - 0 | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | 0 | test0 | | 0 - 0 | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | 0 | test10 | {"f1": 10, "f2": 180, "f3": "test10"} | 0 -(5 rows) - --- cte used in HAVING subquery just works fine --- even if it is inlined -WITH cte_1 AS (SELECT max(key) as max FROM test_table) -SELECT - key, count(*) -FROM - test_table -GROUP BY - key -HAVING - (count(*) > (SELECT max FROM cte_1)) -ORDER BY 2 DESC, 1 DESC -LIMIT 5; -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT max(key) AS max FROM cte_inline.test_table -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, count(*) AS count FROM cte_inline.test_table GROUP BY key HAVING (count(*) OPERATOR(pg_catalog.>) (SELECT cte_1.max FROM (SELECT intermediate_result.max FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(max integer)) cte_1)) ORDER BY (count(*)) DESC, key DESC LIMIT 5 -DEBUG: Router planner cannot handle multi-shard select queries - key | count ---------------------------------------------------------------------- - 0 | 44 - 9 | 40 - 8 | 40 - 6 | 40 - 5 | 40 -(5 rows) - --- cte used in ORDER BY just works fine --- even if it is inlined -WITH cte_1 AS (SELECT max(key) as max FROM test_table) -SELECT - key -FROM - test_table JOIN cte_1 ON (key = max) -ORDER BY - cte_1.max -LIMIT 3; -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT max(key) AS max FROM cte_inline.test_table -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT test_table.key FROM (cte_inline.test_table JOIN (SELECT intermediate_result.max FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(max integer)) cte_1 ON ((test_table.key OPERATOR(pg_catalog.=) cte_1.max))) ORDER BY cte_1.max LIMIT 3 -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 - key ---------------------------------------------------------------------- - 9 - 9 - 9 -(3 rows) - -PREPARE inlined_cte_without_params AS - WITH cte_1 AS (SELECT count(*) FROM test_table GROUP BY key) - SELECT * FROM cte_1 ORDER BY 1 DESC LIMIT 3; -PREPARE non_inlined_cte_without_params AS - WITH cte_1 AS (SELECT * FROM test_table) - SELECT - *, (SELECT 1) - FROM - cte_1 ORDER BY 1 DESC, 2 DESC, 3 DESC LIMIT 3; -PREPARE inlined_cte_has_parameter_on_non_dist_key(text) AS - WITH cte_1 AS (SELECT count(*) FROM test_table WHERE value = $1 GROUP BY key) - SELECT * FROM cte_1 ORDER BY 1 DESC LIMIT 3; -PREPARE inlined_cte_has_parameter_on_dist_key(int) AS - WITH cte_1 AS (SELECT count(*) FROM test_table WHERE key > $1 GROUP BY key) - SELECT * FROM cte_1 ORDER BY 1 DESC LIMIT 3; -PREPARE non_inlined_cte_has_parameter_on_dist_key(int) AS - WITH cte_1 AS (SELECT * FROM test_table where key > $1) - SELECT - *, (SELECT 1) - FROM - cte_1 ORDER BY 1 DESC, 2 DESC, 3 DESC LIMIT 3; -PREPARE retry_planning(int) AS - WITH cte_1 AS (SELECT * FROM test_table WHERE key > $1) - SELECT json_object_agg(DISTINCT key, value) FROM cte_1 ORDER BY max(key), min(value) DESC LIMIT 3; -EXECUTE inlined_cte_without_params; -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 - count ---------------------------------------------------------------------- - 44 - 40 - 40 -(3 rows) - -EXECUTE inlined_cte_without_params; - count ---------------------------------------------------------------------- - 44 - 40 - 40 -(3 rows) - -EXECUTE inlined_cte_without_params; - count ---------------------------------------------------------------------- - 44 - 40 - 40 -(3 rows) - -EXECUTE inlined_cte_without_params; - count ---------------------------------------------------------------------- - 44 - 40 - 40 -(3 rows) - -EXECUTE inlined_cte_without_params; - count ---------------------------------------------------------------------- - 44 - 40 - 40 -(3 rows) - -EXECUTE inlined_cte_without_params; - count ---------------------------------------------------------------------- - 44 - 40 - 40 -(3 rows) - -EXECUTE non_inlined_cte_without_params; -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, other_value, (SELECT 1) FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 ORDER BY key DESC, value DESC, other_value DESC LIMIT 3 -DEBUG: Creating router plan -DEBUG: Plan is router executable - key | value | other_value | ?column? ---------------------------------------------------------------------- - 9 | test99 | | 1 - 9 | test99 | | 1 - 9 | test99 | | 1 -(3 rows) - -EXECUTE non_inlined_cte_without_params; - key | value | other_value | ?column? ---------------------------------------------------------------------- - 9 | test99 | | 1 - 9 | test99 | | 1 - 9 | test99 | | 1 -(3 rows) - -EXECUTE non_inlined_cte_without_params; - key | value | other_value | ?column? ---------------------------------------------------------------------- - 9 | test99 | | 1 - 9 | test99 | | 1 - 9 | test99 | | 1 -(3 rows) - -EXECUTE non_inlined_cte_without_params; - key | value | other_value | ?column? ---------------------------------------------------------------------- - 9 | test99 | | 1 - 9 | test99 | | 1 - 9 | test99 | | 1 -(3 rows) - -EXECUTE non_inlined_cte_without_params; - key | value | other_value | ?column? ---------------------------------------------------------------------- - 9 | test99 | | 1 - 9 | test99 | | 1 - 9 | test99 | | 1 -(3 rows) - -EXECUTE non_inlined_cte_without_params; - key | value | other_value | ?column? ---------------------------------------------------------------------- - 9 | test99 | | 1 - 9 | test99 | | 1 - 9 | test99 | | 1 -(3 rows) - -EXECUTE inlined_cte_has_parameter_on_non_dist_key('test1'); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 - count ---------------------------------------------------------------------- -(0 rows) - -EXECUTE inlined_cte_has_parameter_on_non_dist_key('test2'); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 - count ---------------------------------------------------------------------- - 4 -(1 row) - -EXECUTE inlined_cte_has_parameter_on_non_dist_key('test3'); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 - count ---------------------------------------------------------------------- - 4 -(1 row) - -EXECUTE inlined_cte_has_parameter_on_non_dist_key('test4'); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 - count ---------------------------------------------------------------------- -(0 rows) - -EXECUTE inlined_cte_has_parameter_on_non_dist_key('test5'); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 - count ---------------------------------------------------------------------- - 4 -(1 row) - -EXECUTE inlined_cte_has_parameter_on_non_dist_key('test6'); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 - count ---------------------------------------------------------------------- - 4 -(1 row) - -EXECUTE inlined_cte_has_parameter_on_dist_key(1); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 - count ---------------------------------------------------------------------- - 40 - 40 - 40 -(3 rows) - -EXECUTE inlined_cte_has_parameter_on_dist_key(2); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 - count ---------------------------------------------------------------------- - 40 - 40 - 40 -(3 rows) - -EXECUTE inlined_cte_has_parameter_on_dist_key(3); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 - count ---------------------------------------------------------------------- - 40 - 40 - 40 -(3 rows) - -EXECUTE inlined_cte_has_parameter_on_dist_key(4); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 - count ---------------------------------------------------------------------- - 40 - 40 - 40 -(3 rows) - -EXECUTE inlined_cte_has_parameter_on_dist_key(5); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 - count ---------------------------------------------------------------------- - 40 - 40 - 40 -(3 rows) - -EXECUTE inlined_cte_has_parameter_on_dist_key(6); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 - count ---------------------------------------------------------------------- - 40 - 40 -(2 rows) - -EXECUTE non_inlined_cte_has_parameter_on_dist_key(1); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.>) 1) -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, other_value, (SELECT 1) FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 ORDER BY key DESC, value DESC, other_value DESC LIMIT 3 -DEBUG: Creating router plan -DEBUG: Plan is router executable - key | value | other_value | ?column? ---------------------------------------------------------------------- - 9 | test99 | | 1 - 9 | test99 | | 1 - 9 | test99 | | 1 -(3 rows) - -EXECUTE non_inlined_cte_has_parameter_on_dist_key(2); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.>) 2) -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, other_value, (SELECT 1) FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 ORDER BY key DESC, value DESC, other_value DESC LIMIT 3 -DEBUG: Creating router plan -DEBUG: Plan is router executable - key | value | other_value | ?column? ---------------------------------------------------------------------- - 9 | test99 | | 1 - 9 | test99 | | 1 - 9 | test99 | | 1 -(3 rows) - -EXECUTE non_inlined_cte_has_parameter_on_dist_key(3); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.>) 3) -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, other_value, (SELECT 1) FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 ORDER BY key DESC, value DESC, other_value DESC LIMIT 3 -DEBUG: Creating router plan -DEBUG: Plan is router executable - key | value | other_value | ?column? ---------------------------------------------------------------------- - 9 | test99 | | 1 - 9 | test99 | | 1 - 9 | test99 | | 1 -(3 rows) - -EXECUTE non_inlined_cte_has_parameter_on_dist_key(4); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.>) 4) -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, other_value, (SELECT 1) FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 ORDER BY key DESC, value DESC, other_value DESC LIMIT 3 -DEBUG: Creating router plan -DEBUG: Plan is router executable - key | value | other_value | ?column? ---------------------------------------------------------------------- - 9 | test99 | | 1 - 9 | test99 | | 1 - 9 | test99 | | 1 -(3 rows) - -EXECUTE non_inlined_cte_has_parameter_on_dist_key(5); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.>) 5) -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, other_value, (SELECT 1) FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 ORDER BY key DESC, value DESC, other_value DESC LIMIT 3 -DEBUG: Creating router plan -DEBUG: Plan is router executable - key | value | other_value | ?column? ---------------------------------------------------------------------- - 9 | test99 | | 1 - 9 | test99 | | 1 - 9 | test99 | | 1 -(3 rows) - -EXECUTE non_inlined_cte_has_parameter_on_dist_key(6); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.>) 6) -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, other_value, (SELECT 1) FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 ORDER BY key DESC, value DESC, other_value DESC LIMIT 3 -DEBUG: Creating router plan -DEBUG: Plan is router executable - key | value | other_value | ?column? ---------------------------------------------------------------------- - 9 | test99 | | 1 - 9 | test99 | | 1 - 9 | test99 | | 1 -(3 rows) - -EXECUTE retry_planning(1); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries - json_object_agg ---------------------------------------------------------------------- - { "2" : "test12", "2" : "test2", "2" : "test22", "2" : "test32", "2" : "test42", "2" : "test52", "2" : "test62", "2" : "test72", "2" : "test82", "2" : "test92", "3" : "test13", "3" : "test23", "3" : "test3", "3" : "test33", "3" : "test43", "3" : "test53", "3" : "test63", "3" : "test73", "3" : "test83", "3" : "test93", "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "6" : "test16", "6" : "test26", "6" : "test36", "6" : "test46", "6" : "test56", "6" : "test6", "6" : "test66", "6" : "test76", "6" : "test86", "6" : "test96", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } -(1 row) - -EXECUTE retry_planning(2); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries - json_object_agg ---------------------------------------------------------------------- - { "3" : "test13", "3" : "test23", "3" : "test3", "3" : "test33", "3" : "test43", "3" : "test53", "3" : "test63", "3" : "test73", "3" : "test83", "3" : "test93", "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "6" : "test16", "6" : "test26", "6" : "test36", "6" : "test46", "6" : "test56", "6" : "test6", "6" : "test66", "6" : "test76", "6" : "test86", "6" : "test96", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } -(1 row) - -EXECUTE retry_planning(3); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries - json_object_agg ---------------------------------------------------------------------- - { "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "6" : "test16", "6" : "test26", "6" : "test36", "6" : "test46", "6" : "test56", "6" : "test6", "6" : "test66", "6" : "test76", "6" : "test86", "6" : "test96", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } -(1 row) - -EXECUTE retry_planning(4); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries - json_object_agg ---------------------------------------------------------------------- - { "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "6" : "test16", "6" : "test26", "6" : "test36", "6" : "test46", "6" : "test56", "6" : "test6", "6" : "test66", "6" : "test76", "6" : "test86", "6" : "test96", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } -(1 row) - -EXECUTE retry_planning(5); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries - json_object_agg ---------------------------------------------------------------------- - { "6" : "test16", "6" : "test26", "6" : "test36", "6" : "test46", "6" : "test56", "6" : "test6", "6" : "test66", "6" : "test76", "6" : "test86", "6" : "test96", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } -(1 row) - -EXECUTE retry_planning(6); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries - json_object_agg ---------------------------------------------------------------------- - { "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } -(1 row) - --- this test can only work if the CTE is recursively --- planned -WITH b AS (SELECT * FROM test_table) -SELECT count(*) FROM (SELECT key as x FROM test_table OFFSET 0) as ref LEFT JOIN b ON (ref.x = b.key); -DEBUG: CTE b is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT key AS x FROM cte_inline.test_table OFFSET 0 -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT intermediate_result.x FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(x integer)) ref LEFT JOIN (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) b ON ((ref.x OPERATOR(pg_catalog.=) b.key))) -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE b: SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_2 for subquery SELECT key AS x FROM cte_inline.test_table OFFSET 0 -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT intermediate_result.x FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(x integer)) ref LEFT JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) b ON ((ref.x OPERATOR(pg_catalog.=) b.key))) -DEBUG: Creating router plan -DEBUG: Plan is router executable - count ---------------------------------------------------------------------- - 11536 -(1 row) - --- this becomes a non-colocated subquery join --- because after the CTEs are inlined the joins --- become a non-colocated subquery join -WITH a AS (SELECT * FROM test_table), -b AS (SELECT * FROM test_table) -SELECT count(*) FROM a LEFT JOIN b ON (a.value = b.value); -DEBUG: CTE a is going to be inlined via distributed planning -DEBUG: CTE b is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) a LEFT JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) b ON ((a.value OPERATOR(pg_catalog.=) b.value))) -DEBUG: Router planner cannot handle multi-shard select queries - count ---------------------------------------------------------------------- - 1136 -(1 row) - --- cte a has to be recursively planned because of OFFSET 0 --- after that, cte b also requires recursive planning -WITH a AS (SELECT * FROM test_table OFFSET 0), -b AS (SELECT * FROM test_table) -SELECT min(a.key) FROM a LEFT JOIN b ON (a.value = b.value); -DEBUG: CTE a is going to be inlined via distributed planning -DEBUG: CTE b is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM cte_inline.test_table OFFSET 0 -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT min(a.key) AS min FROM ((SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) a LEFT JOIN (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) b ON ((a.value OPERATOR(pg_catalog.=) b.value))) -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE a: SELECT key, value, other_value FROM cte_inline.test_table OFFSET 0 -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_2 for CTE b: SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT min(a.key) AS min FROM ((SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) a LEFT JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) b ON ((a.value OPERATOR(pg_catalog.=) b.value))) -DEBUG: Creating router plan -DEBUG: Plan is router executable - min ---------------------------------------------------------------------- - 0 -(1 row) - --- after both CTEs are inlined, this becomes non-colocated subquery join -WITH cte_1 AS (SELECT * FROM test_table), -cte_2 AS (SELECT * FROM test_table) -SELECT * FROM cte_1 JOIN cte_2 ON (cte_1.value > cte_2.value) ORDER BY 1,2,3,4,5,6 DESC LIMIT 3;; -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: CTE cte_2 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT cte_1.key, cte_1.value, cte_1.other_value, cte_2.key, cte_2.value, cte_2.other_value FROM ((SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) cte_1 JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_2 ON ((cte_1.value OPERATOR(pg_catalog.>) cte_2.value))) ORDER BY cte_1.key, cte_1.value, cte_1.other_value, cte_2.key, cte_2.value, cte_2.other_value DESC LIMIT 3 -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: push down of limit count: 3 - key | value | other_value | key | value | other_value ---------------------------------------------------------------------- - 0 | test10 | {"f1": 10, "f2": 180, "f3": "test10"} | 0 | test0 | - 0 | test10 | {"f1": 10, "f2": 180, "f3": "test10"} | 0 | test0 | - 0 | test10 | {"f1": 10, "f2": 180, "f3": "test10"} | 0 | test0 | -(3 rows) - --- full join is only supported when both sides are --- recursively planned -WITH cte_1 AS (SELECT value FROM test_table WHERE key > 1), - cte_2 AS (SELECT value FROM test_table WHERE key > 3) -SELECT * FROM cte_1 FULL JOIN cte_2 USING (value) ORDER BY 1 DESC LIMIT 3;; -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: CTE cte_2 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT value FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.>) 3) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT value FROM ((SELECT test_table.value FROM cte_inline.test_table WHERE (test_table.key OPERATOR(pg_catalog.>) 1)) cte_1 FULL JOIN (SELECT intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text)) cte_2 USING (value)) ORDER BY value DESC LIMIT 3 -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT value FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.>) 1) -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_2 for CTE cte_2: SELECT value FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.>) 3) -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT value FROM ((SELECT intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text)) cte_1 FULL JOIN (SELECT intermediate_result.value FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(value text)) cte_2 USING (value)) ORDER BY value DESC LIMIT 3 -DEBUG: Creating router plan -DEBUG: Plan is router executable - value ---------------------------------------------------------------------- - test99 - test99 - test99 -(3 rows) - --- an unsupported agg. for multi-shard queries --- so CTE has to be recursively planned -WITH cte_1 AS (SELECT * FROM test_table WHERE key > 1) -SELECT json_object_agg(DISTINCT key, value) FROM cte_1; -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries - json_object_agg ---------------------------------------------------------------------- - { "2" : "test12", "2" : "test2", "2" : "test22", "2" : "test32", "2" : "test42", "2" : "test52", "2" : "test62", "2" : "test72", "2" : "test82", "2" : "test92", "3" : "test13", "3" : "test23", "3" : "test3", "3" : "test33", "3" : "test43", "3" : "test53", "3" : "test63", "3" : "test73", "3" : "test83", "3" : "test93", "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "6" : "test16", "6" : "test26", "6" : "test36", "6" : "test46", "6" : "test56", "6" : "test6", "6" : "test66", "6" : "test76", "6" : "test86", "6" : "test96", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } -(1 row) - --- both cte_1 and cte_2 are going to be inlined. --- later, cte_2 is recursively planned since it doesn't have --- GROUP BY but aggragate in a subquery. --- this is an important example of being able to recursively plan --- "some" of the CTEs -WITH cte_1 AS (SELECT value FROM test_table WHERE key > 1), - cte_2 AS (SELECT max(value) as value FROM test_table WHERE key > 3) -SELECT count(*) FROM cte_1 JOIN cte_2 USING (value); -DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: CTE cte_2 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT max(value) AS value FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.>) 3) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT test_table.value FROM cte_inline.test_table WHERE (test_table.key OPERATOR(pg_catalog.>) 1)) cte_1 JOIN (SELECT intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text)) cte_2 USING (value)) -DEBUG: Router planner cannot handle multi-shard select queries - count ---------------------------------------------------------------------- - 4 -(1 row) - --- prevent DROP CASCADE to give notices -SET client_min_messages TO ERROR; -DROP SCHEMA cte_inline CASCADE; diff --git a/src/test/regress/sql/cte_inline.sql b/src/test/regress/sql/cte_inline.sql index 057fc8d53..bf0d9885d 100644 --- a/src/test/regress/sql/cte_inline.sql +++ b/src/test/regress/sql/cte_inline.sql @@ -9,7 +9,7 @@ INSERT INTO test_table SELECT i % 10, 'test' || i, row_to_json(row(i, i*18, 'tes -- server version because CTE inlining might produce -- different debug messages in PG 11 vs PG 12 SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int; +SELECT substring(:'server_version', '\d+')::int >= 12; SET client_min_messages TO DEBUG; @@ -363,7 +363,8 @@ WITH cte_1 AS (SELECT * FROM test_table), cte_2 AS (SELECT * FROM test_table ORDER BY 1 DESC LIMIT 3) (SELECT *, (SELECT 1) FROM cte_1 EXCEPT SELECT *, 1 FROM test_table) UNION -(SELECT *, 1 FROM cte_2); +(SELECT *, 1 FROM cte_2) +ORDER BY 1,2; -- cte_1 is safe to inline, even if because after inlining From 87088d92bc6864dc61b2bea00fdf52d86620eaea Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Thu, 18 Jun 2020 15:49:45 +0300 Subject: [PATCH 35/52] Changelog: handle VACUUM PARALLEL option Postgres 13 added a new VACUUM option, PARALLEL. It is now supported in our code as well. Relevant changelog message on postgres: Allow VACUUM to process indexes in parallel (Masahiko Sawada, Amit Kapila) --- src/backend/distributed/commands/vacuum.c | 46 ++++++++++++++++++++++- src/test/regress/expected/pg13.out | 36 ++++++++++++++++++ src/test/regress/multi_schedule | 2 +- src/test/regress/sql/pg13.sql | 27 +++++++++++++ 4 files changed, 109 insertions(+), 2 deletions(-) create mode 100644 src/test/regress/expected/pg13.out create mode 100644 src/test/regress/sql/pg13.sql diff --git a/src/backend/distributed/commands/vacuum.c b/src/backend/distributed/commands/vacuum.c index db32e23fe..52b102395 100644 --- a/src/backend/distributed/commands/vacuum.c +++ b/src/backend/distributed/commands/vacuum.c @@ -29,7 +29,10 @@ #include "storage/lmgr.h" #include "utils/builtins.h" #include "utils/lsyscache.h" +#include "postmaster/bgworker_internals.h" + +#define VACUUM_PARALLEL_NOTSET -2 /* * Subset of VacuumParams we care about */ @@ -40,8 +43,11 @@ typedef struct CitusVacuumParams VacOptTernaryValue truncate; VacOptTernaryValue index_cleanup; #endif -} CitusVacuumParams; +#if PG_VERSION_NUM >= PG_VERSION_13 + int nworkers; +#endif +} CitusVacuumParams; /* Local functions forward declarations for processing distributed table commands */ static bool IsDistributedVacuumStmt(int vacuumOptions, List *vacuumRelationIdList); @@ -285,6 +291,9 @@ DeparseVacuumStmtPrefix(CitusVacuumParams vacuumParams) && vacuumParams.truncate == VACOPT_TERNARY_DEFAULT && vacuumParams.index_cleanup == VACOPT_TERNARY_DEFAULT #endif +#if PG_VERSION_NUM >= PG_VERSION_13 + && vacuumParams.nworkers == VACUUM_PARALLEL_NOTSET +#endif ) { return vacuumPrefix->data; @@ -341,6 +350,12 @@ DeparseVacuumStmtPrefix(CitusVacuumParams vacuumParams) } #endif +#if PG_VERSION_NUM >= PG_VERSION_13 + if (vacuumParams.nworkers != VACUUM_PARALLEL_NOTSET) { + appendStringInfo(vacuumPrefix, "PARALLEL %d,", vacuumParams.nworkers); + } +#endif + vacuumPrefix->data[vacuumPrefix->len - 1] = ')'; appendStringInfoChar(vacuumPrefix, ' '); @@ -421,6 +436,8 @@ ExtractVacuumTargetRels(VacuumStmt *vacuumStmt) /* * This is mostly ExecVacuum from Postgres's commands/vacuum.c + * Note that ExecVacuum does an actual vacuum as well and we don't want + * that to happen in the coordinator hence we copied the rest here. */ static CitusVacuumParams VacuumStmtParams(VacuumStmt *vacstmt) @@ -436,6 +453,9 @@ VacuumStmtParams(VacuumStmt *vacstmt) /* Set default value */ params.index_cleanup = VACOPT_TERNARY_DEFAULT; params.truncate = VACOPT_TERNARY_DEFAULT; + #if PG_VERSION_NUM >= PG_VERSION_13 + params.nworkers = VACUUM_PARALLEL_NOTSET; + #endif /* Parse options list */ DefElem *opt = NULL; @@ -484,6 +504,30 @@ VacuumStmtParams(VacuumStmt *vacstmt) params.truncate = defGetBoolean(opt) ? VACOPT_TERNARY_ENABLED : VACOPT_TERNARY_DISABLED; } + #if PG_VERSION_NUM >= PG_VERSION_13 + else if (strcmp(opt->defname, "parallel") == 0) { + + if (opt->arg == NULL) + { + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("parallel option requires a value between 0 and %d", + MAX_PARALLEL_WORKER_LIMIT))); + } + else + { + int nworkers; + nworkers = defGetInt32(opt); + if (nworkers < 0 || nworkers > MAX_PARALLEL_WORKER_LIMIT) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("parallel vacuum degree must be between 0 and %d", + MAX_PARALLEL_WORKER_LIMIT))); + + params.nworkers = nworkers; + } + } + #endif else { ereport(ERROR, diff --git a/src/test/regress/expected/pg13.out b/src/test/regress/expected/pg13.out new file mode 100644 index 000000000..860ed3a20 --- /dev/null +++ b/src/test/regress/expected/pg13.out @@ -0,0 +1,36 @@ +create schema test_pg13; +set search_path to test_pg13; +SET citus.shard_replication_factor to 1; +SET citus.shard_count to 2; +SET citus.next_shard_id TO 65000; +CREATE TABLE dist_table (name char, age int); +CREATE INDEX name_index on dist_table(name); +SELECT create_distributed_table('dist_table', 'name'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SET client_min_messages to DEBUG1; +SET citus.log_remote_commands to ON; +-- make sure vacuum parallel doesn't error out +VACUUM (PARALLEL 2) dist_table; +NOTICE: issuing VACUUM (PARALLEL 2) test_pg13.dist_table_65000 +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing VACUUM (PARALLEL 2) test_pg13.dist_table_65001 +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +VACUUM (PARALLEL 0) dist_table; +NOTICE: issuing VACUUM (PARALLEL 0) test_pg13.dist_table_65000 +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing VACUUM (PARALLEL 0) test_pg13.dist_table_65001 +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +-- This should error out since -5 is not valid. +VACUUM (PARALLEL -5) dist_table; +ERROR: parallel vacuum degree must be between 0 and 1024 +-- This should error out since no number is given +VACUUM (PARALLEL) dist_table; +ERROR: parallel option requires a value between 0 and 1024 +RESET client_min_messages; +RESET citus.log_remote_commands; +drop schema test_pg13 cascade; +NOTICE: drop cascades to table dist_table diff --git a/src/test/regress/multi_schedule b/src/test/regress/multi_schedule index dde15a85d..e79c34592 100644 --- a/src/test/regress/multi_schedule +++ b/src/test/regress/multi_schedule @@ -80,7 +80,7 @@ test: set_operation_and_local_tables test: subqueries_deep subquery_view subquery_partitioning subquery_complex_target_list subqueries_not_supported subquery_in_where test: non_colocated_leaf_subquery_joins non_colocated_subquery_joins non_colocated_join_order -test: subquery_prepared_statements pg12 cte_inline +test: subquery_prepared_statements pg12 cte_inline pg13 # ---------- # Miscellaneous tests to check our query planning behavior diff --git a/src/test/regress/sql/pg13.sql b/src/test/regress/sql/pg13.sql new file mode 100644 index 000000000..2b75b8b4e --- /dev/null +++ b/src/test/regress/sql/pg13.sql @@ -0,0 +1,27 @@ + +create schema test_pg13; +set search_path to test_pg13; + +SET citus.shard_replication_factor to 1; +SET citus.shard_count to 2; +SET citus.next_shard_id TO 65000; + +CREATE TABLE dist_table (name char, age int); +CREATE INDEX name_index on dist_table(name); + +SELECT create_distributed_table('dist_table', 'name'); + +SET client_min_messages to DEBUG1; +SET citus.log_remote_commands to ON; +-- make sure vacuum parallel doesn't error out +VACUUM (PARALLEL 2) dist_table; +VACUUM (PARALLEL 0) dist_table; +-- This should error out since -5 is not valid. +VACUUM (PARALLEL -5) dist_table; +-- This should error out since no number is given +VACUUM (PARALLEL) dist_table; + +RESET client_min_messages; +RESET citus.log_remote_commands; + +drop schema test_pg13 cascade; \ No newline at end of file From 920d7211e4c847866c63e8e6c247bbb31753197e Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Fri, 19 Jun 2020 14:03:45 +0300 Subject: [PATCH 36/52] Changelog: Test that we error out for DROP EXPRESSION PG13 now supports dropping expression from a column such as generated columns. We error out with this currently. Changelog entry in postgres: Add ALTER TABLE clause DROP EXPRESSION to remove generated properties from columns (Peter Eisentraut) --- src/test/regress/expected/pg13.out | 35 +++++++++++++++++++++++++++++- src/test/regress/sql/pg13.sql | 8 +++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/test/regress/expected/pg13.out b/src/test/regress/expected/pg13.out index 860ed3a20..3b72db614 100644 --- a/src/test/regress/expected/pg13.out +++ b/src/test/regress/expected/pg13.out @@ -31,6 +31,39 @@ ERROR: parallel vacuum degree must be between 0 and 1024 VACUUM (PARALLEL) dist_table; ERROR: parallel option requires a value between 0 and 1024 RESET client_min_messages; +-- test alter table alter column drop expression +CREATE TABLE generated_col_table(a int, b int GENERATED ALWAYS AS (a * 10) STORED); +SELECT create_distributed_table('generated_col_table', 'a'); +NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx'); +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx'); +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SELECT worker_apply_shard_ddl_command (65002, 'test_pg13', 'CREATE TABLE test_pg13.generated_col_table (a integer, b integer GENERATED ALWAYS AS ((a * 10)) STORED)');SELECT worker_apply_shard_ddl_command (65002, 'test_pg13', 'ALTER TABLE test_pg13.generated_col_table OWNER TO postgres') +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SELECT worker_apply_shard_ddl_command (65003, 'test_pg13', 'CREATE TABLE test_pg13.generated_col_table (a integer, b integer GENERATED ALWAYS AS ((a * 10)) STORED)');SELECT worker_apply_shard_ddl_command (65003, 'test_pg13', 'ALTER TABLE test_pg13.generated_col_table OWNER TO postgres') +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing PREPARE TRANSACTION 'citus_0_13977_182_2' +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing PREPARE TRANSACTION 'citus_0_13977_182_3' +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing COMMIT PREPARED 'citus_0_13977_182_2' +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing COMMIT PREPARED 'citus_0_13977_182_3' +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO generated_col_table VALUES (1); +NOTICE: issuing INSERT INTO test_pg13.generated_col_table_65002 (a) VALUES (1) +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +-- Make sure that we currently error out +ALTER TABLE generated_col_table ALTER COLUMN b DROP EXPRESSION; +ERROR: alter table command is currently unsupported +DETAIL: Only ADD|DROP COLUMN, SET|DROP NOT NULL, SET|DROP DEFAULT, ADD|DROP|VALIDATE CONSTRAINT, SET (), RESET (), ATTACH|DETACH PARTITION and TYPE subcommands are supported. RESET citus.log_remote_commands; drop schema test_pg13 cascade; -NOTICE: drop cascades to table dist_table +NOTICE: drop cascades to 2 other objects +DETAIL: drop cascades to table dist_table +drop cascades to table generated_col_table diff --git a/src/test/regress/sql/pg13.sql b/src/test/regress/sql/pg13.sql index 2b75b8b4e..e34490926 100644 --- a/src/test/regress/sql/pg13.sql +++ b/src/test/regress/sql/pg13.sql @@ -22,6 +22,14 @@ VACUUM (PARALLEL -5) dist_table; VACUUM (PARALLEL) dist_table; RESET client_min_messages; + +-- test alter table alter column drop expression +CREATE TABLE generated_col_table(a int, b int GENERATED ALWAYS AS (a * 10) STORED); +SELECT create_distributed_table('generated_col_table', 'a'); +INSERT INTO generated_col_table VALUES (1); +-- Make sure that we currently error out +ALTER TABLE generated_col_table ALTER COLUMN b DROP EXPRESSION; + RESET citus.log_remote_commands; drop schema test_pg13 cascade; \ No newline at end of file From 275ccd0400d0caba69e85fe0ff619861f5262e48 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Fri, 19 Jun 2020 14:37:45 +0300 Subject: [PATCH 37/52] Changelog: Test that alter view rename column works Changelog entry in PG13: Add ALTER VIEW syntax to rename view columns (Fujii Masao) --- src/test/regress/expected/pg13.out | 23 ++++++++++++++++++----- src/test/regress/sql/pg13.sql | 5 +++++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/src/test/regress/expected/pg13.out b/src/test/regress/expected/pg13.out index 3b72db614..a8b3bcd85 100644 --- a/src/test/regress/expected/pg13.out +++ b/src/test/regress/expected/pg13.out @@ -42,13 +42,13 @@ NOTICE: issuing SELECT worker_apply_shard_ddl_command (65002, 'test_pg13', 'CRE DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx NOTICE: issuing SELECT worker_apply_shard_ddl_command (65003, 'test_pg13', 'CREATE TABLE test_pg13.generated_col_table (a integer, b integer GENERATED ALWAYS AS ((a * 10)) STORED)');SELECT worker_apply_shard_ddl_command (65003, 'test_pg13', 'ALTER TABLE test_pg13.generated_col_table OWNER TO postgres') DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing PREPARE TRANSACTION 'citus_0_13977_182_2' +NOTICE: issuing PREPARE TRANSACTION 'citus_0_18072_182_2' DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing PREPARE TRANSACTION 'citus_0_13977_182_3' +NOTICE: issuing PREPARE TRANSACTION 'citus_0_18072_182_3' DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing COMMIT PREPARED 'citus_0_13977_182_2' +NOTICE: issuing COMMIT PREPARED 'citus_0_18072_182_2' DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing COMMIT PREPARED 'citus_0_13977_182_3' +NOTICE: issuing COMMIT PREPARED 'citus_0_18072_182_3' DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx create_distributed_table --------------------------------------------------------------------- @@ -62,8 +62,21 @@ DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx ALTER TABLE generated_col_table ALTER COLUMN b DROP EXPRESSION; ERROR: alter table command is currently unsupported DETAIL: Only ADD|DROP COLUMN, SET|DROP NOT NULL, SET|DROP DEFAULT, ADD|DROP|VALIDATE CONSTRAINT, SET (), RESET (), ATTACH|DETACH PARTITION and TYPE subcommands are supported. +-- alter view rename column works fine +CREATE VIEW v AS SELECT * FROM dist_table; +ALTER VIEW v RENAME age to new_age; +SELECT * FROM v; +NOTICE: issuing SELECT name, age AS new_age FROM test_pg13.dist_table_65000 dist_table WHERE true +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing SELECT name, age AS new_age FROM test_pg13.dist_table_65001 dist_table WHERE true +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx + name | new_age +--------------------------------------------------------------------- +(0 rows) + RESET citus.log_remote_commands; drop schema test_pg13 cascade; -NOTICE: drop cascades to 2 other objects +NOTICE: drop cascades to 3 other objects DETAIL: drop cascades to table dist_table drop cascades to table generated_col_table +drop cascades to view v diff --git a/src/test/regress/sql/pg13.sql b/src/test/regress/sql/pg13.sql index e34490926..bf9d6b1e6 100644 --- a/src/test/regress/sql/pg13.sql +++ b/src/test/regress/sql/pg13.sql @@ -30,6 +30,11 @@ INSERT INTO generated_col_table VALUES (1); -- Make sure that we currently error out ALTER TABLE generated_col_table ALTER COLUMN b DROP EXPRESSION; +-- alter view rename column works fine +CREATE VIEW v AS SELECT * FROM dist_table; +ALTER VIEW v RENAME age to new_age; +SELECT * FROM v; + RESET citus.log_remote_commands; drop schema test_pg13 cascade; \ No newline at end of file From ebabca16b77c97ff9f4e7e95dfe136de7b36b550 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Sat, 20 Jun 2020 18:39:11 +0300 Subject: [PATCH 38/52] Changelog: Test row suffix notation It seems that row suffix notation is working fine with our code, a test is added. Changelog entry in PG13: Allow ROW values values to have their members extracted with suffix notation (Tom Lane) --- src/test/regress/expected/pg13.out | 57 +++++++++++++++++------------- src/test/regress/sql/pg13.sql | 8 ++++- 2 files changed, 40 insertions(+), 25 deletions(-) diff --git a/src/test/regress/expected/pg13.out b/src/test/regress/expected/pg13.out index a8b3bcd85..8b3a4b0dd 100644 --- a/src/test/regress/expected/pg13.out +++ b/src/test/regress/expected/pg13.out @@ -31,33 +31,16 @@ ERROR: parallel vacuum degree must be between 0 and 1024 VACUUM (PARALLEL) dist_table; ERROR: parallel option requires a value between 0 and 1024 RESET client_min_messages; +RESET citus.log_remote_commands; -- test alter table alter column drop expression CREATE TABLE generated_col_table(a int, b int GENERATED ALWAYS AS (a * 10) STORED); SELECT create_distributed_table('generated_col_table', 'a'); -NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx'); -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx'); -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SELECT worker_apply_shard_ddl_command (65002, 'test_pg13', 'CREATE TABLE test_pg13.generated_col_table (a integer, b integer GENERATED ALWAYS AS ((a * 10)) STORED)');SELECT worker_apply_shard_ddl_command (65002, 'test_pg13', 'ALTER TABLE test_pg13.generated_col_table OWNER TO postgres') -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SELECT worker_apply_shard_ddl_command (65003, 'test_pg13', 'CREATE TABLE test_pg13.generated_col_table (a integer, b integer GENERATED ALWAYS AS ((a * 10)) STORED)');SELECT worker_apply_shard_ddl_command (65003, 'test_pg13', 'ALTER TABLE test_pg13.generated_col_table OWNER TO postgres') -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing PREPARE TRANSACTION 'citus_0_18072_182_2' -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing PREPARE TRANSACTION 'citus_0_18072_182_3' -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing COMMIT PREPARED 'citus_0_18072_182_2' -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing COMMIT PREPARED 'citus_0_18072_182_3' -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx create_distributed_table --------------------------------------------------------------------- (1 row) INSERT INTO generated_col_table VALUES (1); -NOTICE: issuing INSERT INTO test_pg13.generated_col_table_65002 (a) VALUES (1) -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -- Make sure that we currently error out ALTER TABLE generated_col_table ALTER COLUMN b DROP EXPRESSION; ERROR: alter table command is currently unsupported @@ -66,17 +49,43 @@ DETAIL: Only ADD|DROP COLUMN, SET|DROP NOT NULL, SET|DROP DEFAULT, ADD|DROP|VAL CREATE VIEW v AS SELECT * FROM dist_table; ALTER VIEW v RENAME age to new_age; SELECT * FROM v; -NOTICE: issuing SELECT name, age AS new_age FROM test_pg13.dist_table_65000 dist_table WHERE true -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing SELECT name, age AS new_age FROM test_pg13.dist_table_65001 dist_table WHERE true -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx name | new_age --------------------------------------------------------------------- (0 rows) -RESET citus.log_remote_commands; +-- row suffix notation works fine +CREATE TABLE ab (a int, b int); +SELECT create_distributed_table('ab','a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO ab SELECT i, 2 * i FROM generate_series(1,20)i; +SELECT * FROM ab WHERE (ROW(a,b)).f1 > (ROW(10,30)).f1 ORDER BY 1,2; + a | b +--------------------------------------------------------------------- + 11 | 22 + 12 | 24 + 13 | 26 + 14 | 28 + 15 | 30 + 16 | 32 + 17 | 34 + 18 | 36 + 19 | 38 + 20 | 40 +(10 rows) + +SELECT * FROM ab WHERE (ROW(a,b)).f2 > (ROW(0,38)).f2 ORDER BY 1,2; + a | b +--------------------------------------------------------------------- + 20 | 40 +(1 row) + drop schema test_pg13 cascade; -NOTICE: drop cascades to 3 other objects +NOTICE: drop cascades to 4 other objects DETAIL: drop cascades to table dist_table drop cascades to table generated_col_table drop cascades to view v +drop cascades to table ab diff --git a/src/test/regress/sql/pg13.sql b/src/test/regress/sql/pg13.sql index bf9d6b1e6..00a229780 100644 --- a/src/test/regress/sql/pg13.sql +++ b/src/test/regress/sql/pg13.sql @@ -22,6 +22,7 @@ VACUUM (PARALLEL -5) dist_table; VACUUM (PARALLEL) dist_table; RESET client_min_messages; +RESET citus.log_remote_commands; -- test alter table alter column drop expression CREATE TABLE generated_col_table(a int, b int GENERATED ALWAYS AS (a * 10) STORED); @@ -35,6 +36,11 @@ CREATE VIEW v AS SELECT * FROM dist_table; ALTER VIEW v RENAME age to new_age; SELECT * FROM v; -RESET citus.log_remote_commands; +-- row suffix notation works fine +CREATE TABLE ab (a int, b int); +SELECT create_distributed_table('ab','a'); +INSERT INTO ab SELECT i, 2 * i FROM generate_series(1,20)i; +SELECT * FROM ab WHERE (ROW(a,b)).f1 > (ROW(10,30)).f1 ORDER BY 1,2; +SELECT * FROM ab WHERE (ROW(a,b)).f2 > (ROW(0,38)).f2 ORDER BY 1,2; drop schema test_pg13 cascade; \ No newline at end of file From 79dcb8014075f6a2e6867e0f8d39bb390e0ebc92 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Sat, 20 Jun 2020 20:55:59 +0300 Subject: [PATCH 39/52] Changelog: Test IS NORMALIZED for pg13 Tests for is_normalized and normalized ar eadded. One thing that seems to be because of existent bug is that when we don't give the second argument to normalize or is_normalized, which is optional, it crashes. Because in the executor part, in the expression we don't have the default argument. Changelog entry in PG-13: Add SQL functions NORMALIZE() to normalize Unicode strings, and IS NORMALIZED to check for normalization (Peter Eisentraut) Commit on Postgres: 2991ac5fc9b3904ca4582be6d323497d7c3d17c9 --- src/test/regress/expected/pg13.out | 82 +++++++++++++++++++++++++++++- src/test/regress/sql/pg13.sql | 10 ++++ 2 files changed, 91 insertions(+), 1 deletion(-) diff --git a/src/test/regress/expected/pg13.out b/src/test/regress/expected/pg13.out index 8b3a4b0dd..77250a647 100644 --- a/src/test/regress/expected/pg13.out +++ b/src/test/regress/expected/pg13.out @@ -83,9 +83,89 @@ SELECT * FROM ab WHERE (ROW(a,b)).f2 > (ROW(0,38)).f2 ORDER BY 1,2; 20 | 40 (1 row) +CREATE TABLE text_table (name text); +SELECT create_distributed_table('text_table', 'name'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO text_table VALUES ('abc'); +-- not normalized +INSERT INTO text_table VALUES (U&'\0061\0308bc'); +SELECT name IS NORMALIZED FROM text_table ORDER BY 1; + is_normalized +--------------------------------------------------------------------- + f + t +(2 rows) + +SELECT is_normalized(name) FROM text_table ORDER BY 1; + is_normalized +--------------------------------------------------------------------- + f + t +(2 rows) + +SELECT normalize(name) FROM text_table ORDER BY 1; + normalize +--------------------------------------------------------------------- + abc + äbc +(2 rows) + +INSERT INTO text_table VALUES (normalize(U&'\0061\0308bc', NFC)); +-- test unicode escape +-- insert the word 'data' with unicode escapes +INSERT INTO text_table VALUES(U&'d\0061t\+000061'); +-- insert the word слон +INSERT INTO text_table VALUES(U&'\0441\043B\043E\043D'); +SELECT * FROM text_table ORDER BY 1; + name +--------------------------------------------------------------------- + abc + äbc + data + äbc + слон +(5 rows) + +-- Test that we don't propagate base types +CREATE TYPE myvarchar; +CREATE FUNCTION myvarcharin(cstring, oid, integer) RETURNS myvarchar +LANGUAGE internal IMMUTABLE PARALLEL SAFE STRICT AS 'varcharin'; +NOTICE: return type myvarchar is only a shell +CREATE FUNCTION myvarcharout(myvarchar) RETURNS cstring +LANGUAGE internal IMMUTABLE PARALLEL SAFE STRICT AS 'varcharout'; +NOTICE: argument type myvarchar is only a shell +CREATE TYPE myvarchar ( + input = myvarcharin, + output = myvarcharout, + alignment = integer, + storage = main +); +CREATE TABLE my_table (a int, b myvarchar); +-- this will error because it seems that we don't propagate the "BASE TYPES" +-- Alter table also errors out so this doesn't seem to apply to use: +-- """Add ALTER TYPE options useful for extensions, +-- like TOAST and I/O functions control (Tomas Vondra, Tom Lane)""" +SELECT create_distributed_table('my_table', 'a'); +ERROR: type "test_pg13.myvarchar" does not exist +CONTEXT: while executing command on localhost:xxxxx +CREATE TABLE test_table(a int, b tsvector); +SELECT create_distributed_table('test_table', 'a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- we currently don't support this +CREATE INDEX test_table_index ON test_table USING gist (b tsvector_ops(siglen = 100)); +ERROR: citus currently doesn't support this index arguments drop schema test_pg13 cascade; -NOTICE: drop cascades to 4 other objects +NOTICE: drop cascades to 5 other objects DETAIL: drop cascades to table dist_table drop cascades to table generated_col_table drop cascades to view v drop cascades to table ab +drop cascades to table text_table diff --git a/src/test/regress/sql/pg13.sql b/src/test/regress/sql/pg13.sql index 00a229780..73eae1d4b 100644 --- a/src/test/regress/sql/pg13.sql +++ b/src/test/regress/sql/pg13.sql @@ -43,4 +43,14 @@ INSERT INTO ab SELECT i, 2 * i FROM generate_series(1,20)i; SELECT * FROM ab WHERE (ROW(a,b)).f1 > (ROW(10,30)).f1 ORDER BY 1,2; SELECT * FROM ab WHERE (ROW(a,b)).f2 > (ROW(0,38)).f2 ORDER BY 1,2; +CREATE TABLE text_table (name text); +SELECT create_distributed_table('text_table', 'name'); +INSERT INTO text_table VALUES ('abc'); +-- not normalized +INSERT INTO text_table VALUES (U&'\0061\0308bc'); +SELECT name IS NORMALIZED FROM text_table ORDER BY 1; +SELECT is_normalized(name) FROM text_table ORDER BY 1; +SELECT normalize(name) FROM text_table ORDER BY 1; +INSERT INTO text_table VALUES (normalize(U&'\0061\0308bc', NFC)); + drop schema test_pg13 cascade; \ No newline at end of file From 00633165fcf2178783cac3bee5a59b4e32715651 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Sat, 20 Jun 2020 21:06:16 +0300 Subject: [PATCH 40/52] Changelog: Test unicode escapes Unicode escapes work as expected, related tests are added. Changelog entry on PG13: Allow Unicode escapes, e.g., E'\u####', U&'\####', to specify any character available in the database encoding, even when the database encoding is not UTF-8 (Tom Lane) --- src/test/regress/expected/pg13.out | 33 +----------------------------- src/test/regress/sql/pg13.sql | 8 ++++++++ 2 files changed, 9 insertions(+), 32 deletions(-) diff --git a/src/test/regress/expected/pg13.out b/src/test/regress/expected/pg13.out index 77250a647..716062a54 100644 --- a/src/test/regress/expected/pg13.out +++ b/src/test/regress/expected/pg13.out @@ -83,6 +83,7 @@ SELECT * FROM ab WHERE (ROW(a,b)).f2 > (ROW(0,38)).f2 ORDER BY 1,2; 20 | 40 (1 row) +-- test normalized CREATE TABLE text_table (name text); SELECT create_distributed_table('text_table', 'name'); create_distributed_table @@ -130,38 +131,6 @@ SELECT * FROM text_table ORDER BY 1; слон (5 rows) --- Test that we don't propagate base types -CREATE TYPE myvarchar; -CREATE FUNCTION myvarcharin(cstring, oid, integer) RETURNS myvarchar -LANGUAGE internal IMMUTABLE PARALLEL SAFE STRICT AS 'varcharin'; -NOTICE: return type myvarchar is only a shell -CREATE FUNCTION myvarcharout(myvarchar) RETURNS cstring -LANGUAGE internal IMMUTABLE PARALLEL SAFE STRICT AS 'varcharout'; -NOTICE: argument type myvarchar is only a shell -CREATE TYPE myvarchar ( - input = myvarcharin, - output = myvarcharout, - alignment = integer, - storage = main -); -CREATE TABLE my_table (a int, b myvarchar); --- this will error because it seems that we don't propagate the "BASE TYPES" --- Alter table also errors out so this doesn't seem to apply to use: --- """Add ALTER TYPE options useful for extensions, --- like TOAST and I/O functions control (Tomas Vondra, Tom Lane)""" -SELECT create_distributed_table('my_table', 'a'); -ERROR: type "test_pg13.myvarchar" does not exist -CONTEXT: while executing command on localhost:xxxxx -CREATE TABLE test_table(a int, b tsvector); -SELECT create_distributed_table('test_table', 'a'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - --- we currently don't support this -CREATE INDEX test_table_index ON test_table USING gist (b tsvector_ops(siglen = 100)); -ERROR: citus currently doesn't support this index arguments drop schema test_pg13 cascade; NOTICE: drop cascades to 5 other objects DETAIL: drop cascades to table dist_table diff --git a/src/test/regress/sql/pg13.sql b/src/test/regress/sql/pg13.sql index 73eae1d4b..387f6c894 100644 --- a/src/test/regress/sql/pg13.sql +++ b/src/test/regress/sql/pg13.sql @@ -43,6 +43,7 @@ INSERT INTO ab SELECT i, 2 * i FROM generate_series(1,20)i; SELECT * FROM ab WHERE (ROW(a,b)).f1 > (ROW(10,30)).f1 ORDER BY 1,2; SELECT * FROM ab WHERE (ROW(a,b)).f2 > (ROW(0,38)).f2 ORDER BY 1,2; +-- test normalized CREATE TABLE text_table (name text); SELECT create_distributed_table('text_table', 'name'); INSERT INTO text_table VALUES ('abc'); @@ -53,4 +54,11 @@ SELECT is_normalized(name) FROM text_table ORDER BY 1; SELECT normalize(name) FROM text_table ORDER BY 1; INSERT INTO text_table VALUES (normalize(U&'\0061\0308bc', NFC)); +-- test unicode escape +-- insert the word 'data' with unicode escapes +INSERT INTO text_table VALUES(U&'d\0061t\+000061'); +-- insert the word слон +INSERT INTO text_table VALUES(U&'\0441\043B\043E\043D'); +SELECT * FROM text_table ORDER BY 1; + drop schema test_pg13 cascade; \ No newline at end of file From f7a19713614490376ecad77a9f0bae318a68b08e Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Sat, 20 Jun 2020 21:53:28 +0300 Subject: [PATCH 41/52] Changelog: Alter type options It seems that we don't support propagating commands related to base types. Therefore Alter TYPE options doesn't seem to apply to us. I have added a test to verify that we don't propagate them. Changelog entry on pg13: Add ALTER TYPE options useful for extensions, like TOAST and I/O functions control (Tomas Vondra, Tom Lane) --- src/test/regress/expected/pg13.out | 28 +++++++++++++++++++++++++++- src/test/regress/sql/pg13.sql | 20 ++++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/src/test/regress/expected/pg13.out b/src/test/regress/expected/pg13.out index 716062a54..26bc678fd 100644 --- a/src/test/regress/expected/pg13.out +++ b/src/test/regress/expected/pg13.out @@ -131,10 +131,36 @@ SELECT * FROM text_table ORDER BY 1; слон (5 rows) +-- Test that we don't propagate base types +CREATE TYPE myvarchar; +CREATE FUNCTION myvarcharin(cstring, oid, integer) RETURNS myvarchar +LANGUAGE internal IMMUTABLE PARALLEL SAFE STRICT AS 'varcharin'; +NOTICE: return type myvarchar is only a shell +CREATE FUNCTION myvarcharout(myvarchar) RETURNS cstring +LANGUAGE internal IMMUTABLE PARALLEL SAFE STRICT AS 'varcharout'; +NOTICE: argument type myvarchar is only a shell +CREATE TYPE myvarchar ( + input = myvarcharin, + output = myvarcharout, + alignment = integer, + storage = main +); +CREATE TABLE my_table (a int, b myvarchar); +-- this will error because it seems that we don't propagate the "BASE TYPES" +-- Alter table also errors out so this doesn't seem to apply to use: +-- """Add ALTER TYPE options useful for extensions, +-- like TOAST and I/O functions control (Tomas Vondra, Tom Lane)""" +SELECT create_distributed_table('my_table', 'a'); +ERROR: type "test_pg13.myvarchar" does not exist +CONTEXT: while executing command on localhost:xxxxx drop schema test_pg13 cascade; -NOTICE: drop cascades to 5 other objects +NOTICE: drop cascades to 9 other objects DETAIL: drop cascades to table dist_table drop cascades to table generated_col_table drop cascades to view v drop cascades to table ab drop cascades to table text_table +drop cascades to function myvarcharout(myvarchar) +drop cascades to type myvarchar +drop cascades to function myvarcharin(cstring,oid,integer) +drop cascades to table my_table diff --git a/src/test/regress/sql/pg13.sql b/src/test/regress/sql/pg13.sql index 387f6c894..8509654d2 100644 --- a/src/test/regress/sql/pg13.sql +++ b/src/test/regress/sql/pg13.sql @@ -61,4 +61,24 @@ INSERT INTO text_table VALUES(U&'d\0061t\+000061'); INSERT INTO text_table VALUES(U&'\0441\043B\043E\043D'); SELECT * FROM text_table ORDER BY 1; +-- Test that we don't propagate base types +CREATE TYPE myvarchar; +CREATE FUNCTION myvarcharin(cstring, oid, integer) RETURNS myvarchar +LANGUAGE internal IMMUTABLE PARALLEL SAFE STRICT AS 'varcharin'; +CREATE FUNCTION myvarcharout(myvarchar) RETURNS cstring +LANGUAGE internal IMMUTABLE PARALLEL SAFE STRICT AS 'varcharout'; +CREATE TYPE myvarchar ( + input = myvarcharin, + output = myvarcharout, + alignment = integer, + storage = main +); + +CREATE TABLE my_table (a int, b myvarchar); +-- this will error because it seems that we don't propagate the "BASE TYPES" +-- Alter table also errors out so this doesn't seem to apply to use: +-- """Add ALTER TYPE options useful for extensions, +-- like TOAST and I/O functions control (Tomas Vondra, Tom Lane)""" +SELECT create_distributed_table('my_table', 'a'); + drop schema test_pg13 cascade; \ No newline at end of file From d0b0c8892099cc93c066cd110bcff749d7105867 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Sat, 20 Jun 2020 23:53:47 +0300 Subject: [PATCH 42/52] Changelog: error out if index has opclassopts Error out if index has opclassopts. Changelog entry on PG13: Allow CREATE INDEX to specify the GiST signature length and maximum number of integer ranges (Nikita Glukhov) --- src/backend/distributed/deparser/citus_ruleutils.c | 6 ++++++ src/test/regress/expected/pg13.out | 13 ++++++++++++- src/test/regress/sql/pg13.sql | 5 +++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/backend/distributed/deparser/citus_ruleutils.c b/src/backend/distributed/deparser/citus_ruleutils.c index 9f5b0f905..9203d9364 100644 --- a/src/backend/distributed/deparser/citus_ruleutils.c +++ b/src/backend/distributed/deparser/citus_ruleutils.c @@ -37,6 +37,7 @@ #include "commands/defrem.h" #include "commands/extension.h" #include "distributed/citus_ruleutils.h" +#include "distributed/listutils.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/relay_utility.h" #include "distributed/metadata_utility.h" @@ -813,6 +814,11 @@ deparse_index_columns(StringInfo buffer, List *indexParameterList, List *deparse appendStringInfo(buffer, "%s ", NameListToQuotedString(indexElement->opclass)); } +#if PG_VERSION_NUM >= PG_VERSION_13 + if (indexElement->opclassopts != NIL) { + ereport(ERROR, errmsg("citus currently doesn't support this index arguments")); + } +#endif if (indexElement->ordering != SORTBY_DEFAULT) { diff --git a/src/test/regress/expected/pg13.out b/src/test/regress/expected/pg13.out index 26bc678fd..8bc788ae1 100644 --- a/src/test/regress/expected/pg13.out +++ b/src/test/regress/expected/pg13.out @@ -153,8 +153,18 @@ CREATE TABLE my_table (a int, b myvarchar); SELECT create_distributed_table('my_table', 'a'); ERROR: type "test_pg13.myvarchar" does not exist CONTEXT: while executing command on localhost:xxxxx +CREATE TABLE test_table(a int, b tsvector); +SELECT create_distributed_table('test_table', 'a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- we currently don't support this +CREATE INDEX test_table_index ON test_table USING gist (b tsvector_ops(siglen = 100)); +ERROR: citus currently doesn't support this index arguments drop schema test_pg13 cascade; -NOTICE: drop cascades to 9 other objects +NOTICE: drop cascades to 10 other objects DETAIL: drop cascades to table dist_table drop cascades to table generated_col_table drop cascades to view v @@ -164,3 +174,4 @@ drop cascades to function myvarcharout(myvarchar) drop cascades to type myvarchar drop cascades to function myvarcharin(cstring,oid,integer) drop cascades to table my_table +drop cascades to table test_table diff --git a/src/test/regress/sql/pg13.sql b/src/test/regress/sql/pg13.sql index 8509654d2..40d6d08e5 100644 --- a/src/test/regress/sql/pg13.sql +++ b/src/test/regress/sql/pg13.sql @@ -81,4 +81,9 @@ CREATE TABLE my_table (a int, b myvarchar); -- like TOAST and I/O functions control (Tomas Vondra, Tom Lane)""" SELECT create_distributed_table('my_table', 'a'); +CREATE TABLE test_table(a int, b tsvector); +SELECT create_distributed_table('test_table', 'a'); +-- we currently don't support this +CREATE INDEX test_table_index ON test_table USING gist (b tsvector_ops(siglen = 100)); + drop schema test_pg13 cascade; \ No newline at end of file From 288aa586035e52700ce92b1fd0012866b1d4e6d8 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Sun, 21 Jun 2020 00:03:52 +0300 Subject: [PATCH 43/52] add alternative out for pg13 test --- src/test/regress/expected/pg13.out | 7 +++++++ src/test/regress/expected/pg13_0.out | 6 ++++++ src/test/regress/sql/pg13.sql | 8 ++++++++ 3 files changed, 21 insertions(+) create mode 100644 src/test/regress/expected/pg13_0.out diff --git a/src/test/regress/expected/pg13.out b/src/test/regress/expected/pg13.out index 8bc788ae1..db8e29f00 100644 --- a/src/test/regress/expected/pg13.out +++ b/src/test/regress/expected/pg13.out @@ -1,3 +1,10 @@ +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int > 12 AS server_version_above_twelve +\gset +\if :server_version_above_twelve +\else +\q +\endif create schema test_pg13; set search_path to test_pg13; SET citus.shard_replication_factor to 1; diff --git a/src/test/regress/expected/pg13_0.out b/src/test/regress/expected/pg13_0.out new file mode 100644 index 000000000..e25fbb82d --- /dev/null +++ b/src/test/regress/expected/pg13_0.out @@ -0,0 +1,6 @@ +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int > 12 AS server_version_above_twelve +\gset +\if :server_version_above_twelve +\else +\q diff --git a/src/test/regress/sql/pg13.sql b/src/test/regress/sql/pg13.sql index 40d6d08e5..7877eb9e1 100644 --- a/src/test/regress/sql/pg13.sql +++ b/src/test/regress/sql/pg13.sql @@ -1,3 +1,11 @@ +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int > 12 AS server_version_above_twelve +\gset +\if :server_version_above_twelve +\else +\q +\endif + create schema test_pg13; set search_path to test_pg13; From d68bfc5687987cc52e3473875984ff4dcb04b89b Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Sun, 21 Jun 2020 00:05:39 +0300 Subject: [PATCH 44/52] Improve error for index operator class parameters The error message when index has opclassopts is improved and the commit from postgres side is also included for future reference. Also some minor style related changes are applied. --- src/backend/distributed/commands/vacuum.c | 17 ++++++---- .../distributed/deparser/citus_ruleutils.c | 10 ++++-- .../distributed/executor/local_executor.c | 4 +-- .../planner/combine_query_planner.c | 2 +- .../distributed/planner/distributed_planner.c | 3 +- .../planner/insert_select_planner.c | 2 +- .../distributed/planner/local_plan_cache.c | 2 +- .../distributed/planner/multi_explain.c | 7 ++-- .../planner/multi_physical_planner.c | 11 +++---- .../distributed/planner/recursive_planning.c | 2 +- .../test/distributed_intermediate_results.c | 6 ++-- src/include/distributed/version_compat.h | 24 +++++++------- .../regress/expected/multi_partitioning_1.out | 4 +-- src/test/regress/expected/pg13.out | 2 +- .../multi_complex_count_distinct.source | 33 ------------------- src/test/regress/sql/pg13.sql | 2 +- 16 files changed, 50 insertions(+), 81 deletions(-) diff --git a/src/backend/distributed/commands/vacuum.c b/src/backend/distributed/commands/vacuum.c index 52b102395..706fea00c 100644 --- a/src/backend/distributed/commands/vacuum.c +++ b/src/backend/distributed/commands/vacuum.c @@ -33,6 +33,7 @@ #define VACUUM_PARALLEL_NOTSET -2 + /* * Subset of VacuumParams we care about */ @@ -293,7 +294,7 @@ DeparseVacuumStmtPrefix(CitusVacuumParams vacuumParams) #endif #if PG_VERSION_NUM >= PG_VERSION_13 && vacuumParams.nworkers == VACUUM_PARALLEL_NOTSET -#endif +#endif ) { return vacuumPrefix->data; @@ -351,7 +352,8 @@ DeparseVacuumStmtPrefix(CitusVacuumParams vacuumParams) #endif #if PG_VERSION_NUM >= PG_VERSION_13 - if (vacuumParams.nworkers != VACUUM_PARALLEL_NOTSET) { + if (vacuumParams.nworkers != VACUUM_PARALLEL_NOTSET) + { appendStringInfo(vacuumPrefix, "PARALLEL %d,", vacuumParams.nworkers); } #endif @@ -454,7 +456,7 @@ VacuumStmtParams(VacuumStmt *vacstmt) params.index_cleanup = VACOPT_TERNARY_DEFAULT; params.truncate = VACOPT_TERNARY_DEFAULT; #if PG_VERSION_NUM >= PG_VERSION_13 - params.nworkers = VACUUM_PARALLEL_NOTSET; + params.nworkers = VACUUM_PARALLEL_NOTSET; #endif /* Parse options list */ @@ -505,8 +507,8 @@ VacuumStmtParams(VacuumStmt *vacstmt) VACOPT_TERNARY_DISABLED; } #if PG_VERSION_NUM >= PG_VERSION_13 - else if (strcmp(opt->defname, "parallel") == 0) { - + else if (strcmp(opt->defname, "parallel") == 0) + { if (opt->arg == NULL) { ereport(ERROR, @@ -516,13 +518,14 @@ VacuumStmtParams(VacuumStmt *vacstmt) } else { - int nworkers; - nworkers = defGetInt32(opt); + int nworkers = defGetInt32(opt); if (nworkers < 0 || nworkers > MAX_PARALLEL_WORKER_LIMIT) + { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("parallel vacuum degree must be between 0 and %d", MAX_PARALLEL_WORKER_LIMIT))); + } params.nworkers = nworkers; } diff --git a/src/backend/distributed/deparser/citus_ruleutils.c b/src/backend/distributed/deparser/citus_ruleutils.c index 9203d9364..2dd90a722 100644 --- a/src/backend/distributed/deparser/citus_ruleutils.c +++ b/src/backend/distributed/deparser/citus_ruleutils.c @@ -815,9 +815,13 @@ deparse_index_columns(StringInfo buffer, List *indexParameterList, List *deparse NameListToQuotedString(indexElement->opclass)); } #if PG_VERSION_NUM >= PG_VERSION_13 - if (indexElement->opclassopts != NIL) { - ereport(ERROR, errmsg("citus currently doesn't support this index arguments")); - } + + /* Commit on postgres: 911e70207703799605f5a0e8aad9f06cff067c63*/ + if (indexElement->opclassopts != NIL) + { + ereport(ERROR, errmsg( + "citus currently doesn't support operator class parameters in indexes")); + } #endif if (indexElement->ordering != SORTBY_DEFAULT) diff --git a/src/backend/distributed/executor/local_executor.c b/src/backend/distributed/executor/local_executor.c index e15624776..1e6b1f2c1 100644 --- a/src/backend/distributed/executor/local_executor.c +++ b/src/backend/distributed/executor/local_executor.c @@ -294,7 +294,7 @@ ExecuteLocalTaskListExtended(List *taskList, * implemented. So, let planner to call distributed_planner() which * eventually calls standard_planner(). */ - localPlan = planner_compat(shardQuery, NULL, cursorOptions, paramListInfo); + localPlan = planner_compat(shardQuery, cursorOptions, paramListInfo); } char *shardQueryString = NULL; @@ -334,7 +334,7 @@ LocallyPlanAndExecuteMultipleQueries(List *queryStrings, TupleDestination *tuple 0); int cursorOptions = 0; ParamListInfo paramListInfo = NULL; - PlannedStmt *localPlan = planner_compat(shardQuery, NULL, cursorOptions, + PlannedStmt *localPlan = planner_compat(shardQuery, cursorOptions, paramListInfo); totalProcessedRows += ExecuteLocalTaskPlan(localPlan, queryString, tupleDest, task, diff --git a/src/backend/distributed/planner/combine_query_planner.c b/src/backend/distributed/planner/combine_query_planner.c index 0db7d4b6f..f0f2e4860 100644 --- a/src/backend/distributed/planner/combine_query_planner.c +++ b/src/backend/distributed/planner/combine_query_planner.c @@ -295,7 +295,7 @@ BuildSelectStatementViaStdPlanner(Query *combineQuery, List *remoteScanTargetLis ReplaceCitusExtraDataContainer = true; ReplaceCitusExtraDataContainerWithCustomScan = remoteScan; - standardStmt = standard_planner_compat(combineQuery, NULL, 0, NULL); + standardStmt = standard_planner_compat(combineQuery, 0, NULL); ReplaceCitusExtraDataContainer = false; ReplaceCitusExtraDataContainerWithCustomScan = NULL; diff --git a/src/backend/distributed/planner/distributed_planner.c b/src/backend/distributed/planner/distributed_planner.c index 3a7f77a95..cdcac8f81 100644 --- a/src/backend/distributed/planner/distributed_planner.c +++ b/src/backend/distributed/planner/distributed_planner.c @@ -223,7 +223,6 @@ distributed_planner(Query *parse, * postgres' planner. */ planContext.plan = standard_planner_compat(planContext.query, - NULL, planContext.cursorOptions, planContext.boundParams); if (needsDistributedPlanning) @@ -1053,7 +1052,7 @@ CreateDistributedPlan(uint64 planId, Query *originalQuery, Query *query, ParamLi * being contiguous. */ - standard_planner_compat(newQuery, NULL, 0, boundParams); + standard_planner_compat(newQuery, 0, boundParams); /* overwrite the old transformed query with the new transformed query */ *query = *newQuery; diff --git a/src/backend/distributed/planner/insert_select_planner.c b/src/backend/distributed/planner/insert_select_planner.c index 67037916b..bd3501b8e 100644 --- a/src/backend/distributed/planner/insert_select_planner.c +++ b/src/backend/distributed/planner/insert_select_planner.c @@ -1389,7 +1389,7 @@ CreateNonPushableInsertSelectPlan(uint64 planId, Query *parse, ParamListInfo bou /* plan the subquery, this may be another distributed query */ int cursorOptions = CURSOR_OPT_PARALLEL_OK; PlannedStmt *selectPlan = pg_plan_query_compat(selectQueryCopy, NULL, cursorOptions, - boundParams); + boundParams); bool repartitioned = IsRedistributablePlan(selectPlan->planTree) && IsSupportedRedistributionTarget(targetRelationId); diff --git a/src/backend/distributed/planner/local_plan_cache.c b/src/backend/distributed/planner/local_plan_cache.c index 380cb0952..a9b139eea 100644 --- a/src/backend/distributed/planner/local_plan_cache.c +++ b/src/backend/distributed/planner/local_plan_cache.c @@ -90,7 +90,7 @@ CacheLocalPlanForShardQuery(Task *task, DistributedPlan *originalDistributedPlan LockRelationOid(rangeTableEntry->relid, lockMode); LocalPlannedStatement *localPlannedStatement = CitusMakeNode(LocalPlannedStatement); - localPlan = planner_compat(shardQuery, NULL, 0, NULL); + localPlan = planner_compat(shardQuery, 0, NULL); localPlannedStatement->localPlan = localPlan; localPlannedStatement->shardId = task->anchorShardId; localPlannedStatement->localGroupId = GetLocalGroupId(); diff --git a/src/backend/distributed/planner/multi_explain.c b/src/backend/distributed/planner/multi_explain.c index c89e40d1c..f386f3123 100644 --- a/src/backend/distributed/planner/multi_explain.c +++ b/src/backend/distributed/planner/multi_explain.c @@ -314,8 +314,7 @@ ExplainSubPlans(DistributedPlan *distributedPlan, ExplainState *es) INSTR_TIME_SET_ZERO(planduration); - ExplainOnePlanCompat(plan, into, es, queryString, params, NULL, &planduration, - NULL); + ExplainOnePlanCompat(plan, into, es, queryString, params, NULL, &planduration); if (es->format == EXPLAIN_FORMAT_TEXT) { @@ -1135,7 +1134,7 @@ CitusExplainOneQuery(Query *query, int cursorOptions, IntoClause *into, /* run it (if needed) and produce output */ ExplainOnePlanCompat(plan, into, es, queryString, params, queryEnv, - &planduration, NULL); + &planduration); } @@ -1466,7 +1465,7 @@ ExplainOneQuery(Query *query, int cursorOptions, /* run it (if needed) and produce output */ ExplainOnePlanCompat(plan, into, es, queryString, params, queryEnv, - &planduration, NULL); + &planduration); } } diff --git a/src/backend/distributed/planner/multi_physical_planner.c b/src/backend/distributed/planner/multi_physical_planner.c index 764d89e6a..8729a4bb5 100644 --- a/src/backend/distributed/planner/multi_physical_planner.c +++ b/src/backend/distributed/planner/multi_physical_planner.c @@ -1279,19 +1279,16 @@ SetJoinRelatedColumnsCompat(RangeTblEntry *rangeTableEntry, /* We don't have any merged columns so set it to 0 */ rangeTableEntry->joinmergedcols = 0; - Var *var = NULL; - int varId = 1; - foreach_ptr(var, leftColumnVars) + int numvars = list_length(leftColumnVars); + for (int varId = 1; varId <= numvars; varId++) { rangeTableEntry->joinleftcols = lappend_int(rangeTableEntry->joinleftcols, varId); - varId++; } - varId = 1; - foreach_ptr(var, rightColumnVars) + numvars = list_length(rightColumnVars); + for (int varId = 1; varId <= numvars; varId++) { rangeTableEntry->joinrightcols = lappend_int(rangeTableEntry->joinrightcols, varId); - varId++; } #endif } diff --git a/src/backend/distributed/planner/recursive_planning.c b/src/backend/distributed/planner/recursive_planning.c index d1c07adb1..9e844a437 100644 --- a/src/backend/distributed/planner/recursive_planning.c +++ b/src/backend/distributed/planner/recursive_planning.c @@ -1174,7 +1174,7 @@ CreateDistributedSubPlan(uint32 subPlanId, Query *subPlanQuery) } DistributedSubPlan *subPlan = CitusMakeNode(DistributedSubPlan); - subPlan->plan = planner_compat(subPlanQuery, NULL, cursorOptions, NULL); + subPlan->plan = planner_compat(subPlanQuery, cursorOptions, NULL); subPlan->subPlanId = subPlanId; return subPlan; diff --git a/src/backend/distributed/test/distributed_intermediate_results.c b/src/backend/distributed/test/distributed_intermediate_results.c index 2ab999685..120cea563 100644 --- a/src/backend/distributed/test/distributed_intermediate_results.c +++ b/src/backend/distributed/test/distributed_intermediate_results.c @@ -50,7 +50,8 @@ partition_task_list_results(PG_FUNCTION_ARGS) bool binaryFormat = PG_GETARG_BOOL(3); Query *parsedQuery = ParseQueryString(queryString, NULL, 0); - PlannedStmt *queryPlan = pg_plan_query_compat(parsedQuery, queryString, + PlannedStmt *queryPlan = pg_plan_query_compat(parsedQuery, + queryString, CURSOR_OPT_PARALLEL_OK, NULL); if (!IsCitusCustomScan(queryPlan->planTree)) @@ -123,7 +124,8 @@ redistribute_task_list_results(PG_FUNCTION_ARGS) bool binaryFormat = PG_GETARG_BOOL(3); Query *parsedQuery = ParseQueryString(queryString, NULL, 0); - PlannedStmt *queryPlan = pg_plan_query_compat(parsedQuery, queryString, + PlannedStmt *queryPlan = pg_plan_query_compat(parsedQuery, + queryString, CURSOR_OPT_PARALLEL_OK, NULL); if (!IsCitusCustomScan(queryPlan->planTree)) diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index 62849040a..75bef4f04 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -32,14 +32,14 @@ #define lnext_compat(l, r) lnext(l, r) #define list_delete_cell_compat(l, c, p) list_delete_cell(l, c) #define pg_plan_query_compat(p, q, c, b) pg_plan_query(p, q, c, b) -#define planner_compat(p, q, c, b) planner(p, q, c, b) -#define standard_planner_compat(a, b, c, d) standard_planner(a, b, c, d) -#define PortalDefineQuerySelectCompat(a, b, c, e, f) PortalDefineQuery(a, b, c, \ - CMDTAG_SELECT, e, \ - f) +#define planner_compat(p, c, b) planner(p, NULL, c, b) +#define standard_planner_compat(a, c, d) standard_planner(a, NULL, c, d) +#define PortalDefineQuerySelectCompat(a, b, c, d, e) PortalDefineQuery(a, b, c, \ + CMDTAG_SELECT, d, \ + e) #define getOwnedSequencesCompat(a, b) getOwnedSequences(a) -#define ExplainOnePlanCompat(a, b, c, d, e, f, g, h) ExplainOnePlan(a, b, c, d, e, f, g, \ - h) +#define ExplainOnePlanCompat(a, b, c, d, e, f, g) ExplainOnePlan(a, b, c, d, e, f, g, \ + NULL) #define varoattno varattnosyn #define varnoold varnosyn #define Set_ptr_value(a, b) ((a)->ptr_value = (b)) @@ -49,12 +49,12 @@ #define lnext_compat(l, r) lnext(r) #define list_delete_cell_compat(l, c, p) list_delete_cell(l, c, p) #define pg_plan_query_compat(p, q, c, b) pg_plan_query(p, c, b) -#define planner_compat(p, q, c, b) planner(p, c, b) -#define standard_planner_compat(a, b, c, d) standard_planner(a, c, d) -#define PortalDefineQuerySelectCompat(a, b, c, e, f) PortalDefineQuery(a, b, c, "SELECT", \ - e, f) +#define planner_compat(p, c, b) planner(p, c, b) +#define standard_planner_compat(a, c, d) standard_planner(a, c, d) +#define PortalDefineQuerySelectCompat(a, b, c, d, e) PortalDefineQuery(a, b, c, "SELECT", \ + d, e) #define getOwnedSequencesCompat(a, b) getOwnedSequences(a, b) -#define ExplainOnePlanCompat(a, b, c, d, e, f, g, h) ExplainOnePlan(a, b, c, d, e, f, g) +#define ExplainOnePlanCompat(a, b, c, d, e, f, g) ExplainOnePlan(a, b, c, d, e, f, g) #define Set_ptr_value(a, b) ((a)->data.ptr_value = (b)) #define RangeTableEntryFromNSItem(a) (a) #define QueryCompletionCompat char diff --git a/src/test/regress/expected/multi_partitioning_1.out b/src/test/regress/expected/multi_partitioning_1.out index 18691286c..e85a4a2a9 100644 --- a/src/test/regress/expected/multi_partitioning_1.out +++ b/src/test/regress/expected/multi_partitioning_1.out @@ -4,6 +4,7 @@ SET citus.next_shard_id TO 1660000; SET citus.shard_count TO 4; SET citus.shard_replication_factor TO 1; +SET citus.enable_repartition_joins to ON; -- -- Distributed Partitioned Table Creation Tests -- @@ -1298,8 +1299,6 @@ SELECT relation::regclass, locktype, mode FROM pg_locks WHERE relation::regclass (3 rows) COMMIT; --- test locks on task-tracker SELECT -SET citus.task_executor_type TO 'task-tracker'; BEGIN; SELECT * FROM partitioning_locks AS pl1 JOIN partitioning_locks AS pl2 ON pl1.id = pl2.ref_id ORDER BY 1, 2; id | ref_id | time | id | ref_id | time @@ -1315,7 +1314,6 @@ SELECT relation::regclass, locktype, mode FROM pg_locks WHERE relation::regclass (3 rows) COMMIT; -RESET citus.task_executor_type; -- test locks on INSERT BEGIN; INSERT INTO partitioning_locks VALUES(1, 1, '2009-01-01'); diff --git a/src/test/regress/expected/pg13.out b/src/test/regress/expected/pg13.out index db8e29f00..938604788 100644 --- a/src/test/regress/expected/pg13.out +++ b/src/test/regress/expected/pg13.out @@ -169,7 +169,7 @@ SELECT create_distributed_table('test_table', 'a'); -- we currently don't support this CREATE INDEX test_table_index ON test_table USING gist (b tsvector_ops(siglen = 100)); -ERROR: citus currently doesn't support this index arguments +ERROR: citus currently doesn't support operator class parameters in indexes drop schema test_pg13 cascade; NOTICE: drop cascades to 10 other objects DETAIL: drop cascades to table dist_table diff --git a/src/test/regress/output/multi_complex_count_distinct.source b/src/test/regress/output/multi_complex_count_distinct.source index 6a3f0a098..47c4c5bdb 100644 --- a/src/test/regress/output/multi_complex_count_distinct.source +++ b/src/test/regress/output/multi_complex_count_distinct.source @@ -355,39 +355,6 @@ SELECT TRUCK | 1757 (7 rows) -EXPLAIN (COSTS false, VERBOSE true) -SELECT - l_shipmode, count(distinct l_partkey) - FROM lineitem_hash - GROUP BY l_shipmode - HAVING count(distinct l_suppkey) > 1550 - ORDER BY 1, 2 DESC; - QUERY PLAN ---------------------------------------------------------------------- - Sort - Output: remote_scan.l_shipmode, (count(DISTINCT remote_scan.count)) - Sort Key: remote_scan.l_shipmode, (count(DISTINCT remote_scan.count)) DESC - -> GroupAggregate - Output: remote_scan.l_shipmode, count(DISTINCT remote_scan.count) - Group Key: remote_scan.l_shipmode - Filter: (count(DISTINCT remote_scan.worker_column_3) > 1550) - -> Sort - Output: remote_scan.l_shipmode, remote_scan.count, remote_scan.worker_column_3 - Sort Key: remote_scan.l_shipmode - -> Custom Scan (Citus Adaptive) - Output: remote_scan.l_shipmode, remote_scan.count, remote_scan.worker_column_3 - Task Count: 8 - Tasks Shown: One of 8 - -> Task - Query: SELECT l_shipmode, l_partkey AS count, l_suppkey AS worker_column_3 FROM lineitem_hash_240000 lineitem_hash WHERE true GROUP BY l_shipmode, l_partkey, l_suppkey - Node: host=localhost port=xxxxx dbname=regression - -> HashAggregate - Output: l_shipmode, l_partkey, l_suppkey - Group Key: lineitem_hash.l_shipmode, lineitem_hash.l_partkey, lineitem_hash.l_suppkey - -> Seq Scan on public.lineitem_hash_240000 lineitem_hash - Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment -(22 rows) - -- count distinct is supported on single table subqueries SELECT * FROM ( diff --git a/src/test/regress/sql/pg13.sql b/src/test/regress/sql/pg13.sql index 7877eb9e1..3c423c96e 100644 --- a/src/test/regress/sql/pg13.sql +++ b/src/test/regress/sql/pg13.sql @@ -94,4 +94,4 @@ SELECT create_distributed_table('test_table', 'a'); -- we currently don't support this CREATE INDEX test_table_index ON test_table USING gist (b tsvector_ops(siglen = 100)); -drop schema test_pg13 cascade; \ No newline at end of file +drop schema test_pg13 cascade; From b641f63bfd79510471c178587e664fcfd77da7b0 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Fri, 24 Jul 2020 13:08:21 +0300 Subject: [PATCH 45/52] Use CMDTAG_SELECT_COMPAT CMDTAG_SELECT exists in PG12 hence defining a MACRO such as CMDTAG_SELECT -> "SELECT" is not possible. I chose CMDTAG_SELECT_COMPAT because with the COMPAT suffix it is explicit that it maps to different things in different versions and also has a less chance of mapping something irrevelant. For example if we used SELECT as a macro, then it would map every SELECT to whatever it is mapping to, which might have unexpected/undesired behaviour. --- src/backend/distributed/commands/multi_copy.c | 4 ++-- src/backend/distributed/executor/multi_executor.c | 11 ++++++----- .../executor/partitioned_intermediate_results.c | 3 ++- src/backend/distributed/metadata/metadata_utility.c | 2 +- .../distributed/transaction/transaction_recovery.c | 5 +++-- src/include/distributed/version_compat.h | 7 ++----- 6 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/backend/distributed/commands/multi_copy.c b/src/backend/distributed/commands/multi_copy.c index c857e754f..dfb7ec164 100644 --- a/src/backend/distributed/commands/multi_copy.c +++ b/src/backend/distributed/commands/multi_copy.c @@ -2821,8 +2821,8 @@ ProcessCopyStmt(CopyStmt *copyStatement, QueryCompletionCompat *completionTag, c /* consider using RangeVarGetRelidExtended to check perms before locking */ Relation copiedRelation = table_openrv(copyStatement->relation, - isFrom ? RowExclusiveLock : - AccessShareLock); + isFrom ? RowExclusiveLock : + AccessShareLock); bool isCitusRelation = IsCitusTable(RelationGetRelid(copiedRelation)); diff --git a/src/backend/distributed/executor/multi_executor.c b/src/backend/distributed/executor/multi_executor.c index d0c04e396..9296397cb 100644 --- a/src/backend/distributed/executor/multi_executor.c +++ b/src/backend/distributed/executor/multi_executor.c @@ -628,11 +628,12 @@ ExecutePlanIntoDestReceiver(PlannedStmt *queryPlan, ParamListInfo params, /* don't display the portal in pg_cursors, it is for internal use only */ portal->visible = false; - PortalDefineQuerySelectCompat(portal, - NULL, - "", - list_make1(queryPlan), - NULL); + PortalDefineQuery(portal, + NULL, + "", + CMDTAG_SELECT_COMPAT, + list_make1(queryPlan), + NULL); PortalStart(portal, params, eflags, GetActiveSnapshot()); PortalRun(portal, count, false, true, dest, dest, NULL); diff --git a/src/backend/distributed/executor/partitioned_intermediate_results.c b/src/backend/distributed/executor/partitioned_intermediate_results.c index 21722dbfc..d0dec1dc3 100644 --- a/src/backend/distributed/executor/partitioned_intermediate_results.c +++ b/src/backend/distributed/executor/partitioned_intermediate_results.c @@ -266,7 +266,8 @@ StartPortalForQueryExecution(const char *queryString) /* don't display the portal in pg_cursors, it is for internal use only */ portal->visible = false; - PortalDefineQuerySelectCompat(portal, NULL, queryString, list_make1(queryPlan), NULL); + PortalDefineQuery(portal, NULL, queryString, CMDTAG_SELECT_COMPAT, list_make1( + queryPlan), NULL); int eflags = 0; PortalStart(portal, NULL, eflags, GetActiveSnapshot()); diff --git a/src/backend/distributed/metadata/metadata_utility.c b/src/backend/distributed/metadata/metadata_utility.c index b3655853b..53774bc09 100644 --- a/src/backend/distributed/metadata/metadata_utility.c +++ b/src/backend/distributed/metadata/metadata_utility.c @@ -634,7 +634,7 @@ NodeGroupHasShardPlacements(int32 groupId, bool onlyConsiderActivePlacements) ScanKeyData scanKey[2]; Relation pgPlacement = table_open(DistPlacementRelationId(), - AccessShareLock); + AccessShareLock); ScanKeyInit(&scanKey[0], Anum_pg_dist_placement_groupid, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(groupId)); diff --git a/src/backend/distributed/transaction/transaction_recovery.c b/src/backend/distributed/transaction/transaction_recovery.c index f868efffb..4fc90f705 100644 --- a/src/backend/distributed/transaction/transaction_recovery.c +++ b/src/backend/distributed/transaction/transaction_recovery.c @@ -95,7 +95,8 @@ LogTransactionRecord(int32 groupId, char *transactionName) values[Anum_pg_dist_transaction_gid - 1] = CStringGetTextDatum(transactionName); /* open transaction relation and insert new tuple */ - Relation pgDistTransaction = table_open(DistTransactionRelationId(), RowExclusiveLock); + Relation pgDistTransaction = table_open(DistTransactionRelationId(), + RowExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistTransaction); HeapTuple heapTuple = heap_form_tuple(tupleDescriptor, values, isNulls); @@ -172,7 +173,7 @@ RecoverWorkerTransactions(WorkerNode *workerNode) /* take table lock first to avoid running concurrently */ Relation pgDistTransaction = table_open(DistTransactionRelationId(), - ShareUpdateExclusiveLock); + ShareUpdateExclusiveLock); TupleDesc tupleDescriptor = RelationGetDescr(pgDistTransaction); /* diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index 75bef4f04..99f147070 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -34,10 +34,8 @@ #define pg_plan_query_compat(p, q, c, b) pg_plan_query(p, q, c, b) #define planner_compat(p, c, b) planner(p, NULL, c, b) #define standard_planner_compat(a, c, d) standard_planner(a, NULL, c, d) -#define PortalDefineQuerySelectCompat(a, b, c, d, e) PortalDefineQuery(a, b, c, \ - CMDTAG_SELECT, d, \ - e) #define getOwnedSequencesCompat(a, b) getOwnedSequences(a) +#define CMDTAG_SELECT_COMPAT CMDTAG_SELECT #define ExplainOnePlanCompat(a, b, c, d, e, f, g) ExplainOnePlan(a, b, c, d, e, f, g, \ NULL) #define varoattno varattnosyn @@ -51,8 +49,7 @@ #define pg_plan_query_compat(p, q, c, b) pg_plan_query(p, c, b) #define planner_compat(p, c, b) planner(p, c, b) #define standard_planner_compat(a, c, d) standard_planner(a, c, d) -#define PortalDefineQuerySelectCompat(a, b, c, d, e) PortalDefineQuery(a, b, c, "SELECT", \ - d, e) +#define CMDTAG_SELECT_COMPAT "SELECT" #define getOwnedSequencesCompat(a, b) getOwnedSequences(a, b) #define ExplainOnePlanCompat(a, b, c, d, e, f, g) ExplainOnePlan(a, b, c, d, e, f, g) #define Set_ptr_value(a, b) ((a)->data.ptr_value = (b)) From 8e9b52971cc8f48eccaabe48d09665564321745c Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Fri, 24 Jul 2020 13:17:17 +0300 Subject: [PATCH 46/52] Use new var field names in the codebase The codebase is updated to use varattnosync and varnosyn and we defined the macros for older versions. This way we can just remove the macros when we drop an older version. --- .../planner/combine_query_planner.c | 2 +- .../planner/insert_select_planner.c | 2 +- .../distributed/planner/multi_join_order.c | 2 +- .../planner/multi_logical_optimizer.c | 6 +++--- .../planner/multi_physical_planner.c | 20 +++++++++---------- .../distributed/planner/recursive_planning.c | 4 ++-- src/include/distributed/version_compat.h | 4 ++-- 7 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/backend/distributed/planner/combine_query_planner.c b/src/backend/distributed/planner/combine_query_planner.c index f0f2e4860..8f2512753 100644 --- a/src/backend/distributed/planner/combine_query_planner.c +++ b/src/backend/distributed/planner/combine_query_planner.c @@ -93,7 +93,7 @@ RemoteScanTargetList(List *workerTargetList) Var *remoteScanColumn = makeVarFromTargetEntry(tableId, workerTargetEntry); remoteScanColumn->varattno = columnId; - remoteScanColumn->varoattno = columnId; + remoteScanColumn->varattnosyn = columnId; columnId++; if (remoteScanColumn->vartype == RECORDOID || remoteScanColumn->vartype == diff --git a/src/backend/distributed/planner/insert_select_planner.c b/src/backend/distributed/planner/insert_select_planner.c index bd3501b8e..6f25b1a59 100644 --- a/src/backend/distributed/planner/insert_select_planner.c +++ b/src/backend/distributed/planner/insert_select_planner.c @@ -514,7 +514,7 @@ CreateTargetListForCombineQuery(List *targetList) Var *column = makeVarFromTargetEntry(masterTableId, originalTargetEntry); column->varattno = columnId; - column->varoattno = columnId; + column->varattnosyn = columnId; columnId++; if (column->vartype == RECORDOID || column->vartype == RECORDARRAYOID) diff --git a/src/backend/distributed/planner/multi_join_order.c b/src/backend/distributed/planner/multi_join_order.c index e5b84a014..9b5dfde7d 100644 --- a/src/backend/distributed/planner/multi_join_order.c +++ b/src/backend/distributed/planner/multi_join_order.c @@ -1365,7 +1365,7 @@ PartitionColumn(Oid relationId, uint32 rangeTableId) partitionColumn = partitionKey; partitionColumn->varno = rangeTableId; - partitionColumn->varnoold = rangeTableId; + partitionColumn->varnosyn = rangeTableId; return partitionColumn; } diff --git a/src/backend/distributed/planner/multi_logical_optimizer.c b/src/backend/distributed/planner/multi_logical_optimizer.c index 4d0fbc514..f9021b8bd 100644 --- a/src/backend/distributed/planner/multi_logical_optimizer.c +++ b/src/backend/distributed/planner/multi_logical_optimizer.c @@ -1438,7 +1438,7 @@ MasterExtendedOpNode(MultiExtendedOp *originalOpNode, */ Var *column = makeVarFromTargetEntry(masterTableId, originalTargetEntry); column->varattno = walkerContext.columnId; - column->varoattno = walkerContext.columnId; + column->varattnosyn = walkerContext.columnId; walkerContext.columnId++; if (column->vartype == RECORDOID || column->vartype == RECORDARRAYOID) @@ -1673,9 +1673,9 @@ MasterAggregateExpression(Aggref *originalAggregate, } columnToUpdate->varno = masterTableId; - columnToUpdate->varnoold = masterTableId; + columnToUpdate->varnosyn = masterTableId; columnToUpdate->varattno = startColumnCount + columnIndex; - columnToUpdate->varoattno = startColumnCount + columnIndex; + columnToUpdate->varattnosyn = startColumnCount + columnIndex; } /* we added that many columns */ diff --git a/src/backend/distributed/planner/multi_physical_planner.c b/src/backend/distributed/planner/multi_physical_planner.c index 8729a4bb5..4716ad468 100644 --- a/src/backend/distributed/planner/multi_physical_planner.c +++ b/src/backend/distributed/planner/multi_physical_planner.c @@ -1086,8 +1086,8 @@ QueryJoinTree(MultiNode *multiNode, List *dependentJobList, List **rangeTableLis UpdateColumnAttributes(column, *rangeTableList, dependentJobList); /* adjust our column old attributes for partition pruning to work */ - column->varnoold = column->varno; - column->varoattno = column->varattno; + column->varnosyn = column->varno; + column->varattnosyn = column->varattno; } /* make AND clauses explicit after fixing them */ @@ -1561,8 +1561,8 @@ UpdateAllColumnAttributes(Node *columnContainer, List *rangeTableList, static void UpdateColumnAttributes(Var *column, List *rangeTableList, List *dependentJobList) { - Index originalTableId = column->varnoold; - AttrNumber originalColumnId = column->varoattno; + Index originalTableId = column->varnosyn; + AttrNumber originalColumnId = column->varattnosyn; /* find the new table identifier */ Index newTableId = NewTableId(originalTableId, rangeTableList); @@ -1646,8 +1646,8 @@ NewColumnId(Index originalTableId, AttrNumber originalColumnId, * Check against the *old* values for this column, as the new values * would have been updated already. */ - if (column->varnoold == originalTableId && - column->varoattno == originalColumnId) + if (column->varnosyn == originalTableId && + column->varattnosyn == originalColumnId) { newColumnId = columnIndex; break; @@ -2977,8 +2977,8 @@ AnchorRangeTableIdList(List *rangeTableList, List *baseRangeTableIdList) /* - * AdjustColumnOldAttributes adjust the old tableId (varnoold) and old columnId - * (varoattno), and sets them equal to the new values. We need this adjustment + * AdjustColumnOldAttributes adjust the old tableId (varnosyn) and old columnId + * (varattnosyn), and sets them equal to the new values. We need this adjustment * for partition pruning where we compare these columns with partition columns * loaded from system catalogs. Since columns loaded from system catalogs always * have the same old and new values, we also need to adjust column values here. @@ -2992,8 +2992,8 @@ AdjustColumnOldAttributes(List *expressionList) foreach(columnCell, columnList) { Var *column = (Var *) lfirst(columnCell); - column->varnoold = column->varno; - column->varoattno = column->varattno; + column->varnosyn = column->varno; + column->varattnosyn = column->varattno; } } diff --git a/src/backend/distributed/planner/recursive_planning.c b/src/backend/distributed/planner/recursive_planning.c index 9e844a437..253ec2aaf 100644 --- a/src/backend/distributed/planner/recursive_planning.c +++ b/src/backend/distributed/planner/recursive_planning.c @@ -1681,8 +1681,8 @@ BuildReadIntermediateResultsQuery(List *targetEntryList, List *columnAliasList, functionColumnVar->vartypmod = columnTypMod; functionColumnVar->varcollid = columnCollation; functionColumnVar->varlevelsup = 0; - functionColumnVar->varnoold = 1; - functionColumnVar->varoattno = columnNumber; + functionColumnVar->varnosyn = 1; + functionColumnVar->varattnosyn = columnNumber; functionColumnVar->location = -1; TargetEntry *newTargetEntry = makeNode(TargetEntry); diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index 99f147070..102ce8401 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -38,8 +38,6 @@ #define CMDTAG_SELECT_COMPAT CMDTAG_SELECT #define ExplainOnePlanCompat(a, b, c, d, e, f, g) ExplainOnePlan(a, b, c, d, e, f, g, \ NULL) -#define varoattno varattnosyn -#define varnoold varnosyn #define Set_ptr_value(a, b) ((a)->ptr_value = (b)) #define RangeTableEntryFromNSItem(a) ((a)->p_rte) #define QueryCompletionCompat QueryCompletion @@ -55,6 +53,8 @@ #define Set_ptr_value(a, b) ((a)->data.ptr_value = (b)) #define RangeTableEntryFromNSItem(a) (a) #define QueryCompletionCompat char +#define varattnosyn varoattno +#define varnosyn varnoold #endif #if PG_VERSION_NUM >= PG_VERSION_12 From fe1e1c9b68372bc7e953a69f185b267eb70c619d Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Fri, 24 Jul 2020 13:21:07 +0300 Subject: [PATCH 47/52] Replace Set_ptr_value as SetListCellPtr to be more explicit Move header to right place and fix comment style --- src/backend/distributed/commands/multi_copy.c | 7 ++++--- src/backend/distributed/deparser/ruleutils_13.c | 14 ++++++++------ .../distributed/planner/multi_physical_planner.c | 2 +- .../distributed/planner/multi_router_planner.c | 2 +- src/include/distributed/version_compat.h | 6 +++--- 5 files changed, 17 insertions(+), 14 deletions(-) diff --git a/src/backend/distributed/commands/multi_copy.c b/src/backend/distributed/commands/multi_copy.c index dfb7ec164..473303cc9 100644 --- a/src/backend/distributed/commands/multi_copy.c +++ b/src/backend/distributed/commands/multi_copy.c @@ -93,13 +93,14 @@ #include "distributed/hash_helpers.h" #include "executor/executor.h" #include "foreign/foreign.h" -#if PG_VERSION_NUM >= PG_VERSION_13 -#include "tcop/cmdtag.h" -#endif + #include "libpq/libpq.h" #include "libpq/pqformat.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" +#if PG_VERSION_NUM >= PG_VERSION_13 +#include "tcop/cmdtag.h" +#endif #include "tsearch/ts_locale.h" #include "utils/builtins.h" #include "utils/lsyscache.h" diff --git a/src/backend/distributed/deparser/ruleutils_13.c b/src/backend/distributed/deparser/ruleutils_13.c index e19a2d6a8..25607f7e6 100644 --- a/src/backend/distributed/deparser/ruleutils_13.c +++ b/src/backend/distributed/deparser/ruleutils_13.c @@ -3556,9 +3556,10 @@ get_variable(Var *var, int levelsup, bool istoplevel, deparse_context *context) if (var->varnosyn > 0 && var->varnosyn <= list_length(dpns->rtable) && dpns->plan == NULL) { rte = rt_fetch(var->varnosyn, dpns->rtable); - // if the rte var->varnosync points to is not a regular table and it is a join - // then the correct relname will be found with var->varnosync and var->varattnosync - // TODO:: this is a workaround and it can be simplified. + /* + * if the rte var->varnosyn points to is not a regular table and it is a join + * then the correct relname will be found with var->varnosyn and var->varattnosyn + */ if (rte->rtekind == RTE_JOIN && rte->relid == 0 && var->varnosyn != var->varno) { varno = var->varnosyn; varattno = var->varattnosyn; @@ -3969,9 +3970,10 @@ get_name_for_var_field(Var *var, int fieldno, if (var->varnosyn > 0 && var->varnosyn <= list_length(dpns->rtable) && dpns->plan == NULL) { rte = rt_fetch(var->varnosyn, dpns->rtable); - // if the rte var->varnosync points to is not a regular table and it is a join - // then the correct relname will be found with var->varnosync and var->varattnosync - // TODO:: this is a workaround and it can be simplified. + /* + * if the rte var->varnosyn points to is not a regular table and it is a join + * then the correct relname will be found with var->varnosyn and var->varattnosyn + */ if (rte->rtekind == RTE_JOIN && rte->relid == 0 && var->varnosyn != var->varno) { varno = var->varnosyn; varattno = var->varattnosyn; diff --git a/src/backend/distributed/planner/multi_physical_planner.c b/src/backend/distributed/planner/multi_physical_planner.c index 4716ad468..07f73db88 100644 --- a/src/backend/distributed/planner/multi_physical_planner.c +++ b/src/backend/distributed/planner/multi_physical_planner.c @@ -5171,7 +5171,7 @@ GreedyAssignTask(WorkerNode *workerNode, List *taskList, List *activeShardPlacem rotatePlacementListBy = replicaIndex; /* overwrite task list to signal that this task is assigned */ - Set_ptr_value(taskCell, NULL); + SetListCellPtr(taskCell, NULL); break; } } diff --git a/src/backend/distributed/planner/multi_router_planner.c b/src/backend/distributed/planner/multi_router_planner.c index 75493e91a..e322622bc 100644 --- a/src/backend/distributed/planner/multi_router_planner.c +++ b/src/backend/distributed/planner/multi_router_planner.c @@ -2976,7 +2976,7 @@ NormalizeMultiRowInsertTargetList(Query *query) expandedValuesList = lappend(expandedValuesList, targetExpr); } - Set_ptr_value(valuesListCell, (void *) expandedValuesList); + SetListCellPtr(valuesListCell, (void *) expandedValuesList); } /* reset coltypes, coltypmods, colcollations and rebuild them below */ diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index 102ce8401..4bda13761 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -38,7 +38,7 @@ #define CMDTAG_SELECT_COMPAT CMDTAG_SELECT #define ExplainOnePlanCompat(a, b, c, d, e, f, g) ExplainOnePlan(a, b, c, d, e, f, g, \ NULL) -#define Set_ptr_value(a, b) ((a)->ptr_value = (b)) +#define SetListCellPtr(a, b) ((a)->ptr_value = (b)) #define RangeTableEntryFromNSItem(a) ((a)->p_rte) #define QueryCompletionCompat QueryCompletion #else /* pre PG13 */ @@ -50,11 +50,11 @@ #define CMDTAG_SELECT_COMPAT "SELECT" #define getOwnedSequencesCompat(a, b) getOwnedSequences(a, b) #define ExplainOnePlanCompat(a, b, c, d, e, f, g) ExplainOnePlan(a, b, c, d, e, f, g) -#define Set_ptr_value(a, b) ((a)->data.ptr_value = (b)) +#define SetListCellPtr(a, b) ((a)->data.ptr_value = (b)) #define RangeTableEntryFromNSItem(a) (a) #define QueryCompletionCompat char #define varattnosyn varoattno -#define varnosyn varnoold +#define varnosyn varnoold #endif #if PG_VERSION_NUM >= PG_VERSION_12 From 63ed126ad4ffad793292c8688d6fb1f9f29fdd16 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Fri, 24 Jul 2020 16:44:50 +0300 Subject: [PATCH 48/52] Set buffer usage with explain It seems that currently we process even postgres tables in explain commands. This is because we register a hook for explain and we don't have any check to see if the query has any citus table. With this commit, we now send the buffer usage as well to the relevant API. There is some duplicate in the code but it is because of the existing structure, we can refactor this separately. --- .../distributed/planner/multi_explain.c | 60 ++++++++++++++++++- src/include/distributed/version_compat.h | 6 +- 2 files changed, 60 insertions(+), 6 deletions(-) diff --git a/src/backend/distributed/planner/multi_explain.c b/src/backend/distributed/planner/multi_explain.c index f386f3123..414531c56 100644 --- a/src/backend/distributed/planner/multi_explain.c +++ b/src/backend/distributed/planner/multi_explain.c @@ -272,7 +272,16 @@ ExplainSubPlans(DistributedPlan *distributedPlan, ExplainState *es) ParamListInfo params = NULL; char *queryString = NULL; instr_time planduration; + #if PG_VERSION_NUM >= PG_VERSION_13 + BufferUsage bufusage_start, + bufusage; + + if (es->buffers) + { + bufusage_start = pgBufferUsage; + } + #endif if (es->format == EXPLAIN_FORMAT_TEXT) { char *resultId = GenerateResultId(planId, subPlan->subPlanId); @@ -314,7 +323,18 @@ ExplainSubPlans(DistributedPlan *distributedPlan, ExplainState *es) INSTR_TIME_SET_ZERO(planduration); - ExplainOnePlanCompat(plan, into, es, queryString, params, NULL, &planduration); + #if PG_VERSION_NUM >= PG_VERSION_13 + + /* calc differences of buffer counters. */ + if (es->buffers) + { + memset(&bufusage, 0, sizeof(BufferUsage)); + BufferUsageAccumDiff(&bufusage, &pgBufferUsage, &bufusage_start); + } + #endif + + ExplainOnePlanCompat(plan, into, es, queryString, params, NULL, &planduration, + (es->buffers ? &bufusage : NULL)); if (es->format == EXPLAIN_FORMAT_TEXT) { @@ -1123,6 +1143,15 @@ CitusExplainOneQuery(Query *query, int cursorOptions, IntoClause *into, /* rest is copied from ExplainOneQuery() */ instr_time planstart, planduration; + #if PG_VERSION_NUM >= PG_VERSION_13 + BufferUsage bufusage_start, + bufusage; + + if (es->buffers) + { + bufusage_start = pgBufferUsage; + } + #endif INSTR_TIME_SET_CURRENT(planstart); @@ -1132,9 +1161,19 @@ CitusExplainOneQuery(Query *query, int cursorOptions, IntoClause *into, INSTR_TIME_SET_CURRENT(planduration); INSTR_TIME_SUBTRACT(planduration, planstart); + #if PG_VERSION_NUM >= PG_VERSION_13 + + /* calc differences of buffer counters. */ + if (es->buffers) + { + memset(&bufusage, 0, sizeof(BufferUsage)); + BufferUsageAccumDiff(&bufusage, &pgBufferUsage, &bufusage_start); + } + #endif + /* run it (if needed) and produce output */ ExplainOnePlanCompat(plan, into, es, queryString, params, queryEnv, - &planduration); + &planduration, (es->buffers ? &bufusage : NULL)); } @@ -1454,7 +1493,13 @@ ExplainOneQuery(Query *query, int cursorOptions, { instr_time planstart, planduration; + #if PG_VERSION_NUM >= PG_VERSION_13 + BufferUsage bufusage_start, + bufusage; + if (es->buffers) + bufusage_start = pgBufferUsage; + #endif INSTR_TIME_SET_CURRENT(planstart); /* plan the query */ @@ -1463,9 +1508,18 @@ ExplainOneQuery(Query *query, int cursorOptions, INSTR_TIME_SET_CURRENT(planduration); INSTR_TIME_SUBTRACT(planduration, planstart); + #if PG_VERSION_NUM >= PG_VERSION_13 + + /* calc differences of buffer counters. */ + if (es->buffers) + { + memset(&bufusage, 0, sizeof(BufferUsage)); + BufferUsageAccumDiff(&bufusage, &pgBufferUsage, &bufusage_start); + } + #endif /* run it (if needed) and produce output */ ExplainOnePlanCompat(plan, into, es, queryString, params, queryEnv, - &planduration); + &planduration, (es->buffers ? &bufusage : NULL)); } } diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index 4bda13761..d140a66b7 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -36,8 +36,8 @@ #define standard_planner_compat(a, c, d) standard_planner(a, NULL, c, d) #define getOwnedSequencesCompat(a, b) getOwnedSequences(a) #define CMDTAG_SELECT_COMPAT CMDTAG_SELECT -#define ExplainOnePlanCompat(a, b, c, d, e, f, g) ExplainOnePlan(a, b, c, d, e, f, g, \ - NULL) +#define ExplainOnePlanCompat(a, b, c, d, e, f, g, h) ExplainOnePlan(a, b, c, d, e, f, g, \ + h) #define SetListCellPtr(a, b) ((a)->ptr_value = (b)) #define RangeTableEntryFromNSItem(a) ((a)->p_rte) #define QueryCompletionCompat QueryCompletion @@ -49,7 +49,7 @@ #define standard_planner_compat(a, c, d) standard_planner(a, c, d) #define CMDTAG_SELECT_COMPAT "SELECT" #define getOwnedSequencesCompat(a, b) getOwnedSequences(a, b) -#define ExplainOnePlanCompat(a, b, c, d, e, f, g) ExplainOnePlan(a, b, c, d, e, f, g) +#define ExplainOnePlanCompat(a, b, c, d, e, f, g, h) ExplainOnePlan(a, b, c, d, e, f, g) #define SetListCellPtr(a, b) ((a)->data.ptr_value = (b)) #define RangeTableEntryFromNSItem(a) (a) #define QueryCompletionCompat char From 33406598e37cd8041d67e448f90ebd9edbb9309c Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Mon, 27 Jul 2020 12:31:06 +0300 Subject: [PATCH 49/52] Add ruleutils changes from 3977 and 4011 --- .../distributed/deparser/ruleutils_13.c | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/backend/distributed/deparser/ruleutils_13.c b/src/backend/distributed/deparser/ruleutils_13.c index 25607f7e6..5d0ec331f 100644 --- a/src/backend/distributed/deparser/ruleutils_13.c +++ b/src/backend/distributed/deparser/ruleutils_13.c @@ -984,6 +984,7 @@ set_relation_column_names(deparse_namespace *dpns, RangeTblEntry *rte, int ncolumns; char **real_colnames; bool changed_any; + bool has_anonymous; int noldcolumns; int i; int j; @@ -1071,6 +1072,7 @@ set_relation_column_names(deparse_namespace *dpns, RangeTblEntry *rte, */ noldcolumns = list_length(rte->eref->colnames); changed_any = false; + has_anonymous = false; j = 0; for (i = 0; i < ncolumns; i++) { @@ -1108,6 +1110,13 @@ set_relation_column_names(deparse_namespace *dpns, RangeTblEntry *rte, /* Remember if any assigned aliases differ from "real" name */ if (!changed_any && strcmp(colname, real_colname) != 0) changed_any = true; + + /* + * Remember if there is a reference to an anonymous column as named by + * char * FigureColname(Node *node) + */ + if (!has_anonymous && strcmp(real_colname, "?column?") == 0) + has_anonymous = true; } /* @@ -1137,7 +1146,7 @@ set_relation_column_names(deparse_namespace *dpns, RangeTblEntry *rte, else if (rte->alias && rte->alias->colnames != NIL) colinfo->printaliases = true; else - colinfo->printaliases = changed_any; + colinfo->printaliases = changed_any || has_anonymous; } /* @@ -2975,7 +2984,7 @@ get_insert_query_def(Query *query, deparse_context *context) /* INSERT requires AS keyword for target alias */ if (rte->alias != NULL) appendStringInfo(buf, "AS %s ", - quote_identifier(rte->alias->aliasname)); + quote_identifier(get_rtable_name(query->resultRelation, context))); /* * Add the insert-column-names list. Any indirection decoration needed on @@ -3174,7 +3183,7 @@ get_update_query_def(Query *query, deparse_context *context) if(rte->eref != NULL) appendStringInfo(buf, " %s", - quote_identifier(rte->eref->aliasname)); + quote_identifier(get_rtable_name(query->resultRelation, context))); } else { @@ -3186,7 +3195,7 @@ get_update_query_def(Query *query, deparse_context *context) if (rte->alias != NULL) appendStringInfo(buf, " %s", - quote_identifier(rte->alias->aliasname)); + quote_identifier(get_rtable_name(query->resultRelation, context))); } appendStringInfoString(buf, " SET "); @@ -3406,7 +3415,7 @@ get_delete_query_def(Query *query, deparse_context *context) if(rte->eref != NULL) appendStringInfo(buf, " %s", - quote_identifier(rte->eref->aliasname)); + quote_identifier(get_rtable_name(query->resultRelation, context))); } else { @@ -3418,7 +3427,7 @@ get_delete_query_def(Query *query, deparse_context *context) if (rte->alias != NULL) appendStringInfo(buf, " %s", - quote_identifier(rte->alias->aliasname)); + quote_identifier(get_rtable_name(query->resultRelation, context))); } /* Add the USING clause if given */ From 283b1db6a464773f96234bc93b07c118a122c855 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Tue, 28 Jul 2020 23:24:28 +0300 Subject: [PATCH 50/52] add pg13 to CI --- .circleci/config.yml | 105 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 104 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 84050db1c..ed3c2b421 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,7 +6,7 @@ orbs: jobs: build: docker: - - image: 'citus/extbuilder:latest' + - image: 'heisenberg302/extbuilder-13:latest' steps: - checkout - run: @@ -81,6 +81,8 @@ jobs: - codecov/upload: flags: 'test_11,multi' + + test-11_check-van-mx: docker: - image: 'citus/exttester-11:latest' @@ -154,6 +156,18 @@ jobs: command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext --target check-pg-upgrade --old-pg-version 11 --new-pg-version 12' no_output_timeout: 2m + test-12-13_check-pg-upgrade: + docker: + - image: 'heisenberg302/pgupgradetester:latest' + working_directory: /home/circleci/project + steps: + - attach_workspace: + at: . + - run: + name: 'Install and test postgres upgrade' + command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext --target check-pg-upgrade --old-pg-version 12 --new-pg-version 13' + no_output_timeout: 2m + test-12_check-multi: docker: - image: 'citus/exttester-12:latest' @@ -250,6 +264,82 @@ jobs: install-and-test-ext --target check-citus-upgrade-mixed --citus-pre-tar /install-pg11-citusv8.3.0.tar no_output_timeout: 2m + test-13_check-multi: + docker: + - image: 'heisenberg302/exttester-13:latest' + working_directory: /home/circleci/project + steps: + - attach_workspace: + at: . + - run: + name: 'Install and Test (check-multi)' + command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-multi' + no_output_timeout: 2m + - codecov/upload: + flags: 'test_13,multi' + + test-13_check-van-mx: + docker: + - image: 'heisenberg302/exttester-13:latest' + working_directory: /home/circleci/project + steps: + - attach_workspace: + at: . + - run: + name: 'Install and Test (check-van-mx)' + command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-vanilla check-multi-mx' + no_output_timeout: 2m + - codecov/upload: + flags: 'test_13,vanilla,mx' + test-13_check-iso-work-fol: + docker: + - image: 'heisenberg302/exttester-13:latest' + working_directory: /home/circleci/project + steps: + - attach_workspace: + at: . + - run: + name: 'Install and Test (check-iso-work-fol)' + command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-isolation check-worker' + no_output_timeout: 2m + - codecov/upload: + flags: 'test_13,isolation,worker' + test-13_check-fol: + docker: + - image: 'heisenberg302/exttester-13:latest' + working_directory: /home/circleci/project + steps: + - attach_workspace: + at: . + - run: + name: 'Enable core dumps' + command: 'ulimit -c unlimited' + - run: + name: 'Install and Test (fol)' + command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-follower-cluster' + no_output_timeout: 2m + - run: + command: | + mkdir -p /tmp/core_dumps + cp core.* /tmp/core_dumps + when: on_fail + - codecov/upload: + flags: 'test_13,follower' + - store_artifacts: + path: '/tmp/core_dumps' + + test-13_check-failure: + docker: + - image: 'heisenberg302/failtester-13:latest' + working_directory: /home/circleci/project + steps: + - attach_workspace: + at: . + - run: + name: 'Install and Test (check-failure)' + command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext check-failure' + no_output_timeout: 2m + check-merge-to-enterprise: docker: - image: buildpack-deps:stretch @@ -325,8 +415,21 @@ workflows: - test-12_check-failure: requires: [build] + - test-13_check-multi: + requires: [build] + - test-13_check-van-mx: + requires: [build] + - test-13_check-iso-work-fol: + requires: [build] + - test-13_check-fol: + requires: [build] + - test-13_check-failure: + requires: [build] + - test-11-12_check-pg-upgrade: requires: [build] + - test-12-13_check-pg-upgrade: + requires: [build] - test-11_check-citus-upgrade: requires: [build] From fe4ac51d8c9ad9a36b7a784db192439d2189ed72 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Thu, 30 Jul 2020 10:46:32 +0300 Subject: [PATCH 51/52] Normalize Output:.. since it changes with pg13 Fix indentation for better readability --- .circleci/config.yml | 2 +- .../partitioned_intermediate_results.c | 4 +- src/include/distributed/version_compat.h | 4 +- src/test/regress/bin/normalize.sed | 2 + .../regress/expected/multi_data_types.out | 4 +- src/test/regress/expected/multi_explain.out | 68 +++---- .../regress/expected/multi_mx_explain.out | 8 +- .../multi_subquery_window_functions.out | 44 ++--- .../expected/subquery_complex_target_list.out | 6 +- .../expected/tdigest_aggregate_support.out | 180 +++++++++--------- .../regress/spec/isolation_cancellation.spec | 5 - 11 files changed, 162 insertions(+), 165 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ed3c2b421..13e5264ce 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -166,7 +166,7 @@ jobs: - run: name: 'Install and test postgres upgrade' command: 'chown -R circleci:circleci /home/circleci && install-and-test-ext --target check-pg-upgrade --old-pg-version 12 --new-pg-version 13' - no_output_timeout: 2m + no_output_timeout: 2m test-12_check-multi: docker: diff --git a/src/backend/distributed/executor/partitioned_intermediate_results.c b/src/backend/distributed/executor/partitioned_intermediate_results.c index d0dec1dc3..3b481a231 100644 --- a/src/backend/distributed/executor/partitioned_intermediate_results.c +++ b/src/backend/distributed/executor/partitioned_intermediate_results.c @@ -266,8 +266,8 @@ StartPortalForQueryExecution(const char *queryString) /* don't display the portal in pg_cursors, it is for internal use only */ portal->visible = false; - PortalDefineQuery(portal, NULL, queryString, CMDTAG_SELECT_COMPAT, list_make1( - queryPlan), NULL); + PortalDefineQuery(portal, NULL, queryString, CMDTAG_SELECT_COMPAT, + list_make1(queryPlan), NULL); int eflags = 0; PortalStart(portal, NULL, eflags, GetActiveSnapshot()); diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index d140a66b7..69034c110 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -36,8 +36,8 @@ #define standard_planner_compat(a, c, d) standard_planner(a, NULL, c, d) #define getOwnedSequencesCompat(a, b) getOwnedSequences(a) #define CMDTAG_SELECT_COMPAT CMDTAG_SELECT -#define ExplainOnePlanCompat(a, b, c, d, e, f, g, h) ExplainOnePlan(a, b, c, d, e, f, g, \ - h) +#define ExplainOnePlanCompat(a, b, c, d, e, f, g, h) \ + ExplainOnePlan(a, b, c, d, e, f, g, h) #define SetListCellPtr(a, b) ((a)->ptr_value = (b)) #define RangeTableEntryFromNSItem(a) ((a)->p_rte) #define QueryCompletionCompat QueryCompletion diff --git a/src/test/regress/bin/normalize.sed b/src/test/regress/bin/normalize.sed index c982066cf..4579ab4f7 100644 --- a/src/test/regress/bin/normalize.sed +++ b/src/test/regress/bin/normalize.sed @@ -100,6 +100,8 @@ s/partition ".*" would be violated by some row/partition would be violated by so /.*Peak Memory Usage:.*$/d s/of relation ".*" contains null values/contains null values/g s/of relation "t1" is violated by some row/is violated by some row/g +# can be removed when we remove PG_VERSION_NUM >= 120000 +s/(.*)Output:.*$/\1Output: xxxxxx/g # intermediate_results diff --git a/src/test/regress/expected/multi_data_types.out b/src/test/regress/expected/multi_data_types.out index db1cfedcc..ab75277e3 100644 --- a/src/test/regress/expected/multi_data_types.out +++ b/src/test/regress/expected/multi_data_types.out @@ -174,7 +174,7 @@ INSERT INTO composite_type_partitioned_table VALUES (123, '(123, 456)'::other_co Node: host=localhost port=xxxxx dbname=regression -> Insert on public.composite_type_partitioned_table_530003 (actual rows=0 loops=1) -> Result (actual rows=1 loops=1) - Output: 123, '(123,456)'::test_composite_type + Output: xxxxxx (9 rows) SELECT run_command_on_coordinator_and_workers($cf$ @@ -218,7 +218,7 @@ INSERT INTO composite_type_partitioned_table VALUES (123, '(456, 678)'::other_co Node: host=localhost port=xxxxx dbname=regression -> Insert on public.composite_type_partitioned_table_530000 (actual rows=0 loops=1) -> Result (actual rows=1 loops=1) - Output: 123, '(456,678)'::test_composite_type + Output: xxxxxx (9 rows) -- create and distribute a table on enum type column diff --git a/src/test/regress/expected/multi_explain.out b/src/test/regress/expected/multi_explain.out index 084a2867a..85fc6dbb7 100644 --- a/src/test/regress/expected/multi_explain.out +++ b/src/test/regress/expected/multi_explain.out @@ -345,14 +345,14 @@ EXPLAIN (COSTS FALSE, ANALYZE TRUE, TIMING FALSE, SUMMARY FALSE, VERBOSE TRUE) SELECT l_quantity, count(*) count_quantity FROM lineitem GROUP BY l_quantity ORDER BY count_quantity, l_quantity; Sort (actual rows=50 loops=1) - Output: remote_scan.l_quantity, (COALESCE((pg_catalog.sum(remote_scan.count_quantity))::bigint, '0'::bigint)) + Output: xxxxxx Sort Key: (COALESCE((pg_catalog.sum(remote_scan.count_quantity))::bigint, '0'::bigint)), remote_scan.l_quantity Sort Method: quicksort Memory: 27kB -> HashAggregate (actual rows=50 loops=1) - Output: remote_scan.l_quantity, COALESCE((pg_catalog.sum(remote_scan.count_quantity))::bigint, '0'::bigint) + Output: xxxxxx Group Key: remote_scan.l_quantity -> Custom Scan (Citus Adaptive) (actual rows=100 loops=1) - Output: remote_scan.l_quantity, remote_scan.count_quantity + Output: xxxxxx Task Count: 2 Tuple data received from nodes: 780 bytes Tasks Shown: One of 2 @@ -361,48 +361,48 @@ Sort (actual rows=50 loops=1) Tuple data received from node: 390 bytes Node: host=localhost port=xxxxx dbname=regression -> HashAggregate (actual rows=50 loops=1) - Output: l_quantity, count(*) + Output: xxxxxx Group Key: lineitem.l_quantity -> Seq Scan on public.lineitem_290000 lineitem (actual rows=6000 loops=1) - Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment + Output: xxxxxx -- Test query text output, with ANALYZE OFF EXPLAIN (COSTS FALSE, ANALYZE FALSE, TIMING FALSE, SUMMARY FALSE, VERBOSE TRUE) SELECT l_quantity, count(*) count_quantity FROM lineitem GROUP BY l_quantity ORDER BY count_quantity, l_quantity; Sort - Output: remote_scan.l_quantity, (COALESCE((pg_catalog.sum(remote_scan.count_quantity))::bigint, '0'::bigint)) + Output: xxxxxx Sort Key: (COALESCE((pg_catalog.sum(remote_scan.count_quantity))::bigint, '0'::bigint)), remote_scan.l_quantity -> HashAggregate - Output: remote_scan.l_quantity, COALESCE((pg_catalog.sum(remote_scan.count_quantity))::bigint, '0'::bigint) + Output: xxxxxx Group Key: remote_scan.l_quantity -> Custom Scan (Citus Adaptive) - Output: remote_scan.l_quantity, remote_scan.count_quantity + Output: xxxxxx Task Count: 2 Tasks Shown: One of 2 -> Task Query: SELECT l_quantity, count(*) AS count_quantity FROM lineitem_290000 lineitem WHERE true GROUP BY l_quantity Node: host=localhost port=xxxxx dbname=regression -> HashAggregate - Output: l_quantity, count(*) + Output: xxxxxx Group Key: lineitem.l_quantity -> Seq Scan on public.lineitem_290000 lineitem - Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment + Output: xxxxxx -- Test verbose EXPLAIN (COSTS FALSE, VERBOSE TRUE) SELECT sum(l_quantity) / avg(l_quantity) FROM lineitem; Aggregate - Output: (sum(remote_scan."?column?") / (sum(remote_scan."?column?_1") / pg_catalog.sum(remote_scan."?column?_2"))) + Output: xxxxxx -> Custom Scan (Citus Adaptive) - Output: remote_scan."?column?", remote_scan."?column?_1", remote_scan."?column?_2" + Output: xxxxxx Task Count: 2 Tasks Shown: One of 2 -> Task Query: SELECT sum(l_quantity), sum(l_quantity), count(l_quantity) FROM lineitem_290000 lineitem WHERE true Node: host=localhost port=xxxxx dbname=regression -> Aggregate - Output: sum(l_quantity), sum(l_quantity), count(l_quantity) + Output: xxxxxx -> Seq Scan on public.lineitem_290000 lineitem - Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment + Output: xxxxxx -- Test join EXPLAIN (COSTS FALSE) SELECT * FROM lineitem @@ -525,40 +525,40 @@ EXPLAIN (COSTS FALSE, VERBOSE TRUE) SELECT sum(l_quantity) / avg(l_quantity) FROM lineitem HAVING sum(l_quantity) > 100; Aggregate - Output: (sum(remote_scan."?column?") / (sum(remote_scan."?column?_1") / pg_catalog.sum(remote_scan."?column?_2"))) + Output: xxxxxx Filter: (sum(remote_scan.worker_column_4) > '100'::numeric) -> Custom Scan (Citus Adaptive) - Output: remote_scan."?column?", remote_scan."?column?_1", remote_scan."?column?_2", remote_scan.worker_column_4 + Output: xxxxxx Task Count: 2 Tasks Shown: One of 2 -> Task Query: SELECT sum(l_quantity), sum(l_quantity), count(l_quantity), sum(l_quantity) AS worker_column_4 FROM lineitem_290000 lineitem WHERE true Node: host=localhost port=xxxxx dbname=regression -> Aggregate - Output: sum(l_quantity), sum(l_quantity), count(l_quantity), sum(l_quantity) + Output: xxxxxx -> Seq Scan on public.lineitem_290000 lineitem - Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment + Output: xxxxxx -- Test having without aggregate EXPLAIN (COSTS FALSE, VERBOSE TRUE) SELECT l_quantity FROM lineitem GROUP BY l_quantity HAVING l_quantity > (100 * random()); HashAggregate - Output: remote_scan.l_quantity + Output: xxxxxx Group Key: remote_scan.l_quantity Filter: ((remote_scan.worker_column_2)::double precision > ('100'::double precision * random())) -> Custom Scan (Citus Adaptive) - Output: remote_scan.l_quantity, remote_scan.worker_column_2 + Output: xxxxxx Task Count: 2 Tasks Shown: One of 2 -> Task Query: SELECT l_quantity, l_quantity AS worker_column_2 FROM lineitem_290000 lineitem WHERE true GROUP BY l_quantity Node: host=localhost port=xxxxx dbname=regression -> HashAggregate - Output: l_quantity, l_quantity + Output: xxxxxx Group Key: lineitem.l_quantity -> Seq Scan on public.lineitem_290000 lineitem - Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment + Output: xxxxxx -- Subquery pushdown tests with explain EXPLAIN (COSTS OFF) SELECT @@ -1395,26 +1395,26 @@ series AS ( SELECT l_orderkey FROM series JOIN keys ON (s = l_orderkey) ORDER BY s; Custom Scan (Citus Adaptive) - Output: remote_scan.l_orderkey + Output: xxxxxx -> Distributed Subplan XXX_1 -> HashAggregate - Output: remote_scan.l_orderkey + Output: xxxxxx Group Key: remote_scan.l_orderkey -> Custom Scan (Citus Adaptive) - Output: remote_scan.l_orderkey + Output: xxxxxx Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT DISTINCT l_orderkey FROM lineitem_hash_part_360041 lineitem_hash_part WHERE true Node: host=localhost port=xxxxx dbname=regression -> HashAggregate - Output: l_orderkey + Output: xxxxxx Group Key: lineitem_hash_part.l_orderkey -> Seq Scan on public.lineitem_hash_part_360041 lineitem_hash_part - Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment + Output: xxxxxx -> Distributed Subplan XXX_2 -> Function Scan on pg_catalog.generate_series s - Output: s + Output: xxxxxx Function Call: generate_series(1, 10) Task Count: 1 Tasks Shown: All @@ -1422,19 +1422,19 @@ Custom Scan (Citus Adaptive) Query: SELECT keys.l_orderkey FROM ((SELECT intermediate_result.s FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(s integer)) series JOIN (SELECT intermediate_result.l_orderkey FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(l_orderkey bigint)) keys ON ((series.s OPERATOR(pg_catalog.=) keys.l_orderkey))) ORDER BY series.s Node: host=localhost port=xxxxx dbname=regression -> Merge Join - Output: intermediate_result_1.l_orderkey, intermediate_result.s + Output: xxxxxx Merge Cond: (intermediate_result.s = intermediate_result_1.l_orderkey) -> Sort - Output: intermediate_result.s + Output: xxxxxx Sort Key: intermediate_result.s -> Function Scan on pg_catalog.read_intermediate_result intermediate_result - Output: intermediate_result.s + Output: xxxxxx Function Call: read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) -> Sort - Output: intermediate_result_1.l_orderkey + Output: xxxxxx Sort Key: intermediate_result_1.l_orderkey -> Function Scan on pg_catalog.read_intermediate_result intermediate_result_1 - Output: intermediate_result_1.l_orderkey + Output: xxxxxx Function Call: read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) SET citus.enable_cte_inlining TO true; SELECT true AS valid FROM explain_json($$ @@ -1755,7 +1755,7 @@ SELECT * FROM worker_save_query_explain_analyze('SELECT * FROM explain_analyze_t 4 (4 rows) -SELECT explain_analyze_output ~ 'Output: a, b' FROM worker_last_saved_explain_analyze(); +SELECT explain_analyze_output ~ 'Output: xxxxxx ?column? --------------------------------------------------------------------- t diff --git a/src/test/regress/expected/multi_mx_explain.out b/src/test/regress/expected/multi_mx_explain.out index c5495a839..e0ffa7618 100644 --- a/src/test/regress/expected/multi_mx_explain.out +++ b/src/test/regress/expected/multi_mx_explain.out @@ -285,18 +285,18 @@ Sort EXPLAIN (COSTS FALSE, VERBOSE TRUE) SELECT sum(l_quantity) / avg(l_quantity) FROM lineitem_mx; Aggregate - Output: (sum(remote_scan."?column?") / (sum(remote_scan."?column?_1") / pg_catalog.sum(remote_scan."?column?_2"))) + Output: xxxxxx -> Custom Scan (Citus Adaptive) - Output: remote_scan."?column?", remote_scan."?column?_1", remote_scan."?column?_2" + Output: xxxxxx Task Count: 16 Tasks Shown: One of 16 -> Task Query: SELECT sum(l_quantity), sum(l_quantity), count(l_quantity) FROM lineitem_mx_1220052 lineitem_mx WHERE true Node: host=localhost port=xxxxx dbname=regression -> Aggregate - Output: sum(l_quantity), sum(l_quantity), count(l_quantity) + Output: xxxxxx -> Seq Scan on public.lineitem_mx_1220052 lineitem_mx - Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment + Output: xxxxxx -- Test join EXPLAIN (COSTS FALSE) SELECT * FROM lineitem_mx diff --git a/src/test/regress/expected/multi_subquery_window_functions.out b/src/test/regress/expected/multi_subquery_window_functions.out index 97a481df0..eab938e6c 100644 --- a/src/test/regress/expected/multi_subquery_window_functions.out +++ b/src/test/regress/expected/multi_subquery_window_functions.out @@ -712,68 +712,68 @@ EXPLAIN (COSTS FALSE, VERBOSE TRUE) QUERY PLAN --------------------------------------------------------------------- Limit - Output: remote_scan.user_id, remote_scan.sum + Output: xxxxxx -> Sort - Output: remote_scan.user_id, remote_scan.sum + Output: xxxxxx Sort Key: remote_scan.sum DESC, remote_scan.user_id DESC -> Custom Scan (Citus Adaptive) - Output: remote_scan.user_id, remote_scan.sum + Output: xxxxxx Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT worker_column_1 AS user_id, worker_column_2 AS sum FROM (SELECT ftop.user_id AS worker_column_1, ftop.sum AS worker_column_2 FROM (SELECT user_id_1.user_id, sum(user_id_1.counter) AS sum FROM (SELECT users_table.user_id, sum(users_table.value_2) OVER (PARTITION BY users_table.user_id) AS counter FROM public.users_table_1400256 users_table UNION SELECT events_table.user_id, sum(events_table.value_2) OVER (PARTITION BY events_table.user_id) AS counter FROM public.events_table_1400260 events_table) user_id_1 GROUP BY user_id_1.user_id UNION SELECT user_id_2.user_id, sum(user_id_2.counter) AS sum FROM (SELECT users_table.user_id, sum(users_table.value_2) OVER (PARTITION BY users_table.user_id) AS counter FROM public.users_table_1400256 users_table UNION SELECT events_table.user_id, sum(events_table.value_2) OVER (PARTITION BY events_table.user_id) AS counter FROM public.events_table_1400260 events_table) user_id_2 GROUP BY user_id_2.user_id) ftop) worker_subquery ORDER BY worker_column_2 DESC, worker_column_1 DESC LIMIT '5'::bigint Node: host=localhost port=xxxxx dbname=regression -> Limit - Output: users_table.user_id, (sum((sum(users_table.value_2) OVER (?)))) + Output: xxxxxx -> Sort - Output: users_table.user_id, (sum((sum(users_table.value_2) OVER (?)))) + Output: xxxxxx Sort Key: (sum((sum(users_table.value_2) OVER (?)))) DESC, users_table.user_id DESC -> HashAggregate - Output: users_table.user_id, (sum((sum(users_table.value_2) OVER (?)))) + Output: xxxxxx Group Key: users_table.user_id, (sum((sum(users_table.value_2) OVER (?)))) -> Append -> HashAggregate - Output: users_table.user_id, sum((sum(users_table.value_2) OVER (?))) + Output: xxxxxx Group Key: users_table.user_id -> HashAggregate - Output: users_table.user_id, (sum(users_table.value_2) OVER (?)) + Output: xxxxxx Group Key: users_table.user_id, (sum(users_table.value_2) OVER (?)) -> Append -> WindowAgg - Output: users_table.user_id, sum(users_table.value_2) OVER (?) + Output: xxxxxx -> Sort - Output: users_table.user_id, users_table.value_2 + Output: xxxxxx Sort Key: users_table.user_id -> Seq Scan on public.users_table_1400256 users_table - Output: users_table.user_id, users_table.value_2 + Output: xxxxxx -> WindowAgg - Output: events_table.user_id, sum(events_table.value_2) OVER (?) + Output: xxxxxx -> Sort - Output: events_table.user_id, events_table.value_2 + Output: xxxxxx Sort Key: events_table.user_id -> Seq Scan on public.events_table_1400260 events_table - Output: events_table.user_id, events_table.value_2 + Output: xxxxxx -> HashAggregate - Output: users_table_1.user_id, sum((sum(users_table_1.value_2) OVER (?))) + Output: xxxxxx Group Key: users_table_1.user_id -> HashAggregate - Output: users_table_1.user_id, (sum(users_table_1.value_2) OVER (?)) + Output: xxxxxx Group Key: users_table_1.user_id, (sum(users_table_1.value_2) OVER (?)) -> Append -> WindowAgg - Output: users_table_1.user_id, sum(users_table_1.value_2) OVER (?) + Output: xxxxxx -> Sort - Output: users_table_1.user_id, users_table_1.value_2 + Output: xxxxxx Sort Key: users_table_1.user_id -> Seq Scan on public.users_table_1400256 users_table_1 - Output: users_table_1.user_id, users_table_1.value_2 + Output: xxxxxx -> WindowAgg - Output: events_table_1.user_id, sum(events_table_1.value_2) OVER (?) + Output: xxxxxx -> Sort - Output: events_table_1.user_id, events_table_1.value_2 + Output: xxxxxx Sort Key: events_table_1.user_id -> Seq Scan on public.events_table_1400260 events_table_1 - Output: events_table_1.user_id, events_table_1.value_2 + Output: xxxxxx (63 rows) -- test with window functions which aren't pushed down diff --git a/src/test/regress/expected/subquery_complex_target_list.out b/src/test/regress/expected/subquery_complex_target_list.out index f1009d06d..e8296f588 100644 --- a/src/test/regress/expected/subquery_complex_target_list.out +++ b/src/test/regress/expected/subquery_complex_target_list.out @@ -554,14 +554,14 @@ $$); coordinator_plan --------------------------------------------------------------------- Sort - Output: remote_scan.k1, remote_scan.k2, remote_scan.k3, (any_value(remote_scan.v1)), (any_value(remote_scan.v2)), ((any_value(remote_scan.v3) || '_notgrouped'::text)), remote_scan.va1, remote_scan.va2, remote_scan.va3, (COALESCE((pg_catalog.sum(remote_scan.count))::bigint, '0'::bigint)) + Output: xxxxxx Sort Key: remote_scan.k1 -> HashAggregate - Output: remote_scan.k1, remote_scan.k2, remote_scan.k3, any_value(remote_scan.v1), any_value(remote_scan.v2), (any_value(remote_scan.v3) || '_notgrouped'::text), remote_scan.va1, remote_scan.va2, remote_scan.va3, COALESCE((pg_catalog.sum(remote_scan.count))::bigint, '0'::bigint) + Output: xxxxxx Group Key: remote_scan.k1, remote_scan.va1 Filter: ((length(remote_scan.worker_column_11) + length(any_value(remote_scan.worker_column_12))) < length((any_value(remote_scan.worker_column_13) || '_append'::text))) -> Custom Scan (Citus Adaptive) - Output: remote_scan.k1, remote_scan.k2, remote_scan.k3, remote_scan.v1, remote_scan.v2, remote_scan.v3, remote_scan.va1, remote_scan.va2, remote_scan.va3, remote_scan.count, remote_scan.worker_column_11, remote_scan.worker_column_12, remote_scan.worker_column_13 + Output: xxxxxx Task Count: 4 (10 rows) diff --git a/src/test/regress/expected/tdigest_aggregate_support.out b/src/test/regress/expected/tdigest_aggregate_support.out index a575dcb78..7ad937ce5 100644 --- a/src/test/regress/expected/tdigest_aggregate_support.out +++ b/src/test/regress/expected/tdigest_aggregate_support.out @@ -42,18 +42,18 @@ FROM latencies; QUERY PLAN --------------------------------------------------------------------- Aggregate - Output: tdigest(remote_scan.tdigest) + Output: xxxxxx -> Custom Scan (Citus Adaptive) - Output: remote_scan.tdigest + Output: xxxxxx Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT public.tdigest(latency, 100) AS tdigest FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true Node: host=localhost port=xxxxx dbname=regression -> Aggregate - Output: tdigest(latency, 100) + Output: xxxxxx -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies - Output: a, b, latency + Output: xxxxxx (13 rows) -- explain grouping by distribution column is completely pushed down for tdigest(value, compression) @@ -64,17 +64,17 @@ GROUP BY a; QUERY PLAN --------------------------------------------------------------------- Custom Scan (Citus Adaptive) - Output: remote_scan.a, remote_scan.tdigest + Output: xxxxxx Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT a, public.tdigest(latency, 100) AS tdigest FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true GROUP BY a Node: host=localhost port=xxxxx dbname=regression -> HashAggregate - Output: a, tdigest(latency, 100) + Output: xxxxxx Group Key: latencies.a -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies - Output: a, b, latency + Output: xxxxxx (12 rows) -- explain grouping by non-distribution column is partially pushed down for tdigest(value, compression) @@ -85,20 +85,20 @@ GROUP BY b; QUERY PLAN --------------------------------------------------------------------- HashAggregate - Output: remote_scan.b, tdigest(remote_scan.tdigest) + Output: xxxxxx Group Key: remote_scan.b -> Custom Scan (Citus Adaptive) - Output: remote_scan.b, remote_scan.tdigest + Output: xxxxxx Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT b, public.tdigest(latency, 100) AS tdigest FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true GROUP BY b Node: host=localhost port=xxxxx dbname=regression -> HashAggregate - Output: b, tdigest(latency, 100) + Output: xxxxxx Group Key: latencies.b -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies - Output: a, b, latency + Output: xxxxxx (15 rows) -- explain no grouping to verify partially pushed down for tdigest_precentile(value, compression, quantile) @@ -108,18 +108,18 @@ FROM latencies; QUERY PLAN --------------------------------------------------------------------- Aggregate - Output: tdigest_percentile(remote_scan.tdigest_percentile, '0.99'::double precision) + Output: xxxxxx -> Custom Scan (Citus Adaptive) - Output: remote_scan.tdigest_percentile + Output: xxxxxx Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT public.tdigest(latency, 100) AS tdigest_percentile FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true Node: host=localhost port=xxxxx dbname=regression -> Aggregate - Output: tdigest(latency, 100) + Output: xxxxxx -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies - Output: a, b, latency + Output: xxxxxx (13 rows) -- explain grouping by distribution column is completely pushed down for tdigest_precentile(value, compression, quantile) @@ -130,17 +130,17 @@ GROUP BY a; QUERY PLAN --------------------------------------------------------------------- Custom Scan (Citus Adaptive) - Output: remote_scan.a, remote_scan.tdigest_percentile + Output: xxxxxx Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT a, public.tdigest_percentile(latency, 100, '0.99'::double precision) AS tdigest_percentile FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true GROUP BY a Node: host=localhost port=xxxxx dbname=regression -> HashAggregate - Output: a, tdigest_percentile(latency, 100, '0.99'::double precision) + Output: xxxxxx Group Key: latencies.a -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies - Output: a, b, latency + Output: xxxxxx (12 rows) -- explain grouping by non-distribution column is partially pushed down for tdigest_precentile(value, compression, quantile) @@ -151,20 +151,20 @@ GROUP BY b; QUERY PLAN --------------------------------------------------------------------- HashAggregate - Output: remote_scan.b, tdigest_percentile(remote_scan.tdigest_percentile, '0.99'::double precision) + Output: xxxxxx Group Key: remote_scan.b -> Custom Scan (Citus Adaptive) - Output: remote_scan.b, remote_scan.tdigest_percentile + Output: xxxxxx Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT b, public.tdigest(latency, 100) AS tdigest_percentile FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true GROUP BY b Node: host=localhost port=xxxxx dbname=regression -> HashAggregate - Output: b, tdigest(latency, 100) + Output: xxxxxx Group Key: latencies.b -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies - Output: a, b, latency + Output: xxxxxx (15 rows) -- explain no grouping to verify partially pushed down for tdigest_precentile(value, compression, quantiles[]) @@ -174,18 +174,18 @@ FROM latencies; QUERY PLAN --------------------------------------------------------------------- Aggregate - Output: tdigest_percentile(remote_scan.tdigest_percentile, '{0.99,0.95}'::double precision[]) + Output: xxxxxx -> Custom Scan (Citus Adaptive) - Output: remote_scan.tdigest_percentile + Output: xxxxxx Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT public.tdigest(latency, 100) AS tdigest_percentile FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true Node: host=localhost port=xxxxx dbname=regression -> Aggregate - Output: tdigest(latency, 100) + Output: xxxxxx -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies - Output: a, b, latency + Output: xxxxxx (13 rows) -- explain grouping by distribution column is completely pushed down for tdigest_precentile(value, compression, quantiles[]) @@ -196,17 +196,17 @@ GROUP BY a; QUERY PLAN --------------------------------------------------------------------- Custom Scan (Citus Adaptive) - Output: remote_scan.a, remote_scan.tdigest_percentile + Output: xxxxxx Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT a, public.tdigest_percentile(latency, 100, '{0.99,0.95}'::double precision[]) AS tdigest_percentile FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true GROUP BY a Node: host=localhost port=xxxxx dbname=regression -> HashAggregate - Output: a, tdigest_percentile(latency, 100, '{0.99,0.95}'::double precision[]) + Output: xxxxxx Group Key: latencies.a -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies - Output: a, b, latency + Output: xxxxxx (12 rows) -- explain grouping by non-distribution column is partially pushed down for tdigest_precentile(value, compression, quantiles[]) @@ -217,20 +217,20 @@ GROUP BY b; QUERY PLAN --------------------------------------------------------------------- HashAggregate - Output: remote_scan.b, tdigest_percentile(remote_scan.tdigest_percentile, '{0.99,0.95}'::double precision[]) + Output: xxxxxx Group Key: remote_scan.b -> Custom Scan (Citus Adaptive) - Output: remote_scan.b, remote_scan.tdigest_percentile + Output: xxxxxx Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT b, public.tdigest(latency, 100) AS tdigest_percentile FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true GROUP BY b Node: host=localhost port=xxxxx dbname=regression -> HashAggregate - Output: b, tdigest(latency, 100) + Output: xxxxxx Group Key: latencies.b -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies - Output: a, b, latency + Output: xxxxxx (15 rows) -- explain no grouping to verify partially pushed down for tdigest_precentile_of(value, compression, hypotetical_value) @@ -240,18 +240,18 @@ FROM latencies; QUERY PLAN --------------------------------------------------------------------- Aggregate - Output: tdigest_percentile_of(remote_scan.tdigest_percentile_of, '9000'::double precision) + Output: xxxxxx -> Custom Scan (Citus Adaptive) - Output: remote_scan.tdigest_percentile_of + Output: xxxxxx Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT public.tdigest(latency, 100) AS tdigest_percentile_of FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true Node: host=localhost port=xxxxx dbname=regression -> Aggregate - Output: tdigest(latency, 100) + Output: xxxxxx -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies - Output: a, b, latency + Output: xxxxxx (13 rows) -- explain grouping by distribution column is completely pushed down for tdigest_precentile_of(value, compression, hypotetical_value) @@ -262,17 +262,17 @@ GROUP BY a; QUERY PLAN --------------------------------------------------------------------- Custom Scan (Citus Adaptive) - Output: remote_scan.a, remote_scan.tdigest_percentile_of + Output: xxxxxx Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT a, public.tdigest_percentile_of(latency, 100, '9000'::double precision) AS tdigest_percentile_of FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true GROUP BY a Node: host=localhost port=xxxxx dbname=regression -> HashAggregate - Output: a, tdigest_percentile_of(latency, 100, '9000'::double precision) + Output: xxxxxx Group Key: latencies.a -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies - Output: a, b, latency + Output: xxxxxx (12 rows) -- explain grouping by non-distribution column is partially pushed down for tdigest_precentile_of(value, compression, hypotetical_value) @@ -283,20 +283,20 @@ GROUP BY b; QUERY PLAN --------------------------------------------------------------------- HashAggregate - Output: remote_scan.b, tdigest_percentile_of(remote_scan.tdigest_percentile_of, '9000'::double precision) + Output: xxxxxx Group Key: remote_scan.b -> Custom Scan (Citus Adaptive) - Output: remote_scan.b, remote_scan.tdigest_percentile_of + Output: xxxxxx Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT b, public.tdigest(latency, 100) AS tdigest_percentile_of FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true GROUP BY b Node: host=localhost port=xxxxx dbname=regression -> HashAggregate - Output: b, tdigest(latency, 100) + Output: xxxxxx Group Key: latencies.b -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies - Output: a, b, latency + Output: xxxxxx (15 rows) -- explain no grouping to verify partially pushed down for tdigest_precentile_of(value, compression, hypotetical_values[]) @@ -306,18 +306,18 @@ FROM latencies; QUERY PLAN --------------------------------------------------------------------- Aggregate - Output: tdigest_percentile_of(remote_scan.tdigest_percentile_of, '{9000,9500}'::double precision[]) + Output: xxxxxx -> Custom Scan (Citus Adaptive) - Output: remote_scan.tdigest_percentile_of + Output: xxxxxx Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT public.tdigest(latency, 100) AS tdigest_percentile_of FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true Node: host=localhost port=xxxxx dbname=regression -> Aggregate - Output: tdigest(latency, 100) + Output: xxxxxx -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies - Output: a, b, latency + Output: xxxxxx (13 rows) -- explain grouping by distribution column is completely pushed down for tdigest_precentile_of(value, compression, hypotetical_values[]) @@ -328,17 +328,17 @@ GROUP BY a; QUERY PLAN --------------------------------------------------------------------- Custom Scan (Citus Adaptive) - Output: remote_scan.a, remote_scan.tdigest_percentile_of + Output: xxxxxx Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT a, public.tdigest_percentile_of(latency, 100, '{9000,9500}'::double precision[]) AS tdigest_percentile_of FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true GROUP BY a Node: host=localhost port=xxxxx dbname=regression -> HashAggregate - Output: a, tdigest_percentile_of(latency, 100, '{9000,9500}'::double precision[]) + Output: xxxxxx Group Key: latencies.a -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies - Output: a, b, latency + Output: xxxxxx (12 rows) -- explain grouping by non-distribution column is partially pushed down for tdigest_precentile_of(value, compression, hypotetical_values[]) @@ -349,20 +349,20 @@ GROUP BY b; QUERY PLAN --------------------------------------------------------------------- HashAggregate - Output: remote_scan.b, tdigest_percentile_of(remote_scan.tdigest_percentile_of, '{9000,9500}'::double precision[]) + Output: xxxxxx Group Key: remote_scan.b -> Custom Scan (Citus Adaptive) - Output: remote_scan.b, remote_scan.tdigest_percentile_of + Output: xxxxxx Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT b, public.tdigest(latency, 100) AS tdigest_percentile_of FROM tdigest_aggregate_support.latencies_20070000 latencies WHERE true GROUP BY b Node: host=localhost port=xxxxx dbname=regression -> HashAggregate - Output: b, tdigest(latency, 100) + Output: xxxxxx Group Key: latencies.b -> Seq Scan on tdigest_aggregate_support.latencies_20070000 latencies - Output: a, b, latency + Output: xxxxxx (15 rows) -- verifying results - should be stable due to seed while inserting the data, if failure due to data these queries could be removed or check for certain ranges @@ -413,18 +413,18 @@ FROM latencies_rollup; QUERY PLAN --------------------------------------------------------------------- Aggregate - Output: tdigest(remote_scan.tdigest) + Output: xxxxxx -> Custom Scan (Citus Adaptive) - Output: remote_scan.tdigest + Output: xxxxxx Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT public.tdigest(tdigest) AS tdigest FROM tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup WHERE true Node: host=localhost port=xxxxx dbname=regression -> Aggregate - Output: tdigest(tdigest) + Output: xxxxxx -> Seq Scan on tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup - Output: a, tdigest + Output: xxxxxx (13 rows) -- explain grouping by distribution column is completely pushed down for tdigest(tdigest) @@ -435,17 +435,17 @@ GROUP BY a; QUERY PLAN --------------------------------------------------------------------- Custom Scan (Citus Adaptive) - Output: remote_scan.a, remote_scan.tdigest + Output: xxxxxx Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT a, public.tdigest(tdigest) AS tdigest FROM tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup WHERE true GROUP BY a Node: host=localhost port=xxxxx dbname=regression -> HashAggregate - Output: a, tdigest(tdigest) + Output: xxxxxx Group Key: latencies_rollup.a -> Seq Scan on tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup - Output: a, tdigest + Output: xxxxxx (12 rows) -- explain no grouping to verify partially pushed down for tdigest_precentile(tdigest, quantile) @@ -455,18 +455,18 @@ FROM latencies_rollup; QUERY PLAN --------------------------------------------------------------------- Aggregate - Output: tdigest_percentile(remote_scan.tdigest_percentile, '0.99'::double precision) + Output: xxxxxx -> Custom Scan (Citus Adaptive) - Output: remote_scan.tdigest_percentile + Output: xxxxxx Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT public.tdigest(tdigest) AS tdigest_percentile FROM tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup WHERE true Node: host=localhost port=xxxxx dbname=regression -> Aggregate - Output: tdigest(tdigest) + Output: xxxxxx -> Seq Scan on tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup - Output: a, tdigest + Output: xxxxxx (13 rows) -- explain grouping by distribution column is completely pushed down for tdigest_precentile(tdigest, quantile) @@ -477,17 +477,17 @@ GROUP BY a; QUERY PLAN --------------------------------------------------------------------- Custom Scan (Citus Adaptive) - Output: remote_scan.a, remote_scan.tdigest_percentile + Output: xxxxxx Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT a, public.tdigest_percentile(tdigest, '0.99'::double precision) AS tdigest_percentile FROM tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup WHERE true GROUP BY a Node: host=localhost port=xxxxx dbname=regression -> HashAggregate - Output: a, tdigest_percentile(tdigest, '0.99'::double precision) + Output: xxxxxx Group Key: latencies_rollup.a -> Seq Scan on tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup - Output: a, tdigest + Output: xxxxxx (12 rows) -- explain no grouping to verify partially pushed down for tdigest_precentile(value, compression, quantiles[]) @@ -497,18 +497,18 @@ FROM latencies_rollup; QUERY PLAN --------------------------------------------------------------------- Aggregate - Output: tdigest_percentile(remote_scan.tdigest_percentile, '{0.99,0.95}'::double precision[]) + Output: xxxxxx -> Custom Scan (Citus Adaptive) - Output: remote_scan.tdigest_percentile + Output: xxxxxx Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT public.tdigest(tdigest) AS tdigest_percentile FROM tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup WHERE true Node: host=localhost port=xxxxx dbname=regression -> Aggregate - Output: tdigest(tdigest) + Output: xxxxxx -> Seq Scan on tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup - Output: a, tdigest + Output: xxxxxx (13 rows) -- explain grouping by distribution column is completely pushed down for tdigest_precentile(value, compression, quantiles[]) @@ -519,17 +519,17 @@ GROUP BY a; QUERY PLAN --------------------------------------------------------------------- Custom Scan (Citus Adaptive) - Output: remote_scan.a, remote_scan.tdigest_percentile + Output: xxxxxx Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT a, public.tdigest_percentile(tdigest, '{0.99,0.95}'::double precision[]) AS tdigest_percentile FROM tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup WHERE true GROUP BY a Node: host=localhost port=xxxxx dbname=regression -> HashAggregate - Output: a, tdigest_percentile(tdigest, '{0.99,0.95}'::double precision[]) + Output: xxxxxx Group Key: latencies_rollup.a -> Seq Scan on tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup - Output: a, tdigest + Output: xxxxxx (12 rows) -- explain no grouping to verify partially pushed down for tdigest_precentile_of(value, compression, hypotetical_value) @@ -539,18 +539,18 @@ FROM latencies_rollup; QUERY PLAN --------------------------------------------------------------------- Aggregate - Output: tdigest_percentile_of(remote_scan.tdigest_percentile_of, '9000'::double precision) + Output: xxxxxx -> Custom Scan (Citus Adaptive) - Output: remote_scan.tdigest_percentile_of + Output: xxxxxx Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT public.tdigest(tdigest) AS tdigest_percentile_of FROM tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup WHERE true Node: host=localhost port=xxxxx dbname=regression -> Aggregate - Output: tdigest(tdigest) + Output: xxxxxx -> Seq Scan on tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup - Output: a, tdigest + Output: xxxxxx (13 rows) -- explain grouping by distribution column is completely pushed down for tdigest_precentile_of(value, compression, hypotetical_value) @@ -561,17 +561,17 @@ GROUP BY a; QUERY PLAN --------------------------------------------------------------------- Custom Scan (Citus Adaptive) - Output: remote_scan.a, remote_scan.tdigest_percentile_of + Output: xxxxxx Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT a, public.tdigest_percentile_of(tdigest, '9000'::double precision) AS tdigest_percentile_of FROM tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup WHERE true GROUP BY a Node: host=localhost port=xxxxx dbname=regression -> HashAggregate - Output: a, tdigest_percentile_of(tdigest, '9000'::double precision) + Output: xxxxxx Group Key: latencies_rollup.a -> Seq Scan on tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup - Output: a, tdigest + Output: xxxxxx (12 rows) -- explain no grouping to verify partially pushed down for tdigest_precentile_of(value, compression, hypotetical_values[]) @@ -581,18 +581,18 @@ FROM latencies_rollup; QUERY PLAN --------------------------------------------------------------------- Aggregate - Output: tdigest_percentile_of(remote_scan.tdigest_percentile_of, '{9000,9500}'::double precision[]) + Output: xxxxxx -> Custom Scan (Citus Adaptive) - Output: remote_scan.tdigest_percentile_of + Output: xxxxxx Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT public.tdigest(tdigest) AS tdigest_percentile_of FROM tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup WHERE true Node: host=localhost port=xxxxx dbname=regression -> Aggregate - Output: tdigest(tdigest) + Output: xxxxxx -> Seq Scan on tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup - Output: a, tdigest + Output: xxxxxx (13 rows) -- explain grouping by distribution column is completely pushed down for tdigest_precentile_of(value, compression, hypotetical_values[]) @@ -603,17 +603,17 @@ GROUP BY a; QUERY PLAN --------------------------------------------------------------------- Custom Scan (Citus Adaptive) - Output: remote_scan.a, remote_scan.tdigest_percentile_of + Output: xxxxxx Task Count: 4 Tasks Shown: One of 4 -> Task Query: SELECT a, public.tdigest_percentile_of(tdigest, '{9000,9500}'::double precision[]) AS tdigest_percentile_of FROM tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup WHERE true GROUP BY a Node: host=localhost port=xxxxx dbname=regression -> HashAggregate - Output: a, tdigest_percentile_of(tdigest, '{9000,9500}'::double precision[]) + Output: xxxxxx Group Key: latencies_rollup.a -> Seq Scan on tdigest_aggregate_support.latencies_rollup_20070004 latencies_rollup - Output: a, tdigest + Output: xxxxxx (12 rows) -- verifying results - should be stable due to seed while inserting the data, if failure due to data these queries could be removed or check for certain ranges diff --git a/src/test/regress/spec/isolation_cancellation.spec b/src/test/regress/spec/isolation_cancellation.spec index bcfb2fb37..e71e8bd58 100644 --- a/src/test/regress/spec/isolation_cancellation.spec +++ b/src/test/regress/spec/isolation_cancellation.spec @@ -21,11 +21,6 @@ step "s1-begin" BEGIN; } -step "s1-commit" -{ - COMMIT; -} - step "s1-rollback" { ROLLBACK; From 195c2a91e2560b4801679b3962d57df15806a56a Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Tue, 4 Aug 2020 23:07:00 +0300 Subject: [PATCH 52/52] Use citus docker repo instead of personal account --- .circleci/config.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 13e5264ce..e07584d6d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,7 +6,7 @@ orbs: jobs: build: docker: - - image: 'heisenberg302/extbuilder-13:latest' + - image: 'citus/extbuilder-13:latest' steps: - checkout - run: @@ -158,7 +158,7 @@ jobs: test-12-13_check-pg-upgrade: docker: - - image: 'heisenberg302/pgupgradetester:latest' + - image: 'citus/pgupgradetester:latest' working_directory: /home/circleci/project steps: - attach_workspace: @@ -266,7 +266,7 @@ jobs: test-13_check-multi: docker: - - image: 'heisenberg302/exttester-13:latest' + - image: 'citus/exttester-13:latest' working_directory: /home/circleci/project steps: - attach_workspace: @@ -280,7 +280,7 @@ jobs: test-13_check-van-mx: docker: - - image: 'heisenberg302/exttester-13:latest' + - image: 'citus/exttester-13:latest' working_directory: /home/circleci/project steps: - attach_workspace: @@ -293,7 +293,7 @@ jobs: flags: 'test_13,vanilla,mx' test-13_check-iso-work-fol: docker: - - image: 'heisenberg302/exttester-13:latest' + - image: 'citus/exttester-13:latest' working_directory: /home/circleci/project steps: - attach_workspace: @@ -306,7 +306,7 @@ jobs: flags: 'test_13,isolation,worker' test-13_check-fol: docker: - - image: 'heisenberg302/exttester-13:latest' + - image: 'citus/exttester-13:latest' working_directory: /home/circleci/project steps: - attach_workspace: @@ -330,7 +330,7 @@ jobs: test-13_check-failure: docker: - - image: 'heisenberg302/failtester-13:latest' + - image: 'citus/failtester-13:latest' working_directory: /home/circleci/project steps: - attach_workspace: