From 5aace9bb37b907c221f3375f8a9d3c50192f24b9 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Thu, 29 Jul 2021 16:14:05 +0300 Subject: [PATCH 001/104] Enables Postgres 14 in configure --- configure | 2 +- configure.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configure b/configure index a5aa66ee4..6c5d4f3d6 100755 --- a/configure +++ b/configure @@ -2555,7 +2555,7 @@ if test -z "$version_num"; then as_fn_error $? "Could not detect PostgreSQL version from pg_config." "$LINENO" 5 fi -if test "$version_num" != '12' -a "$version_num" != '13'; then +if test "$version_num" != '12' -a "$version_num" != '13' -a "$version_num" != '14'; then as_fn_error $? "Citus is not compatible with the detected PostgreSQL version ${version_num}." "$LINENO" 5 else { $as_echo "$as_me:${as_lineno-$LINENO}: building against PostgreSQL $version_num" >&5 diff --git a/configure.in b/configure.in index f92b5214c..2d7588fe7 100644 --- a/configure.in +++ b/configure.in @@ -74,7 +74,7 @@ if test -z "$version_num"; then AC_MSG_ERROR([Could not detect PostgreSQL version from pg_config.]) fi -if test "$version_num" != '12' -a "$version_num" != '13'; then +if test "$version_num" != '12' -a "$version_num" != '13' -a "$version_num" != '14'; then AC_MSG_ERROR([Citus is not compatible with the detected PostgreSQL version ${version_num}.]) else AC_MSG_NOTICE([building against PostgreSQL $version_num]) From 7a27d7cee3d8b290c29c66db0f8d246c23676dde Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Thu, 29 Jul 2021 17:14:14 +0300 Subject: [PATCH 002/104] Adds copy of ruleutils_13.c as ruleutils_14.c --- .gitattributes | 1 + .../distributed/deparser/ruleutils_14.c | 8562 +++++++++++++++++ .../distributed/pg_version_constants.h | 1 + 3 files changed, 8564 insertions(+) create mode 100644 src/backend/distributed/deparser/ruleutils_14.c diff --git a/.gitattributes b/.gitattributes index ac1ca0c17..454a83448 100644 --- a/.gitattributes +++ b/.gitattributes @@ -30,6 +30,7 @@ src/backend/distributed/utils/pg11_snprintf.c -citus-style src/backend/distributed/deparser/ruleutils_11.c -citus-style src/backend/distributed/deparser/ruleutils_12.c -citus-style src/backend/distributed/deparser/ruleutils_13.c -citus-style +src/backend/distributed/deparser/ruleutils_14.c -citus-style src/backend/distributed/commands/index_pg_source.c -citus-style src/include/distributed/citus_nodes.h -citus-style diff --git a/src/backend/distributed/deparser/ruleutils_14.c b/src/backend/distributed/deparser/ruleutils_14.c new file mode 100644 index 000000000..6eef32c53 --- /dev/null +++ b/src/backend/distributed/deparser/ruleutils_14.c @@ -0,0 +1,8562 @@ +/*------------------------------------------------------------------------- + * + * ruleutils_14.c + * Functions to convert stored expressions/querytrees back to + * source text + * + * Portions Copyright (c) 1996-2018, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/backend/distributed/deparser/ruleutils_14.c + * + * This needs to be closely in sync with the core code. + *------------------------------------------------------------------------- + */ +#include "distributed/pg_version_constants.h" + +#include "pg_config.h" + +#if (PG_VERSION_NUM >= PG_VERSION_14) && (PG_VERSION_NUM < PG_VERSION_15) + +#include "postgres.h" + +#include +#include +#include + +#include "access/amapi.h" +#include "access/htup_details.h" +#include "access/relation.h" +#include "access/sysattr.h" +#include "access/table.h" +#include "catalog/dependency.h" +#include "catalog/indexing.h" +#include "catalog/pg_aggregate.h" +#include "catalog/pg_am.h" +#include "catalog/pg_authid.h" +#include "catalog/pg_collation.h" +#include "catalog/pg_constraint.h" +#include "catalog/pg_depend.h" +#include "catalog/pg_extension.h" +#include "catalog/pg_foreign_data_wrapper.h" +#include "catalog/pg_language.h" +#include "catalog/pg_opclass.h" +#include "catalog/pg_operator.h" +#include "catalog/pg_partitioned_table.h" +#include "catalog/pg_proc.h" +#include "catalog/pg_statistic_ext.h" +#include "catalog/pg_trigger.h" +#include "catalog/pg_type.h" +#include "commands/defrem.h" +#include "commands/extension.h" +#include "commands/tablespace.h" +#include "common/keywords.h" +#include "distributed/citus_nodefuncs.h" +#include "distributed/citus_ruleutils.h" +#include "executor/spi.h" +#include "foreign/foreign.h" +#include "funcapi.h" +#include "mb/pg_wchar.h" +#include "miscadmin.h" +#include "nodes/makefuncs.h" +#include "nodes/nodeFuncs.h" +#include "nodes/pathnodes.h" +#include "optimizer/optimizer.h" +#include "parser/parse_node.h" +#include "parser/parse_agg.h" +#include "parser/parse_func.h" +#include "parser/parse_node.h" +#include "parser/parse_oper.h" +#include "parser/parser.h" +#include "parser/parsetree.h" +#include "rewrite/rewriteHandler.h" +#include "rewrite/rewriteManip.h" +#include "rewrite/rewriteSupport.h" +#include "utils/array.h" +#include "utils/builtins.h" +#include "utils/fmgroids.h" +#include "utils/hsearch.h" +#include "utils/lsyscache.h" +#include "utils/rel.h" +#include "utils/ruleutils.h" +#include "utils/snapmgr.h" +#include "utils/syscache.h" +#include "utils/typcache.h" +#include "utils/varlena.h" +#include "utils/xml.h" + + +/* ---------- + * Pretty formatting constants + * ---------- + */ + +/* Indent counts */ +#define PRETTYINDENT_STD 8 +#define PRETTYINDENT_JOIN 4 +#define PRETTYINDENT_VAR 4 + +#define PRETTYINDENT_LIMIT 40 /* wrap limit */ + +/* Pretty flags */ +#define PRETTYFLAG_PAREN 0x0001 +#define PRETTYFLAG_INDENT 0x0002 +#define PRETTYFLAG_SCHEMA 0x0004 + +/* Default line length for pretty-print wrapping: 0 means wrap always */ +#define WRAP_COLUMN_DEFAULT 0 + +/* macros to test if pretty action needed */ +#define PRETTY_PAREN(context) ((context)->prettyFlags & PRETTYFLAG_PAREN) +#define PRETTY_INDENT(context) ((context)->prettyFlags & PRETTYFLAG_INDENT) +#define PRETTY_SCHEMA(context) ((context)->prettyFlags & PRETTYFLAG_SCHEMA) + + +/* ---------- + * Local data types + * ---------- + */ + +/* Context info needed for invoking a recursive querytree display routine */ +typedef struct +{ + StringInfo buf; /* output buffer to append to */ + List *namespaces; /* List of deparse_namespace nodes */ + List *windowClause; /* Current query level's WINDOW clause */ + List *windowTList; /* targetlist for resolving WINDOW clause */ + int prettyFlags; /* enabling of pretty-print functions */ + int wrapColumn; /* max line length, or -1 for no limit */ + int indentLevel; /* current indent level for prettyprint */ + bool varprefix; /* true to print prefixes on Vars */ + Oid distrelid; /* the distributed table being modified, if valid */ + int64 shardid; /* a distributed table's shardid, if positive */ + ParseExprKind special_exprkind; /* set only for exprkinds needing special + * handling */ + Bitmapset *appendparents; /* if not null, map child Vars of these relids + * back to the parent rel */ +} deparse_context; + +/* + * Each level of query context around a subtree needs a level of Var namespace. + * A Var having varlevelsup=N refers to the N'th item (counting from 0) in + * the current context's namespaces list. + * + * The rangetable is the list of actual RTEs from the query tree, and the + * cte list is the list of actual CTEs. + * + * rtable_names holds the alias name to be used for each RTE (either a C + * string, or NULL for nameless RTEs such as unnamed joins). + * rtable_columns holds the column alias names to be used for each RTE. + * + * In some cases we need to make names of merged JOIN USING columns unique + * across the whole query, not only per-RTE. If so, unique_using is true + * and using_names is a list of C strings representing names already assigned + * to USING columns. + * + * When deparsing plan trees, there is always just a single item in the + * deparse_namespace list (since a plan tree never contains Vars with + * varlevelsup > 0). We store the PlanState node that is the immediate + * parent of the expression to be deparsed, as well as a list of that + * PlanState's ancestors. In addition, we store its outer and inner subplan + * state nodes, as well as their plan nodes' targetlists, and the index tlist + * if the current plan node might contain INDEX_VAR Vars. (These fields could + * be derived on-the-fly from the current PlanState, but it seems notationally + * clearer to set them up as separate fields.) + */ +typedef struct +{ + List *rtable; /* List of RangeTblEntry nodes */ + List *rtable_names; /* Parallel list of names for RTEs */ + List *rtable_columns; /* Parallel list of deparse_columns structs */ + List *subplans; /* List of Plan trees for SubPlans */ + List *ctes; /* List of CommonTableExpr nodes */ + AppendRelInfo **appendrels; /* Array of AppendRelInfo nodes, or NULL */ + /* Workspace for column alias assignment: */ + bool unique_using; /* Are we making USING names globally unique */ + List *using_names; /* List of assigned names for USING columns */ + /* Remaining fields are used only when deparsing a Plan tree: */ + Plan *plan; /* immediate parent of current expression */ + List *ancestors; /* ancestors of planstate */ + Plan *outer_plan; /* outer subnode, or NULL if none */ + Plan *inner_plan; /* inner subnode, or NULL if none */ + List *outer_tlist; /* referent for OUTER_VAR Vars */ + List *inner_tlist; /* referent for INNER_VAR Vars */ + List *index_tlist; /* referent for INDEX_VAR Vars */ +} deparse_namespace; + +/* Callback signature for resolve_special_varno() */ +typedef void (*rsv_callback) (Node *node, deparse_context *context, + void *callback_arg); + +/* + * Per-relation data about column alias names. + * + * Selecting aliases is unreasonably complicated because of the need to dump + * rules/views whose underlying tables may have had columns added, deleted, or + * renamed since the query was parsed. We must nonetheless print the rule/view + * in a form that can be reloaded and will produce the same results as before. + * + * For each RTE used in the query, we must assign column aliases that are + * unique within that RTE. SQL does not require this of the original query, + * but due to factors such as *-expansion we need to be able to uniquely + * reference every column in a decompiled query. As long as we qualify all + * column references, per-RTE uniqueness is sufficient for that. + * + * However, we can't ensure per-column name uniqueness for unnamed join RTEs, + * since they just inherit column names from their input RTEs, and we can't + * rename the columns at the join level. Most of the time this isn't an issue + * because we don't need to reference the join's output columns as such; we + * can reference the input columns instead. That approach can fail for merged + * JOIN USING columns, however, so when we have one of those in an unnamed + * join, we have to make that column's alias globally unique across the whole + * query to ensure it can be referenced unambiguously. + * + * Another problem is that a JOIN USING clause requires the columns to be + * merged to have the same aliases in both input RTEs, and that no other + * columns in those RTEs or their children conflict with the USING names. + * To handle that, we do USING-column alias assignment in a recursive + * traversal of the query's jointree. When descending through a JOIN with + * USING, we preassign the USING column names to the child columns, overriding + * other rules for column alias assignment. We also mark each RTE with a list + * of all USING column names selected for joins containing that RTE, so that + * when we assign other columns' aliases later, we can avoid conflicts. + * + * Another problem is that if a JOIN's input tables have had columns added or + * deleted since the query was parsed, we must generate a column alias list + * for the join that matches the current set of input columns --- otherwise, a + * change in the number of columns in the left input would throw off matching + * of aliases to columns of the right input. Thus, positions in the printable + * column alias list are not necessarily one-for-one with varattnos of the + * JOIN, so we need a separate new_colnames[] array for printing purposes. + */ +typedef struct +{ + /* + * colnames is an array containing column aliases to use for columns that + * existed when the query was parsed. Dropped columns have NULL entries. + * This array can be directly indexed by varattno to get a Var's name. + * + * Non-NULL entries are guaranteed unique within the RTE, *except* when + * this is for an unnamed JOIN RTE. In that case we merely copy up names + * from the two input RTEs. + * + * During the recursive descent in set_using_names(), forcible assignment + * of a child RTE's column name is represented by pre-setting that element + * of the child's colnames array. So at that stage, NULL entries in this + * array just mean that no name has been preassigned, not necessarily that + * the column is dropped. + */ + int num_cols; /* length of colnames[] array */ + char **colnames; /* array of C strings and NULLs */ + + /* + * new_colnames is an array containing column aliases to use for columns + * that would exist if the query was re-parsed against the current + * definitions of its base tables. This is what to print as the column + * alias list for the RTE. This array does not include dropped columns, + * but it will include columns added since original parsing. Indexes in + * it therefore have little to do with current varattno values. As above, + * entries are unique unless this is for an unnamed JOIN RTE. (In such an + * RTE, we never actually print this array, but we must compute it anyway + * for possible use in computing column names of upper joins.) The + * parallel array is_new_col marks which of these columns are new since + * original parsing. Entries with is_new_col false must match the + * non-NULL colnames entries one-for-one. + */ + int num_new_cols; /* length of new_colnames[] array */ + char **new_colnames; /* array of C strings */ + bool *is_new_col; /* array of bool flags */ + + /* This flag tells whether we should actually print a column alias list */ + bool printaliases; + + /* This list has all names used as USING names in joins above this RTE */ + List *parentUsing; /* names assigned to parent merged columns */ + + /* + * If this struct is for a JOIN RTE, we fill these fields during the + * set_using_names() pass to describe its relationship to its child RTEs. + * + * leftattnos and rightattnos are arrays with one entry per existing + * output column of the join (hence, indexable by join varattno). For a + * simple reference to a column of the left child, leftattnos[i] is the + * child RTE's attno and rightattnos[i] is zero; and conversely for a + * column of the right child. But for merged columns produced by JOIN + * USING/NATURAL JOIN, both leftattnos[i] and rightattnos[i] are nonzero. + * Also, if the column has been dropped, both are zero. + * + * If it's a JOIN USING, usingNames holds the alias names selected for the + * merged columns (these might be different from the original USING list, + * if we had to modify names to achieve uniqueness). + */ + int leftrti; /* rangetable index of left child */ + int rightrti; /* rangetable index of right child */ + int *leftattnos; /* left-child varattnos of join cols, or 0 */ + int *rightattnos; /* right-child varattnos of join cols, or 0 */ + List *usingNames; /* names assigned to merged columns */ +} deparse_columns; + +/* This macro is analogous to rt_fetch(), but for deparse_columns structs */ +#define deparse_columns_fetch(rangetable_index, dpns) \ + ((deparse_columns *) list_nth((dpns)->rtable_columns, (rangetable_index)-1)) + +/* + * Entry in set_rtable_names' hash table + */ +typedef struct +{ + char name[NAMEDATALEN]; /* Hash key --- must be first */ + int counter; /* Largest addition used so far for name */ +} NameHashEntry; + + +/* ---------- + * Local functions + * + * Most of these functions used to use fixed-size buffers to build their + * results. Now, they take an (already initialized) StringInfo object + * as a parameter, and append their text output to its contents. + * ---------- + */ +static void set_rtable_names(deparse_namespace *dpns, List *parent_namespaces, + Bitmapset *rels_used); +static void set_deparse_for_query(deparse_namespace *dpns, Query *query, + List *parent_namespaces); +static bool has_dangerous_join_using(deparse_namespace *dpns, Node *jtnode); +static void set_using_names(deparse_namespace *dpns, Node *jtnode, + List *parentUsing); +static void set_relation_column_names(deparse_namespace *dpns, + RangeTblEntry *rte, + deparse_columns *colinfo); +static void set_join_column_names(deparse_namespace *dpns, RangeTblEntry *rte, + deparse_columns *colinfo); +static bool colname_is_unique(const char *colname, deparse_namespace *dpns, + deparse_columns *colinfo); +static char *make_colname_unique(char *colname, deparse_namespace *dpns, + deparse_columns *colinfo); +static void expand_colnames_array_to(deparse_columns *colinfo, int n); +static void identify_join_columns(JoinExpr *j, RangeTblEntry *jrte, + deparse_columns *colinfo); +static char *get_rtable_name(int rtindex, deparse_context *context); +static void set_deparse_plan(deparse_namespace *dpns, Plan *plan); +static void push_child_plan(deparse_namespace *dpns, Plan *plan, + deparse_namespace *save_dpns); +static void pop_child_plan(deparse_namespace *dpns, + deparse_namespace *save_dpns); +static void push_ancestor_plan(deparse_namespace *dpns, ListCell *ancestor_cell, + deparse_namespace *save_dpns); +static void pop_ancestor_plan(deparse_namespace *dpns, + deparse_namespace *save_dpns); +static void get_query_def(Query *query, StringInfo buf, List *parentnamespace, + TupleDesc resultDesc, + int prettyFlags, int wrapColumn, int startIndent); +static void get_query_def_extended(Query *query, StringInfo buf, + List *parentnamespace, Oid distrelid, int64 shardid, + TupleDesc resultDesc, int prettyFlags, int wrapColumn, + int startIndent); +static void get_values_def(List *values_lists, deparse_context *context); +static void get_with_clause(Query *query, deparse_context *context); +static void get_select_query_def(Query *query, deparse_context *context, + TupleDesc resultDesc); +static void get_insert_query_def(Query *query, deparse_context *context); +static void get_update_query_def(Query *query, deparse_context *context); +static void get_update_query_targetlist_def(Query *query, List *targetList, + deparse_context *context, + RangeTblEntry *rte); +static void get_delete_query_def(Query *query, deparse_context *context); +static void get_utility_query_def(Query *query, deparse_context *context); +static void get_basic_select_query(Query *query, deparse_context *context, + TupleDesc resultDesc); +static void get_target_list(List *targetList, deparse_context *context, + TupleDesc resultDesc); +static void get_setop_query(Node *setOp, Query *query, + deparse_context *context, + TupleDesc resultDesc); +static Node *get_rule_sortgroupclause(Index ref, List *tlist, + bool force_colno, + deparse_context *context); +static void get_rule_groupingset(GroupingSet *gset, List *targetlist, + bool omit_parens, deparse_context *context); +static void get_rule_orderby(List *orderList, List *targetList, + bool force_colno, deparse_context *context); +static void get_rule_windowclause(Query *query, deparse_context *context); +static void get_rule_windowspec(WindowClause *wc, List *targetList, + deparse_context *context); +static char *get_variable(Var *var, int levelsup, bool istoplevel, + deparse_context *context); +static void get_special_variable(Node *node, deparse_context *context, + void *callback_arg); +static void resolve_special_varno(Node *node, deparse_context *context, + rsv_callback callback, void *callback_arg); +static Node *find_param_referent(Param *param, deparse_context *context, + deparse_namespace **dpns_p, ListCell **ancestor_cell_p); +static void get_parameter(Param *param, deparse_context *context); +static const char *get_simple_binary_op_name(OpExpr *expr); +static bool isSimpleNode(Node *node, Node *parentNode, int prettyFlags); +static void appendContextKeyword(deparse_context *context, const char *str, + int indentBefore, int indentAfter, int indentPlus); +static void removeStringInfoSpaces(StringInfo str); +static void get_rule_expr(Node *node, deparse_context *context, + bool showimplicit); +static void get_rule_expr_toplevel(Node *node, deparse_context *context, + bool showimplicit); +static void get_rule_expr_funccall(Node *node, deparse_context *context, + bool showimplicit); +static bool looks_like_function(Node *node); +static void get_oper_expr(OpExpr *expr, deparse_context *context); +static void get_func_expr(FuncExpr *expr, deparse_context *context, + bool showimplicit); +static void get_agg_expr(Aggref *aggref, deparse_context *context, + Aggref *original_aggref); +static void get_agg_combine_expr(Node *node, deparse_context *context, + void *callback_arg); +static void get_windowfunc_expr(WindowFunc *wfunc, deparse_context *context); +static void get_coercion_expr(Node *arg, deparse_context *context, + Oid resulttype, int32 resulttypmod, + Node *parentNode); +static void get_const_expr(Const *constval, deparse_context *context, + int showtype); +static void get_const_collation(Const *constval, deparse_context *context); +static void simple_quote_literal(StringInfo buf, const char *val); +static void get_sublink_expr(SubLink *sublink, deparse_context *context); +static void get_tablefunc(TableFunc *tf, deparse_context *context, + bool showimplicit); +static void get_from_clause(Query *query, const char *prefix, + deparse_context *context); +static void get_from_clause_item(Node *jtnode, Query *query, + deparse_context *context); +static void get_column_alias_list(deparse_columns *colinfo, + deparse_context *context); +static void get_from_clause_coldeflist(RangeTblFunction *rtfunc, + deparse_columns *colinfo, + deparse_context *context); +static void get_tablesample_def(TableSampleClause *tablesample, + deparse_context *context); +char *pg_get_statisticsobj_worker(Oid statextid, bool missing_ok); +static char *pg_get_triggerdef_worker(Oid trigid, bool pretty); +static void set_simple_column_names(deparse_namespace *dpns); +static void get_opclass_name(Oid opclass, Oid actual_datatype, + StringInfo buf); +static Node *processIndirection(Node *node, deparse_context *context); +static void printSubscripts(SubscriptingRef *aref, deparse_context *context); +static char *get_relation_name(Oid relid); +static char *generate_relation_or_shard_name(Oid relid, Oid distrelid, + int64 shardid, List *namespaces); +static char *generate_rte_shard_name(RangeTblEntry *rangeTableEntry); +static char *generate_fragment_name(char *schemaName, char *tableName); +static char *generate_function_name(Oid funcid, int nargs, + List *argnames, Oid *argtypes, + bool has_variadic, bool *use_variadic_p, + ParseExprKind special_exprkind); + +#define only_marker(rte) ((rte)->inh ? "" : "ONLY ") + + + +/* + * pg_get_query_def parses back one query tree, and outputs the resulting query + * string into given buffer. + */ +void +pg_get_query_def(Query *query, StringInfo buffer) +{ + get_query_def(query, buffer, NIL, NULL, 0, WRAP_COLUMN_DEFAULT, 0); +} + + +/* + * pg_get_rule_expr deparses an expression and returns the result as a string. + */ +char * +pg_get_rule_expr(Node *expression) +{ + bool showImplicitCasts = true; + deparse_context context; + OverrideSearchPath *overridePath = NULL; + StringInfo buffer = makeStringInfo(); + + /* + * Set search_path to NIL so that all objects outside of pg_catalog will be + * schema-prefixed. pg_catalog will be added automatically when we call + * PushOverrideSearchPath(), since we set addCatalog to true; + */ + overridePath = GetOverrideSearchPath(CurrentMemoryContext); + overridePath->schemas = NIL; + overridePath->addCatalog = true; + PushOverrideSearchPath(overridePath); + + context.buf = buffer; + context.namespaces = NIL; + context.windowClause = NIL; + context.windowTList = NIL; + context.varprefix = false; + context.prettyFlags = 0; + context.wrapColumn = WRAP_COLUMN_DEFAULT; + context.indentLevel = 0; + context.special_exprkind = EXPR_KIND_NONE; + context.distrelid = InvalidOid; + context.shardid = INVALID_SHARD_ID; + + get_rule_expr(expression, &context, showImplicitCasts); + + /* revert back to original search_path */ + PopOverrideSearchPath(); + + return buffer->data; +} + + +/* + * set_rtable_names: select RTE aliases to be used in printing a query + * + * We fill in dpns->rtable_names with a list of names that is one-for-one with + * the already-filled dpns->rtable list. Each RTE name is unique among those + * in the new namespace plus any ancestor namespaces listed in + * parent_namespaces. + * + * If rels_used isn't NULL, only RTE indexes listed in it are given aliases. + * + * Note that this function is only concerned with relation names, not column + * names. + */ +static void +set_rtable_names(deparse_namespace *dpns, List *parent_namespaces, + Bitmapset *rels_used) +{ + HASHCTL hash_ctl; + HTAB *names_hash; + NameHashEntry *hentry; + bool found; + int rtindex; + ListCell *lc; + + dpns->rtable_names = NIL; + /* nothing more to do if empty rtable */ + if (dpns->rtable == NIL) + return; + + /* + * We use a hash table to hold known names, so that this process is O(N) + * not O(N^2) for N names. + */ + MemSet(&hash_ctl, 0, sizeof(hash_ctl)); + hash_ctl.keysize = NAMEDATALEN; + hash_ctl.entrysize = sizeof(NameHashEntry); + hash_ctl.hcxt = CurrentMemoryContext; + names_hash = hash_create("set_rtable_names names", + list_length(dpns->rtable), + &hash_ctl, + HASH_ELEM | HASH_CONTEXT); + /* Preload the hash table with names appearing in parent_namespaces */ + foreach(lc, parent_namespaces) + { + deparse_namespace *olddpns = (deparse_namespace *) lfirst(lc); + ListCell *lc2; + + foreach(lc2, olddpns->rtable_names) + { + char *oldname = (char *) lfirst(lc2); + + if (oldname == NULL) + continue; + hentry = (NameHashEntry *) hash_search(names_hash, + oldname, + HASH_ENTER, + &found); + /* we do not complain about duplicate names in parent namespaces */ + hentry->counter = 0; + } + } + + /* Now we can scan the rtable */ + rtindex = 1; + foreach(lc, dpns->rtable) + { + RangeTblEntry *rte = (RangeTblEntry *) lfirst(lc); + char *refname; + + /* Just in case this takes an unreasonable amount of time ... */ + CHECK_FOR_INTERRUPTS(); + + if (rels_used && !bms_is_member(rtindex, rels_used)) + { + /* Ignore unreferenced RTE */ + refname = NULL; + } + else if (rte->alias) + { + /* If RTE has a user-defined alias, prefer that */ + refname = rte->alias->aliasname; + } + else if (rte->rtekind == RTE_RELATION) + { + /* Use the current actual name of the relation */ + refname = get_rel_name(rte->relid); + } + else if (rte->rtekind == RTE_JOIN) + { + /* Unnamed join has no refname */ + refname = NULL; + } + else + { + /* Otherwise use whatever the parser assigned */ + refname = rte->eref->aliasname; + } + + /* + * If the selected name isn't unique, append digits to make it so, and + * make a new hash entry for it once we've got a unique name. For a + * very long input name, we might have to truncate to stay within + * NAMEDATALEN. + */ + if (refname) + { + hentry = (NameHashEntry *) hash_search(names_hash, + refname, + HASH_ENTER, + &found); + if (found) + { + /* Name already in use, must choose a new one */ + int refnamelen = strlen(refname); + char *modname = (char *) palloc(refnamelen + 16); + NameHashEntry *hentry2; + + do + { + hentry->counter++; + for (;;) + { + /* + * We avoid using %.*s here because it can misbehave + * if the data is not valid in what libc thinks is the + * prevailing encoding. + */ + memcpy(modname, refname, refnamelen); + sprintf(modname + refnamelen, "_%d", hentry->counter); + if (strlen(modname) < NAMEDATALEN) + break; + /* drop chars from refname to keep all the digits */ + refnamelen = pg_mbcliplen(refname, refnamelen, + refnamelen - 1); + } + hentry2 = (NameHashEntry *) hash_search(names_hash, + modname, + HASH_ENTER, + &found); + } while (found); + hentry2->counter = 0; /* init new hash entry */ + refname = modname; + } + else + { + /* Name not previously used, need only initialize hentry */ + hentry->counter = 0; + } + } + + dpns->rtable_names = lappend(dpns->rtable_names, refname); + rtindex++; + } + + hash_destroy(names_hash); +} + +/* + * set_deparse_for_query: set up deparse_namespace for deparsing a Query tree + * + * For convenience, this is defined to initialize the deparse_namespace struct + * from scratch. + */ +static void +set_deparse_for_query(deparse_namespace *dpns, Query *query, + List *parent_namespaces) +{ + ListCell *lc; + ListCell *lc2; + + /* Initialize *dpns and fill rtable/ctes links */ + memset(dpns, 0, sizeof(deparse_namespace)); + dpns->rtable = query->rtable; + dpns->subplans = NIL; + dpns->ctes = query->cteList; + dpns->appendrels = NULL; + + /* Assign a unique relation alias to each RTE */ + set_rtable_names(dpns, parent_namespaces, NULL); + + /* Initialize dpns->rtable_columns to contain zeroed structs */ + dpns->rtable_columns = NIL; + while (list_length(dpns->rtable_columns) < list_length(dpns->rtable)) + dpns->rtable_columns = lappend(dpns->rtable_columns, + palloc0(sizeof(deparse_columns))); + + /* If it's a utility query, it won't have a jointree */ + if (query->jointree) + { + /* Detect whether global uniqueness of USING names is needed */ + dpns->unique_using = + has_dangerous_join_using(dpns, (Node *) query->jointree); + + /* + * Select names for columns merged by USING, via a recursive pass over + * the query jointree. + */ + set_using_names(dpns, (Node *) query->jointree, NIL); + } + + /* + * Now assign remaining column aliases for each RTE. We do this in a + * linear scan of the rtable, so as to process RTEs whether or not they + * are in the jointree (we mustn't miss NEW.*, INSERT target relations, + * etc). JOIN RTEs must be processed after their children, but this is + * okay because they appear later in the rtable list than their children + * (cf Asserts in identify_join_columns()). + */ + forboth(lc, dpns->rtable, lc2, dpns->rtable_columns) + { + RangeTblEntry *rte = (RangeTblEntry *) lfirst(lc); + deparse_columns *colinfo = (deparse_columns *) lfirst(lc2); + + if (rte->rtekind == RTE_JOIN) + set_join_column_names(dpns, rte, colinfo); + else + set_relation_column_names(dpns, rte, colinfo); + } +} + +/* + * has_dangerous_join_using: search jointree for unnamed JOIN USING + * + * Merged columns of a JOIN USING may act differently from either of the input + * columns, either because they are merged with COALESCE (in a FULL JOIN) or + * because an implicit coercion of the underlying input column is required. + * In such a case the column must be referenced as a column of the JOIN not as + * a column of either input. And this is problematic if the join is unnamed + * (alias-less): we cannot qualify the column's name with an RTE name, since + * there is none. (Forcibly assigning an alias to the join is not a solution, + * since that will prevent legal references to tables below the join.) + * To ensure that every column in the query is unambiguously referenceable, + * we must assign such merged columns names that are globally unique across + * the whole query, aliasing other columns out of the way as necessary. + * + * Because the ensuing re-aliasing is fairly damaging to the readability of + * the query, we don't do this unless we have to. So, we must pre-scan + * the join tree to see if we have to, before starting set_using_names(). + */ +static bool +has_dangerous_join_using(deparse_namespace *dpns, Node *jtnode) +{ + if (IsA(jtnode, RangeTblRef)) + { + /* nothing to do here */ + } + else if (IsA(jtnode, FromExpr)) + { + FromExpr *f = (FromExpr *) jtnode; + ListCell *lc; + + foreach(lc, f->fromlist) + { + if (has_dangerous_join_using(dpns, (Node *) lfirst(lc))) + return true; + } + } + else if (IsA(jtnode, JoinExpr)) + { + JoinExpr *j = (JoinExpr *) jtnode; + + /* Is it an unnamed JOIN with USING? */ + if (j->alias == NULL && j->usingClause) + { + /* + * Yes, so check each join alias var to see if any of them are not + * simple references to underlying columns. If so, we have a + * dangerous situation and must pick unique aliases. + */ + RangeTblEntry *jrte = rt_fetch(j->rtindex, dpns->rtable); + + /* We need only examine the merged columns */ + for (int i = 0; i < jrte->joinmergedcols; i++) + { + Node *aliasvar = list_nth(jrte->joinaliasvars, i); + + if (!IsA(aliasvar, Var)) + return true; + } + } + + /* Nope, but inspect children */ + if (has_dangerous_join_using(dpns, j->larg)) + return true; + if (has_dangerous_join_using(dpns, j->rarg)) + return true; + } + else + elog(ERROR, "unrecognized node type: %d", + (int) nodeTag(jtnode)); + return false; +} + +/* + * set_using_names: select column aliases to be used for merged USING columns + * + * We do this during a recursive descent of the query jointree. + * dpns->unique_using must already be set to determine the global strategy. + * + * Column alias info is saved in the dpns->rtable_columns list, which is + * assumed to be filled with pre-zeroed deparse_columns structs. + * + * parentUsing is a list of all USING aliases assigned in parent joins of + * the current jointree node. (The passed-in list must not be modified.) + */ +static void +set_using_names(deparse_namespace *dpns, Node *jtnode, List *parentUsing) +{ + if (IsA(jtnode, RangeTblRef)) + { + /* nothing to do now */ + } + else if (IsA(jtnode, FromExpr)) + { + FromExpr *f = (FromExpr *) jtnode; + ListCell *lc; + + foreach(lc, f->fromlist) + set_using_names(dpns, (Node *) lfirst(lc), parentUsing); + } + else if (IsA(jtnode, JoinExpr)) + { + JoinExpr *j = (JoinExpr *) jtnode; + RangeTblEntry *rte = rt_fetch(j->rtindex, dpns->rtable); + deparse_columns *colinfo = deparse_columns_fetch(j->rtindex, dpns); + int *leftattnos; + int *rightattnos; + deparse_columns *leftcolinfo; + deparse_columns *rightcolinfo; + int i; + ListCell *lc; + + /* Get info about the shape of the join */ + identify_join_columns(j, rte, colinfo); + leftattnos = colinfo->leftattnos; + rightattnos = colinfo->rightattnos; + + /* Look up the not-yet-filled-in child deparse_columns structs */ + leftcolinfo = deparse_columns_fetch(colinfo->leftrti, dpns); + rightcolinfo = deparse_columns_fetch(colinfo->rightrti, dpns); + + /* + * If this join is unnamed, then we cannot substitute new aliases at + * this level, so any name requirements pushed down to here must be + * pushed down again to the children. + */ + if (rte->alias == NULL) + { + for (i = 0; i < colinfo->num_cols; i++) + { + char *colname = colinfo->colnames[i]; + + if (colname == NULL) + continue; + + /* Push down to left column, unless it's a system column */ + if (leftattnos[i] > 0) + { + expand_colnames_array_to(leftcolinfo, leftattnos[i]); + leftcolinfo->colnames[leftattnos[i] - 1] = colname; + } + + /* Same on the righthand side */ + if (rightattnos[i] > 0) + { + expand_colnames_array_to(rightcolinfo, rightattnos[i]); + rightcolinfo->colnames[rightattnos[i] - 1] = colname; + } + } + } + + /* + * If there's a USING clause, select the USING column names and push + * those names down to the children. We have two strategies: + * + * If dpns->unique_using is true, we force all USING names to be + * unique across the whole query level. In principle we'd only need + * the names of dangerous USING columns to be globally unique, but to + * safely assign all USING names in a single pass, we have to enforce + * the same uniqueness rule for all of them. However, if a USING + * column's name has been pushed down from the parent, we should use + * it as-is rather than making a uniqueness adjustment. This is + * necessary when we're at an unnamed join, and it creates no risk of + * ambiguity. Also, if there's a user-written output alias for a + * merged column, we prefer to use that rather than the input name; + * this simplifies the logic and seems likely to lead to less aliasing + * overall. + * + * If dpns->unique_using is false, we only need USING names to be + * unique within their own join RTE. We still need to honor + * pushed-down names, though. + * + * Though significantly different in results, these two strategies are + * implemented by the same code, with only the difference of whether + * to put assigned names into dpns->using_names. + */ + if (j->usingClause) + { + /* Copy the input parentUsing list so we don't modify it */ + parentUsing = list_copy(parentUsing); + + /* USING names must correspond to the first join output columns */ + expand_colnames_array_to(colinfo, list_length(j->usingClause)); + i = 0; + foreach(lc, j->usingClause) + { + char *colname = strVal(lfirst(lc)); + + /* Assert it's a merged column */ + Assert(leftattnos[i] != 0 && rightattnos[i] != 0); + + /* Adopt passed-down name if any, else select unique name */ + if (colinfo->colnames[i] != NULL) + colname = colinfo->colnames[i]; + else + { + /* Prefer user-written output alias if any */ + if (rte->alias && i < list_length(rte->alias->colnames)) + colname = strVal(list_nth(rte->alias->colnames, i)); + /* Make it appropriately unique */ + colname = make_colname_unique(colname, dpns, colinfo); + if (dpns->unique_using) + dpns->using_names = lappend(dpns->using_names, + colname); + /* Save it as output column name, too */ + colinfo->colnames[i] = colname; + } + + /* Remember selected names for use later */ + colinfo->usingNames = lappend(colinfo->usingNames, colname); + parentUsing = lappend(parentUsing, colname); + + /* Push down to left column, unless it's a system column */ + if (leftattnos[i] > 0) + { + expand_colnames_array_to(leftcolinfo, leftattnos[i]); + leftcolinfo->colnames[leftattnos[i] - 1] = colname; + } + + /* Same on the righthand side */ + if (rightattnos[i] > 0) + { + expand_colnames_array_to(rightcolinfo, rightattnos[i]); + rightcolinfo->colnames[rightattnos[i] - 1] = colname; + } + + i++; + } + } + + /* Mark child deparse_columns structs with correct parentUsing info */ + leftcolinfo->parentUsing = parentUsing; + rightcolinfo->parentUsing = parentUsing; + + /* Now recursively assign USING column names in children */ + set_using_names(dpns, j->larg, parentUsing); + set_using_names(dpns, j->rarg, parentUsing); + } + else + elog(ERROR, "unrecognized node type: %d", + (int) nodeTag(jtnode)); +} + +/* + * set_relation_column_names: select column aliases for a non-join RTE + * + * Column alias info is saved in *colinfo, which is assumed to be pre-zeroed. + * If any colnames entries are already filled in, those override local + * choices. + */ +static void +set_relation_column_names(deparse_namespace *dpns, RangeTblEntry *rte, + deparse_columns *colinfo) +{ + int ncolumns; + char **real_colnames; + bool changed_any; + bool has_anonymous; + int noldcolumns; + int i; + int j; + + /* + * Extract the RTE's "real" column names. This is comparable to + * get_rte_attribute_name, except that it's important to disregard dropped + * columns. We put NULL into the array for a dropped column. + */ + if (rte->rtekind == RTE_RELATION) + { + /* Relation --- look to the system catalogs for up-to-date info */ + Relation rel; + TupleDesc tupdesc; + + rel = relation_open(rte->relid, AccessShareLock); + tupdesc = RelationGetDescr(rel); + + ncolumns = tupdesc->natts; + real_colnames = (char **) palloc(ncolumns * sizeof(char *)); + + for (i = 0; i < ncolumns; i++) + { + Form_pg_attribute attr = TupleDescAttr(tupdesc, i); + + if (attr->attisdropped) + real_colnames[i] = NULL; + else + real_colnames[i] = pstrdup(NameStr(attr->attname)); + } + relation_close(rel, AccessShareLock); + } + else + { + /* Otherwise use the column names from eref */ + ListCell *lc; + + ncolumns = list_length(rte->eref->colnames); + real_colnames = (char **) palloc(ncolumns * sizeof(char *)); + + i = 0; + foreach(lc, rte->eref->colnames) + { + /* + * If the column name shown in eref is an empty string, then it's + * a column that was dropped at the time of parsing the query, so + * treat it as dropped. + */ + char *cname = strVal(lfirst(lc)); + + if (cname[0] == '\0') + cname = NULL; + real_colnames[i] = cname; + i++; + } + } + + /* + * Ensure colinfo->colnames has a slot for each column. (It could be long + * enough already, if we pushed down a name for the last column.) Note: + * it's possible that there are now more columns than there were when the + * query was parsed, ie colnames could be longer than rte->eref->colnames. + * We must assign unique aliases to the new columns too, else there could + * be unresolved conflicts when the view/rule is reloaded. + */ + expand_colnames_array_to(colinfo, ncolumns); + Assert(colinfo->num_cols == ncolumns); + + /* + * Make sufficiently large new_colnames and is_new_col arrays, too. + * + * Note: because we leave colinfo->num_new_cols zero until after the loop, + * colname_is_unique will not consult that array, which is fine because it + * would only be duplicate effort. + */ + colinfo->new_colnames = (char **) palloc(ncolumns * sizeof(char *)); + colinfo->is_new_col = (bool *) palloc(ncolumns * sizeof(bool)); + + /* + * Scan the columns, select a unique alias for each one, and store it in + * colinfo->colnames and colinfo->new_colnames. The former array has NULL + * entries for dropped columns, the latter omits them. Also mark + * new_colnames entries as to whether they are new since parse time; this + * is the case for entries beyond the length of rte->eref->colnames. + */ + noldcolumns = list_length(rte->eref->colnames); + changed_any = false; + has_anonymous = false; + j = 0; + for (i = 0; i < ncolumns; i++) + { + char *real_colname = real_colnames[i]; + char *colname = colinfo->colnames[i]; + + /* Skip dropped columns */ + if (real_colname == NULL) + { + Assert(colname == NULL); /* colnames[i] is already NULL */ + continue; + } + + /* If alias already assigned, that's what to use */ + if (colname == NULL) + { + /* If user wrote an alias, prefer that over real column name */ + if (rte->alias && i < list_length(rte->alias->colnames)) + colname = strVal(list_nth(rte->alias->colnames, i)); + else + colname = real_colname; + + /* Unique-ify and insert into colinfo */ + colname = make_colname_unique(colname, dpns, colinfo); + + colinfo->colnames[i] = colname; + } + + /* Put names of non-dropped columns in new_colnames[] too */ + colinfo->new_colnames[j] = colname; + /* And mark them as new or not */ + colinfo->is_new_col[j] = (i >= noldcolumns); + j++; + + /* Remember if any assigned aliases differ from "real" name */ + if (!changed_any && strcmp(colname, real_colname) != 0) + changed_any = true; + + /* + * Remember if there is a reference to an anonymous column as named by + * char * FigureColname(Node *node) + */ + if (!has_anonymous && strcmp(real_colname, "?column?") == 0) + has_anonymous = true; + } + + /* + * Set correct length for new_colnames[] array. (Note: if columns have + * been added, colinfo->num_cols includes them, which is not really quite + * right but is harmless, since any new columns must be at the end where + * they won't affect varattnos of pre-existing columns.) + */ + colinfo->num_new_cols = j; + + /* + * For a relation RTE, we need only print the alias column names if any + * are different from the underlying "real" names. For a function RTE, + * always emit a complete column alias list; this is to protect against + * possible instability of the default column names (eg, from altering + * parameter names). For tablefunc RTEs, we never print aliases, because + * the column names are part of the clause itself. For other RTE types, + * print if we changed anything OR if there were user-written column + * aliases (since the latter would be part of the underlying "reality"). + */ + if (rte->rtekind == RTE_RELATION) + colinfo->printaliases = changed_any; + else if (rte->rtekind == RTE_FUNCTION) + colinfo->printaliases = true; + else if (rte->rtekind == RTE_TABLEFUNC) + colinfo->printaliases = false; + else if (rte->alias && rte->alias->colnames != NIL) + colinfo->printaliases = true; + else + colinfo->printaliases = changed_any || has_anonymous; +} + +/* + * set_join_column_names: select column aliases for a join RTE + * + * Column alias info is saved in *colinfo, which is assumed to be pre-zeroed. + * If any colnames entries are already filled in, those override local + * choices. Also, names for USING columns were already chosen by + * set_using_names(). We further expect that column alias selection has been + * completed for both input RTEs. + */ +static void +set_join_column_names(deparse_namespace *dpns, RangeTblEntry *rte, + deparse_columns *colinfo) +{ + deparse_columns *leftcolinfo; + deparse_columns *rightcolinfo; + bool changed_any; + int noldcolumns; + int nnewcolumns; + Bitmapset *leftmerged = NULL; + Bitmapset *rightmerged = NULL; + int i; + int j; + int ic; + int jc; + + /* Look up the previously-filled-in child deparse_columns structs */ + leftcolinfo = deparse_columns_fetch(colinfo->leftrti, dpns); + rightcolinfo = deparse_columns_fetch(colinfo->rightrti, dpns); + + /* + * Ensure colinfo->colnames has a slot for each column. (It could be long + * enough already, if we pushed down a name for the last column.) Note: + * it's possible that one or both inputs now have more columns than there + * were when the query was parsed, but we'll deal with that below. We + * only need entries in colnames for pre-existing columns. + */ + noldcolumns = list_length(rte->eref->colnames); + expand_colnames_array_to(colinfo, noldcolumns); + Assert(colinfo->num_cols == noldcolumns); + + /* + * Scan the join output columns, select an alias for each one, and store + * it in colinfo->colnames. If there are USING columns, set_using_names() + * already selected their names, so we can start the loop at the first + * non-merged column. + */ + changed_any = false; + for (i = list_length(colinfo->usingNames); i < noldcolumns; i++) + { + char *colname = colinfo->colnames[i]; + char *real_colname; + + /* Join column must refer to at least one input column */ + Assert(colinfo->leftattnos[i] != 0 || colinfo->rightattnos[i] != 0); + + /* Get the child column name */ + if (colinfo->leftattnos[i] > 0) + real_colname = leftcolinfo->colnames[colinfo->leftattnos[i] - 1]; + else if (colinfo->rightattnos[i] > 0) + real_colname = rightcolinfo->colnames[colinfo->rightattnos[i] - 1]; + else + { + /* We're joining system columns --- use eref name */ + real_colname = strVal(list_nth(rte->eref->colnames, i)); + } + /* If child col has been dropped, no need to assign a join colname */ + if (real_colname == NULL) + { + colinfo->colnames[i] = NULL; + continue; + } + + /* In an unnamed join, just report child column names as-is */ + if (rte->alias == NULL) + { + colinfo->colnames[i] = real_colname; + continue; + } + + /* If alias already assigned, that's what to use */ + if (colname == NULL) + { + /* If user wrote an alias, prefer that over real column name */ + if (rte->alias && i < list_length(rte->alias->colnames)) + colname = strVal(list_nth(rte->alias->colnames, i)); + else + colname = real_colname; + + /* Unique-ify and insert into colinfo */ + colname = make_colname_unique(colname, dpns, colinfo); + + colinfo->colnames[i] = colname; + } + + /* Remember if any assigned aliases differ from "real" name */ + if (!changed_any && strcmp(colname, real_colname) != 0) + changed_any = true; + } + + /* + * Calculate number of columns the join would have if it were re-parsed + * now, and create storage for the new_colnames and is_new_col arrays. + * + * Note: colname_is_unique will be consulting new_colnames[] during the + * loops below, so its not-yet-filled entries must be zeroes. + */ + nnewcolumns = leftcolinfo->num_new_cols + rightcolinfo->num_new_cols - + list_length(colinfo->usingNames); + colinfo->num_new_cols = nnewcolumns; + colinfo->new_colnames = (char **) palloc0(nnewcolumns * sizeof(char *)); + colinfo->is_new_col = (bool *) palloc0(nnewcolumns * sizeof(bool)); + + /* + * Generating the new_colnames array is a bit tricky since any new columns + * added since parse time must be inserted in the right places. This code + * must match the parser, which will order a join's columns as merged + * columns first (in USING-clause order), then non-merged columns from the + * left input (in attnum order), then non-merged columns from the right + * input (ditto). If one of the inputs is itself a join, its columns will + * be ordered according to the same rule, which means newly-added columns + * might not be at the end. We can figure out what's what by consulting + * the leftattnos and rightattnos arrays plus the input is_new_col arrays. + * + * In these loops, i indexes leftattnos/rightattnos (so it's join varattno + * less one), j indexes new_colnames/is_new_col, and ic/jc have similar + * meanings for the current child RTE. + */ + + /* Handle merged columns; they are first and can't be new */ + i = j = 0; + while (i < noldcolumns && + colinfo->leftattnos[i] != 0 && + colinfo->rightattnos[i] != 0) + { + /* column name is already determined and known unique */ + colinfo->new_colnames[j] = colinfo->colnames[i]; + colinfo->is_new_col[j] = false; + + /* build bitmapsets of child attnums of merged columns */ + if (colinfo->leftattnos[i] > 0) + leftmerged = bms_add_member(leftmerged, colinfo->leftattnos[i]); + if (colinfo->rightattnos[i] > 0) + rightmerged = bms_add_member(rightmerged, colinfo->rightattnos[i]); + + i++, j++; + } + + /* Handle non-merged left-child columns */ + ic = 0; + for (jc = 0; jc < leftcolinfo->num_new_cols; jc++) + { + char *child_colname = leftcolinfo->new_colnames[jc]; + + if (!leftcolinfo->is_new_col[jc]) + { + /* Advance ic to next non-dropped old column of left child */ + while (ic < leftcolinfo->num_cols && + leftcolinfo->colnames[ic] == NULL) + ic++; + Assert(ic < leftcolinfo->num_cols); + ic++; + /* If it is a merged column, we already processed it */ + if (bms_is_member(ic, leftmerged)) + continue; + /* Else, advance i to the corresponding existing join column */ + while (i < colinfo->num_cols && + colinfo->colnames[i] == NULL) + i++; + Assert(i < colinfo->num_cols); + Assert(ic == colinfo->leftattnos[i]); + /* Use the already-assigned name of this column */ + colinfo->new_colnames[j] = colinfo->colnames[i]; + i++; + } + else + { + /* + * Unique-ify the new child column name and assign, unless we're + * in an unnamed join, in which case just copy + */ + if (rte->alias != NULL) + { + colinfo->new_colnames[j] = + make_colname_unique(child_colname, dpns, colinfo); + if (!changed_any && + strcmp(colinfo->new_colnames[j], child_colname) != 0) + changed_any = true; + } + else + colinfo->new_colnames[j] = child_colname; + } + + colinfo->is_new_col[j] = leftcolinfo->is_new_col[jc]; + j++; + } + + /* Handle non-merged right-child columns in exactly the same way */ + ic = 0; + for (jc = 0; jc < rightcolinfo->num_new_cols; jc++) + { + char *child_colname = rightcolinfo->new_colnames[jc]; + + if (!rightcolinfo->is_new_col[jc]) + { + /* Advance ic to next non-dropped old column of right child */ + while (ic < rightcolinfo->num_cols && + rightcolinfo->colnames[ic] == NULL) + ic++; + Assert(ic < rightcolinfo->num_cols); + ic++; + /* If it is a merged column, we already processed it */ + if (bms_is_member(ic, rightmerged)) + continue; + /* Else, advance i to the corresponding existing join column */ + while (i < colinfo->num_cols && + colinfo->colnames[i] == NULL) + i++; + Assert(i < colinfo->num_cols); + Assert(ic == colinfo->rightattnos[i]); + /* Use the already-assigned name of this column */ + colinfo->new_colnames[j] = colinfo->colnames[i]; + i++; + } + else + { + /* + * Unique-ify the new child column name and assign, unless we're + * in an unnamed join, in which case just copy + */ + if (rte->alias != NULL) + { + colinfo->new_colnames[j] = + make_colname_unique(child_colname, dpns, colinfo); + if (!changed_any && + strcmp(colinfo->new_colnames[j], child_colname) != 0) + changed_any = true; + } + else + colinfo->new_colnames[j] = child_colname; + } + + colinfo->is_new_col[j] = rightcolinfo->is_new_col[jc]; + j++; + } + + /* Assert we processed the right number of columns */ +#ifdef USE_ASSERT_CHECKING + while (i < colinfo->num_cols && colinfo->colnames[i] == NULL) + i++; + Assert(i == colinfo->num_cols); + Assert(j == nnewcolumns); +#endif + + /* + * For a named join, print column aliases if we changed any from the child + * names. Unnamed joins cannot print aliases. + */ + if (rte->alias != NULL) + colinfo->printaliases = changed_any; + else + colinfo->printaliases = false; +} + +/* + * colname_is_unique: is colname distinct from already-chosen column names? + * + * dpns is query-wide info, colinfo is for the column's RTE + */ +static bool +colname_is_unique(const char *colname, deparse_namespace *dpns, + deparse_columns *colinfo) +{ + int i; + ListCell *lc; + + /* Check against already-assigned column aliases within RTE */ + for (i = 0; i < colinfo->num_cols; i++) + { + char *oldname = colinfo->colnames[i]; + + if (oldname && strcmp(oldname, colname) == 0) + return false; + } + + /* + * If we're building a new_colnames array, check that too (this will be + * partially but not completely redundant with the previous checks) + */ + for (i = 0; i < colinfo->num_new_cols; i++) + { + char *oldname = colinfo->new_colnames[i]; + + if (oldname && strcmp(oldname, colname) == 0) + return false; + } + + /* Also check against USING-column names that must be globally unique */ + foreach(lc, dpns->using_names) + { + char *oldname = (char *) lfirst(lc); + + if (strcmp(oldname, colname) == 0) + return false; + } + + /* Also check against names already assigned for parent-join USING cols */ + foreach(lc, colinfo->parentUsing) + { + char *oldname = (char *) lfirst(lc); + + if (strcmp(oldname, colname) == 0) + return false; + } + + return true; +} + +/* + * make_colname_unique: modify colname if necessary to make it unique + * + * dpns is query-wide info, colinfo is for the column's RTE + */ +static char * +make_colname_unique(char *colname, deparse_namespace *dpns, + deparse_columns *colinfo) +{ + /* + * If the selected name isn't unique, append digits to make it so. For a + * very long input name, we might have to truncate to stay within + * NAMEDATALEN. + */ + if (!colname_is_unique(colname, dpns, colinfo)) + { + int colnamelen = strlen(colname); + char *modname = (char *) palloc(colnamelen + 16); + int i = 0; + + do + { + i++; + for (;;) + { + /* + * We avoid using %.*s here because it can misbehave if the + * data is not valid in what libc thinks is the prevailing + * encoding. + */ + memcpy(modname, colname, colnamelen); + sprintf(modname + colnamelen, "_%d", i); + if (strlen(modname) < NAMEDATALEN) + break; + /* drop chars from colname to keep all the digits */ + colnamelen = pg_mbcliplen(colname, colnamelen, + colnamelen - 1); + } + } while (!colname_is_unique(modname, dpns, colinfo)); + colname = modname; + } + return colname; +} + +/* + * expand_colnames_array_to: make colinfo->colnames at least n items long + * + * Any added array entries are initialized to zero. + */ +static void +expand_colnames_array_to(deparse_columns *colinfo, int n) +{ + if (n > colinfo->num_cols) + { + if (colinfo->colnames == NULL) + colinfo->colnames = (char **) palloc0(n * sizeof(char *)); + else + { + colinfo->colnames = (char **) repalloc(colinfo->colnames, + n * sizeof(char *)); + memset(colinfo->colnames + colinfo->num_cols, 0, + (n - colinfo->num_cols) * sizeof(char *)); + } + colinfo->num_cols = n; + } +} + +/* + * identify_join_columns: figure out where columns of a join come from + * + * Fills the join-specific fields of the colinfo struct, except for + * usingNames which is filled later. + */ +static void +identify_join_columns(JoinExpr *j, RangeTblEntry *jrte, + deparse_columns *colinfo) +{ + int numjoincols; + int jcolno; + int rcolno; + ListCell *lc; + + /* Extract left/right child RT indexes */ + if (IsA(j->larg, RangeTblRef)) + colinfo->leftrti = ((RangeTblRef *) j->larg)->rtindex; + else if (IsA(j->larg, JoinExpr)) + colinfo->leftrti = ((JoinExpr *) j->larg)->rtindex; + else + elog(ERROR, "unrecognized node type in jointree: %d", + (int) nodeTag(j->larg)); + if (IsA(j->rarg, RangeTblRef)) + colinfo->rightrti = ((RangeTblRef *) j->rarg)->rtindex; + else if (IsA(j->rarg, JoinExpr)) + colinfo->rightrti = ((JoinExpr *) j->rarg)->rtindex; + else + elog(ERROR, "unrecognized node type in jointree: %d", + (int) nodeTag(j->rarg)); + + /* Assert children will be processed earlier than join in second pass */ + Assert(colinfo->leftrti < j->rtindex); + Assert(colinfo->rightrti < j->rtindex); + + /* Initialize result arrays with zeroes */ + numjoincols = list_length(jrte->joinaliasvars); + Assert(numjoincols == list_length(jrte->eref->colnames)); + colinfo->leftattnos = (int *) palloc0(numjoincols * sizeof(int)); + colinfo->rightattnos = (int *) palloc0(numjoincols * sizeof(int)); + + /* + * Deconstruct RTE's joinleftcols/joinrightcols into desired format. + * Recall that the column(s) merged due to USING are the first column(s) + * of the join output. We need not do anything special while scanning + * joinleftcols, but while scanning joinrightcols we must distinguish + * merged from unmerged columns. + */ + jcolno = 0; + foreach(lc, jrte->joinleftcols) + { + int leftattno = lfirst_int(lc); + + colinfo->leftattnos[jcolno++] = leftattno; + } + rcolno = 0; + foreach(lc, jrte->joinrightcols) + { + int rightattno = lfirst_int(lc); + + if (rcolno < jrte->joinmergedcols) /* merged column? */ + colinfo->rightattnos[rcolno] = rightattno; + else + colinfo->rightattnos[jcolno++] = rightattno; + rcolno++; + } + Assert(jcolno == numjoincols); +} + +/* + * get_rtable_name: convenience function to get a previously assigned RTE alias + * + * The RTE must belong to the topmost namespace level in "context". + */ +static char * +get_rtable_name(int rtindex, deparse_context *context) +{ + deparse_namespace *dpns = (deparse_namespace *) linitial(context->namespaces); + + Assert(rtindex > 0 && rtindex <= list_length(dpns->rtable_names)); + return (char *) list_nth(dpns->rtable_names, rtindex - 1); +} + +/* + * set_deparse_plan: set up deparse_namespace to parse subexpressions + * of a given Plan node + * + * This sets the plan, outer_planstate, inner_planstate, outer_tlist, + * inner_tlist, and index_tlist fields. Caller is responsible for adjusting + * the ancestors list if necessary. Note that the rtable and ctes fields do + * not need to change when shifting attention to different plan nodes in a + * single plan tree. + */ +static void +set_deparse_plan(deparse_namespace *dpns, Plan *plan) +{ + dpns->plan = plan; + + /* + * We special-case Append and MergeAppend to pretend that the first child + * plan is the OUTER referent; we have to interpret OUTER Vars in their + * tlists according to one of the children, and the first one is the most + * natural choice. Likewise special-case ModifyTable to pretend that the + * first child plan is the OUTER referent; this is to support RETURNING + * lists containing references to non-target relations. + */ + if (IsA(plan, Append)) + dpns->outer_plan = linitial(((Append *) plan)->appendplans); + else if (IsA(plan, MergeAppend)) + dpns->outer_plan = linitial(((MergeAppend *) plan)->mergeplans); + else if (IsA(plan, ModifyTable)) + dpns->outer_plan = linitial(((ModifyTable *) plan)->plans); + else + dpns->outer_plan = outerPlan(plan); + + if (dpns->outer_plan) + dpns->outer_tlist = dpns->outer_plan->targetlist; + else + dpns->outer_tlist = NIL; + + /* + * For a SubqueryScan, pretend the subplan is INNER referent. (We don't + * use OUTER because that could someday conflict with the normal meaning.) + * Likewise, for a CteScan, pretend the subquery's plan is INNER referent. + * For ON CONFLICT .. UPDATE we just need the inner tlist to point to the + * excluded expression's tlist. (Similar to the SubqueryScan we don't want + * to reuse OUTER, it's used for RETURNING in some modify table cases, + * although not INSERT .. CONFLICT). + */ + if (IsA(plan, SubqueryScan)) + dpns->inner_plan = ((SubqueryScan *) plan)->subplan; + else if (IsA(plan, CteScan)) + dpns->inner_plan = list_nth(dpns->subplans, + ((CteScan *) plan)->ctePlanId - 1); + else if (IsA(plan, ModifyTable)) + dpns->inner_plan = plan; + else + dpns->inner_plan = innerPlan(plan); + + if (IsA(plan, ModifyTable)) + dpns->inner_tlist = ((ModifyTable *) plan)->exclRelTlist; + else if (dpns->inner_plan) + dpns->inner_tlist = dpns->inner_plan->targetlist; + else + dpns->inner_tlist = NIL; + + /* Set up referent for INDEX_VAR Vars, if needed */ + if (IsA(plan, IndexOnlyScan)) + dpns->index_tlist = ((IndexOnlyScan *) plan)->indextlist; + else if (IsA(plan, ForeignScan)) + dpns->index_tlist = ((ForeignScan *) plan)->fdw_scan_tlist; + else if (IsA(plan, CustomScan)) + dpns->index_tlist = ((CustomScan *) plan)->custom_scan_tlist; + else + dpns->index_tlist = NIL; +} + +/* + * push_child_plan: temporarily transfer deparsing attention to a child plan + * + * When expanding an OUTER_VAR or INNER_VAR reference, we must adjust the + * deparse context in case the referenced expression itself uses + * OUTER_VAR/INNER_VAR. We modify the top stack entry in-place to avoid + * affecting levelsup issues (although in a Plan tree there really shouldn't + * be any). + * + * Caller must provide a local deparse_namespace variable to save the + * previous state for pop_child_plan. + */ +static void +push_child_plan(deparse_namespace *dpns, Plan *plan, + deparse_namespace *save_dpns) +{ + /* Save state for restoration later */ + *save_dpns = *dpns; + + /* Link current plan node into ancestors list */ + dpns->ancestors = lcons(dpns->plan, dpns->ancestors); + + /* Set attention on selected child */ + set_deparse_plan(dpns, plan); +} + +/* + * pop_child_plan: undo the effects of push_child_plan + */ +static void +pop_child_plan(deparse_namespace *dpns, deparse_namespace *save_dpns) +{ + List *ancestors; + + /* Get rid of ancestors list cell added by push_child_plan */ + ancestors = list_delete_first(dpns->ancestors); + + /* Restore fields changed by push_child_plan */ + *dpns = *save_dpns; + + /* Make sure dpns->ancestors is right (may be unnecessary) */ + dpns->ancestors = ancestors; +} + +/* + * push_ancestor_plan: temporarily transfer deparsing attention to an + * ancestor plan + * + * When expanding a Param reference, we must adjust the deparse context + * to match the plan node that contains the expression being printed; + * otherwise we'd fail if that expression itself contains a Param or + * OUTER_VAR/INNER_VAR/INDEX_VAR variable. + * + * The target ancestor is conveniently identified by the ListCell holding it + * in dpns->ancestors. + * + * Caller must provide a local deparse_namespace variable to save the + * previous state for pop_ancestor_plan. + */ +static void +push_ancestor_plan(deparse_namespace *dpns, ListCell *ancestor_cell, + deparse_namespace *save_dpns) +{ + Plan *plan = (Plan *) lfirst(ancestor_cell); + + /* Save state for restoration later */ + *save_dpns = *dpns; + + /* Build a new ancestor list with just this node's ancestors */ + dpns->ancestors = + list_copy_tail(dpns->ancestors, + list_cell_number(dpns->ancestors, ancestor_cell) + 1); + + /* Set attention on selected ancestor */ + set_deparse_plan(dpns, plan); +} + +/* + * pop_ancestor_plan: undo the effects of push_ancestor_plan + */ +static void +pop_ancestor_plan(deparse_namespace *dpns, deparse_namespace *save_dpns) +{ + /* Free the ancestor list made in push_ancestor_plan */ + list_free(dpns->ancestors); + + /* Restore fields changed by push_ancestor_plan */ + *dpns = *save_dpns; +} + + +/* ---------- + * deparse_shard_query - Parse back a query for execution on a shard + * + * Builds an SQL string to perform the provided query on a specific shard and + * places this string into the provided buffer. + * ---------- + */ +void +deparse_shard_query(Query *query, Oid distrelid, int64 shardid, + StringInfo buffer) +{ + get_query_def_extended(query, buffer, NIL, distrelid, shardid, NULL, 0, + WRAP_COLUMN_DEFAULT, 0); +} + + +/* ---------- + * get_query_def - Parse back one query parsetree + * + * If resultDesc is not NULL, then it is the output tuple descriptor for + * the view represented by a SELECT query. + * ---------- + */ +static void +get_query_def(Query *query, StringInfo buf, List *parentnamespace, + TupleDesc resultDesc, + int prettyFlags, int wrapColumn, int startIndent) +{ + get_query_def_extended(query, buf, parentnamespace, InvalidOid, 0, resultDesc, + prettyFlags, wrapColumn, startIndent); +} + + +/* ---------- + * get_query_def_extended - Parse back one query parsetree, optionally + * with extension using a shard identifier. + * + * If distrelid is valid and shardid is positive, the provided shardid is added + * any time the provided relid is deparsed, so that the query may be executed + * on a placement for the given shard. + * ---------- + */ +static void +get_query_def_extended(Query *query, StringInfo buf, List *parentnamespace, + Oid distrelid, int64 shardid, TupleDesc resultDesc, + int prettyFlags, int wrapColumn, int startIndent) +{ + deparse_context context; + deparse_namespace dpns; + + OverrideSearchPath *overridePath = NULL; + + /* Guard against excessively long or deeply-nested queries */ + CHECK_FOR_INTERRUPTS(); + check_stack_depth(); + + /* + * Before we begin to examine the query, acquire locks on referenced + * relations, and fix up deleted columns in JOIN RTEs. This ensures + * consistent results. Note we assume it's OK to scribble on the passed + * querytree! + * + * We are only deparsing the query (we are not about to execute it), so we + * only need AccessShareLock on the relations it mentions. + */ + AcquireRewriteLocks(query, false, false); + + /* + * Set search_path to NIL so that all objects outside of pg_catalog will be + * schema-prefixed. pg_catalog will be added automatically when we call + * PushOverrideSearchPath(), since we set addCatalog to true; + */ + overridePath = GetOverrideSearchPath(CurrentMemoryContext); + overridePath->schemas = NIL; + overridePath->addCatalog = true; + PushOverrideSearchPath(overridePath); + + context.buf = buf; + context.namespaces = lcons(&dpns, list_copy(parentnamespace)); + context.windowClause = NIL; + context.windowTList = NIL; + context.varprefix = (parentnamespace != NIL || + list_length(query->rtable) != 1); + context.prettyFlags = prettyFlags; + context.wrapColumn = wrapColumn; + context.indentLevel = startIndent; + context.special_exprkind = EXPR_KIND_NONE; + context.appendparents = NULL; + context.distrelid = distrelid; + context.shardid = shardid; + + set_deparse_for_query(&dpns, query, parentnamespace); + + switch (query->commandType) + { + case CMD_SELECT: + get_select_query_def(query, &context, resultDesc); + break; + + case CMD_UPDATE: + get_update_query_def(query, &context); + break; + + case CMD_INSERT: + get_insert_query_def(query, &context); + break; + + case CMD_DELETE: + get_delete_query_def(query, &context); + break; + + case CMD_NOTHING: + appendStringInfoString(buf, "NOTHING"); + break; + + case CMD_UTILITY: + get_utility_query_def(query, &context); + break; + + default: + elog(ERROR, "unrecognized query command type: %d", + query->commandType); + break; + } + + /* revert back to original search_path */ + PopOverrideSearchPath(); +} + +/* ---------- + * get_values_def - Parse back a VALUES list + * ---------- + */ +static void +get_values_def(List *values_lists, deparse_context *context) +{ + StringInfo buf = context->buf; + bool first_list = true; + ListCell *vtl; + + appendStringInfoString(buf, "VALUES "); + + foreach(vtl, values_lists) + { + List *sublist = (List *) lfirst(vtl); + bool first_col = true; + ListCell *lc; + + if (first_list) + first_list = false; + else + appendStringInfoString(buf, ", "); + + appendStringInfoChar(buf, '('); + foreach(lc, sublist) + { + Node *col = (Node *) lfirst(lc); + + if (first_col) + first_col = false; + else + appendStringInfoChar(buf, ','); + + /* + * Print the value. Whole-row Vars need special treatment. + */ + get_rule_expr_toplevel(col, context, false); + } + appendStringInfoChar(buf, ')'); + } +} + +/* ---------- + * get_with_clause - Parse back a WITH clause + * ---------- + */ +static void +get_with_clause(Query *query, deparse_context *context) +{ + StringInfo buf = context->buf; + const char *sep; + ListCell *l; + + if (query->cteList == NIL) + return; + + if (PRETTY_INDENT(context)) + { + context->indentLevel += PRETTYINDENT_STD; + appendStringInfoChar(buf, ' '); + } + + if (query->hasRecursive) + sep = "WITH RECURSIVE "; + else + sep = "WITH "; + foreach(l, query->cteList) + { + CommonTableExpr *cte = (CommonTableExpr *) lfirst(l); + + appendStringInfoString(buf, sep); + appendStringInfoString(buf, quote_identifier(cte->ctename)); + if (cte->aliascolnames) + { + bool first = true; + ListCell *col; + + appendStringInfoChar(buf, '('); + foreach(col, cte->aliascolnames) + { + if (first) + first = false; + else + appendStringInfoString(buf, ", "); + appendStringInfoString(buf, + quote_identifier(strVal(lfirst(col)))); + } + appendStringInfoChar(buf, ')'); + } + appendStringInfoString(buf, " AS "); + switch (cte->ctematerialized) + { + case CTEMaterializeDefault: + break; + case CTEMaterializeAlways: + appendStringInfoString(buf, "MATERIALIZED "); + break; + case CTEMaterializeNever: + appendStringInfoString(buf, "NOT MATERIALIZED "); + break; + } + appendStringInfoChar(buf, '('); + if (PRETTY_INDENT(context)) + appendContextKeyword(context, "", 0, 0, 0); + get_query_def((Query *) cte->ctequery, buf, context->namespaces, NULL, + context->prettyFlags, context->wrapColumn, + context->indentLevel); + if (PRETTY_INDENT(context)) + appendContextKeyword(context, "", 0, 0, 0); + appendStringInfoChar(buf, ')'); + sep = ", "; + } + + if (PRETTY_INDENT(context)) + { + context->indentLevel -= PRETTYINDENT_STD; + appendContextKeyword(context, "", 0, 0, 0); + } + else + appendStringInfoChar(buf, ' '); +} + +/* ---------- + * get_select_query_def - Parse back a SELECT parsetree + * ---------- + */ +static void +get_select_query_def(Query *query, deparse_context *context, + TupleDesc resultDesc) +{ + StringInfo buf = context->buf; + List *save_windowclause; + List *save_windowtlist; + bool force_colno; + ListCell *l; + + /* Insert the WITH clause if given */ + get_with_clause(query, context); + + /* Set up context for possible window functions */ + save_windowclause = context->windowClause; + context->windowClause = query->windowClause; + save_windowtlist = context->windowTList; + context->windowTList = query->targetList; + + /* + * If the Query node has a setOperations tree, then it's the top level of + * a UNION/INTERSECT/EXCEPT query; only the WITH, ORDER BY and LIMIT + * fields are interesting in the top query itself. + */ + if (query->setOperations) + { + get_setop_query(query->setOperations, query, context, resultDesc); + /* ORDER BY clauses must be simple in this case */ + force_colno = true; + } + else + { + get_basic_select_query(query, context, resultDesc); + force_colno = false; + } + + /* Add the ORDER BY clause if given */ + if (query->sortClause != NIL) + { + appendContextKeyword(context, " ORDER BY ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_rule_orderby(query->sortClause, query->targetList, + force_colno, context); + } + + /* + * Add the LIMIT/OFFSET clauses if given. If non-default options, use the + * standard spelling of LIMIT. + */ + if (query->limitOffset != NULL) + { + appendContextKeyword(context, " OFFSET ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); + get_rule_expr(query->limitOffset, context, false); + } + if (query->limitCount != NULL) + { + if (query->limitOption == LIMIT_OPTION_WITH_TIES) + { + // had to add '(' and ')' here because it fails with casting + appendContextKeyword(context, " FETCH FIRST (", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); + get_rule_expr(query->limitCount, context, false); + appendStringInfo(buf, ") ROWS WITH TIES"); + } + else + { + appendContextKeyword(context, " LIMIT ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); + if (IsA(query->limitCount, Const) && + ((Const *) query->limitCount)->constisnull) + appendStringInfoString(buf, "ALL"); + else + get_rule_expr(query->limitCount, context, false); + } + } + + /* Add FOR [KEY] UPDATE/SHARE clauses if present */ + if (query->hasForUpdate) + { + foreach(l, query->rowMarks) + { + RowMarkClause *rc = (RowMarkClause *) lfirst(l); + + /* don't print implicit clauses */ + if (rc->pushedDown) + continue; + + switch (rc->strength) + { + case LCS_NONE: + /* we intentionally throw an error for LCS_NONE */ + elog(ERROR, "unrecognized LockClauseStrength %d", + (int) rc->strength); + break; + case LCS_FORKEYSHARE: + appendContextKeyword(context, " FOR KEY SHARE", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); + break; + case LCS_FORSHARE: + appendContextKeyword(context, " FOR SHARE", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); + break; + case LCS_FORNOKEYUPDATE: + appendContextKeyword(context, " FOR NO KEY UPDATE", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); + break; + case LCS_FORUPDATE: + appendContextKeyword(context, " FOR UPDATE", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); + break; + } + + appendStringInfo(buf, " OF %s", + quote_identifier(get_rtable_name(rc->rti, + context))); + if (rc->waitPolicy == LockWaitError) + appendStringInfoString(buf, " NOWAIT"); + else if (rc->waitPolicy == LockWaitSkip) + appendStringInfoString(buf, " SKIP LOCKED"); + } + } + + context->windowClause = save_windowclause; + context->windowTList = save_windowtlist; +} + +/* + * Detect whether query looks like SELECT ... FROM VALUES(); + * if so, return the VALUES RTE. Otherwise return NULL. + */ +static RangeTblEntry * +get_simple_values_rte(Query *query, TupleDesc resultDesc) +{ + RangeTblEntry *result = NULL; + ListCell *lc; + int colno; + + /* + * We want to return true even if the Query also contains OLD or NEW rule + * RTEs. So the idea is to scan the rtable and see if there is only one + * inFromCl RTE that is a VALUES RTE. + */ + foreach(lc, query->rtable) + { + RangeTblEntry *rte = (RangeTblEntry *) lfirst(lc); + + if (rte->rtekind == RTE_VALUES && rte->inFromCl) + { + if (result) + return NULL; /* multiple VALUES (probably not possible) */ + result = rte; + } + else if (rte->rtekind == RTE_RELATION && !rte->inFromCl) + continue; /* ignore rule entries */ + else + return NULL; /* something else -> not simple VALUES */ + } + + /* + * We don't need to check the targetlist in any great detail, because + * parser/analyze.c will never generate a "bare" VALUES RTE --- they only + * appear inside auto-generated sub-queries with very restricted + * structure. However, DefineView might have modified the tlist by + * injecting new column aliases; so compare tlist resnames against the + * RTE's names to detect that. + */ + if (result) + { + ListCell *lcn; + + if (list_length(query->targetList) != list_length(result->eref->colnames)) + return NULL; /* this probably cannot happen */ + colno = 0; + forboth(lc, query->targetList, lcn, result->eref->colnames) + { + TargetEntry *tle = (TargetEntry *) lfirst(lc); + char *cname = strVal(lfirst(lcn)); + char *colname; + + if (tle->resjunk) + return NULL; /* this probably cannot happen */ + /* compute name that get_target_list would use for column */ + colno++; + if (resultDesc && colno <= resultDesc->natts) + colname = NameStr(TupleDescAttr(resultDesc, colno - 1)->attname); + else + colname = tle->resname; + + /* does it match the VALUES RTE? */ + if (colname == NULL || strcmp(colname, cname) != 0) + return NULL; /* column name has been changed */ + } + } + + return result; +} + +static void +get_basic_select_query(Query *query, deparse_context *context, + TupleDesc resultDesc) +{ + StringInfo buf = context->buf; + RangeTblEntry *values_rte; + char *sep; + ListCell *l; + + if (PRETTY_INDENT(context)) + { + context->indentLevel += PRETTYINDENT_STD; + appendStringInfoChar(buf, ' '); + } + + /* + * If the query looks like SELECT * FROM (VALUES ...), then print just the + * VALUES part. This reverses what transformValuesClause() did at parse + * time. + */ + values_rte = get_simple_values_rte(query, resultDesc); + if (values_rte) + { + get_values_def(values_rte->values_lists, context); + return; + } + + /* + * Build up the query string - first we say SELECT + */ + appendStringInfoString(buf, "SELECT"); + + /* Add the DISTINCT clause if given */ + if (query->distinctClause != NIL) + { + if (query->hasDistinctOn) + { + appendStringInfoString(buf, " DISTINCT ON ("); + sep = ""; + foreach(l, query->distinctClause) + { + SortGroupClause *srt = (SortGroupClause *) lfirst(l); + + appendStringInfoString(buf, sep); + get_rule_sortgroupclause(srt->tleSortGroupRef, query->targetList, + false, context); + sep = ", "; + } + appendStringInfoChar(buf, ')'); + } + else + appendStringInfoString(buf, " DISTINCT"); + } + + /* Then we tell what to select (the targetlist) */ + get_target_list(query->targetList, context, resultDesc); + + /* Add the FROM clause if needed */ + get_from_clause(query, " FROM ", context); + + /* Add the WHERE clause if given */ + if (query->jointree->quals != NULL) + { + appendContextKeyword(context, " WHERE ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_rule_expr(query->jointree->quals, context, false); + } + + /* Add the GROUP BY clause if given */ + if (query->groupClause != NULL || query->groupingSets != NULL) + { + ParseExprKind save_exprkind; + + appendContextKeyword(context, " GROUP BY ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + + save_exprkind = context->special_exprkind; + context->special_exprkind = EXPR_KIND_GROUP_BY; + + if (query->groupingSets == NIL) + { + sep = ""; + foreach(l, query->groupClause) + { + SortGroupClause *grp = (SortGroupClause *) lfirst(l); + + appendStringInfoString(buf, sep); + get_rule_sortgroupclause(grp->tleSortGroupRef, query->targetList, + false, context); + sep = ", "; + } + } + else + { + sep = ""; + foreach(l, query->groupingSets) + { + GroupingSet *grp = lfirst(l); + + appendStringInfoString(buf, sep); + get_rule_groupingset(grp, query->targetList, true, context); + sep = ", "; + } + } + + context->special_exprkind = save_exprkind; + } + + /* Add the HAVING clause if given */ + if (query->havingQual != NULL) + { + appendContextKeyword(context, " HAVING ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); + get_rule_expr(query->havingQual, context, false); + } + + /* Add the WINDOW clause if needed */ + if (query->windowClause != NIL) + get_rule_windowclause(query, context); +} + +/* ---------- + * get_target_list - Parse back a SELECT target list + * + * This is also used for RETURNING lists in INSERT/UPDATE/DELETE. + * ---------- + */ +static void +get_target_list(List *targetList, deparse_context *context, + TupleDesc resultDesc) +{ + StringInfo buf = context->buf; + StringInfoData targetbuf; + bool last_was_multiline = false; + char *sep; + int colno; + ListCell *l; + + /* we use targetbuf to hold each TLE's text temporarily */ + initStringInfo(&targetbuf); + + sep = " "; + colno = 0; + foreach(l, targetList) + { + TargetEntry *tle = (TargetEntry *) lfirst(l); + char *colname; + char *attname; + + if (tle->resjunk) + continue; /* ignore junk entries */ + + appendStringInfoString(buf, sep); + sep = ", "; + colno++; + + /* + * Put the new field text into targetbuf so we can decide after we've + * got it whether or not it needs to go on a new line. + */ + resetStringInfo(&targetbuf); + context->buf = &targetbuf; + + /* + * We special-case Var nodes rather than using get_rule_expr. This is + * needed because get_rule_expr will display a whole-row Var as + * "foo.*", which is the preferred notation in most contexts, but at + * the top level of a SELECT list it's not right (the parser will + * expand that notation into multiple columns, yielding behavior + * different from a whole-row Var). We need to call get_variable + * directly so that we can tell it to do the right thing, and so that + * we can get the attribute name which is the default AS label. + */ + if (tle->expr && (IsA(tle->expr, Var))) + { + attname = get_variable((Var *) tle->expr, 0, true, context); + } + else + { + get_rule_expr((Node *) tle->expr, context, true); + /* We'll show the AS name unless it's this: */ + attname = "?column?"; + } + + /* + * Figure out what the result column should be called. In the context + * of a view, use the view's tuple descriptor (so as to pick up the + * effects of any column RENAME that's been done on the view). + * Otherwise, just use what we can find in the TLE. + */ + if (resultDesc && colno <= resultDesc->natts) + colname = NameStr(TupleDescAttr(resultDesc, colno - 1)->attname); + else + colname = tle->resname; + + /* Show AS unless the column's name is correct as-is */ + if (colname) /* resname could be NULL */ + { + if (attname == NULL || strcmp(attname, colname) != 0) + appendStringInfo(&targetbuf, " AS %s", quote_identifier(colname)); + } + + /* Restore context's output buffer */ + context->buf = buf; + + /* Consider line-wrapping if enabled */ + if (PRETTY_INDENT(context) && context->wrapColumn >= 0) + { + int leading_nl_pos; + + /* Does the new field start with a new line? */ + if (targetbuf.len > 0 && targetbuf.data[0] == '\n') + leading_nl_pos = 0; + else + leading_nl_pos = -1; + + /* If so, we shouldn't add anything */ + if (leading_nl_pos >= 0) + { + /* instead, remove any trailing spaces currently in buf */ + removeStringInfoSpaces(buf); + } + else + { + char *trailing_nl; + + /* Locate the start of the current line in the output buffer */ + trailing_nl = strrchr(buf->data, '\n'); + if (trailing_nl == NULL) + trailing_nl = buf->data; + else + trailing_nl++; + + /* + * Add a newline, plus some indentation, if the new field is + * not the first and either the new field would cause an + * overflow or the last field used more than one line. + */ + if (colno > 1 && + ((strlen(trailing_nl) + targetbuf.len > context->wrapColumn) || + last_was_multiline)) + appendContextKeyword(context, "", -PRETTYINDENT_STD, + PRETTYINDENT_STD, PRETTYINDENT_VAR); + } + + /* Remember this field's multiline status for next iteration */ + last_was_multiline = + (strchr(targetbuf.data + leading_nl_pos + 1, '\n') != NULL); + } + + /* Add the new field */ + appendStringInfoString(buf, targetbuf.data); + } + + /* clean up */ + pfree(targetbuf.data); +} + +static void +get_setop_query(Node *setOp, Query *query, deparse_context *context, + TupleDesc resultDesc) +{ + StringInfo buf = context->buf; + bool need_paren; + + /* Guard against excessively long or deeply-nested queries */ + CHECK_FOR_INTERRUPTS(); + check_stack_depth(); + + if (IsA(setOp, RangeTblRef)) + { + RangeTblRef *rtr = (RangeTblRef *) setOp; + RangeTblEntry *rte = rt_fetch(rtr->rtindex, query->rtable); + Query *subquery = rte->subquery; + + Assert(subquery != NULL); + Assert(subquery->setOperations == NULL); + /* Need parens if WITH, ORDER BY, FOR UPDATE, or LIMIT; see gram.y */ + need_paren = (subquery->cteList || + subquery->sortClause || + subquery->rowMarks || + subquery->limitOffset || + subquery->limitCount); + if (need_paren) + appendStringInfoChar(buf, '('); + get_query_def(subquery, buf, context->namespaces, resultDesc, + context->prettyFlags, context->wrapColumn, + context->indentLevel); + if (need_paren) + appendStringInfoChar(buf, ')'); + } + else if (IsA(setOp, SetOperationStmt)) + { + SetOperationStmt *op = (SetOperationStmt *) setOp; + int subindent; + + /* + * We force parens when nesting two SetOperationStmts, except when the + * lefthand input is another setop of the same kind. Syntactically, + * we could omit parens in rather more cases, but it seems best to use + * parens to flag cases where the setop operator changes. If we use + * parens, we also increase the indentation level for the child query. + * + * There are some cases in which parens are needed around a leaf query + * too, but those are more easily handled at the next level down (see + * code above). + */ + if (IsA(op->larg, SetOperationStmt)) + { + SetOperationStmt *lop = (SetOperationStmt *) op->larg; + + if (op->op == lop->op && op->all == lop->all) + need_paren = false; + else + need_paren = true; + } + else + need_paren = false; + + if (need_paren) + { + appendStringInfoChar(buf, '('); + subindent = PRETTYINDENT_STD; + appendContextKeyword(context, "", subindent, 0, 0); + } + else + subindent = 0; + + get_setop_query(op->larg, query, context, resultDesc); + + if (need_paren) + appendContextKeyword(context, ") ", -subindent, 0, 0); + else if (PRETTY_INDENT(context)) + appendContextKeyword(context, "", -subindent, 0, 0); + else + appendStringInfoChar(buf, ' '); + + switch (op->op) + { + case SETOP_UNION: + appendStringInfoString(buf, "UNION "); + break; + case SETOP_INTERSECT: + appendStringInfoString(buf, "INTERSECT "); + break; + case SETOP_EXCEPT: + appendStringInfoString(buf, "EXCEPT "); + break; + default: + elog(ERROR, "unrecognized set op: %d", + (int) op->op); + } + if (op->all) + appendStringInfoString(buf, "ALL "); + + /* Always parenthesize if RHS is another setop */ + need_paren = IsA(op->rarg, SetOperationStmt); + + /* + * The indentation code here is deliberately a bit different from that + * for the lefthand input, because we want the line breaks in + * different places. + */ + if (need_paren) + { + appendStringInfoChar(buf, '('); + subindent = PRETTYINDENT_STD; + } + else + subindent = 0; + appendContextKeyword(context, "", subindent, 0, 0); + + get_setop_query(op->rarg, query, context, resultDesc); + + if (PRETTY_INDENT(context)) + context->indentLevel -= subindent; + if (need_paren) + appendContextKeyword(context, ")", 0, 0, 0); + } + else + { + elog(ERROR, "unrecognized node type: %d", + (int) nodeTag(setOp)); + } +} + +/* + * Display a sort/group clause. + * + * Also returns the expression tree, so caller need not find it again. + */ +static Node * +get_rule_sortgroupclause(Index ref, List *tlist, bool force_colno, + deparse_context *context) +{ + StringInfo buf = context->buf; + TargetEntry *tle; + Node *expr; + + tle = get_sortgroupref_tle(ref, tlist); + expr = (Node *) tle->expr; + + /* + * Use column-number form if requested by caller. Otherwise, if + * expression is a constant, force it to be dumped with an explicit cast + * as decoration --- this is because a simple integer constant is + * ambiguous (and will be misinterpreted by findTargetlistEntry()) if we + * dump it without any decoration. If it's anything more complex than a + * simple Var, then force extra parens around it, to ensure it can't be + * misinterpreted as a cube() or rollup() construct. + */ + if (force_colno) + { + Assert(!tle->resjunk); + appendStringInfo(buf, "%d", tle->resno); + } + else if (expr && IsA(expr, Const)) + get_const_expr((Const *) expr, context, 1); + else if (!expr || IsA(expr, Var)) + get_rule_expr(expr, context, true); + else + { + /* + * We must force parens for function-like expressions even if + * PRETTY_PAREN is off, since those are the ones in danger of + * misparsing. For other expressions we need to force them only if + * PRETTY_PAREN is on, since otherwise the expression will output them + * itself. (We can't skip the parens.) + */ + bool need_paren = (PRETTY_PAREN(context) + || IsA(expr, FuncExpr) + ||IsA(expr, Aggref) + ||IsA(expr, WindowFunc)); + + if (need_paren) + appendStringInfoChar(context->buf, '('); + get_rule_expr(expr, context, true); + if (need_paren) + appendStringInfoChar(context->buf, ')'); + } + + return expr; +} + +/* + * Display a GroupingSet + */ +static void +get_rule_groupingset(GroupingSet *gset, List *targetlist, + bool omit_parens, deparse_context *context) +{ + ListCell *l; + StringInfo buf = context->buf; + bool omit_child_parens = true; + char *sep = ""; + + switch (gset->kind) + { + case GROUPING_SET_EMPTY: + appendStringInfoString(buf, "()"); + return; + + case GROUPING_SET_SIMPLE: + { + if (!omit_parens || list_length(gset->content) != 1) + appendStringInfoChar(buf, '('); + + foreach(l, gset->content) + { + Index ref = lfirst_int(l); + + appendStringInfoString(buf, sep); + get_rule_sortgroupclause(ref, targetlist, + false, context); + sep = ", "; + } + + if (!omit_parens || list_length(gset->content) != 1) + appendStringInfoChar(buf, ')'); + } + return; + + case GROUPING_SET_ROLLUP: + appendStringInfoString(buf, "ROLLUP("); + break; + case GROUPING_SET_CUBE: + appendStringInfoString(buf, "CUBE("); + break; + case GROUPING_SET_SETS: + appendStringInfoString(buf, "GROUPING SETS ("); + omit_child_parens = false; + break; + } + + foreach(l, gset->content) + { + appendStringInfoString(buf, sep); + get_rule_groupingset(lfirst(l), targetlist, omit_child_parens, context); + sep = ", "; + } + + appendStringInfoChar(buf, ')'); +} + +/* + * Display an ORDER BY list. + */ +static void +get_rule_orderby(List *orderList, List *targetList, + bool force_colno, deparse_context *context) +{ + StringInfo buf = context->buf; + const char *sep; + ListCell *l; + + sep = ""; + foreach(l, orderList) + { + SortGroupClause *srt = (SortGroupClause *) lfirst(l); + Node *sortexpr; + Oid sortcoltype; + TypeCacheEntry *typentry; + + appendStringInfoString(buf, sep); + sortexpr = get_rule_sortgroupclause(srt->tleSortGroupRef, targetList, + force_colno, context); + sortcoltype = exprType(sortexpr); + /* See whether operator is default < or > for datatype */ + typentry = lookup_type_cache(sortcoltype, + TYPECACHE_LT_OPR | TYPECACHE_GT_OPR); + if (srt->sortop == typentry->lt_opr) + { + /* ASC is default, so emit nothing for it */ + if (srt->nulls_first) + appendStringInfoString(buf, " NULLS FIRST"); + } + else if (srt->sortop == typentry->gt_opr) + { + appendStringInfoString(buf, " DESC"); + /* DESC defaults to NULLS FIRST */ + if (!srt->nulls_first) + appendStringInfoString(buf, " NULLS LAST"); + } + else + { + appendStringInfo(buf, " USING %s", + generate_operator_name(srt->sortop, + sortcoltype, + sortcoltype)); + /* be specific to eliminate ambiguity */ + if (srt->nulls_first) + appendStringInfoString(buf, " NULLS FIRST"); + else + appendStringInfoString(buf, " NULLS LAST"); + } + sep = ", "; + } +} + +/* + * Display a WINDOW clause. + * + * Note that the windowClause list might contain only anonymous window + * specifications, in which case we should print nothing here. + */ +static void +get_rule_windowclause(Query *query, deparse_context *context) +{ + StringInfo buf = context->buf; + const char *sep; + ListCell *l; + + sep = NULL; + foreach(l, query->windowClause) + { + WindowClause *wc = (WindowClause *) lfirst(l); + + if (wc->name == NULL) + continue; /* ignore anonymous windows */ + + if (sep == NULL) + appendContextKeyword(context, " WINDOW ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + else + appendStringInfoString(buf, sep); + + appendStringInfo(buf, "%s AS ", quote_identifier(wc->name)); + + get_rule_windowspec(wc, query->targetList, context); + + sep = ", "; + } +} + +/* + * Display a window definition + */ +static void +get_rule_windowspec(WindowClause *wc, List *targetList, + deparse_context *context) +{ + StringInfo buf = context->buf; + bool needspace = false; + const char *sep; + ListCell *l; + + appendStringInfoChar(buf, '('); + if (wc->refname) + { + appendStringInfoString(buf, quote_identifier(wc->refname)); + needspace = true; + } + /* partition clauses are always inherited, so only print if no refname */ + if (wc->partitionClause && !wc->refname) + { + if (needspace) + appendStringInfoChar(buf, ' '); + appendStringInfoString(buf, "PARTITION BY "); + sep = ""; + foreach(l, wc->partitionClause) + { + SortGroupClause *grp = (SortGroupClause *) lfirst(l); + + appendStringInfoString(buf, sep); + get_rule_sortgroupclause(grp->tleSortGroupRef, targetList, + false, context); + sep = ", "; + } + needspace = true; + } + /* print ordering clause only if not inherited */ + if (wc->orderClause && !wc->copiedOrder) + { + if (needspace) + appendStringInfoChar(buf, ' '); + appendStringInfoString(buf, "ORDER BY "); + get_rule_orderby(wc->orderClause, targetList, false, context); + needspace = true; + } + /* framing clause is never inherited, so print unless it's default */ + if (wc->frameOptions & FRAMEOPTION_NONDEFAULT) + { + if (needspace) + appendStringInfoChar(buf, ' '); + if (wc->frameOptions & FRAMEOPTION_RANGE) + appendStringInfoString(buf, "RANGE "); + else if (wc->frameOptions & FRAMEOPTION_ROWS) + appendStringInfoString(buf, "ROWS "); + else if (wc->frameOptions & FRAMEOPTION_GROUPS) + appendStringInfoString(buf, "GROUPS "); + else + Assert(false); + if (wc->frameOptions & FRAMEOPTION_BETWEEN) + appendStringInfoString(buf, "BETWEEN "); + if (wc->frameOptions & FRAMEOPTION_START_UNBOUNDED_PRECEDING) + appendStringInfoString(buf, "UNBOUNDED PRECEDING "); + else if (wc->frameOptions & FRAMEOPTION_START_CURRENT_ROW) + appendStringInfoString(buf, "CURRENT ROW "); + else if (wc->frameOptions & FRAMEOPTION_START_OFFSET) + { + get_rule_expr(wc->startOffset, context, false); + if (wc->frameOptions & FRAMEOPTION_START_OFFSET_PRECEDING) + appendStringInfoString(buf, " PRECEDING "); + else if (wc->frameOptions & FRAMEOPTION_START_OFFSET_FOLLOWING) + appendStringInfoString(buf, " FOLLOWING "); + else + Assert(false); + } + else + Assert(false); + if (wc->frameOptions & FRAMEOPTION_BETWEEN) + { + appendStringInfoString(buf, "AND "); + if (wc->frameOptions & FRAMEOPTION_END_UNBOUNDED_FOLLOWING) + appendStringInfoString(buf, "UNBOUNDED FOLLOWING "); + else if (wc->frameOptions & FRAMEOPTION_END_CURRENT_ROW) + appendStringInfoString(buf, "CURRENT ROW "); + else if (wc->frameOptions & FRAMEOPTION_END_OFFSET) + { + get_rule_expr(wc->endOffset, context, false); + if (wc->frameOptions & FRAMEOPTION_END_OFFSET_PRECEDING) + appendStringInfoString(buf, " PRECEDING "); + else if (wc->frameOptions & FRAMEOPTION_END_OFFSET_FOLLOWING) + appendStringInfoString(buf, " FOLLOWING "); + else + Assert(false); + } + else + Assert(false); + } + if (wc->frameOptions & FRAMEOPTION_EXCLUDE_CURRENT_ROW) + appendStringInfoString(buf, "EXCLUDE CURRENT ROW "); + else if (wc->frameOptions & FRAMEOPTION_EXCLUDE_GROUP) + appendStringInfoString(buf, "EXCLUDE GROUP "); + else if (wc->frameOptions & FRAMEOPTION_EXCLUDE_TIES) + appendStringInfoString(buf, "EXCLUDE TIES "); + /* we will now have a trailing space; remove it */ + buf->len--; + } + appendStringInfoChar(buf, ')'); +} + +/* ---------- + * get_insert_query_def - Parse back an INSERT parsetree + * ---------- + */ +static void +get_insert_query_def(Query *query, deparse_context *context) +{ + StringInfo buf = context->buf; + RangeTblEntry *select_rte = NULL; + RangeTblEntry *values_rte = NULL; + RangeTblEntry *rte; + char *sep; + ListCell *l; + List *strippedexprs; + + /* Insert the WITH clause if given */ + get_with_clause(query, context); + + /* + * If it's an INSERT ... SELECT or multi-row VALUES, there will be a + * single RTE for the SELECT or VALUES. Plain VALUES has neither. + */ + foreach(l, query->rtable) + { + rte = (RangeTblEntry *) lfirst(l); + + if (rte->rtekind == RTE_SUBQUERY) + { + if (select_rte) + elog(ERROR, "too many subquery RTEs in INSERT"); + select_rte = rte; + } + + if (rte->rtekind == RTE_VALUES) + { + if (values_rte) + elog(ERROR, "too many values RTEs in INSERT"); + values_rte = rte; + } + } + if (select_rte && values_rte) + elog(ERROR, "both subquery and values RTEs in INSERT"); + + /* + * Start the query with INSERT INTO relname + */ + rte = rt_fetch(query->resultRelation, query->rtable); + Assert(rte->rtekind == RTE_RELATION); + + if (PRETTY_INDENT(context)) + { + context->indentLevel += PRETTYINDENT_STD; + appendStringInfoChar(buf, ' '); + } + appendStringInfo(buf, "INSERT INTO %s ", + generate_relation_or_shard_name(rte->relid, + context->distrelid, + context->shardid, NIL)); + /* INSERT requires AS keyword for target alias */ + if (rte->alias != NULL) + appendStringInfo(buf, "AS %s ", + quote_identifier(get_rtable_name(query->resultRelation, context))); + + /* + * Add the insert-column-names list. Any indirection decoration needed on + * the column names can be inferred from the top targetlist. + */ + strippedexprs = NIL; + sep = ""; + if (query->targetList) + appendStringInfoChar(buf, '('); + foreach(l, query->targetList) + { + TargetEntry *tle = (TargetEntry *) lfirst(l); + + if (tle->resjunk) + continue; /* ignore junk entries */ + + appendStringInfoString(buf, sep); + sep = ", "; + + /* + * Put out name of target column; look in the catalogs, not at + * tle->resname, since resname will fail to track RENAME. + */ + appendStringInfoString(buf, + quote_identifier(get_attname(rte->relid, + tle->resno, + false))); + + /* + * Print any indirection needed (subfields or subscripts), and strip + * off the top-level nodes representing the indirection assignments. + * Add the stripped expressions to strippedexprs. (If it's a + * single-VALUES statement, the stripped expressions are the VALUES to + * print below. Otherwise they're just Vars and not really + * interesting.) + */ + strippedexprs = lappend(strippedexprs, + processIndirection((Node *) tle->expr, + context)); + } + if (query->targetList) + appendStringInfoString(buf, ") "); + + if (query->override) + { + if (query->override == OVERRIDING_SYSTEM_VALUE) + appendStringInfoString(buf, "OVERRIDING SYSTEM VALUE "); + else if (query->override == OVERRIDING_USER_VALUE) + appendStringInfoString(buf, "OVERRIDING USER VALUE "); + } + + if (select_rte) + { + /* Add the SELECT */ + get_query_def(select_rte->subquery, buf, NIL, NULL, + context->prettyFlags, context->wrapColumn, + context->indentLevel); + } + else if (values_rte) + { + /* Add the multi-VALUES expression lists */ + get_values_def(values_rte->values_lists, context); + } + else if (strippedexprs) + { + /* Add the single-VALUES expression list */ + appendContextKeyword(context, "VALUES (", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 2); + get_rule_expr((Node *) strippedexprs, context, false); + appendStringInfoChar(buf, ')'); + } + else + { + /* No expressions, so it must be DEFAULT VALUES */ + appendStringInfoString(buf, "DEFAULT VALUES"); + } + + /* Add ON CONFLICT if present */ + if (query->onConflict) + { + OnConflictExpr *confl = query->onConflict; + + appendStringInfoString(buf, " ON CONFLICT"); + + if (confl->arbiterElems) + { + /* Add the single-VALUES expression list */ + appendStringInfoChar(buf, '('); + get_rule_expr((Node *) confl->arbiterElems, context, false); + appendStringInfoChar(buf, ')'); + + /* Add a WHERE clause (for partial indexes) if given */ + if (confl->arbiterWhere != NULL) + { + bool save_varprefix; + + /* + * Force non-prefixing of Vars, since parser assumes that they + * belong to target relation. WHERE clause does not use + * InferenceElem, so this is separately required. + */ + save_varprefix = context->varprefix; + context->varprefix = false; + + appendContextKeyword(context, " WHERE ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_rule_expr(confl->arbiterWhere, context, false); + + context->varprefix = save_varprefix; + } + } + else if (OidIsValid(confl->constraint)) + { + char *constraint = get_constraint_name(confl->constraint); + int64 shardId = context->shardid; + + if (shardId > 0) + { + AppendShardIdToName(&constraint, shardId); + } + + if (!constraint) + elog(ERROR, "cache lookup failed for constraint %u", + confl->constraint); + appendStringInfo(buf, " ON CONSTRAINT %s", + quote_identifier(constraint)); + } + + if (confl->action == ONCONFLICT_NOTHING) + { + appendStringInfoString(buf, " DO NOTHING"); + } + else + { + appendStringInfoString(buf, " DO UPDATE SET "); + /* Deparse targetlist */ + get_update_query_targetlist_def(query, confl->onConflictSet, + context, rte); + + /* Add a WHERE clause if given */ + if (confl->onConflictWhere != NULL) + { + appendContextKeyword(context, " WHERE ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_rule_expr(confl->onConflictWhere, context, false); + } + } + } + + /* Add RETURNING if present */ + if (query->returningList) + { + appendContextKeyword(context, " RETURNING", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_target_list(query->returningList, context, NULL); + } +} + + +/* ---------- + * get_update_query_def - Parse back an UPDATE parsetree + * ---------- + */ +static void +get_update_query_def(Query *query, deparse_context *context) +{ + StringInfo buf = context->buf; + RangeTblEntry *rte; + + /* Insert the WITH clause if given */ + get_with_clause(query, context); + + /* + * Start the query with UPDATE relname SET + */ + rte = rt_fetch(query->resultRelation, query->rtable); + + if (PRETTY_INDENT(context)) + { + appendStringInfoChar(buf, ' '); + context->indentLevel += PRETTYINDENT_STD; + } + + /* if it's a shard, do differently */ + if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) + { + char *fragmentSchemaName = NULL; + char *fragmentTableName = NULL; + + ExtractRangeTblExtraData(rte, NULL, &fragmentSchemaName, &fragmentTableName, NULL); + + /* use schema and table name from the remote alias */ + appendStringInfo(buf, "UPDATE %s%s", + only_marker(rte), + generate_fragment_name(fragmentSchemaName, fragmentTableName)); + + if(rte->eref != NULL) + appendStringInfo(buf, " %s", + quote_identifier(get_rtable_name(query->resultRelation, context))); + } + else + { + appendStringInfo(buf, "UPDATE %s%s", + only_marker(rte), + generate_relation_or_shard_name(rte->relid, + context->distrelid, + context->shardid, NIL)); + + if (rte->alias != NULL) + appendStringInfo(buf, " %s", + quote_identifier(get_rtable_name(query->resultRelation, context))); + } + + appendStringInfoString(buf, " SET "); + + /* Deparse targetlist */ + get_update_query_targetlist_def(query, query->targetList, context, rte); + + /* Add the FROM clause if needed */ + get_from_clause(query, " FROM ", context); + + /* Add a WHERE clause if given */ + if (query->jointree->quals != NULL) + { + appendContextKeyword(context, " WHERE ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_rule_expr(query->jointree->quals, context, false); + } + + /* Add RETURNING if present */ + if (query->returningList) + { + appendContextKeyword(context, " RETURNING", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_target_list(query->returningList, context, NULL); + } +} + + +/* ---------- + * get_update_query_targetlist_def - Parse back an UPDATE targetlist + * ---------- + */ +static void +get_update_query_targetlist_def(Query *query, List *targetList, + deparse_context *context, RangeTblEntry *rte) +{ + StringInfo buf = context->buf; + ListCell *l; + ListCell *next_ma_cell; + int remaining_ma_columns; + const char *sep; + SubLink *cur_ma_sublink; + List *ma_sublinks; + + /* + * Prepare to deal with MULTIEXPR assignments: collect the source SubLinks + * into a list. We expect them to appear, in ID order, in resjunk tlist + * entries. + */ + ma_sublinks = NIL; + if (query->hasSubLinks) /* else there can't be any */ + { + foreach(l, targetList) + { + TargetEntry *tle = (TargetEntry *) lfirst(l); + + if (tle->resjunk && IsA(tle->expr, SubLink)) + { + SubLink *sl = (SubLink *) tle->expr; + + if (sl->subLinkType == MULTIEXPR_SUBLINK) + { + ma_sublinks = lappend(ma_sublinks, sl); + Assert(sl->subLinkId == list_length(ma_sublinks)); + } + } + } + } + next_ma_cell = list_head(ma_sublinks); + cur_ma_sublink = NULL; + remaining_ma_columns = 0; + + /* Add the comma separated list of 'attname = value' */ + sep = ""; + foreach(l, targetList) + { + TargetEntry *tle = (TargetEntry *) lfirst(l); + Node *expr; + + if (tle->resjunk) + continue; /* ignore junk entries */ + + /* Emit separator (OK whether we're in multiassignment or not) */ + appendStringInfoString(buf, sep); + sep = ", "; + + /* + * Check to see if we're starting a multiassignment group: if so, + * output a left paren. + */ + if (next_ma_cell != NULL && cur_ma_sublink == NULL) + { + /* + * We must dig down into the expr to see if it's a PARAM_MULTIEXPR + * Param. That could be buried under FieldStores and + * SubscriptingRefs and CoerceToDomains (cf processIndirection()), + * and underneath those there could be an implicit type coercion. + * Because we would ignore implicit type coercions anyway, we + * don't need to be as careful as processIndirection() is about + * descending past implicit CoerceToDomains. + */ + expr = (Node *) tle->expr; + while (expr) + { + if (IsA(expr, FieldStore)) + { + FieldStore *fstore = (FieldStore *) expr; + + expr = (Node *) linitial(fstore->newvals); + } + else if (IsA(expr, SubscriptingRef)) + { + SubscriptingRef *sbsref = (SubscriptingRef *) expr; + + if (sbsref->refassgnexpr == NULL) + break; + expr = (Node *) sbsref->refassgnexpr; + } + else if (IsA(expr, CoerceToDomain)) + { + CoerceToDomain *cdomain = (CoerceToDomain *) expr; + + if (cdomain->coercionformat != COERCE_IMPLICIT_CAST) + break; + expr = (Node *) cdomain->arg; + } + else + break; + } + expr = strip_implicit_coercions(expr); + + if (expr && IsA(expr, Param) && + ((Param *) expr)->paramkind == PARAM_MULTIEXPR) + { + cur_ma_sublink = (SubLink *) lfirst(next_ma_cell); + next_ma_cell = lnext(ma_sublinks, next_ma_cell); + remaining_ma_columns = count_nonjunk_tlist_entries( + ((Query *) cur_ma_sublink->subselect)->targetList); + Assert(((Param *) expr)->paramid == + ((cur_ma_sublink->subLinkId << 16) | 1)); + appendStringInfoChar(buf, '('); + } + } + + /* + * Put out name of target column; look in the catalogs, not at + * tle->resname, since resname will fail to track RENAME. + */ + appendStringInfoString(buf, + quote_identifier(get_attname(rte->relid, + tle->resno, + false))); + + /* + * Print any indirection needed (subfields or subscripts), and strip + * off the top-level nodes representing the indirection assignments. + */ + expr = processIndirection((Node *) tle->expr, context); + + /* + * If we're in a multiassignment, skip printing anything more, unless + * this is the last column; in which case, what we print should be the + * sublink, not the Param. + */ + if (cur_ma_sublink != NULL) + { + if (--remaining_ma_columns > 0) + continue; /* not the last column of multiassignment */ + appendStringInfoChar(buf, ')'); + expr = (Node *) cur_ma_sublink; + cur_ma_sublink = NULL; + } + + appendStringInfoString(buf, " = "); + + get_rule_expr(expr, context, false); + } +} + + +/* ---------- + * get_delete_query_def - Parse back a DELETE parsetree + * ---------- + */ +static void +get_delete_query_def(Query *query, deparse_context *context) +{ + StringInfo buf = context->buf; + RangeTblEntry *rte; + + /* Insert the WITH clause if given */ + get_with_clause(query, context); + + /* + * Start the query with DELETE FROM relname + */ + rte = rt_fetch(query->resultRelation, query->rtable); + + if (PRETTY_INDENT(context)) + { + appendStringInfoChar(buf, ' '); + context->indentLevel += PRETTYINDENT_STD; + } + + /* if it's a shard, do differently */ + if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) + { + char *fragmentSchemaName = NULL; + char *fragmentTableName = NULL; + + ExtractRangeTblExtraData(rte, NULL, &fragmentSchemaName, &fragmentTableName, NULL); + + /* use schema and table name from the remote alias */ + appendStringInfo(buf, "DELETE FROM %s%s", + only_marker(rte), + generate_fragment_name(fragmentSchemaName, fragmentTableName)); + + if(rte->eref != NULL) + appendStringInfo(buf, " %s", + quote_identifier(get_rtable_name(query->resultRelation, context))); + } + else + { + appendStringInfo(buf, "DELETE FROM %s%s", + only_marker(rte), + generate_relation_or_shard_name(rte->relid, + context->distrelid, + context->shardid, NIL)); + + if (rte->alias != NULL) + appendStringInfo(buf, " %s", + quote_identifier(get_rtable_name(query->resultRelation, context))); + } + + /* Add the USING clause if given */ + get_from_clause(query, " USING ", context); + + /* Add a WHERE clause if given */ + if (query->jointree->quals != NULL) + { + appendContextKeyword(context, " WHERE ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_rule_expr(query->jointree->quals, context, false); + } + + /* Add RETURNING if present */ + if (query->returningList) + { + appendContextKeyword(context, " RETURNING", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_target_list(query->returningList, context, NULL); + } +} + + +/* ---------- + * get_utility_query_def - Parse back a UTILITY parsetree + * ---------- + */ +static void +get_utility_query_def(Query *query, deparse_context *context) +{ + StringInfo buf = context->buf; + + if (query->utilityStmt && IsA(query->utilityStmt, NotifyStmt)) + { + NotifyStmt *stmt = (NotifyStmt *) query->utilityStmt; + + appendContextKeyword(context, "", + 0, PRETTYINDENT_STD, 1); + appendStringInfo(buf, "NOTIFY %s", + quote_identifier(stmt->conditionname)); + if (stmt->payload) + { + appendStringInfoString(buf, ", "); + simple_quote_literal(buf, stmt->payload); + } + } + else if (query->utilityStmt && IsA(query->utilityStmt, TruncateStmt)) + { + TruncateStmt *stmt = (TruncateStmt *) query->utilityStmt; + List *relationList = stmt->relations; + ListCell *relationCell = NULL; + + appendContextKeyword(context, "", + 0, PRETTYINDENT_STD, 1); + + appendStringInfo(buf, "TRUNCATE TABLE"); + + foreach(relationCell, relationList) + { + RangeVar *relationVar = (RangeVar *) lfirst(relationCell); + Oid relationId = RangeVarGetRelid(relationVar, NoLock, false); + char *relationName = generate_relation_or_shard_name(relationId, + context->distrelid, + context->shardid, NIL); + appendStringInfo(buf, " %s", relationName); + + if (lnext(relationList, relationCell) != NULL) + { + appendStringInfo(buf, ","); + } + } + + if (stmt->restart_seqs) + { + appendStringInfo(buf, " RESTART IDENTITY"); + } + + if (stmt->behavior == DROP_CASCADE) + { + appendStringInfo(buf, " CASCADE"); + } + } + else + { + /* Currently only NOTIFY utility commands can appear in rules */ + elog(ERROR, "unexpected utility statement type"); + } +} + +/* + * Display a Var appropriately. + * + * In some cases (currently only when recursing into an unnamed join) + * the Var's varlevelsup has to be interpreted with respect to a context + * above the current one; levelsup indicates the offset. + * + * If istoplevel is true, the Var is at the top level of a SELECT's + * targetlist, which means we need special treatment of whole-row Vars. + * Instead of the normal "tab.*", we'll print "tab.*::typename", which is a + * dirty hack to prevent "tab.*" from being expanded into multiple columns. + * (The parser will strip the useless coercion, so no inefficiency is added in + * dump and reload.) We used to print just "tab" in such cases, but that is + * ambiguous and will yield the wrong result if "tab" is also a plain column + * name in the query. + * + * Returns the attname of the Var, or NULL if the Var has no attname (because + * it is a whole-row Var or a subplan output reference). + */ +static char * +get_variable(Var *var, int levelsup, bool istoplevel, deparse_context *context) +{ + StringInfo buf = context->buf; + RangeTblEntry *rte; + AttrNumber attnum; + Index varno; + AttrNumber varattno; + int netlevelsup; + deparse_namespace *dpns; + deparse_columns *colinfo; + char *refname; + char *attname; + + /* Find appropriate nesting depth */ + netlevelsup = var->varlevelsup + levelsup; + if (netlevelsup >= list_length(context->namespaces)) + elog(ERROR, "bogus varlevelsup: %d offset %d", + var->varlevelsup, levelsup); + dpns = (deparse_namespace *) list_nth(context->namespaces, + netlevelsup); + + varno = var->varno; + varattno = var->varattno; + + + if (var->varnosyn > 0 && var->varnosyn <= list_length(dpns->rtable) && dpns->plan == NULL) { + rte = rt_fetch(var->varnosyn, dpns->rtable); + + /* + * if the rte var->varnosyn points to is not a regular table and it is a join + * then the correct relname will be found with var->varnosyn and var->varattnosyn + */ + if (rte->rtekind == RTE_JOIN && rte->relid == 0 && var->varnosyn != var->varno) { + varno = var->varnosyn; + varattno = var->varattnosyn; + } + } + + /* + * Try to find the relevant RTE in this rtable. In a plan tree, it's + * likely that varno is OUTER_VAR or INNER_VAR, in which case we must dig + * down into the subplans, or INDEX_VAR, which is resolved similarly. Also + * find the aliases previously assigned for this RTE. + */ + if (varno >= 1 && varno <= list_length(dpns->rtable)) + { + + /* + * We might have been asked to map child Vars to some parent relation. + */ + if (context->appendparents && dpns->appendrels) + { + + Index pvarno = varno; + AttrNumber pvarattno = varattno; + AppendRelInfo *appinfo = dpns->appendrels[pvarno]; + bool found = false; + + /* Only map up to inheritance parents, not UNION ALL appendrels */ + while (appinfo && + rt_fetch(appinfo->parent_relid, + dpns->rtable)->rtekind == RTE_RELATION) + { + found = false; + if (pvarattno > 0) /* system columns stay as-is */ + { + if (pvarattno > appinfo->num_child_cols) + break; /* safety check */ + pvarattno = appinfo->parent_colnos[pvarattno - 1]; + if (pvarattno == 0) + break; /* Var is local to child */ + } + + pvarno = appinfo->parent_relid; + found = true; + + /* If the parent is itself a child, continue up. */ + Assert(pvarno > 0 && pvarno <= list_length(dpns->rtable)); + appinfo = dpns->appendrels[pvarno]; + } + + /* + * If we found an ancestral rel, and that rel is included in + * appendparents, print that column not the original one. + */ + if (found && bms_is_member(pvarno, context->appendparents)) + { + varno = pvarno; + varattno = pvarattno; + } + } + + rte = rt_fetch(varno, dpns->rtable); + refname = (char *) list_nth(dpns->rtable_names, varno - 1); + colinfo = deparse_columns_fetch(varno, dpns); + attnum = varattno; + } + else + { + resolve_special_varno((Node *) var, context, get_special_variable, + NULL); + return NULL; + } + + /* + * The planner will sometimes emit Vars referencing resjunk elements of a + * subquery's target list (this is currently only possible if it chooses + * to generate a "physical tlist" for a SubqueryScan or CteScan node). + * Although we prefer to print subquery-referencing Vars using the + * subquery's alias, that's not possible for resjunk items since they have + * no alias. So in that case, drill down to the subplan and print the + * contents of the referenced tlist item. This works because in a plan + * tree, such Vars can only occur in a SubqueryScan or CteScan node, and + * we'll have set dpns->inner_plan to reference the child plan node. + */ + if ((rte->rtekind == RTE_SUBQUERY || rte->rtekind == RTE_CTE) && + attnum > list_length(rte->eref->colnames) && + dpns->inner_plan) + { + TargetEntry *tle; + deparse_namespace save_dpns; + + tle = get_tle_by_resno(dpns->inner_tlist, attnum); + if (!tle) + elog(ERROR, "invalid attnum %d for relation \"%s\"", + attnum, rte->eref->aliasname); + + Assert(netlevelsup == 0); + push_child_plan(dpns, dpns->inner_plan, &save_dpns); + + /* + * Force parentheses because our caller probably assumed a Var is a + * simple expression. + */ + if (!IsA(tle->expr, Var)) + appendStringInfoChar(buf, '('); + get_rule_expr((Node *) tle->expr, context, true); + if (!IsA(tle->expr, Var)) + appendStringInfoChar(buf, ')'); + + pop_child_plan(dpns, &save_dpns); + return NULL; + } + + /* + * If it's an unnamed join, look at the expansion of the alias variable. + * If it's a simple reference to one of the input vars, then recursively + * print the name of that var instead. When it's not a simple reference, + * we have to just print the unqualified join column name. (This can only + * happen with "dangerous" merged columns in a JOIN USING; we took pains + * previously to make the unqualified column name unique in such cases.) + * + * This wouldn't work in decompiling plan trees, because we don't store + * joinaliasvars lists after planning; but a plan tree should never + * contain a join alias variable. + */ + if (rte->rtekind == RTE_JOIN && rte->alias == NULL) + { + if (rte->joinaliasvars == NIL) + elog(ERROR, "cannot decompile join alias var in plan tree"); + if (attnum > 0) + { + Var *aliasvar; + + aliasvar = (Var *) list_nth(rte->joinaliasvars, attnum - 1); + /* we intentionally don't strip implicit coercions here */ + if (aliasvar && IsA(aliasvar, Var)) + { + return get_variable(aliasvar, var->varlevelsup + levelsup, + istoplevel, context); + } + } + + /* + * Unnamed join has no refname. (Note: since it's unnamed, there is + * no way the user could have referenced it to create a whole-row Var + * for it. So we don't have to cover that case below.) + */ + Assert(refname == NULL); + } + + if (attnum == InvalidAttrNumber) + attname = NULL; + else if (attnum > 0) + { + /* Get column name to use from the colinfo struct */ + if (attnum > colinfo->num_cols) + elog(ERROR, "invalid attnum %d for relation \"%s\"", + attnum, rte->eref->aliasname); + attname = colinfo->colnames[attnum - 1]; + if (attname == NULL) /* dropped column? */ + elog(ERROR, "invalid attnum %d for relation \"%s\"", + attnum, rte->eref->aliasname); + } + else if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) + { + /* System column on a Citus shard */ + attname = get_attname(rte->relid, attnum, false); + } + else + { + /* System column - name is fixed, get it from the catalog */ + attname = get_rte_attribute_name(rte, attnum); + } + + if (refname && (context->varprefix || attname == NULL)) + { + appendStringInfoString(buf, quote_identifier(refname)); + appendStringInfoChar(buf, '.'); + } + if (attname) + appendStringInfoString(buf, quote_identifier(attname)); + else + { + appendStringInfoChar(buf, '*'); + + if (istoplevel) + { + if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) + { + /* use rel.*::shard_name instead of rel.*::table_name */ + appendStringInfo(buf, "::%s", + generate_rte_shard_name(rte)); + } + else + { + appendStringInfo(buf, "::%s", + format_type_with_typemod(var->vartype, + var->vartypmod)); + } + } + } + + return attname; +} + +/* + * Deparse a Var which references OUTER_VAR, INNER_VAR, or INDEX_VAR. This + * routine is actually a callback for get_special_varno, which handles finding + * the correct TargetEntry. We get the expression contained in that + * TargetEntry and just need to deparse it, a job we can throw back on + * get_rule_expr. + */ +static void +get_special_variable(Node *node, deparse_context *context, void *callback_arg) +{ + StringInfo buf = context->buf; + + /* + * For a non-Var referent, force parentheses because our caller probably + * assumed a Var is a simple expression. + */ + if (!IsA(node, Var)) + appendStringInfoChar(buf, '('); + get_rule_expr(node, context, true); + if (!IsA(node, Var)) + appendStringInfoChar(buf, ')'); +} + +/* + * Chase through plan references to special varnos (OUTER_VAR, INNER_VAR, + * INDEX_VAR) until we find a real Var or some kind of non-Var node; then, + * invoke the callback provided. + */ +static void +resolve_special_varno(Node *node, deparse_context *context, rsv_callback callback, void *callback_arg) +{ + Var *var; + deparse_namespace *dpns; + + /* This function is recursive, so let's be paranoid. */ + check_stack_depth(); + + /* If it's not a Var, invoke the callback. */ + if (!IsA(node, Var)) + { + (*callback) (node, context, callback_arg); + return; + } + + /* Find appropriate nesting depth */ + var = (Var *) node; + dpns = (deparse_namespace *) list_nth(context->namespaces, + var->varlevelsup); + + /* + * It's a special RTE, so recurse. + */ + if (var->varno == OUTER_VAR && dpns->outer_tlist) + { + TargetEntry *tle; + deparse_namespace save_dpns; + Bitmapset *save_appendparents; + + tle = get_tle_by_resno(dpns->outer_tlist, var->varattno); + if (!tle) + elog(ERROR, "bogus varattno for OUTER_VAR var: %d", var->varattno); + + /* If we're descending to the first child of an Append or MergeAppend, + * update appendparents. This will affect deparsing of all Vars + * appearing within the eventually-resolved subexpression. + */ + save_appendparents = context->appendparents; + + if (IsA(dpns->plan, Append)) + context->appendparents = bms_union(context->appendparents, + ((Append *) dpns->plan)->apprelids); + else if (IsA(dpns->plan, MergeAppend)) + context->appendparents = bms_union(context->appendparents, + ((MergeAppend *) dpns->plan)->apprelids); + + push_child_plan(dpns, dpns->outer_plan, &save_dpns); + resolve_special_varno((Node *) tle->expr, context, + callback, callback_arg); + pop_child_plan(dpns, &save_dpns); + context->appendparents = save_appendparents; + return; + } + else if (var->varno == INNER_VAR && dpns->inner_tlist) + { + TargetEntry *tle; + deparse_namespace save_dpns; + + tle = get_tle_by_resno(dpns->inner_tlist, var->varattno); + if (!tle) + elog(ERROR, "bogus varattno for INNER_VAR var: %d", var->varattno); + + push_child_plan(dpns, dpns->inner_plan, &save_dpns); + resolve_special_varno((Node *) tle->expr, context, callback, callback_arg); + pop_child_plan(dpns, &save_dpns); + return; + } + else if (var->varno == INDEX_VAR && dpns->index_tlist) + { + TargetEntry *tle; + + tle = get_tle_by_resno(dpns->index_tlist, var->varattno); + if (!tle) + elog(ERROR, "bogus varattno for INDEX_VAR var: %d", var->varattno); + + resolve_special_varno((Node *) tle->expr, context, callback, callback_arg); + return; + } + else if (var->varno < 1 || var->varno > list_length(dpns->rtable)) + elog(ERROR, "bogus varno: %d", var->varno); + + /* Not special. Just invoke the callback. */ + (*callback) (node, context, callback_arg); +} + +/* + * Get the name of a field of an expression of composite type. The + * expression is usually a Var, but we handle other cases too. + * + * levelsup is an extra offset to interpret the Var's varlevelsup correctly. + * + * This is fairly straightforward when the expression has a named composite + * type; we need only look up the type in the catalogs. However, the type + * could also be RECORD. Since no actual table or view column is allowed to + * have type RECORD, a Var of type RECORD must refer to a JOIN or FUNCTION RTE + * or to a subquery output. We drill down to find the ultimate defining + * expression and attempt to infer the field name from it. We ereport if we + * can't determine the name. + * + * Similarly, a PARAM of type RECORD has to refer to some expression of + * a determinable composite type. + */ +static const char * +get_name_for_var_field(Var *var, int fieldno, + int levelsup, deparse_context *context) +{ + RangeTblEntry *rte; + AttrNumber attnum; + int netlevelsup; + deparse_namespace *dpns; + Index varno; + AttrNumber varattno; + TupleDesc tupleDesc; + Node *expr; + + /* + * If it's a RowExpr that was expanded from a whole-row Var, use the + * column names attached to it. + */ + if (IsA(var, RowExpr)) + { + RowExpr *r = (RowExpr *) var; + + if (fieldno > 0 && fieldno <= list_length(r->colnames)) + return strVal(list_nth(r->colnames, fieldno - 1)); + } + + /* + * If it's a Param of type RECORD, try to find what the Param refers to. + */ + if (IsA(var, Param)) + { + Param *param = (Param *) var; + ListCell *ancestor_cell; + + expr = find_param_referent(param, context, &dpns, &ancestor_cell); + if (expr) + { + /* Found a match, so recurse to decipher the field name */ + deparse_namespace save_dpns; + const char *result; + + push_ancestor_plan(dpns, ancestor_cell, &save_dpns); + result = get_name_for_var_field((Var *) expr, fieldno, + 0, context); + pop_ancestor_plan(dpns, &save_dpns); + return result; + } + } + + /* + * If it's a Var of type RECORD, we have to find what the Var refers to; + * if not, we can use get_expr_result_tupdesc(). + */ + if (!IsA(var, Var) || + var->vartype != RECORDOID) + { + tupleDesc = get_expr_result_tupdesc((Node *) var, false); + /* Got the tupdesc, so we can extract the field name */ + Assert(fieldno >= 1 && fieldno <= tupleDesc->natts); + return NameStr(TupleDescAttr(tupleDesc, fieldno - 1)->attname); + } + + /* Find appropriate nesting depth */ + netlevelsup = var->varlevelsup + levelsup; + if (netlevelsup >= list_length(context->namespaces)) + elog(ERROR, "bogus varlevelsup: %d offset %d", + var->varlevelsup, levelsup); + dpns = (deparse_namespace *) list_nth(context->namespaces, + netlevelsup); + + varno = var->varno; + varattno = var->varattno; + + if (var->varnosyn > 0 && var->varnosyn <= list_length(dpns->rtable) && dpns->plan == NULL) { + rte = rt_fetch(var->varnosyn, dpns->rtable); + + /* + * if the rte var->varnosyn points to is not a regular table and it is a join + * then the correct relname will be found with var->varnosyn and var->varattnosyn + */ + if (rte->rtekind == RTE_JOIN && rte->relid == 0 && var->varnosyn != var->varno) { + varno = var->varnosyn; + varattno = var->varattnosyn; + } + } + + /* + * Try to find the relevant RTE in this rtable. In a plan tree, it's + * likely that varno is OUTER_VAR or INNER_VAR, in which case we must dig + * down into the subplans, or INDEX_VAR, which is resolved similarly. + */ + if (varno >= 1 && varno <= list_length(dpns->rtable)) + { + rte = rt_fetch(varno, dpns->rtable); + attnum = varattno; + } + else if (varno == OUTER_VAR && dpns->outer_tlist) + { + TargetEntry *tle; + deparse_namespace save_dpns; + const char *result; + + tle = get_tle_by_resno(dpns->outer_tlist, varattno); + if (!tle) + elog(ERROR, "bogus varattno for OUTER_VAR var: %d", varattno); + + Assert(netlevelsup == 0); + push_child_plan(dpns, dpns->outer_plan, &save_dpns); + + result = get_name_for_var_field((Var *) tle->expr, fieldno, + levelsup, context); + + pop_child_plan(dpns, &save_dpns); + return result; + } + else if (varno == INNER_VAR && dpns->inner_tlist) + { + TargetEntry *tle; + deparse_namespace save_dpns; + const char *result; + + tle = get_tle_by_resno(dpns->inner_tlist, varattno); + if (!tle) + elog(ERROR, "bogus varattno for INNER_VAR var: %d", varattno); + + Assert(netlevelsup == 0); + push_child_plan(dpns, dpns->inner_plan, &save_dpns); + + result = get_name_for_var_field((Var *) tle->expr, fieldno, + levelsup, context); + + pop_child_plan(dpns, &save_dpns); + return result; + } + else if (varno == INDEX_VAR && dpns->index_tlist) + { + TargetEntry *tle; + const char *result; + + tle = get_tle_by_resno(dpns->index_tlist, varattno); + if (!tle) + elog(ERROR, "bogus varattno for INDEX_VAR var: %d", varattno); + + Assert(netlevelsup == 0); + + result = get_name_for_var_field((Var *) tle->expr, fieldno, + levelsup, context); + + return result; + } + else + { + elog(ERROR, "bogus varno: %d", varno); + return NULL; /* keep compiler quiet */ + } + + if (attnum == InvalidAttrNumber) + { + /* Var is whole-row reference to RTE, so select the right field */ + return get_rte_attribute_name(rte, fieldno); + } + + /* + * This part has essentially the same logic as the parser's + * expandRecordVariable() function, but we are dealing with a different + * representation of the input context, and we only need one field name + * not a TupleDesc. Also, we need special cases for finding subquery and + * CTE subplans when deparsing Plan trees. + */ + expr = (Node *) var; /* default if we can't drill down */ + + switch (rte->rtekind) + { + case RTE_RELATION: + case RTE_VALUES: + case RTE_NAMEDTUPLESTORE: + case RTE_RESULT: + + /* + * This case should not occur: a column of a table or values list + * shouldn't have type RECORD. Fall through and fail (most + * likely) at the bottom. + */ + break; + case RTE_SUBQUERY: + /* Subselect-in-FROM: examine sub-select's output expr */ + { + if (rte->subquery) + { + TargetEntry *ste = get_tle_by_resno(rte->subquery->targetList, + attnum); + + if (ste == NULL || ste->resjunk) + elog(ERROR, "subquery %s does not have attribute %d", + rte->eref->aliasname, attnum); + expr = (Node *) ste->expr; + if (IsA(expr, Var)) + { + /* + * Recurse into the sub-select to see what its Var + * refers to. We have to build an additional level of + * namespace to keep in step with varlevelsup in the + * subselect. + */ + deparse_namespace mydpns; + const char *result; + + set_deparse_for_query(&mydpns, rte->subquery, + context->namespaces); + + context->namespaces = lcons(&mydpns, + context->namespaces); + + result = get_name_for_var_field((Var *) expr, fieldno, + 0, context); + + context->namespaces = + list_delete_first(context->namespaces); + + return result; + } + /* else fall through to inspect the expression */ + } + else + { + /* + * We're deparsing a Plan tree so we don't have complete + * RTE entries (in particular, rte->subquery is NULL). But + * the only place we'd see a Var directly referencing a + * SUBQUERY RTE is in a SubqueryScan plan node, and we can + * look into the child plan's tlist instead. + */ + TargetEntry *tle; + deparse_namespace save_dpns; + const char *result; + + if (!dpns->inner_plan) + elog(ERROR, "failed to find plan for subquery %s", + rte->eref->aliasname); + tle = get_tle_by_resno(dpns->inner_tlist, attnum); + if (!tle) + elog(ERROR, "bogus varattno for subquery var: %d", + attnum); + Assert(netlevelsup == 0); + push_child_plan(dpns, dpns->inner_plan, &save_dpns); + + result = get_name_for_var_field((Var *) tle->expr, fieldno, + levelsup, context); + + pop_child_plan(dpns, &save_dpns); + return result; + } + } + break; + case RTE_JOIN: + /* Join RTE --- recursively inspect the alias variable */ + if (rte->joinaliasvars == NIL) + elog(ERROR, "cannot decompile join alias var in plan tree"); + Assert(attnum > 0 && attnum <= list_length(rte->joinaliasvars)); + expr = (Node *) list_nth(rte->joinaliasvars, attnum - 1); + Assert(expr != NULL); + /* we intentionally don't strip implicit coercions here */ + if (IsA(expr, Var)) + return get_name_for_var_field((Var *) expr, fieldno, + var->varlevelsup + levelsup, + context); + /* else fall through to inspect the expression */ + break; + case RTE_FUNCTION: + case RTE_TABLEFUNC: + + /* + * We couldn't get here unless a function is declared with one of + * its result columns as RECORD, which is not allowed. + */ + break; + case RTE_CTE: + /* CTE reference: examine subquery's output expr */ + { + CommonTableExpr *cte = NULL; + Index ctelevelsup; + ListCell *lc; + + /* + * Try to find the referenced CTE using the namespace stack. + */ + ctelevelsup = rte->ctelevelsup + netlevelsup; + if (ctelevelsup >= list_length(context->namespaces)) + lc = NULL; + else + { + deparse_namespace *ctedpns; + + ctedpns = (deparse_namespace *) + list_nth(context->namespaces, ctelevelsup); + foreach(lc, ctedpns->ctes) + { + cte = (CommonTableExpr *) lfirst(lc); + if (strcmp(cte->ctename, rte->ctename) == 0) + break; + } + } + if (lc != NULL) + { + Query *ctequery = (Query *) cte->ctequery; + TargetEntry *ste = get_tle_by_resno(GetCTETargetList(cte), + attnum); + + if (ste == NULL || ste->resjunk) + elog(ERROR, "subquery %s does not have attribute %d", + rte->eref->aliasname, attnum); + expr = (Node *) ste->expr; + if (IsA(expr, Var)) + { + /* + * Recurse into the CTE to see what its Var refers to. + * We have to build an additional level of namespace + * to keep in step with varlevelsup in the CTE. + * Furthermore it could be an outer CTE, so we may + * have to delete some levels of namespace. + */ + List *save_nslist = context->namespaces; + List *new_nslist; + deparse_namespace mydpns; + const char *result; + + set_deparse_for_query(&mydpns, ctequery, + context->namespaces); + + new_nslist = list_copy_tail(context->namespaces, + ctelevelsup); + context->namespaces = lcons(&mydpns, new_nslist); + + result = get_name_for_var_field((Var *) expr, fieldno, + 0, context); + + context->namespaces = save_nslist; + + return result; + } + /* else fall through to inspect the expression */ + } + else + { + /* + * We're deparsing a Plan tree so we don't have a CTE + * list. But the only place we'd see a Var directly + * referencing a CTE RTE is in a CteScan plan node, and we + * can look into the subplan's tlist instead. + */ + TargetEntry *tle; + deparse_namespace save_dpns; + const char *result; + + if (!dpns->inner_plan) + elog(ERROR, "failed to find plan for CTE %s", + rte->eref->aliasname); + tle = get_tle_by_resno(dpns->inner_tlist, attnum); + if (!tle) + elog(ERROR, "bogus varattno for subquery var: %d", + attnum); + Assert(netlevelsup == 0); + push_child_plan(dpns, dpns->inner_plan, &save_dpns); + + result = get_name_for_var_field((Var *) tle->expr, fieldno, + levelsup, context); + + pop_child_plan(dpns, &save_dpns); + return result; + } + } + break; + } + + /* + * We now have an expression we can't expand any more, so see if + * get_expr_result_tupdesc() can do anything with it. + */ + tupleDesc = get_expr_result_tupdesc(expr, false); + /* Got the tupdesc, so we can extract the field name */ + Assert(fieldno >= 1 && fieldno <= tupleDesc->natts); + return NameStr(TupleDescAttr(tupleDesc, fieldno - 1)->attname); +} + +/* + * Try to find the referenced expression for a PARAM_EXEC Param that might + * reference a parameter supplied by an upper NestLoop or SubPlan plan node. + * + * If successful, return the expression and set *dpns_p and *ancestor_cell_p + * appropriately for calling push_ancestor_plan(). If no referent can be + * found, return NULL. + */ +static Node * +find_param_referent(Param *param, deparse_context *context, + deparse_namespace **dpns_p, ListCell **ancestor_cell_p) +{ + /* Initialize output parameters to prevent compiler warnings */ + *dpns_p = NULL; + *ancestor_cell_p = NULL; + + /* + * If it's a PARAM_EXEC parameter, look for a matching NestLoopParam or + * SubPlan argument. This will necessarily be in some ancestor of the + * current expression's Plan. + */ + if (param->paramkind == PARAM_EXEC) + { + deparse_namespace *dpns; + Plan *child_plan; + bool in_same_plan_level; + ListCell *lc; + + dpns = (deparse_namespace *) linitial(context->namespaces); + child_plan = dpns->plan; + in_same_plan_level = true; + + foreach(lc, dpns->ancestors) + { + Node *ancestor = (Node *) lfirst(lc); + ListCell *lc2; + + /* + * NestLoops transmit params to their inner child only; also, once + * we've crawled up out of a subplan, this couldn't possibly be + * the right match. + */ + if (IsA(ancestor, NestLoop) && + child_plan == innerPlan(ancestor) && + in_same_plan_level) + { + NestLoop *nl = (NestLoop *) ancestor; + + foreach(lc2, nl->nestParams) + { + NestLoopParam *nlp = (NestLoopParam *) lfirst(lc2); + + if (nlp->paramno == param->paramid) + { + /* Found a match, so return it */ + *dpns_p = dpns; + *ancestor_cell_p = lc; + return (Node *) nlp->paramval; + } + } + } + + /* + * Check to see if we're crawling up from a subplan. + */ + if(IsA(ancestor, SubPlan)) + { + SubPlan *subplan = (SubPlan *) ancestor; + ListCell *lc3; + ListCell *lc4; + + /* Matched subplan, so check its arguments */ + forboth(lc3, subplan->parParam, lc4, subplan->args) + { + int paramid = lfirst_int(lc3); + Node *arg = (Node *) lfirst(lc4); + + if (paramid == param->paramid) + { + /* + * Found a match, so return it. But, since Vars in + * the arg are to be evaluated in the surrounding + * context, we have to point to the next ancestor item + * that is *not* a SubPlan. + */ + ListCell *rest; + + for_each_cell(rest, dpns->ancestors, + lnext(dpns->ancestors, lc)) + { + Node *ancestor2 = (Node *) lfirst(rest); + + if (!IsA(ancestor2, SubPlan)) + { + *dpns_p = dpns; + *ancestor_cell_p = rest; + return arg; + } + } + elog(ERROR, "SubPlan cannot be outermost ancestor"); + } + } + + /* We have emerged from a subplan. */ + in_same_plan_level = false; + + /* SubPlan isn't a kind of Plan, so skip the rest */ + continue; + } + + /* + * Check to see if we're emerging from an initplan of the current + * ancestor plan. Initplans never have any parParams, so no need + * to search that list, but we need to know if we should reset + * in_same_plan_level. + */ + foreach(lc2, ((Plan *) ancestor)->initPlan) + { + SubPlan *subplan = castNode(SubPlan, lfirst(lc2)); + + if (child_plan != (Plan *) list_nth(dpns->subplans, + subplan->plan_id - 1)) + continue; + + /* No parameters to be had here. */ + Assert(subplan->parParam == NIL); + + /* We have emerged from an initplan. */ + in_same_plan_level = false; + break; + } + + /* No luck, crawl up to next ancestor */ + child_plan = (Plan *) ancestor; + } + } + + /* No referent found */ + return NULL; +} + +/* + * Display a Param appropriately. + */ +static void +get_parameter(Param *param, deparse_context *context) +{ + Node *expr; + deparse_namespace *dpns; + ListCell *ancestor_cell; + + /* + * If it's a PARAM_EXEC parameter, try to locate the expression from which + * the parameter was computed. Note that failing to find a referent isn't + * an error, since the Param might well be a subplan output rather than an + * input. + */ + expr = find_param_referent(param, context, &dpns, &ancestor_cell); + if (expr) + { + /* Found a match, so print it */ + deparse_namespace save_dpns; + bool save_varprefix; + bool need_paren; + + /* Switch attention to the ancestor plan node */ + push_ancestor_plan(dpns, ancestor_cell, &save_dpns); + + /* + * Force prefixing of Vars, since they won't belong to the relation + * being scanned in the original plan node. + */ + save_varprefix = context->varprefix; + context->varprefix = true; + + /* + * A Param's expansion is typically a Var, Aggref, or upper-level + * Param, which wouldn't need extra parentheses. Otherwise, insert + * parens to ensure the expression looks atomic. + */ + need_paren = !(IsA(expr, Var) || + IsA(expr, Aggref) || + IsA(expr, Param)); + if (need_paren) + appendStringInfoChar(context->buf, '('); + + get_rule_expr(expr, context, false); + + if (need_paren) + appendStringInfoChar(context->buf, ')'); + + context->varprefix = save_varprefix; + + pop_ancestor_plan(dpns, &save_dpns); + + return; + } + + /* + * Not PARAM_EXEC, or couldn't find referent: for base types just print $N. + * For composite types, add cast to the parameter to ease remote node detect + * the type. + */ + if (param->paramtype >= FirstNormalObjectId) + { + char *typeName = format_type_with_typemod(param->paramtype, param->paramtypmod); + + appendStringInfo(context->buf, "$%d::%s", param->paramid, typeName); + } + else + { + appendStringInfo(context->buf, "$%d", param->paramid); + } +} + +/* + * get_simple_binary_op_name + * + * helper function for isSimpleNode + * will return single char binary operator name, or NULL if it's not + */ +static const char * +get_simple_binary_op_name(OpExpr *expr) +{ + List *args = expr->args; + + if (list_length(args) == 2) + { + /* binary operator */ + Node *arg1 = (Node *) linitial(args); + Node *arg2 = (Node *) lsecond(args); + const char *op; + + op = generate_operator_name(expr->opno, exprType(arg1), exprType(arg2)); + if (strlen(op) == 1) + return op; + } + return NULL; +} + + +/* + * isSimpleNode - check if given node is simple (doesn't need parenthesizing) + * + * true : simple in the context of parent node's type + * false : not simple + */ +static bool +isSimpleNode(Node *node, Node *parentNode, int prettyFlags) +{ + if (!node) + return false; + + switch (nodeTag(node)) + { + case T_Var: + case T_Const: + case T_Param: + case T_CoerceToDomainValue: + case T_SetToDefault: + case T_CurrentOfExpr: + /* single words: always simple */ + return true; + + case T_SubscriptingRef: + case T_ArrayExpr: + case T_RowExpr: + case T_CoalesceExpr: + case T_MinMaxExpr: + case T_SQLValueFunction: + case T_XmlExpr: + case T_NextValueExpr: + case T_NullIfExpr: + case T_Aggref: + case T_WindowFunc: + case T_FuncExpr: + /* function-like: name(..) or name[..] */ + return true; + + /* CASE keywords act as parentheses */ + case T_CaseExpr: + return true; + + case T_FieldSelect: + + /* + * appears simple since . has top precedence, unless parent is + * T_FieldSelect itself! + */ + return (IsA(parentNode, FieldSelect) ? false : true); + + case T_FieldStore: + + /* + * treat like FieldSelect (probably doesn't matter) + */ + return (IsA(parentNode, FieldStore) ? false : true); + + case T_CoerceToDomain: + /* maybe simple, check args */ + return isSimpleNode((Node *) ((CoerceToDomain *) node)->arg, + node, prettyFlags); + case T_RelabelType: + return isSimpleNode((Node *) ((RelabelType *) node)->arg, + node, prettyFlags); + case T_CoerceViaIO: + return isSimpleNode((Node *) ((CoerceViaIO *) node)->arg, + node, prettyFlags); + case T_ArrayCoerceExpr: + return isSimpleNode((Node *) ((ArrayCoerceExpr *) node)->arg, + node, prettyFlags); + case T_ConvertRowtypeExpr: + return isSimpleNode((Node *) ((ConvertRowtypeExpr *) node)->arg, + node, prettyFlags); + + case T_OpExpr: + { + /* depends on parent node type; needs further checking */ + if (prettyFlags & PRETTYFLAG_PAREN && IsA(parentNode, OpExpr)) + { + const char *op; + const char *parentOp; + bool is_lopriop; + bool is_hipriop; + bool is_lopriparent; + bool is_hipriparent; + + op = get_simple_binary_op_name((OpExpr *) node); + if (!op) + return false; + + /* We know only the basic operators + - and * / % */ + is_lopriop = (strchr("+-", *op) != NULL); + is_hipriop = (strchr("*/%", *op) != NULL); + if (!(is_lopriop || is_hipriop)) + return false; + + parentOp = get_simple_binary_op_name((OpExpr *) parentNode); + if (!parentOp) + return false; + + is_lopriparent = (strchr("+-", *parentOp) != NULL); + is_hipriparent = (strchr("*/%", *parentOp) != NULL); + if (!(is_lopriparent || is_hipriparent)) + return false; + + if (is_hipriop && is_lopriparent) + return true; /* op binds tighter than parent */ + + if (is_lopriop && is_hipriparent) + return false; + + /* + * Operators are same priority --- can skip parens only if + * we have (a - b) - c, not a - (b - c). + */ + if (node == (Node *) linitial(((OpExpr *) parentNode)->args)) + return true; + + return false; + } + /* else do the same stuff as for T_SubLink et al. */ + } + /* FALLTHROUGH */ + + case T_SubLink: + case T_NullTest: + case T_BooleanTest: + case T_DistinctExpr: + switch (nodeTag(parentNode)) + { + case T_FuncExpr: + { + /* special handling for casts */ + CoercionForm type = ((FuncExpr *) parentNode)->funcformat; + + if (type == COERCE_EXPLICIT_CAST || + type == COERCE_IMPLICIT_CAST) + return false; + return true; /* own parentheses */ + } + case T_BoolExpr: /* lower precedence */ + case T_SubscriptingRef: /* other separators */ + case T_ArrayExpr: /* other separators */ + case T_RowExpr: /* other separators */ + case T_CoalesceExpr: /* own parentheses */ + case T_MinMaxExpr: /* own parentheses */ + case T_XmlExpr: /* own parentheses */ + case T_NullIfExpr: /* other separators */ + case T_Aggref: /* own parentheses */ + case T_WindowFunc: /* own parentheses */ + case T_CaseExpr: /* other separators */ + return true; + default: + return false; + } + + case T_BoolExpr: + switch (nodeTag(parentNode)) + { + case T_BoolExpr: + if (prettyFlags & PRETTYFLAG_PAREN) + { + BoolExprType type; + BoolExprType parentType; + + type = ((BoolExpr *) node)->boolop; + parentType = ((BoolExpr *) parentNode)->boolop; + switch (type) + { + case NOT_EXPR: + case AND_EXPR: + if (parentType == AND_EXPR || parentType == OR_EXPR) + return true; + break; + case OR_EXPR: + if (parentType == OR_EXPR) + return true; + break; + } + } + return false; + case T_FuncExpr: + { + /* special handling for casts */ + CoercionForm type = ((FuncExpr *) parentNode)->funcformat; + + if (type == COERCE_EXPLICIT_CAST || + type == COERCE_IMPLICIT_CAST) + return false; + return true; /* own parentheses */ + } + case T_SubscriptingRef: /* other separators */ + case T_ArrayExpr: /* other separators */ + case T_RowExpr: /* other separators */ + case T_CoalesceExpr: /* own parentheses */ + case T_MinMaxExpr: /* own parentheses */ + case T_XmlExpr: /* own parentheses */ + case T_NullIfExpr: /* other separators */ + case T_Aggref: /* own parentheses */ + case T_WindowFunc: /* own parentheses */ + case T_CaseExpr: /* other separators */ + return true; + default: + return false; + } + + default: + break; + } + /* those we don't know: in dubio complexo */ + return false; +} + + +/* + * appendContextKeyword - append a keyword to buffer + * + * If prettyPrint is enabled, perform a line break, and adjust indentation. + * Otherwise, just append the keyword. + */ +static void +appendContextKeyword(deparse_context *context, const char *str, + int indentBefore, int indentAfter, int indentPlus) +{ + StringInfo buf = context->buf; + + if (PRETTY_INDENT(context)) + { + int indentAmount; + + context->indentLevel += indentBefore; + + /* remove any trailing spaces currently in the buffer ... */ + removeStringInfoSpaces(buf); + /* ... then add a newline and some spaces */ + appendStringInfoChar(buf, '\n'); + + if (context->indentLevel < PRETTYINDENT_LIMIT) + indentAmount = Max(context->indentLevel, 0) + indentPlus; + else + { + /* + * If we're indented more than PRETTYINDENT_LIMIT characters, try + * to conserve horizontal space by reducing the per-level + * indentation. For best results the scale factor here should + * divide all the indent amounts that get added to indentLevel + * (PRETTYINDENT_STD, etc). It's important that the indentation + * not grow unboundedly, else deeply-nested trees use O(N^2) + * whitespace; so we also wrap modulo PRETTYINDENT_LIMIT. + */ + indentAmount = PRETTYINDENT_LIMIT + + (context->indentLevel - PRETTYINDENT_LIMIT) / + (PRETTYINDENT_STD / 2); + indentAmount %= PRETTYINDENT_LIMIT; + /* scale/wrap logic affects indentLevel, but not indentPlus */ + indentAmount += indentPlus; + } + appendStringInfoSpaces(buf, indentAmount); + + appendStringInfoString(buf, str); + + context->indentLevel += indentAfter; + if (context->indentLevel < 0) + context->indentLevel = 0; + } + else + appendStringInfoString(buf, str); +} + +/* + * removeStringInfoSpaces - delete trailing spaces from a buffer. + * + * Possibly this should move to stringinfo.c at some point. + */ +static void +removeStringInfoSpaces(StringInfo str) +{ + while (str->len > 0 && str->data[str->len - 1] == ' ') + str->data[--(str->len)] = '\0'; +} + + +/* + * get_rule_expr_paren - deparse expr using get_rule_expr, + * embracing the string with parentheses if necessary for prettyPrint. + * + * Never embrace if prettyFlags=0, because it's done in the calling node. + * + * Any node that does *not* embrace its argument node by sql syntax (with + * parentheses, non-operator keywords like CASE/WHEN/ON, or comma etc) should + * use get_rule_expr_paren instead of get_rule_expr so parentheses can be + * added. + */ +static void +get_rule_expr_paren(Node *node, deparse_context *context, + bool showimplicit, Node *parentNode) +{ + bool need_paren; + + need_paren = PRETTY_PAREN(context) && + !isSimpleNode(node, parentNode, context->prettyFlags); + + if (need_paren) + appendStringInfoChar(context->buf, '('); + + get_rule_expr(node, context, showimplicit); + + if (need_paren) + appendStringInfoChar(context->buf, ')'); +} + + +/* ---------- + * get_rule_expr - Parse back an expression + * + * Note: showimplicit determines whether we display any implicit cast that + * is present at the top of the expression tree. It is a passed argument, + * not a field of the context struct, because we change the value as we + * recurse down into the expression. In general we suppress implicit casts + * when the result type is known with certainty (eg, the arguments of an + * OR must be boolean). We display implicit casts for arguments of functions + * and operators, since this is needed to be certain that the same function + * or operator will be chosen when the expression is re-parsed. + * ---------- + */ +static void +get_rule_expr(Node *node, deparse_context *context, + bool showimplicit) +{ + StringInfo buf = context->buf; + + if (node == NULL) + return; + + /* Guard against excessively long or deeply-nested queries */ + CHECK_FOR_INTERRUPTS(); + check_stack_depth(); + + /* + * Each level of get_rule_expr must emit an indivisible term + * (parenthesized if necessary) to ensure result is reparsed into the same + * expression tree. The only exception is that when the input is a List, + * we emit the component items comma-separated with no surrounding + * decoration; this is convenient for most callers. + */ + switch (nodeTag(node)) + { + case T_Var: + (void) get_variable((Var *) node, 0, false, context); + break; + + case T_Const: + get_const_expr((Const *) node, context, 0); + break; + + case T_Param: + get_parameter((Param *) node, context); + break; + + case T_Aggref: + get_agg_expr((Aggref *) node, context, (Aggref *) node); + break; + + case T_GroupingFunc: + { + GroupingFunc *gexpr = (GroupingFunc *) node; + + appendStringInfoString(buf, "GROUPING("); + get_rule_expr((Node *) gexpr->args, context, true); + appendStringInfoChar(buf, ')'); + } + break; + + case T_WindowFunc: + get_windowfunc_expr((WindowFunc *) node, context); + break; + + case T_SubscriptingRef: + { + SubscriptingRef *sbsref = (SubscriptingRef *) node; + bool need_parens; + + /* + * If the argument is a CaseTestExpr, we must be inside a + * FieldStore, ie, we are assigning to an element of an array + * within a composite column. Since we already punted on + * displaying the FieldStore's target information, just punt + * here too, and display only the assignment source + * expression. + */ + if (IsA(sbsref->refexpr, CaseTestExpr)) + { + Assert(sbsref->refassgnexpr); + get_rule_expr((Node *) sbsref->refassgnexpr, + context, showimplicit); + break; + } + + /* + * Parenthesize the argument unless it's a simple Var or a + * FieldSelect. (In particular, if it's another + * SubscriptingRef, we *must* parenthesize to avoid + * confusion.) + */ + need_parens = !IsA(sbsref->refexpr, Var) && + !IsA(sbsref->refexpr, FieldSelect); + if (need_parens) + appendStringInfoChar(buf, '('); + get_rule_expr((Node *) sbsref->refexpr, context, showimplicit); + if (need_parens) + appendStringInfoChar(buf, ')'); + + /* + * If there's a refassgnexpr, we want to print the node in the + * format "container[subscripts] := refassgnexpr". This is + * not legal SQL, so decompilation of INSERT or UPDATE + * statements should always use processIndirection as part of + * the statement-level syntax. We should only see this when + * EXPLAIN tries to print the targetlist of a plan resulting + * from such a statement. + */ + if (sbsref->refassgnexpr) + { + Node *refassgnexpr; + + /* + * Use processIndirection to print this node's subscripts + * as well as any additional field selections or + * subscripting in immediate descendants. It returns the + * RHS expr that is actually being "assigned". + */ + refassgnexpr = processIndirection(node, context); + appendStringInfoString(buf, " := "); + get_rule_expr(refassgnexpr, context, showimplicit); + } + else + { + /* Just an ordinary container fetch, so print subscripts */ + printSubscripts(sbsref, context); + } + } + break; + + case T_FuncExpr: + get_func_expr((FuncExpr *) node, context, showimplicit); + break; + + case T_NamedArgExpr: + { + NamedArgExpr *na = (NamedArgExpr *) node; + + appendStringInfo(buf, "%s => ", quote_identifier(na->name)); + get_rule_expr((Node *) na->arg, context, showimplicit); + } + break; + + case T_OpExpr: + get_oper_expr((OpExpr *) node, context); + break; + + case T_DistinctExpr: + { + DistinctExpr *expr = (DistinctExpr *) node; + List *args = expr->args; + Node *arg1 = (Node *) linitial(args); + Node *arg2 = (Node *) lsecond(args); + + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren(arg1, context, true, node); + appendStringInfoString(buf, " IS DISTINCT FROM "); + get_rule_expr_paren(arg2, context, true, node); + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + } + break; + + case T_NullIfExpr: + { + NullIfExpr *nullifexpr = (NullIfExpr *) node; + + appendStringInfoString(buf, "NULLIF("); + get_rule_expr((Node *) nullifexpr->args, context, true); + appendStringInfoChar(buf, ')'); + } + break; + + case T_ScalarArrayOpExpr: + { + ScalarArrayOpExpr *expr = (ScalarArrayOpExpr *) node; + List *args = expr->args; + Node *arg1 = (Node *) linitial(args); + Node *arg2 = (Node *) lsecond(args); + + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren(arg1, context, true, node); + appendStringInfo(buf, " %s %s (", + generate_operator_name(expr->opno, + exprType(arg1), + get_base_element_type(exprType(arg2))), + expr->useOr ? "ANY" : "ALL"); + get_rule_expr_paren(arg2, context, true, node); + + /* + * There's inherent ambiguity in "x op ANY/ALL (y)" when y is + * a bare sub-SELECT. Since we're here, the sub-SELECT must + * be meant as a scalar sub-SELECT yielding an array value to + * be used in ScalarArrayOpExpr; but the grammar will + * preferentially interpret such a construct as an ANY/ALL + * SubLink. To prevent misparsing the output that way, insert + * a dummy coercion (which will be stripped by parse analysis, + * so no inefficiency is added in dump and reload). This is + * indeed most likely what the user wrote to get the construct + * accepted in the first place. + */ + if (IsA(arg2, SubLink) && + ((SubLink *) arg2)->subLinkType == EXPR_SUBLINK) + appendStringInfo(buf, "::%s", + format_type_with_typemod(exprType(arg2), + exprTypmod(arg2))); + appendStringInfoChar(buf, ')'); + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + } + break; + + case T_BoolExpr: + { + BoolExpr *expr = (BoolExpr *) node; + Node *first_arg = linitial(expr->args); + ListCell *arg = list_second_cell(expr->args); + + switch (expr->boolop) + { + case AND_EXPR: + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren(first_arg, context, + false, node); + while (arg) + { + appendStringInfoString(buf, " AND "); + get_rule_expr_paren((Node *) lfirst(arg), context, + false, node); + arg = lnext(expr->args, arg); + } + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + break; + + case OR_EXPR: + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren(first_arg, context, + false, node); + while (arg) + { + appendStringInfoString(buf, " OR "); + get_rule_expr_paren((Node *) lfirst(arg), context, + false, node); + arg = lnext(expr->args, arg); + } + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + break; + + case NOT_EXPR: + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + appendStringInfoString(buf, "NOT "); + get_rule_expr_paren(first_arg, context, + false, node); + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + break; + + default: + elog(ERROR, "unrecognized boolop: %d", + (int) expr->boolop); + } + } + break; + + case T_SubLink: + get_sublink_expr((SubLink *) node, context); + break; + + case T_SubPlan: + { + SubPlan *subplan = (SubPlan *) node; + + /* + * We cannot see an already-planned subplan in rule deparsing, + * only while EXPLAINing a query plan. We don't try to + * reconstruct the original SQL, just reference the subplan + * that appears elsewhere in EXPLAIN's result. + */ + if (subplan->useHashTable) + appendStringInfo(buf, "(hashed %s)", subplan->plan_name); + else + appendStringInfo(buf, "(%s)", subplan->plan_name); + } + break; + + case T_AlternativeSubPlan: + { + AlternativeSubPlan *asplan = (AlternativeSubPlan *) node; + ListCell *lc; + + /* As above, this can only happen during EXPLAIN */ + appendStringInfoString(buf, "(alternatives: "); + foreach(lc, asplan->subplans) + { + SubPlan *splan = lfirst_node(SubPlan, lc); + + if (splan->useHashTable) + appendStringInfo(buf, "hashed %s", splan->plan_name); + else + appendStringInfoString(buf, splan->plan_name); + if (lnext(asplan->subplans, lc)) + appendStringInfoString(buf, " or "); + } + appendStringInfoChar(buf, ')'); + } + break; + + case T_FieldSelect: + { + FieldSelect *fselect = (FieldSelect *) node; + Node *arg = (Node *) fselect->arg; + int fno = fselect->fieldnum; + const char *fieldname; + bool need_parens; + + /* + * Parenthesize the argument unless it's an SubscriptingRef or + * another FieldSelect. Note in particular that it would be + * WRONG to not parenthesize a Var argument; simplicity is not + * the issue here, having the right number of names is. + */ + need_parens = !IsA(arg, SubscriptingRef) && + !IsA(arg, FieldSelect); + if (need_parens) + appendStringInfoChar(buf, '('); + get_rule_expr(arg, context, true); + if (need_parens) + appendStringInfoChar(buf, ')'); + + /* + * Get and print the field name. + */ + fieldname = get_name_for_var_field((Var *) arg, fno, + 0, context); + appendStringInfo(buf, ".%s", quote_identifier(fieldname)); + } + break; + + case T_FieldStore: + { + FieldStore *fstore = (FieldStore *) node; + bool need_parens; + + /* + * There is no good way to represent a FieldStore as real SQL, + * so decompilation of INSERT or UPDATE statements should + * always use processIndirection as part of the + * statement-level syntax. We should only get here when + * EXPLAIN tries to print the targetlist of a plan resulting + * from such a statement. The plan case is even harder than + * ordinary rules would be, because the planner tries to + * collapse multiple assignments to the same field or subfield + * into one FieldStore; so we can see a list of target fields + * not just one, and the arguments could be FieldStores + * themselves. We don't bother to try to print the target + * field names; we just print the source arguments, with a + * ROW() around them if there's more than one. This isn't + * terribly complete, but it's probably good enough for + * EXPLAIN's purposes; especially since anything more would be + * either hopelessly confusing or an even poorer + * representation of what the plan is actually doing. + */ + need_parens = (list_length(fstore->newvals) != 1); + if (need_parens) + appendStringInfoString(buf, "ROW("); + get_rule_expr((Node *) fstore->newvals, context, showimplicit); + if (need_parens) + appendStringInfoChar(buf, ')'); + } + break; + + case T_RelabelType: + { + RelabelType *relabel = (RelabelType *) node; + + /* + * This is a Citus specific modification + * The planner converts CollateExpr to RelabelType + * and here we convert back. + */ + if (relabel->resultcollid != InvalidOid) + { + CollateExpr *collate = RelabelTypeToCollateExpr(relabel); + Node *arg = (Node *) collate->arg; + + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren(arg, context, showimplicit, node); + appendStringInfo(buf, " COLLATE %s", + generate_collation_name(collate->collOid)); + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + } + else + { + Node *arg = (Node *) relabel->arg; + + if (relabel->relabelformat == COERCE_IMPLICIT_CAST && + !showimplicit) + { + /* don't show the implicit cast */ + get_rule_expr_paren(arg, context, false, node); + } + else + { + get_coercion_expr(arg, context, + relabel->resulttype, + relabel->resulttypmod, + node); + } + } + } + break; + + case T_CoerceViaIO: + { + CoerceViaIO *iocoerce = (CoerceViaIO *) node; + Node *arg = (Node *) iocoerce->arg; + + if (iocoerce->coerceformat == COERCE_IMPLICIT_CAST && + !showimplicit) + { + /* don't show the implicit cast */ + get_rule_expr_paren(arg, context, false, node); + } + else + { + get_coercion_expr(arg, context, + iocoerce->resulttype, + -1, + node); + } + } + break; + + case T_ArrayCoerceExpr: + { + ArrayCoerceExpr *acoerce = (ArrayCoerceExpr *) node; + Node *arg = (Node *) acoerce->arg; + + if (acoerce->coerceformat == COERCE_IMPLICIT_CAST && + !showimplicit) + { + /* don't show the implicit cast */ + get_rule_expr_paren(arg, context, false, node); + } + else + { + get_coercion_expr(arg, context, + acoerce->resulttype, + acoerce->resulttypmod, + node); + } + } + break; + + case T_ConvertRowtypeExpr: + { + ConvertRowtypeExpr *convert = (ConvertRowtypeExpr *) node; + Node *arg = (Node *) convert->arg; + + if (convert->convertformat == COERCE_IMPLICIT_CAST && + !showimplicit) + { + /* don't show the implicit cast */ + get_rule_expr_paren(arg, context, false, node); + } + else + { + get_coercion_expr(arg, context, + convert->resulttype, -1, + node); + } + } + break; + + case T_CollateExpr: + { + CollateExpr *collate = (CollateExpr *) node; + Node *arg = (Node *) collate->arg; + + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren(arg, context, showimplicit, node); + appendStringInfo(buf, " COLLATE %s", + generate_collation_name(collate->collOid)); + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + } + break; + + case T_CaseExpr: + { + CaseExpr *caseexpr = (CaseExpr *) node; + ListCell *temp; + + appendContextKeyword(context, "CASE", + 0, PRETTYINDENT_VAR, 0); + if (caseexpr->arg) + { + appendStringInfoChar(buf, ' '); + get_rule_expr((Node *) caseexpr->arg, context, true); + } + foreach(temp, caseexpr->args) + { + CaseWhen *when = (CaseWhen *) lfirst(temp); + Node *w = (Node *) when->expr; + + if (caseexpr->arg) + { + /* + * The parser should have produced WHEN clauses of the + * form "CaseTestExpr = RHS", possibly with an + * implicit coercion inserted above the CaseTestExpr. + * For accurate decompilation of rules it's essential + * that we show just the RHS. However in an + * expression that's been through the optimizer, the + * WHEN clause could be almost anything (since the + * equality operator could have been expanded into an + * inline function). If we don't recognize the form + * of the WHEN clause, just punt and display it as-is. + */ + if (IsA(w, OpExpr)) + { + List *args = ((OpExpr *) w)->args; + + if (list_length(args) == 2 && + IsA(strip_implicit_coercions(linitial(args)), + CaseTestExpr)) + w = (Node *) lsecond(args); + } + } + + if (!PRETTY_INDENT(context)) + appendStringInfoChar(buf, ' '); + appendContextKeyword(context, "WHEN ", + 0, 0, 0); + get_rule_expr(w, context, false); + appendStringInfoString(buf, " THEN "); + get_rule_expr((Node *) when->result, context, true); + } + if (!PRETTY_INDENT(context)) + appendStringInfoChar(buf, ' '); + appendContextKeyword(context, "ELSE ", + 0, 0, 0); + get_rule_expr((Node *) caseexpr->defresult, context, true); + if (!PRETTY_INDENT(context)) + appendStringInfoChar(buf, ' '); + appendContextKeyword(context, "END", + -PRETTYINDENT_VAR, 0, 0); + } + break; + + case T_CaseTestExpr: + { + /* + * Normally we should never get here, since for expressions + * that can contain this node type we attempt to avoid + * recursing to it. But in an optimized expression we might + * be unable to avoid that (see comments for CaseExpr). If we + * do see one, print it as CASE_TEST_EXPR. + */ + appendStringInfoString(buf, "CASE_TEST_EXPR"); + } + break; + + case T_ArrayExpr: + { + ArrayExpr *arrayexpr = (ArrayExpr *) node; + + appendStringInfoString(buf, "ARRAY["); + get_rule_expr((Node *) arrayexpr->elements, context, true); + appendStringInfoChar(buf, ']'); + + /* + * If the array isn't empty, we assume its elements are + * coerced to the desired type. If it's empty, though, we + * need an explicit coercion to the array type. + */ + if (arrayexpr->elements == NIL) + appendStringInfo(buf, "::%s", + format_type_with_typemod(arrayexpr->array_typeid, -1)); + } + break; + + case T_RowExpr: + { + RowExpr *rowexpr = (RowExpr *) node; + TupleDesc tupdesc = NULL; + ListCell *arg; + int i; + char *sep; + + /* + * If it's a named type and not RECORD, we may have to skip + * dropped columns and/or claim there are NULLs for added + * columns. + */ + if (rowexpr->row_typeid != RECORDOID) + { + tupdesc = lookup_rowtype_tupdesc(rowexpr->row_typeid, -1); + Assert(list_length(rowexpr->args) <= tupdesc->natts); + } + + /* + * SQL99 allows "ROW" to be omitted when there is more than + * one column, but for simplicity we always print it. + */ + appendStringInfoString(buf, "ROW("); + sep = ""; + i = 0; + foreach(arg, rowexpr->args) + { + Node *e = (Node *) lfirst(arg); + + if (tupdesc == NULL || + !TupleDescAttr(tupdesc, i)->attisdropped) + { + appendStringInfoString(buf, sep); + /* Whole-row Vars need special treatment here */ + get_rule_expr_toplevel(e, context, true); + sep = ", "; + } + i++; + } + if (tupdesc != NULL) + { + while (i < tupdesc->natts) + { + if (!TupleDescAttr(tupdesc, i)->attisdropped) + { + appendStringInfoString(buf, sep); + appendStringInfoString(buf, "NULL"); + sep = ", "; + } + i++; + } + + ReleaseTupleDesc(tupdesc); + } + appendStringInfoChar(buf, ')'); + if (rowexpr->row_format == COERCE_EXPLICIT_CAST) + appendStringInfo(buf, "::%s", + format_type_with_typemod(rowexpr->row_typeid, -1)); + } + break; + + case T_RowCompareExpr: + { + RowCompareExpr *rcexpr = (RowCompareExpr *) node; + ListCell *arg; + char *sep; + + /* + * SQL99 allows "ROW" to be omitted when there is more than + * one column, but for simplicity we always print it. + */ + appendStringInfoString(buf, "(ROW("); + sep = ""; + foreach(arg, rcexpr->largs) + { + Node *e = (Node *) lfirst(arg); + + appendStringInfoString(buf, sep); + get_rule_expr(e, context, true); + sep = ", "; + } + + /* + * We assume that the name of the first-column operator will + * do for all the rest too. This is definitely open to + * failure, eg if some but not all operators were renamed + * since the construct was parsed, but there seems no way to + * be perfect. + */ + appendStringInfo(buf, ") %s ROW(", + generate_operator_name(linitial_oid(rcexpr->opnos), + exprType(linitial(rcexpr->largs)), + exprType(linitial(rcexpr->rargs)))); + sep = ""; + foreach(arg, rcexpr->rargs) + { + Node *e = (Node *) lfirst(arg); + + appendStringInfoString(buf, sep); + get_rule_expr(e, context, true); + sep = ", "; + } + appendStringInfoString(buf, "))"); + } + break; + + case T_CoalesceExpr: + { + CoalesceExpr *coalesceexpr = (CoalesceExpr *) node; + + appendStringInfoString(buf, "COALESCE("); + get_rule_expr((Node *) coalesceexpr->args, context, true); + appendStringInfoChar(buf, ')'); + } + break; + + case T_MinMaxExpr: + { + MinMaxExpr *minmaxexpr = (MinMaxExpr *) node; + + switch (minmaxexpr->op) + { + case IS_GREATEST: + appendStringInfoString(buf, "GREATEST("); + break; + case IS_LEAST: + appendStringInfoString(buf, "LEAST("); + break; + } + get_rule_expr((Node *) minmaxexpr->args, context, true); + appendStringInfoChar(buf, ')'); + } + break; + + case T_SQLValueFunction: + { + SQLValueFunction *svf = (SQLValueFunction *) node; + + /* + * Note: this code knows that typmod for time, timestamp, and + * timestamptz just prints as integer. + */ + switch (svf->op) + { + case SVFOP_CURRENT_DATE: + appendStringInfoString(buf, "CURRENT_DATE"); + break; + case SVFOP_CURRENT_TIME: + appendStringInfoString(buf, "CURRENT_TIME"); + break; + case SVFOP_CURRENT_TIME_N: + appendStringInfo(buf, "CURRENT_TIME(%d)", svf->typmod); + break; + case SVFOP_CURRENT_TIMESTAMP: + appendStringInfoString(buf, "CURRENT_TIMESTAMP"); + break; + case SVFOP_CURRENT_TIMESTAMP_N: + appendStringInfo(buf, "CURRENT_TIMESTAMP(%d)", + svf->typmod); + break; + case SVFOP_LOCALTIME: + appendStringInfoString(buf, "LOCALTIME"); + break; + case SVFOP_LOCALTIME_N: + appendStringInfo(buf, "LOCALTIME(%d)", svf->typmod); + break; + case SVFOP_LOCALTIMESTAMP: + appendStringInfoString(buf, "LOCALTIMESTAMP"); + break; + case SVFOP_LOCALTIMESTAMP_N: + appendStringInfo(buf, "LOCALTIMESTAMP(%d)", + svf->typmod); + break; + case SVFOP_CURRENT_ROLE: + appendStringInfoString(buf, "CURRENT_ROLE"); + break; + case SVFOP_CURRENT_USER: + appendStringInfoString(buf, "CURRENT_USER"); + break; + case SVFOP_USER: + appendStringInfoString(buf, "USER"); + break; + case SVFOP_SESSION_USER: + appendStringInfoString(buf, "SESSION_USER"); + break; + case SVFOP_CURRENT_CATALOG: + appendStringInfoString(buf, "CURRENT_CATALOG"); + break; + case SVFOP_CURRENT_SCHEMA: + appendStringInfoString(buf, "CURRENT_SCHEMA"); + break; + } + } + break; + + case T_XmlExpr: + { + XmlExpr *xexpr = (XmlExpr *) node; + bool needcomma = false; + ListCell *arg; + ListCell *narg; + Const *con; + + switch (xexpr->op) + { + case IS_XMLCONCAT: + appendStringInfoString(buf, "XMLCONCAT("); + break; + case IS_XMLELEMENT: + appendStringInfoString(buf, "XMLELEMENT("); + break; + case IS_XMLFOREST: + appendStringInfoString(buf, "XMLFOREST("); + break; + case IS_XMLPARSE: + appendStringInfoString(buf, "XMLPARSE("); + break; + case IS_XMLPI: + appendStringInfoString(buf, "XMLPI("); + break; + case IS_XMLROOT: + appendStringInfoString(buf, "XMLROOT("); + break; + case IS_XMLSERIALIZE: + appendStringInfoString(buf, "XMLSERIALIZE("); + break; + case IS_DOCUMENT: + break; + } + if (xexpr->op == IS_XMLPARSE || xexpr->op == IS_XMLSERIALIZE) + { + if (xexpr->xmloption == XMLOPTION_DOCUMENT) + appendStringInfoString(buf, "DOCUMENT "); + else + appendStringInfoString(buf, "CONTENT "); + } + if (xexpr->name) + { + appendStringInfo(buf, "NAME %s", + quote_identifier(map_xml_name_to_sql_identifier(xexpr->name))); + needcomma = true; + } + if (xexpr->named_args) + { + if (xexpr->op != IS_XMLFOREST) + { + if (needcomma) + appendStringInfoString(buf, ", "); + appendStringInfoString(buf, "XMLATTRIBUTES("); + needcomma = false; + } + forboth(arg, xexpr->named_args, narg, xexpr->arg_names) + { + Node *e = (Node *) lfirst(arg); + char *argname = strVal(lfirst(narg)); + + if (needcomma) + appendStringInfoString(buf, ", "); + get_rule_expr((Node *) e, context, true); + appendStringInfo(buf, " AS %s", + quote_identifier(map_xml_name_to_sql_identifier(argname))); + needcomma = true; + } + if (xexpr->op != IS_XMLFOREST) + appendStringInfoChar(buf, ')'); + } + if (xexpr->args) + { + if (needcomma) + appendStringInfoString(buf, ", "); + switch (xexpr->op) + { + case IS_XMLCONCAT: + case IS_XMLELEMENT: + case IS_XMLFOREST: + case IS_XMLPI: + case IS_XMLSERIALIZE: + /* no extra decoration needed */ + get_rule_expr((Node *) xexpr->args, context, true); + break; + case IS_XMLPARSE: + Assert(list_length(xexpr->args) == 2); + + get_rule_expr((Node *) linitial(xexpr->args), + context, true); + + con = lsecond_node(Const, xexpr->args); + Assert(!con->constisnull); + if (DatumGetBool(con->constvalue)) + appendStringInfoString(buf, + " PRESERVE WHITESPACE"); + else + appendStringInfoString(buf, + " STRIP WHITESPACE"); + break; + case IS_XMLROOT: + Assert(list_length(xexpr->args) == 3); + + get_rule_expr((Node *) linitial(xexpr->args), + context, true); + + appendStringInfoString(buf, ", VERSION "); + con = (Const *) lsecond(xexpr->args); + if (IsA(con, Const) && + con->constisnull) + appendStringInfoString(buf, "NO VALUE"); + else + get_rule_expr((Node *) con, context, false); + + con = lthird_node(Const, xexpr->args); + if (con->constisnull) + /* suppress STANDALONE NO VALUE */ ; + else + { + switch (DatumGetInt32(con->constvalue)) + { + case XML_STANDALONE_YES: + appendStringInfoString(buf, + ", STANDALONE YES"); + break; + case XML_STANDALONE_NO: + appendStringInfoString(buf, + ", STANDALONE NO"); + break; + case XML_STANDALONE_NO_VALUE: + appendStringInfoString(buf, + ", STANDALONE NO VALUE"); + break; + default: + break; + } + } + break; + case IS_DOCUMENT: + get_rule_expr_paren((Node *) xexpr->args, context, false, node); + break; + } + + } + if (xexpr->op == IS_XMLSERIALIZE) + appendStringInfo(buf, " AS %s", + format_type_with_typemod(xexpr->type, + xexpr->typmod)); + if (xexpr->op == IS_DOCUMENT) + appendStringInfoString(buf, " IS DOCUMENT"); + else + appendStringInfoChar(buf, ')'); + } + break; + + case T_NullTest: + { + NullTest *ntest = (NullTest *) node; + + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren((Node *) ntest->arg, context, true, node); + + /* + * For scalar inputs, we prefer to print as IS [NOT] NULL, + * which is shorter and traditional. If it's a rowtype input + * but we're applying a scalar test, must print IS [NOT] + * DISTINCT FROM NULL to be semantically correct. + */ + if (ntest->argisrow || + !type_is_rowtype(exprType((Node *) ntest->arg))) + { + switch (ntest->nulltesttype) + { + case IS_NULL: + appendStringInfoString(buf, " IS NULL"); + break; + case IS_NOT_NULL: + appendStringInfoString(buf, " IS NOT NULL"); + break; + default: + elog(ERROR, "unrecognized nulltesttype: %d", + (int) ntest->nulltesttype); + } + } + else + { + switch (ntest->nulltesttype) + { + case IS_NULL: + appendStringInfoString(buf, " IS NOT DISTINCT FROM NULL"); + break; + case IS_NOT_NULL: + appendStringInfoString(buf, " IS DISTINCT FROM NULL"); + break; + default: + elog(ERROR, "unrecognized nulltesttype: %d", + (int) ntest->nulltesttype); + } + } + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + } + break; + + case T_BooleanTest: + { + BooleanTest *btest = (BooleanTest *) node; + + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren((Node *) btest->arg, context, false, node); + switch (btest->booltesttype) + { + case IS_TRUE: + appendStringInfoString(buf, " IS TRUE"); + break; + case IS_NOT_TRUE: + appendStringInfoString(buf, " IS NOT TRUE"); + break; + case IS_FALSE: + appendStringInfoString(buf, " IS FALSE"); + break; + case IS_NOT_FALSE: + appendStringInfoString(buf, " IS NOT FALSE"); + break; + case IS_UNKNOWN: + appendStringInfoString(buf, " IS UNKNOWN"); + break; + case IS_NOT_UNKNOWN: + appendStringInfoString(buf, " IS NOT UNKNOWN"); + break; + default: + elog(ERROR, "unrecognized booltesttype: %d", + (int) btest->booltesttype); + } + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + } + break; + + case T_CoerceToDomain: + { + CoerceToDomain *ctest = (CoerceToDomain *) node; + Node *arg = (Node *) ctest->arg; + + if (ctest->coercionformat == COERCE_IMPLICIT_CAST && + !showimplicit) + { + /* don't show the implicit cast */ + get_rule_expr(arg, context, false); + } + else + { + get_coercion_expr(arg, context, + ctest->resulttype, + ctest->resulttypmod, + node); + } + } + break; + + case T_CoerceToDomainValue: + appendStringInfoString(buf, "VALUE"); + break; + + case T_SetToDefault: + appendStringInfoString(buf, "DEFAULT"); + break; + + case T_CurrentOfExpr: + { + CurrentOfExpr *cexpr = (CurrentOfExpr *) node; + + if (cexpr->cursor_name) + appendStringInfo(buf, "CURRENT OF %s", + quote_identifier(cexpr->cursor_name)); + else + appendStringInfo(buf, "CURRENT OF $%d", + cexpr->cursor_param); + } + break; + + case T_NextValueExpr: + { + NextValueExpr *nvexpr = (NextValueExpr *) node; + + /* + * This isn't exactly nextval(), but that seems close enough + * for EXPLAIN's purposes. + */ + appendStringInfoString(buf, "nextval("); + simple_quote_literal(buf, + generate_relation_name(nvexpr->seqid, + NIL)); + appendStringInfoChar(buf, ')'); + } + break; + + case T_InferenceElem: + { + InferenceElem *iexpr = (InferenceElem *) node; + bool save_varprefix; + bool need_parens; + + /* + * InferenceElem can only refer to target relation, so a + * prefix is not useful, and indeed would cause parse errors. + */ + save_varprefix = context->varprefix; + context->varprefix = false; + + /* + * Parenthesize the element unless it's a simple Var or a bare + * function call. Follows pg_get_indexdef_worker(). + */ + need_parens = !IsA(iexpr->expr, Var); + if (IsA(iexpr->expr, FuncExpr) && + ((FuncExpr *) iexpr->expr)->funcformat == + COERCE_EXPLICIT_CALL) + need_parens = false; + + if (need_parens) + appendStringInfoChar(buf, '('); + get_rule_expr((Node *) iexpr->expr, + context, false); + if (need_parens) + appendStringInfoChar(buf, ')'); + + context->varprefix = save_varprefix; + + if (iexpr->infercollid) + appendStringInfo(buf, " COLLATE %s", + generate_collation_name(iexpr->infercollid)); + + /* Add the operator class name, if not default */ + if (iexpr->inferopclass) + { + Oid inferopclass = iexpr->inferopclass; + Oid inferopcinputtype = get_opclass_input_type(iexpr->inferopclass); + + get_opclass_name(inferopclass, inferopcinputtype, buf); + } + } + break; + + case T_PartitionBoundSpec: + { + PartitionBoundSpec *spec = (PartitionBoundSpec *) node; + ListCell *cell; + char *sep; + + if (spec->is_default) + { + appendStringInfoString(buf, "DEFAULT"); + break; + } + + switch (spec->strategy) + { + case PARTITION_STRATEGY_HASH: + Assert(spec->modulus > 0 && spec->remainder >= 0); + Assert(spec->modulus > spec->remainder); + + appendStringInfoString(buf, "FOR VALUES"); + appendStringInfo(buf, " WITH (modulus %d, remainder %d)", + spec->modulus, spec->remainder); + break; + + case PARTITION_STRATEGY_LIST: + Assert(spec->listdatums != NIL); + + appendStringInfoString(buf, "FOR VALUES IN ("); + sep = ""; + foreach(cell, spec->listdatums) + { + Const *val = castNode(Const, lfirst(cell)); + + appendStringInfoString(buf, sep); + get_const_expr(val, context, -1); + sep = ", "; + } + + appendStringInfoChar(buf, ')'); + break; + + case PARTITION_STRATEGY_RANGE: + Assert(spec->lowerdatums != NIL && + spec->upperdatums != NIL && + list_length(spec->lowerdatums) == + list_length(spec->upperdatums)); + + appendStringInfo(buf, "FOR VALUES FROM %s TO %s", + get_range_partbound_string(spec->lowerdatums), + get_range_partbound_string(spec->upperdatums)); + break; + + default: + elog(ERROR, "unrecognized partition strategy: %d", + (int) spec->strategy); + break; + } + } + break; + + case T_List: + { + char *sep; + ListCell *l; + + sep = ""; + foreach(l, (List *) node) + { + appendStringInfoString(buf, sep); + get_rule_expr((Node *) lfirst(l), context, showimplicit); + sep = ", "; + } + } + break; + + case T_TableFunc: + get_tablefunc((TableFunc *) node, context, showimplicit); + break; + + default: + elog(ERROR, "unrecognized node type: %d", (int) nodeTag(node)); + break; + } +} + +/* + * get_rule_expr_toplevel - Parse back a toplevel expression + * + * Same as get_rule_expr(), except that if the expr is just a Var, we pass + * istoplevel = true not false to get_variable(). This causes whole-row Vars + * to get printed with decoration that will prevent expansion of "*". + * We need to use this in contexts such as ROW() and VALUES(), where the + * parser would expand "foo.*" appearing at top level. (In principle we'd + * use this in get_target_list() too, but that has additional worries about + * whether to print AS, so it needs to invoke get_variable() directly anyway.) + */ +static void +get_rule_expr_toplevel(Node *node, deparse_context *context, + bool showimplicit) +{ + if (node && IsA(node, Var)) + (void) get_variable((Var *) node, 0, true, context); + else + get_rule_expr(node, context, showimplicit); +} + +/* + * get_rule_expr_funccall - Parse back a function-call expression + * + * Same as get_rule_expr(), except that we guarantee that the output will + * look like a function call, or like one of the things the grammar treats as + * equivalent to a function call (see the func_expr_windowless production). + * This is needed in places where the grammar uses func_expr_windowless and + * you can't substitute a parenthesized a_expr. If what we have isn't going + * to look like a function call, wrap it in a dummy CAST() expression, which + * will satisfy the grammar --- and, indeed, is likely what the user wrote to + * produce such a thing. + */ +static void +get_rule_expr_funccall(Node *node, deparse_context *context, + bool showimplicit) +{ + if (looks_like_function(node)) + get_rule_expr(node, context, showimplicit); + else + { + StringInfo buf = context->buf; + + appendStringInfoString(buf, "CAST("); + /* no point in showing any top-level implicit cast */ + get_rule_expr(node, context, false); + appendStringInfo(buf, " AS %s)", + format_type_with_typemod(exprType(node), + exprTypmod(node))); + } +} + +/* + * Helper function to identify node types that satisfy func_expr_windowless. + * If in doubt, "false" is always a safe answer. + */ +static bool +looks_like_function(Node *node) +{ + if (node == NULL) + return false; /* probably shouldn't happen */ + switch (nodeTag(node)) + { + case T_FuncExpr: + /* OK, unless it's going to deparse as a cast */ + return (((FuncExpr *) node)->funcformat == COERCE_EXPLICIT_CALL); + case T_NullIfExpr: + case T_CoalesceExpr: + case T_MinMaxExpr: + case T_SQLValueFunction: + case T_XmlExpr: + /* these are all accepted by func_expr_common_subexpr */ + return true; + default: + break; + } + return false; +} + + +/* + * get_oper_expr - Parse back an OpExpr node + */ +static void +get_oper_expr(OpExpr *expr, deparse_context *context) +{ + StringInfo buf = context->buf; + Oid opno = expr->opno; + List *args = expr->args; + + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + if (list_length(args) == 2) + { + /* binary operator */ + Node *arg1 = (Node *) linitial(args); + Node *arg2 = (Node *) lsecond(args); + + get_rule_expr_paren(arg1, context, true, (Node *) expr); + appendStringInfo(buf, " %s ", + generate_operator_name(opno, + exprType(arg1), + exprType(arg2))); + get_rule_expr_paren(arg2, context, true, (Node *) expr); + } + else + { + /* unary operator --- but which side? */ + Node *arg = (Node *) linitial(args); + HeapTuple tp; + Form_pg_operator optup; + + tp = SearchSysCache1(OPEROID, ObjectIdGetDatum(opno)); + if (!HeapTupleIsValid(tp)) + elog(ERROR, "cache lookup failed for operator %u", opno); + optup = (Form_pg_operator) GETSTRUCT(tp); + switch (optup->oprkind) + { + case 'l': + appendStringInfo(buf, "%s ", + generate_operator_name(opno, + InvalidOid, + exprType(arg))); + get_rule_expr_paren(arg, context, true, (Node *) expr); + break; + case 'r': + get_rule_expr_paren(arg, context, true, (Node *) expr); + appendStringInfo(buf, " %s", + generate_operator_name(opno, + exprType(arg), + InvalidOid)); + break; + default: + elog(ERROR, "bogus oprkind: %d", optup->oprkind); + } + ReleaseSysCache(tp); + } + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); +} + +/* + * get_func_expr - Parse back a FuncExpr node + */ +static void +get_func_expr(FuncExpr *expr, deparse_context *context, + bool showimplicit) +{ + StringInfo buf = context->buf; + Oid funcoid = expr->funcid; + Oid argtypes[FUNC_MAX_ARGS]; + int nargs; + List *argnames; + bool use_variadic; + ListCell *l; + + /* + * If the function call came from an implicit coercion, then just show the + * first argument --- unless caller wants to see implicit coercions. + */ + if (expr->funcformat == COERCE_IMPLICIT_CAST && !showimplicit) + { + get_rule_expr_paren((Node *) linitial(expr->args), context, + false, (Node *) expr); + return; + } + + /* + * If the function call came from a cast, then show the first argument + * plus an explicit cast operation. + */ + if (expr->funcformat == COERCE_EXPLICIT_CAST || + expr->funcformat == COERCE_IMPLICIT_CAST) + { + Node *arg = linitial(expr->args); + Oid rettype = expr->funcresulttype; + int32 coercedTypmod; + + /* Get the typmod if this is a length-coercion function */ + (void) exprIsLengthCoercion((Node *) expr, &coercedTypmod); + + get_coercion_expr(arg, context, + rettype, coercedTypmod, + (Node *) expr); + + return; + } + + /* + * Normal function: display as proname(args). First we need to extract + * the argument datatypes. + */ + if (list_length(expr->args) > FUNC_MAX_ARGS) + ereport(ERROR, + (errcode(ERRCODE_TOO_MANY_ARGUMENTS), + errmsg("too many arguments"))); + nargs = 0; + argnames = NIL; + foreach(l, expr->args) + { + Node *arg = (Node *) lfirst(l); + + if (IsA(arg, NamedArgExpr)) + argnames = lappend(argnames, ((NamedArgExpr *) arg)->name); + argtypes[nargs] = exprType(arg); + nargs++; + } + + appendStringInfo(buf, "%s(", + generate_function_name(funcoid, nargs, + argnames, argtypes, + expr->funcvariadic, + &use_variadic, + context->special_exprkind)); + nargs = 0; + foreach(l, expr->args) + { + if (nargs++ > 0) + appendStringInfoString(buf, ", "); + if (use_variadic && lnext(expr->args, l) == NULL) + appendStringInfoString(buf, "VARIADIC "); + get_rule_expr((Node *) lfirst(l), context, true); + } + appendStringInfoChar(buf, ')'); +} + +/* + * get_agg_expr - Parse back an Aggref node + */ +static void +get_agg_expr(Aggref *aggref, deparse_context *context, + Aggref *original_aggref) +{ + StringInfo buf = context->buf; + Oid argtypes[FUNC_MAX_ARGS]; + int nargs; + bool use_variadic; + + /* + * For a combining aggregate, we look up and deparse the corresponding + * partial aggregate instead. This is necessary because our input + * argument list has been replaced; the new argument list always has just + * one element, which will point to a partial Aggref that supplies us with + * transition states to combine. + */ + if (DO_AGGSPLIT_COMBINE(aggref->aggsplit)) + { + TargetEntry *tle; + + + Assert(list_length(aggref->args) == 1); + tle = linitial_node(TargetEntry, aggref->args); + resolve_special_varno((Node *) tle->expr, context, + get_agg_combine_expr, original_aggref); + return; + } + + /* + * Mark as PARTIAL, if appropriate. We look to the original aggref so as + * to avoid printing this when recursing from the code just above. + */ + if (DO_AGGSPLIT_SKIPFINAL(original_aggref->aggsplit)) + appendStringInfoString(buf, "PARTIAL "); + + /* Extract the argument types as seen by the parser */ + nargs = get_aggregate_argtypes(aggref, argtypes); + + /* Print the aggregate name, schema-qualified if needed */ + appendStringInfo(buf, "%s(%s", + generate_function_name(aggref->aggfnoid, nargs, + NIL, argtypes, + aggref->aggvariadic, + &use_variadic, + context->special_exprkind), + (aggref->aggdistinct != NIL) ? "DISTINCT " : ""); + + if (AGGKIND_IS_ORDERED_SET(aggref->aggkind)) + { + /* + * Ordered-set aggregates do not use "*" syntax. Also, we needn't + * worry about inserting VARIADIC. So we can just dump the direct + * args as-is. + */ + Assert(!aggref->aggvariadic); + get_rule_expr((Node *) aggref->aggdirectargs, context, true); + Assert(aggref->aggorder != NIL); + appendStringInfoString(buf, ") WITHIN GROUP (ORDER BY "); + get_rule_orderby(aggref->aggorder, aggref->args, false, context); + } + else + { + /* aggstar can be set only in zero-argument aggregates */ + if (aggref->aggstar) + appendStringInfoChar(buf, '*'); + else + { + ListCell *l; + int i; + + i = 0; + foreach(l, aggref->args) + { + TargetEntry *tle = (TargetEntry *) lfirst(l); + Node *arg = (Node *) tle->expr; + + Assert(!IsA(arg, NamedArgExpr)); + if (tle->resjunk) + continue; + if (i++ > 0) + appendStringInfoString(buf, ", "); + if (use_variadic && i == nargs) + appendStringInfoString(buf, "VARIADIC "); + get_rule_expr(arg, context, true); + } + } + + if (aggref->aggorder != NIL) + { + appendStringInfoString(buf, " ORDER BY "); + get_rule_orderby(aggref->aggorder, aggref->args, false, context); + } + } + + if (aggref->aggfilter != NULL) + { + appendStringInfoString(buf, ") FILTER (WHERE "); + get_rule_expr((Node *) aggref->aggfilter, context, false); + } + + appendStringInfoChar(buf, ')'); +} + +/* + * This is a helper function for get_agg_expr(). It's used when we deparse + * a combining Aggref; resolve_special_varno locates the corresponding partial + * Aggref and then calls this. + */ +static void +get_agg_combine_expr(Node *node, deparse_context *context, void *callback_arg) +{ + Aggref *aggref; + Aggref *original_aggref = callback_arg; + + if (!IsA(node, Aggref)) + elog(ERROR, "combining Aggref does not point to an Aggref"); + + aggref = (Aggref *) node; + get_agg_expr(aggref, context, original_aggref); +} + +/* + * get_windowfunc_expr - Parse back a WindowFunc node + */ +static void +get_windowfunc_expr(WindowFunc *wfunc, deparse_context *context) +{ + StringInfo buf = context->buf; + Oid argtypes[FUNC_MAX_ARGS]; + int nargs; + List *argnames; + ListCell *l; + + if (list_length(wfunc->args) > FUNC_MAX_ARGS) + ereport(ERROR, + (errcode(ERRCODE_TOO_MANY_ARGUMENTS), + errmsg("too many arguments"))); + nargs = 0; + argnames = NIL; + foreach(l, wfunc->args) + { + Node *arg = (Node *) lfirst(l); + + if (IsA(arg, NamedArgExpr)) + argnames = lappend(argnames, ((NamedArgExpr *) arg)->name); + argtypes[nargs] = exprType(arg); + nargs++; + } + + appendStringInfo(buf, "%s(", + generate_function_name(wfunc->winfnoid, nargs, + argnames, argtypes, + false, NULL, + context->special_exprkind)); + /* winstar can be set only in zero-argument aggregates */ + if (wfunc->winstar) + appendStringInfoChar(buf, '*'); + else + get_rule_expr((Node *) wfunc->args, context, true); + + if (wfunc->aggfilter != NULL) + { + appendStringInfoString(buf, ") FILTER (WHERE "); + get_rule_expr((Node *) wfunc->aggfilter, context, false); + } + + appendStringInfoString(buf, ") OVER "); + + foreach(l, context->windowClause) + { + WindowClause *wc = (WindowClause *) lfirst(l); + + if (wc->winref == wfunc->winref) + { + if (wc->name) + appendStringInfoString(buf, quote_identifier(wc->name)); + else + get_rule_windowspec(wc, context->windowTList, context); + break; + } + } + if (l == NULL) + { + if (context->windowClause) + elog(ERROR, "could not find window clause for winref %u", + wfunc->winref); + + /* + * In EXPLAIN, we don't have window context information available, so + * we have to settle for this: + */ + appendStringInfoString(buf, "(?)"); + } +} + +/* ---------- + * get_coercion_expr + * + * Make a string representation of a value coerced to a specific type + * ---------- + */ +static void +get_coercion_expr(Node *arg, deparse_context *context, + Oid resulttype, int32 resulttypmod, + Node *parentNode) +{ + StringInfo buf = context->buf; + + /* + * Since parse_coerce.c doesn't immediately collapse application of + * length-coercion functions to constants, what we'll typically see in + * such cases is a Const with typmod -1 and a length-coercion function + * right above it. Avoid generating redundant output. However, beware of + * suppressing casts when the user actually wrote something like + * 'foo'::text::char(3). + * + * Note: it might seem that we are missing the possibility of needing to + * print a COLLATE clause for such a Const. However, a Const could only + * have nondefault collation in a post-constant-folding tree, in which the + * length coercion would have been folded too. See also the special + * handling of CollateExpr in coerce_to_target_type(): any collation + * marking will be above the coercion node, not below it. + */ + if (arg && IsA(arg, Const) && + ((Const *) arg)->consttype == resulttype && + ((Const *) arg)->consttypmod == -1) + { + /* Show the constant without normal ::typename decoration */ + get_const_expr((Const *) arg, context, -1); + } + else + { + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren(arg, context, false, parentNode); + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + } + appendStringInfo(buf, "::%s", + format_type_with_typemod(resulttype, resulttypmod)); +} + +/* ---------- + * get_const_expr + * + * Make a string representation of a Const + * + * showtype can be -1 to never show "::typename" decoration, or +1 to always + * show it, or 0 to show it only if the constant wouldn't be assumed to be + * the right type by default. + * + * If the Const's collation isn't default for its type, show that too. + * We mustn't do this when showtype is -1 (since that means the caller will + * print "::typename", and we can't put a COLLATE clause in between). It's + * caller's responsibility that collation isn't missed in such cases. + * ---------- + */ +static void +get_const_expr(Const *constval, deparse_context *context, int showtype) +{ + StringInfo buf = context->buf; + Oid typoutput; + bool typIsVarlena; + char *extval; + bool needlabel = false; + + if (constval->constisnull) + { + /* + * Always label the type of a NULL constant to prevent misdecisions + * about type when reparsing. + */ + appendStringInfoString(buf, "NULL"); + if (showtype >= 0) + { + appendStringInfo(buf, "::%s", + format_type_with_typemod(constval->consttype, + constval->consttypmod)); + get_const_collation(constval, context); + } + return; + } + + getTypeOutputInfo(constval->consttype, + &typoutput, &typIsVarlena); + + extval = OidOutputFunctionCall(typoutput, constval->constvalue); + + switch (constval->consttype) + { + case INT4OID: + + /* + * INT4 can be printed without any decoration, unless it is + * negative; in that case print it as '-nnn'::integer to ensure + * that the output will re-parse as a constant, not as a constant + * plus operator. In most cases we could get away with printing + * (-nnn) instead, because of the way that gram.y handles negative + * literals; but that doesn't work for INT_MIN, and it doesn't + * seem that much prettier anyway. + */ + if (extval[0] != '-') + appendStringInfoString(buf, extval); + else + { + appendStringInfo(buf, "'%s'", extval); + needlabel = true; /* we must attach a cast */ + } + break; + + case NUMERICOID: + + /* + * NUMERIC can be printed without quotes if it looks like a float + * constant (not an integer, and not Infinity or NaN) and doesn't + * have a leading sign (for the same reason as for INT4). + */ + if (isdigit((unsigned char) extval[0]) && + strcspn(extval, "eE.") != strlen(extval)) + { + appendStringInfoString(buf, extval); + } + else + { + appendStringInfo(buf, "'%s'", extval); + needlabel = true; /* we must attach a cast */ + } + break; + + case BITOID: + case VARBITOID: + appendStringInfo(buf, "B'%s'", extval); + break; + + case BOOLOID: + if (strcmp(extval, "t") == 0) + appendStringInfoString(buf, "true"); + else + appendStringInfoString(buf, "false"); + break; + + default: + simple_quote_literal(buf, extval); + break; + } + + pfree(extval); + + if (showtype < 0) + return; + + /* + * For showtype == 0, append ::typename unless the constant will be + * implicitly typed as the right type when it is read in. + * + * XXX this code has to be kept in sync with the behavior of the parser, + * especially make_const. + */ + switch (constval->consttype) + { + case BOOLOID: + case UNKNOWNOID: + /* These types can be left unlabeled */ + needlabel = false; + break; + case INT4OID: + /* We determined above whether a label is needed */ + break; + case NUMERICOID: + + /* + * Float-looking constants will be typed as numeric, which we + * checked above; but if there's a nondefault typmod we need to + * show it. + */ + needlabel |= (constval->consttypmod >= 0); + break; + default: + needlabel = true; + break; + } + if (needlabel || showtype > 0) + appendStringInfo(buf, "::%s", + format_type_with_typemod(constval->consttype, + constval->consttypmod)); + + get_const_collation(constval, context); +} + +/* + * helper for get_const_expr: append COLLATE if needed + */ +static void +get_const_collation(Const *constval, deparse_context *context) +{ + StringInfo buf = context->buf; + + if (OidIsValid(constval->constcollid)) + { + Oid typcollation = get_typcollation(constval->consttype); + + if (constval->constcollid != typcollation) + { + appendStringInfo(buf, " COLLATE %s", + generate_collation_name(constval->constcollid)); + } + } +} + +/* + * simple_quote_literal - Format a string as a SQL literal, append to buf + */ +static void +simple_quote_literal(StringInfo buf, const char *val) +{ + const char *valptr; + + /* + * We form the string literal according to the prevailing setting of + * standard_conforming_strings; we never use E''. User is responsible for + * making sure result is used correctly. + */ + appendStringInfoChar(buf, '\''); + for (valptr = val; *valptr; valptr++) + { + char ch = *valptr; + + if (SQL_STR_DOUBLE(ch, !standard_conforming_strings)) + appendStringInfoChar(buf, ch); + appendStringInfoChar(buf, ch); + } + appendStringInfoChar(buf, '\''); +} + + +/* ---------- + * get_sublink_expr - Parse back a sublink + * ---------- + */ +static void +get_sublink_expr(SubLink *sublink, deparse_context *context) +{ + StringInfo buf = context->buf; + Query *query = (Query *) (sublink->subselect); + char *opname = NULL; + bool need_paren; + + if (sublink->subLinkType == ARRAY_SUBLINK) + appendStringInfoString(buf, "ARRAY("); + else + appendStringInfoChar(buf, '('); + + /* + * Note that we print the name of only the first operator, when there are + * multiple combining operators. This is an approximation that could go + * wrong in various scenarios (operators in different schemas, renamed + * operators, etc) but there is not a whole lot we can do about it, since + * the syntax allows only one operator to be shown. + */ + if (sublink->testexpr) + { + if (IsA(sublink->testexpr, OpExpr)) + { + /* single combining operator */ + OpExpr *opexpr = (OpExpr *) sublink->testexpr; + + get_rule_expr(linitial(opexpr->args), context, true); + opname = generate_operator_name(opexpr->opno, + exprType(linitial(opexpr->args)), + exprType(lsecond(opexpr->args))); + } + else if (IsA(sublink->testexpr, BoolExpr)) + { + /* multiple combining operators, = or <> cases */ + char *sep; + ListCell *l; + + appendStringInfoChar(buf, '('); + sep = ""; + foreach(l, ((BoolExpr *) sublink->testexpr)->args) + { + OpExpr *opexpr = lfirst_node(OpExpr, l); + + appendStringInfoString(buf, sep); + get_rule_expr(linitial(opexpr->args), context, true); + if (!opname) + opname = generate_operator_name(opexpr->opno, + exprType(linitial(opexpr->args)), + exprType(lsecond(opexpr->args))); + sep = ", "; + } + appendStringInfoChar(buf, ')'); + } + else if (IsA(sublink->testexpr, RowCompareExpr)) + { + /* multiple combining operators, < <= > >= cases */ + RowCompareExpr *rcexpr = (RowCompareExpr *) sublink->testexpr; + + appendStringInfoChar(buf, '('); + get_rule_expr((Node *) rcexpr->largs, context, true); + opname = generate_operator_name(linitial_oid(rcexpr->opnos), + exprType(linitial(rcexpr->largs)), + exprType(linitial(rcexpr->rargs))); + appendStringInfoChar(buf, ')'); + } + else + elog(ERROR, "unrecognized testexpr type: %d", + (int) nodeTag(sublink->testexpr)); + } + + need_paren = true; + + switch (sublink->subLinkType) + { + case EXISTS_SUBLINK: + appendStringInfoString(buf, "EXISTS "); + break; + + case ANY_SUBLINK: + if (strcmp(opname, "=") == 0) /* Represent = ANY as IN */ + appendStringInfoString(buf, " IN "); + else + appendStringInfo(buf, " %s ANY ", opname); + break; + + case ALL_SUBLINK: + appendStringInfo(buf, " %s ALL ", opname); + break; + + case ROWCOMPARE_SUBLINK: + appendStringInfo(buf, " %s ", opname); + break; + + case EXPR_SUBLINK: + case MULTIEXPR_SUBLINK: + case ARRAY_SUBLINK: + need_paren = false; + break; + + case CTE_SUBLINK: /* shouldn't occur in a SubLink */ + default: + elog(ERROR, "unrecognized sublink type: %d", + (int) sublink->subLinkType); + break; + } + + if (need_paren) + appendStringInfoChar(buf, '('); + + get_query_def(query, buf, context->namespaces, NULL, + context->prettyFlags, context->wrapColumn, + context->indentLevel); + + if (need_paren) + appendStringInfoString(buf, "))"); + else + appendStringInfoChar(buf, ')'); +} + + +/* ---------- + * get_tablefunc - Parse back a table function + * ---------- + */ +static void +get_tablefunc(TableFunc *tf, deparse_context *context, bool showimplicit) +{ + StringInfo buf = context->buf; + + /* XMLTABLE is the only existing implementation. */ + + appendStringInfoString(buf, "XMLTABLE("); + + if (tf->ns_uris != NIL) + { + ListCell *lc1, + *lc2; + bool first = true; + + appendStringInfoString(buf, "XMLNAMESPACES ("); + forboth(lc1, tf->ns_uris, lc2, tf->ns_names) + { + Node *expr = (Node *) lfirst(lc1); + char *name = strVal(lfirst(lc2)); + + if (!first) + appendStringInfoString(buf, ", "); + else + first = false; + + if (name != NULL) + { + get_rule_expr(expr, context, showimplicit); + appendStringInfo(buf, " AS %s", name); + } + else + { + appendStringInfoString(buf, "DEFAULT "); + get_rule_expr(expr, context, showimplicit); + } + } + appendStringInfoString(buf, "), "); + } + + appendStringInfoChar(buf, '('); + get_rule_expr((Node *) tf->rowexpr, context, showimplicit); + appendStringInfoString(buf, ") PASSING ("); + get_rule_expr((Node *) tf->docexpr, context, showimplicit); + appendStringInfoChar(buf, ')'); + + if (tf->colexprs != NIL) + { + ListCell *l1; + ListCell *l2; + ListCell *l3; + ListCell *l4; + ListCell *l5; + int colnum = 0; + + appendStringInfoString(buf, " COLUMNS "); + forfive(l1, tf->colnames, l2, tf->coltypes, l3, tf->coltypmods, + l4, tf->colexprs, l5, tf->coldefexprs) + { + char *colname = strVal(lfirst(l1)); + Oid typid = lfirst_oid(l2); + int32 typmod = lfirst_int(l3); + Node *colexpr = (Node *) lfirst(l4); + Node *coldefexpr = (Node *) lfirst(l5); + bool ordinality = (tf->ordinalitycol == colnum); + bool notnull = bms_is_member(colnum, tf->notnulls); + + if (colnum > 0) + appendStringInfoString(buf, ", "); + colnum++; + + appendStringInfo(buf, "%s %s", quote_identifier(colname), + ordinality ? "FOR ORDINALITY" : + format_type_with_typemod(typid, typmod)); + if (ordinality) + continue; + + if (coldefexpr != NULL) + { + appendStringInfoString(buf, " DEFAULT ("); + get_rule_expr((Node *) coldefexpr, context, showimplicit); + appendStringInfoChar(buf, ')'); + } + if (colexpr != NULL) + { + appendStringInfoString(buf, " PATH ("); + get_rule_expr((Node *) colexpr, context, showimplicit); + appendStringInfoChar(buf, ')'); + } + if (notnull) + appendStringInfoString(buf, " NOT NULL"); + } + } + + appendStringInfoChar(buf, ')'); +} + +/* ---------- + * get_from_clause - Parse back a FROM clause + * + * "prefix" is the keyword that denotes the start of the list of FROM + * elements. It is FROM when used to parse back SELECT and UPDATE, but + * is USING when parsing back DELETE. + * ---------- + */ +static void +get_from_clause(Query *query, const char *prefix, deparse_context *context) +{ + StringInfo buf = context->buf; + bool first = true; + ListCell *l; + + /* + * We use the query's jointree as a guide to what to print. However, we + * must ignore auto-added RTEs that are marked not inFromCl. (These can + * only appear at the top level of the jointree, so it's sufficient to + * check here.) This check also ensures we ignore the rule pseudo-RTEs + * for NEW and OLD. + */ + foreach(l, query->jointree->fromlist) + { + Node *jtnode = (Node *) lfirst(l); + + if (IsA(jtnode, RangeTblRef)) + { + int varno = ((RangeTblRef *) jtnode)->rtindex; + RangeTblEntry *rte = rt_fetch(varno, query->rtable); + + if (!rte->inFromCl) + continue; + } + + if (first) + { + appendContextKeyword(context, prefix, + -PRETTYINDENT_STD, PRETTYINDENT_STD, 2); + first = false; + + get_from_clause_item(jtnode, query, context); + } + else + { + StringInfoData itembuf; + + appendStringInfoString(buf, ", "); + + /* + * Put the new FROM item's text into itembuf so we can decide + * after we've got it whether or not it needs to go on a new line. + */ + initStringInfo(&itembuf); + context->buf = &itembuf; + + get_from_clause_item(jtnode, query, context); + + /* Restore context's output buffer */ + context->buf = buf; + + /* Consider line-wrapping if enabled */ + if (PRETTY_INDENT(context) && context->wrapColumn >= 0) + { + /* Does the new item start with a new line? */ + if (itembuf.len > 0 && itembuf.data[0] == '\n') + { + /* If so, we shouldn't add anything */ + /* instead, remove any trailing spaces currently in buf */ + removeStringInfoSpaces(buf); + } + else + { + char *trailing_nl; + + /* Locate the start of the current line in the buffer */ + trailing_nl = strrchr(buf->data, '\n'); + if (trailing_nl == NULL) + trailing_nl = buf->data; + else + trailing_nl++; + + /* + * Add a newline, plus some indentation, if the new item + * would cause an overflow. + */ + if (strlen(trailing_nl) + itembuf.len > context->wrapColumn) + appendContextKeyword(context, "", -PRETTYINDENT_STD, + PRETTYINDENT_STD, + PRETTYINDENT_VAR); + } + } + + /* Add the new item */ + appendStringInfoString(buf, itembuf.data); + + /* clean up */ + pfree(itembuf.data); + } + } +} + +static void +get_from_clause_item(Node *jtnode, Query *query, deparse_context *context) +{ + StringInfo buf = context->buf; + deparse_namespace *dpns = (deparse_namespace *) linitial(context->namespaces); + + if (IsA(jtnode, RangeTblRef)) + { + int varno = ((RangeTblRef *) jtnode)->rtindex; + RangeTblEntry *rte = rt_fetch(varno, query->rtable); + char *refname = get_rtable_name(varno, context); + deparse_columns *colinfo = deparse_columns_fetch(varno, dpns); + RangeTblFunction *rtfunc1 = NULL; + bool printalias; + CitusRTEKind rteKind = GetRangeTblKind(rte); + + if (rte->lateral) + appendStringInfoString(buf, "LATERAL "); + + /* Print the FROM item proper */ + switch (rte->rtekind) + { + case RTE_RELATION: + /* Normal relation RTE */ + appendStringInfo(buf, "%s%s", + only_marker(rte), + generate_relation_or_shard_name(rte->relid, + context->distrelid, + context->shardid, + context->namespaces)); + break; + case RTE_SUBQUERY: + /* Subquery RTE */ + appendStringInfoChar(buf, '('); + get_query_def(rte->subquery, buf, context->namespaces, NULL, + context->prettyFlags, context->wrapColumn, + context->indentLevel); + appendStringInfoChar(buf, ')'); + break; + case RTE_FUNCTION: + /* if it's a shard, do differently */ + if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) + { + char *fragmentSchemaName = NULL; + char *fragmentTableName = NULL; + + ExtractRangeTblExtraData(rte, NULL, &fragmentSchemaName, &fragmentTableName, NULL); + + /* use schema and table name from the remote alias */ + appendStringInfo(buf, "%s%s", + only_marker(rte), + generate_fragment_name(fragmentSchemaName, + fragmentTableName)); + break; + } + + /* Function RTE */ + rtfunc1 = (RangeTblFunction *) linitial(rte->functions); + + /* + * Omit ROWS FROM() syntax for just one function, unless it + * has both a coldeflist and WITH ORDINALITY. If it has both, + * we must use ROWS FROM() syntax to avoid ambiguity about + * whether the coldeflist includes the ordinality column. + */ + if (list_length(rte->functions) == 1 && + (rtfunc1->funccolnames == NIL || !rte->funcordinality)) + { + get_rule_expr_funccall(rtfunc1->funcexpr, context, true); + /* we'll print the coldeflist below, if it has one */ + } + else + { + bool all_unnest; + ListCell *lc; + + /* + * If all the function calls in the list are to unnest, + * and none need a coldeflist, then collapse the list back + * down to UNNEST(args). (If we had more than one + * built-in unnest function, this would get more + * difficult.) + * + * XXX This is pretty ugly, since it makes not-terribly- + * future-proof assumptions about what the parser would do + * with the output; but the alternative is to emit our + * nonstandard ROWS FROM() notation for what might have + * been a perfectly spec-compliant multi-argument + * UNNEST(). + */ + all_unnest = true; + foreach(lc, rte->functions) + { + RangeTblFunction *rtfunc = (RangeTblFunction *) lfirst(lc); + + if (!IsA(rtfunc->funcexpr, FuncExpr) || + ((FuncExpr *) rtfunc->funcexpr)->funcid != F_ARRAY_UNNEST || + rtfunc->funccolnames != NIL) + { + all_unnest = false; + break; + } + } + + if (all_unnest) + { + List *allargs = NIL; + + foreach(lc, rte->functions) + { + RangeTblFunction *rtfunc = (RangeTblFunction *) lfirst(lc); + List *args = ((FuncExpr *) rtfunc->funcexpr)->args; + + allargs = list_concat(allargs, args); + } + + appendStringInfoString(buf, "UNNEST("); + get_rule_expr((Node *) allargs, context, true); + appendStringInfoChar(buf, ')'); + } + else + { + int funcno = 0; + + appendStringInfoString(buf, "ROWS FROM("); + foreach(lc, rte->functions) + { + RangeTblFunction *rtfunc = (RangeTblFunction *) lfirst(lc); + + if (funcno > 0) + appendStringInfoString(buf, ", "); + get_rule_expr_funccall(rtfunc->funcexpr, context, true); + if (rtfunc->funccolnames != NIL) + { + /* Reconstruct the column definition list */ + appendStringInfoString(buf, " AS "); + get_from_clause_coldeflist(rtfunc, + NULL, + context); + } + funcno++; + } + appendStringInfoChar(buf, ')'); + } + /* prevent printing duplicate coldeflist below */ + rtfunc1 = NULL; + } + if (rte->funcordinality) + appendStringInfoString(buf, " WITH ORDINALITY"); + break; + case RTE_TABLEFUNC: + get_tablefunc(rte->tablefunc, context, true); + break; + case RTE_VALUES: + /* Values list RTE */ + appendStringInfoChar(buf, '('); + get_values_def(rte->values_lists, context); + appendStringInfoChar(buf, ')'); + break; + case RTE_CTE: + appendStringInfoString(buf, quote_identifier(rte->ctename)); + break; + default: + elog(ERROR, "unrecognized RTE kind: %d", (int) rte->rtekind); + break; + } + + /* Print the relation alias, if needed */ + printalias = false; + if (rte->alias != NULL) + { + /* Always print alias if user provided one */ + printalias = true; + } + else if (colinfo->printaliases) + { + /* Always print alias if we need to print column aliases */ + printalias = true; + } + else if (rte->rtekind == RTE_RELATION) + { + /* + * No need to print alias if it's same as relation name (this + * would normally be the case, but not if set_rtable_names had to + * resolve a conflict). + */ + if (strcmp(refname, get_relation_name(rte->relid)) != 0) + printalias = true; + } + else if (rte->rtekind == RTE_FUNCTION) + { + /* + * For a function RTE, always print alias. This covers possible + * renaming of the function and/or instability of the + * FigureColname rules for things that aren't simple functions. + * Note we'd need to force it anyway for the columndef list case. + */ + printalias = true; + } + else if (rte->rtekind == RTE_VALUES) + { + /* Alias is syntactically required for VALUES */ + printalias = true; + } + else if (rte->rtekind == RTE_CTE) + { + /* + * No need to print alias if it's same as CTE name (this would + * normally be the case, but not if set_rtable_names had to + * resolve a conflict). + */ + if (strcmp(refname, rte->ctename) != 0) + printalias = true; + } + else if (rte->rtekind == RTE_SUBQUERY) + { + /* subquery requires alias too */ + printalias = true; + } + if (printalias) + appendStringInfo(buf, " %s", quote_identifier(refname)); + + /* Print the column definitions or aliases, if needed */ + if (rtfunc1 && rtfunc1->funccolnames != NIL) + { + /* Reconstruct the columndef list, which is also the aliases */ + get_from_clause_coldeflist(rtfunc1, colinfo, context); + } + else if (GetRangeTblKind(rte) != CITUS_RTE_SHARD) + { + /* Else print column aliases as needed */ + get_column_alias_list(colinfo, context); + } + /* check if column's are given aliases in distributed tables */ + else if (colinfo->parentUsing != NIL) + { + Assert(colinfo->printaliases); + get_column_alias_list(colinfo, context); + } + + /* Tablesample clause must go after any alias */ + if ((rteKind == CITUS_RTE_RELATION || rteKind == CITUS_RTE_SHARD) && + rte->tablesample) + { + get_tablesample_def(rte->tablesample, context); + } + } + else if (IsA(jtnode, JoinExpr)) + { + JoinExpr *j = (JoinExpr *) jtnode; + deparse_columns *colinfo = deparse_columns_fetch(j->rtindex, dpns); + bool need_paren_on_right; + + need_paren_on_right = PRETTY_PAREN(context) && + !IsA(j->rarg, RangeTblRef) && + !(IsA(j->rarg, JoinExpr) &&((JoinExpr *) j->rarg)->alias != NULL); + + if (!PRETTY_PAREN(context) || j->alias != NULL) + appendStringInfoChar(buf, '('); + + get_from_clause_item(j->larg, query, context); + + switch (j->jointype) + { + case JOIN_INNER: + if (j->quals) + appendContextKeyword(context, " JOIN ", + -PRETTYINDENT_STD, + PRETTYINDENT_STD, + PRETTYINDENT_JOIN); + else + appendContextKeyword(context, " CROSS JOIN ", + -PRETTYINDENT_STD, + PRETTYINDENT_STD, + PRETTYINDENT_JOIN); + break; + case JOIN_LEFT: + appendContextKeyword(context, " LEFT JOIN ", + -PRETTYINDENT_STD, + PRETTYINDENT_STD, + PRETTYINDENT_JOIN); + break; + case JOIN_FULL: + appendContextKeyword(context, " FULL JOIN ", + -PRETTYINDENT_STD, + PRETTYINDENT_STD, + PRETTYINDENT_JOIN); + break; + case JOIN_RIGHT: + appendContextKeyword(context, " RIGHT JOIN ", + -PRETTYINDENT_STD, + PRETTYINDENT_STD, + PRETTYINDENT_JOIN); + break; + default: + elog(ERROR, "unrecognized join type: %d", + (int) j->jointype); + } + + if (need_paren_on_right) + appendStringInfoChar(buf, '('); + get_from_clause_item(j->rarg, query, context); + if (need_paren_on_right) + appendStringInfoChar(buf, ')'); + + if (j->usingClause) + { + ListCell *lc; + bool first = true; + + appendStringInfoString(buf, " USING ("); + /* Use the assigned names, not what's in usingClause */ + foreach(lc, colinfo->usingNames) + { + char *colname = (char *) lfirst(lc); + + if (first) + first = false; + else + appendStringInfoString(buf, ", "); + appendStringInfoString(buf, quote_identifier(colname)); + } + appendStringInfoChar(buf, ')'); + } + else if (j->quals) + { + appendStringInfoString(buf, " ON "); + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr(j->quals, context, false); + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + } + else if (j->jointype != JOIN_INNER) + { + /* If we didn't say CROSS JOIN above, we must provide an ON */ + appendStringInfoString(buf, " ON TRUE"); + } + + if (!PRETTY_PAREN(context) || j->alias != NULL) + appendStringInfoChar(buf, ')'); + + /* Yes, it's correct to put alias after the right paren ... */ + if (j->alias != NULL) + { + /* + * Note that it's correct to emit an alias clause if and only if + * there was one originally. Otherwise we'd be converting a named + * join to unnamed or vice versa, which creates semantic + * subtleties we don't want. However, we might print a different + * alias name than was there originally. + */ + appendStringInfo(buf, " %s", + quote_identifier(get_rtable_name(j->rtindex, + context))); + get_column_alias_list(colinfo, context); + } + } + else + elog(ERROR, "unrecognized node type: %d", + (int) nodeTag(jtnode)); +} + +/* + * get_column_alias_list - print column alias list for an RTE + * + * Caller must already have printed the relation's alias name. + */ +static void +get_column_alias_list(deparse_columns *colinfo, deparse_context *context) +{ + StringInfo buf = context->buf; + int i; + bool first = true; + + /* Don't print aliases if not needed */ + if (!colinfo->printaliases) + return; + + for (i = 0; i < colinfo->num_new_cols; i++) + { + char *colname = colinfo->new_colnames[i]; + + if (first) + { + appendStringInfoChar(buf, '('); + first = false; + } + else + appendStringInfoString(buf, ", "); + appendStringInfoString(buf, quote_identifier(colname)); + } + if (!first) + appendStringInfoChar(buf, ')'); +} + +/* + * get_from_clause_coldeflist - reproduce FROM clause coldeflist + * + * When printing a top-level coldeflist (which is syntactically also the + * relation's column alias list), use column names from colinfo. But when + * printing a coldeflist embedded inside ROWS FROM(), we prefer to use the + * original coldeflist's names, which are available in rtfunc->funccolnames. + * Pass NULL for colinfo to select the latter behavior. + * + * The coldeflist is appended immediately (no space) to buf. Caller is + * responsible for ensuring that an alias or AS is present before it. + */ +static void +get_from_clause_coldeflist(RangeTblFunction *rtfunc, + deparse_columns *colinfo, + deparse_context *context) +{ + StringInfo buf = context->buf; + ListCell *l1; + ListCell *l2; + ListCell *l3; + ListCell *l4; + int i; + + appendStringInfoChar(buf, '('); + + i = 0; + forfour(l1, rtfunc->funccoltypes, + l2, rtfunc->funccoltypmods, + l3, rtfunc->funccolcollations, + l4, rtfunc->funccolnames) + { + Oid atttypid = lfirst_oid(l1); + int32 atttypmod = lfirst_int(l2); + Oid attcollation = lfirst_oid(l3); + char *attname; + + if (colinfo) + attname = colinfo->colnames[i]; + else + attname = strVal(lfirst(l4)); + + Assert(attname); /* shouldn't be any dropped columns here */ + + if (i > 0) + appendStringInfoString(buf, ", "); + appendStringInfo(buf, "%s %s", + quote_identifier(attname), + format_type_with_typemod(atttypid, atttypmod)); + if (OidIsValid(attcollation) && + attcollation != get_typcollation(atttypid)) + appendStringInfo(buf, " COLLATE %s", + generate_collation_name(attcollation)); + + i++; + } + + appendStringInfoChar(buf, ')'); +} + +/* + * get_tablesample_def - print a TableSampleClause + */ +static void +get_tablesample_def(TableSampleClause *tablesample, deparse_context *context) +{ + StringInfo buf = context->buf; + Oid argtypes[1]; + int nargs; + ListCell *l; + + /* + * We should qualify the handler's function name if it wouldn't be + * resolved by lookup in the current search path. + */ + argtypes[0] = INTERNALOID; + appendStringInfo(buf, " TABLESAMPLE %s (", + generate_function_name(tablesample->tsmhandler, 1, + NIL, argtypes, + false, NULL, EXPR_KIND_NONE)); + + nargs = 0; + foreach(l, tablesample->args) + { + if (nargs++ > 0) + appendStringInfoString(buf, ", "); + get_rule_expr((Node *) lfirst(l), context, false); + } + appendStringInfoChar(buf, ')'); + + if (tablesample->repeatable != NULL) + { + appendStringInfoString(buf, " REPEATABLE ("); + get_rule_expr((Node *) tablesample->repeatable, context, false); + appendStringInfoChar(buf, ')'); + } +} + +char * +pg_get_triggerdef_command(Oid triggerId) +{ + Assert(OidIsValid(triggerId)); + + /* no need to have pretty SQL command */ + bool prettyOutput = false; + return pg_get_triggerdef_worker(triggerId, prettyOutput); +} + + +char * +pg_get_statisticsobj_worker(Oid statextid, bool missing_ok) +{ + StringInfoData buf; + int colno; + bool isnull; + int i; + + HeapTuple statexttup = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(statextid)); + + if (!HeapTupleIsValid(statexttup)) + { + if (missing_ok) + { + return NULL; + } + elog(ERROR, "cache lookup failed for statistics object %u", statextid); + } + + Form_pg_statistic_ext statextrec = (Form_pg_statistic_ext) GETSTRUCT(statexttup); + + initStringInfo(&buf); + + char *nsp = get_namespace_name(statextrec->stxnamespace); + appendStringInfo(&buf, "CREATE STATISTICS %s", + quote_qualified_identifier(nsp, + NameStr(statextrec->stxname))); + + /* + * Decode the stxkind column so that we know which stats types to print. + */ + Datum datum = SysCacheGetAttr(STATEXTOID, statexttup, + Anum_pg_statistic_ext_stxkind, &isnull); + Assert(!isnull); + ArrayType *arr = DatumGetArrayTypeP(datum); + if (ARR_NDIM(arr) != 1 || + ARR_HASNULL(arr) || + ARR_ELEMTYPE(arr) != CHAROID) + { + elog(ERROR, "stxkind is not a 1-D char array"); + } + char *enabled = (char *) ARR_DATA_PTR(arr); + + bool ndistinct_enabled = false; + bool dependencies_enabled = false; + bool mcv_enabled = false; + + for (i = 0; i < ARR_DIMS(arr)[0]; i++) + { + if (enabled[i] == STATS_EXT_NDISTINCT) + { + ndistinct_enabled = true; + } + if (enabled[i] == STATS_EXT_DEPENDENCIES) + { + dependencies_enabled = true; + } + if (enabled[i] == STATS_EXT_MCV) + { + mcv_enabled = true; + } + } + + /* + * If any option is disabled, then we'll need to append the types clause + * to show which options are enabled. We omit the types clause on purpose + * when all options are enabled, so a pg_dump/pg_restore will create all + * statistics types on a newer postgres version, if the statistics had all + * options enabled on the original version. + */ + if (!ndistinct_enabled || !dependencies_enabled || !mcv_enabled) + { + bool gotone = false; + + appendStringInfoString(&buf, " ("); + + if (ndistinct_enabled) + { + appendStringInfoString(&buf, "ndistinct"); + gotone = true; + } + + if (dependencies_enabled) + { + appendStringInfo(&buf, "%sdependencies", gotone ? ", " : ""); + gotone = true; + } + + if (mcv_enabled) + { + appendStringInfo(&buf, "%smcv", gotone ? ", " : ""); + } + + appendStringInfoChar(&buf, ')'); + } + + appendStringInfoString(&buf, " ON "); + + for (colno = 0; colno < statextrec->stxkeys.dim1; colno++) + { + AttrNumber attnum = statextrec->stxkeys.values[colno]; + + if (colno > 0) + { + appendStringInfoString(&buf, ", "); + } + + char *attname = get_attname(statextrec->stxrelid, attnum, false); + + appendStringInfoString(&buf, quote_identifier(attname)); + } + + appendStringInfo(&buf, " FROM %s", + generate_relation_name(statextrec->stxrelid, NIL)); + + ReleaseSysCache(statexttup); + + return buf.data; +} + + +static char * +pg_get_triggerdef_worker(Oid trigid, bool pretty) +{ + HeapTuple ht_trig; + Form_pg_trigger trigrec; + StringInfoData buf; + Relation tgrel; + ScanKeyData skey[1]; + SysScanDesc tgscan; + int findx = 0; + char *tgname; + char *tgoldtable; + char *tgnewtable; + Oid argtypes[1]; /* dummy */ + Datum value; + bool isnull; + + /* + * Fetch the pg_trigger tuple by the Oid of the trigger + */ + tgrel = table_open(TriggerRelationId, AccessShareLock); + + ScanKeyInit(&skey[0], + Anum_pg_trigger_oid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(trigid)); + + tgscan = systable_beginscan(tgrel, TriggerOidIndexId, true, + NULL, 1, skey); + + ht_trig = systable_getnext(tgscan); + + if (!HeapTupleIsValid(ht_trig)) + { + systable_endscan(tgscan); + table_close(tgrel, AccessShareLock); + return NULL; + } + + trigrec = (Form_pg_trigger) GETSTRUCT(ht_trig); + + /* + * Start the trigger definition. Note that the trigger's name should never + * be schema-qualified, but the trigger rel's name may be. + */ + initStringInfo(&buf); + + tgname = NameStr(trigrec->tgname); + appendStringInfo(&buf, "CREATE %sTRIGGER %s ", + OidIsValid(trigrec->tgconstraint) ? "CONSTRAINT " : "", + quote_identifier(tgname)); + + if (TRIGGER_FOR_BEFORE(trigrec->tgtype)) + appendStringInfoString(&buf, "BEFORE"); + else if (TRIGGER_FOR_AFTER(trigrec->tgtype)) + appendStringInfoString(&buf, "AFTER"); + else if (TRIGGER_FOR_INSTEAD(trigrec->tgtype)) + appendStringInfoString(&buf, "INSTEAD OF"); + else + elog(ERROR, "unexpected tgtype value: %d", trigrec->tgtype); + + if (TRIGGER_FOR_INSERT(trigrec->tgtype)) + { + appendStringInfoString(&buf, " INSERT"); + findx++; + } + if (TRIGGER_FOR_DELETE(trigrec->tgtype)) + { + if (findx > 0) + appendStringInfoString(&buf, " OR DELETE"); + else + appendStringInfoString(&buf, " DELETE"); + findx++; + } + if (TRIGGER_FOR_UPDATE(trigrec->tgtype)) + { + if (findx > 0) + appendStringInfoString(&buf, " OR UPDATE"); + else + appendStringInfoString(&buf, " UPDATE"); + findx++; + /* tgattr is first var-width field, so OK to access directly */ + if (trigrec->tgattr.dim1 > 0) + { + int i; + + appendStringInfoString(&buf, " OF "); + for (i = 0; i < trigrec->tgattr.dim1; i++) + { + char *attname; + + if (i > 0) + appendStringInfoString(&buf, ", "); + attname = get_attname(trigrec->tgrelid, + trigrec->tgattr.values[i], false); + appendStringInfoString(&buf, quote_identifier(attname)); + } + } + } + if (TRIGGER_FOR_TRUNCATE(trigrec->tgtype)) + { + if (findx > 0) + appendStringInfoString(&buf, " OR TRUNCATE"); + else + appendStringInfoString(&buf, " TRUNCATE"); + findx++; + } + + /* + * In non-pretty mode, always schema-qualify the target table name for + * safety. In pretty mode, schema-qualify only if not visible. + */ + appendStringInfo(&buf, " ON %s ", + pretty ? + generate_relation_name(trigrec->tgrelid, NIL) : + generate_qualified_relation_name(trigrec->tgrelid)); + + if (OidIsValid(trigrec->tgconstraint)) + { + if (OidIsValid(trigrec->tgconstrrelid)) + appendStringInfo(&buf, "FROM %s ", + generate_relation_name(trigrec->tgconstrrelid, NIL)); + if (!trigrec->tgdeferrable) + appendStringInfoString(&buf, "NOT "); + appendStringInfoString(&buf, "DEFERRABLE INITIALLY "); + if (trigrec->tginitdeferred) + appendStringInfoString(&buf, "DEFERRED "); + else + appendStringInfoString(&buf, "IMMEDIATE "); + } + + value = fastgetattr(ht_trig, Anum_pg_trigger_tgoldtable, + tgrel->rd_att, &isnull); + if (!isnull) + tgoldtable = NameStr(*DatumGetName(value)); + else + tgoldtable = NULL; + value = fastgetattr(ht_trig, Anum_pg_trigger_tgnewtable, + tgrel->rd_att, &isnull); + if (!isnull) + tgnewtable = NameStr(*DatumGetName(value)); + else + tgnewtable = NULL; + if (tgoldtable != NULL || tgnewtable != NULL) + { + appendStringInfoString(&buf, "REFERENCING "); + if (tgoldtable != NULL) + appendStringInfo(&buf, "OLD TABLE AS %s ", + quote_identifier(tgoldtable)); + if (tgnewtable != NULL) + appendStringInfo(&buf, "NEW TABLE AS %s ", + quote_identifier(tgnewtable)); + } + + if (TRIGGER_FOR_ROW(trigrec->tgtype)) + appendStringInfoString(&buf, "FOR EACH ROW "); + else + appendStringInfoString(&buf, "FOR EACH STATEMENT "); + + /* If the trigger has a WHEN qualification, add that */ + value = fastgetattr(ht_trig, Anum_pg_trigger_tgqual, + tgrel->rd_att, &isnull); + if (!isnull) + { + Node *qual; + char relkind; + deparse_context context; + deparse_namespace dpns; + RangeTblEntry *oldrte; + RangeTblEntry *newrte; + + appendStringInfoString(&buf, "WHEN ("); + + qual = stringToNode(TextDatumGetCString(value)); + + relkind = get_rel_relkind(trigrec->tgrelid); + + /* Build minimal OLD and NEW RTEs for the rel */ + oldrte = makeNode(RangeTblEntry); + oldrte->rtekind = RTE_RELATION; + oldrte->relid = trigrec->tgrelid; + oldrte->relkind = relkind; + oldrte->rellockmode = AccessShareLock; + oldrte->alias = makeAlias("old", NIL); + oldrte->eref = oldrte->alias; + oldrte->lateral = false; + oldrte->inh = false; + oldrte->inFromCl = true; + + newrte = makeNode(RangeTblEntry); + newrte->rtekind = RTE_RELATION; + newrte->relid = trigrec->tgrelid; + newrte->relkind = relkind; + newrte->rellockmode = AccessShareLock; + newrte->alias = makeAlias("new", NIL); + newrte->eref = newrte->alias; + newrte->lateral = false; + newrte->inh = false; + newrte->inFromCl = true; + + /* Build two-element rtable */ + memset(&dpns, 0, sizeof(dpns)); + dpns.rtable = list_make2(oldrte, newrte); + dpns.ctes = NIL; + set_rtable_names(&dpns, NIL, NULL); + set_simple_column_names(&dpns); + + /* Set up context with one-deep namespace stack */ + context.buf = &buf; + context.namespaces = list_make1(&dpns); + context.windowClause = NIL; + context.windowTList = NIL; + context.varprefix = true; + context.prettyFlags = pretty ? (PRETTYFLAG_PAREN | PRETTYFLAG_INDENT | PRETTYFLAG_SCHEMA) : PRETTYFLAG_INDENT; + context.wrapColumn = WRAP_COLUMN_DEFAULT; + context.indentLevel = PRETTYINDENT_STD; + context.special_exprkind = EXPR_KIND_NONE; + context.appendparents = NULL; + + get_rule_expr(qual, &context, false); + + appendStringInfoString(&buf, ") "); + } + + appendStringInfo(&buf, "EXECUTE FUNCTION %s(", + generate_function_name(trigrec->tgfoid, 0, + NIL, argtypes, + false, NULL, EXPR_KIND_NONE)); + + if (trigrec->tgnargs > 0) + { + char *p; + int i; + + value = fastgetattr(ht_trig, Anum_pg_trigger_tgargs, + tgrel->rd_att, &isnull); + if (isnull) + elog(ERROR, "tgargs is null for trigger %u", trigid); + p = (char *) VARDATA_ANY(DatumGetByteaPP(value)); + for (i = 0; i < trigrec->tgnargs; i++) + { + if (i > 0) + appendStringInfoString(&buf, ", "); + simple_quote_literal(&buf, p); + /* advance p to next string embedded in tgargs */ + while (*p) + p++; + p++; + } + } + + /* We deliberately do not put semi-colon at end */ + appendStringInfoChar(&buf, ')'); + + /* Clean up */ + systable_endscan(tgscan); + + table_close(tgrel, AccessShareLock); + + return buf.data; +} + +/* + * set_simple_column_names: fill in column aliases for non-query situations + * + * This handles EXPLAIN and cases where we only have relation RTEs. Without + * a join tree, we can't do anything smart about join RTEs, but we don't + * need to (note that EXPLAIN should never see join alias Vars anyway). + * If we do hit a join RTE we'll just process it like a non-table base RTE. + */ +static void +set_simple_column_names(deparse_namespace *dpns) +{ + ListCell *lc; + ListCell *lc2; + + /* Initialize dpns->rtable_columns to contain zeroed structs */ + dpns->rtable_columns = NIL; + while (list_length(dpns->rtable_columns) < list_length(dpns->rtable)) + dpns->rtable_columns = lappend(dpns->rtable_columns, + palloc0(sizeof(deparse_columns))); + + /* Assign unique column aliases within each RTE */ + forboth(lc, dpns->rtable, lc2, dpns->rtable_columns) + { + RangeTblEntry *rte = (RangeTblEntry *) lfirst(lc); + deparse_columns *colinfo = (deparse_columns *) lfirst(lc2); + + set_relation_column_names(dpns, rte, colinfo); + } +} + +/* + * get_opclass_name - fetch name of an index operator class + * + * The opclass name is appended (after a space) to buf. + * + * Output is suppressed if the opclass is the default for the given + * actual_datatype. (If you don't want this behavior, just pass + * InvalidOid for actual_datatype.) + */ +static void +get_opclass_name(Oid opclass, Oid actual_datatype, + StringInfo buf) +{ + HeapTuple ht_opc; + Form_pg_opclass opcrec; + char *opcname; + char *nspname; + + ht_opc = SearchSysCache1(CLAOID, ObjectIdGetDatum(opclass)); + if (!HeapTupleIsValid(ht_opc)) + elog(ERROR, "cache lookup failed for opclass %u", opclass); + opcrec = (Form_pg_opclass) GETSTRUCT(ht_opc); + + if (!OidIsValid(actual_datatype) || + GetDefaultOpClass(actual_datatype, opcrec->opcmethod) != opclass) + { + /* Okay, we need the opclass name. Do we need to qualify it? */ + opcname = NameStr(opcrec->opcname); + if (OpclassIsVisible(opclass)) + appendStringInfo(buf, " %s", quote_identifier(opcname)); + else + { + nspname = get_namespace_name(opcrec->opcnamespace); + appendStringInfo(buf, " %s.%s", + quote_identifier(nspname), + quote_identifier(opcname)); + } + } + ReleaseSysCache(ht_opc); +} + +/* + * processIndirection - take care of array and subfield assignment + * + * We strip any top-level FieldStore or assignment SubscriptingRef nodes that + * appear in the input, printing them as decoration for the base column + * name (which we assume the caller just printed). We might also need to + * strip CoerceToDomain nodes, but only ones that appear above assignment + * nodes. + * + * Returns the subexpression that's to be assigned. + */ +static Node * +processIndirection(Node *node, deparse_context *context) +{ + StringInfo buf = context->buf; + CoerceToDomain *cdomain = NULL; + + for (;;) + { + if (node == NULL) + break; + if (IsA(node, FieldStore)) + { + FieldStore *fstore = (FieldStore *) node; + Oid typrelid; + char *fieldname; + + /* lookup tuple type */ + typrelid = get_typ_typrelid(fstore->resulttype); + if (!OidIsValid(typrelid)) + elog(ERROR, "argument type %s of FieldStore is not a tuple type", + format_type_be(fstore->resulttype)); + + /* + * Print the field name. There should only be one target field in + * stored rules. There could be more than that in executable + * target lists, but this function cannot be used for that case. + */ + Assert(list_length(fstore->fieldnums) == 1); + fieldname = get_attname(typrelid, + linitial_int(fstore->fieldnums), false); + appendStringInfo(buf, ".%s", quote_identifier(fieldname)); + + /* + * We ignore arg since it should be an uninteresting reference to + * the target column or subcolumn. + */ + node = (Node *) linitial(fstore->newvals); + } + else if (IsA(node, SubscriptingRef)) + { + SubscriptingRef *sbsref = (SubscriptingRef *) node; + + if (sbsref->refassgnexpr == NULL) + break; + printSubscripts(sbsref, context); + + /* + * We ignore refexpr since it should be an uninteresting reference + * to the target column or subcolumn. + */ + node = (Node *) sbsref->refassgnexpr; + } + else if (IsA(node, CoerceToDomain)) + { + cdomain = (CoerceToDomain *) node; + /* If it's an explicit domain coercion, we're done */ + if (cdomain->coercionformat != COERCE_IMPLICIT_CAST) + break; + /* Tentatively descend past the CoerceToDomain */ + node = (Node *) cdomain->arg; + } + else + break; + } + + /* + * If we descended past a CoerceToDomain whose argument turned out not to + * be a FieldStore or array assignment, back up to the CoerceToDomain. + * (This is not enough to be fully correct if there are nested implicit + * CoerceToDomains, but such cases shouldn't ever occur.) + */ + if (cdomain && node == (Node *) cdomain->arg) + node = (Node *) cdomain; + + return node; +} + +static void +printSubscripts(SubscriptingRef *sbsref, deparse_context *context) +{ + StringInfo buf = context->buf; + ListCell *lowlist_item; + ListCell *uplist_item; + + lowlist_item = list_head(sbsref->reflowerindexpr); /* could be NULL */ + foreach(uplist_item, sbsref->refupperindexpr) + { + appendStringInfoChar(buf, '['); + if (lowlist_item) + { + /* If subexpression is NULL, get_rule_expr prints nothing */ + get_rule_expr((Node *) lfirst(lowlist_item), context, false); + appendStringInfoChar(buf, ':'); + lowlist_item = lnext(sbsref->reflowerindexpr, lowlist_item); + } + /* If subexpression is NULL, get_rule_expr prints nothing */ + get_rule_expr((Node *) lfirst(uplist_item), context, false); + appendStringInfoChar(buf, ']'); + } +} + +/* + * get_relation_name + * Get the unqualified name of a relation specified by OID + * + * This differs from the underlying get_rel_name() function in that it will + * throw error instead of silently returning NULL if the OID is bad. + */ +static char * +get_relation_name(Oid relid) +{ + char *relname = get_rel_name(relid); + + if (!relname) + elog(ERROR, "cache lookup failed for relation %u", relid); + return relname; +} + +/* + * generate_relation_or_shard_name + * Compute the name to display for a relation or shard + * + * If the provided relid is equal to the provided distrelid, this function + * returns a shard-extended relation name; otherwise, it falls through to a + * simple generate_relation_name call. + */ +static char * +generate_relation_or_shard_name(Oid relid, Oid distrelid, int64 shardid, + List *namespaces) +{ + char *relname = NULL; + + if (relid == distrelid) + { + relname = get_relation_name(relid); + + if (shardid > 0) + { + Oid schemaOid = get_rel_namespace(relid); + char *schemaName = get_namespace_name(schemaOid); + + AppendShardIdToName(&relname, shardid); + + relname = quote_qualified_identifier(schemaName, relname); + } + } + else + { + relname = generate_relation_name(relid, namespaces); + } + + return relname; +} + +/* + * generate_relation_name + * Compute the name to display for a relation specified by OID + * + * The result includes all necessary quoting and schema-prefixing. + * + * If namespaces isn't NIL, it must be a list of deparse_namespace nodes. + * We will forcibly qualify the relation name if it equals any CTE name + * visible in the namespace list. + */ +char * +generate_relation_name(Oid relid, List *namespaces) +{ + HeapTuple tp; + Form_pg_class reltup; + bool need_qual; + ListCell *nslist; + char *relname; + char *nspname; + char *result; + + tp = SearchSysCache1(RELOID, ObjectIdGetDatum(relid)); + if (!HeapTupleIsValid(tp)) + elog(ERROR, "cache lookup failed for relation %u", relid); + reltup = (Form_pg_class) GETSTRUCT(tp); + relname = NameStr(reltup->relname); + + /* Check for conflicting CTE name */ + need_qual = false; + foreach(nslist, namespaces) + { + deparse_namespace *dpns = (deparse_namespace *) lfirst(nslist); + ListCell *ctlist; + + foreach(ctlist, dpns->ctes) + { + CommonTableExpr *cte = (CommonTableExpr *) lfirst(ctlist); + + if (strcmp(cte->ctename, relname) == 0) + { + need_qual = true; + break; + } + } + if (need_qual) + break; + } + + /* Otherwise, qualify the name if not visible in search path */ + if (!need_qual) + need_qual = !RelationIsVisible(relid); + + if (need_qual) + nspname = get_namespace_name(reltup->relnamespace); + else + nspname = NULL; + + result = quote_qualified_identifier(nspname, relname); + + ReleaseSysCache(tp); + + return result; +} + + +/* + * generate_rte_shard_name returns the qualified name of the shard given a + * CITUS_RTE_SHARD range table entry. + */ +static char * +generate_rte_shard_name(RangeTblEntry *rangeTableEntry) +{ + char *shardSchemaName = NULL; + char *shardTableName = NULL; + + Assert(GetRangeTblKind(rangeTableEntry) == CITUS_RTE_SHARD); + + ExtractRangeTblExtraData(rangeTableEntry, NULL, &shardSchemaName, &shardTableName, + NULL); + + return generate_fragment_name(shardSchemaName, shardTableName); +} + + +/* + * generate_fragment_name + * Compute the name to display for a shard or merged table + * + * The result includes all necessary quoting and schema-prefixing. The schema + * name can be NULL for regular shards. For merged tables, they are always + * declared within a job-specific schema, and therefore can't have null schema + * names. + */ +static char * +generate_fragment_name(char *schemaName, char *tableName) +{ + StringInfo fragmentNameString = makeStringInfo(); + + if (schemaName != NULL) + { + appendStringInfo(fragmentNameString, "%s.%s", quote_identifier(schemaName), + quote_identifier(tableName)); + } + else + { + appendStringInfoString(fragmentNameString, quote_identifier(tableName)); + } + + return fragmentNameString->data; +} + +/* + * generate_function_name + * Compute the name to display for a function specified by OID, + * given that it is being called with the specified actual arg names and + * types. (Those matter because of ambiguous-function resolution rules.) + * + * If we're dealing with a potentially variadic function (in practice, this + * means a FuncExpr or Aggref, not some other way of calling a function), then + * has_variadic must specify whether variadic arguments have been merged, + * and *use_variadic_p will be set to indicate whether to print VARIADIC in + * the output. For non-FuncExpr cases, has_variadic should be false and + * use_variadic_p can be NULL. + * + * The result includes all necessary quoting and schema-prefixing. + */ +static char * +generate_function_name(Oid funcid, int nargs, List *argnames, Oid *argtypes, + bool has_variadic, bool *use_variadic_p, + ParseExprKind special_exprkind) +{ + char *result; + HeapTuple proctup; + Form_pg_proc procform; + char *proname; + bool use_variadic; + char *nspname; + FuncDetailCode p_result; + Oid p_funcid; + Oid p_rettype; + bool p_retset; + int p_nvargs; + Oid p_vatype; + Oid *p_true_typeids; + bool force_qualify = false; + + proctup = SearchSysCache1(PROCOID, ObjectIdGetDatum(funcid)); + if (!HeapTupleIsValid(proctup)) + elog(ERROR, "cache lookup failed for function %u", funcid); + procform = (Form_pg_proc) GETSTRUCT(proctup); + proname = NameStr(procform->proname); + + /* + * Due to parser hacks to avoid needing to reserve CUBE, we need to force + * qualification in some special cases. + */ + if (special_exprkind == EXPR_KIND_GROUP_BY) + { + if (strcmp(proname, "cube") == 0 || strcmp(proname, "rollup") == 0) + force_qualify = true; + } + + /* + * Determine whether VARIADIC should be printed. We must do this first + * since it affects the lookup rules in func_get_detail(). + * + * Currently, we always print VARIADIC if the function has a merged + * variadic-array argument. Note that this is always the case for + * functions taking a VARIADIC argument type other than VARIADIC ANY. + * + * In principle, if VARIADIC wasn't originally specified and the array + * actual argument is deconstructable, we could print the array elements + * separately and not print VARIADIC, thus more nearly reproducing the + * original input. For the moment that seems like too much complication + * for the benefit, and anyway we do not know whether VARIADIC was + * originally specified if it's a non-ANY type. + */ + if (use_variadic_p) + { + /* Parser should not have set funcvariadic unless fn is variadic */ + Assert(!has_variadic || OidIsValid(procform->provariadic)); + use_variadic = has_variadic; + *use_variadic_p = use_variadic; + } + else + { + Assert(!has_variadic); + use_variadic = false; + } + + /* + * The idea here is to schema-qualify only if the parser would fail to + * resolve the correct function given the unqualified func name with the + * specified argtypes and VARIADIC flag. But if we already decided to + * force qualification, then we can skip the lookup and pretend we didn't + * find it. + */ + if (!force_qualify) + p_result = func_get_detail(list_make1(makeString(proname)), + NIL, argnames, nargs, argtypes, + !use_variadic, true, + &p_funcid, &p_rettype, + &p_retset, &p_nvargs, &p_vatype, + &p_true_typeids, NULL); + else + { + p_result = FUNCDETAIL_NOTFOUND; + p_funcid = InvalidOid; + } + + if ((p_result == FUNCDETAIL_NORMAL || + p_result == FUNCDETAIL_AGGREGATE || + p_result == FUNCDETAIL_WINDOWFUNC) && + p_funcid == funcid) + nspname = NULL; + else + nspname = get_namespace_name(procform->pronamespace); + + result = quote_qualified_identifier(nspname, proname); + + ReleaseSysCache(proctup); + + return result; +} + +/* + * generate_operator_name + * Compute the name to display for an operator specified by OID, + * given that it is being called with the specified actual arg types. + * (Arg types matter because of ambiguous-operator resolution rules. + * Pass InvalidOid for unused arg of a unary operator.) + * + * The result includes all necessary quoting and schema-prefixing, + * plus the OPERATOR() decoration needed to use a qualified operator name + * in an expression. + */ +char * +generate_operator_name(Oid operid, Oid arg1, Oid arg2) +{ + StringInfoData buf; + HeapTuple opertup; + Form_pg_operator operform; + char *oprname; + char *nspname; + + initStringInfo(&buf); + + opertup = SearchSysCache1(OPEROID, ObjectIdGetDatum(operid)); + if (!HeapTupleIsValid(opertup)) + elog(ERROR, "cache lookup failed for operator %u", operid); + operform = (Form_pg_operator) GETSTRUCT(opertup); + oprname = NameStr(operform->oprname); + + /* + * Unlike generate_operator_name() in postgres/src/backend/utils/adt/ruleutils.c, + * we don't check if the operator is in current namespace or not. This is + * because this check is costly when the operator is not in current namespace. + */ + nspname = get_namespace_name(operform->oprnamespace); + Assert(nspname != NULL); + appendStringInfo(&buf, "OPERATOR(%s.", quote_identifier(nspname)); + appendStringInfoString(&buf, oprname); + appendStringInfoChar(&buf, ')'); + + ReleaseSysCache(opertup); + + return buf.data; +} + +/* + * get_one_range_partition_bound_string + * A C string representation of one range partition bound + */ +char * +get_range_partbound_string(List *bound_datums) +{ + deparse_context context; + StringInfo buf = makeStringInfo(); + ListCell *cell; + char *sep; + + memset(&context, 0, sizeof(deparse_context)); + context.buf = buf; + + appendStringInfoString(buf, "("); + sep = ""; + foreach(cell, bound_datums) + { + PartitionRangeDatum *datum = + castNode(PartitionRangeDatum, lfirst(cell)); + + appendStringInfoString(buf, sep); + if (datum->kind == PARTITION_RANGE_DATUM_MINVALUE) + appendStringInfoString(buf, "MINVALUE"); + else if (datum->kind == PARTITION_RANGE_DATUM_MAXVALUE) + appendStringInfoString(buf, "MAXVALUE"); + else + { + Const *val = castNode(Const, datum->value); + + get_const_expr(val, &context, -1); + } + sep = ", "; + } + appendStringInfoChar(buf, ')'); + + return buf->data; +} + +/* + * Collect a list of OIDs of all sequences owned by the specified relation, + * and column if specified. If deptype is not zero, then only find sequences + * with the specified dependency type. + */ +List * +getOwnedSequences_internal(Oid relid, AttrNumber attnum, char deptype) +{ + List *result = NIL; + Relation depRel; + ScanKeyData key[3]; + SysScanDesc scan; + HeapTuple tup; + + depRel = table_open(DependRelationId, AccessShareLock); + + ScanKeyInit(&key[0], + Anum_pg_depend_refclassid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(RelationRelationId)); + ScanKeyInit(&key[1], + Anum_pg_depend_refobjid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(relid)); + if (attnum) + ScanKeyInit(&key[2], + Anum_pg_depend_refobjsubid, + BTEqualStrategyNumber, F_INT4EQ, + Int32GetDatum(attnum)); + + scan = systable_beginscan(depRel, DependReferenceIndexId, true, + NULL, attnum ? 3 : 2, key); + + while (HeapTupleIsValid(tup = systable_getnext(scan))) + { + Form_pg_depend deprec = (Form_pg_depend) GETSTRUCT(tup); + + /* + * We assume any auto or internal dependency of a sequence on a column + * must be what we are looking for. (We need the relkind test because + * indexes can also have auto dependencies on columns.) + */ + if (deprec->classid == RelationRelationId && + deprec->objsubid == 0 && + deprec->refobjsubid != 0 && + (deprec->deptype == DEPENDENCY_AUTO || deprec->deptype == DEPENDENCY_INTERNAL) && + get_rel_relkind(deprec->objid) == RELKIND_SEQUENCE) + { + if (!deptype || deprec->deptype == deptype) + result = lappend_oid(result, deprec->objid); + } + } + + systable_endscan(scan); + + table_close(depRel, AccessShareLock); + + return result; +} + +#endif /* (PG_VERSION_NUM >= PG_VERSION_14) && (PG_VERSION_NUM < PG_VERSION_15) */ diff --git a/src/include/distributed/pg_version_constants.h b/src/include/distributed/pg_version_constants.h index 046beea09..6595c0c28 100644 --- a/src/include/distributed/pg_version_constants.h +++ b/src/include/distributed/pg_version_constants.h @@ -14,5 +14,6 @@ #define PG_VERSION_12 120000 #define PG_VERSION_13 130000 #define PG_VERSION_14 140000 +#define PG_VERSION_15 150000 #endif /* PG_VERSION_CONSTANTS */ From 1b6c8348fbf8894e8849cfafd33f6a47404e52ff Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Tue, 10 Aug 2021 16:49:04 +0300 Subject: [PATCH 003/104] Adds PG14 to version_compat.h and columnar_version_compat.h files --- src/include/columnar/columnar_version_compat.h | 4 ++++ src/include/distributed/version_compat.h | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/include/columnar/columnar_version_compat.h b/src/include/columnar/columnar_version_compat.h index 48bd2203e..29a0a42d5 100644 --- a/src/include/columnar/columnar_version_compat.h +++ b/src/include/columnar/columnar_version_compat.h @@ -12,6 +12,10 @@ #ifndef COLUMNAR_COMPAT_H #define COLUMNAR_COMPAT_H +#if PG_VERSION_NUM >= PG_VERSION_14 +#else +#endif + #define ACLCHECK_OBJECT_TABLE OBJECT_TABLE #define ExplainPropertyLong(qlabel, value, es) \ diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index 01b2e66cb..50692ad08 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -30,6 +30,10 @@ #include "tcop/tcopprot.h" #endif +#if PG_VERSION_NUM >= PG_VERSION_14 +#else +#endif + #if PG_VERSION_NUM >= PG_VERSION_13 #define lnext_compat(l, r) lnext(l, r) #define list_delete_cell_compat(l, c, p) list_delete_cell(l, c) From 63cdb4b70adef90f298f1da94c3c355beacf8ea8 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Tue, 10 Aug 2021 17:18:35 +0300 Subject: [PATCH 004/104] Adds AlterTableStmtObjType macro AlterTableStmt's relkind field is changed into objtype New AlterTableStmtObjType macro uses the appropriate one Relevant PG commit: cc35d8933a211d9965eb1c1d2749a903d5735db2 --- src/backend/distributed/commands/distribute_object_ops.c | 3 ++- src/backend/distributed/commands/sequence.c | 6 +++--- src/backend/distributed/commands/table.c | 4 ++-- src/backend/distributed/commands/type.c | 4 ++-- src/backend/distributed/commands/utility_hook.c | 4 ++-- src/backend/distributed/deparser/deparse_sequence_stmts.c | 5 +++-- src/backend/distributed/deparser/deparse_table_stmts.c | 5 +++-- src/backend/distributed/deparser/deparse_type_stmts.c | 5 +++-- src/backend/distributed/deparser/qualify_sequence_stmt.c | 3 ++- src/backend/distributed/deparser/qualify_type_stmt.c | 3 ++- src/include/distributed/version_compat.h | 2 ++ 11 files changed, 26 insertions(+), 18 deletions(-) diff --git a/src/backend/distributed/commands/distribute_object_ops.c b/src/backend/distributed/commands/distribute_object_ops.c index a0b0f91ef..a275e1282 100644 --- a/src/backend/distributed/commands/distribute_object_ops.c +++ b/src/backend/distributed/commands/distribute_object_ops.c @@ -15,6 +15,7 @@ #include "distributed/commands.h" #include "distributed/deparser.h" #include "distributed/pg_version_constants.h" +#include "distributed/version_compat.h" static DistributeObjectOps NoDistributeOps = { .deparse = NULL, @@ -772,7 +773,7 @@ GetDistributeObjectOps(Node *node) case T_AlterTableStmt: { AlterTableStmt *stmt = castNode(AlterTableStmt, node); - switch (stmt->relkind) + switch (AlterTableStmtObjType_compat(stmt)) { case OBJECT_TYPE: { diff --git a/src/backend/distributed/commands/sequence.c b/src/backend/distributed/commands/sequence.c index 94c0867f0..dd7390b45 100644 --- a/src/backend/distributed/commands/sequence.c +++ b/src/backend/distributed/commands/sequence.c @@ -595,7 +595,7 @@ PreprocessAlterSequenceOwnerStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { AlterTableStmt *stmt = castNode(AlterTableStmt, node); - Assert(stmt->relkind == OBJECT_SEQUENCE); + Assert(AlterTableStmtObjType_compat(stmt) == OBJECT_SEQUENCE); ObjectAddress sequenceAddress = GetObjectAddressFromParseTree((Node *) stmt, false); if (!ShouldPropagateObject(&sequenceAddress)) @@ -623,7 +623,7 @@ ObjectAddress AlterSequenceOwnerStmtObjectAddress(Node *node, bool missing_ok) { AlterTableStmt *stmt = castNode(AlterTableStmt, node); - Assert(stmt->relkind == OBJECT_SEQUENCE); + Assert(AlterTableStmtObjType_compat(stmt) == OBJECT_SEQUENCE); RangeVar *sequence = stmt->relation; Oid seqOid = RangeVarGetRelid(sequence, NoLock, missing_ok); @@ -643,7 +643,7 @@ List * PostprocessAlterSequenceOwnerStmt(Node *node, const char *queryString) { AlterTableStmt *stmt = castNode(AlterTableStmt, node); - Assert(stmt->relkind == OBJECT_SEQUENCE); + Assert(AlterTableStmtObjType_compat(stmt) == OBJECT_SEQUENCE); ObjectAddress sequenceAddress = GetObjectAddressFromParseTree((Node *) stmt, false); if (!ShouldPropagateObject(&sequenceAddress)) diff --git a/src/backend/distributed/commands/table.c b/src/backend/distributed/commands/table.c index f064e4334..1442c6c4c 100644 --- a/src/backend/distributed/commands/table.c +++ b/src/backend/distributed/commands/table.c @@ -536,7 +536,7 @@ PreprocessAlterTableStmt(Node *node, const char *alterTableCommand, if (get_rel_relkind(leftRelationId) == RELKIND_SEQUENCE) { AlterTableStmt *stmtCopy = copyObject(alterTableStatement); - stmtCopy->relkind = OBJECT_SEQUENCE; + AlterTableStmtObjType_compat(stmtCopy) = OBJECT_SEQUENCE; return PreprocessAlterSequenceOwnerStmt((Node *) stmtCopy, alterTableCommand, processUtilityContext); } @@ -1629,7 +1629,7 @@ PostprocessAlterTableStmt(AlterTableStmt *alterTableStatement) */ if (get_rel_relkind(relationId) == RELKIND_SEQUENCE) { - alterTableStatement->relkind = OBJECT_SEQUENCE; + AlterTableStmtObjType_compat(alterTableStatement) = OBJECT_SEQUENCE; PostprocessAlterSequenceOwnerStmt((Node *) alterTableStatement, NULL); return; } diff --git a/src/backend/distributed/commands/type.c b/src/backend/distributed/commands/type.c index 81cad3a49..b98d894a4 100644 --- a/src/backend/distributed/commands/type.c +++ b/src/backend/distributed/commands/type.c @@ -206,7 +206,7 @@ PreprocessAlterTypeStmt(Node *node, const char *queryString, ProcessUtilityContext processUtilityContext) { AlterTableStmt *stmt = castNode(AlterTableStmt, node); - Assert(stmt->relkind == OBJECT_TYPE); + Assert(AlterTableStmtObjType_compat(stmt) == OBJECT_TYPE); ObjectAddress typeAddress = GetObjectAddressFromParseTree((Node *) stmt, false); if (!ShouldPropagateObject(&typeAddress)) @@ -789,7 +789,7 @@ ObjectAddress AlterTypeStmtObjectAddress(Node *node, bool missing_ok) { AlterTableStmt *stmt = castNode(AlterTableStmt, node); - Assert(stmt->relkind == OBJECT_TYPE); + Assert(AlterTableStmtObjType_compat(stmt) == OBJECT_TYPE); TypeName *typeName = MakeTypeNameFromRangeVar(stmt->relation); Oid typeOid = LookupTypeNameOid(NULL, typeName, missing_ok); diff --git a/src/backend/distributed/commands/utility_hook.c b/src/backend/distributed/commands/utility_hook.c index 8f976ceb0..26b162cd1 100644 --- a/src/backend/distributed/commands/utility_hook.c +++ b/src/backend/distributed/commands/utility_hook.c @@ -484,8 +484,8 @@ ProcessUtilityInternal(PlannedStmt *pstmt, if (IsA(parsetree, AlterTableStmt)) { AlterTableStmt *alterTableStmt = (AlterTableStmt *) parsetree; - if (alterTableStmt->relkind == OBJECT_TABLE || - alterTableStmt->relkind == OBJECT_FOREIGN_TABLE) + if (AlterTableStmtObjType_compat(alterTableStmt) == OBJECT_TABLE || + AlterTableStmtObjType_compat(alterTableStmt) == OBJECT_FOREIGN_TABLE) { ErrorIfAlterDropsPartitionColumn(alterTableStmt); diff --git a/src/backend/distributed/deparser/deparse_sequence_stmts.c b/src/backend/distributed/deparser/deparse_sequence_stmts.c index 3fbabf962..e6cb36146 100644 --- a/src/backend/distributed/deparser/deparse_sequence_stmts.c +++ b/src/backend/distributed/deparser/deparse_sequence_stmts.c @@ -15,6 +15,7 @@ #include "catalog/namespace.h" #include "distributed/deparser.h" +#include "distributed/version_compat.h" #include "utils/acl.h" #include "utils/builtins.h" #include "utils/lsyscache.h" @@ -193,7 +194,7 @@ DeparseAlterSequenceOwnerStmt(Node *node) StringInfoData str = { 0 }; initStringInfo(&str); - Assert(stmt->relkind == OBJECT_SEQUENCE); + Assert(AlterTableStmtObjType_compat(stmt) == OBJECT_SEQUENCE); AppendAlterSequenceOwnerStmt(&str, stmt); @@ -208,7 +209,7 @@ DeparseAlterSequenceOwnerStmt(Node *node) static void AppendAlterSequenceOwnerStmt(StringInfo buf, AlterTableStmt *stmt) { - Assert(stmt->relkind == OBJECT_SEQUENCE); + Assert(AlterTableStmtObjType_compat(stmt) == OBJECT_SEQUENCE); RangeVar *seq = stmt->relation; char *qualifiedSequenceName = quote_qualified_identifier(seq->schemaname, seq->relname); diff --git a/src/backend/distributed/deparser/deparse_table_stmts.c b/src/backend/distributed/deparser/deparse_table_stmts.c index 8b63207f4..26e2bd8a9 100644 --- a/src/backend/distributed/deparser/deparse_table_stmts.c +++ b/src/backend/distributed/deparser/deparse_table_stmts.c @@ -12,6 +12,7 @@ #include "postgres.h" #include "distributed/deparser.h" +#include "distributed/version_compat.h" #include "nodes/nodes.h" #include "nodes/parsenodes.h" #include "parser/parse_type.h" @@ -63,7 +64,7 @@ DeparseAlterTableStmt(Node *node) StringInfoData str = { 0 }; initStringInfo(&str); - Assert(stmt->relkind == OBJECT_TABLE); + Assert(AlterTableStmtObjType_compat(stmt) == OBJECT_TABLE); AppendAlterTableStmt(&str, stmt); return str.data; @@ -82,7 +83,7 @@ AppendAlterTableStmt(StringInfo buf, AlterTableStmt *stmt) stmt->relation->relname); ListCell *cmdCell = NULL; - Assert(stmt->relkind == OBJECT_TABLE); + Assert(AlterTableStmtObjType_compat(stmt) == OBJECT_TABLE); appendStringInfo(buf, "ALTER TABLE %s", identifier); foreach(cmdCell, stmt->cmds) diff --git a/src/backend/distributed/deparser/deparse_type_stmts.c b/src/backend/distributed/deparser/deparse_type_stmts.c index 07f84e185..e12d96ad9 100644 --- a/src/backend/distributed/deparser/deparse_type_stmts.c +++ b/src/backend/distributed/deparser/deparse_type_stmts.c @@ -26,6 +26,7 @@ #include "distributed/citus_ruleutils.h" #include "distributed/commands.h" #include "distributed/deparser.h" +#include "distributed/version_compat.h" #define AlterEnumIsRename(stmt) (stmt->oldVal != NULL) #define AlterEnumIsAddValue(stmt) (stmt->oldVal == NULL) @@ -121,7 +122,7 @@ DeparseAlterTypeStmt(Node *node) StringInfoData str = { 0 }; initStringInfo(&str); - Assert(stmt->relkind == OBJECT_TYPE); + Assert(AlterTableStmtObjType_compat(stmt) == OBJECT_TYPE); AppendAlterTypeStmt(&str, stmt); @@ -136,7 +137,7 @@ AppendAlterTypeStmt(StringInfo buf, AlterTableStmt *stmt) stmt->relation->relname); ListCell *cmdCell = NULL; - Assert(stmt->relkind == OBJECT_TYPE); + Assert(AlterTableStmtObjType_compat(stmt) == OBJECT_TYPE); appendStringInfo(buf, "ALTER TYPE %s", identifier); foreach(cmdCell, stmt->cmds) diff --git a/src/backend/distributed/deparser/qualify_sequence_stmt.c b/src/backend/distributed/deparser/qualify_sequence_stmt.c index 1ad6a9995..efff68c72 100644 --- a/src/backend/distributed/deparser/qualify_sequence_stmt.c +++ b/src/backend/distributed/deparser/qualify_sequence_stmt.c @@ -18,6 +18,7 @@ #include "postgres.h" #include "distributed/deparser.h" +#include "distributed/version_compat.h" #include "parser/parse_func.h" #include "utils/lsyscache.h" @@ -31,7 +32,7 @@ void QualifyAlterSequenceOwnerStmt(Node *node) { AlterTableStmt *stmt = castNode(AlterTableStmt, node); - Assert(stmt->relkind == OBJECT_SEQUENCE); + Assert(AlterTableStmtObjType_compat(stmt) == OBJECT_SEQUENCE); RangeVar *seq = stmt->relation; diff --git a/src/backend/distributed/deparser/qualify_type_stmt.c b/src/backend/distributed/deparser/qualify_type_stmt.c index cc8c2e04e..506491f47 100644 --- a/src/backend/distributed/deparser/qualify_type_stmt.c +++ b/src/backend/distributed/deparser/qualify_type_stmt.c @@ -25,6 +25,7 @@ #include "catalog/pg_type.h" #include "distributed/commands.h" #include "distributed/deparser.h" +#include "distributed/version_compat.h" #include "nodes/makefuncs.h" #include "parser/parse_type.h" #include "utils/syscache.h" @@ -125,7 +126,7 @@ void QualifyAlterTypeStmt(Node *node) { AlterTableStmt *stmt = castNode(AlterTableStmt, node); - Assert(stmt->relkind == OBJECT_TYPE); + Assert(AlterTableStmtObjType_compat(stmt) == OBJECT_TYPE); if (stmt->relation->schemaname == NULL) { diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index 50692ad08..66e6add86 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -31,7 +31,9 @@ #endif #if PG_VERSION_NUM >= PG_VERSION_14 +#define AlterTableStmtObjType(a) ((a)->objtype) #else +#define AlterTableStmtObjType(a) ((a)->relkind) #endif #if PG_VERSION_NUM >= PG_VERSION_13 From f933d2a57a9b25e6e878ae2025fd20f23de1fa18 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Tue, 10 Aug 2021 17:31:19 +0300 Subject: [PATCH 005/104] Includes defrem.h in index.c --- src/backend/distributed/commands/index.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/backend/distributed/commands/index.c b/src/backend/distributed/commands/index.c index bd1b9eb88..73f572d47 100644 --- a/src/backend/distributed/commands/index.c +++ b/src/backend/distributed/commands/index.c @@ -18,6 +18,7 @@ #include "catalog/index.h" #include "catalog/namespace.h" #include "catalog/pg_class.h" +#include "commands/defrem.h" #include "commands/tablecmds.h" #include "distributed/citus_ruleutils.h" #include "distributed/commands.h" From b790ecf180133673d732d375e5cb2ca697e6b1c9 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Wed, 11 Aug 2021 15:20:22 +0300 Subject: [PATCH 006/104] Introduces F_NEXTVAL_COMPAT macro Name of F_NEXTVAL_OID is changed to F_NEXTVAL Relevant PG commit: 8e1f37c07aafd4bb7aa6e1e1982010af11f8b5c7 --- src/backend/distributed/deparser/citus_ruleutils.c | 2 +- src/include/distributed/version_compat.h | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/backend/distributed/deparser/citus_ruleutils.c b/src/backend/distributed/deparser/citus_ruleutils.c index 6e02a5a07..1626e1add 100644 --- a/src/backend/distributed/deparser/citus_ruleutils.c +++ b/src/backend/distributed/deparser/citus_ruleutils.c @@ -1020,7 +1020,7 @@ contain_nextval_expression_walker(Node *node, void *context) { FuncExpr *funcExpr = (FuncExpr *) node; - if (funcExpr->funcid == F_NEXTVAL_OID) + if (funcExpr->funcid == F_NEXTVAL) { return true; } diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index 66e6add86..b2d3425b5 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -32,8 +32,10 @@ #if PG_VERSION_NUM >= PG_VERSION_14 #define AlterTableStmtObjType(a) ((a)->objtype) +#define F_NEXTVAL_COMPAT F_NEXTVAL #else #define AlterTableStmtObjType(a) ((a)->relkind) +#define F_NEXTVAL_COMPAT F_NEXTVAL_OID #endif #if PG_VERSION_NUM >= PG_VERSION_13 From 4bc0c80bba5ce3096fa03ab70f41df488c0086f5 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Wed, 11 Aug 2021 15:29:47 +0300 Subject: [PATCH 007/104] Adds index_delete_tuples instead of compute_xid_horizon_for_tuples Relevant PG commit: d168b666823b6e0bcf60ed19ce24fb5fb91b8ccf --- src/backend/columnar/columnar_tableam.c | 17 +++++++++++++++++ src/backend/distributed/test/fake_am.c | 18 ++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/src/backend/columnar/columnar_tableam.c b/src/backend/columnar/columnar_tableam.c index 65cbc8e8f..43c4d141f 100644 --- a/src/backend/columnar/columnar_tableam.c +++ b/src/backend/columnar/columnar_tableam.c @@ -634,6 +634,16 @@ columnar_tuple_satisfies_snapshot(Relation rel, TupleTableSlot *slot, } +#if PG_VERSION_NUM >= PG_VERSION_14 +static TransactionId +columnar_index_delete_tuples(Relation rel, + TM_IndexDeleteOp *delstate) +{ + elog(ERROR, "columnar_index_delete_tuples not implemented"); +} + + +#else static TransactionId columnar_compute_xid_horizon_for_tuples(Relation rel, ItemPointerData *tids, @@ -643,6 +653,9 @@ columnar_compute_xid_horizon_for_tuples(Relation rel, } +#endif + + static void columnar_tuple_insert(Relation relation, TupleTableSlot *slot, CommandId cid, int options, BulkInsertState bistate) @@ -2096,7 +2109,11 @@ static const TableAmRoutine columnar_am_methods = { .tuple_get_latest_tid = columnar_get_latest_tid, .tuple_tid_valid = columnar_tuple_tid_valid, .tuple_satisfies_snapshot = columnar_tuple_satisfies_snapshot, +#if PG_VERSION_NUM >= PG_VERSION_14 + .index_delete_tuples = columnar_index_delete_tuples, +#else .compute_xid_horizon_for_tuples = columnar_compute_xid_horizon_for_tuples, +#endif .tuple_insert = columnar_tuple_insert, .tuple_insert_speculative = columnar_tuple_insert_speculative, diff --git a/src/backend/distributed/test/fake_am.c b/src/backend/distributed/test/fake_am.c index 39140eab7..ce7784510 100644 --- a/src/backend/distributed/test/fake_am.c +++ b/src/backend/distributed/test/fake_am.c @@ -168,6 +168,17 @@ fake_tuple_satisfies_snapshot(Relation rel, TupleTableSlot *slot, } +#if PG_VERSION_NUM >= PG_VERSION_14 +static TransactionId +fake_index_delete_tuples(Relation rel, + TM_IndexDeleteOp *delstate) +{ + elog(ERROR, "fake_index_delete_tuples not implemented"); + return InvalidTransactionId; +} + + +#else static TransactionId fake_compute_xid_horizon_for_tuples(Relation rel, ItemPointerData *tids, @@ -178,6 +189,9 @@ fake_compute_xid_horizon_for_tuples(Relation rel, } +#endif + + /* ---------------------------------------------------------------------------- * Functions for manipulations of physical tuples for fake AM. * ---------------------------------------------------------------------------- @@ -556,7 +570,11 @@ static const TableAmRoutine fake_methods = { .tuple_get_latest_tid = fake_get_latest_tid, .tuple_tid_valid = fake_tuple_tid_valid, .tuple_satisfies_snapshot = fake_tuple_satisfies_snapshot, +#if PG_VERSION_NUM >= PG_VERSION_14 + .index_delete_tuples = fake_index_delete_tuples, +#else .compute_xid_horizon_for_tuples = fake_compute_xid_horizon_for_tuples, +#endif .relation_set_new_filenode = fake_relation_set_new_filenode, .relation_nontransactional_truncate = fake_relation_nontransactional_truncate, From 3c10e0f5689232c2d6cde9e167dd32569bf2fae4 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Wed, 11 Aug 2021 15:38:21 +0300 Subject: [PATCH 008/104] Introduces ROLE_MONITOR_COMPAT macro DEFAULT_ROLE_MONITOR is renamed to ROLE_PG_MONITOR This macro uses appropriate one Relevant PG commit: c9c41c7a337d3e2deb0b2a193e9ecfb865d8f52b --- src/backend/distributed/transaction/backend_data.c | 2 +- src/include/distributed/version_compat.h | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/backend/distributed/transaction/backend_data.c b/src/backend/distributed/transaction/backend_data.c index 6efe81e6f..26d853507 100644 --- a/src/backend/distributed/transaction/backend_data.c +++ b/src/backend/distributed/transaction/backend_data.c @@ -371,7 +371,7 @@ StoreAllActiveTransactions(Tuplestorestate *tupleStore, TupleDesc tupleDescripto memset(values, 0, sizeof(values)); memset(isNulls, false, sizeof(isNulls)); - if (is_member_of_role(userId, DEFAULT_ROLE_MONITOR)) + if (is_member_of_role(userId, ROLE_PG_MONITOR)) { showAllTransactions = true; } diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index b2d3425b5..5bea3c427 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -33,9 +33,11 @@ #if PG_VERSION_NUM >= PG_VERSION_14 #define AlterTableStmtObjType(a) ((a)->objtype) #define F_NEXTVAL_COMPAT F_NEXTVAL +#define ROLE_MONITOR_COMPAT ROLE_PG_MONITOR #else #define AlterTableStmtObjType(a) ((a)->relkind) #define F_NEXTVAL_COMPAT F_NEXTVAL_OID +#define ROLE_MONITOR_COMPAT DEFAULT_ROLE_MONITOR #endif #if PG_VERSION_NUM >= PG_VERSION_13 From f8d3e50f255fe0acb6cc9fd13cc4630e6cb74631 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Wed, 11 Aug 2021 15:47:41 +0300 Subject: [PATCH 009/104] Introduces STATUS_WAITING_COMPAT macro The STATUS_WAITING define is removed and an enum with PROC_WAIT_STATUS_WAITING is added instead This macro uses appropriate one Relevant PG commit: a513f1dfbf2c29a51b0f7cbd5913ce2d2ee452c5 --- src/backend/distributed/transaction/lock_graph.c | 4 ++-- src/include/distributed/version_compat.h | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/backend/distributed/transaction/lock_graph.c b/src/backend/distributed/transaction/lock_graph.c index aed021ae0..9265bd320 100644 --- a/src/backend/distributed/transaction/lock_graph.c +++ b/src/backend/distributed/transaction/lock_graph.c @@ -456,7 +456,7 @@ BuildLocalWaitGraph(void) static bool IsProcessWaitingForSafeOperations(PGPROC *proc) { - if (proc->waitStatus != STATUS_WAITING) + if (proc->waitStatus != PROC_WAIT_STATUS_WAITING) { return false; } @@ -715,7 +715,7 @@ AddProcToVisit(PROCStack *remaining, PGPROC *proc) bool IsProcessWaitingForLock(PGPROC *proc) { - return proc->waitStatus == STATUS_WAITING; + return proc->waitStatus == PROC_WAIT_STATUS_WAITING; } diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index 5bea3c427..9581d3804 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -34,10 +34,12 @@ #define AlterTableStmtObjType(a) ((a)->objtype) #define F_NEXTVAL_COMPAT F_NEXTVAL #define ROLE_MONITOR_COMPAT ROLE_PG_MONITOR +#define STATUS_WAITING_COMPAT PROC_WAIT_STATUS_WAITING #else #define AlterTableStmtObjType(a) ((a)->relkind) #define F_NEXTVAL_COMPAT F_NEXTVAL_OID #define ROLE_MONITOR_COMPAT DEFAULT_ROLE_MONITOR +#define STATUS_WAITING_COMPAT STATUS_WAITING #endif #if PG_VERSION_NUM >= PG_VERSION_13 From 54ee93885ade6a8b5a4a07d044badb832044e622 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Wed, 11 Aug 2021 16:02:14 +0300 Subject: [PATCH 010/104] Introduces getObjectTypeDescription_compat and getObjectIdentity_compat macros getObjectTypeDescription and getObjectIdentity functions now have a new bool missing_ok parameter These new macros give us the ability to use this new parameter for PG14 and they don't give the parameter for previous versions Currently all missing_ok parameters are set to false to keep current behavior Relevant PG commit: 2a10fdc4307a667883f7a3369cb93a721ade9680 --- src/backend/distributed/commands/dependencies.c | 2 +- src/backend/distributed/commands/function.c | 9 ++++++--- src/backend/distributed/commands/type.c | 3 ++- src/backend/distributed/metadata/distobject.c | 4 ++-- src/include/distributed/version_compat.h | 4 ++++ 5 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/backend/distributed/commands/dependencies.c b/src/backend/distributed/commands/dependencies.c index 0b19ee729..50e459512 100644 --- a/src/backend/distributed/commands/dependencies.c +++ b/src/backend/distributed/commands/dependencies.c @@ -251,7 +251,7 @@ GetDependencyCreateDDLCommands(const ObjectAddress *dependency) */ Assert(false); ereport(ERROR, (errmsg("unsupported object %s for distribution by citus", - getObjectTypeDescription(dependency)), + getObjectTypeDescription_compat(dependency, /* missingOk: */ false)), errdetail( "citus tries to recreate an unsupported object on its workers"), errhint("please report a bug as this should not be happening"))); diff --git a/src/backend/distributed/commands/function.c b/src/backend/distributed/commands/function.c index 1f8a86e40..6aedce934 100644 --- a/src/backend/distributed/commands/function.c +++ b/src/backend/distributed/commands/function.c @@ -1613,7 +1613,8 @@ PreprocessAlterFunctionDependsStmt(Node *node, const char *queryString, * workers */ - const char *functionName = getObjectIdentity(&address); + const char *functionName = + getObjectIdentity_compat(&address, /* missingOk: */ false); ereport(ERROR, (errmsg("distrtibuted functions are not allowed to depend on an " "extension"), errdetail("Function \"%s\" is already distributed. Functions from " @@ -1932,8 +1933,10 @@ ErrorIfFunctionDependsOnExtension(const ObjectAddress *functionAddress) if (IsObjectAddressOwnedByExtension(functionAddress, &extensionAddress)) { - char *functionName = getObjectIdentity(functionAddress); - char *extensionName = getObjectIdentity(&extensionAddress); + char *functionName = + getObjectIdentity_compat(functionAddress, /* missingOk: */ false); + char *extensionName = + getObjectIdentity_compat(&extensionAddress, /* missingOk: */ false); ereport(ERROR, (errmsg("unable to create a distributed function from functions " "owned by an extension"), errdetail("Function \"%s\" has a dependency on extension \"%s\". " diff --git a/src/backend/distributed/commands/type.c b/src/backend/distributed/commands/type.c index b98d894a4..83cdc1a6b 100644 --- a/src/backend/distributed/commands/type.c +++ b/src/backend/distributed/commands/type.c @@ -973,7 +973,8 @@ CreateTypeDDLCommandsIdempotent(const ObjectAddress *typeAddress) /* add owner ship change so the creation command can be run as a different user */ const char *username = GetUserNameFromId(GetTypeOwner(typeAddress->objectId), false); initStringInfo(&buf); - appendStringInfo(&buf, ALTER_TYPE_OWNER_COMMAND, getObjectIdentity(typeAddress), + appendStringInfo(&buf, ALTER_TYPE_OWNER_COMMAND, + getObjectIdentity_compat(typeAddress, false), quote_identifier(username)); ddlCommands = lappend(ddlCommands, buf.data); diff --git a/src/backend/distributed/metadata/distobject.c b/src/backend/distributed/metadata/distobject.c index ef2adf641..b96db6ed0 100644 --- a/src/backend/distributed/metadata/distobject.c +++ b/src/backend/distributed/metadata/distobject.c @@ -75,8 +75,8 @@ citus_unmark_object_distributed(PG_FUNCTION_ARGS) { ereport(ERROR, (errmsg("object still exists"), errdetail("the %s \"%s\" still exists", - getObjectTypeDescription(&address), - getObjectIdentity(&address)), + getObjectTypeDescription_compat(&address, /* missingOk: */ false), + getObjectIdentity_compat(&address, /* missingOk: */ false)), errhint("drop the object via a DROP command"))); } diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index 9581d3804..aa438ad0e 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -35,11 +35,15 @@ #define F_NEXTVAL_COMPAT F_NEXTVAL #define ROLE_MONITOR_COMPAT ROLE_PG_MONITOR #define STATUS_WAITING_COMPAT PROC_WAIT_STATUS_WAITING +#define getObjectTypeDescription_compat(a, b) getObjectTypeDescription(a, b) +#define getObjectIdentity_compat(a, b) getObjectIdentity(a, b) #else #define AlterTableStmtObjType(a) ((a)->relkind) #define F_NEXTVAL_COMPAT F_NEXTVAL_OID #define ROLE_MONITOR_COMPAT DEFAULT_ROLE_MONITOR #define STATUS_WAITING_COMPAT STATUS_WAITING +#define getObjectTypeDescription_compat(a, b) getObjectTypeDescription(a) +#define getObjectIdentity_compat(a, b) getObjectIdentity(a) #endif #if PG_VERSION_NUM >= PG_VERSION_13 From 347ae2928f9d9b7328e08eec887324cfa5d335a0 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Wed, 11 Aug 2021 16:34:35 +0300 Subject: [PATCH 011/104] Introduces stats_compat macro for MemoryContextMethods->stats stats function now have a new bool print_to_stderr parameter This new macro gives us the ability to use this new parameter for PG14 and it doesn't give the parameter for previous versions Existing print_to_stderr parameter is set to true to keep current behavior Relevant PG commit: 43620e328617c1f41a2a54c8cee01723064e3ffa --- src/backend/columnar/columnar_debug.c | 3 ++- src/backend/distributed/test/xact_stats.c | 4 +++- src/include/distributed/version_compat.h | 6 ++++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/backend/columnar/columnar_debug.c b/src/backend/columnar/columnar_debug.c index 5b62a3b56..5525bb032 100644 --- a/src/backend/columnar/columnar_debug.c +++ b/src/backend/columnar/columnar_debug.c @@ -17,6 +17,7 @@ #include "catalog/pg_type.h" #include "distributed/pg_version_constants.h" #include "distributed/tuplestore.h" +#include "distributed/version_compat.h" #include "miscadmin.h" #include "storage/fd.h" #include "storage/smgr.h" @@ -161,5 +162,5 @@ MemoryContextTotals(MemoryContext context, MemoryContextCounters *counters) MemoryContextTotals(child, counters); } - context->methods->stats(context, NULL, NULL, counters); + context->methods->stats_compat(context, NULL, NULL, counters, true); } diff --git a/src/backend/distributed/test/xact_stats.c b/src/backend/distributed/test/xact_stats.c index 05723aa35..c31a17b7f 100644 --- a/src/backend/distributed/test/xact_stats.c +++ b/src/backend/distributed/test/xact_stats.c @@ -19,6 +19,7 @@ #include "pgstat.h" #include "distributed/transaction_management.h" +#include "distributed/version_compat.h" static Size MemoryContextTotalSpace(MemoryContext context); @@ -47,7 +48,8 @@ MemoryContextTotalSpace(MemoryContext context) Size totalSpace = 0; MemoryContextCounters totals = { 0 }; - TopTransactionContext->methods->stats(TopTransactionContext, NULL, NULL, &totals); + TopTransactionContext->methods->stats_compat(TopTransactionContext, NULL, NULL, + &totals, true); totalSpace += totals.totalspace; for (MemoryContext child = context->firstchild; diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index aa438ad0e..f4a036a01 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -37,6 +37,9 @@ #define STATUS_WAITING_COMPAT PROC_WAIT_STATUS_WAITING #define getObjectTypeDescription_compat(a, b) getObjectTypeDescription(a, b) #define getObjectIdentity_compat(a, b) getObjectIdentity(a, b) + +/* for MemoryContextMethods->stats */ +#define stats_compat(a, b, c, d, e) stats(a, b, c, d, e) #else #define AlterTableStmtObjType(a) ((a)->relkind) #define F_NEXTVAL_COMPAT F_NEXTVAL_OID @@ -44,6 +47,9 @@ #define STATUS_WAITING_COMPAT STATUS_WAITING #define getObjectTypeDescription_compat(a, b) getObjectTypeDescription(a) #define getObjectIdentity_compat(a, b) getObjectIdentity(a) + +/* for MemoryContextMethods->stats */ +#define stats_compat(a, b, c, d, e) stats(a, b, c, d) #endif #if PG_VERSION_NUM >= PG_VERSION_13 From ebf1b7e23faf9ef03c258b102184d7bd623fbc2e Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Thu, 12 Aug 2021 13:16:42 +0300 Subject: [PATCH 012/104] Introduces macros for functions that now have include_out_arguments argument New macros: FuncnameGetCandidates_compat and expand_function_arguments_compat The functions (the ones without _compat) now have a new bool include_out_arguments parameter These new macros give us the ability to use this new parameter for PG14 and it doesn't give the parameter for previous versions Existing include_out_arguments parameters are set to 'false' to keep current behavior Relevant PG commit: e56bce5d43789cce95d099554ae9593ada92b3b7 --- src/backend/distributed/commands/function.c | 4 ++-- .../distributed/planner/multi_logical_optimizer.c | 4 ++-- src/backend/distributed/utils/citus_clauses.c | 6 ++++-- src/backend/distributed/utils/function_utils.c | 12 ++++++++---- src/include/distributed/version_compat.h | 6 ++++++ 5 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/backend/distributed/commands/function.c b/src/backend/distributed/commands/function.c index 6aedce934..e6a646735 100644 --- a/src/backend/distributed/commands/function.c +++ b/src/backend/distributed/commands/function.c @@ -1808,8 +1808,8 @@ GenerateBackupNameForProcCollision(const ObjectAddress *address) List *newProcName = list_make2(namespace, makeString(newName)); /* don't need to rename if the input arguments don't match */ - FuncCandidateList clist = FuncnameGetCandidates(newProcName, numargs, NIL, false, - false, true); + FuncCandidateList clist = FuncnameGetCandidates_compat(newProcName, numargs, NIL, + false, false, false, true); for (; clist; clist = clist->next) { if (memcmp(clist->args, argtypes, sizeof(Oid) * numargs) == 0) diff --git a/src/backend/distributed/planner/multi_logical_optimizer.c b/src/backend/distributed/planner/multi_logical_optimizer.c index 16c300bc8..bd3739717 100644 --- a/src/backend/distributed/planner/multi_logical_optimizer.c +++ b/src/backend/distributed/planner/multi_logical_optimizer.c @@ -3585,8 +3585,8 @@ static Oid CitusFunctionOidWithSignature(char *functionName, int numargs, Oid *argtypes) { List *aggregateName = list_make2(makeString("pg_catalog"), makeString(functionName)); - FuncCandidateList clist = FuncnameGetCandidates(aggregateName, numargs, NIL, false, - false, true); + FuncCandidateList clist = FuncnameGetCandidates_compat(aggregateName, numargs, NIL, + false, false, false, true); for (; clist; clist = clist->next) { diff --git a/src/backend/distributed/utils/citus_clauses.c b/src/backend/distributed/utils/citus_clauses.c index 99f1a3ac6..bd1b409b2 100644 --- a/src/backend/distributed/utils/citus_clauses.c +++ b/src/backend/distributed/utils/citus_clauses.c @@ -12,6 +12,7 @@ #include "distributed/insert_select_planner.h" #include "distributed/metadata_cache.h" #include "distributed/multi_router_planner.h" +#include "distributed/version_compat.h" #include "catalog/pg_proc.h" #include "catalog/pg_type.h" @@ -526,8 +527,9 @@ FixFunctionArgumentsWalker(Node *expr, void *context) elog(ERROR, "cache lookup failed for function %u", funcExpr->funcid); } - funcExpr->args = expand_function_arguments(funcExpr->args, - funcExpr->funcresulttype, func_tuple); + funcExpr->args = expand_function_arguments_compat(funcExpr->args, false, + funcExpr->funcresulttype, + func_tuple); ReleaseSysCache(func_tuple); } diff --git a/src/backend/distributed/utils/function_utils.c b/src/backend/distributed/utils/function_utils.c index 07c85b796..f0818d0c8 100644 --- a/src/backend/distributed/utils/function_utils.c +++ b/src/backend/distributed/utils/function_utils.c @@ -45,10 +45,14 @@ FunctionOidExtended(const char *schemaName, const char *functionName, int argume const bool findVariadics = false; const bool findDefaults = false; - FuncCandidateList functionList = FuncnameGetCandidates(qualifiedFunctionNameList, - argumentCount, - argumentList, findVariadics, - findDefaults, true); + FuncCandidateList functionList = FuncnameGetCandidates_compat( + qualifiedFunctionNameList, + argumentCount, + argumentList, + findVariadics, + findDefaults, + false, + true); if (functionList == NULL) { diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index f4a036a01..750a73aeb 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -40,6 +40,9 @@ /* for MemoryContextMethods->stats */ #define stats_compat(a, b, c, d, e) stats(a, b, c, d, e) +#define FuncnameGetCandidates_compat(a, b, c, d, e, f, g) \ + FuncnameGetCandidates(a, b, c, d, e, f, g) +#define expand_function_arguments_compat(a, b, c, d) expand_function_arguments(a, b, c, d) #else #define AlterTableStmtObjType(a) ((a)->relkind) #define F_NEXTVAL_COMPAT F_NEXTVAL_OID @@ -50,6 +53,9 @@ /* for MemoryContextMethods->stats */ #define stats_compat(a, b, c, d, e) stats(a, b, c, d) +#define FuncnameGetCandidates_compat(a, b, c, d, e, f, g) \ + FuncnameGetCandidates(a, b, c, d, e, g) +#define expand_function_arguments_compat(a, b, c, d) expand_function_arguments(a, c, d) #endif #if PG_VERSION_NUM >= PG_VERSION_13 From 37ae22ce3e0fd3eedf33a0019a0aecba1bbcfa91 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Thu, 12 Aug 2021 16:03:58 +0300 Subject: [PATCH 013/104] Introduces macros for vacuum options VacOptTernaryValue enum is renamed to VacOptValue. In the enum there were three values, VACOPT_TERNARY_DEFAULT, VACOPT_TERNARY_DISABLED, and VACOPT_TERNARY_ENABLED Now there are four values VACOPTVALUE_UNSPECIFIED, VACOPTVALUE_AUTO, VACOPTVALUE_DISABLED, and VACOPTVALUE_ENABLED New macros are VacOptValue_compat, VACOPTVALUE_UNSPECIFIED_COMPAT, VACOPTVALUE_DISABLED_COMPAT, and VACOPTVALUE_ENABLED_COMPAT The VACOPTVALUE_UNSPECIFIED_COMPAT matches VACOPT_TERNARY_DEFAULT and VACOPTVALUE_UNSPECIFIED. And there are no macro for VACOPTVALUE_AUTO. Relevant PG commit: 3499df0dee8c4ea51d264a674df5b5e31991319a --- src/backend/columnar/columnar_tableam.c | 4 ++-- src/backend/distributed/commands/vacuum.c | 28 +++++++++++------------ src/include/distributed/version_compat.h | 8 +++++++ 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/src/backend/columnar/columnar_tableam.c b/src/backend/columnar/columnar_tableam.c index 43c4d141f..9c7c42bf4 100644 --- a/src/backend/columnar/columnar_tableam.c +++ b/src/backend/columnar/columnar_tableam.c @@ -966,7 +966,7 @@ columnar_vacuum_rel(Relation rel, VacuumParams *params, int elevel = (params->options & VACOPT_VERBOSE) ? INFO : DEBUG2; /* this should have been resolved by vacuum.c until now */ - Assert(params->truncate != VACOPT_TERNARY_DEFAULT); + Assert(params->truncate != VACOPTVALUE_UNSPECIFIED); LogRelationStats(rel, elevel); @@ -974,7 +974,7 @@ columnar_vacuum_rel(Relation rel, VacuumParams *params, * We don't have updates, deletes, or concurrent updates, so all we * care for now is truncating the unused space at the end of storage. */ - if (params->truncate == VACOPT_TERNARY_ENABLED) + if (params->truncate == VACOPTVALUE_ENABLED) { TruncateColumnar(rel, elevel); } diff --git a/src/backend/distributed/commands/vacuum.c b/src/backend/distributed/commands/vacuum.c index e4569e3f7..f6fabfe2b 100644 --- a/src/backend/distributed/commands/vacuum.c +++ b/src/backend/distributed/commands/vacuum.c @@ -39,8 +39,8 @@ typedef struct CitusVacuumParams { int options; - VacOptTernaryValue truncate; - VacOptTernaryValue index_cleanup; + VacOptValue truncate; + VacOptValue index_cleanup; #if PG_VERSION_NUM >= PG_VERSION_13 int nworkers; @@ -346,8 +346,8 @@ DeparseVacuumStmtPrefix(CitusVacuumParams vacuumParams) /* if no flags remain, exit early */ if (vacuumFlags == 0 && - vacuumParams.truncate == VACOPT_TERNARY_DEFAULT && - vacuumParams.index_cleanup == VACOPT_TERNARY_DEFAULT + vacuumParams.truncate == VACOPTVALUE_UNSPECIFIED && + vacuumParams.index_cleanup == VACOPTVALUE_UNSPECIFIED #if PG_VERSION_NUM >= PG_VERSION_13 && vacuumParams.nworkers == VACUUM_PARALLEL_NOTSET #endif @@ -389,18 +389,18 @@ DeparseVacuumStmtPrefix(CitusVacuumParams vacuumParams) appendStringInfoString(vacuumPrefix, "SKIP_LOCKED,"); } - if (vacuumParams.truncate != VACOPT_TERNARY_DEFAULT) + if (vacuumParams.truncate != VACOPTVALUE_UNSPECIFIED) { appendStringInfoString(vacuumPrefix, - vacuumParams.truncate == VACOPT_TERNARY_ENABLED ? + vacuumParams.truncate == VACOPTVALUE_ENABLED ? "TRUNCATE," : "TRUNCATE false," ); } - if (vacuumParams.index_cleanup != VACOPT_TERNARY_DEFAULT) + if (vacuumParams.index_cleanup != VACOPTVALUE_UNSPECIFIED) { appendStringInfoString(vacuumPrefix, - vacuumParams.index_cleanup == VACOPT_TERNARY_ENABLED ? + vacuumParams.index_cleanup == VACOPTVALUE_ENABLED ? "INDEX_CLEANUP," : "INDEX_CLEANUP false," ); } @@ -506,8 +506,8 @@ VacuumStmtParams(VacuumStmt *vacstmt) bool disable_page_skipping = false; /* Set default value */ - params.index_cleanup = VACOPT_TERNARY_DEFAULT; - params.truncate = VACOPT_TERNARY_DEFAULT; + params.index_cleanup = VACOPTVALUE_UNSPECIFIED; + params.truncate = VACOPTVALUE_UNSPECIFIED; #if PG_VERSION_NUM >= PG_VERSION_13 params.nworkers = VACUUM_PARALLEL_NOTSET; #endif @@ -551,13 +551,13 @@ VacuumStmtParams(VacuumStmt *vacstmt) } else if (strcmp(opt->defname, "index_cleanup") == 0) { - params.index_cleanup = defGetBoolean(opt) ? VACOPT_TERNARY_ENABLED : - VACOPT_TERNARY_DISABLED; + params.index_cleanup = defGetBoolean(opt) ? VACOPTVALUE_ENABLED : + VACOPTVALUE_DISABLED; } else if (strcmp(opt->defname, "truncate") == 0) { - params.truncate = defGetBoolean(opt) ? VACOPT_TERNARY_ENABLED : - VACOPT_TERNARY_DISABLED; + params.truncate = defGetBoolean(opt) ? VACOPTVALUE_ENABLED : + VACOPTVALUE_DISABLED; } #if PG_VERSION_NUM >= PG_VERSION_13 else if (strcmp(opt->defname, "parallel") == 0) diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index 750a73aeb..5dc552fb2 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -43,6 +43,10 @@ #define FuncnameGetCandidates_compat(a, b, c, d, e, f, g) \ FuncnameGetCandidates(a, b, c, d, e, f, g) #define expand_function_arguments_compat(a, b, c, d) expand_function_arguments(a, b, c, d) +#define VacOptValue_compat VacOptValue +#define VACOPTVALUE_UNSPECIFIED_COMPAT VACOPTVALUE_UNSPECIFIED +#define VACOPTVALUE_DISABLED_COMPAT VACOPTVALUE_DISABLED +#define VACOPTVALUE_ENABLED_COMPAT VACOPTVALUE_ENABLED #else #define AlterTableStmtObjType(a) ((a)->relkind) #define F_NEXTVAL_COMPAT F_NEXTVAL_OID @@ -56,6 +60,10 @@ #define FuncnameGetCandidates_compat(a, b, c, d, e, f, g) \ FuncnameGetCandidates(a, b, c, d, e, g) #define expand_function_arguments_compat(a, b, c, d) expand_function_arguments(a, c, d) +#define VacOptValue_compat VacOptTernaryValue +#define VACOPTVALUE_UNSPECIFIED_COMPAT VACOPT_TERNARY_DEFAULT +#define VACOPTVALUE_DISABLED_COMPAT VACOPT_TERNARY_DISABLED +#define VACOPTVALUE_ENABLED_COMPAT VACOPT_TERNARY_ENABLED #endif #if PG_VERSION_NUM >= PG_VERSION_13 From 8f34f84ce6cee2a81aadfd22e9574c0f6aad4dba Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Fri, 13 Aug 2021 10:42:00 +0300 Subject: [PATCH 014/104] Introduces IsReindexWithParam_compat macro In ReindexStmt concurrent field is moved to options and then options are converted to params list. This macro uses previous fields for previous versions and the new params list with a new function named IsReindexWithParam for PG14 Relevant PG commits: 844c05abc3f1c1703bf17cf44ab66351ed9711d2 b5913f6120792465f4394b93c15c2e2ac0c08376 --- src/backend/distributed/commands/index.c | 13 +++++--- .../distributed/deparser/citus_ruleutils.c | 32 +++++++++++++++++-- src/include/distributed/commands.h | 1 + src/include/distributed/version_compat.h | 5 +++ 4 files changed, 44 insertions(+), 7 deletions(-) diff --git a/src/backend/distributed/commands/index.c b/src/backend/distributed/commands/index.c index 73f572d47..6a7431528 100644 --- a/src/backend/distributed/commands/index.c +++ b/src/backend/distributed/commands/index.c @@ -529,8 +529,8 @@ PreprocessReindexStmt(Node *node, const char *reindexCommand, { Relation relation = NULL; Oid relationId = InvalidOid; - LOCKMODE lockmode = reindexStatement->concurrent ? ShareUpdateExclusiveLock : - AccessExclusiveLock; + LOCKMODE lockmode = IsReindexWithParam_compat(reindexStatement, "concurrently") ? + ShareUpdateExclusiveLock : AccessExclusiveLock; MemoryContext relationContext = NULL; Assert(reindexStatement->kind == REINDEX_OBJECT_INDEX || @@ -539,7 +539,8 @@ PreprocessReindexStmt(Node *node, const char *reindexCommand, if (reindexStatement->kind == REINDEX_OBJECT_INDEX) { struct ReindexIndexCallbackState state; - state.concurrent = reindexStatement->concurrent; + state.concurrent = IsReindexWithParam_compat(reindexStatement, + "concurrently"); state.locked_table_oid = InvalidOid; Oid indOid = RangeVarGetRelidExtended(reindexStatement->relation, @@ -590,8 +591,10 @@ PreprocessReindexStmt(Node *node, const char *reindexCommand, { DDLJob *ddlJob = palloc0(sizeof(DDLJob)); ddlJob->targetRelationId = relationId; - ddlJob->concurrentIndexCmd = reindexStatement->concurrent; - ddlJob->startNewTransaction = reindexStatement->concurrent; + ddlJob->concurrentIndexCmd = IsReindexWithParam_compat(reindexStatement, + "concurrently"); + ddlJob->startNewTransaction = IsReindexWithParam_compat(reindexStatement, + "concurrently"); ddlJob->commandString = reindexCommand; ddlJob->taskList = CreateReindexTaskList(relationId, reindexStatement); diff --git a/src/backend/distributed/deparser/citus_ruleutils.c b/src/backend/distributed/deparser/citus_ruleutils.c index 1626e1add..70bacd204 100644 --- a/src/backend/distributed/deparser/citus_ruleutils.c +++ b/src/backend/distributed/deparser/citus_ruleutils.c @@ -39,6 +39,7 @@ #include "commands/defrem.h" #include "commands/extension.h" #include "distributed/citus_ruleutils.h" +#include "distributed/commands.h" #include "distributed/listutils.h" #include "distributed/multi_partitioning_utils.h" #include "distributed/metadata_cache.h" @@ -740,7 +741,8 @@ deparse_shard_reindex_statement(ReindexStmt *origStmt, Oid distrelid, int64 shar { ReindexStmt *reindexStmt = copyObject(origStmt); /* copy to avoid modifications */ char *relationName = NULL; - const char *concurrentlyString = reindexStmt->concurrent ? "CONCURRENTLY " : ""; + const char *concurrentlyString = + IsReindexWithParam_compat(reindexStmt, "concurrently") ? "CONCURRENTLY " : ""; if (reindexStmt->kind == REINDEX_OBJECT_INDEX || @@ -754,7 +756,7 @@ deparse_shard_reindex_statement(ReindexStmt *origStmt, Oid distrelid, int64 shar appendStringInfoString(buffer, "REINDEX "); - if (reindexStmt->options == REINDEXOPT_VERBOSE) + if (IsReindexWithParam_compat(reindexStmt, "verbose")) { appendStringInfoString(buffer, "(VERBOSE) "); } @@ -800,6 +802,32 @@ deparse_shard_reindex_statement(ReindexStmt *origStmt, Oid distrelid, int64 shar } } +/* + * IsReindexWithParam_compat returns true if the given parameter + * exists for the given reindexStmt. + */ +bool IsReindexWithParam_compat(ReindexStmt* reindexStmt, char* param) { +#if PG_VERSION_NUM < PG_VERSION_14 + if (strcmp(param, "concurrently") == 0) { + return reindexStmt->concurrent; + }else if (strcmp(param, "verbose") == 0) { + return reindexStmt->options & REINDEXOPT_VERBOSE; + } + return false; +#else + DefElem *opt = NULL; + foreach_ptr(opt, reindexStmt->params) + { + if (strcmp(opt->defname, param) == 0) + { + return defGetBoolean(opt); + } + } + return false; +#endif + +} + /* deparse_index_columns appends index or include parameters to the provided buffer */ static void diff --git a/src/include/distributed/commands.h b/src/include/distributed/commands.h index a01d51387..bb73e8764 100644 --- a/src/include/distributed/commands.h +++ b/src/include/distributed/commands.h @@ -288,6 +288,7 @@ extern void ErrorIfUnsupportedAlterIndexStmt(AlterTableStmt *alterTableStatement extern void MarkIndexValid(IndexStmt *indexStmt); extern List * ExecuteFunctionOnEachTableIndex(Oid relationId, PGIndexProcessor pgIndexProcessor, int flags); +extern bool IsReindexWithParam_compat(ReindexStmt *stmt, char *paramName); /* objectaddress.c - forward declarations */ extern ObjectAddress CreateExtensionStmtObjectAddress(Node *stmt, bool missing_ok); diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index 5dc552fb2..0694219be 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -47,6 +47,7 @@ #define VACOPTVALUE_UNSPECIFIED_COMPAT VACOPTVALUE_UNSPECIFIED #define VACOPTVALUE_DISABLED_COMPAT VACOPTVALUE_DISABLED #define VACOPTVALUE_ENABLED_COMPAT VACOPTVALUE_ENABLED +#define IsReindexWithParam_compat(reindex, param) IsReindexWithParam(reindex, param) #else #define AlterTableStmtObjType(a) ((a)->relkind) #define F_NEXTVAL_COMPAT F_NEXTVAL_OID @@ -64,6 +65,10 @@ #define VACOPTVALUE_UNSPECIFIED_COMPAT VACOPT_TERNARY_DEFAULT #define VACOPTVALUE_DISABLED_COMPAT VACOPT_TERNARY_DISABLED #define VACOPTVALUE_ENABLED_COMPAT VACOPT_TERNARY_ENABLED +#define IsReindexWithParam_compat(reindex, param) \ + ((strcmp(param, "concurrently") == 0) ? ((reindex)->concurrent) : \ + ((strcmp(param, "verbose") == 0) ? ((reindex)->options == REINDEXOPT_VERBOSE) : \ + false)) #endif #if PG_VERSION_NUM >= PG_VERSION_13 From 35cfa5d7b9f8c935c3dfd8274819864c3dd56499 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Fri, 13 Aug 2021 15:32:18 +0300 Subject: [PATCH 015/104] Introduces CopyFromState_compat macro CopyState struct is divided into parts and one of them is CopyFromState This macro uses the appropriate one for PG versions Relevant PG commit: c532d15dddff14b01fe9ef1d465013cb8ef186df --- .../distributed/commands/local_multi_copy.c | 13 ++++---- src/backend/distributed/commands/multi_copy.c | 30 ++++++++++--------- .../distributed/executor/multi_executor.c | 5 ++-- src/include/distributed/version_compat.h | 2 ++ 4 files changed, 27 insertions(+), 23 deletions(-) diff --git a/src/backend/distributed/commands/local_multi_copy.c b/src/backend/distributed/commands/local_multi_copy.c index ca532ef70..12b53e90a 100644 --- a/src/backend/distributed/commands/local_multi_copy.c +++ b/src/backend/distributed/commands/local_multi_copy.c @@ -209,13 +209,12 @@ DoLocalCopy(StringInfo buffer, Oid relationId, int64 shardId, CopyStmt *copyStat Oid shardOid = GetTableLocalShardOid(relationId, shardId); Relation shard = table_open(shardOid, RowExclusiveLock); ParseState *pState = make_parsestate(NULL); - - /* p_rtable of pState is set so that we can check constraints. */ - pState->p_rtable = CreateRangeTable(shard, ACL_INSERT); - - CopyState cstate = BeginCopyFrom(pState, shard, NULL, false, - ReadFromLocalBufferCallback, - copyStatement->attlist, copyStatement->options); + (void) addRangeTableEntryForRelation(pState, shard, AccessShareLock, + NULL, false, false); + CopyFromState cstate = BeginCopyFrom_compat(pState, shard, NULL, NULL, false, + ReadFromLocalBufferCallback, + copyStatement->attlist, + copyStatement->options); CopyFrom(cstate); EndCopyFrom(cstate); diff --git a/src/backend/distributed/commands/multi_copy.c b/src/backend/distributed/commands/multi_copy.c index 98449d0d3..ebc9f3fa6 100644 --- a/src/backend/distributed/commands/multi_copy.c +++ b/src/backend/distributed/commands/multi_copy.c @@ -520,13 +520,14 @@ CopyToExistingShards(CopyStmt *copyStatement, QueryCompletionCompat *completionT } /* initialize copy state to read from COPY data source */ - CopyState copyState = BeginCopyFrom(NULL, - copiedDistributedRelation, - copyStatement->filename, - copyStatement->is_program, - NULL, - copyStatement->attlist, - copyStatement->options); + CopyFromState copyState = BeginCopyFrom_compat(NULL, + copiedDistributedRelation, + NULL, + copyStatement->filename, + copyStatement->is_program, + NULL, + copyStatement->attlist, + copyStatement->options); /* set up callback to identify error line number */ errorCallback.callback = CopyFromErrorCallback; @@ -617,13 +618,14 @@ CopyToNewShards(CopyStmt *copyStatement, QueryCompletionCompat *completionTag, O (ShardConnections *) palloc0(sizeof(ShardConnections)); /* initialize copy state to read from COPY data source */ - CopyState copyState = BeginCopyFrom(NULL, - distributedRelation, - copyStatement->filename, - copyStatement->is_program, - NULL, - copyStatement->attlist, - copyStatement->options); + CopyFromState copyState = BeginCopyFrom_compat(NULL, + distributedRelation, + NULL, + copyStatement->filename, + copyStatement->is_program, + NULL, + copyStatement->attlist, + copyStatement->options); CopyOutState copyOutState = (CopyOutState) palloc0(sizeof(CopyOutStateData)); copyOutState->delim = (char *) delimiterCharacter; diff --git a/src/backend/distributed/executor/multi_executor.c b/src/backend/distributed/executor/multi_executor.c index 121766a02..cdbafacc0 100644 --- a/src/backend/distributed/executor/multi_executor.c +++ b/src/backend/distributed/executor/multi_executor.c @@ -410,8 +410,9 @@ ReadFileIntoTupleStore(char *fileName, char *copyFormat, TupleDesc tupleDescript location); copyOptions = lappend(copyOptions, copyOption); - CopyState copyState = BeginCopyFrom(NULL, stubRelation, fileName, false, NULL, - NULL, copyOptions); + CopyFromState copyState = BeginCopyFrom_compat(NULL, stubRelation, NULL, + fileName, false, NULL, + NULL, copyOptions); while (true) { diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index 0694219be..994da5f2d 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -48,6 +48,7 @@ #define VACOPTVALUE_DISABLED_COMPAT VACOPTVALUE_DISABLED #define VACOPTVALUE_ENABLED_COMPAT VACOPTVALUE_ENABLED #define IsReindexWithParam_compat(reindex, param) IsReindexWithParam(reindex, param) +#define CopyFromState_compat CopyFromState #else #define AlterTableStmtObjType(a) ((a)->relkind) #define F_NEXTVAL_COMPAT F_NEXTVAL_OID @@ -69,6 +70,7 @@ ((strcmp(param, "concurrently") == 0) ? ((reindex)->concurrent) : \ ((strcmp(param, "verbose") == 0) ? ((reindex)->options == REINDEXOPT_VERBOSE) : \ false)) +#define CopyFromState_compat CopyState #endif #if PG_VERSION_NUM >= PG_VERSION_13 From db2d9af863f30a18efaeb2b3c61308fe1d98fc46 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Fri, 13 Aug 2021 15:58:21 +0300 Subject: [PATCH 016/104] Introduces BeginCopyFrom_compat macro BeginCopyFrom function now has a new whereClause parameter. In the function this parameter is assigned to the whereClause field of the CopyFromState returned Currently in Postgres there is only one place where this argument isn't NULL, and in previous PG version the whereClause argument of copy state is set right after the function call Since we don't have such example all current whereClause parameters are set to NULL Relevant PG commit: c532d15dddff14b01fe9ef1d465013cb8ef186df --- src/include/distributed/version_compat.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index 994da5f2d..588676929 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -49,6 +49,7 @@ #define VACOPTVALUE_ENABLED_COMPAT VACOPTVALUE_ENABLED #define IsReindexWithParam_compat(reindex, param) IsReindexWithParam(reindex, param) #define CopyFromState_compat CopyFromState +#define BeginCopyFrom_compat(a, b, c, d, e, f, g, h) BeginCopyFrom(a, b, c, d, e, f, g, h) #else #define AlterTableStmtObjType(a) ((a)->relkind) #define F_NEXTVAL_COMPAT F_NEXTVAL_OID @@ -71,6 +72,7 @@ ((strcmp(param, "verbose") == 0) ? ((reindex)->options == REINDEXOPT_VERBOSE) : \ false)) #define CopyFromState_compat CopyState +#define BeginCopyFrom_compat(a, b, c, d, e, f, g, h) BeginCopyFrom(a, b, d, e, f, g, h) #endif #if PG_VERSION_NUM >= PG_VERSION_13 From 5df625161918bc4e55a0f5c962296198cce9a1f0 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Wed, 11 Aug 2021 14:59:40 +0300 Subject: [PATCH 017/104] Removes CopyGetAttnums function definition for PG14 This function was copied from Postgres but it is not static at PG14 So we keep the definition only for previous versions Relevant PG commit: c532d15dddff14b01fe9ef1d465013cb8ef186df --- src/backend/distributed/commands/multi_copy.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/backend/distributed/commands/multi_copy.c b/src/backend/distributed/commands/multi_copy.c index ebc9f3fa6..bdfccda23 100644 --- a/src/backend/distributed/commands/multi_copy.c +++ b/src/backend/distributed/commands/multi_copy.c @@ -267,7 +267,9 @@ static CopyCoercionData * ColumnCoercionPaths(TupleDesc destTupleDescriptor, Oid *finalColumnTypeArray); static FmgrInfo * TypeOutputFunctions(uint32 columnCount, Oid *typeIdArray, bool binaryFormat); +#if PG_VERSION_NUM < PG_VERSION_14 static List * CopyGetAttnums(TupleDesc tupDesc, Relation rel, List *attnamelist); +#endif static bool CopyStatementHasFormat(CopyStmt *copyStatement, char *formatName); static void CitusCopyFrom(CopyStmt *copyStatement, QueryCompletionCompat *completionTag); static HTAB * CreateConnectionStateHash(MemoryContext memoryContext); @@ -3277,6 +3279,8 @@ CreateRangeTable(Relation rel, AclMode requiredAccess) } +#if PG_VERSION_NUM < PG_VERSION_14 + /* Helper for CheckCopyPermissions(), copied from postgres */ static List * CopyGetAttnums(TupleDesc tupDesc, Relation rel, List *attnamelist) @@ -3358,6 +3362,9 @@ CopyGetAttnums(TupleDesc tupDesc, Relation rel, List *attnamelist) } +#endif + + /* * CreateConnectionStateHash constructs a hash table which maps from socket * number to CopyConnectionState, passing the provided MemoryContext to From 82858ca8fe9986e7da76de9151d81a263f4e8350 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Fri, 13 Aug 2021 19:48:50 +0300 Subject: [PATCH 018/104] Introduces ProcessUtility macros for readOnlyTree parameter New macros: standard_ProcessUtility_compat, ProcessUtility_compat, ColumnarProcessUtility_compat, PrevProcessUtilityHook_compat The functions now have a new bool parameter: readOnlyTree These new macros give us the ability to use this new parameter for PG14 and it doesn't give the parameter for previous versions In multi_ProcessUtility and ColumnarProcessUtility, before doing anything else, we check if readOnlyTree parameter is true and create a copy of pstmt Existing readOnlyTree parameters are set to false since we already handle the read only case at multi_ProcessUtility and ColumnarProcessUtility Relevant PG commit: 7c337b6b527b7052e6a751f966d5734c56f668b5 --- src/backend/columnar/columnar_tableam.c | 18 ++++++++- .../distributed/commands/utility_hook.c | 38 ++++++++++++------- .../columnar/columnar_version_compat.h | 8 ++++ .../distributed/commands/utility_hook.h | 3 ++ src/include/distributed/version_compat.h | 7 ++++ 5 files changed, 59 insertions(+), 15 deletions(-) diff --git a/src/backend/columnar/columnar_tableam.c b/src/backend/columnar/columnar_tableam.c index 9c7c42bf4..7577deb05 100644 --- a/src/backend/columnar/columnar_tableam.c +++ b/src/backend/columnar/columnar_tableam.c @@ -118,6 +118,9 @@ static void ColumnarTableAMObjectAccessHook(ObjectAccessType access, Oid classId void *arg); static void ColumnarProcessUtility(PlannedStmt *pstmt, const char *queryString, +#if PG_VERSION_NUM >= PG_VERSION_14 + bool readOnlyTree, +#endif ProcessUtilityContext context, ParamListInfo params, struct QueryEnvironment *queryEnv, @@ -2006,12 +2009,23 @@ ColumnarTableAMObjectAccessHook(ObjectAccessType access, Oid classId, Oid object static void ColumnarProcessUtility(PlannedStmt *pstmt, const char *queryString, +#if PG_VERSION_NUM >= PG_VERSION_14 + bool readOnlyTree, +#endif ProcessUtilityContext context, ParamListInfo params, struct QueryEnvironment *queryEnv, DestReceiver *dest, QueryCompletionCompat *completionTag) { + +#if PG_VERSION_NUM >= PG_VERSION_14 + if (readOnlyTree) + { + pstmt = copyObject(pstmt); + } +#endif + Node *parsetree = pstmt->utilityStmt; if (IsA(parsetree, IndexStmt)) @@ -2034,8 +2048,8 @@ ColumnarProcessUtility(PlannedStmt *pstmt, RelationClose(rel); } - PrevProcessUtilityHook(pstmt, queryString, context, - params, queryEnv, dest, completionTag); + PrevProcessUtilityHook_compat(pstmt, queryString, false, context, + params, queryEnv, dest, completionTag); } diff --git a/src/backend/distributed/commands/utility_hook.c b/src/backend/distributed/commands/utility_hook.c index 26b162cd1..82837ba59 100644 --- a/src/backend/distributed/commands/utility_hook.c +++ b/src/backend/distributed/commands/utility_hook.c @@ -111,8 +111,8 @@ ProcessUtilityParseTree(Node *node, const char *queryString, ProcessUtilityConte plannedStmt->commandType = CMD_UTILITY; plannedStmt->utilityStmt = node; - ProcessUtility(plannedStmt, queryString, context, params, NULL, dest, - completionTag); + ProcessUtility_compat(plannedStmt, queryString, false, context, params, NULL, dest, + completionTag); } @@ -128,13 +128,25 @@ ProcessUtilityParseTree(Node *node, const char *queryString, ProcessUtilityConte void multi_ProcessUtility(PlannedStmt *pstmt, const char *queryString, +#if PG_VERSION_NUM >= PG_VERSION_14 + bool readOnlyTree, +#endif ProcessUtilityContext context, ParamListInfo params, struct QueryEnvironment *queryEnv, DestReceiver *dest, QueryCompletionCompat *completionTag) { - Node *parsetree = pstmt->utilityStmt; + Node *parsetree; + +#if PG_VERSION_NUM >= PG_VERSION_14 + if (readOnlyTree) + { + pstmt = copyObject(pstmt); + } +#endif + + parsetree = pstmt->utilityStmt; if (IsA(parsetree, TransactionStmt) || IsA(parsetree, LockStmt) || @@ -154,8 +166,8 @@ multi_ProcessUtility(PlannedStmt *pstmt, * that state. Since we never need to intercept transaction statements, * skip our checks and immediately fall into standard_ProcessUtility. */ - standard_ProcessUtility(pstmt, queryString, context, - params, queryEnv, dest, completionTag); + standard_ProcessUtility_compat(pstmt, queryString, false, context, + params, queryEnv, dest, completionTag); return; } @@ -173,8 +185,8 @@ multi_ProcessUtility(PlannedStmt *pstmt, * Ensure that utility commands do not behave any differently until CREATE * EXTENSION is invoked. */ - standard_ProcessUtility(pstmt, queryString, context, - params, queryEnv, dest, completionTag); + standard_ProcessUtility_compat(pstmt, queryString, false, context, + params, queryEnv, dest, completionTag); return; } @@ -205,8 +217,8 @@ multi_ProcessUtility(PlannedStmt *pstmt, PG_TRY(); { - standard_ProcessUtility(pstmt, queryString, context, - params, queryEnv, dest, completionTag); + standard_ProcessUtility_compat(pstmt, queryString, false, context, + params, queryEnv, dest, completionTag); StoredProcedureLevel -= 1; } @@ -229,8 +241,8 @@ multi_ProcessUtility(PlannedStmt *pstmt, PG_TRY(); { - standard_ProcessUtility(pstmt, queryString, context, - params, queryEnv, dest, completionTag); + standard_ProcessUtility_compat(pstmt, queryString, false, context, + params, queryEnv, dest, completionTag); DoBlockLevel -= 1; } @@ -566,8 +578,8 @@ ProcessUtilityInternal(PlannedStmt *pstmt, citusCanBeUpdatedToAvailableVersion = !InstalledAndAvailableVersionsSame(); } - standard_ProcessUtility(pstmt, queryString, context, - params, queryEnv, dest, completionTag); + standard_ProcessUtility_compat(pstmt, queryString, false, context, + params, queryEnv, dest, completionTag); /* * if we are running ALTER EXTENSION citus UPDATE (to "") command, we may need diff --git a/src/include/columnar/columnar_version_compat.h b/src/include/columnar/columnar_version_compat.h index 29a0a42d5..36b7f5068 100644 --- a/src/include/columnar/columnar_version_compat.h +++ b/src/include/columnar/columnar_version_compat.h @@ -13,7 +13,15 @@ #define COLUMNAR_COMPAT_H #if PG_VERSION_NUM >= PG_VERSION_14 +#define ColumnarProcessUtility_compat(a, b, c, d, e, f, g, h) \ + ColumnarProcessUtility(a, b, c, d, e, f, g, h) +#define PrevProcessUtilityHook_compat(a, b, c, d, e, f, g, h) \ + PrevProcessUtilityHook(a, b, c, d, e, f, g, h) #else +#define ColumnarProcessUtility_compat(a, b, c, d, e, f, g, h) \ + ColumnarProcessUtility(a, b, d, e, f, g, h) +#define PrevProcessUtilityHook_compat(a, b, c, d, e, f, g, h) \ + PrevProcessUtilityHook(a, b, d, e, f, g, h) #endif #define ACLCHECK_OBJECT_TABLE OBJECT_TABLE diff --git a/src/include/distributed/commands/utility_hook.h b/src/include/distributed/commands/utility_hook.h index 24717986e..22f8a8cf1 100644 --- a/src/include/distributed/commands/utility_hook.h +++ b/src/include/distributed/commands/utility_hook.h @@ -64,6 +64,9 @@ typedef struct DDLJob extern void multi_ProcessUtility(PlannedStmt *pstmt, const char *queryString, +#if PG_VERSION_NUM >= PG_VERSION_14 + bool readOnlyTree, +#endif ProcessUtilityContext context, ParamListInfo params, struct QueryEnvironment *queryEnv, DestReceiver *dest, QueryCompletionCompat *completionTag diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index 588676929..1f97ff901 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -50,6 +50,10 @@ #define IsReindexWithParam_compat(reindex, param) IsReindexWithParam(reindex, param) #define CopyFromState_compat CopyFromState #define BeginCopyFrom_compat(a, b, c, d, e, f, g, h) BeginCopyFrom(a, b, c, d, e, f, g, h) +#define standard_ProcessUtility_compat(a, b, c, d, e, f, g, h) \ + standard_ProcessUtility(a, b, c, d, e, f, g, h) +#define ProcessUtility_compat(a, b, c, d, e, f, g, h) \ + ProcessUtility(a, b, c, d, e, f, g, h) #else #define AlterTableStmtObjType(a) ((a)->relkind) #define F_NEXTVAL_COMPAT F_NEXTVAL_OID @@ -73,6 +77,9 @@ false)) #define CopyFromState_compat CopyState #define BeginCopyFrom_compat(a, b, c, d, e, f, g, h) BeginCopyFrom(a, b, d, e, f, g, h) +#define standard_ProcessUtility_compat(a, b, c, d, e, f, g, h) \ + standard_ProcessUtility(a, b, d, e, f, g, h) +#define ProcessUtility_compat(a, b, c, d, e, f, g, h) ProcessUtility(a, b, d, e, f, g, h) #endif #if PG_VERSION_NUM >= PG_VERSION_13 From 1d5053b652185b1f17dbdeea2901ee65fd86342a Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Sat, 14 Aug 2021 12:22:40 +0300 Subject: [PATCH 019/104] Removes support for old protocols in Copy functions from PG14 Some Copy related functions copied from Postgres had support for both old and new protocols Postgres removed support for old version so we remove it too Relevant PG commit: 3174d69fb96a66173224e60ec7053b988d5ed4d9 --- src/backend/distributed/commands/multi_copy.c | 55 ++++++++++--------- src/include/distributed/commands/multi_copy.h | 4 ++ src/include/distributed/version_compat.h | 2 + 3 files changed, 34 insertions(+), 27 deletions(-) diff --git a/src/backend/distributed/commands/multi_copy.c b/src/backend/distributed/commands/multi_copy.c index bdfccda23..ed616780c 100644 --- a/src/backend/distributed/commands/multi_copy.c +++ b/src/backend/distributed/commands/multi_copy.c @@ -67,6 +67,7 @@ #include "catalog/pg_type.h" #include "commands/copy.h" #include "commands/defrem.h" +#include "commands/progress.h" #include "distributed/citus_safe_lib.h" #include "distributed/commands/multi_copy.h" #include "distributed/commands/utility_hook.h" @@ -1806,24 +1807,8 @@ CreateEmptyShard(char *relationName) static void SendCopyBegin(CopyOutState cstate) { - if (PG_PROTOCOL_MAJOR(FrontendProtocol) >= 3) - { - /* new way */ - StringInfoData buf; - int natts = list_length(cstate->attnumlist); - int16 format = (cstate->binary ? 1 : 0); - int i; - - pq_beginmessage(&buf, 'H'); - pq_sendbyte(&buf, format); /* overall format */ - pq_sendint16(&buf, natts); - for (i = 0; i < natts; i++) - pq_sendint16(&buf, format); /* per-column formats */ - pq_endmessage(&buf); - cstate->copy_dest = COPY_NEW_FE; - } - else - { +#if PG_VERSION_NUM < PG_VERSION_14 + if (PG_PROTOCOL_MAJOR(FrontendProtocol) < 3) { /* old way */ if (cstate->binary) ereport(ERROR, @@ -1833,7 +1818,21 @@ SendCopyBegin(CopyOutState cstate) /* grottiness needed for old COPY OUT protocol */ pq_startcopyout(); cstate->copy_dest = COPY_OLD_FE; + return; } +#endif + StringInfoData buf; + int natts = list_length(cstate->attnumlist); + int16 format = (cstate->binary ? 1 : 0); + int i; + + pq_beginmessage(&buf, 'H'); + pq_sendbyte(&buf, format); /* overall format */ + pq_sendint16(&buf, natts); + for (i = 0; i < natts; i++) + pq_sendint16(&buf, format); /* per-column formats */ + pq_endmessage(&buf); + cstate->copy_dest = COPY_FRONTEND; } @@ -1841,20 +1840,20 @@ SendCopyBegin(CopyOutState cstate) static void SendCopyEnd(CopyOutState cstate) { - if (cstate->copy_dest == COPY_NEW_FE) - { - /* Shouldn't have any unsent data */ - Assert(cstate->fe_msgbuf->len == 0); - /* Send Copy Done message */ - pq_putemptymessage('c'); - } - else +#if PG_VERSION_NUM < PG_VERSION_14 + if (cstate->copy_dest != COPY_NEW_FE) { CopySendData(cstate, "\\.", 2); /* Need to flush out the trailer (this also appends a newline) */ CopySendEndOfRow(cstate, true); pq_endcopyout(false); + return; } +#endif + /* Shouldn't have any unsent data */ + Assert(cstate->fe_msgbuf->len == 0); + /* Send Copy Done message */ + pq_putemptymessage('c'); } @@ -1908,6 +1907,7 @@ CopySendEndOfRow(CopyOutState cstate, bool includeEndOfLine) switch (cstate->copy_dest) { +#if PG_VERSION_NUM < PG_VERSION_14 case COPY_OLD_FE: /* The FE/BE protocol uses \n as newline for all platforms */ if (!cstate->binary && includeEndOfLine) @@ -1921,7 +1921,8 @@ CopySendEndOfRow(CopyOutState cstate, bool includeEndOfLine) errmsg("connection lost during COPY to stdout"))); } break; - case COPY_NEW_FE: +#endif + case COPY_FRONTEND: /* The FE/BE protocol uses \n as newline for all platforms */ if (!cstate->binary && includeEndOfLine) CopySendChar(cstate, '\n'); diff --git a/src/include/distributed/commands/multi_copy.h b/src/include/distributed/commands/multi_copy.h index a5f414208..4d1988347 100644 --- a/src/include/distributed/commands/multi_copy.h +++ b/src/include/distributed/commands/multi_copy.h @@ -31,8 +31,12 @@ typedef enum CitusCopyDest { COPY_FILE, /* to/from file (or a piped program) */ +#if PG_VERSION_NUM >= PG_VERSION_14 + COPY_FRONTEND, /* to frontend */ +#else COPY_OLD_FE, /* to/from frontend (2.0 protocol) */ COPY_NEW_FE, /* to/from frontend (3.0 protocol) */ +#endif COPY_CALLBACK /* to/from callback function */ } CitusCopyDest; diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index 1f97ff901..e5df7a264 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -54,6 +54,7 @@ standard_ProcessUtility(a, b, c, d, e, f, g, h) #define ProcessUtility_compat(a, b, c, d, e, f, g, h) \ ProcessUtility(a, b, c, d, e, f, g, h) +#define COPY_FRONTEND_COMPAT COPY_FRONTEND #else #define AlterTableStmtObjType(a) ((a)->relkind) #define F_NEXTVAL_COMPAT F_NEXTVAL_OID @@ -80,6 +81,7 @@ #define standard_ProcessUtility_compat(a, b, c, d, e, f, g, h) \ standard_ProcessUtility(a, b, d, e, f, g, h) #define ProcessUtility_compat(a, b, c, d, e, f, g, h) ProcessUtility(a, b, d, e, f, g, h) +#define COPY_FRONTEND_COMPAT COPY_NEW_FE #endif #if PG_VERSION_NUM >= PG_VERSION_13 From e38b75799d34479cf08a606086610285f180e6a8 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Mon, 16 Aug 2021 10:52:55 +0300 Subject: [PATCH 020/104] Fixes some indentation in ruleutils_14.c Relevant PG commit: fa27dd40d5c5f56a1ee837a75c97549e992e32a4 --- src/backend/distributed/deparser/ruleutils_14.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/backend/distributed/deparser/ruleutils_14.c b/src/backend/distributed/deparser/ruleutils_14.c index 6eef32c53..b6bbff803 100644 --- a/src/backend/distributed/deparser/ruleutils_14.c +++ b/src/backend/distributed/deparser/ruleutils_14.c @@ -2667,8 +2667,8 @@ get_rule_sortgroupclause(Index ref, List *tlist, bool force_colno, */ bool need_paren = (PRETTY_PAREN(context) || IsA(expr, FuncExpr) - ||IsA(expr, Aggref) - ||IsA(expr, WindowFunc)); + || IsA(expr, Aggref) + || IsA(expr, WindowFunc)); if (need_paren) appendStringInfoChar(context->buf, '('); @@ -7321,7 +7321,7 @@ get_from_clause_item(Node *jtnode, Query *query, deparse_context *context) need_paren_on_right = PRETTY_PAREN(context) && !IsA(j->rarg, RangeTblRef) && - !(IsA(j->rarg, JoinExpr) &&((JoinExpr *) j->rarg)->alias != NULL); + !(IsA(j->rarg, JoinExpr) && ((JoinExpr *) j->rarg)->alias != NULL); if (!PRETTY_PAREN(context) || j->alias != NULL) appendStringInfoChar(buf, '('); From a710b3b9494973463b5bb8eac9a32a9d540e931f Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Mon, 16 Aug 2021 10:57:01 +0300 Subject: [PATCH 021/104] Removes some comments with printf %.*s format from ruleutils_14.c Relevant PG commit: c410af098c46949e36607eb13689e697fa2def97 --- src/backend/distributed/deparser/ruleutils_14.c | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/backend/distributed/deparser/ruleutils_14.c b/src/backend/distributed/deparser/ruleutils_14.c index b6bbff803..071ed0722 100644 --- a/src/backend/distributed/deparser/ruleutils_14.c +++ b/src/backend/distributed/deparser/ruleutils_14.c @@ -630,11 +630,6 @@ set_rtable_names(deparse_namespace *dpns, List *parent_namespaces, hentry->counter++; for (;;) { - /* - * We avoid using %.*s here because it can misbehave - * if the data is not valid in what libc thinks is the - * prevailing encoding. - */ memcpy(modname, refname, refnamelen); sprintf(modname + refnamelen, "_%d", hentry->counter); if (strlen(modname) < NAMEDATALEN) @@ -1492,11 +1487,6 @@ make_colname_unique(char *colname, deparse_namespace *dpns, i++; for (;;) { - /* - * We avoid using %.*s here because it can misbehave if the - * data is not valid in what libc thinks is the prevailing - * encoding. - */ memcpy(modname, colname, colnamelen); sprintf(modname + colnamelen, "_%d", i); if (strlen(modname) < NAMEDATALEN) From e642f6c97f78d45d6b11c53464152b25184e7dce Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Mon, 16 Aug 2021 11:14:09 +0300 Subject: [PATCH 022/104] Removes support for postfix operators from ruleutils_14.c Relevant PG commit: 1ed6b895634ce0dc5fd4bd040e87252b32182cba --- .../distributed/deparser/ruleutils_14.c | 33 ++++--------------- 1 file changed, 6 insertions(+), 27 deletions(-) diff --git a/src/backend/distributed/deparser/ruleutils_14.c b/src/backend/distributed/deparser/ruleutils_14.c index 071ed0722..2fe4b13b3 100644 --- a/src/backend/distributed/deparser/ruleutils_14.c +++ b/src/backend/distributed/deparser/ruleutils_14.c @@ -6171,35 +6171,14 @@ get_oper_expr(OpExpr *expr, deparse_context *context) } else { - /* unary operator --- but which side? */ + /* prefix operator */ Node *arg = (Node *) linitial(args); - HeapTuple tp; - Form_pg_operator optup; - tp = SearchSysCache1(OPEROID, ObjectIdGetDatum(opno)); - if (!HeapTupleIsValid(tp)) - elog(ERROR, "cache lookup failed for operator %u", opno); - optup = (Form_pg_operator) GETSTRUCT(tp); - switch (optup->oprkind) - { - case 'l': - appendStringInfo(buf, "%s ", - generate_operator_name(opno, - InvalidOid, - exprType(arg))); - get_rule_expr_paren(arg, context, true, (Node *) expr); - break; - case 'r': - get_rule_expr_paren(arg, context, true, (Node *) expr); - appendStringInfo(buf, " %s", - generate_operator_name(opno, - exprType(arg), - InvalidOid)); - break; - default: - elog(ERROR, "bogus oprkind: %d", optup->oprkind); - } - ReleaseSysCache(tp); + appendStringInfo(buf, "%s ", + generate_operator_name(opno, + InvalidOid, + exprType(arg))); + get_rule_expr_paren(arg, context, true, (Node *) expr); } if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, ')'); From beb49f0d53d6c372b0a21750ad362363aed21361 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Mon, 16 Aug 2021 12:46:22 +0300 Subject: [PATCH 023/104] Updates AlternativeSubPlan comment in ruleutils_14.c Relevant PG commit: 41efb8340877e8ffd0023bb6b2ef22ffd1ca014d --- src/backend/distributed/deparser/ruleutils_14.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/backend/distributed/deparser/ruleutils_14.c b/src/backend/distributed/deparser/ruleutils_14.c index 2fe4b13b3..ec4ee679b 100644 --- a/src/backend/distributed/deparser/ruleutils_14.c +++ b/src/backend/distributed/deparser/ruleutils_14.c @@ -5143,7 +5143,12 @@ get_rule_expr(Node *node, deparse_context *context, AlternativeSubPlan *asplan = (AlternativeSubPlan *) node; ListCell *lc; - /* As above, this can only happen during EXPLAIN */ + /* + * This case cannot be reached in normal usage, since no + * AlternativeSubPlan can appear either in parsetrees or + * finished plan trees. We keep it just in case somebody + * wants to use this code to print planner data structures. + */ appendStringInfoString(buf, "(alternatives: "); foreach(lc, asplan->subplans) { From 69aa240b99c41a44849054688f5f8cea37768889 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Mon, 16 Aug 2021 13:20:27 +0300 Subject: [PATCH 024/104] Adds for_each_from to ruleutils_14.c Relevant PG commit: 56fe008996bc1a547ce60c8dddd2ca821cac163e --- src/backend/distributed/deparser/ruleutils_14.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/backend/distributed/deparser/ruleutils_14.c b/src/backend/distributed/deparser/ruleutils_14.c index ec4ee679b..229bba8f3 100644 --- a/src/backend/distributed/deparser/ruleutils_14.c +++ b/src/backend/distributed/deparser/ruleutils_14.c @@ -5064,7 +5064,7 @@ get_rule_expr(Node *node, deparse_context *context, { BoolExpr *expr = (BoolExpr *) node; Node *first_arg = linitial(expr->args); - ListCell *arg = list_second_cell(expr->args); + ListCell *arg; switch (expr->boolop) { @@ -5073,12 +5073,11 @@ get_rule_expr(Node *node, deparse_context *context, appendStringInfoChar(buf, '('); get_rule_expr_paren(first_arg, context, false, node); - while (arg) + for_each_from(arg, expr->args, 1) { appendStringInfoString(buf, " AND "); get_rule_expr_paren((Node *) lfirst(arg), context, false, node); - arg = lnext(expr->args, arg); } if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, ')'); @@ -5089,12 +5088,11 @@ get_rule_expr(Node *node, deparse_context *context, appendStringInfoChar(buf, '('); get_rule_expr_paren(first_arg, context, false, node); - while (arg) + for_each_from(arg, expr->args, 1) { appendStringInfoString(buf, " OR "); get_rule_expr_paren((Node *) lfirst(arg), context, false, node); - arg = lnext(expr->args, arg); } if (!PRETTY_PAREN(context)) appendStringInfoChar(buf, ')'); From 30f77b29a71f9ea7876bff1a963b54fd7ac04cc9 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Mon, 16 Aug 2021 13:27:03 +0300 Subject: [PATCH 025/104] Fixes some appendStringInfos in ruleutils_14.c Relevant PG commit: 110d81728a0a006abcf654543fc15346f8043dc0 --- src/backend/distributed/deparser/ruleutils_14.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/distributed/deparser/ruleutils_14.c b/src/backend/distributed/deparser/ruleutils_14.c index 229bba8f3..ef3b26cfb 100644 --- a/src/backend/distributed/deparser/ruleutils_14.c +++ b/src/backend/distributed/deparser/ruleutils_14.c @@ -2091,7 +2091,7 @@ get_select_query_def(Query *query, deparse_context *context, appendContextKeyword(context, " FETCH FIRST (", -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); get_rule_expr(query->limitCount, context, false); - appendStringInfo(buf, ") ROWS WITH TIES"); + appendStringInfoString(buf, ") ROWS WITH TIES"); } else { @@ -8446,7 +8446,7 @@ get_range_partbound_string(List *bound_datums) memset(&context, 0, sizeof(deparse_context)); context.buf = buf; - appendStringInfoString(buf, "("); + appendStringInfoChar(buf, '('); sep = ""; foreach(cell, bound_datums) { From b4f76303c6ad3155d18dd7227bcc1c5521e739d8 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Mon, 16 Aug 2021 13:34:21 +0300 Subject: [PATCH 026/104] Updates F_ARRAY_UNNEST to F_UNNEST_ANYARRAY in ruleutils_14.c Relevant PG commit: 8e1f37c07aafd4bb7aa6e1e1982010af11f8b5c7 --- src/backend/distributed/deparser/ruleutils_14.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/distributed/deparser/ruleutils_14.c b/src/backend/distributed/deparser/ruleutils_14.c index ef3b26cfb..fba572053 100644 --- a/src/backend/distributed/deparser/ruleutils_14.c +++ b/src/backend/distributed/deparser/ruleutils_14.c @@ -7134,7 +7134,7 @@ get_from_clause_item(Node *jtnode, Query *query, deparse_context *context) RangeTblFunction *rtfunc = (RangeTblFunction *) lfirst(lc); if (!IsA(rtfunc->funcexpr, FuncExpr) || - ((FuncExpr *) rtfunc->funcexpr)->funcid != F_ARRAY_UNNEST || + ((FuncExpr *) rtfunc->funcexpr)->funcid != F_UNNEST_ANYARRAY || rtfunc->funccolnames != NIL) { all_unnest = false; From 1cb865deb817cd5d2afe5bb6e149f5c72500c623 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Mon, 16 Aug 2021 14:54:43 +0300 Subject: [PATCH 027/104] Adds SQL syntax function calls related changes to ruleutils_14.c Relevant PG commit: 40c24bfef92530bd846e111c1742c2a54441c62c --- .../distributed/deparser/ruleutils_14.c | 235 +++++++++++++++++- 1 file changed, 234 insertions(+), 1 deletion(-) diff --git a/src/backend/distributed/deparser/ruleutils_14.c b/src/backend/distributed/deparser/ruleutils_14.c index fba572053..075d1332f 100644 --- a/src/backend/distributed/deparser/ruleutils_14.c +++ b/src/backend/distributed/deparser/ruleutils_14.c @@ -413,6 +413,7 @@ static void get_agg_expr(Aggref *aggref, deparse_context *context, static void get_agg_combine_expr(Node *node, deparse_context *context, void *callback_arg); static void get_windowfunc_expr(WindowFunc *wfunc, deparse_context *context); +static bool get_func_sql_syntax(FuncExpr *expr, deparse_context *context); static void get_coercion_expr(Node *arg, deparse_context *context, Oid resulttype, int32 resulttypmod, Node *parentNode); @@ -6132,7 +6133,8 @@ looks_like_function(Node *node) { case T_FuncExpr: /* OK, unless it's going to deparse as a cast */ - return (((FuncExpr *) node)->funcformat == COERCE_EXPLICIT_CALL); + return (((FuncExpr *) node)->funcformat == COERCE_EXPLICIT_CALL || + ((FuncExpr *) node)->funcformat == COERCE_SQL_SYNTAX); case T_NullIfExpr: case T_CoalesceExpr: case T_MinMaxExpr: @@ -6234,6 +6236,18 @@ get_func_expr(FuncExpr *expr, deparse_context *context, return; } + /* + * If the function was called using one of the SQL spec's random special + * syntaxes, try to reproduce that. If we don't recognize the function, + * fall through. + */ + if (expr->funcformat == COERCE_SQL_SYNTAX) + { + if (get_func_sql_syntax(expr, context)) + return; + } + + /* * Normal function: display as proname(args). First we need to extract * the argument datatypes. @@ -6470,6 +6484,225 @@ get_windowfunc_expr(WindowFunc *wfunc, deparse_context *context) } } + +/* + * get_func_sql_syntax - Parse back a SQL-syntax function call + * + * Returns true if we successfully deparsed, false if we did not + * recognize the function. + */ +static bool +get_func_sql_syntax(FuncExpr *expr, deparse_context *context) +{ + StringInfo buf = context->buf; + Oid funcoid = expr->funcid; + + switch (funcoid) + { + case F_TIMEZONE_INTERVAL_TIMESTAMP: + case F_TIMEZONE_INTERVAL_TIMESTAMPTZ: + case F_TIMEZONE_INTERVAL_TIMETZ: + case F_TIMEZONE_TEXT_TIMESTAMP: + case F_TIMEZONE_TEXT_TIMESTAMPTZ: + case F_TIMEZONE_TEXT_TIMETZ: + /* AT TIME ZONE ... note reversed argument order */ + appendStringInfoChar(buf, '('); + get_rule_expr((Node *) lsecond(expr->args), context, false); + appendStringInfoString(buf, " AT TIME ZONE "); + get_rule_expr((Node *) linitial(expr->args), context, false); + appendStringInfoChar(buf, ')'); + return true; + + case F_OVERLAPS_TIMESTAMPTZ_INTERVAL_TIMESTAMPTZ_INTERVAL: + case F_OVERLAPS_TIMESTAMPTZ_INTERVAL_TIMESTAMPTZ_TIMESTAMPTZ: + case F_OVERLAPS_TIMESTAMPTZ_TIMESTAMPTZ_TIMESTAMPTZ_INTERVAL: + case F_OVERLAPS_TIMESTAMPTZ_TIMESTAMPTZ_TIMESTAMPTZ_TIMESTAMPTZ: + case F_OVERLAPS_TIMESTAMP_INTERVAL_TIMESTAMP_INTERVAL: + case F_OVERLAPS_TIMESTAMP_INTERVAL_TIMESTAMP_TIMESTAMP: + case F_OVERLAPS_TIMESTAMP_TIMESTAMP_TIMESTAMP_INTERVAL: + case F_OVERLAPS_TIMESTAMP_TIMESTAMP_TIMESTAMP_TIMESTAMP: + case F_OVERLAPS_TIMETZ_TIMETZ_TIMETZ_TIMETZ: + case F_OVERLAPS_TIME_INTERVAL_TIME_INTERVAL: + case F_OVERLAPS_TIME_INTERVAL_TIME_TIME: + case F_OVERLAPS_TIME_TIME_TIME_INTERVAL: + case F_OVERLAPS_TIME_TIME_TIME_TIME: + /* (x1, x2) OVERLAPS (y1, y2) */ + appendStringInfoString(buf, "(("); + get_rule_expr((Node *) linitial(expr->args), context, false); + appendStringInfoString(buf, ", "); + get_rule_expr((Node *) lsecond(expr->args), context, false); + appendStringInfoString(buf, ") OVERLAPS ("); + get_rule_expr((Node *) lthird(expr->args), context, false); + appendStringInfoString(buf, ", "); + get_rule_expr((Node *) lfourth(expr->args), context, false); + appendStringInfoString(buf, "))"); + return true; + + case F_IS_NORMALIZED: + /* IS xxx NORMALIZED */ + appendStringInfoString(buf, "(("); + get_rule_expr((Node *) linitial(expr->args), context, false); + appendStringInfoString(buf, ") IS"); + if (list_length(expr->args) == 2) + { + Const *con = (Const *) lsecond(expr->args); + + Assert(IsA(con, Const) && + con->consttype == TEXTOID && + !con->constisnull); + appendStringInfo(buf, " %s", + TextDatumGetCString(con->constvalue)); + } + appendStringInfoString(buf, " NORMALIZED)"); + return true; + + case F_PG_COLLATION_FOR: + /* COLLATION FOR */ + appendStringInfoString(buf, "COLLATION FOR ("); + get_rule_expr((Node *) linitial(expr->args), context, false); + appendStringInfoChar(buf, ')'); + return true; + + /* + * XXX EXTRACT, a/k/a date_part(), is intentionally not covered + * yet. Add it after we change the return type to numeric. + */ + + case F_NORMALIZE: + /* NORMALIZE() */ + appendStringInfoString(buf, "NORMALIZE("); + get_rule_expr((Node *) linitial(expr->args), context, false); + if (list_length(expr->args) == 2) + { + Const *con = (Const *) lsecond(expr->args); + + Assert(IsA(con, Const) && + con->consttype == TEXTOID && + !con->constisnull); + appendStringInfo(buf, ", %s", + TextDatumGetCString(con->constvalue)); + } + appendStringInfoChar(buf, ')'); + return true; + + case F_OVERLAY_BIT_BIT_INT4: + case F_OVERLAY_BIT_BIT_INT4_INT4: + case F_OVERLAY_BYTEA_BYTEA_INT4: + case F_OVERLAY_BYTEA_BYTEA_INT4_INT4: + case F_OVERLAY_TEXT_TEXT_INT4: + case F_OVERLAY_TEXT_TEXT_INT4_INT4: + /* OVERLAY() */ + appendStringInfoString(buf, "OVERLAY("); + get_rule_expr((Node *) linitial(expr->args), context, false); + appendStringInfoString(buf, " PLACING "); + get_rule_expr((Node *) lsecond(expr->args), context, false); + appendStringInfoString(buf, " FROM "); + get_rule_expr((Node *) lthird(expr->args), context, false); + if (list_length(expr->args) == 4) + { + appendStringInfoString(buf, " FOR "); + get_rule_expr((Node *) lfourth(expr->args), context, false); + } + appendStringInfoChar(buf, ')'); + return true; + + case F_POSITION_BIT_BIT: + case F_POSITION_BYTEA_BYTEA: + case F_POSITION_TEXT_TEXT: + /* POSITION() ... extra parens since args are b_expr not a_expr */ + appendStringInfoString(buf, "POSITION(("); + get_rule_expr((Node *) lsecond(expr->args), context, false); + appendStringInfoString(buf, ") IN ("); + get_rule_expr((Node *) linitial(expr->args), context, false); + appendStringInfoString(buf, "))"); + return true; + + case F_SUBSTRING_BIT_INT4: + case F_SUBSTRING_BIT_INT4_INT4: + case F_SUBSTRING_BYTEA_INT4: + case F_SUBSTRING_BYTEA_INT4_INT4: + case F_SUBSTRING_TEXT_INT4: + case F_SUBSTRING_TEXT_INT4_INT4: + /* SUBSTRING FROM/FOR (i.e., integer-position variants) */ + appendStringInfoString(buf, "SUBSTRING("); + get_rule_expr((Node *) linitial(expr->args), context, false); + appendStringInfoString(buf, " FROM "); + get_rule_expr((Node *) lsecond(expr->args), context, false); + if (list_length(expr->args) == 3) + { + appendStringInfoString(buf, " FOR "); + get_rule_expr((Node *) lthird(expr->args), context, false); + } + appendStringInfoChar(buf, ')'); + return true; + + case F_SUBSTRING_TEXT_TEXT_TEXT: + /* SUBSTRING SIMILAR/ESCAPE */ + appendStringInfoString(buf, "SUBSTRING("); + get_rule_expr((Node *) linitial(expr->args), context, false); + appendStringInfoString(buf, " SIMILAR "); + get_rule_expr((Node *) lsecond(expr->args), context, false); + appendStringInfoString(buf, " ESCAPE "); + get_rule_expr((Node *) lthird(expr->args), context, false); + appendStringInfoChar(buf, ')'); + return true; + + case F_BTRIM_BYTEA_BYTEA: + case F_BTRIM_TEXT: + case F_BTRIM_TEXT_TEXT: + /* TRIM() */ + appendStringInfoString(buf, "TRIM(BOTH"); + if (list_length(expr->args) == 2) + { + appendStringInfoChar(buf, ' '); + get_rule_expr((Node *) lsecond(expr->args), context, false); + } + appendStringInfoString(buf, " FROM "); + get_rule_expr((Node *) linitial(expr->args), context, false); + appendStringInfoChar(buf, ')'); + return true; + + case F_LTRIM_TEXT: + case F_LTRIM_TEXT_TEXT: + /* TRIM() */ + appendStringInfoString(buf, "TRIM(LEADING"); + if (list_length(expr->args) == 2) + { + appendStringInfoChar(buf, ' '); + get_rule_expr((Node *) lsecond(expr->args), context, false); + } + appendStringInfoString(buf, " FROM "); + get_rule_expr((Node *) linitial(expr->args), context, false); + appendStringInfoChar(buf, ')'); + return true; + + case F_RTRIM_TEXT: + case F_RTRIM_TEXT_TEXT: + /* TRIM() */ + appendStringInfoString(buf, "TRIM(TRAILING"); + if (list_length(expr->args) == 2) + { + appendStringInfoChar(buf, ' '); + get_rule_expr((Node *) lsecond(expr->args), context, false); + } + appendStringInfoString(buf, " FROM "); + get_rule_expr((Node *) linitial(expr->args), context, false); + appendStringInfoChar(buf, ')'); + return true; + + case F_XMLEXISTS: + /* XMLEXISTS ... extra parens because args are c_expr */ + appendStringInfoString(buf, "XMLEXISTS(("); + get_rule_expr((Node *) linitial(expr->args), context, false); + appendStringInfoString(buf, ") PASSING ("); + get_rule_expr((Node *) lsecond(expr->args), context, false); + appendStringInfoString(buf, "))"); + return true; + } + return false; +} + + /* ---------- * get_coercion_expr * From d4874f5ad28953d8818457845ba40a2ba19d5035 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Mon, 16 Aug 2021 14:58:12 +0300 Subject: [PATCH 028/104] Removes indexing.h header from ruleutils_14.c Relevant PG commit: bdc4edbea6fc847f806e1e7118d730e159512bfc --- src/backend/distributed/deparser/ruleutils_14.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/backend/distributed/deparser/ruleutils_14.c b/src/backend/distributed/deparser/ruleutils_14.c index 075d1332f..9a1c9e4e8 100644 --- a/src/backend/distributed/deparser/ruleutils_14.c +++ b/src/backend/distributed/deparser/ruleutils_14.c @@ -32,7 +32,6 @@ #include "access/sysattr.h" #include "access/table.h" #include "catalog/dependency.h" -#include "catalog/indexing.h" #include "catalog/pg_aggregate.h" #include "catalog/pg_am.h" #include "catalog/pg_authid.h" From e72bd0c1a1257fbc2c787ec7ef8f16118fd3f640 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Mon, 16 Aug 2021 15:03:00 +0300 Subject: [PATCH 029/104] Removes dependency.h from ruleutils_14.c Relevant PG commit: 8b069ef5dca97cd737a5fd64c420df3cd61ec1c9 --- src/backend/distributed/deparser/ruleutils_14.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/backend/distributed/deparser/ruleutils_14.c b/src/backend/distributed/deparser/ruleutils_14.c index 9a1c9e4e8..8dae1c75e 100644 --- a/src/backend/distributed/deparser/ruleutils_14.c +++ b/src/backend/distributed/deparser/ruleutils_14.c @@ -31,7 +31,6 @@ #include "access/relation.h" #include "access/sysattr.h" #include "access/table.h" -#include "catalog/dependency.h" #include "catalog/pg_aggregate.h" #include "catalog/pg_am.h" #include "catalog/pg_authid.h" From 71691ecf06bd48f91713ba2e72d5b73c2dcaef41 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Mon, 16 Aug 2021 15:07:05 +0300 Subject: [PATCH 030/104] Adds HASH_STRINGS flag to ruleutils_14.c Relevant PG commit: b3817f5f774663d55931dd4fab9c5a94a15ae7ab --- src/backend/distributed/deparser/ruleutils_14.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/distributed/deparser/ruleutils_14.c b/src/backend/distributed/deparser/ruleutils_14.c index 8dae1c75e..75bbe90ad 100644 --- a/src/backend/distributed/deparser/ruleutils_14.c +++ b/src/backend/distributed/deparser/ruleutils_14.c @@ -540,14 +540,14 @@ set_rtable_names(deparse_namespace *dpns, List *parent_namespaces, * We use a hash table to hold known names, so that this process is O(N) * not O(N^2) for N names. */ - MemSet(&hash_ctl, 0, sizeof(hash_ctl)); hash_ctl.keysize = NAMEDATALEN; hash_ctl.entrysize = sizeof(NameHashEntry); hash_ctl.hcxt = CurrentMemoryContext; names_hash = hash_create("set_rtable_names names", list_length(dpns->rtable), &hash_ctl, - HASH_ELEM | HASH_CONTEXT); + HASH_ELEM | HASH_STRINGS | HASH_CONTEXT); + /* Preload the hash table with names appearing in parent_namespaces */ foreach(lc, parent_namespaces) { From 1174046a339c722d6412af8eb553ca0b938fd934 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Mon, 16 Aug 2021 15:14:47 +0300 Subject: [PATCH 031/104] Adds bytea equivalents of ltrim() and rtrim() to ruleutils_14.c Relevant PG commit: a6cf3df4ebdcbc7857910a67f259705645383e9f --- src/backend/distributed/deparser/ruleutils_14.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/backend/distributed/deparser/ruleutils_14.c b/src/backend/distributed/deparser/ruleutils_14.c index 75bbe90ad..117421c40 100644 --- a/src/backend/distributed/deparser/ruleutils_14.c +++ b/src/backend/distributed/deparser/ruleutils_14.c @@ -6660,6 +6660,7 @@ get_func_sql_syntax(FuncExpr *expr, deparse_context *context) appendStringInfoChar(buf, ')'); return true; + case F_LTRIM_BYTEA_BYTEA: case F_LTRIM_TEXT: case F_LTRIM_TEXT_TEXT: /* TRIM() */ @@ -6674,6 +6675,7 @@ get_func_sql_syntax(FuncExpr *expr, deparse_context *context) appendStringInfoChar(buf, ')'); return true; + case F_RTRIM_BYTEA_BYTEA: case F_RTRIM_TEXT: case F_RTRIM_TEXT_TEXT: /* TRIM() */ From 12b3c04fe36ed511f8de1d571a51525644d58c5b Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Mon, 16 Aug 2021 15:21:32 +0300 Subject: [PATCH 032/104] Adds SEARCH and CYCLE clauses to ruleutils_14.c Relevant PG commit: 3696a600e2292d43c00949ddf0352e4ebb487e5b --- .../distributed/deparser/ruleutils_14.c | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/src/backend/distributed/deparser/ruleutils_14.c b/src/backend/distributed/deparser/ruleutils_14.c index 117421c40..61efb468b 100644 --- a/src/backend/distributed/deparser/ruleutils_14.c +++ b/src/backend/distributed/deparser/ruleutils_14.c @@ -2011,6 +2011,53 @@ get_with_clause(Query *query, deparse_context *context) if (PRETTY_INDENT(context)) appendContextKeyword(context, "", 0, 0, 0); appendStringInfoChar(buf, ')'); + + if (cte->search_clause) + { + bool first = true; + ListCell *lc; + + appendStringInfo(buf, " SEARCH %s FIRST BY ", + cte->search_clause->search_breadth_first ? "BREADTH" : "DEPTH"); + + foreach(lc, cte->search_clause->search_col_list) + { + if (first) + first = false; + else + appendStringInfoString(buf, ", "); + appendStringInfoString(buf, + quote_identifier(strVal(lfirst(lc)))); + } + + appendStringInfo(buf, " SET %s", quote_identifier(cte->search_clause->search_seq_column)); + } + + if (cte->cycle_clause) + { + bool first = true; + ListCell *lc; + + appendStringInfoString(buf, " CYCLE "); + + foreach(lc, cte->cycle_clause->cycle_col_list) + { + if (first) + first = false; + else + appendStringInfoString(buf, ", "); + appendStringInfoString(buf, + quote_identifier(strVal(lfirst(lc)))); + } + + appendStringInfo(buf, " SET %s", quote_identifier(cte->cycle_clause->cycle_mark_column)); + appendStringInfoString(buf, " TO "); + get_rule_expr(cte->cycle_clause->cycle_mark_value, context, false); + appendStringInfoString(buf, " DEFAULT "); + get_rule_expr(cte->cycle_clause->cycle_mark_default, context, false); + appendStringInfo(buf, " USING %s", quote_identifier(cte->cycle_clause->cycle_path_column)); + } + sep = ", "; } From 5bb538543d394c6dedf7d5b08a6b240f622d30b4 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Mon, 16 Aug 2021 15:26:55 +0300 Subject: [PATCH 033/104] Enhances cycle mark values at ruleutils_14.c Relevant PG commit: f4adc41c4f92cc91d507b19e397140c35bb9fd71 --- .../distributed/deparser/ruleutils_14.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/backend/distributed/deparser/ruleutils_14.c b/src/backend/distributed/deparser/ruleutils_14.c index 61efb468b..5e85ffe7b 100644 --- a/src/backend/distributed/deparser/ruleutils_14.c +++ b/src/backend/distributed/deparser/ruleutils_14.c @@ -2051,10 +2051,21 @@ get_with_clause(Query *query, deparse_context *context) } appendStringInfo(buf, " SET %s", quote_identifier(cte->cycle_clause->cycle_mark_column)); - appendStringInfoString(buf, " TO "); - get_rule_expr(cte->cycle_clause->cycle_mark_value, context, false); - appendStringInfoString(buf, " DEFAULT "); - get_rule_expr(cte->cycle_clause->cycle_mark_default, context, false); + + { + Const *cmv = castNode(Const, cte->cycle_clause->cycle_mark_value); + Const *cmd = castNode(Const, cte->cycle_clause->cycle_mark_default); + + if (!(cmv->consttype == BOOLOID && !cmv->constisnull && DatumGetBool(cmv->constvalue) == true && + cmd->consttype == BOOLOID && !cmd->constisnull && DatumGetBool(cmd->constvalue) == false)) + { + appendStringInfoString(buf, " TO "); + get_rule_expr(cte->cycle_clause->cycle_mark_value, context, false); + appendStringInfoString(buf, " DEFAULT "); + get_rule_expr(cte->cycle_clause->cycle_mark_default, context, false); + } + } + appendStringInfo(buf, " USING %s", quote_identifier(cte->cycle_clause->cycle_path_column)); } From af2853d1de9d193f34ac724835feb19eeb977327 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Mon, 16 Aug 2021 15:31:17 +0300 Subject: [PATCH 034/104] Adds GROUP BY DISTINCT to ruleutils_14.c Relevant PG commit: be45be9c33a85e72cdaeb9967e9f6d2d00199e09 --- src/backend/distributed/deparser/ruleutils_14.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/backend/distributed/deparser/ruleutils_14.c b/src/backend/distributed/deparser/ruleutils_14.c index 5e85ffe7b..c02801f7a 100644 --- a/src/backend/distributed/deparser/ruleutils_14.c +++ b/src/backend/distributed/deparser/ruleutils_14.c @@ -2358,6 +2358,8 @@ get_basic_select_query(Query *query, deparse_context *context, appendContextKeyword(context, " GROUP BY ", -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + if (query->groupDistinct) + appendStringInfoString(buf, "DISTINCT "); save_exprkind = context->special_exprkind; context->special_exprkind = EXPR_KIND_GROUP_BY; From c3f05286075e784f60c9a0ff8da973b0a2f3a13c Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Mon, 16 Aug 2021 16:16:44 +0300 Subject: [PATCH 035/104] Extends statistics on expressions in ruleutils_14.c Relevant PG commit: a4d75c86bf15220df22de0a92c819ecef9db3849 --- src/backend/distributed/commands/statistics.c | 2 +- .../distributed/deparser/ruleutils_14.c | 257 +++++++++++++----- src/include/distributed/citus_ruleutils.h | 3 +- 3 files changed, 190 insertions(+), 72 deletions(-) diff --git a/src/backend/distributed/commands/statistics.c b/src/backend/distributed/commands/statistics.c index fb5406786..371b6c098 100644 --- a/src/backend/distributed/commands/statistics.c +++ b/src/backend/distributed/commands/statistics.c @@ -443,7 +443,7 @@ GetExplicitStatisticsCommandList(Oid relationId) { /* we need create commands for already created stats before distribution */ char *createStatisticsCommand = pg_get_statisticsobj_worker(statisticsId, - false); + false, false); explicitStatisticsCommandList = lappend(explicitStatisticsCommandList, diff --git a/src/backend/distributed/deparser/ruleutils_14.c b/src/backend/distributed/deparser/ruleutils_14.c index c02801f7a..bdfe06275 100644 --- a/src/backend/distributed/deparser/ruleutils_14.c +++ b/src/backend/distributed/deparser/ruleutils_14.c @@ -318,6 +318,9 @@ typedef struct * as a parameter, and append their text output to its contents. * ---------- */ +static char *deparse_expression_pretty(Node *expr, List *dpcontext, + bool forceprefix, bool showimplicit, + int prettyFlags, int startIndent); static void set_rtable_names(deparse_namespace *dpns, List *parent_namespaces, Bitmapset *rels_used); static void set_deparse_for_query(deparse_namespace *dpns, Query *query, @@ -433,7 +436,8 @@ static void get_from_clause_coldeflist(RangeTblFunction *rtfunc, deparse_context *context); static void get_tablesample_def(TableSampleClause *tablesample, deparse_context *context); -char *pg_get_statisticsobj_worker(Oid statextid, bool missing_ok); +char *pg_get_statisticsobj_worker(Oid statextid, bool columns_only, + bool missing_ok); static char *pg_get_triggerdef_worker(Oid trigid, bool pretty); static void set_simple_column_names(deparse_namespace *dpns); static void get_opclass_name(Oid opclass, Oid actual_datatype, @@ -507,6 +511,51 @@ pg_get_rule_expr(Node *expression) } +/* ---------- + * deparse_expression_pretty - General utility for deparsing expressions + * + * expr is the node tree to be deparsed. It must be a transformed expression + * tree (ie, not the raw output of gram.y). + * + * dpcontext is a list of deparse_namespace nodes representing the context + * for interpreting Vars in the node tree. It can be NIL if no Vars are + * expected. + * + * forceprefix is true to force all Vars to be prefixed with their table names. + * + * showimplicit is true to force all implicit casts to be shown explicitly. + * + * Tries to pretty up the output according to prettyFlags and startIndent. + * + * The result is a palloc'd string. + * ---------- + */ +static char * +deparse_expression_pretty(Node *expr, List *dpcontext, + bool forceprefix, bool showimplicit, + int prettyFlags, int startIndent) +{ + StringInfoData buf; + deparse_context context; + + initStringInfo(&buf); + context.buf = &buf; + context.namespaces = dpcontext; + context.windowClause = NIL; + context.windowTList = NIL; + context.varprefix = forceprefix; + context.prettyFlags = prettyFlags; + context.wrapColumn = WRAP_COLUMN_DEFAULT; + context.indentLevel = startIndent; + context.special_exprkind = EXPR_KIND_NONE; + context.appendparents = NULL; + + get_rule_expr(expr, &context, showimplicit); + + return buf.data; +} + + /* * set_rtable_names: select RTE aliases to be used in printing a query * @@ -7837,12 +7886,24 @@ pg_get_triggerdef_command(Oid triggerId) char * -pg_get_statisticsobj_worker(Oid statextid, bool missing_ok) +pg_get_statisticsobj_worker(Oid statextid, bool columns_only, bool missing_ok) { StringInfoData buf; int colno; + char *nsp; + ArrayType *arr; + char *enabled; + Datum datum; bool isnull; + bool ndistinct_enabled; + bool dependencies_enabled; + bool mcv_enabled; int i; + List *context; + ListCell *lc; + List *exprs = NIL; + bool has_exprs; + int ncolumns; HeapTuple statexttup = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(statextid)); @@ -7855,85 +7916,116 @@ pg_get_statisticsobj_worker(Oid statextid, bool missing_ok) elog(ERROR, "cache lookup failed for statistics object %u", statextid); } + /* has the statistics expressions? */ + has_exprs = !heap_attisnull(statexttup, Anum_pg_statistic_ext_stxexprs, NULL); + Form_pg_statistic_ext statextrec = (Form_pg_statistic_ext) GETSTRUCT(statexttup); + /* + * Get the statistics expressions, if any. (NOTE: we do not use the + * relcache versions of the expressions, because we want to display + * non-const-folded expressions.) + */ + if (has_exprs) + { + Datum exprsDatum; + bool isnull; + char *exprsString; + + exprsDatum = SysCacheGetAttr(STATEXTOID, statexttup, + Anum_pg_statistic_ext_stxexprs, &isnull); + Assert(!isnull); + exprsString = TextDatumGetCString(exprsDatum); + exprs = (List *) stringToNode(exprsString); + pfree(exprsString); + } + else + { + exprs = NIL; + } + + /* count the number of columns (attributes and expressions) */ + ncolumns = statextrec->stxkeys.dim1 + list_length(exprs); + initStringInfo(&buf); - char *nsp = get_namespace_name(statextrec->stxnamespace); - appendStringInfo(&buf, "CREATE STATISTICS %s", - quote_qualified_identifier(nsp, - NameStr(statextrec->stxname))); - - /* - * Decode the stxkind column so that we know which stats types to print. - */ - Datum datum = SysCacheGetAttr(STATEXTOID, statexttup, - Anum_pg_statistic_ext_stxkind, &isnull); - Assert(!isnull); - ArrayType *arr = DatumGetArrayTypeP(datum); - if (ARR_NDIM(arr) != 1 || - ARR_HASNULL(arr) || - ARR_ELEMTYPE(arr) != CHAROID) + if (!columns_only) { - elog(ERROR, "stxkind is not a 1-D char array"); - } - char *enabled = (char *) ARR_DATA_PTR(arr); + nsp = get_namespace_name(statextrec->stxnamespace); + appendStringInfo(&buf, "CREATE STATISTICS %s", + quote_qualified_identifier(nsp, + NameStr(statextrec->stxname))); - bool ndistinct_enabled = false; - bool dependencies_enabled = false; - bool mcv_enabled = false; + /* + * Decode the stxkind column so that we know which stats types to + * print. + */ + datum = SysCacheGetAttr(STATEXTOID, statexttup, + Anum_pg_statistic_ext_stxkind, &isnull); + Assert(!isnull); + arr = DatumGetArrayTypeP(datum); + if (ARR_NDIM(arr) != 1 || + ARR_HASNULL(arr) || + ARR_ELEMTYPE(arr) != CHAROID) + elog(ERROR, "stxkind is not a 1-D char array"); + enabled = (char *) ARR_DATA_PTR(arr); - for (i = 0; i < ARR_DIMS(arr)[0]; i++) - { - if (enabled[i] == STATS_EXT_NDISTINCT) + ndistinct_enabled = false; + dependencies_enabled = false; + mcv_enabled = false; + + for (i = 0; i < ARR_DIMS(arr)[0]; i++) { - ndistinct_enabled = true; + if (enabled[i] == STATS_EXT_NDISTINCT) + ndistinct_enabled = true; + else if (enabled[i] == STATS_EXT_DEPENDENCIES) + dependencies_enabled = true; + else if (enabled[i] == STATS_EXT_MCV) + mcv_enabled = true; + + /* ignore STATS_EXT_EXPRESSIONS (it's built automatically) */ } - if (enabled[i] == STATS_EXT_DEPENDENCIES) + + /* + * If any option is disabled, then we'll need to append the types + * clause to show which options are enabled. We omit the types clause + * on purpose when all options are enabled, so a pg_dump/pg_restore + * will create all statistics types on a newer postgres version, if + * the statistics had all options enabled on the original version. + * + * But if the statistics is defined on just a single column, it has to + * be an expression statistics. In that case we don't need to specify + * kinds. + */ + if ((!ndistinct_enabled || !dependencies_enabled || !mcv_enabled) && + (ncolumns > 1)) { - dependencies_enabled = true; - } - if (enabled[i] == STATS_EXT_MCV) - { - mcv_enabled = true; + bool gotone = false; + + appendStringInfoString(&buf, " ("); + + if (ndistinct_enabled) + { + appendStringInfoString(&buf, "ndistinct"); + gotone = true; + } + + if (dependencies_enabled) + { + appendStringInfo(&buf, "%sdependencies", gotone ? ", " : ""); + gotone = true; + } + + if (mcv_enabled) + appendStringInfo(&buf, "%smcv", gotone ? ", " : ""); + + appendStringInfoChar(&buf, ')'); } + + appendStringInfoString(&buf, " ON "); } - /* - * If any option is disabled, then we'll need to append the types clause - * to show which options are enabled. We omit the types clause on purpose - * when all options are enabled, so a pg_dump/pg_restore will create all - * statistics types on a newer postgres version, if the statistics had all - * options enabled on the original version. - */ - if (!ndistinct_enabled || !dependencies_enabled || !mcv_enabled) - { - bool gotone = false; - - appendStringInfoString(&buf, " ("); - - if (ndistinct_enabled) - { - appendStringInfoString(&buf, "ndistinct"); - gotone = true; - } - - if (dependencies_enabled) - { - appendStringInfo(&buf, "%sdependencies", gotone ? ", " : ""); - gotone = true; - } - - if (mcv_enabled) - { - appendStringInfo(&buf, "%smcv", gotone ? ", " : ""); - } - - appendStringInfoChar(&buf, ')'); - } - - appendStringInfoString(&buf, " ON "); - + /* decode simple column references */ for (colno = 0; colno < statextrec->stxkeys.dim1; colno++) { AttrNumber attnum = statextrec->stxkeys.values[colno]; @@ -7948,8 +8040,33 @@ pg_get_statisticsobj_worker(Oid statextid, bool missing_ok) appendStringInfoString(&buf, quote_identifier(attname)); } - appendStringInfo(&buf, " FROM %s", - generate_relation_name(statextrec->stxrelid, NIL)); + context = deparse_context_for(get_relation_name(statextrec->stxrelid), + statextrec->stxrelid); + + foreach(lc, exprs) + { + Node *expr = (Node *) lfirst(lc); + char *str; + int prettyFlags = PRETTYFLAG_INDENT; + + str = deparse_expression_pretty(expr, context, false, false, + prettyFlags, 0); + + if (colno > 0) + appendStringInfoString(&buf, ", "); + + /* Need parens if it's not a bare function call */ + if (looks_like_function(expr)) + appendStringInfoString(&buf, str); + else + appendStringInfo(&buf, "(%s)", str); + + colno++; + } + + if (!columns_only) + appendStringInfo(&buf, " FROM %s", + generate_relation_name(statextrec->stxrelid, NIL)); ReleaseSysCache(statexttup); diff --git a/src/include/distributed/citus_ruleutils.h b/src/include/distributed/citus_ruleutils.h index 008838af4..2bb008c5e 100644 --- a/src/include/distributed/citus_ruleutils.h +++ b/src/include/distributed/citus_ruleutils.h @@ -50,7 +50,8 @@ char * pg_get_rule_expr(Node *expression); extern void deparse_shard_query(Query *query, Oid distrelid, int64 shardid, StringInfo buffer); extern char * pg_get_triggerdef_command(Oid triggerId); -extern char * pg_get_statisticsobj_worker(Oid statextid, bool missing_ok); +extern char * pg_get_statisticsobj_worker(Oid statextid, bool columns_only, + bool missing_ok); extern char * generate_relation_name(Oid relid, List *namespaces); extern char * generate_qualified_relation_name(Oid relid); extern char * generate_operator_name(Oid operid, Oid arg1, Oid arg2); From f557bae64cc96e2c40e0a1bd6377390ac4dab2d6 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Mon, 16 Aug 2021 16:19:34 +0300 Subject: [PATCH 036/104] Adds JOIN ... USING alias to ruleutils_14.c Relevant PG commit: 055fee7eb4dcc78e58672aef146334275e1cc40d --- src/backend/distributed/deparser/ruleutils_14.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/backend/distributed/deparser/ruleutils_14.c b/src/backend/distributed/deparser/ruleutils_14.c index bdfe06275..1c1c7576b 100644 --- a/src/backend/distributed/deparser/ruleutils_14.c +++ b/src/backend/distributed/deparser/ruleutils_14.c @@ -7703,6 +7703,10 @@ get_from_clause_item(Node *jtnode, Query *query, deparse_context *context) appendStringInfoString(buf, quote_identifier(colname)); } appendStringInfoChar(buf, ')'); + + if (j->join_using_alias) + appendStringInfo(buf, " AS %s", + quote_identifier(j->join_using_alias->aliasname)); } else if (j->quals) { From 131062d6b57064ddc6b36b3789e42fcfa00b6314 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Mon, 16 Aug 2021 16:23:19 +0300 Subject: [PATCH 037/104] Removes ModifyTable check from set_deparse_plan in ruleutils_14.c Relevant PG commit: 86dc90056dfdbd9d1b891718d2e5614e3e432f35 --- src/backend/distributed/deparser/ruleutils_14.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/backend/distributed/deparser/ruleutils_14.c b/src/backend/distributed/deparser/ruleutils_14.c index 1c1c7576b..75b4b9115 100644 --- a/src/backend/distributed/deparser/ruleutils_14.c +++ b/src/backend/distributed/deparser/ruleutils_14.c @@ -1674,16 +1674,12 @@ set_deparse_plan(deparse_namespace *dpns, Plan *plan) * We special-case Append and MergeAppend to pretend that the first child * plan is the OUTER referent; we have to interpret OUTER Vars in their * tlists according to one of the children, and the first one is the most - * natural choice. Likewise special-case ModifyTable to pretend that the - * first child plan is the OUTER referent; this is to support RETURNING - * lists containing references to non-target relations. + * natural choice. */ if (IsA(plan, Append)) dpns->outer_plan = linitial(((Append *) plan)->appendplans); else if (IsA(plan, MergeAppend)) dpns->outer_plan = linitial(((MergeAppend *) plan)->mergeplans); - else if (IsA(plan, ModifyTable)) - dpns->outer_plan = linitial(((ModifyTable *) plan)->plans); else dpns->outer_plan = outerPlan(plan); From 84f0be56c36f5b2a4359cd22402083dcab1f4ce6 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Mon, 16 Aug 2021 16:28:30 +0300 Subject: [PATCH 038/104] Adds EXTRACT cases to get_func_sql_syntax in ruleutils_14.c Relevant PG commit: a2da77cdb4661826482ebf2ddba1f953bc74afe4 --- .../distributed/deparser/ruleutils_14.c | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/backend/distributed/deparser/ruleutils_14.c b/src/backend/distributed/deparser/ruleutils_14.c index 75b4b9115..6354bf8a8 100644 --- a/src/backend/distributed/deparser/ruleutils_14.c +++ b/src/backend/distributed/deparser/ruleutils_14.c @@ -6641,6 +6641,27 @@ get_func_sql_syntax(FuncExpr *expr, deparse_context *context) appendStringInfoString(buf, "))"); return true; + case F_EXTRACT_TEXT_DATE: + case F_EXTRACT_TEXT_TIME: + case F_EXTRACT_TEXT_TIMETZ: + case F_EXTRACT_TEXT_TIMESTAMP: + case F_EXTRACT_TEXT_TIMESTAMPTZ: + case F_EXTRACT_TEXT_INTERVAL: + /* EXTRACT (x FROM y) */ + appendStringInfoString(buf, "EXTRACT("); + { + Const *con = (Const *) linitial(expr->args); + + Assert(IsA(con, Const) && + con->consttype == TEXTOID && + !con->constisnull); + appendStringInfoString(buf, TextDatumGetCString(con->constvalue)); + } + appendStringInfoString(buf, " FROM "); + get_rule_expr((Node *) lsecond(expr->args), context, false); + appendStringInfoChar(buf, ')'); + return true; + case F_IS_NORMALIZED: /* IS xxx NORMALIZED */ appendStringInfoString(buf, "(("); From 2990cfb6c900ce81472b4b8c54bf5769aa5f88c5 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Mon, 16 Aug 2021 17:10:29 +0300 Subject: [PATCH 039/104] Adds SQL-standard function body support to ruleutils_14.c Relevant PG commit: e717a9a18b2e34c9c40e5259ad4d31cd7e420750 --- .../distributed/deparser/ruleutils_14.c | 53 ++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/src/backend/distributed/deparser/ruleutils_14.c b/src/backend/distributed/deparser/ruleutils_14.c index 6354bf8a8..68669b21a 100644 --- a/src/backend/distributed/deparser/ruleutils_14.c +++ b/src/backend/distributed/deparser/ruleutils_14.c @@ -182,6 +182,10 @@ typedef struct List *outer_tlist; /* referent for OUTER_VAR Vars */ List *inner_tlist; /* referent for INNER_VAR Vars */ List *index_tlist; /* referent for INDEX_VAR Vars */ + /* Special namespace representing a function signature: */ + char *funcname; + int numargs; + char **argnames; } deparse_namespace; /* Callback signature for resolve_special_varno() */ @@ -2358,7 +2362,10 @@ get_basic_select_query(Query *query, deparse_context *context, /* * Build up the query string - first we say SELECT */ - appendStringInfoString(buf, "SELECT"); + if (query->isReturn) + appendStringInfoString(buf, "RETURN"); + else + appendStringInfoString(buf, "SELECT"); /* Add the DISTINCT clause if given */ if (query->distinctClause != NIL) @@ -4594,6 +4601,50 @@ get_parameter(Param *param, deparse_context *context) return; } + /* + * If it's an external parameter, see if the outermost namespace provides + * function argument names. + */ + if (param->paramkind == PARAM_EXTERN) + { + dpns = lfirst(list_tail(context->namespaces)); + if (dpns->argnames) + { + char *argname = dpns->argnames[param->paramid - 1]; + + if (argname) + { + bool should_qualify = false; + ListCell *lc; + + /* + * Qualify the parameter name if there are any other deparse + * namespaces with range tables. This avoids qualifying in + * trivial cases like "RETURN a + b", but makes it safe in all + * other cases. + */ + foreach(lc, context->namespaces) + { + deparse_namespace *dpns = lfirst(lc); + + if (list_length(dpns->rtable_names) > 0) + { + should_qualify = true; + break; + } + } + if (should_qualify) + { + appendStringInfoString(context->buf, quote_identifier(dpns->funcname)); + appendStringInfoChar(context->buf, '.'); + } + + appendStringInfoString(context->buf, quote_identifier(argname)); + return; + } + } + } + /* * Not PARAM_EXEC, or couldn't find referent: for base types just print $N. * For composite types, add cast to the parameter to ease remote node detect From 3b7bcf755525ddef83efa649347997d9221e6188 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Mon, 16 Aug 2021 17:43:55 +0300 Subject: [PATCH 040/104] Adds missing include_out_argument parameter to func_get_detail in ruleutils_14.c Relevant PG commit: e56bce5d43789cce95d099554ae9593ada92b3b7 --- src/backend/distributed/deparser/ruleutils_14.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/distributed/deparser/ruleutils_14.c b/src/backend/distributed/deparser/ruleutils_14.c index 68669b21a..ec24e7001 100644 --- a/src/backend/distributed/deparser/ruleutils_14.c +++ b/src/backend/distributed/deparser/ruleutils_14.c @@ -8844,7 +8844,7 @@ generate_function_name(Oid funcid, int nargs, List *argnames, Oid *argtypes, if (!force_qualify) p_result = func_get_detail(list_make1(makeString(proname)), NIL, argnames, nargs, argtypes, - !use_variadic, true, + !use_variadic, true, false, &p_funcid, &p_rettype, &p_retset, &p_nvargs, &p_vatype, &p_true_typeids, NULL); From 86d9260781097432b897388782c8b5cb260c6251 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Mon, 16 Aug 2021 18:05:03 +0300 Subject: [PATCH 041/104] Uses lfirst_node in ruleutils_14.c Relevant PG commit: 2b00db4fb0c7f02f000276bfadaab65a14059168 --- src/backend/distributed/deparser/ruleutils_14.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/backend/distributed/deparser/ruleutils_14.c b/src/backend/distributed/deparser/ruleutils_14.c index ec24e7001..9dc69f164 100644 --- a/src/backend/distributed/deparser/ruleutils_14.c +++ b/src/backend/distributed/deparser/ruleutils_14.c @@ -4521,7 +4521,7 @@ find_param_referent(Param *param, deparse_context *context, */ foreach(lc2, ((Plan *) ancestor)->initPlan) { - SubPlan *subplan = castNode(SubPlan, lfirst(lc2)); + SubPlan *subplan = lfirst_node(SubPlan, lc2); if (child_plan != (Plan *) list_nth(dpns->subplans, subplan->plan_id - 1)) @@ -6168,7 +6168,7 @@ get_rule_expr(Node *node, deparse_context *context, sep = ""; foreach(cell, spec->listdatums) { - Const *val = castNode(Const, lfirst(cell)); + Const *val = lfirst_node(Const, cell); appendStringInfoString(buf, sep); get_const_expr(val, context, -1); @@ -8933,7 +8933,7 @@ get_range_partbound_string(List *bound_datums) foreach(cell, bound_datums) { PartitionRangeDatum *datum = - castNode(PartitionRangeDatum, lfirst(cell)); + lfirst_node(PartitionRangeDatum, cell); appendStringInfoString(buf, sep); if (datum->kind == PARTITION_RANGE_DATUM_MINVALUE) From b01e7e884c9ab9bc8cf5520e76c6b2bf531fc8b2 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Tue, 17 Aug 2021 13:18:46 +0300 Subject: [PATCH 042/104] Pass NULL for plannerInfo as we don't generate PlaceHolderVars --- .../distributed/operations/delete_protocol.c | 16 +++++++++++++--- .../distributed/planner/insert_select_planner.c | 11 ++++++++++- .../planner/relation_restriction_equivalence.c | 2 +- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/src/backend/distributed/operations/delete_protocol.c b/src/backend/distributed/operations/delete_protocol.c index 24763e73b..b5254b545 100644 --- a/src/backend/distributed/operations/delete_protocol.c +++ b/src/backend/distributed/operations/delete_protocol.c @@ -688,6 +688,8 @@ ShardsMatchingDeleteCriteria(Oid relationId, List *shardIntervalList, Assert(deleteCriteria != NULL); List *deleteCriteriaList = list_make1(deleteCriteria); + + /* walk over shard list and check if shards can be dropped */ ShardInterval *shardInterval = NULL; foreach_ptr(shardInterval, shardIntervalList) @@ -703,9 +705,17 @@ ShardsMatchingDeleteCriteria(Oid relationId, List *shardIntervalList, Expr *lessThanExpr = (Expr *) linitial(andExpr->args); Expr *greaterThanExpr = (Expr *) lsecond(andExpr->args); - RestrictInfo *lessThanRestrictInfo = make_simple_restrictinfo(lessThanExpr); - RestrictInfo *greaterThanRestrictInfo = make_simple_restrictinfo( - greaterThanExpr); + /* + * passing NULL for plannerInfo will be problematic if we have placeholder + * vars. However, it won't be the case here because we are building + * the expression from shard intervals which don't have placeholder vars. + * Note that this is only the case with PG14 as the parameter doesn't exist + * prior to that. + */ + RestrictInfo *lessThanRestrictInfo = make_simple_restrictinfo_compat(NULL, + lessThanExpr); + RestrictInfo *greaterThanRestrictInfo = make_simple_restrictinfo_compat(NULL, + greaterThanExpr); restrictInfoList = lappend(restrictInfoList, lessThanRestrictInfo); restrictInfoList = lappend(restrictInfoList, greaterThanRestrictInfo); diff --git a/src/backend/distributed/planner/insert_select_planner.c b/src/backend/distributed/planner/insert_select_planner.c index e7260946f..55559ce58 100644 --- a/src/backend/distributed/planner/insert_select_planner.c +++ b/src/backend/distributed/planner/insert_select_planner.c @@ -747,7 +747,16 @@ RouterModifyTaskForShardInterval(Query *originalQuery, continue; } - shardRestrictionList = make_simple_restrictinfo((Expr *) shardOpExpressions); + + /* + * passing NULL for plannerInfo will be problematic if we have placeholder + * vars. However, it won't be the case here because we are building + * the expression from shard intervals which don't have placeholder vars. + * Note that this is only the case with PG14 as the parameter doesn't exist + * prior to that. + */ + shardRestrictionList = make_simple_restrictinfo_compat(NULL, + (Expr *) shardOpExpressions); extendedBaseRestrictInfo = lappend(extendedBaseRestrictInfo, shardRestrictionList); diff --git a/src/backend/distributed/planner/relation_restriction_equivalence.c b/src/backend/distributed/planner/relation_restriction_equivalence.c index 5cbd2e53d..1d13102fb 100644 --- a/src/backend/distributed/planner/relation_restriction_equivalence.c +++ b/src/backend/distributed/planner/relation_restriction_equivalence.c @@ -2054,7 +2054,7 @@ GetRestrictInfoListForRelation(RangeTblEntry *rangeTblEntry, * If the restriction involves multiple tables, we cannot add it to * input relation's expression list. */ - Relids varnos = pull_varnos((Node *) restrictionClause); + Relids varnos = pull_varnos_compat(relationRestriction->plannerInfo, (Node *) restrictionClause); if (bms_num_members(varnos) != 1) { continue; From 287706b7177538d4fd2078696b7323805ac4973a Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Tue, 17 Aug 2021 14:04:50 +0300 Subject: [PATCH 043/104] Introduces SetTuplestoreDestReceiverParams_compat macro SetTuplestoreDestReceiverParams function now has two new parameters This new macro give us the ability to use this new parameter for PG14 and it doesn't give the parameter for previous versions Existing parameters are set to NULL to keep previous behavior Relevant PG commit: 2f48ede080f42b97b594fb14102c82ca1001b80c --- src/backend/distributed/planner/multi_explain.c | 4 ++-- src/include/distributed/version_compat.h | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/backend/distributed/planner/multi_explain.c b/src/backend/distributed/planner/multi_explain.c index df7db64a0..084700fa0 100644 --- a/src/backend/distributed/planner/multi_explain.c +++ b/src/backend/distributed/planner/multi_explain.c @@ -1024,8 +1024,8 @@ worker_save_query_explain_analyze(PG_FUNCTION_ARGS) TupleDesc tupleDescriptor = NULL; Tuplestorestate *tupleStore = SetupTuplestore(fcinfo, &tupleDescriptor); DestReceiver *tupleStoreDest = CreateTuplestoreDestReceiver(); - SetTuplestoreDestReceiverParams(tupleStoreDest, tupleStore, - CurrentMemoryContext, false); + SetTuplestoreDestReceiverParams_compat(tupleStoreDest, tupleStore, + CurrentMemoryContext, false, NULL, NULL); List *parseTreeList = pg_parse_query(queryString); if (list_length(parseTreeList) != 1) diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index e5df7a264..29fa67c5f 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -55,6 +55,8 @@ #define ProcessUtility_compat(a, b, c, d, e, f, g, h) \ ProcessUtility(a, b, c, d, e, f, g, h) #define COPY_FRONTEND_COMPAT COPY_FRONTEND +#define SetTuplestoreDestReceiverParams_compat(a, b, c, d, e, f) \ + SetTuplestoreDestReceiverParams(a, b, c, d, e, f) #else #define AlterTableStmtObjType(a) ((a)->relkind) #define F_NEXTVAL_COMPAT F_NEXTVAL_OID @@ -82,6 +84,8 @@ standard_ProcessUtility(a, b, d, e, f, g, h) #define ProcessUtility_compat(a, b, c, d, e, f, g, h) ProcessUtility(a, b, d, e, f, g, h) #define COPY_FRONTEND_COMPAT COPY_NEW_FE +#define SetTuplestoreDestReceiverParams_compat(a, b, c, d, e, f) \ + SetTuplestoreDestReceiverParams(a, b, c, d) #endif #if PG_VERSION_NUM >= PG_VERSION_13 From 898d3bb8d3ef4fa2bf538b928300070aad7814a2 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Tue, 17 Aug 2021 14:37:05 +0300 Subject: [PATCH 044/104] Introduces proc_statusflags_compat macro In two commits vacuumFlags in PGXACT is moved and then renamed to status flags This macro uses the appropriate version of the flag Relevant PG commits: 5788e258bb26495fab65ff3aa486268d1c50b123 cd9c1b3e197a9b53b840dcc87eb41b04d601a5f9 --- src/backend/distributed/transaction/lock_graph.c | 4 +--- src/include/distributed/version_compat.h | 3 +++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/backend/distributed/transaction/lock_graph.c b/src/backend/distributed/transaction/lock_graph.c index 9265bd320..30dec5d80 100644 --- a/src/backend/distributed/transaction/lock_graph.c +++ b/src/backend/distributed/transaction/lock_graph.c @@ -461,9 +461,7 @@ IsProcessWaitingForSafeOperations(PGPROC *proc) return false; } - /* get the transaction that the backend associated with */ - PGXACT *pgxact = &ProcGlobal->allPgXact[proc->pgprocno]; - if (pgxact->vacuumFlags & PROC_IS_AUTOVACUUM) + if (pgproc_statusflags_compat(proc) & PROC_IS_AUTOVACUUM) { return true; } diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index 29fa67c5f..1155488c4 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -57,6 +57,7 @@ #define COPY_FRONTEND_COMPAT COPY_FRONTEND #define SetTuplestoreDestReceiverParams_compat(a, b, c, d, e, f) \ SetTuplestoreDestReceiverParams(a, b, c, d, e, f) +#define pgproc_statusflags_compat(pgproc) ((pgproc)->statusFlags) #else #define AlterTableStmtObjType(a) ((a)->relkind) #define F_NEXTVAL_COMPAT F_NEXTVAL_OID @@ -86,6 +87,8 @@ #define COPY_FRONTEND_COMPAT COPY_NEW_FE #define SetTuplestoreDestReceiverParams_compat(a, b, c, d, e, f) \ SetTuplestoreDestReceiverParams(a, b, c, d) +#define pgproc_statusflags_compat(pgproc) \ + ((&ProcGlobal->allPgXact[(pgproc)->pgprocno])->vacuumFlags) #endif #if PG_VERSION_NUM >= PG_VERSION_13 From cb3b76ed24b21471cc870e91d55cf595e2a2d908 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Tue, 17 Aug 2021 15:15:27 +0300 Subject: [PATCH 045/104] Introduces get_partition_parent_compat and RelationGetPartitionDesc_compat macros get_partition_parent and RelationGetPartitionDesc functions now have new parameters to also include detached partitions Thess new macros give us the ability to use these new parameter for PG14 and they don't give the parameters for previous versions Existing parameters are set to not accept detached partitions Relevant PG commit: 71f4c8c6f74ba021e55d35b1128d22fb8c6e1629 --- src/backend/distributed/utils/multi_partitioning_utils.c | 8 ++++---- src/include/distributed/version_compat.h | 4 ++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/backend/distributed/utils/multi_partitioning_utils.c b/src/backend/distributed/utils/multi_partitioning_utils.c index fbe21e68c..50d77a84a 100644 --- a/src/backend/distributed/utils/multi_partitioning_utils.c +++ b/src/backend/distributed/utils/multi_partitioning_utils.c @@ -539,7 +539,7 @@ IsParentTable(Oid relationId) Oid PartitionParentOid(Oid partitionOid) { - Oid partitionParentOid = get_partition_parent(partitionOid); + Oid partitionParentOid = get_partition_parent_compat(partitionOid, false); return partitionParentOid; } @@ -590,7 +590,7 @@ PartitionList(Oid parentRelationId) ereport(ERROR, (errmsg("\"%s\" is not a parent table", relationName))); } - PartitionDesc partDesc = RelationGetPartitionDesc(rel); + PartitionDesc partDesc = RelationGetPartitionDesc_compat(rel, true); Assert(partDesc != NULL); int partitionCount = partDesc->nparts; @@ -623,7 +623,7 @@ GenerateDetachPartitionCommand(Oid partitionTableId) ereport(ERROR, (errmsg("\"%s\" is not a partition", relationName))); } - Oid parentId = get_partition_parent(partitionTableId); + Oid parentId = get_partition_parent_compat(partitionTableId, false); char *tableQualifiedName = generate_qualified_relation_name(partitionTableId); char *parentTableQualifiedName = generate_qualified_relation_name(parentId); @@ -717,7 +717,7 @@ GenerateAlterTableAttachPartitionCommand(Oid partitionTableId) ereport(ERROR, (errmsg("\"%s\" is not a partition", relationName))); } - Oid parentId = get_partition_parent(partitionTableId); + Oid parentId = get_partition_parent_compat(partitionTableId, false); char *tableQualifiedName = generate_qualified_relation_name(partitionTableId); char *parentTableQualifiedName = generate_qualified_relation_name(parentId); diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index 1155488c4..960ef8eef 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -58,6 +58,8 @@ #define SetTuplestoreDestReceiverParams_compat(a, b, c, d, e, f) \ SetTuplestoreDestReceiverParams(a, b, c, d, e, f) #define pgproc_statusflags_compat(pgproc) ((pgproc)->statusFlags) +#define get_partition_parent_compat(a, b) get_partition_parent(a, b) +#define RelationGetPartitionDesc_compat(a, b) RelationGetPartitionDesc(a, b) #else #define AlterTableStmtObjType(a) ((a)->relkind) #define F_NEXTVAL_COMPAT F_NEXTVAL_OID @@ -89,6 +91,8 @@ SetTuplestoreDestReceiverParams(a, b, c, d) #define pgproc_statusflags_compat(pgproc) \ ((&ProcGlobal->allPgXact[(pgproc)->pgprocno])->vacuumFlags) +#define get_partition_parent_compat(a, b) get_partition_parent(a) +#define RelationGetPartitionDesc_compat(a, b) RelationGetPartitionDesc(a) #endif #if PG_VERSION_NUM >= PG_VERSION_13 From b644ac55c61b1d1d1fef6c0c51b32949987913c6 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Tue, 17 Aug 2021 17:27:12 +0300 Subject: [PATCH 046/104] Introduces GetOldestNonRemovableTransactionId_compat macro GetOldestXmin function is removed so we use GetOldestNonRemovableTransactionId functions instead GetOldestNonRemovableTransactionId_compat picks the appropriate one Relevant PG commit: dc7420c2c9274a283779ec19718d2d16323640c0 --- src/backend/columnar/columnar_tableam.c | 3 ++- src/include/columnar/columnar_version_compat.h | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/backend/columnar/columnar_tableam.c b/src/backend/columnar/columnar_tableam.c index 7577deb05..783e674a1 100644 --- a/src/backend/columnar/columnar_tableam.c +++ b/src/backend/columnar/columnar_tableam.c @@ -1306,7 +1306,8 @@ columnar_index_build_range_scan(Relation columnarRelation, if (!IsBootstrapProcessingMode() && !indexInfo->ii_Concurrent) { /* ignore lazy VACUUM's */ - OldestXmin = GetOldestXmin(columnarRelation, PROCARRAY_FLAGS_VACUUM); + OldestXmin = GetOldestNonRemovableTransactionId_compat(columnarRelation, + PROCARRAY_FLAGS_VACUUM); } Snapshot snapshot = { 0 }; diff --git a/src/include/columnar/columnar_version_compat.h b/src/include/columnar/columnar_version_compat.h index 36b7f5068..8e301f3ac 100644 --- a/src/include/columnar/columnar_version_compat.h +++ b/src/include/columnar/columnar_version_compat.h @@ -17,11 +17,14 @@ ColumnarProcessUtility(a, b, c, d, e, f, g, h) #define PrevProcessUtilityHook_compat(a, b, c, d, e, f, g, h) \ PrevProcessUtilityHook(a, b, c, d, e, f, g, h) +#define GetOldestNonRemovableTransactionId_compat(a, b) \ + GetOldestNonRemovableTransactionId(a) #else #define ColumnarProcessUtility_compat(a, b, c, d, e, f, g, h) \ ColumnarProcessUtility(a, b, d, e, f, g, h) #define PrevProcessUtilityHook_compat(a, b, c, d, e, f, g, h) \ PrevProcessUtilityHook(a, b, d, e, f, g, h) +#define GetOldestNonRemovableTransactionId_compat(a, b) GetOldestXmin(a, b) #endif #define ACLCHECK_OBJECT_TABLE OBJECT_TABLE From fd2ca2825b7371804a8c811d4a163ab8546d314f Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Wed, 18 Aug 2021 15:51:22 +0300 Subject: [PATCH 047/104] Introduces ExecSimpleRelationInsert_compat and modifyStateResultRelInfo macros es_result_relation_info is removed from Estate. In this commit we make some changes to handle that. resultRelationInfo filed is added to ModifyState to support the removed field. Relevant PG commits: 1375422c7826a2bf387be29895e961614f69de4b a04daa97a4339c38e304cd6164d37da540d665a8 --- src/backend/columnar/columnar_metadata.c | 29 ++++++++++++------- .../columnar/columnar_version_compat.h | 4 +++ 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/src/backend/columnar/columnar_metadata.c b/src/backend/columnar/columnar_metadata.c index 936fe39a5..3f622a44d 100644 --- a/src/backend/columnar/columnar_metadata.c +++ b/src/backend/columnar/columnar_metadata.c @@ -64,6 +64,7 @@ typedef struct { Relation rel; EState *estate; + ResultRelInfo *resultRelInfo; } ModifyState; /* RowNumberLookupMode to be used in StripeMetadataLookupRowNumber */ @@ -1314,12 +1315,20 @@ StartModifyRelation(Relation rel) { EState *estate = create_estate_for_relation(rel); +#if PG_VERSION_NUM >= PG_VERSION_14 + ResultRelInfo *resultRelInfo = makeNode(ResultRelInfo); + InitResultRelInfo(resultRelInfo, rel, 1, NULL, 0); +#else + ResultRelInfo *resultRelInfo = estate->es_result_relation_info; +#endif + /* ExecSimpleRelationInsert, ... require caller to open indexes */ - ExecOpenIndices(estate->es_result_relation_info, false); + ExecOpenIndices(resultRelInfo, false); ModifyState *modifyState = palloc(sizeof(ModifyState)); modifyState->rel = rel; modifyState->estate = estate; + modifyState->resultRelInfo = resultRelInfo; return modifyState; } @@ -1341,7 +1350,7 @@ InsertTupleAndEnforceConstraints(ModifyState *state, Datum *values, bool *nulls) ExecStoreHeapTuple(tuple, slot, false); /* use ExecSimpleRelationInsert to enforce constraints */ - ExecSimpleRelationInsert(state->estate, slot); + ExecSimpleRelationInsert_compat(state->resultRelInfo, state->estate, slot); } @@ -1353,7 +1362,7 @@ static void DeleteTupleAndEnforceConstraints(ModifyState *state, HeapTuple heapTuple) { EState *estate = state->estate; - ResultRelInfo *resultRelInfo = estate->es_result_relation_info; + ResultRelInfo *resultRelInfo = state->resultRelInfo; ItemPointer tid = &(heapTuple->t_self); simple_heap_delete(state->rel, tid); @@ -1369,10 +1378,15 @@ DeleteTupleAndEnforceConstraints(ModifyState *state, HeapTuple heapTuple) static void FinishModifyRelation(ModifyState *state) { - ExecCloseIndices(state->estate->es_result_relation_info); + ExecCloseIndices(state->resultRelInfo); AfterTriggerEndQuery(state->estate); +#if PG_VERSION_NUM >= PG_VERSION_14 + ExecCloseResultRelations(state->estate); + ExecCloseRangeTableRelations(state->estate); +#else ExecCleanUpTriggerState(state->estate); +#endif ExecResetTupleTable(state->estate->es_tupleTable, false); FreeExecutorState(state->estate); @@ -1401,13 +1415,6 @@ create_estate_for_relation(Relation rel) rte->rellockmode = AccessShareLock; ExecInitRangeTable(estate, list_make1(rte)); - ResultRelInfo *resultRelInfo = makeNode(ResultRelInfo); - InitResultRelInfo(resultRelInfo, rel, 1, NULL, 0); - - estate->es_result_relations = resultRelInfo; - estate->es_num_result_relations = 1; - estate->es_result_relation_info = resultRelInfo; - estate->es_output_cid = GetCurrentCommandId(true); /* Prepare to catch AFTER triggers. */ diff --git a/src/include/columnar/columnar_version_compat.h b/src/include/columnar/columnar_version_compat.h index 8e301f3ac..e418561d0 100644 --- a/src/include/columnar/columnar_version_compat.h +++ b/src/include/columnar/columnar_version_compat.h @@ -19,12 +19,16 @@ PrevProcessUtilityHook(a, b, c, d, e, f, g, h) #define GetOldestNonRemovableTransactionId_compat(a, b) \ GetOldestNonRemovableTransactionId(a) +#define ExecSimpleRelationInsert_compat(a, b, c) \ + ExecSimpleRelationInsert(a, b, c) #else #define ColumnarProcessUtility_compat(a, b, c, d, e, f, g, h) \ ColumnarProcessUtility(a, b, d, e, f, g, h) #define PrevProcessUtilityHook_compat(a, b, c, d, e, f, g, h) \ PrevProcessUtilityHook(a, b, d, e, f, g, h) #define GetOldestNonRemovableTransactionId_compat(a, b) GetOldestXmin(a, b) +#define ExecSimpleRelationInsert_compat(a, b, c) \ + ExecSimpleRelationInsert(b, c) #endif #define ACLCHECK_OBJECT_TABLE OBJECT_TABLE From b21a00e7750905c2324583a9f637395848b7ff97 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Wed, 18 Aug 2021 16:28:45 +0300 Subject: [PATCH 048/104] Introduces index_insert_compat macro index_insert function now has a new parameter, indexUnchanged This new macro give us the ability to use these new parameter for PG14 and they don't give the parameters for previous versions Existing parameter is set to false Relevant PG commit: 9dc718bdf2b1a574481a45624d42b674332e2903 --- src/backend/columnar/columnar_tableam.c | 4 ++-- src/include/columnar/columnar_version_compat.h | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/backend/columnar/columnar_tableam.c b/src/backend/columnar/columnar_tableam.c index 783e674a1..e50a855fe 100644 --- a/src/backend/columnar/columnar_tableam.c +++ b/src/backend/columnar/columnar_tableam.c @@ -1653,8 +1653,8 @@ ColumnarReadMissingRowsIntoIndex(TableScanDesc scan, Relation indexRelation, Relation columnarRelation = scan->rs_rd; IndexUniqueCheck indexUniqueCheck = indexInfo->ii_Unique ? UNIQUE_CHECK_YES : UNIQUE_CHECK_NO; - index_insert(indexRelation, indexValues, indexNulls, columnarItemPointer, - columnarRelation, indexUniqueCheck, indexInfo); + index_insert_compat(indexRelation, indexValues, indexNulls, columnarItemPointer, + columnarRelation, indexUniqueCheck, false, indexInfo); validateIndexState->tups_inserted += 1; } diff --git a/src/include/columnar/columnar_version_compat.h b/src/include/columnar/columnar_version_compat.h index e418561d0..6b503b73f 100644 --- a/src/include/columnar/columnar_version_compat.h +++ b/src/include/columnar/columnar_version_compat.h @@ -21,6 +21,8 @@ GetOldestNonRemovableTransactionId(a) #define ExecSimpleRelationInsert_compat(a, b, c) \ ExecSimpleRelationInsert(a, b, c) +#define index_insert_compat(a, b, c, d, e, f, g, h) \ + index_insert(a, b, c, d, e, f, g, h) #else #define ColumnarProcessUtility_compat(a, b, c, d, e, f, g, h) \ ColumnarProcessUtility(a, b, d, e, f, g, h) @@ -29,6 +31,8 @@ #define GetOldestNonRemovableTransactionId_compat(a, b) GetOldestXmin(a, b) #define ExecSimpleRelationInsert_compat(a, b, c) \ ExecSimpleRelationInsert(b, c) +#define index_insert_compat(a, b, c, d, e, f, g, h) \ + index_insert(a, b, c, d, e, f, h) #endif #define ACLCHECK_OBJECT_TABLE OBJECT_TABLE From 5930378f61e0bf5e825879a8b9fd932c6375f011 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Wed, 18 Aug 2021 16:58:43 +0300 Subject: [PATCH 049/104] Renames shadowing ruleutils_14.c variables --- src/backend/distributed/deparser/ruleutils_14.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/backend/distributed/deparser/ruleutils_14.c b/src/backend/distributed/deparser/ruleutils_14.c index 9dc69f164..7d8213155 100644 --- a/src/backend/distributed/deparser/ruleutils_14.c +++ b/src/backend/distributed/deparser/ruleutils_14.c @@ -4625,9 +4625,9 @@ get_parameter(Param *param, deparse_context *context) */ foreach(lc, context->namespaces) { - deparse_namespace *dpns = lfirst(lc); + deparse_namespace *dp_ns = lfirst(lc); - if (list_length(dpns->rtable_names) > 0) + if (list_length(dp_ns->rtable_names) > 0) { should_qualify = true; break; @@ -8001,12 +8001,12 @@ pg_get_statisticsobj_worker(Oid statextid, bool columns_only, bool missing_ok) if (has_exprs) { Datum exprsDatum; - bool isnull; + bool isNull; char *exprsString; exprsDatum = SysCacheGetAttr(STATEXTOID, statexttup, - Anum_pg_statistic_ext_stxexprs, &isnull); - Assert(!isnull); + Anum_pg_statistic_ext_stxexprs, &isNull); + Assert(!isNull); exprsString = TextDatumGetCString(exprsDatum); exprs = (List *) stringToNode(exprsString); pfree(exprsString); From 96833e2b8f044a2a0e236a3a69ca5c59dd1fd35e Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Wed, 18 Aug 2021 15:13:57 +0300 Subject: [PATCH 050/104] Use HASH_STRINGS explicitly in hash functions Postgres expects to set the HASH_STRINGS explicitly in case of the default behaivor for string hash function. Postgres Commit b3817f5f774663d55931dd4fab9c5a94a15ae7ab --- src/backend/columnar/write_state_management.c | 2 +- src/backend/distributed/utils/listutils.c | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/backend/columnar/write_state_management.c b/src/backend/columnar/write_state_management.c index 69860ad57..4d815c2a0 100644 --- a/src/backend/columnar/write_state_management.c +++ b/src/backend/columnar/write_state_management.c @@ -132,7 +132,7 @@ columnar_init_write_state(Relation relation, TupleDesc tupdesc, "Column Store Write State Management Context", ALLOCSET_DEFAULT_SIZES); HASHCTL info; - uint32 hashFlags = (HASH_ELEM | HASH_CONTEXT); + uint32 hashFlags = (HASH_ELEM | HASH_STRINGS | HASH_CONTEXT); memset(&info, 0, sizeof(info)); info.keysize = sizeof(Oid); info.entrysize = sizeof(WriteStateMapEntry); diff --git a/src/backend/distributed/utils/listutils.c b/src/backend/distributed/utils/listutils.c index c58be3e46..836a4bff6 100644 --- a/src/backend/distributed/utils/listutils.c +++ b/src/backend/distributed/utils/listutils.c @@ -135,7 +135,13 @@ ListToHashSet(List *itemList, Size keySize, bool isStringList) info.entrysize = keySize; info.hcxt = CurrentMemoryContext; - if (!isStringList) + if (isStringList) + { +#if PG_VERSION_NUM >= PG_VERSION_14 + flags |= HASH_STRINGS; +#endif + } + else { flags |= HASH_BLOBS; } From 29f5b999518b2bd1ad033b3056c5e5c651b21e55 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Wed, 18 Aug 2021 16:20:12 +0300 Subject: [PATCH 051/104] Use empty string instead of NULL for queryString Postgres doesn't accept NULL for queryStrings in explain plans anymore. Internally, there are some places in Postgres where they modified the NULLS to ""(the empty string). So we do the same on citus side. Commit on Postgres: 1111b2668d89bfcb6f502789158b1233ab4217a6 --- src/backend/distributed/planner/multi_explain.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/backend/distributed/planner/multi_explain.c b/src/backend/distributed/planner/multi_explain.c index 084700fa0..0443523e0 100644 --- a/src/backend/distributed/planner/multi_explain.c +++ b/src/backend/distributed/planner/multi_explain.c @@ -251,7 +251,13 @@ NonPushableInsertSelectExplainScan(CustomScanState *node, List *ancestors, /* explain the inner SELECT query */ IntoClause *into = NULL; ParamListInfo params = NULL; - char *queryString = NULL; + + /* + * With PG14, we need to provide a string here, + * for now we put an empty string, which is valid according to postgres. + */ + char *queryString = pstrdup(""); + ExplainOneQuery(queryCopy, 0, into, es, queryString, params, NULL); ExplainCloseGroup("Select Query", "Select Query", false, es); @@ -278,7 +284,11 @@ ExplainSubPlans(DistributedPlan *distributedPlan, ExplainState *es) PlannedStmt *plan = subPlan->plan; IntoClause *into = NULL; ParamListInfo params = NULL; - char *queryString = NULL; + /* + * With PG14, we need to provide a string here, + * for now we put an empty string, which is valid according to postgres. + */ + char *queryString = pstrdup(""); instr_time planduration; #if PG_VERSION_NUM >= PG_VERSION_13 From a1bfb4f31bc7c727cedaa654cb8202aa4e4acfe1 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Wed, 18 Aug 2021 17:02:15 +0300 Subject: [PATCH 052/104] Fix unlimited copy size variable's value --- src/backend/distributed/commands/transmit.c | 2 +- src/include/distributed/version_compat.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/backend/distributed/commands/transmit.c b/src/backend/distributed/commands/transmit.c index 8ce727017..be6f56d0d 100644 --- a/src/backend/distributed/commands/transmit.c +++ b/src/backend/distributed/commands/transmit.c @@ -245,7 +245,7 @@ static bool ReceiveCopyData(StringInfo copyData) { bool copyDone = true; - const int unlimitedSize = 0; + const int unlimitedSize = PQ_LARGE_MESSAGE_LIMIT; HOLD_CANCEL_INTERRUPTS(); pq_startmsgread(); diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index 960ef8eef..fba6cfe99 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -93,6 +93,7 @@ ((&ProcGlobal->allPgXact[(pgproc)->pgprocno])->vacuumFlags) #define get_partition_parent_compat(a, b) get_partition_parent(a) #define RelationGetPartitionDesc_compat(a, b) RelationGetPartitionDesc(a) +#define PQ_LARGE_MESSAGE_LIMIT 0 #endif #if PG_VERSION_NUM >= PG_VERSION_13 From 8ef94dc1f503d14451f65d1574973bfc3840b7c1 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Wed, 18 Aug 2021 18:17:54 +0300 Subject: [PATCH 053/104] Changes array_cat argument type from anyarray to anycompatiblearray Relevant PG commit: 9e38c2bb5093ceb0c04d6315ccd8975bd17add66 fix array_cat_agg for pg upgrades array_cat_agg now needs to take anycompatiblearray instead of anyarray because array_cat changed its type from anyarray to anycompatiblearray with pg14. To handle upgrades correctly, we drop the aggregate in citus_pg_prepare_upgrade. To be able to drop it, we first remove the dependency from pg_depend. Then we create the right aggregate in citus_finish_pg_upgrade and we also add the dependency back to pg_depend. --- .../planner/multi_logical_optimizer.c | 24 ++- .../distributed/sql/citus--10.1-1--10.2-1.sql | 2 + src/backend/distributed/sql/citus--8.0-1.sql | 14 ++ .../udfs/citus_finish_pg_upgrade/10.2-1.sql | 141 ++++++++++++++++++ .../udfs/citus_finish_pg_upgrade/latest.sql | 36 +++++ .../udfs/citus_prepare_pg_upgrade/10.2-1.sql | 74 +++++++++ .../udfs/citus_prepare_pg_upgrade/latest.sql | 13 ++ src/test/regress/bin/normalize.sed | 1 + src/test/regress/expected/multi_extension.out | 2 +- .../expected/upgrade_list_citus_objects.out | 2 +- .../expected/upgrade_list_citus_objects_0.out | 2 +- 11 files changed, 307 insertions(+), 4 deletions(-) create mode 100644 src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/10.2-1.sql create mode 100644 src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/10.2-1.sql diff --git a/src/backend/distributed/planner/multi_logical_optimizer.c b/src/backend/distributed/planner/multi_logical_optimizer.c index bd3739717..5d464f94a 100644 --- a/src/backend/distributed/planner/multi_logical_optimizer.c +++ b/src/backend/distributed/planner/multi_logical_optimizer.c @@ -1847,7 +1847,11 @@ MasterAggregateExpression(Aggref *originalAggregate, { /* array_cat_agg() takes anyarray as input */ catAggregateName = ARRAY_CAT_AGGREGATE_NAME; +#if PG_VERSION_NUM >= PG_VERSION_14 + catInputType = ANYCOMPATIBLEARRAYOID; +#else catInputType = ANYARRAYOID; +#endif } else if (aggregateType == AGGREGATE_JSONB_AGG || aggregateType == AGGREGATE_JSONB_OBJECT_AGG) @@ -1882,7 +1886,25 @@ MasterAggregateExpression(Aggref *originalAggregate, newMasterAggregate->args = list_make1(catAggArgument); newMasterAggregate->aggfilter = NULL; newMasterAggregate->aggtranstype = InvalidOid; - newMasterAggregate->aggargtypes = list_make1_oid(ANYARRAYOID); + + if (aggregateType == AGGREGATE_ARRAY_AGG) + { +#if PG_VERSION_NUM >= PG_VERSION_14 + /* + * Postgres expects the type of the array here such as INT4ARRAYOID. + * Hence we set it to workerReturnType. If we set this to + * ANYCOMPATIBLEARRAYOID then we will get the following error: + * "argument declared anycompatiblearray is not an array but type anycompatiblearray" + */ + newMasterAggregate->aggargtypes = list_make1_oid(workerReturnType); +#else + newMasterAggregate->aggargtypes = list_make1_oid(ANYARRAYOID); +#endif + } + else + { + newMasterAggregate->aggargtypes = list_make1_oid(ANYARRAYOID); + } newMasterAggregate->aggsplit = AGGSPLIT_SIMPLE; newMasterExpression = (Expr *) newMasterAggregate; diff --git a/src/backend/distributed/sql/citus--10.1-1--10.2-1.sql b/src/backend/distributed/sql/citus--10.1-1--10.2-1.sql index 7f0f8dfb0..ec5d162eb 100644 --- a/src/backend/distributed/sql/citus--10.1-1--10.2-1.sql +++ b/src/backend/distributed/sql/citus--10.1-1--10.2-1.sql @@ -28,3 +28,5 @@ CREATE FUNCTION pg_catalog.citus_drop_all_shards(logicalrelid regclass, COMMENT ON FUNCTION pg_catalog.citus_drop_all_shards(regclass, text, text, boolean) IS 'drop all shards in a relation and update metadata'; #include "udfs/citus_drop_trigger/10.2-1.sql"; +#include "udfs/citus_prepare_pg_upgrade/10.2-1.sql" +#include "udfs/citus_finish_pg_upgrade/10.2-1.sql" diff --git a/src/backend/distributed/sql/citus--8.0-1.sql b/src/backend/distributed/sql/citus--8.0-1.sql index 5647e01fd..e27c773d7 100644 --- a/src/backend/distributed/sql/citus--8.0-1.sql +++ b/src/backend/distributed/sql/citus--8.0-1.sql @@ -320,9 +320,23 @@ CREATE TRIGGER dist_shard_cache_invalidate -- Citus aggregates + +DO $proc$ +BEGIN +IF substring(current_Setting('server_version'), '\d+')::int >= 14 THEN + EXECUTE $$ +CREATE AGGREGATE array_cat_agg(anycompatiblearray) (SFUNC = array_cat, STYPE = anycompatiblearray); +COMMENT ON AGGREGATE array_cat_agg(anycompatiblearray) + IS 'concatenate input arrays into a single array'; + $$; +ELSE + EXECUTE $$ CREATE AGGREGATE array_cat_agg(anyarray) (SFUNC = array_cat, STYPE = anyarray); COMMENT ON AGGREGATE array_cat_agg(anyarray) IS 'concatenate input arrays into a single array'; + $$; +END IF; +END$proc$; GRANT SELECT ON pg_catalog.pg_dist_partition TO public; GRANT SELECT ON pg_catalog.pg_dist_shard TO public; diff --git a/src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/10.2-1.sql b/src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/10.2-1.sql new file mode 100644 index 000000000..ae3b0b900 --- /dev/null +++ b/src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/10.2-1.sql @@ -0,0 +1,141 @@ +CREATE OR REPLACE FUNCTION pg_catalog.citus_finish_pg_upgrade() + RETURNS void + LANGUAGE plpgsql + SET search_path = pg_catalog + AS $cppu$ +DECLARE + table_name regclass; + command text; + trigger_name text; +BEGIN + + + IF substring(current_Setting('server_version'), '\d+')::int >= 14 THEN + EXECUTE $cmd$ + CREATE AGGREGATE array_cat_agg(anycompatiblearray) (SFUNC = array_cat, STYPE = anycompatiblearray); + COMMENT ON AGGREGATE array_cat_agg(anycompatiblearray) + IS 'concatenate input arrays into a single array'; + $cmd$; + ELSE + EXECUTE $cmd$ + CREATE AGGREGATE array_cat_agg(anyarray) (SFUNC = array_cat, STYPE = anyarray); + COMMENT ON AGGREGATE array_cat_agg(anyarray) + IS 'concatenate input arrays into a single array'; + $cmd$; + END IF; + + /* + * Citus creates the array_cat_agg but because of a compatibility + * issue between pg13-pg14, we drop and create it during upgrade. + * And as Citus creates it, there needs to be a dependency to the + * Citus extension, so we create that dependency here. + * We are not using: + * ALTER EXENSION citus DROP/CREATE AGGREGATE array_cat_agg + * because we don't have an easy way to check if the aggregate + * exists with anyarray type or anycompatiblearray type. + */ + INSERT INTO pg_depend + SELECT + 'pg_proc'::regclass::oid as classid, + (SELECT oid FROM pg_proc WHERE proname = 'array_cat_agg') as objid, + 0 as objsubid, + 'pg_extension'::regclass::oid as refclassid, + (select oid from pg_extension where extname = 'citus') as refobjid, + 0 as refobjsubid , + 'e' as deptype; + + -- + -- restore citus catalog tables + -- + INSERT INTO pg_catalog.pg_dist_partition SELECT * FROM public.pg_dist_partition; + INSERT INTO pg_catalog.pg_dist_shard SELECT * FROM public.pg_dist_shard; + INSERT INTO pg_catalog.pg_dist_placement SELECT * FROM public.pg_dist_placement; + INSERT INTO pg_catalog.pg_dist_node_metadata SELECT * FROM public.pg_dist_node_metadata; + INSERT INTO pg_catalog.pg_dist_node SELECT * FROM public.pg_dist_node; + INSERT INTO pg_catalog.pg_dist_local_group SELECT * FROM public.pg_dist_local_group; + INSERT INTO pg_catalog.pg_dist_transaction SELECT * FROM public.pg_dist_transaction; + INSERT INTO pg_catalog.pg_dist_colocation SELECT * FROM public.pg_dist_colocation; + -- enterprise catalog tables + INSERT INTO pg_catalog.pg_dist_authinfo SELECT * FROM public.pg_dist_authinfo; + INSERT INTO pg_catalog.pg_dist_poolinfo SELECT * FROM public.pg_dist_poolinfo; + + INSERT INTO pg_catalog.pg_dist_rebalance_strategy SELECT + name, + default_strategy, + shard_cost_function::regprocedure::regproc, + node_capacity_function::regprocedure::regproc, + shard_allowed_on_node_function::regprocedure::regproc, + default_threshold, + minimum_threshold, + improvement_threshold + FROM public.pg_dist_rebalance_strategy; + + -- + -- drop backup tables + -- + DROP TABLE public.pg_dist_authinfo; + DROP TABLE public.pg_dist_colocation; + DROP TABLE public.pg_dist_local_group; + DROP TABLE public.pg_dist_node; + DROP TABLE public.pg_dist_node_metadata; + DROP TABLE public.pg_dist_partition; + DROP TABLE public.pg_dist_placement; + DROP TABLE public.pg_dist_poolinfo; + DROP TABLE public.pg_dist_shard; + DROP TABLE public.pg_dist_transaction; + DROP TABLE public.pg_dist_rebalance_strategy; + + -- + -- reset sequences + -- + PERFORM setval('pg_catalog.pg_dist_shardid_seq', (SELECT MAX(shardid)+1 AS max_shard_id FROM pg_dist_shard), false); + PERFORM setval('pg_catalog.pg_dist_placement_placementid_seq', (SELECT MAX(placementid)+1 AS max_placement_id FROM pg_dist_placement), false); + PERFORM setval('pg_catalog.pg_dist_groupid_seq', (SELECT MAX(groupid)+1 AS max_group_id FROM pg_dist_node), false); + PERFORM setval('pg_catalog.pg_dist_node_nodeid_seq', (SELECT MAX(nodeid)+1 AS max_node_id FROM pg_dist_node), false); + PERFORM setval('pg_catalog.pg_dist_colocationid_seq', (SELECT MAX(colocationid)+1 AS max_colocation_id FROM pg_dist_colocation), false); + + -- + -- register triggers + -- + FOR table_name IN SELECT logicalrelid FROM pg_catalog.pg_dist_partition + LOOP + trigger_name := 'truncate_trigger_' || table_name::oid; + command := 'create trigger ' || trigger_name || ' after truncate on ' || table_name || ' execute procedure pg_catalog.citus_truncate_trigger()'; + EXECUTE command; + command := 'update pg_trigger set tgisinternal = true where tgname = ' || quote_literal(trigger_name); + EXECUTE command; + END LOOP; + + -- + -- set dependencies + -- + INSERT INTO pg_depend + SELECT + 'pg_class'::regclass::oid as classid, + p.logicalrelid::regclass::oid as objid, + 0 as objsubid, + 'pg_extension'::regclass::oid as refclassid, + (select oid from pg_extension where extname = 'citus') as refobjid, + 0 as refobjsubid , + 'n' as deptype + FROM pg_catalog.pg_dist_partition p; + + -- restore pg_dist_object from the stable identifiers + TRUNCATE citus.pg_dist_object; + INSERT INTO citus.pg_dist_object (classid, objid, objsubid, distribution_argument_index, colocationid) + SELECT + address.classid, + address.objid, + address.objsubid, + naming.distribution_argument_index, + naming.colocationid + FROM + public.pg_dist_object naming, + pg_catalog.pg_get_object_address(naming.type, naming.object_names, naming.object_args) address; + + DROP TABLE public.pg_dist_object; +END; +$cppu$; + +COMMENT ON FUNCTION pg_catalog.citus_finish_pg_upgrade() + IS 'perform tasks to restore citus settings from a location that has been prepared before pg_upgrade'; diff --git a/src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/latest.sql b/src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/latest.sql index 5902b646f..ae3b0b900 100644 --- a/src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/latest.sql +++ b/src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/latest.sql @@ -8,6 +8,42 @@ DECLARE command text; trigger_name text; BEGIN + + + IF substring(current_Setting('server_version'), '\d+')::int >= 14 THEN + EXECUTE $cmd$ + CREATE AGGREGATE array_cat_agg(anycompatiblearray) (SFUNC = array_cat, STYPE = anycompatiblearray); + COMMENT ON AGGREGATE array_cat_agg(anycompatiblearray) + IS 'concatenate input arrays into a single array'; + $cmd$; + ELSE + EXECUTE $cmd$ + CREATE AGGREGATE array_cat_agg(anyarray) (SFUNC = array_cat, STYPE = anyarray); + COMMENT ON AGGREGATE array_cat_agg(anyarray) + IS 'concatenate input arrays into a single array'; + $cmd$; + END IF; + + /* + * Citus creates the array_cat_agg but because of a compatibility + * issue between pg13-pg14, we drop and create it during upgrade. + * And as Citus creates it, there needs to be a dependency to the + * Citus extension, so we create that dependency here. + * We are not using: + * ALTER EXENSION citus DROP/CREATE AGGREGATE array_cat_agg + * because we don't have an easy way to check if the aggregate + * exists with anyarray type or anycompatiblearray type. + */ + INSERT INTO pg_depend + SELECT + 'pg_proc'::regclass::oid as classid, + (SELECT oid FROM pg_proc WHERE proname = 'array_cat_agg') as objid, + 0 as objsubid, + 'pg_extension'::regclass::oid as refclassid, + (select oid from pg_extension where extname = 'citus') as refobjid, + 0 as refobjsubid , + 'e' as deptype; + -- -- restore citus catalog tables -- diff --git a/src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/10.2-1.sql b/src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/10.2-1.sql new file mode 100644 index 000000000..8b5e13bc0 --- /dev/null +++ b/src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/10.2-1.sql @@ -0,0 +1,74 @@ +CREATE OR REPLACE FUNCTION pg_catalog.citus_prepare_pg_upgrade() + RETURNS void + LANGUAGE plpgsql + SET search_path = pg_catalog + AS $cppu$ +BEGIN + + DELETE FROM pg_depend WHERE + objid IN (SELECT oid FROM pg_proc WHERE proname = 'array_cat_agg') AND + refobjid IN (select oid from pg_extension where extname = 'citus'); + /* + * We are dropping the aggregates because postgres 14 changed + * array_cat type from anyarray to anycompatiblearray. When + * upgrading to pg14, spegifically when running pg_restore on + * array_cat_agg we would get an error. So we drop the aggregate + * and create the right one on citus_finish_pg_upgrade. + */ + DROP AGGREGATE IF EXISTS array_cat_agg(anyarray); + DROP AGGREGATE IF EXISTS array_cat_agg(anycompatiblearray); + -- + -- Drop existing backup tables + -- + DROP TABLE IF EXISTS public.pg_dist_partition; + DROP TABLE IF EXISTS public.pg_dist_shard; + DROP TABLE IF EXISTS public.pg_dist_placement; + DROP TABLE IF EXISTS public.pg_dist_node_metadata; + DROP TABLE IF EXISTS public.pg_dist_node; + DROP TABLE IF EXISTS public.pg_dist_local_group; + DROP TABLE IF EXISTS public.pg_dist_transaction; + DROP TABLE IF EXISTS public.pg_dist_colocation; + DROP TABLE IF EXISTS public.pg_dist_authinfo; + DROP TABLE IF EXISTS public.pg_dist_poolinfo; + DROP TABLE IF EXISTS public.pg_dist_rebalance_strategy; + DROP TABLE IF EXISTS public.pg_dist_object; + + -- + -- backup citus catalog tables + -- + CREATE TABLE public.pg_dist_partition AS SELECT * FROM pg_catalog.pg_dist_partition; + CREATE TABLE public.pg_dist_shard AS SELECT * FROM pg_catalog.pg_dist_shard; + CREATE TABLE public.pg_dist_placement AS SELECT * FROM pg_catalog.pg_dist_placement; + CREATE TABLE public.pg_dist_node_metadata AS SELECT * FROM pg_catalog.pg_dist_node_metadata; + CREATE TABLE public.pg_dist_node AS SELECT * FROM pg_catalog.pg_dist_node; + CREATE TABLE public.pg_dist_local_group AS SELECT * FROM pg_catalog.pg_dist_local_group; + CREATE TABLE public.pg_dist_transaction AS SELECT * FROM pg_catalog.pg_dist_transaction; + CREATE TABLE public.pg_dist_colocation AS SELECT * FROM pg_catalog.pg_dist_colocation; + -- enterprise catalog tables + CREATE TABLE public.pg_dist_authinfo AS SELECT * FROM pg_catalog.pg_dist_authinfo; + CREATE TABLE public.pg_dist_poolinfo AS SELECT * FROM pg_catalog.pg_dist_poolinfo; + CREATE TABLE public.pg_dist_rebalance_strategy AS SELECT + name, + default_strategy, + shard_cost_function::regprocedure::text, + node_capacity_function::regprocedure::text, + shard_allowed_on_node_function::regprocedure::text, + default_threshold, + minimum_threshold, + improvement_threshold + FROM pg_catalog.pg_dist_rebalance_strategy; + + -- store upgrade stable identifiers on pg_dist_object catalog + CREATE TABLE public.pg_dist_object AS SELECT + address.type, + address.object_names, + address.object_args, + objects.distribution_argument_index, + objects.colocationid + FROM citus.pg_dist_object objects, + pg_catalog.pg_identify_object_as_address(objects.classid, objects.objid, objects.objsubid) address; +END; +$cppu$; + +COMMENT ON FUNCTION pg_catalog.citus_prepare_pg_upgrade() + IS 'perform tasks to copy citus settings to a location that could later be restored after pg_upgrade is done'; diff --git a/src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/latest.sql b/src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/latest.sql index fa2014870..8b5e13bc0 100644 --- a/src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/latest.sql +++ b/src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/latest.sql @@ -4,6 +4,19 @@ CREATE OR REPLACE FUNCTION pg_catalog.citus_prepare_pg_upgrade() SET search_path = pg_catalog AS $cppu$ BEGIN + + DELETE FROM pg_depend WHERE + objid IN (SELECT oid FROM pg_proc WHERE proname = 'array_cat_agg') AND + refobjid IN (select oid from pg_extension where extname = 'citus'); + /* + * We are dropping the aggregates because postgres 14 changed + * array_cat type from anyarray to anycompatiblearray. When + * upgrading to pg14, spegifically when running pg_restore on + * array_cat_agg we would get an error. So we drop the aggregate + * and create the right one on citus_finish_pg_upgrade. + */ + DROP AGGREGATE IF EXISTS array_cat_agg(anyarray); + DROP AGGREGATE IF EXISTS array_cat_agg(anycompatiblearray); -- -- Drop existing backup tables -- diff --git a/src/test/regress/bin/normalize.sed b/src/test/regress/bin/normalize.sed index 750a8e676..a0dfb1c4a 100644 --- a/src/test/regress/bin/normalize.sed +++ b/src/test/regress/bin/normalize.sed @@ -229,3 +229,4 @@ s/ERROR: parallel workers for vacuum must/ERROR: parallel vacuum degree must/g # ignore PL/pgSQL line numbers that differ on Mac builds s/(CONTEXT: PL\/pgSQL function .* line )([0-9]+)/\1XX/g +s/function array_cat_agg\(anycompatiblearray\)/function array_cat_agg\(anyarray\)/g diff --git a/src/test/regress/expected/multi_extension.out b/src/test/regress/expected/multi_extension.out index c6a5b1bd1..b115ecfc7 100644 --- a/src/test/regress/expected/multi_extension.out +++ b/src/test/regress/expected/multi_extension.out @@ -139,7 +139,7 @@ SELECT * FROM multi_extension.print_extension_changes(); | function alter_role_if_exists(text,text) boolean | function any_value(anyelement) anyelement | function any_value_agg(anyelement,anyelement) anyelement - | function array_cat_agg(anyarray) anyarray + | function array_cat_agg(anycompatiblearray) anycompatiblearray | function assign_distributed_transaction_id(integer,bigint,timestamp with time zone) void | function authinfo_valid(text) boolean | function broadcast_intermediate_result(text,text) bigint diff --git a/src/test/regress/expected/upgrade_list_citus_objects.out b/src/test/regress/expected/upgrade_list_citus_objects.out index d1212ce27..5983d6fbc 100644 --- a/src/test/regress/expected/upgrade_list_citus_objects.out +++ b/src/test/regress/expected/upgrade_list_citus_objects.out @@ -26,7 +26,7 @@ ORDER BY 1; function alter_table_set_access_method(regclass,text) function any_value(anyelement) function any_value_agg(anyelement,anyelement) - function array_cat_agg(anyarray) + function array_cat_agg(anycompatiblearray) function assign_distributed_transaction_id(integer,bigint,timestamp with time zone) function authinfo_valid(text) function broadcast_intermediate_result(text,text) diff --git a/src/test/regress/expected/upgrade_list_citus_objects_0.out b/src/test/regress/expected/upgrade_list_citus_objects_0.out index 045b538f2..fc2db7cd7 100644 --- a/src/test/regress/expected/upgrade_list_citus_objects_0.out +++ b/src/test/regress/expected/upgrade_list_citus_objects_0.out @@ -23,7 +23,7 @@ ORDER BY 1; function alter_table_set_access_method(regclass,text) function any_value(anyelement) function any_value_agg(anyelement,anyelement) - function array_cat_agg(anyarray) + function array_cat_agg(anycompatiblearray) function assign_distributed_transaction_id(integer,bigint,timestamp with time zone) function authinfo_valid(text) function broadcast_intermediate_result(text,text) From abd3c1089bd43d079d87e7d4ac7baee7f7dc0439 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Thu, 19 Aug 2021 08:34:11 +0300 Subject: [PATCH 054/104] Use oid_hash in write state management --- src/backend/columnar/write_state_management.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/backend/columnar/write_state_management.c b/src/backend/columnar/write_state_management.c index 4d815c2a0..201a1a479 100644 --- a/src/backend/columnar/write_state_management.c +++ b/src/backend/columnar/write_state_management.c @@ -16,6 +16,7 @@ #include "access/tsmapi.h" #if PG_VERSION_NUM >= 130000 #include "access/heaptoast.h" +#include "common/hashfn.h" #else #include "access/tuptoaster.h" #endif @@ -132,9 +133,10 @@ columnar_init_write_state(Relation relation, TupleDesc tupdesc, "Column Store Write State Management Context", ALLOCSET_DEFAULT_SIZES); HASHCTL info; - uint32 hashFlags = (HASH_ELEM | HASH_STRINGS | HASH_CONTEXT); + uint32 hashFlags = (HASH_ELEM | HASH_FUNCTION | HASH_CONTEXT); memset(&info, 0, sizeof(info)); info.keysize = sizeof(Oid); + info.hash = oid_hash; info.entrysize = sizeof(WriteStateMapEntry); info.hcxt = WriteStateContext; From 3f5c178c93b9233a8042d8324517b668715f92d8 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Thu, 19 Aug 2021 10:50:24 +0300 Subject: [PATCH 055/104] Remove VERBOSE output to make pg14 and pg13 output the same --- .../multi_subquery_window_functions.out | 37 ++++--------------- .../sql/multi_subquery_window_functions.sql | 2 +- 2 files changed, 8 insertions(+), 31 deletions(-) diff --git a/src/test/regress/expected/multi_subquery_window_functions.out b/src/test/regress/expected/multi_subquery_window_functions.out index 97a481df0..aa4249efc 100644 --- a/src/test/regress/expected/multi_subquery_window_functions.out +++ b/src/test/regress/expected/multi_subquery_window_functions.out @@ -675,7 +675,7 @@ LIMIT 2 | 1 (5 rows) -EXPLAIN (COSTS FALSE, VERBOSE TRUE) +EXPLAIN (COSTS FALSE) SELECT * FROM ( ( SELECT user_id, @@ -709,72 +709,49 @@ EXPLAIN (COSTS FALSE, VERBOSE TRUE) user_id)) AS ftop ORDER BY 2 DESC, 1 DESC LIMIT 5; - QUERY PLAN + QUERY PLAN --------------------------------------------------------------------- Limit - Output: remote_scan.user_id, remote_scan.sum -> Sort - Output: remote_scan.user_id, remote_scan.sum Sort Key: remote_scan.sum DESC, remote_scan.user_id DESC -> Custom Scan (Citus Adaptive) - Output: remote_scan.user_id, remote_scan.sum Task Count: 4 Tasks Shown: One of 4 -> Task - Query: SELECT worker_column_1 AS user_id, worker_column_2 AS sum FROM (SELECT ftop.user_id AS worker_column_1, ftop.sum AS worker_column_2 FROM (SELECT user_id_1.user_id, sum(user_id_1.counter) AS sum FROM (SELECT users_table.user_id, sum(users_table.value_2) OVER (PARTITION BY users_table.user_id) AS counter FROM public.users_table_1400256 users_table UNION SELECT events_table.user_id, sum(events_table.value_2) OVER (PARTITION BY events_table.user_id) AS counter FROM public.events_table_1400260 events_table) user_id_1 GROUP BY user_id_1.user_id UNION SELECT user_id_2.user_id, sum(user_id_2.counter) AS sum FROM (SELECT users_table.user_id, sum(users_table.value_2) OVER (PARTITION BY users_table.user_id) AS counter FROM public.users_table_1400256 users_table UNION SELECT events_table.user_id, sum(events_table.value_2) OVER (PARTITION BY events_table.user_id) AS counter FROM public.events_table_1400260 events_table) user_id_2 GROUP BY user_id_2.user_id) ftop) worker_subquery ORDER BY worker_column_2 DESC, worker_column_1 DESC LIMIT '5'::bigint Node: host=localhost port=xxxxx dbname=regression -> Limit - Output: users_table.user_id, (sum((sum(users_table.value_2) OVER (?)))) -> Sort - Output: users_table.user_id, (sum((sum(users_table.value_2) OVER (?)))) Sort Key: (sum((sum(users_table.value_2) OVER (?)))) DESC, users_table.user_id DESC -> HashAggregate - Output: users_table.user_id, (sum((sum(users_table.value_2) OVER (?)))) Group Key: users_table.user_id, (sum((sum(users_table.value_2) OVER (?)))) -> Append -> HashAggregate - Output: users_table.user_id, sum((sum(users_table.value_2) OVER (?))) Group Key: users_table.user_id -> HashAggregate - Output: users_table.user_id, (sum(users_table.value_2) OVER (?)) Group Key: users_table.user_id, (sum(users_table.value_2) OVER (?)) -> Append -> WindowAgg - Output: users_table.user_id, sum(users_table.value_2) OVER (?) -> Sort - Output: users_table.user_id, users_table.value_2 Sort Key: users_table.user_id - -> Seq Scan on public.users_table_1400256 users_table - Output: users_table.user_id, users_table.value_2 + -> Seq Scan on users_table_1400256 users_table -> WindowAgg - Output: events_table.user_id, sum(events_table.value_2) OVER (?) -> Sort - Output: events_table.user_id, events_table.value_2 Sort Key: events_table.user_id - -> Seq Scan on public.events_table_1400260 events_table - Output: events_table.user_id, events_table.value_2 + -> Seq Scan on events_table_1400260 events_table -> HashAggregate - Output: users_table_1.user_id, sum((sum(users_table_1.value_2) OVER (?))) Group Key: users_table_1.user_id -> HashAggregate - Output: users_table_1.user_id, (sum(users_table_1.value_2) OVER (?)) Group Key: users_table_1.user_id, (sum(users_table_1.value_2) OVER (?)) -> Append -> WindowAgg - Output: users_table_1.user_id, sum(users_table_1.value_2) OVER (?) -> Sort - Output: users_table_1.user_id, users_table_1.value_2 Sort Key: users_table_1.user_id - -> Seq Scan on public.users_table_1400256 users_table_1 - Output: users_table_1.user_id, users_table_1.value_2 + -> Seq Scan on users_table_1400256 users_table_1 -> WindowAgg - Output: events_table_1.user_id, sum(events_table_1.value_2) OVER (?) -> Sort - Output: events_table_1.user_id, events_table_1.value_2 Sort Key: events_table_1.user_id - -> Seq Scan on public.events_table_1400260 events_table_1 - Output: events_table_1.user_id, events_table_1.value_2 -(63 rows) + -> Seq Scan on events_table_1400260 events_table_1 +(40 rows) -- test with window functions which aren't pushed down SELECT diff --git a/src/test/regress/sql/multi_subquery_window_functions.sql b/src/test/regress/sql/multi_subquery_window_functions.sql index b5c0332d7..706aa64fa 100644 --- a/src/test/regress/sql/multi_subquery_window_functions.sql +++ b/src/test/regress/sql/multi_subquery_window_functions.sql @@ -441,7 +441,7 @@ ORDER BY LIMIT 5; -EXPLAIN (COSTS FALSE, VERBOSE TRUE) +EXPLAIN (COSTS FALSE) SELECT * FROM ( ( SELECT user_id, From fb8671f2912c5540650fe959acdbef5404e1845f Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Thu, 19 Aug 2021 11:13:22 +0300 Subject: [PATCH 056/104] Change pg13 test to not differ with pg14 to avoid adding alternative output --- src/test/regress/expected/pg13.out | 38 +++++++----------------------- src/test/regress/sql/pg13.sql | 6 +++++ 2 files changed, 15 insertions(+), 29 deletions(-) diff --git a/src/test/regress/expected/pg13.out b/src/test/regress/expected/pg13.out index 3933cb45d..7b730ed64 100644 --- a/src/test/regress/expected/pg13.out +++ b/src/test/regress/expected/pg13.out @@ -228,39 +228,19 @@ INSERT INTO test_wal VALUES(3,33),(4,44),(5,55) RETURNING *; (10 rows) -- make sure WAL works in distributed subplans +-- this test has different output for pg14 and here we mostly test that +-- we don't get an error, hence we use explain_has_distributed_subplan. +SELECT public.explain_has_distributed_subplan( +$$ EXPLAIN (ANALYZE TRUE, WAL TRUE, COSTS FALSE, SUMMARY FALSE, BUFFERS FALSE, TIMING FALSE) WITH cte_1 AS (INSERT INTO test_wal VALUES(6,66),(7,77),(8,88) RETURNING *) SELECT * FROM cte_1; - QUERY PLAN +$$ +); + explain_has_distributed_subplan --------------------------------------------------------------------- - Custom Scan (Citus Adaptive) (actual rows=3 loops=1) - -> Distributed Subplan XXX_1 - Intermediate Data Size: 54 bytes - Result destination: Write locally - -> Custom Scan (Citus Adaptive) (actual rows=3 loops=1) - Task Count: 2 - Tuple data received from nodes: 9 bytes - Tasks Shown: All - -> Task - Tuple data received from node: 6 bytes - Node: host=localhost port=xxxxx dbname=regression - -> Insert on test_wal_65012 citus_table_alias (actual rows=2 loops=1) - WAL: records=2 bytes=126 - -> Values Scan on "*VALUES*" (actual rows=2 loops=1) - -> Task - Tuple data received from node: 3 bytes - Node: host=localhost port=xxxxx dbname=regression - -> Insert on test_wal_65013 citus_table_alias (actual rows=1 loops=1) - WAL: records=1 bytes=63 - -> Result (actual rows=1 loops=1) - Task Count: 1 - Tuple data received from nodes: 9 bytes - Tasks Shown: All - -> Task - Tuple data received from node: 9 bytes - Node: host=localhost port=xxxxx dbname=regression - -> Function Scan on read_intermediate_result intermediate_result (actual rows=3 loops=1) -(27 rows) + t +(1 row) SET client_min_messages TO WARNING; drop schema test_pg13 cascade; diff --git a/src/test/regress/sql/pg13.sql b/src/test/regress/sql/pg13.sql index 608292a9e..00c94a605 100644 --- a/src/test/regress/sql/pg13.sql +++ b/src/test/regress/sql/pg13.sql @@ -111,9 +111,15 @@ EXPLAIN (ANALYZE TRUE, WAL TRUE, COSTS FALSE, SUMMARY FALSE, BUFFERS FALSE, TIMI INSERT INTO test_wal VALUES(3,33),(4,44),(5,55) RETURNING *; -- make sure WAL works in distributed subplans +-- this test has different output for pg14 and here we mostly test that +-- we don't get an error, hence we use explain_has_distributed_subplan. +SELECT public.explain_has_distributed_subplan( +$$ EXPLAIN (ANALYZE TRUE, WAL TRUE, COSTS FALSE, SUMMARY FALSE, BUFFERS FALSE, TIMING FALSE) WITH cte_1 AS (INSERT INTO test_wal VALUES(6,66),(7,77),(8,88) RETURNING *) SELECT * FROM cte_1; +$$ +); SET client_min_messages TO WARNING; drop schema test_pg13 cascade; From dc81cae18fdc36ed819c1eb89eaba9f6059f80e0 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Thu, 19 Aug 2021 12:21:03 +0300 Subject: [PATCH 057/104] Turn off COSTS to avoid alternative output for pg14 --- src/test/regress/expected/multi_explain.out | 30 ++++++--------------- src/test/regress/sql/multi_explain.sql | 4 +-- 2 files changed, 10 insertions(+), 24 deletions(-) diff --git a/src/test/regress/expected/multi_explain.out b/src/test/regress/expected/multi_explain.out index beac2eef4..7eb3eb0d4 100644 --- a/src/test/regress/expected/multi_explain.out +++ b/src/test/regress/expected/multi_explain.out @@ -81,13 +81,11 @@ EXPLAIN (COSTS FALSE, FORMAT JSON) "Node Type": "Aggregate", "Strategy": "Hashed", "Partial Mode": "Simple", - "Parent Relationship": "Outer", "Parallel Aware": false, "Group Key": ["remote_scan.l_quantity"], "Plans": [ { "Node Type": "Custom Scan", - "Parent Relationship": "Outer", "Custom Plan Provider": "Citus Adaptive", "Parallel Aware": false, "Distributed Query": { @@ -109,7 +107,6 @@ EXPLAIN (COSTS FALSE, FORMAT JSON) "Plans": [ { "Node Type": "Seq Scan", - "Parent Relationship": "Outer", "Parallel Aware": false, "Relation Name": "lineitem_290000", "Alias": "lineitem" @@ -154,7 +151,6 @@ EXPLAIN (COSTS FALSE, FORMAT XML) Aggregate Hashed Simple - Outer false remote_scan.l_quantity @@ -162,7 +158,6 @@ EXPLAIN (COSTS FALSE, FORMAT XML) Custom Scan - Outer Citus Adaptive false @@ -186,7 +181,6 @@ EXPLAIN (COSTS FALSE, FORMAT XML) Seq Scan - Outer false lineitem_290000 lineitem @@ -226,13 +220,11 @@ EXPLAIN (COSTS FALSE, FORMAT YAML) - Node Type: "Aggregate" Strategy: "Hashed" Partial Mode: "Simple" - Parent Relationship: "Outer" Parallel Aware: false Group Key: - "remote_scan.l_quantity" Plans: - Node Type: "Custom Scan" - Parent Relationship: "Outer" Custom Plan Provider: "Citus Adaptive" Parallel Aware: false Distributed Query: @@ -251,7 +243,6 @@ EXPLAIN (COSTS FALSE, FORMAT YAML) - "l_quantity" Plans: - Node Type: "Seq Scan" - Parent Relationship: "Outer" Parallel Aware: false Relation Name: "lineitem_290000" Alias: "lineitem" @@ -1093,7 +1084,6 @@ EXPLAIN (COSTS FALSE, FORMAT JSON) "Plans": [ { "Node Type": "Custom Scan", - "Parent Relationship": "Outer", "Custom Plan Provider": "Citus Adaptive", "Parallel Aware": false, "Distributed Query": { @@ -1142,7 +1132,6 @@ EXPLAIN (COSTS FALSE, FORMAT XML) Custom Scan - Outer Citus Adaptive false @@ -1201,7 +1190,6 @@ EXPLAIN (COSTS FALSE, FORMAT YAML) Parallel Aware: false Plans: - Node Type: "Custom Scan" - Parent Relationship: "Outer" Custom Plan Provider: "Citus Adaptive" Parallel Aware: false Distributed Query: @@ -1284,14 +1272,14 @@ Custom Scan (Citus Adaptive) (actual rows=3 loops=1) \set VERBOSITY TERSE PREPARE multi_shard_query_param(int) AS UPDATE lineitem SET l_quantity = $1; BEGIN; -EXPLAIN EXECUTE multi_shard_query_param(5); -Custom Scan (Citus Adaptive) (cost=0.00..0.00 rows=0 width=0) +EXPLAIN (COSTS OFF) EXECUTE multi_shard_query_param(5); +Custom Scan (Citus Adaptive) Task Count: 2 Tasks Shown: One of 2 -> Task Node: host=localhost port=xxxxx dbname=regression - -> Update on lineitem_290000 lineitem (cost=0.00..176.00 rows=6000 width=140) - -> Seq Scan on lineitem_290000 lineitem (cost=0.00..176.00 rows=6000 width=140) + -> Update on lineitem_290000 lineitem + -> Seq Scan on lineitem_290000 lineitem ROLLBACK; BEGIN; EXPLAIN (ANALYZE ON, COSTS OFF, TIMING OFF, SUMMARY OFF) EXECUTE multi_shard_query_param(5); @@ -2078,7 +2066,6 @@ EXPLAIN (COSTS off, ANALYZE on, TIMING off, SUMMARY off, FORMAT JSON) INSERT INT "Plans": [ { "Node Type": "Result", - "Parent Relationship": "Member", "Parallel Aware": false, "Actual Rows": 1, "Actual Loops": 1 @@ -2176,7 +2163,6 @@ EXPLAIN (COSTS off, ANALYZE on, TIMING off, SUMMARY off, FORMAT XML) INSERT INTO Result - Member false 1 1 @@ -2784,14 +2770,14 @@ Custom Scan (Citus Adaptive) (actual rows=0 loops=1) deallocate update_query; -- prepared deletes PREPARE delete_query AS DELETE FROM simple WHERE name=$1 OR name=$2; -EXPLAIN EXECUTE delete_query('x', 'y'); -Custom Scan (Citus Adaptive) (cost=0.00..0.00 rows=0 width=0) +EXPLAIN (COSTS OFF) EXECUTE delete_query('x', 'y'); +Custom Scan (Citus Adaptive) Task Count: 2 Tasks Shown: One of 2 -> Task Node: host=localhost port=xxxxx dbname=regression - -> Delete on simple_570026 simple (cost=0.00..29.05 rows=13 width=6) - -> Seq Scan on simple_570026 simple (cost=0.00..29.05 rows=13 width=6) + -> Delete on simple_570026 simple + -> Seq Scan on simple_570026 simple Filter: ((name = 'x'::text) OR (name = 'y'::text)) EXPLAIN :default_analyze_flags EXECUTE delete_query('x', 'y'); Custom Scan (Citus Adaptive) (actual rows=0 loops=1) diff --git a/src/test/regress/sql/multi_explain.sql b/src/test/regress/sql/multi_explain.sql index 1840fa208..d58477f72 100644 --- a/src/test/regress/sql/multi_explain.sql +++ b/src/test/regress/sql/multi_explain.sql @@ -539,7 +539,7 @@ EXPLAIN (ANALYZE ON, COSTS OFF, TIMING OFF, SUMMARY OFF) EXECUTE router_executor \set VERBOSITY TERSE PREPARE multi_shard_query_param(int) AS UPDATE lineitem SET l_quantity = $1; BEGIN; -EXPLAIN EXECUTE multi_shard_query_param(5); +EXPLAIN (COSTS OFF) EXECUTE multi_shard_query_param(5); ROLLBACK; BEGIN; EXPLAIN (ANALYZE ON, COSTS OFF, TIMING OFF, SUMMARY OFF) EXECUTE multi_shard_query_param(5); @@ -990,7 +990,7 @@ deallocate update_query; -- prepared deletes PREPARE delete_query AS DELETE FROM simple WHERE name=$1 OR name=$2; -EXPLAIN EXECUTE delete_query('x', 'y'); +EXPLAIN (COSTS OFF) EXECUTE delete_query('x', 'y'); EXPLAIN :default_analyze_flags EXECUTE delete_query('x', 'y'); deallocate delete_query; From df9b7149c3953c7266ec44260c871c3e75ba031b Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Thu, 19 Aug 2021 12:50:22 +0300 Subject: [PATCH 058/104] Add some normalization rules for pg14 --- src/test/regress/bin/normalize.sed | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/test/regress/bin/normalize.sed b/src/test/regress/bin/normalize.sed index a0dfb1c4a..d15107c5e 100644 --- a/src/test/regress/bin/normalize.sed +++ b/src/test/regress/bin/normalize.sed @@ -176,7 +176,7 @@ s/relation with OID [0-9]+ does not exist/relation with OID XXXX does not exist/ /^DEBUG: EventTriggerInvoke [0-9]+$/d # ignore DEBUG1 messages that Postgres generates -/^DEBUG: rehashing catalog cache id [0-9]+$/d +/^DEBUG: rehashing catalog cache id .*$/d # ignore JIT related messages /^DEBUG: probing availability of JIT.*/d @@ -229,4 +229,15 @@ s/ERROR: parallel workers for vacuum must/ERROR: parallel vacuum degree must/g # ignore PL/pgSQL line numbers that differ on Mac builds s/(CONTEXT: PL\/pgSQL function .* line )([0-9]+)/\1XX/g + +# can be removed after dropping PG13 support +s/ERROR: parallel workers for vacuum must be between/ERROR: parallel vacuum degree must be between/g +s/ERROR: fake_fetch_row_version not implemented/ERROR: fake_tuple_update not implemented/g +s/ERROR: COMMIT is not allowed in an SQL function/ERROR: COMMIT is not allowed in a SQL function/g +s/ERROR: ROLLBACK is not allowed in an SQL function/ERROR: ROLLBACK is not allowed in a SQL function/g +/.*Query Identifier.*/d +/.*Async-Capable.*/d +/.*Async Capable.*/d +/Parent Relationship/d +/Parent-Relationship/d s/function array_cat_agg\(anycompatiblearray\)/function array_cat_agg\(anyarray\)/g From 256e7d1540533ecfe8079e0f9f293c55ad9b800c Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Thu, 19 Aug 2021 13:11:02 +0300 Subject: [PATCH 059/104] Add alternative output for window_functions --- .../expected/insert_select_repartition.out | 1 - .../regress/expected/window_functions.out | 52 ++++++++++++------- .../regress/expected/window_functions_0.out | 20 ++++--- 3 files changed, 46 insertions(+), 27 deletions(-) diff --git a/src/test/regress/expected/insert_select_repartition.out b/src/test/regress/expected/insert_select_repartition.out index 163985ace..afa54b7e8 100644 --- a/src/test/regress/expected/insert_select_repartition.out +++ b/src/test/regress/expected/insert_select_repartition.out @@ -1209,7 +1209,6 @@ ON CONFLICT(c1, c2, c3, c4, c5, c6) DO UPDATE SET cardinality = enriched.cardinality + excluded.cardinality, sum = enriched.sum + excluded.sum; -DEBUG: rehashing catalog cache id 14 for pg_opclass; 17 tups, 8 buckets at character 224 DEBUG: INSERT target table and the source relation of the SELECT partition column value must be colocated in distributed INSERT ... SELECT DEBUG: Router planner cannot handle multi-shard select queries DEBUG: performing repartitioned INSERT ... SELECT diff --git a/src/test/regress/expected/window_functions.out b/src/test/regress/expected/window_functions.out index 0a41bc0cc..6657c3670 100644 --- a/src/test/regress/expected/window_functions.out +++ b/src/test/regress/expected/window_functions.out @@ -1386,12 +1386,16 @@ LIMIT 5; Sort Key: users_table.user_id, (avg(users_table.value_1)) DESC Presorted Key: users_table.user_id -> WindowAgg - -> Sort + -> Incremental Sort Sort Key: users_table.user_id, (('1'::numeric / ('1'::numeric + avg(users_table.value_1)))) - -> HashAggregate + Presorted Key: users_table.user_id + -> GroupAggregate Group Key: users_table.user_id, users_table.value_2 - -> Seq Scan on users_table_1400256 users_table -(18 rows) + -> Incremental Sort + Sort Key: users_table.user_id, users_table.value_2 + Presorted Key: users_table.user_id + -> Index Scan using is_index1_1400256 on users_table_1400256 users_table +(22 rows) EXPLAIN (COSTS FALSE) SELECT @@ -1418,12 +1422,16 @@ LIMIT 5; Sort Key: users_table.user_id, (avg(users_table.value_1)) DESC Presorted Key: users_table.user_id -> WindowAgg - -> Sort + -> Incremental Sort Sort Key: users_table.user_id, (('1'::numeric / ('1'::numeric + avg(users_table.value_1)))) - -> HashAggregate + Presorted Key: users_table.user_id + -> GroupAggregate Group Key: users_table.user_id, users_table.value_2 - -> Seq Scan on users_table_1400256 users_table -(18 rows) + -> Incremental Sort + Sort Key: users_table.user_id, users_table.value_2 + Presorted Key: users_table.user_id + -> Index Scan using is_index1_1400256 on users_table_1400256 users_table +(22 rows) EXPLAIN (COSTS FALSE) SELECT @@ -1435,7 +1443,7 @@ FROM GROUP BY user_id, value_2 ORDER BY user_id, avg(value_1) DESC LIMIT 5; - QUERY PLAN + QUERY PLAN --------------------------------------------------------------------- Limit -> Sort @@ -1450,12 +1458,16 @@ LIMIT 5; Sort Key: users_table.user_id, (avg(users_table.value_1)) DESC Presorted Key: users_table.user_id -> WindowAgg - -> Sort + -> Incremental Sort Sort Key: users_table.user_id, ((1 / (1 + sum(users_table.value_2)))) - -> HashAggregate + Presorted Key: users_table.user_id + -> GroupAggregate Group Key: users_table.user_id, users_table.value_2 - -> Seq Scan on users_table_1400256 users_table -(18 rows) + -> Incremental Sort + Sort Key: users_table.user_id, users_table.value_2 + Presorted Key: users_table.user_id + -> Index Scan using is_index1_1400256 on users_table_1400256 users_table +(22 rows) EXPLAIN (COSTS FALSE) SELECT @@ -1467,7 +1479,7 @@ FROM GROUP BY user_id, value_2 ORDER BY user_id, avg(value_1) DESC LIMIT 5; - QUERY PLAN + QUERY PLAN --------------------------------------------------------------------- Limit -> Sort @@ -1482,12 +1494,16 @@ LIMIT 5; Sort Key: users_table.user_id, (avg(users_table.value_1)) DESC Presorted Key: users_table.user_id -> WindowAgg - -> Sort + -> Incremental Sort Sort Key: users_table.user_id, (sum(users_table.value_2)) - -> HashAggregate + Presorted Key: users_table.user_id + -> GroupAggregate Group Key: users_table.user_id, users_table.value_2 - -> Seq Scan on users_table_1400256 users_table -(18 rows) + -> Incremental Sort + Sort Key: users_table.user_id, users_table.value_2 + Presorted Key: users_table.user_id + -> Index Scan using is_index1_1400256 on users_table_1400256 users_table +(22 rows) -- Grouping can be pushed down with aggregates even when window function can't EXPLAIN (COSTS FALSE) diff --git a/src/test/regress/expected/window_functions_0.out b/src/test/regress/expected/window_functions_0.out index aea319c0b..0a41bc0cc 100644 --- a/src/test/regress/expected/window_functions_0.out +++ b/src/test/regress/expected/window_functions_0.out @@ -1382,15 +1382,16 @@ LIMIT 5; -> Task Node: host=localhost port=xxxxx dbname=regression -> Limit - -> Sort + -> Incremental Sort Sort Key: users_table.user_id, (avg(users_table.value_1)) DESC + Presorted Key: users_table.user_id -> WindowAgg -> Sort Sort Key: users_table.user_id, (('1'::numeric / ('1'::numeric + avg(users_table.value_1)))) -> HashAggregate Group Key: users_table.user_id, users_table.value_2 -> Seq Scan on users_table_1400256 users_table -(17 rows) +(18 rows) EXPLAIN (COSTS FALSE) SELECT @@ -1413,15 +1414,16 @@ LIMIT 5; -> Task Node: host=localhost port=xxxxx dbname=regression -> Limit - -> Sort + -> Incremental Sort Sort Key: users_table.user_id, (avg(users_table.value_1)) DESC + Presorted Key: users_table.user_id -> WindowAgg -> Sort Sort Key: users_table.user_id, (('1'::numeric / ('1'::numeric + avg(users_table.value_1)))) -> HashAggregate Group Key: users_table.user_id, users_table.value_2 -> Seq Scan on users_table_1400256 users_table -(17 rows) +(18 rows) EXPLAIN (COSTS FALSE) SELECT @@ -1444,15 +1446,16 @@ LIMIT 5; -> Task Node: host=localhost port=xxxxx dbname=regression -> Limit - -> Sort + -> Incremental Sort Sort Key: users_table.user_id, (avg(users_table.value_1)) DESC + Presorted Key: users_table.user_id -> WindowAgg -> Sort Sort Key: users_table.user_id, ((1 / (1 + sum(users_table.value_2)))) -> HashAggregate Group Key: users_table.user_id, users_table.value_2 -> Seq Scan on users_table_1400256 users_table -(17 rows) +(18 rows) EXPLAIN (COSTS FALSE) SELECT @@ -1475,15 +1478,16 @@ LIMIT 5; -> Task Node: host=localhost port=xxxxx dbname=regression -> Limit - -> Sort + -> Incremental Sort Sort Key: users_table.user_id, (avg(users_table.value_1)) DESC + Presorted Key: users_table.user_id -> WindowAgg -> Sort Sort Key: users_table.user_id, (sum(users_table.value_2)) -> HashAggregate Group Key: users_table.user_id, users_table.value_2 -> Seq Scan on users_table_1400256 users_table -(17 rows) +(18 rows) -- Grouping can be pushed down with aggregates even when window function can't EXPLAIN (COSTS FALSE) From 20c32a7a1d0708dcbc886eca1ca47714f05e7034 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Thu, 19 Aug 2021 15:59:59 +0300 Subject: [PATCH 060/104] Add alternative output for multi_deparse_function Postgres tightened up its checks for invalid GUC names hence we started to get an alternative output for one of our tests. We add an alternative output since the file is relatively small. Commit on PG: 3db826bd55cd1df0dd8c3d811f8e5b936d7ba1e4 --- .../expected/distributed_functions.out | 17 --- .../expected/multi_deparse_function.out | 114 ++++++------------ .../regress/sql/distributed_functions.sql | 7 -- .../regress/sql/multi_deparse_function.sql | 18 --- 4 files changed, 34 insertions(+), 122 deletions(-) diff --git a/src/test/regress/expected/distributed_functions.out b/src/test/regress/expected/distributed_functions.out index 7fb8166a3..300a87954 100644 --- a/src/test/regress/expected/distributed_functions.out +++ b/src/test/regress/expected/distributed_functions.out @@ -343,20 +343,6 @@ SELECT public.verify_function_is_same_on_workers('function_tests.eq(macaddr,maca t (1 row) -ALTER FUNCTION eq(macaddr,macaddr) SET "citus.setting;'" TO 'hello '' world'; -SELECT public.verify_function_is_same_on_workers('function_tests.eq(macaddr,macaddr)'); - verify_function_is_same_on_workers ---------------------------------------------------------------------- - t -(1 row) - -ALTER FUNCTION eq(macaddr,macaddr) RESET "citus.setting;'"; -SELECT public.verify_function_is_same_on_workers('function_tests.eq(macaddr,macaddr)'); - verify_function_is_same_on_workers ---------------------------------------------------------------------- - t -(1 row) - ALTER FUNCTION eq(macaddr,macaddr) SET search_path TO 'sch'';ma', public; SELECT public.verify_function_is_same_on_workers('function_tests.eq(macaddr,macaddr)'); verify_function_is_same_on_workers @@ -540,9 +526,6 @@ SELECT * FROM run_command_on_workers('SELECT function_tests2.sum2(id) FROM (sele localhost | 57638 | f | ERROR: function function_tests2.sum2(integer) does not exist (2 rows) --- postgres doesn't accept parameter names in the regprocedure input -SELECT create_distributed_function('eq_with_param_names(val1 macaddr, macaddr)', 'val1'); -ERROR: invalid type name "val1 macaddr" -- invalid distribution_arg_name SELECT create_distributed_function('eq_with_param_names(macaddr, macaddr)', distribution_arg_name:='test'); ERROR: cannot distribute the function "eq_with_param_names" since the distribution argument is not valid diff --git a/src/test/regress/expected/multi_deparse_function.out b/src/test/regress/expected/multi_deparse_function.out index 7a7d775c8..24a9b3171 100644 --- a/src/test/regress/expected/multi_deparse_function.out +++ b/src/test/regress/expected/multi_deparse_function.out @@ -70,7 +70,7 @@ SELECT create_distributed_function('add(int,int)'); SELECT deparse_and_run_on_workers($cmd$ ALTER FUNCTION add CALLED ON NULL INPUT $cmd$); -INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer, integer) CALLED ON NULL INPUT; +INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer,integer) CALLED ON NULL INPUT; CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE deparse_and_run_on_workers --------------------------------------------------------------------- @@ -83,7 +83,7 @@ CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE SELECT deparse_and_run_on_workers($cmd$ ALTER FUNCTION add RETURNS NULL ON NULL INPUT $cmd$); -INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer, integer) STRICT; +INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer,integer) STRICT; CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE deparse_and_run_on_workers --------------------------------------------------------------------- @@ -94,7 +94,7 @@ CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE SELECT deparse_and_run_on_workers($cmd$ ALTER FUNCTION add STRICT $cmd$); -INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer, integer) STRICT; +INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer,integer) STRICT; CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE deparse_and_run_on_workers --------------------------------------------------------------------- @@ -105,7 +105,7 @@ CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE SELECT deparse_and_run_on_workers($cmd$ ALTER FUNCTION add IMMUTABLE $cmd$); -INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer, integer) IMMUTABLE; +INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer,integer) IMMUTABLE; CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE deparse_and_run_on_workers --------------------------------------------------------------------- @@ -116,7 +116,7 @@ CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE SELECT deparse_and_run_on_workers($cmd$ ALTER FUNCTION add STABLE $cmd$); -INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer, integer) STABLE; +INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer,integer) STABLE; CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE deparse_and_run_on_workers --------------------------------------------------------------------- @@ -127,7 +127,7 @@ CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE SELECT deparse_and_run_on_workers($cmd$ ALTER FUNCTION add VOLATILE $cmd$); -INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer, integer) VOLATILE; +INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer,integer) VOLATILE; CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE deparse_and_run_on_workers --------------------------------------------------------------------- @@ -138,7 +138,7 @@ CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE SELECT deparse_and_run_on_workers($cmd$ ALTER FUNCTION add LEAKPROOF $cmd$); -INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer, integer) LEAKPROOF; +INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer,integer) LEAKPROOF; CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE deparse_and_run_on_workers --------------------------------------------------------------------- @@ -149,7 +149,7 @@ CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE SELECT deparse_and_run_on_workers($cmd$ ALTER FUNCTION add NOT LEAKPROOF $cmd$); -INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer, integer) NOT LEAKPROOF; +INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer,integer) NOT LEAKPROOF; CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE deparse_and_run_on_workers --------------------------------------------------------------------- @@ -162,7 +162,7 @@ CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE SELECT deparse_and_run_on_workers($cmd$ ALTER FUNCTION add EXTERNAL SECURITY INVOKER $cmd$); -INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer, integer) SECURITY INVOKER; +INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer,integer) SECURITY INVOKER; CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE deparse_and_run_on_workers --------------------------------------------------------------------- @@ -173,7 +173,7 @@ CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE SELECT deparse_and_run_on_workers($cmd$ ALTER FUNCTION add SECURITY INVOKER $cmd$); -INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer, integer) SECURITY INVOKER; +INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer,integer) SECURITY INVOKER; CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE deparse_and_run_on_workers --------------------------------------------------------------------- @@ -184,7 +184,7 @@ CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE SELECT deparse_and_run_on_workers($cmd$ ALTER FUNCTION add EXTERNAL SECURITY DEFINER $cmd$); -INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer, integer) SECURITY DEFINER; +INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer,integer) SECURITY DEFINER; CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE deparse_and_run_on_workers --------------------------------------------------------------------- @@ -195,7 +195,7 @@ CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE SELECT deparse_and_run_on_workers($cmd$ ALTER FUNCTION add SECURITY DEFINER $cmd$); -INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer, integer) SECURITY DEFINER; +INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer,integer) SECURITY DEFINER; CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE deparse_and_run_on_workers --------------------------------------------------------------------- @@ -206,7 +206,7 @@ CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE SELECT deparse_and_run_on_workers($cmd$ ALTER FUNCTION add PARALLEL UNSAFE $cmd$); -INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer, integer) PARALLEL UNSAFE; +INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer,integer) PARALLEL UNSAFE; CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE deparse_and_run_on_workers --------------------------------------------------------------------- @@ -217,7 +217,7 @@ CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE SELECT deparse_and_run_on_workers($cmd$ ALTER FUNCTION add PARALLEL RESTRICTED $cmd$); -INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer, integer) PARALLEL RESTRICTED; +INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer,integer) PARALLEL RESTRICTED; CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE deparse_and_run_on_workers --------------------------------------------------------------------- @@ -228,7 +228,7 @@ CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE SELECT deparse_and_run_on_workers($cmd$ ALTER FUNCTION add PARALLEL SAFE $cmd$); -INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer, integer) PARALLEL SAFE; +INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer,integer) PARALLEL SAFE; CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE deparse_and_run_on_workers --------------------------------------------------------------------- @@ -240,7 +240,7 @@ CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE SELECT deparse_and_run_on_workers($cmd$ ALTER FUNCTION add COST 1234 $cmd$); -INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer, integer) COST 1234.000000; +INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer,integer) COST 1234.000000; CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE deparse_and_run_on_workers --------------------------------------------------------------------- @@ -251,7 +251,7 @@ CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE SELECT deparse_and_run_on_workers($cmd$ ALTER FUNCTION add COST 1234.5 $cmd$); -INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer, integer) COST 1234.500000; +INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer,integer) COST 1234.500000; CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE deparse_and_run_on_workers --------------------------------------------------------------------- @@ -262,7 +262,7 @@ CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE SELECT deparse_and_run_on_workers($cmd$ ALTER FUNCTION add SET log_min_messages = ERROR $cmd$); -INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer, integer) SET log_min_messages = 'error'; +INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer,integer) SET log_min_messages = 'error'; CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE deparse_and_run_on_workers --------------------------------------------------------------------- @@ -273,7 +273,7 @@ CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE SELECT deparse_and_run_on_workers($cmd$ ALTER FUNCTION add SET log_min_messages TO DEFAULT $cmd$); -INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer, integer) SET log_min_messages TO DEFAULT; +INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer,integer) SET log_min_messages TO DEFAULT; CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE deparse_and_run_on_workers --------------------------------------------------------------------- @@ -284,7 +284,7 @@ CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE SELECT deparse_and_run_on_workers($cmd$ ALTER FUNCTION add SET log_min_messages FROM CURRENT $cmd$); -INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer, integer) SET log_min_messages FROM CURRENT; +INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer,integer) SET log_min_messages FROM CURRENT; CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE deparse_and_run_on_workers --------------------------------------------------------------------- @@ -295,7 +295,7 @@ CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE SELECT deparse_and_run_on_workers($cmd$ ALTER FUNCTION add(int, int) SET TIME ZONE INTERVAL '-08:00' HOUR TO MINUTE; $cmd$); -INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer, integer) SET TIME ZONE INTERVAL '@ 8 hours ago'; +INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer,integer) SET TIME ZONE INTERVAL '@ 8 hours ago'; CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE deparse_and_run_on_workers --------------------------------------------------------------------- @@ -306,7 +306,7 @@ CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE SELECT deparse_and_run_on_workers($cmd$ ALTER FUNCTION add(int, int) SET TIME ZONE '-7'; $cmd$); -INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer, integer) SET timezone = '-7'; +INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer,integer) SET timezone = '-7'; CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE deparse_and_run_on_workers --------------------------------------------------------------------- @@ -314,56 +314,10 @@ CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE (localhost,57638,t,"ALTER FUNCTION") (2 rows) -SELECT deparse_and_run_on_workers($cmd$ -ALTER FUNCTION add(int, int) SET "citus.setting;'" TO 'hello '' world'; -$cmd$); -INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer, integer) SET "citus.setting;'" = 'hello '' world'; -CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE - deparse_and_run_on_workers ---------------------------------------------------------------------- - (localhost,57637,t,"ALTER FUNCTION") - (localhost,57638,t,"ALTER FUNCTION") -(2 rows) - -SELECT deparse_and_run_on_workers($cmd$ -ALTER FUNCTION add(int, int) SET "citus.setting;'" TO -3.2; -$cmd$); -INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer, integer) SET "citus.setting;'" = -3.2; -CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE - deparse_and_run_on_workers ---------------------------------------------------------------------- - (localhost,57637,t,"ALTER FUNCTION") - (localhost,57638,t,"ALTER FUNCTION") -(2 rows) - -SELECT deparse_and_run_on_workers($cmd$ -ALTER FUNCTION add(int, int) SET "citus.setting;'" TO -32; -$cmd$); -INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer, integer) SET "citus.setting;'" = -32; -CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE - deparse_and_run_on_workers ---------------------------------------------------------------------- - (localhost,57637,t,"ALTER FUNCTION") - (localhost,57638,t,"ALTER FUNCTION") -(2 rows) - --- This raises an error about only accepting one item, --- that's okay, we're just testing that we don't produce bad syntax. -SELECT deparse_and_run_on_workers($cmd$ -ALTER FUNCTION add(int, int) SET "citus.setting;'" TO 'hello '' world', 'second '' item'; -$cmd$); -INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer, integer) SET "citus.setting;'" = 'hello '' world', 'second '' item'; -CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE - deparse_and_run_on_workers ---------------------------------------------------------------------- - (localhost,57637,f,"ERROR: SET citus.setting;' takes only one argument") - (localhost,57638,f,"ERROR: SET citus.setting;' takes only one argument") -(2 rows) - SELECT deparse_and_run_on_workers($cmd$ ALTER FUNCTION add RESET log_min_messages $cmd$); -INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer, integer) RESET log_min_messages; +INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer,integer) RESET log_min_messages; CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE deparse_and_run_on_workers --------------------------------------------------------------------- @@ -374,7 +328,7 @@ CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE SELECT deparse_and_run_on_workers($cmd$ ALTER FUNCTION add RESET ALL $cmd$); -INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer, integer) RESET ALL; +INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer,integer) RESET ALL; CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE deparse_and_run_on_workers --------------------------------------------------------------------- @@ -386,7 +340,7 @@ CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE SELECT deparse_and_run_on_workers($cmd$ ALTER FUNCTION add RENAME TO summation $cmd$); -INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer, integer) RENAME TO summation; +INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer,integer) RENAME TO summation; CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE deparse_and_run_on_workers --------------------------------------------------------------------- @@ -401,7 +355,7 @@ ALTER FUNCTION add RENAME TO summation; SELECT deparse_and_run_on_workers($cmd$ ALTER FUNCTION summation RENAME TO add $cmd$); -INFO: Propagating deparsed query: ALTER FUNCTION function_tests.summation(integer, integer) RENAME TO add; +INFO: Propagating deparsed query: ALTER FUNCTION function_tests.summation(integer,integer) RENAME TO add; CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE deparse_and_run_on_workers --------------------------------------------------------------------- @@ -424,7 +378,7 @@ SELECT run_command_on_workers('CREATE ROLE function_role'); SELECT deparse_and_run_on_workers($cmd$ ALTER FUNCTION add OWNER TO function_role $cmd$); -INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer, integer) OWNER TO function_role; +INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer,integer) OWNER TO function_role; CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE deparse_and_run_on_workers --------------------------------------------------------------------- @@ -435,7 +389,7 @@ CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE SELECT deparse_and_run_on_workers($cmd$ ALTER FUNCTION add OWNER TO missing_role $cmd$); -INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer, integer) OWNER TO missing_role; +INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer,integer) OWNER TO missing_role; CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE deparse_and_run_on_workers --------------------------------------------------------------------- @@ -447,7 +401,7 @@ CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE SELECT deparse_and_run_on_workers($cmd$ ALTER FUNCTION add SET SCHEMA public $cmd$); -INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer, integer) SET SCHEMA public; +INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer,integer) SET SCHEMA public; CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE deparse_and_run_on_workers --------------------------------------------------------------------- @@ -460,7 +414,7 @@ ALTER FUNCTION add SET SCHEMA public; SELECT deparse_and_run_on_workers($cmd$ ALTER FUNCTION public.add SET SCHEMA function_tests $cmd$); -INFO: Propagating deparsed query: ALTER FUNCTION public.add(integer, integer) SET SCHEMA function_tests; +INFO: Propagating deparsed query: ALTER FUNCTION public.add(integer,integer) SET SCHEMA function_tests; CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE deparse_and_run_on_workers --------------------------------------------------------------------- @@ -472,7 +426,7 @@ ALTER FUNCTION public.add SET SCHEMA function_tests; SELECT deparse_and_run_on_workers($cmd$ ALTER FUNCTION add DEPENDS ON EXTENSION citus $cmd$); -INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer, integer) DEPENDS ON EXTENSION citus; +INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer,integer) DEPENDS ON EXTENSION citus; CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE deparse_and_run_on_workers --------------------------------------------------------------------- @@ -484,7 +438,7 @@ CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE SELECT deparse_and_run_on_workers($cmd$ ALTER FUNCTION pg_catalog.get_shard_id_for_distribution_column(table_name regclass, distribution_value "any") PARALLEL SAFE; $cmd$); -INFO: Propagating deparsed query: ALTER FUNCTION pg_catalog.get_shard_id_for_distribution_column(pg_catalog.regclass, pg_catalog."any") PARALLEL SAFE; +INFO: Propagating deparsed query: ALTER FUNCTION pg_catalog.get_shard_id_for_distribution_column(pg_catalog.regclass,pg_catalog."any") PARALLEL SAFE; CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE deparse_and_run_on_workers --------------------------------------------------------------------- @@ -498,14 +452,14 @@ DROP FUNCTION add(int,int); $cmd$); deparse_test --------------------------------------------------------------------- - DROP FUNCTION function_tests.add(integer, integer); + DROP FUNCTION function_tests.add(integer,integer); (1 row) -- have multiple actions in a single query SELECT deparse_and_run_on_workers($cmd$ ALTER FUNCTION add volatile leakproof SECURITY DEFINER PARALLEL unsafe; $cmd$); -INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer, integer) VOLATILE LEAKPROOF SECURITY DEFINER PARALLEL UNSAFE; +INFO: Propagating deparsed query: ALTER FUNCTION function_tests.add(integer,integer) VOLATILE LEAKPROOF SECURITY DEFINER PARALLEL UNSAFE; CONTEXT: PL/pgSQL function deparse_and_run_on_workers(text) line XX at RAISE deparse_and_run_on_workers --------------------------------------------------------------------- diff --git a/src/test/regress/sql/distributed_functions.sql b/src/test/regress/sql/distributed_functions.sql index f98be1029..9eeb464ea 100644 --- a/src/test/regress/sql/distributed_functions.sql +++ b/src/test/regress/sql/distributed_functions.sql @@ -233,10 +233,6 @@ ALTER ROUTINE eq(macaddr,macaddr) SET client_min_messages TO debug; SELECT public.verify_function_is_same_on_workers('function_tests.eq(macaddr,macaddr)'); ALTER FUNCTION eq(macaddr,macaddr) RESET client_min_messages; SELECT public.verify_function_is_same_on_workers('function_tests.eq(macaddr,macaddr)'); -ALTER FUNCTION eq(macaddr,macaddr) SET "citus.setting;'" TO 'hello '' world'; -SELECT public.verify_function_is_same_on_workers('function_tests.eq(macaddr,macaddr)'); -ALTER FUNCTION eq(macaddr,macaddr) RESET "citus.setting;'"; -SELECT public.verify_function_is_same_on_workers('function_tests.eq(macaddr,macaddr)'); ALTER FUNCTION eq(macaddr,macaddr) SET search_path TO 'sch'';ma', public; SELECT public.verify_function_is_same_on_workers('function_tests.eq(macaddr,macaddr)'); ALTER FUNCTION eq(macaddr,macaddr) RESET search_path; @@ -318,9 +314,6 @@ DROP AGGREGATE function_tests2.sum2(int); -- call should fail as aggregate should have been dropped SELECT * FROM run_command_on_workers('SELECT function_tests2.sum2(id) FROM (select 1 id, 2) subq;') ORDER BY 1,2; --- postgres doesn't accept parameter names in the regprocedure input -SELECT create_distributed_function('eq_with_param_names(val1 macaddr, macaddr)', 'val1'); - -- invalid distribution_arg_name SELECT create_distributed_function('eq_with_param_names(macaddr, macaddr)', distribution_arg_name:='test'); SELECT create_distributed_function('eq_with_param_names(macaddr, macaddr)', distribution_arg_name:='int'); diff --git a/src/test/regress/sql/multi_deparse_function.sql b/src/test/regress/sql/multi_deparse_function.sql index 06de2c0fe..5a9355cc6 100644 --- a/src/test/regress/sql/multi_deparse_function.sql +++ b/src/test/regress/sql/multi_deparse_function.sql @@ -166,24 +166,6 @@ SELECT deparse_and_run_on_workers($cmd$ ALTER FUNCTION add(int, int) SET TIME ZONE '-7'; $cmd$); -SELECT deparse_and_run_on_workers($cmd$ -ALTER FUNCTION add(int, int) SET "citus.setting;'" TO 'hello '' world'; -$cmd$); - -SELECT deparse_and_run_on_workers($cmd$ -ALTER FUNCTION add(int, int) SET "citus.setting;'" TO -3.2; -$cmd$); - -SELECT deparse_and_run_on_workers($cmd$ -ALTER FUNCTION add(int, int) SET "citus.setting;'" TO -32; -$cmd$); - --- This raises an error about only accepting one item, --- that's okay, we're just testing that we don't produce bad syntax. -SELECT deparse_and_run_on_workers($cmd$ -ALTER FUNCTION add(int, int) SET "citus.setting;'" TO 'hello '' world', 'second '' item'; -$cmd$); - SELECT deparse_and_run_on_workers($cmd$ ALTER FUNCTION add RESET log_min_messages $cmd$); From 9b6ce1089231af661e1fe22ffc07f629e0ad163c Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Thu, 19 Aug 2021 18:12:00 +0300 Subject: [PATCH 061/104] Removes password outputs from alter_role_propagation tests --- .../expected/alter_role_propagation.out | 57 ++++++++++++++----- .../regress/sql/alter_role_propagation.sql | 12 ++-- 2 files changed, 50 insertions(+), 19 deletions(-) diff --git a/src/test/regress/expected/alter_role_propagation.out b/src/test/regress/expected/alter_role_propagation.out index 05072ddff..02b798e0b 100644 --- a/src/test/regress/expected/alter_role_propagation.out +++ b/src/test/regress/expected/alter_role_propagation.out @@ -2,11 +2,18 @@ CREATE SCHEMA alter_role; CREATE SCHEMA ",CitUs,.TeeN!?"; -- test if the passowrd of the extension owner can be upgraded ALTER ROLE CURRENT_USER PASSWORD 'password123' VALID UNTIL 'infinity'; -SELECT run_command_on_workers($$SELECT row(rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, EXTRACT (year FROM rolvaliduntil)) FROM pg_authid WHERE rolname = current_user$$); - run_command_on_workers +SELECT run_command_on_workers($$SELECT row(rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, EXTRACT (year FROM rolvaliduntil)) FROM pg_authid WHERE rolname = current_user$$); + run_command_on_workers --------------------------------------------------------------------- - (localhost,57637,t,"(postgres,t,t,t,t,t,t,t,-1,md5891d7d5079424b1cb973187d354d78de,Infinity)") - (localhost,57638,t,"(postgres,t,t,t,t,t,t,t,-1,md5891d7d5079424b1cb973187d354d78de,Infinity)") + (localhost,57637,t,"(postgres,t,t,t,t,t,t,t,-1,Infinity)") + (localhost,57638,t,"(postgres,t,t,t,t,t,t,t,-1,Infinity)") +(2 rows) + +SELECT workers.result = pg_authid.rolpassword AS password_is_same FROM run_command_on_workers($$SELECT rolpassword FROM pg_authid WHERE rolname = current_user$$) workers, pg_authid WHERE pg_authid.rolname = current_user; + password_is_same +--------------------------------------------------------------------- + t + t (2 rows) -- test if the password and some connection settings are propagated when a node gets added @@ -16,11 +23,18 @@ SELECT row(rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlog --------------------------------------------------------------------- (0 rows) -SELECT run_command_on_workers($$SELECT row(rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, EXTRACT (year FROM rolvaliduntil)) FROM pg_authid WHERE rolname = current_user$$); - run_command_on_workers +SELECT run_command_on_workers($$SELECT row(rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, EXTRACT (year FROM rolvaliduntil)) FROM pg_authid WHERE rolname = current_user$$); + run_command_on_workers --------------------------------------------------------------------- - (localhost,57637,t,"(postgres,t,t,t,t,t,t,t,66,md568701dc40be546e0357027fb0109338c,2032)") - (localhost,57638,t,"(postgres,t,t,t,t,t,t,t,66,md568701dc40be546e0357027fb0109338c,2032)") + (localhost,57637,t,"(postgres,t,t,t,t,t,t,t,66,2032)") + (localhost,57638,t,"(postgres,t,t,t,t,t,t,t,66,2032)") +(2 rows) + +SELECT workers.result = pg_authid.rolpassword AS password_is_same FROM run_command_on_workers($$SELECT rolpassword FROM pg_authid WHERE rolname = current_user$$) workers, pg_authid WHERE pg_authid.rolname = current_user; + password_is_same +--------------------------------------------------------------------- + t + t (2 rows) SELECT master_remove_node('localhost', :worker_1_port); @@ -35,10 +49,16 @@ SELECT row(rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlog --------------------------------------------------------------------- (0 rows) -SELECT run_command_on_workers($$SELECT row(rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, EXTRACT (year FROM rolvaliduntil)) FROM pg_authid WHERE rolname = current_user$$); - run_command_on_workers +SELECT run_command_on_workers($$SELECT row(rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, EXTRACT (year FROM rolvaliduntil)) FROM pg_authid WHERE rolname = current_user$$); + run_command_on_workers --------------------------------------------------------------------- - (localhost,57638,t,"(postgres,t,t,t,t,t,t,t,0,md53e559cc1fcf0c70f1f8e05c9a79c3133,2052)") + (localhost,57638,t,"(postgres,t,t,t,t,t,t,t,0,2052)") +(1 row) + +SELECT workers.result = pg_authid.rolpassword AS password_is_same FROM run_command_on_workers($$SELECT rolpassword FROM pg_authid WHERE rolname = current_user$$) workers, pg_authid WHERE pg_authid.rolname = current_user; + password_is_same +--------------------------------------------------------------------- + t (1 row) SELECT 1 FROM master_add_node('localhost', :worker_1_port); @@ -52,11 +72,18 @@ SELECT row(rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlog --------------------------------------------------------------------- (0 rows) -SELECT run_command_on_workers($$SELECT row(rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, EXTRACT (year FROM rolvaliduntil)) FROM pg_authid WHERE rolname = current_user$$); - run_command_on_workers +SELECT run_command_on_workers($$SELECT row(rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, EXTRACT (year FROM rolvaliduntil)) FROM pg_authid WHERE rolname = current_user$$); + run_command_on_workers --------------------------------------------------------------------- - (localhost,57637,t,"(postgres,t,t,t,t,t,t,t,0,md53e559cc1fcf0c70f1f8e05c9a79c3133,2052)") - (localhost,57638,t,"(postgres,t,t,t,t,t,t,t,0,md53e559cc1fcf0c70f1f8e05c9a79c3133,2052)") + (localhost,57637,t,"(postgres,t,t,t,t,t,t,t,0,2052)") + (localhost,57638,t,"(postgres,t,t,t,t,t,t,t,0,2052)") +(2 rows) + +SELECT workers.result = pg_authid.rolpassword AS password_is_same FROM run_command_on_workers($$SELECT rolpassword FROM pg_authid WHERE rolname = current_user$$) workers, pg_authid WHERE pg_authid.rolname = current_user; + password_is_same +--------------------------------------------------------------------- + t + t (2 rows) -- check user, database and postgres wide SET settings. diff --git a/src/test/regress/sql/alter_role_propagation.sql b/src/test/regress/sql/alter_role_propagation.sql index e42cde447..627b0a1f2 100644 --- a/src/test/regress/sql/alter_role_propagation.sql +++ b/src/test/regress/sql/alter_role_propagation.sql @@ -3,19 +3,23 @@ CREATE SCHEMA ",CitUs,.TeeN!?"; -- test if the passowrd of the extension owner can be upgraded ALTER ROLE CURRENT_USER PASSWORD 'password123' VALID UNTIL 'infinity'; -SELECT run_command_on_workers($$SELECT row(rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, EXTRACT (year FROM rolvaliduntil)) FROM pg_authid WHERE rolname = current_user$$); +SELECT run_command_on_workers($$SELECT row(rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, EXTRACT (year FROM rolvaliduntil)) FROM pg_authid WHERE rolname = current_user$$); +SELECT workers.result = pg_authid.rolpassword AS password_is_same FROM run_command_on_workers($$SELECT rolpassword FROM pg_authid WHERE rolname = current_user$$) workers, pg_authid WHERE pg_authid.rolname = current_user; -- test if the password and some connection settings are propagated when a node gets added ALTER ROLE CURRENT_USER WITH CONNECTION LIMIT 66 VALID UNTIL '2032-05-05' PASSWORD 'password456'; SELECT row(rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, EXTRACT (year FROM rolvaliduntil)) FROM pg_authid WHERE rolname = 'alter_role_1'; -SELECT run_command_on_workers($$SELECT row(rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, EXTRACT (year FROM rolvaliduntil)) FROM pg_authid WHERE rolname = current_user$$); +SELECT run_command_on_workers($$SELECT row(rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, EXTRACT (year FROM rolvaliduntil)) FROM pg_authid WHERE rolname = current_user$$); +SELECT workers.result = pg_authid.rolpassword AS password_is_same FROM run_command_on_workers($$SELECT rolpassword FROM pg_authid WHERE rolname = current_user$$) workers, pg_authid WHERE pg_authid.rolname = current_user; SELECT master_remove_node('localhost', :worker_1_port); ALTER ROLE CURRENT_USER WITH CONNECTION LIMIT 0 VALID UNTIL '2052-05-05' PASSWORD 'password789'; SELECT row(rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, EXTRACT (year FROM rolvaliduntil)) FROM pg_authid WHERE rolname = 'alter_role_1'; -SELECT run_command_on_workers($$SELECT row(rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, EXTRACT (year FROM rolvaliduntil)) FROM pg_authid WHERE rolname = current_user$$); +SELECT run_command_on_workers($$SELECT row(rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, EXTRACT (year FROM rolvaliduntil)) FROM pg_authid WHERE rolname = current_user$$); +SELECT workers.result = pg_authid.rolpassword AS password_is_same FROM run_command_on_workers($$SELECT rolpassword FROM pg_authid WHERE rolname = current_user$$) workers, pg_authid WHERE pg_authid.rolname = current_user; SELECT 1 FROM master_add_node('localhost', :worker_1_port); SELECT row(rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, EXTRACT (year FROM rolvaliduntil)) FROM pg_authid WHERE rolname = 'alter_role_1'; -SELECT run_command_on_workers($$SELECT row(rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, EXTRACT (year FROM rolvaliduntil)) FROM pg_authid WHERE rolname = current_user$$); +SELECT run_command_on_workers($$SELECT row(rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, EXTRACT (year FROM rolvaliduntil)) FROM pg_authid WHERE rolname = current_user$$); +SELECT workers.result = pg_authid.rolpassword AS password_is_same FROM run_command_on_workers($$SELECT rolpassword FROM pg_authid WHERE rolname = current_user$$) workers, pg_authid WHERE pg_authid.rolname = current_user; -- check user, database and postgres wide SET settings. -- pre check From 79d1b7d50b5846fd79a7a4883751ff72d20cae16 Mon Sep 17 00:00:00 2001 From: Nils Dijk Date: Thu, 19 Aug 2021 17:25:07 +0200 Subject: [PATCH 062/104] add 14beta3 to CI --- .circleci/config.yml | 81 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 098196b84..5b0a6a746 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -600,6 +600,80 @@ workflows: make: check-failure requires: [build-13] + - test-citus: + name: 'test-14_check-multi' + pg_major: 14 + image_tag: '14beta3-dev202108191715' + make: check-multi + requires: [build-14] + - test-citus: + name: 'test-14_check-multi-1' + pg_major: 14 + image_tag: '14beta3-dev202108191715' + make: check-multi-1 + requires: [build-14] + - test-citus: + name: 'test-14_check-mx' + pg_major: 14 + image_tag: '14beta3-dev202108191715' + make: check-multi-mx + requires: [build-14] + - test-citus: + name: 'test-14_check-vanilla' + pg_major: 14 + image_tag: '14beta3-dev202108191715' + make: check-vanilla + requires: [build-14] + - test-citus: + name: 'test-14_check-isolation' + pg_major: 14 + image_tag: '14beta3-dev202108191715' + make: check-isolation + requires: [build-14] + - test-citus: + name: 'test-14_check-worker' + pg_major: 14 + image_tag: '14beta3-dev202108191715' + make: check-worker + requires: [build-14] + - test-citus: + name: 'test-14_check-operations' + pg_major: 14 + image_tag: '14beta3-dev202108191715' + make: check-operations + requires: [build-14] + - test-citus: + name: 'test-14_check-follower-cluster' + pg_major: 14 + image_tag: '14beta3-dev202108191715' + make: check-follower-cluster + requires: [build-14] + - test-citus: + name: 'test-14_check-columnar' + pg_major: 14 + image_tag: '14beta3-dev202108191715' + make: check-columnar + requires: [build-14] + - test-citus: + name: 'test-14_check-columnar-isolation' + pg_major: 14 + image_tag: '14beta3-dev202108191715' + make: check-columnar-isolation + requires: [build-14] + - tap-test-citus: + name: 'test_14_tap-recovery' + pg_major: 14 + image_tag: '14beta3-dev202108191715' + suite: recovery + requires: [build-14] + - test-citus: + name: 'test-14_check-failure' + pg_major: 14 + image: citus/failtester + image_tag: '14beta3-dev202108191715' + make: check-failure + requires: [build-14] + - test-pg-upgrade: name: 'test-12-13_check-pg-upgrade' old_pg_major: 12 @@ -607,6 +681,13 @@ workflows: image_tag: '12.8-13.4' requires: [build-12,build-13] + - test-pg-upgrade: + name: 'test-13-14_check-pg-upgrade' + old_pg_major: 13 + new_pg_major: 14 + image_tag: 12-13-14 + requires: [build-13,build-14] + - test-citus-upgrade: name: test-12_check-citus-upgrade pg_major: 12 From f16d5e18333886bc2c0fefa732cf792e442fa9dd Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Thu, 19 Aug 2021 19:01:26 +0300 Subject: [PATCH 063/104] Introduces make_simple_restrictinfo_compat and pull_varnos_compat macros make_simple_restrictinfo and pull_varnos functions now have a new parameter These new macros give us the ability to use this new parameter for PG14 and they don't give the parameter for previous versions Relevant PG commit: 55dc86eca70b1dc18a79c141b3567efed910329d --- src/include/distributed/version_compat.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index fba6cfe99..8d5e8d663 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -60,6 +60,8 @@ #define pgproc_statusflags_compat(pgproc) ((pgproc)->statusFlags) #define get_partition_parent_compat(a, b) get_partition_parent(a, b) #define RelationGetPartitionDesc_compat(a, b) RelationGetPartitionDesc(a, b) +#define make_simple_restrictinfo_compat(a, b) make_simple_restrictinfo(a, b) +#define pull_varnos_compat(a, b) pull_varnos(a, b) #else #define AlterTableStmtObjType(a) ((a)->relkind) #define F_NEXTVAL_COMPAT F_NEXTVAL_OID @@ -94,6 +96,8 @@ #define get_partition_parent_compat(a, b) get_partition_parent(a) #define RelationGetPartitionDesc_compat(a, b) RelationGetPartitionDesc(a) #define PQ_LARGE_MESSAGE_LIMIT 0 +#define make_simple_restrictinfo_compat(a, b) make_simple_restrictinfo(b) +#define pull_varnos_compat(a, b) pull_varnos(b) #endif #if PG_VERSION_NUM >= PG_VERSION_13 From 7823e492194ac518aa62551fadd7c948e1e31d59 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Thu, 19 Aug 2021 19:33:10 +0300 Subject: [PATCH 064/104] Introduces pg_get_statisticsobj_worker_compat macro Relevant PG commit: a4d75c86bf15220df22de0a92c819ecef9db3849 --- src/backend/distributed/commands/statistics.c | 4 ++-- src/include/distributed/citus_ruleutils.h | 4 ++++ src/include/distributed/version_compat.h | 2 ++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/backend/distributed/commands/statistics.c b/src/backend/distributed/commands/statistics.c index 371b6c098..ababdd1a8 100644 --- a/src/backend/distributed/commands/statistics.c +++ b/src/backend/distributed/commands/statistics.c @@ -442,8 +442,8 @@ GetExplicitStatisticsCommandList(Oid relationId) foreach_oid(statisticsId, statisticsIdList) { /* we need create commands for already created stats before distribution */ - char *createStatisticsCommand = pg_get_statisticsobj_worker(statisticsId, - false, false); + char *createStatisticsCommand = pg_get_statisticsobj_worker_compat(statisticsId, + false, false); explicitStatisticsCommandList = lappend(explicitStatisticsCommandList, diff --git a/src/include/distributed/citus_ruleutils.h b/src/include/distributed/citus_ruleutils.h index 2bb008c5e..fac19352d 100644 --- a/src/include/distributed/citus_ruleutils.h +++ b/src/include/distributed/citus_ruleutils.h @@ -50,8 +50,12 @@ char * pg_get_rule_expr(Node *expression); extern void deparse_shard_query(Query *query, Oid distrelid, int64 shardid, StringInfo buffer); extern char * pg_get_triggerdef_command(Oid triggerId); +#if PG_VERSION_NUM >= PG_VERSION_14 extern char * pg_get_statisticsobj_worker(Oid statextid, bool columns_only, bool missing_ok); +#else +extern char * pg_get_statisticsobj_worker(Oid statextid, bool missing_ok); +#endif extern char * generate_relation_name(Oid relid, List *namespaces); extern char * generate_qualified_relation_name(Oid relid); extern char * generate_operator_name(Oid operid, Oid arg1, Oid arg2); diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index 8d5e8d663..1bbdd4334 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -62,6 +62,7 @@ #define RelationGetPartitionDesc_compat(a, b) RelationGetPartitionDesc(a, b) #define make_simple_restrictinfo_compat(a, b) make_simple_restrictinfo(a, b) #define pull_varnos_compat(a, b) pull_varnos(a, b) +#define pg_get_statisticsobj_worker_compat(a, b, c) pg_get_statisticsobj_worker(a, b, c) #else #define AlterTableStmtObjType(a) ((a)->relkind) #define F_NEXTVAL_COMPAT F_NEXTVAL_OID @@ -98,6 +99,7 @@ #define PQ_LARGE_MESSAGE_LIMIT 0 #define make_simple_restrictinfo_compat(a, b) make_simple_restrictinfo(b) #define pull_varnos_compat(a, b) pull_varnos(b) +#define pg_get_statisticsobj_worker_compat(a, b, c) pg_get_statisticsobj_worker(a, c) #endif #if PG_VERSION_NUM >= PG_VERSION_13 From ca0d4c3bde425689a5ff826480adc9aa226ccb86 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Thu, 19 Aug 2021 21:42:32 +0300 Subject: [PATCH 065/104] Includes pg_version_constants.h in columnar_version_compat.h --- src/include/columnar/columnar_version_compat.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/include/columnar/columnar_version_compat.h b/src/include/columnar/columnar_version_compat.h index 6b503b73f..45b8a0e55 100644 --- a/src/include/columnar/columnar_version_compat.h +++ b/src/include/columnar/columnar_version_compat.h @@ -12,6 +12,8 @@ #ifndef COLUMNAR_COMPAT_H #define COLUMNAR_COMPAT_H +#include "distributed/pg_version_constants.h" + #if PG_VERSION_NUM >= PG_VERSION_14 #define ColumnarProcessUtility_compat(a, b, c, d, e, f, g, h) \ ColumnarProcessUtility(a, b, c, d, e, f, g, h) From 375a1adc9ef41ba73160bbc467d3c68e25a9cce8 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Fri, 20 Aug 2021 09:16:44 +0300 Subject: [PATCH 066/104] Check if extversion is the same for seg extension When we check the exact version of the seg extension, it becomes a problem when its version changes, such as from 1.3 to 1.4. So now we modified the changes to check for that the version is the same in all the cluster. --- .../expected/propagate_extension_commands.out | 26 ++++++++++--------- .../sql/propagate_extension_commands.sql | 9 ++++--- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/src/test/regress/expected/propagate_extension_commands.out b/src/test/regress/expected/propagate_extension_commands.out index 969a52b97..b7e0618e3 100644 --- a/src/test/regress/expected/propagate_extension_commands.out +++ b/src/test/regress/expected/propagate_extension_commands.out @@ -161,10 +161,11 @@ SELECT run_command_on_workers($$SELECT count(extnamespace) FROM pg_extension WHE (localhost,57637,t,1) (1 row) -SELECT run_command_on_workers($$SELECT extversion FROM pg_extension WHERE extname = 'seg'$$); - run_command_on_workers +SELECT workers.result = pg_extension.extversion AS same_version + FROM run_command_on_workers($$SELECT extversion FROM pg_extension WHERE extname = 'seg'$$) workers, pg_extension WHERE extname = 'seg'; + same_version --------------------------------------------------------------------- - (localhost,57637,t,1.3) + t (1 row) -- now create the reference table @@ -254,11 +255,12 @@ SELECT run_command_on_workers($$SELECT count(extnamespace) FROM pg_extension WHE (localhost,57638,t,1) (2 rows) -SELECT run_command_on_workers($$SELECT extversion FROM pg_extension WHERE extname = 'seg'$$); - run_command_on_workers +SELECT workers.result = pg_extension.extversion AS same_version + FROM run_command_on_workers($$SELECT extversion FROM pg_extension WHERE extname = 'seg'$$) workers, pg_extension WHERE extname = 'seg'; + same_version --------------------------------------------------------------------- - (localhost,57637,t,1.3) - (localhost,57638,t,1.3) + t + t (2 rows) -- check for the unpackaged extension to be created correctly @@ -376,12 +378,12 @@ BEGIN; ROLLBACK; -- show that the CREATE EXTENSION command propagated even if the transaction -- block is rollbacked, that's a shortcoming of dependency creation logic -SELECT run_command_on_workers($$SELECT extversion FROM pg_extension WHERE extname = 'seg'$$); - run_command_on_workers +SELECT COUNT(DISTINCT workers.result) + FROM run_command_on_workers($$SELECT extversion FROM pg_extension WHERE extname = 'seg'$$) workers; + count --------------------------------------------------------------------- - (localhost,57637,t,1.3) - (localhost,57638,t,1.3) -(2 rows) + 1 +(1 row) -- drop the schema and all the objects DROP SCHEMA "extension'test" CASCADE; diff --git a/src/test/regress/sql/propagate_extension_commands.sql b/src/test/regress/sql/propagate_extension_commands.sql index ad7ba749b..0ba17e293 100644 --- a/src/test/regress/sql/propagate_extension_commands.sql +++ b/src/test/regress/sql/propagate_extension_commands.sql @@ -98,7 +98,8 @@ CREATE EXTENSION seg; -- show that the extension is created on existing worker SELECT run_command_on_workers($$SELECT count(extnamespace) FROM pg_extension WHERE extname = 'seg'$$); -SELECT run_command_on_workers($$SELECT extversion FROM pg_extension WHERE extname = 'seg'$$); +SELECT workers.result = pg_extension.extversion AS same_version + FROM run_command_on_workers($$SELECT extversion FROM pg_extension WHERE extname = 'seg'$$) workers, pg_extension WHERE extname = 'seg'; -- now create the reference table CREATE TABLE ref_table_2 (x seg); @@ -144,7 +145,8 @@ SELECT 1 from master_add_node('localhost', :worker_2_port); -- show that the extension is created on both existing and new node SELECT run_command_on_workers($$SELECT count(extnamespace) FROM pg_extension WHERE extname = 'seg'$$); -SELECT run_command_on_workers($$SELECT extversion FROM pg_extension WHERE extname = 'seg'$$); +SELECT workers.result = pg_extension.extversion AS same_version + FROM run_command_on_workers($$SELECT extversion FROM pg_extension WHERE extname = 'seg'$$) workers, pg_extension WHERE extname = 'seg'; -- check for the unpackaged extension to be created correctly SELECT run_command_on_workers($$SELECT count(extnamespace) FROM pg_extension WHERE extname = 'dict_int'$$); @@ -210,7 +212,8 @@ ROLLBACK; -- show that the CREATE EXTENSION command propagated even if the transaction -- block is rollbacked, that's a shortcoming of dependency creation logic -SELECT run_command_on_workers($$SELECT extversion FROM pg_extension WHERE extname = 'seg'$$); +SELECT COUNT(DISTINCT workers.result) + FROM run_command_on_workers($$SELECT extversion FROM pg_extension WHERE extname = 'seg'$$) workers; -- drop the schema and all the objects DROP SCHEMA "extension'test" CASCADE; From 6b65dbc492d937a37b536fc8555881328f52ebbb Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Fri, 20 Aug 2021 09:38:41 +0300 Subject: [PATCH 067/104] Add partition_wise_join to avoid big alternative output There was a small part in multi_partitioning that would need an alternative output for pg14. Instead of adding an alternative for the whole file, we created a new file, called partition_wise_join.sql and added the alternative output for that. --- .../regress/expected/multi_partitioning.out | 192 +--------------- .../regress/expected/partition_wise_join.out | 208 ++++++++++++++++++ .../expected/partition_wise_join_0.out | 208 ++++++++++++++++++ src/test/regress/multi_1_schedule | 1 + src/test/regress/sql/multi_partitioning.sql | 45 ---- src/test/regress/sql/partition_wise_join.sql | 61 +++++ 6 files changed, 479 insertions(+), 236 deletions(-) create mode 100644 src/test/regress/expected/partition_wise_join.out create mode 100644 src/test/regress/expected/partition_wise_join_0.out create mode 100644 src/test/regress/sql/partition_wise_join.sql diff --git a/src/test/regress/expected/multi_partitioning.out b/src/test/regress/expected/multi_partitioning.out index 310436b41..9980b0f85 100644 --- a/src/test/regress/expected/multi_partitioning.out +++ b/src/test/regress/expected/multi_partitioning.out @@ -1616,194 +1616,6 @@ SELECT * FROM lockinfo; (20 rows) COMMIT; --- test partition-wise join -CREATE TABLE partitioning_hash_join_test(id int, subid int) PARTITION BY HASH(subid); -CREATE TABLE partitioning_hash_join_test_0 PARTITION OF partitioning_hash_join_test FOR VALUES WITH (MODULUS 3, REMAINDER 0); -CREATE TABLE partitioning_hash_join_test_1 PARTITION OF partitioning_hash_join_test FOR VALUES WITH (MODULUS 3, REMAINDER 1); -CREATE TABLE partitioning_hash_join_test_2 PARTITION OF partitioning_hash_join_test FOR VALUES WITH (MODULUS 3, REMAINDER 2); -SELECT create_distributed_table('partitioning_hash_join_test', 'id'); - create_distributed_table ---------------------------------------------------------------------- - -(1 row) - -SELECT success FROM run_command_on_workers('alter system set enable_mergejoin to off'); - success ---------------------------------------------------------------------- - t - t -(2 rows) - -SELECT success FROM run_command_on_workers('alter system set enable_nestloop to off'); - success ---------------------------------------------------------------------- - t - t -(2 rows) - -SELECT success FROM run_command_on_workers('alter system set enable_indexscan to off'); - success ---------------------------------------------------------------------- - t - t -(2 rows) - -SELECT success FROM run_command_on_workers('alter system set enable_indexonlyscan to off'); - success ---------------------------------------------------------------------- - t - t -(2 rows) - -SELECT success FROM run_command_on_workers('alter system set enable_partitionwise_join to off'); - success ---------------------------------------------------------------------- - t - t -(2 rows) - -SELECT success FROM run_command_on_workers('select pg_reload_conf()'); - success ---------------------------------------------------------------------- - t - t -(2 rows) - -EXPLAIN (COSTS OFF) -SELECT * FROM partitioning_hash_test JOIN partitioning_hash_join_test USING (id, subid); - QUERY PLAN ---------------------------------------------------------------------- - Custom Scan (Citus Adaptive) - Task Count: 4 - Tasks Shown: One of 4 - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Hash Join - Hash Cond: ((partitioning_hash_join_test_xxx.id = partitioning_hash_test_xxx.id) AND (partitioning_hash_join_test_xxx.subid = partitioning_hash_test_xxx.subid)) - -> Append - -> Seq Scan on partitioning_hash_join_test_0_1660133 partitioning_hash_join_test_xxx - -> Seq Scan on partitioning_hash_join_test_1_1660137 partitioning_hash_join_test_xxx - -> Seq Scan on partitioning_hash_join_test_2_1660141 partitioning_hash_join_test_xxx - -> Hash - -> Append - -> Seq Scan on partitioning_hash_test_0_1660016 partitioning_hash_test_xxx - -> Seq Scan on partitioning_hash_test_1_1660020 partitioning_hash_test_xxx - -> Seq Scan on partitioning_hash_test_2_1660032 partitioning_hash_test_xxx -(16 rows) - --- set partition-wise join on and parallel to off -SELECT success FROM run_command_on_workers('alter system set enable_partitionwise_join to on'); - success ---------------------------------------------------------------------- - t - t -(2 rows) - -SELECT success FROM run_command_on_workers('select pg_reload_conf()'); - success ---------------------------------------------------------------------- - t - t -(2 rows) - -SET enable_partitionwise_join TO on; -ANALYZE partitioning_hash_test, partitioning_hash_join_test; -EXPLAIN (COSTS OFF) -SELECT * FROM partitioning_hash_test JOIN partitioning_hash_join_test USING (id, subid); - QUERY PLAN ---------------------------------------------------------------------- - Custom Scan (Citus Adaptive) - Task Count: 4 - Tasks Shown: One of 4 - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Append - -> Hash Join - Hash Cond: ((partitioning_hash_join_test_xxx.id = partitioning_hash_test_xxx.id) AND (partitioning_hash_join_test_xxx.subid = partitioning_hash_test_xxx.subid)) - -> Seq Scan on partitioning_hash_join_test_0_1660133 partitioning_hash_join_test_xxx - -> Hash - -> Seq Scan on partitioning_hash_test_0_1660016 partitioning_hash_test_xxx - -> Hash Join - Hash Cond: ((partitioning_hash_test_xxx.id = partitioning_hash_join_test_xxx.id) AND (partitioning_hash_test_xxx.subid = partitioning_hash_join_test_xxx.subid)) - -> Seq Scan on partitioning_hash_test_1_1660020 partitioning_hash_test_xxx - -> Hash - -> Seq Scan on partitioning_hash_join_test_1_1660137 partitioning_hash_join_test_xxx - -> Hash Join - Hash Cond: ((partitioning_hash_join_test_xxx.id = partitioning_hash_test_xxx.id) AND (partitioning_hash_join_test_xxx.subid = partitioning_hash_test_xxx.subid)) - -> Seq Scan on partitioning_hash_join_test_2_1660141 partitioning_hash_join_test_xxx - -> Hash - -> Seq Scan on partitioning_hash_test_2_1660032 partitioning_hash_test_xxx -(21 rows) - --- note that partition-wise joins only work when partition key is in the join --- following join does not have that, therefore join will not be pushed down to --- partitions -EXPLAIN (COSTS OFF) -SELECT * FROM partitioning_hash_test JOIN partitioning_hash_join_test USING (id); - QUERY PLAN ---------------------------------------------------------------------- - Custom Scan (Citus Adaptive) - Task Count: 4 - Tasks Shown: One of 4 - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Hash Join - Hash Cond: (partitioning_hash_join_test_xxx.id = partitioning_hash_test_xxx.id) - -> Append - -> Seq Scan on partitioning_hash_join_test_0_1660133 partitioning_hash_join_test_xxx - -> Seq Scan on partitioning_hash_join_test_1_1660137 partitioning_hash_join_test_xxx - -> Seq Scan on partitioning_hash_join_test_2_1660141 partitioning_hash_join_test_xxx - -> Hash - -> Append - -> Seq Scan on partitioning_hash_test_0_1660016 partitioning_hash_test_xxx - -> Seq Scan on partitioning_hash_test_1_1660020 partitioning_hash_test_xxx - -> Seq Scan on partitioning_hash_test_2_1660032 partitioning_hash_test_xxx -(16 rows) - --- reset partition-wise join -SELECT success FROM run_command_on_workers('alter system reset enable_partitionwise_join'); - success ---------------------------------------------------------------------- - t - t -(2 rows) - -SELECT success FROM run_command_on_workers('alter system reset enable_mergejoin'); - success ---------------------------------------------------------------------- - t - t -(2 rows) - -SELECT success FROM run_command_on_workers('alter system reset enable_nestloop'); - success ---------------------------------------------------------------------- - t - t -(2 rows) - -SELECT success FROM run_command_on_workers('alter system reset enable_indexscan'); - success ---------------------------------------------------------------------- - t - t -(2 rows) - -SELECT success FROM run_command_on_workers('alter system reset enable_indexonlyscan'); - success ---------------------------------------------------------------------- - t - t -(2 rows) - -SELECT success FROM run_command_on_workers('select pg_reload_conf()'); - success ---------------------------------------------------------------------- - t - t -(2 rows) - -RESET enable_partitionwise_join; DROP VIEW lockinfo; DROP TABLE IF EXISTS @@ -2069,10 +1881,9 @@ SELECT fix_pre_citus10_partitioned_table_constraint_names(); --------------------------------------------------------------------- partitioning_test "schema-test" - public.partitioning_hash_join_test public.partitioning_hash_test public.partitioning_test_failure -(5 rows) +(4 rows) -- the following should fail SELECT fix_pre_citus10_partitioned_table_constraint_names('public.non_distributed_partitioned_table'); @@ -2248,7 +2059,6 @@ drop cascades to table distributed_parent_table RESET search_path; DROP TABLE IF EXISTS partitioning_hash_test, - partitioning_hash_join_test, partitioning_test_failure, non_distributed_partitioned_table, partitioning_test_foreign_key; diff --git a/src/test/regress/expected/partition_wise_join.out b/src/test/regress/expected/partition_wise_join.out new file mode 100644 index 000000000..1b00fcef3 --- /dev/null +++ b/src/test/regress/expected/partition_wise_join.out @@ -0,0 +1,208 @@ +CREATE SCHEMA partition_wise_join; +SET search_path to partition_wise_join; +CREATE TABLE partitioning_hash_test(id int, subid int) PARTITION BY HASH(subid); +CREATE TABLE partitioning_hash_test_0 PARTITION OF partitioning_hash_test FOR VALUES WITH (MODULUS 3, REMAINDER 0); +CREATE TABLE partitioning_hash_test_1 PARTITION OF partitioning_hash_test FOR VALUES WITH (MODULUS 3, REMAINDER 1); +INSERT INTO partitioning_hash_test VALUES (1, 2); +INSERT INTO partitioning_hash_test VALUES (2, 13); +INSERT INTO partitioning_hash_test VALUES (3, 7); +INSERT INTO partitioning_hash_test VALUES (4, 4); +-- distribute partitioned table +SELECT create_distributed_table('partitioning_hash_test', 'id'); +NOTICE: Copying data from local table... +NOTICE: copying the data has completed +DETAIL: The local data in the table is no longer visible, but is still on disk. +HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$partition_wise_join.partitioning_hash_test_0$$) +NOTICE: Copying data from local table... +NOTICE: copying the data has completed +DETAIL: The local data in the table is no longer visible, but is still on disk. +HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$partition_wise_join.partitioning_hash_test_1$$) + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- test partition-wise join +CREATE TABLE partitioning_hash_join_test(id int, subid int) PARTITION BY HASH(subid); +CREATE TABLE partitioning_hash_join_test_0 PARTITION OF partitioning_hash_join_test FOR VALUES WITH (MODULUS 3, REMAINDER 0); +CREATE TABLE partitioning_hash_join_test_1 PARTITION OF partitioning_hash_join_test FOR VALUES WITH (MODULUS 3, REMAINDER 1); +CREATE TABLE partitioning_hash_join_test_2 PARTITION OF partitioning_hash_join_test FOR VALUES WITH (MODULUS 3, REMAINDER 2); +SELECT create_distributed_table('partitioning_hash_join_test', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT success FROM run_command_on_workers('alter system set enable_mergejoin to off'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +SELECT success FROM run_command_on_workers('alter system set enable_nestloop to off'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +SELECT success FROM run_command_on_workers('alter system set enable_indexscan to off'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +SELECT success FROM run_command_on_workers('alter system set enable_indexonlyscan to off'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +SELECT success FROM run_command_on_workers('alter system set enable_partitionwise_join to off'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +SELECT success FROM run_command_on_workers('select pg_reload_conf()'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +EXPLAIN (COSTS OFF) +SELECT * FROM partitioning_hash_test JOIN partitioning_hash_join_test USING (id, subid); + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Hash Join + Hash Cond: ((partitioning_hash_join_test_xxx.id = partitioning_hash_test_xxx.id) AND (partitioning_hash_join_test_xxx.subid = partitioning_hash_test_xxx.subid)) + -> Append + -> Seq Scan on partitioning_hash_join_test_0_360163 partitioning_hash_join_test_xxx + -> Seq Scan on partitioning_hash_join_test_1_360167 partitioning_hash_join_test_xxx + -> Seq Scan on partitioning_hash_join_test_2_360171 partitioning_hash_join_test_xxx + -> Hash + -> Append + -> Seq Scan on partitioning_hash_test_0_360151 partitioning_hash_test_xxx + -> Seq Scan on partitioning_hash_test_1_360155 partitioning_hash_test_xxx +(15 rows) + +-- set partition-wise join on and parallel to off +SELECT success FROM run_command_on_workers('alter system set enable_partitionwise_join to on'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +SELECT success FROM run_command_on_workers('select pg_reload_conf()'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +-- SET enable_partitionwise_join TO on; +ANALYZE partitioning_hash_test, partitioning_hash_join_test; +EXPLAIN (COSTS OFF) +SELECT * FROM partitioning_hash_test JOIN partitioning_hash_join_test USING (id, subid); + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Hash Join + Hash Cond: ((partitioning_hash_test_xxx.id = partitioning_hash_join_test_xxx.id) AND (partitioning_hash_test_xxx.subid = partitioning_hash_join_test_xxx.subid)) + -> Append + -> Seq Scan on partitioning_hash_test_0_360151 partitioning_hash_test_xxx + -> Seq Scan on partitioning_hash_test_1_360155 partitioning_hash_test_xxx + -> Hash + -> Append + -> Seq Scan on partitioning_hash_join_test_0_360163 partitioning_hash_join_test_xxx + -> Seq Scan on partitioning_hash_join_test_1_360167 partitioning_hash_join_test_xxx + -> Seq Scan on partitioning_hash_join_test_2_360171 partitioning_hash_join_test_xxx +(15 rows) + +-- note that partition-wise joins only work when partition key is in the join +-- following join does not have that, therefore join will not be pushed down to +-- partitions +EXPLAIN (COSTS OFF) +SELECT * FROM partitioning_hash_test JOIN partitioning_hash_join_test USING (id); + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Hash Join + Hash Cond: (partitioning_hash_test_xxx.id = partitioning_hash_join_test_xxx.id) + -> Append + -> Seq Scan on partitioning_hash_test_0_360151 partitioning_hash_test_xxx + -> Seq Scan on partitioning_hash_test_1_360155 partitioning_hash_test_xxx + -> Hash + -> Append + -> Seq Scan on partitioning_hash_join_test_0_360163 partitioning_hash_join_test_xxx + -> Seq Scan on partitioning_hash_join_test_1_360167 partitioning_hash_join_test_xxx + -> Seq Scan on partitioning_hash_join_test_2_360171 partitioning_hash_join_test_xxx +(15 rows) + +-- reset partition-wise join +SELECT success FROM run_command_on_workers('alter system reset enable_partitionwise_join'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +SELECT success FROM run_command_on_workers('alter system reset enable_mergejoin'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +SELECT success FROM run_command_on_workers('alter system reset enable_nestloop'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +SELECT success FROM run_command_on_workers('alter system reset enable_indexscan'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +SELECT success FROM run_command_on_workers('alter system reset enable_indexonlyscan'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +SELECT success FROM run_command_on_workers('select pg_reload_conf()'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +RESET enable_partitionwise_join; +DROP SCHEMA partition_wise_join CASCADE; +NOTICE: drop cascades to 2 other objects +DETAIL: drop cascades to table partitioning_hash_test +drop cascades to table partitioning_hash_join_test diff --git a/src/test/regress/expected/partition_wise_join_0.out b/src/test/regress/expected/partition_wise_join_0.out new file mode 100644 index 000000000..9bda58598 --- /dev/null +++ b/src/test/regress/expected/partition_wise_join_0.out @@ -0,0 +1,208 @@ +CREATE SCHEMA partition_wise_join; +SET search_path to partition_wise_join; +CREATE TABLE partitioning_hash_test(id int, subid int) PARTITION BY HASH(subid); +CREATE TABLE partitioning_hash_test_0 PARTITION OF partitioning_hash_test FOR VALUES WITH (MODULUS 3, REMAINDER 0); +CREATE TABLE partitioning_hash_test_1 PARTITION OF partitioning_hash_test FOR VALUES WITH (MODULUS 3, REMAINDER 1); +INSERT INTO partitioning_hash_test VALUES (1, 2); +INSERT INTO partitioning_hash_test VALUES (2, 13); +INSERT INTO partitioning_hash_test VALUES (3, 7); +INSERT INTO partitioning_hash_test VALUES (4, 4); +-- distribute partitioned table +SELECT create_distributed_table('partitioning_hash_test', 'id'); +NOTICE: Copying data from local table... +NOTICE: copying the data has completed +DETAIL: The local data in the table is no longer visible, but is still on disk. +HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$partition_wise_join.partitioning_hash_test_0$$) +NOTICE: Copying data from local table... +NOTICE: copying the data has completed +DETAIL: The local data in the table is no longer visible, but is still on disk. +HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$partition_wise_join.partitioning_hash_test_1$$) + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- test partition-wise join +CREATE TABLE partitioning_hash_join_test(id int, subid int) PARTITION BY HASH(subid); +CREATE TABLE partitioning_hash_join_test_0 PARTITION OF partitioning_hash_join_test FOR VALUES WITH (MODULUS 3, REMAINDER 0); +CREATE TABLE partitioning_hash_join_test_1 PARTITION OF partitioning_hash_join_test FOR VALUES WITH (MODULUS 3, REMAINDER 1); +CREATE TABLE partitioning_hash_join_test_2 PARTITION OF partitioning_hash_join_test FOR VALUES WITH (MODULUS 3, REMAINDER 2); +SELECT create_distributed_table('partitioning_hash_join_test', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT success FROM run_command_on_workers('alter system set enable_mergejoin to off'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +SELECT success FROM run_command_on_workers('alter system set enable_nestloop to off'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +SELECT success FROM run_command_on_workers('alter system set enable_indexscan to off'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +SELECT success FROM run_command_on_workers('alter system set enable_indexonlyscan to off'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +SELECT success FROM run_command_on_workers('alter system set enable_partitionwise_join to off'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +SELECT success FROM run_command_on_workers('select pg_reload_conf()'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +EXPLAIN (COSTS OFF) +SELECT * FROM partitioning_hash_test JOIN partitioning_hash_join_test USING (id, subid); + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Hash Join + Hash Cond: ((partitioning_hash_join_test_xxx.id = partitioning_hash_test_xxx.id) AND (partitioning_hash_join_test_xxx.subid = partitioning_hash_test_xxx.subid)) + -> Append + -> Seq Scan on partitioning_hash_join_test_0_360163 partitioning_hash_join_test_xxx + -> Seq Scan on partitioning_hash_join_test_1_360167 partitioning_hash_join_test_xxx + -> Seq Scan on partitioning_hash_join_test_2_360171 partitioning_hash_join_test_xxx + -> Hash + -> Append + -> Seq Scan on partitioning_hash_test_0_360151 partitioning_hash_test_xxx + -> Seq Scan on partitioning_hash_test_1_360155 partitioning_hash_test_xxx +(15 rows) + +-- set partition-wise join on and parallel to off +SELECT success FROM run_command_on_workers('alter system set enable_partitionwise_join to on'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +SELECT success FROM run_command_on_workers('select pg_reload_conf()'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +-- SET enable_partitionwise_join TO on; +ANALYZE partitioning_hash_test, partitioning_hash_join_test; +EXPLAIN (COSTS OFF) +SELECT * FROM partitioning_hash_test JOIN partitioning_hash_join_test USING (id, subid); + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Hash Join + Hash Cond: ((partitioning_hash_join_test_xxx.id = partitioning_hash_test_xxx.id) AND (partitioning_hash_join_test_xxx.subid = partitioning_hash_test_xxx.subid)) + -> Append + -> Seq Scan on partitioning_hash_join_test_0_360163 partitioning_hash_join_test_xxx + -> Seq Scan on partitioning_hash_join_test_1_360167 partitioning_hash_join_test_xxx + -> Seq Scan on partitioning_hash_join_test_2_360171 partitioning_hash_join_test_xxx + -> Hash + -> Append + -> Seq Scan on partitioning_hash_test_0_360151 partitioning_hash_test_xxx + -> Seq Scan on partitioning_hash_test_1_360155 partitioning_hash_test_xxx +(15 rows) + +-- note that partition-wise joins only work when partition key is in the join +-- following join does not have that, therefore join will not be pushed down to +-- partitions +EXPLAIN (COSTS OFF) +SELECT * FROM partitioning_hash_test JOIN partitioning_hash_join_test USING (id); + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Hash Join + Hash Cond: (partitioning_hash_join_test_xxx.id = partitioning_hash_test_xxx.id) + -> Append + -> Seq Scan on partitioning_hash_join_test_0_360163 partitioning_hash_join_test_xxx + -> Seq Scan on partitioning_hash_join_test_1_360167 partitioning_hash_join_test_xxx + -> Seq Scan on partitioning_hash_join_test_2_360171 partitioning_hash_join_test_xxx + -> Hash + -> Append + -> Seq Scan on partitioning_hash_test_0_360151 partitioning_hash_test_xxx + -> Seq Scan on partitioning_hash_test_1_360155 partitioning_hash_test_xxx +(15 rows) + +-- reset partition-wise join +SELECT success FROM run_command_on_workers('alter system reset enable_partitionwise_join'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +SELECT success FROM run_command_on_workers('alter system reset enable_mergejoin'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +SELECT success FROM run_command_on_workers('alter system reset enable_nestloop'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +SELECT success FROM run_command_on_workers('alter system reset enable_indexscan'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +SELECT success FROM run_command_on_workers('alter system reset enable_indexonlyscan'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +SELECT success FROM run_command_on_workers('select pg_reload_conf()'); + success +--------------------------------------------------------------------- + t + t +(2 rows) + +RESET enable_partitionwise_join; +DROP SCHEMA partition_wise_join CASCADE; +NOTICE: drop cascades to 2 other objects +DETAIL: drop cascades to table partitioning_hash_test +drop cascades to table partitioning_hash_join_test diff --git a/src/test/regress/multi_1_schedule b/src/test/regress/multi_1_schedule index ef09cb0db..266ee3e12 100644 --- a/src/test/regress/multi_1_schedule +++ b/src/test/regress/multi_1_schedule @@ -67,6 +67,7 @@ test: ensure_no_intermediate_data_leak # ---------- test: multi_partitioning_utils multi_partitioning partitioning_issue_3970 replicated_partitioned_table test: drop_partitioned_table +test: partition_wise_join # ---------- # Tests for foreign data wrapper support diff --git a/src/test/regress/sql/multi_partitioning.sql b/src/test/regress/sql/multi_partitioning.sql index ae48314dd..b480dd7b3 100644 --- a/src/test/regress/sql/multi_partitioning.sql +++ b/src/test/regress/sql/multi_partitioning.sql @@ -1020,50 +1020,6 @@ INSERT INTO partitioning_locks_2009 SELECT * FROM partitioning_locks WHERE time SELECT * FROM lockinfo; COMMIT; --- test partition-wise join -CREATE TABLE partitioning_hash_join_test(id int, subid int) PARTITION BY HASH(subid); -CREATE TABLE partitioning_hash_join_test_0 PARTITION OF partitioning_hash_join_test FOR VALUES WITH (MODULUS 3, REMAINDER 0); -CREATE TABLE partitioning_hash_join_test_1 PARTITION OF partitioning_hash_join_test FOR VALUES WITH (MODULUS 3, REMAINDER 1); -CREATE TABLE partitioning_hash_join_test_2 PARTITION OF partitioning_hash_join_test FOR VALUES WITH (MODULUS 3, REMAINDER 2); - -SELECT create_distributed_table('partitioning_hash_join_test', 'id'); - -SELECT success FROM run_command_on_workers('alter system set enable_mergejoin to off'); -SELECT success FROM run_command_on_workers('alter system set enable_nestloop to off'); -SELECT success FROM run_command_on_workers('alter system set enable_indexscan to off'); -SELECT success FROM run_command_on_workers('alter system set enable_indexonlyscan to off'); -SELECT success FROM run_command_on_workers('alter system set enable_partitionwise_join to off'); -SELECT success FROM run_command_on_workers('select pg_reload_conf()'); - -EXPLAIN (COSTS OFF) -SELECT * FROM partitioning_hash_test JOIN partitioning_hash_join_test USING (id, subid); - --- set partition-wise join on and parallel to off -SELECT success FROM run_command_on_workers('alter system set enable_partitionwise_join to on'); -SELECT success FROM run_command_on_workers('select pg_reload_conf()'); - -SET enable_partitionwise_join TO on; -ANALYZE partitioning_hash_test, partitioning_hash_join_test; - -EXPLAIN (COSTS OFF) -SELECT * FROM partitioning_hash_test JOIN partitioning_hash_join_test USING (id, subid); - --- note that partition-wise joins only work when partition key is in the join --- following join does not have that, therefore join will not be pushed down to --- partitions -EXPLAIN (COSTS OFF) -SELECT * FROM partitioning_hash_test JOIN partitioning_hash_join_test USING (id); - --- reset partition-wise join -SELECT success FROM run_command_on_workers('alter system reset enable_partitionwise_join'); -SELECT success FROM run_command_on_workers('alter system reset enable_mergejoin'); -SELECT success FROM run_command_on_workers('alter system reset enable_nestloop'); -SELECT success FROM run_command_on_workers('alter system reset enable_indexscan'); -SELECT success FROM run_command_on_workers('alter system reset enable_indexonlyscan'); -SELECT success FROM run_command_on_workers('select pg_reload_conf()'); - -RESET enable_partitionwise_join; - DROP VIEW lockinfo; DROP TABLE IF EXISTS @@ -1328,7 +1284,6 @@ DROP SCHEMA partitioning_schema CASCADE; RESET search_path; DROP TABLE IF EXISTS partitioning_hash_test, - partitioning_hash_join_test, partitioning_test_failure, non_distributed_partitioned_table, partitioning_test_foreign_key; diff --git a/src/test/regress/sql/partition_wise_join.sql b/src/test/regress/sql/partition_wise_join.sql new file mode 100644 index 000000000..38e6d7a0b --- /dev/null +++ b/src/test/regress/sql/partition_wise_join.sql @@ -0,0 +1,61 @@ +CREATE SCHEMA partition_wise_join; +SET search_path to partition_wise_join; + +CREATE TABLE partitioning_hash_test(id int, subid int) PARTITION BY HASH(subid); + +CREATE TABLE partitioning_hash_test_0 PARTITION OF partitioning_hash_test FOR VALUES WITH (MODULUS 3, REMAINDER 0); +CREATE TABLE partitioning_hash_test_1 PARTITION OF partitioning_hash_test FOR VALUES WITH (MODULUS 3, REMAINDER 1); + +INSERT INTO partitioning_hash_test VALUES (1, 2); +INSERT INTO partitioning_hash_test VALUES (2, 13); +INSERT INTO partitioning_hash_test VALUES (3, 7); +INSERT INTO partitioning_hash_test VALUES (4, 4); + +-- distribute partitioned table +SELECT create_distributed_table('partitioning_hash_test', 'id'); + +-- test partition-wise join +CREATE TABLE partitioning_hash_join_test(id int, subid int) PARTITION BY HASH(subid); +CREATE TABLE partitioning_hash_join_test_0 PARTITION OF partitioning_hash_join_test FOR VALUES WITH (MODULUS 3, REMAINDER 0); +CREATE TABLE partitioning_hash_join_test_1 PARTITION OF partitioning_hash_join_test FOR VALUES WITH (MODULUS 3, REMAINDER 1); +CREATE TABLE partitioning_hash_join_test_2 PARTITION OF partitioning_hash_join_test FOR VALUES WITH (MODULUS 3, REMAINDER 2); + +SELECT create_distributed_table('partitioning_hash_join_test', 'id'); + +SELECT success FROM run_command_on_workers('alter system set enable_mergejoin to off'); +SELECT success FROM run_command_on_workers('alter system set enable_nestloop to off'); +SELECT success FROM run_command_on_workers('alter system set enable_indexscan to off'); +SELECT success FROM run_command_on_workers('alter system set enable_indexonlyscan to off'); +SELECT success FROM run_command_on_workers('alter system set enable_partitionwise_join to off'); +SELECT success FROM run_command_on_workers('select pg_reload_conf()'); + +EXPLAIN (COSTS OFF) +SELECT * FROM partitioning_hash_test JOIN partitioning_hash_join_test USING (id, subid); + +-- set partition-wise join on and parallel to off +SELECT success FROM run_command_on_workers('alter system set enable_partitionwise_join to on'); +SELECT success FROM run_command_on_workers('select pg_reload_conf()'); + +-- SET enable_partitionwise_join TO on; +ANALYZE partitioning_hash_test, partitioning_hash_join_test; + +EXPLAIN (COSTS OFF) +SELECT * FROM partitioning_hash_test JOIN partitioning_hash_join_test USING (id, subid); + +-- note that partition-wise joins only work when partition key is in the join +-- following join does not have that, therefore join will not be pushed down to +-- partitions +EXPLAIN (COSTS OFF) +SELECT * FROM partitioning_hash_test JOIN partitioning_hash_join_test USING (id); + +-- reset partition-wise join +SELECT success FROM run_command_on_workers('alter system reset enable_partitionwise_join'); +SELECT success FROM run_command_on_workers('alter system reset enable_mergejoin'); +SELECT success FROM run_command_on_workers('alter system reset enable_nestloop'); +SELECT success FROM run_command_on_workers('alter system reset enable_indexscan'); +SELECT success FROM run_command_on_workers('alter system reset enable_indexonlyscan'); +SELECT success FROM run_command_on_workers('select pg_reload_conf()'); + +RESET enable_partitionwise_join; + +DROP SCHEMA partition_wise_join CASCADE; \ No newline at end of file From 75fff147923da11dc8f76bcf80e4db6c6f6a3654 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Fri, 20 Aug 2021 09:50:49 +0300 Subject: [PATCH 068/104] Turn off VERBOSE to avoid alternative output With VERBOSE option, as of PG14, we get a line with "Query Identifier". --- .../regress/expected/multi_data_types.out | 20 ++++++++----------- src/test/regress/sql/multi_data_types.sql | 4 ++-- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/src/test/regress/expected/multi_data_types.out b/src/test/regress/expected/multi_data_types.out index db1cfedcc..4bc7da5c7 100644 --- a/src/test/regress/expected/multi_data_types.out +++ b/src/test/regress/expected/multi_data_types.out @@ -162,20 +162,18 @@ SELECT * FROM composite_type_partitioned_table WHERE id = 123; 123 | (123,456) (1 row) -EXPLAIN (ANALYZE TRUE, COSTS FALSE, VERBOSE TRUE, TIMING FALSE, SUMMARY FALSE) +EXPLAIN (ANALYZE TRUE, COSTS FALSE, VERBOSE FALSE, TIMING FALSE, SUMMARY FALSE) INSERT INTO composite_type_partitioned_table VALUES (123, '(123, 456)'::other_composite_type); - QUERY PLAN + QUERY PLAN --------------------------------------------------------------------- Custom Scan (Citus Adaptive) (actual rows=0 loops=1) Task Count: 1 Tasks Shown: All -> Task - Query: INSERT INTO public.composite_type_partitioned_table_530003 (id, col) VALUES (123, '(123,456)'::public.test_composite_type) Node: host=localhost port=xxxxx dbname=regression - -> Insert on public.composite_type_partitioned_table_530003 (actual rows=0 loops=1) + -> Insert on composite_type_partitioned_table_530003 (actual rows=0 loops=1) -> Result (actual rows=1 loops=1) - Output: 123, '(123,456)'::test_composite_type -(9 rows) +(7 rows) SELECT run_command_on_coordinator_and_workers($cf$ DROP CAST (other_composite_type as test_composite_type); @@ -206,20 +204,18 @@ $cf$); (1 row) INSERT INTO composite_type_partitioned_table VALUES (456, '(456, 678)'::other_composite_type); -EXPLAIN (ANALYZE TRUE, COSTS FALSE, VERBOSE TRUE, TIMING FALSE, SUMMARY FALSE) +EXPLAIN (ANALYZE TRUE, COSTS FALSE, VERBOSE FALSE, TIMING FALSE, SUMMARY FALSE) INSERT INTO composite_type_partitioned_table VALUES (123, '(456, 678)'::other_composite_type); - QUERY PLAN + QUERY PLAN --------------------------------------------------------------------- Custom Scan (Citus Adaptive) (actual rows=0 loops=1) Task Count: 1 Tasks Shown: All -> Task - Query: INSERT INTO public.composite_type_partitioned_table_530000 (id, col) VALUES (123, '(456,678)'::public.other_composite_type) Node: host=localhost port=xxxxx dbname=regression - -> Insert on public.composite_type_partitioned_table_530000 (actual rows=0 loops=1) + -> Insert on composite_type_partitioned_table_530000 (actual rows=0 loops=1) -> Result (actual rows=1 loops=1) - Output: 123, '(456,678)'::test_composite_type -(9 rows) +(7 rows) -- create and distribute a table on enum type column CREATE TYPE bug_status AS ENUM ('new', 'open', 'closed'); diff --git a/src/test/regress/sql/multi_data_types.sql b/src/test/regress/sql/multi_data_types.sql index 847876e83..7601bb319 100644 --- a/src/test/regress/sql/multi_data_types.sql +++ b/src/test/regress/sql/multi_data_types.sql @@ -116,7 +116,7 @@ $cf$); INSERT INTO composite_type_partitioned_table VALUES (123, '(123, 456)'::other_composite_type); SELECT * FROM composite_type_partitioned_table WHERE id = 123; -EXPLAIN (ANALYZE TRUE, COSTS FALSE, VERBOSE TRUE, TIMING FALSE, SUMMARY FALSE) +EXPLAIN (ANALYZE TRUE, COSTS FALSE, VERBOSE FALSE, TIMING FALSE, SUMMARY FALSE) INSERT INTO composite_type_partitioned_table VALUES (123, '(123, 456)'::other_composite_type); SELECT run_command_on_coordinator_and_workers($cf$ @@ -135,7 +135,7 @@ SELECT run_command_on_coordinator_and_workers($cf$ $cf$); INSERT INTO composite_type_partitioned_table VALUES (456, '(456, 678)'::other_composite_type); -EXPLAIN (ANALYZE TRUE, COSTS FALSE, VERBOSE TRUE, TIMING FALSE, SUMMARY FALSE) +EXPLAIN (ANALYZE TRUE, COSTS FALSE, VERBOSE FALSE, TIMING FALSE, SUMMARY FALSE) INSERT INTO composite_type_partitioned_table VALUES (123, '(456, 678)'::other_composite_type); From f3fa133caa17187c3e7a82fc09b4425470cc9aa3 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Fri, 20 Aug 2021 10:26:41 +0300 Subject: [PATCH 069/104] Bind seg version to 1.3 in isolation_textension_commands --- .../regress/expected/isolation_extension_commands.out | 10 +++++----- .../regress/spec/isolation_extension_commands.spec | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/test/regress/expected/isolation_extension_commands.out b/src/test/regress/expected/isolation_extension_commands.out index d944a87cd..4c5bfe3f0 100644 --- a/src/test/regress/expected/isolation_extension_commands.out +++ b/src/test/regress/expected/isolation_extension_commands.out @@ -199,7 +199,7 @@ step s1-add-node-1: (1 row) step s2-create-extension-with-schema1: - CREATE extension seg with schema schema1; + CREATE extension seg with version "1.3" schema schema1; step s1-commit: COMMIT; @@ -317,7 +317,7 @@ step s1-add-node-1: (1 row) step s1-create-extension-with-schema2: - CREATE extension seg with schema schema2; + CREATE extension seg with version "1.3" schema schema2; step s1-begin: BEGIN; @@ -399,7 +399,7 @@ step s1-remove-node-1: (1 row) step s2-create-extension-with-schema1: - CREATE extension seg with schema schema1; + CREATE extension seg with version "1.3" schema schema1; step s1-commit: COMMIT; @@ -674,7 +674,7 @@ step s2-begin: BEGIN; step s2-create-extension-with-schema1: - CREATE extension seg with schema schema1; + CREATE extension seg with version "1.3" schema schema1; step s1-add-node-1: SELECT 1 FROM master_add_node('localhost', 57637); @@ -742,7 +742,7 @@ step s2-add-node-1: (1 row) step s2-create-extension-with-schema2: - CREATE extension seg with schema schema2; + CREATE extension seg with version "1.3" schema schema2; step s2-begin: BEGIN; diff --git a/src/test/regress/spec/isolation_extension_commands.spec b/src/test/regress/spec/isolation_extension_commands.spec index 19d07844b..270a60330 100644 --- a/src/test/regress/spec/isolation_extension_commands.spec +++ b/src/test/regress/spec/isolation_extension_commands.spec @@ -36,7 +36,7 @@ step "s1-commit" step "s1-create-extension-with-schema2" { - CREATE extension seg with schema schema2; + CREATE extension seg with version "1.3" schema schema2; } step "s1-print" @@ -72,12 +72,12 @@ step "s2-alter-extension-version-13" step "s2-create-extension-with-schema1" { - CREATE extension seg with schema schema1; + CREATE extension seg with version "1.3" schema schema1; } step "s2-create-extension-with-schema2" { - CREATE extension seg with schema schema2; + CREATE extension seg with version "1.3" schema schema2; } step "s2-drop-extension" From aca2b8b67554f05d96a9ffd80bc458833357f2a4 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Fri, 20 Aug 2021 10:48:25 +0300 Subject: [PATCH 070/104] Add alternative output for isolation_master_update_node --- src/test/regress/expected/isolation_master_update_node.out | 2 ++ src/test/regress/expected/isolation_master_update_node_0.out | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/test/regress/expected/isolation_master_update_node.out b/src/test/regress/expected/isolation_master_update_node.out index 46e0d23d5..29a5ca2e2 100644 --- a/src/test/regress/expected/isolation_master_update_node.out +++ b/src/test/regress/expected/isolation_master_update_node.out @@ -56,6 +56,8 @@ master_update_node step s2-abort: ABORT; step s1-abort: ABORT; FATAL: terminating connection due to administrator command +FATAL: terminating connection due to administrator command +SSL connection has been closed unexpectedly server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. diff --git a/src/test/regress/expected/isolation_master_update_node_0.out b/src/test/regress/expected/isolation_master_update_node_0.out index 8dbc71029..cc746278e 100644 --- a/src/test/regress/expected/isolation_master_update_node_0.out +++ b/src/test/regress/expected/isolation_master_update_node_0.out @@ -47,7 +47,9 @@ step s2-abort: ABORT; step s1-abort: ABORT; WARNING: this step had a leftover error message FATAL: terminating connection due to administrator command -SSL connection has been closed unexpectedly +server closed the connection unexpectedly + This probably means the server terminated abnormally + before or while processing the request. master_remove_node From b632dd9940286af1a389373de5e56c633ba53405 Mon Sep 17 00:00:00 2001 From: Nils Dijk Date: Fri, 20 Aug 2021 17:06:36 +0200 Subject: [PATCH 071/104] use pg14 image for pg upgrade tests --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5b0a6a746..35f700a04 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -685,7 +685,7 @@ workflows: name: 'test-13-14_check-pg-upgrade' old_pg_major: 13 new_pg_major: 14 - image_tag: 12-13-14 + image_tag: '12-13-14-dev202108191715' requires: [build-13,build-14] - test-citus-upgrade: From 9fc4c27b08611468d8931013d43936bf4eb2aafa Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Fri, 20 Aug 2021 20:17:33 +0300 Subject: [PATCH 072/104] Readds deleted resultRelInfo changes for previos PG versions These changes were removed in commit: Introduces ExecSimpleRelationInsert_compat and modifyStateResultRelInfo macros We shouldn't have removed them but instead kept them for before PG14 --- src/backend/columnar/columnar_metadata.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/backend/columnar/columnar_metadata.c b/src/backend/columnar/columnar_metadata.c index 3f622a44d..7b6425b76 100644 --- a/src/backend/columnar/columnar_metadata.c +++ b/src/backend/columnar/columnar_metadata.c @@ -1415,6 +1415,15 @@ create_estate_for_relation(Relation rel) rte->rellockmode = AccessShareLock; ExecInitRangeTable(estate, list_make1(rte)); +#if PG_VERSION_NUM < PG_VERSION_14 + ResultRelInfo *resultRelInfo = makeNode(ResultRelInfo); + InitResultRelInfo(resultRelInfo, rel, 1, NULL, 0); + + estate->es_result_relations = resultRelInfo; + estate->es_num_result_relations = 1; + estate->es_result_relation_info = resultRelInfo; +#endif + estate->es_output_cid = GetCurrentCommandId(true); /* Prepare to catch AFTER triggers. */ From c31b0c2652c10901c4ff016bc25c87114a91a9d5 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Fri, 20 Aug 2021 20:57:43 +0300 Subject: [PATCH 073/104] Sets next_shard_id at partition_wise_join test --- src/test/regress/expected/partition_wise_join.out | 1 + src/test/regress/expected/partition_wise_join_0.out | 1 + src/test/regress/sql/partition_wise_join.sql | 2 ++ 3 files changed, 4 insertions(+) diff --git a/src/test/regress/expected/partition_wise_join.out b/src/test/regress/expected/partition_wise_join.out index 1b00fcef3..63ae67af3 100644 --- a/src/test/regress/expected/partition_wise_join.out +++ b/src/test/regress/expected/partition_wise_join.out @@ -1,5 +1,6 @@ CREATE SCHEMA partition_wise_join; SET search_path to partition_wise_join; +SET citus.next_shard_id TO 360147; CREATE TABLE partitioning_hash_test(id int, subid int) PARTITION BY HASH(subid); CREATE TABLE partitioning_hash_test_0 PARTITION OF partitioning_hash_test FOR VALUES WITH (MODULUS 3, REMAINDER 0); CREATE TABLE partitioning_hash_test_1 PARTITION OF partitioning_hash_test FOR VALUES WITH (MODULUS 3, REMAINDER 1); diff --git a/src/test/regress/expected/partition_wise_join_0.out b/src/test/regress/expected/partition_wise_join_0.out index 9bda58598..559862094 100644 --- a/src/test/regress/expected/partition_wise_join_0.out +++ b/src/test/regress/expected/partition_wise_join_0.out @@ -1,5 +1,6 @@ CREATE SCHEMA partition_wise_join; SET search_path to partition_wise_join; +SET citus.next_shard_id TO 360147; CREATE TABLE partitioning_hash_test(id int, subid int) PARTITION BY HASH(subid); CREATE TABLE partitioning_hash_test_0 PARTITION OF partitioning_hash_test FOR VALUES WITH (MODULUS 3, REMAINDER 0); CREATE TABLE partitioning_hash_test_1 PARTITION OF partitioning_hash_test FOR VALUES WITH (MODULUS 3, REMAINDER 1); diff --git a/src/test/regress/sql/partition_wise_join.sql b/src/test/regress/sql/partition_wise_join.sql index 38e6d7a0b..0e91240e3 100644 --- a/src/test/regress/sql/partition_wise_join.sql +++ b/src/test/regress/sql/partition_wise_join.sql @@ -1,6 +1,8 @@ CREATE SCHEMA partition_wise_join; SET search_path to partition_wise_join; +SET citus.next_shard_id TO 360147; + CREATE TABLE partitioning_hash_test(id int, subid int) PARTITION BY HASH(subid); CREATE TABLE partitioning_hash_test_0 PARTITION OF partitioning_hash_test FOR VALUES WITH (MODULUS 3, REMAINDER 0); From cd402b6a2bef191df528262bd9e691c1a47c9e6e Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Sat, 21 Aug 2021 18:15:23 +0300 Subject: [PATCH 074/104] Add alternative output for pg12 for window_functions --- .../regress/expected/window_functions_1.out | 1648 +++++++++++++++++ 1 file changed, 1648 insertions(+) create mode 100644 src/test/regress/expected/window_functions_1.out diff --git a/src/test/regress/expected/window_functions_1.out b/src/test/regress/expected/window_functions_1.out new file mode 100644 index 000000000..aea319c0b --- /dev/null +++ b/src/test/regress/expected/window_functions_1.out @@ -0,0 +1,1648 @@ +-- =================================================================== +-- test top level window functions that are pushdownable +-- =================================================================== +-- a very simple window function with an aggregate and a window function +-- distribution column is on the partition by clause +SELECT + user_id, COUNT(*) OVER (PARTITION BY user_id), + rank() OVER (PARTITION BY user_id) +FROM + users_table +ORDER BY + 1 DESC, 2 DESC, 3 DESC +LIMIT 5; + user_id | count | rank +--------------------------------------------------------------------- + 6 | 10 | 1 + 6 | 10 | 1 + 6 | 10 | 1 + 6 | 10 | 1 + 6 | 10 | 1 +(5 rows) + +-- a more complicated window clause, including an aggregate +-- in both the window clause and the target entry +SELECT + user_id, avg(avg(value_3)) OVER (PARTITION BY user_id, MIN(value_2)) +FROM + users_table +GROUP BY + 1 +ORDER BY + 2 DESC NULLS LAST, 1 DESC; + user_id | avg +--------------------------------------------------------------------- + 2 | 3 + 4 | 2.82608695652174 + 3 | 2.70588235294118 + 6 | 2.6 + 1 | 2.57142857142857 + 5 | 2.46153846153846 +(6 rows) + +-- window clause operates on the results of a subquery +SELECT + user_id, max(value_1) OVER (PARTITION BY user_id, MIN(value_2)) +FROM ( + SELECT + DISTINCT us.user_id, us.value_2, value_1, random() as r1 + FROM + users_table as us, events_table + WHERE + us.user_id = events_table.user_id AND event_type IN (1,2) + ORDER BY + user_id, value_2 + ) s +GROUP BY + 1, value_1 +ORDER BY + 2 DESC, 1; + user_id | max +--------------------------------------------------------------------- + 1 | 5 + 3 | 5 + 3 | 5 + 4 | 5 + 5 | 5 + 5 | 5 + 6 | 5 + 6 | 5 + 1 | 4 + 2 | 4 + 3 | 4 + 3 | 4 + 3 | 4 + 4 | 4 + 4 | 4 + 5 | 4 + 5 | 4 + 1 | 3 + 2 | 3 + 2 | 3 + 2 | 3 + 6 | 3 + 2 | 2 + 4 | 2 + 4 | 2 + 4 | 2 + 6 | 2 + 1 | 1 + 3 | 1 + 5 | 1 + 6 | 1 + 5 | 0 +(32 rows) + +-- window function operates on the results of +-- a join +-- we also want to verify that this doesn't crash +-- when the logging level is DEBUG4 +SET log_min_messages TO DEBUG4; +SELECT + us.user_id, + SUM(us.value_1) OVER (PARTITION BY us.user_id) +FROM + users_table us + JOIN + events_table ev + ON (us.user_id = ev.user_id) +GROUP BY + 1, + value_1 +ORDER BY + 1, + 2 +LIMIT 5; + user_id | sum +--------------------------------------------------------------------- + 1 | 13 + 1 | 13 + 1 | 13 + 1 | 13 + 2 | 10 +(5 rows) + +-- the same query, but this time join with an alias +SELECT + user_id, value_1, SUM(j.value_1) OVER (PARTITION BY j.user_id) +FROM + (users_table us + JOIN + events_table ev + USING (user_id ) + ) j +GROUP BY + user_id, + value_1 +ORDER BY + 3 DESC, 2 DESC, 1 DESC +LIMIT 5; + user_id | value_1 | sum +--------------------------------------------------------------------- + 5 | 5 | 15 + 4 | 5 | 15 + 3 | 5 | 15 + 5 | 4 | 15 + 4 | 4 | 15 +(5 rows) + +-- querying views that have window functions should be ok +CREATE VIEW window_view AS +SELECT + DISTINCT user_id, rank() OVER (PARTITION BY user_id ORDER BY value_1) +FROM + users_table +GROUP BY + user_id, value_1 +HAVING count(*) > 1; +-- Window function in View works +SELECT * +FROM + window_view +ORDER BY + 2 DESC, 1 +LIMIT 10; + user_id | rank +--------------------------------------------------------------------- + 5 | 6 + 2 | 5 + 4 | 5 + 5 | 5 + 2 | 4 + 3 | 4 + 4 | 4 + 5 | 4 + 6 | 4 + 2 | 3 +(10 rows) + +-- the other way around also should work fine +-- query a view using window functions +CREATE VIEW users_view AS SELECT * FROM users_table; +SELECT + DISTINCT user_id, rank() OVER (PARTITION BY user_id ORDER BY value_1) +FROM + users_view +GROUP BY + user_id, value_1 +HAVING count(*) > 4 +ORDER BY + 2 DESC, 1; + user_id | rank +--------------------------------------------------------------------- + 4 | 2 + 5 | 2 + 2 | 1 + 3 | 1 + 4 | 1 + 5 | 1 +(6 rows) + +DROP VIEW users_view, window_view; +-- window functions along with subquery in HAVING +SELECT + user_id, count (user_id) OVER (PARTITION BY user_id) +FROM + users_table +GROUP BY + user_id HAVING avg(value_1) < (SELECT min(k_no) FROM users_ref_test_table) +ORDER BY 1 DESC,2 DESC +LIMIT 1; + user_id | count +--------------------------------------------------------------------- + 6 | 1 +(1 row) + +-- window function uses columns from two different tables +SELECT + DISTINCT ON (events_table.user_id, rnk) events_table.user_id, rank() OVER my_win AS rnk +FROM + events_table, users_table +WHERE + users_table.user_id = events_table.user_id +WINDOW + my_win AS (PARTITION BY events_table.user_id, users_table.value_1 ORDER BY events_table.time DESC) +ORDER BY + rnk DESC, 1 DESC +LIMIT 10; + user_id | rnk +--------------------------------------------------------------------- + 3 | 121 + 5 | 118 + 2 | 116 + 3 | 115 + 4 | 113 + 2 | 111 + 5 | 109 + 3 | 109 + 4 | 106 + 2 | 106 +(10 rows) + +-- the same query with reference table column is also on the partition by clause +SELECT + DISTINCT ON (events_table.user_id, rnk) events_table.user_id, rank() OVER my_win AS rnk +FROM + events_table, users_ref_test_table uref +WHERE + uref.id = events_table.user_id +WINDOW + my_win AS (PARTITION BY events_table.user_id, uref.k_no ORDER BY events_table.time DESC) +ORDER BY + rnk DESC, 1 DESC +LIMIT 10; + user_id | rnk +--------------------------------------------------------------------- + 2 | 24 + 2 | 23 + 2 | 22 + 3 | 21 + 2 | 21 + 3 | 20 + 2 | 20 + 3 | 19 + 2 | 19 + 3 | 18 +(10 rows) + +-- similar query with no distribution column on the partition by clause +SELECT + DISTINCT ON (events_table.user_id, rnk) events_table.user_id, rank() OVER my_win AS rnk +FROM + events_table, users_ref_test_table uref +WHERE + uref.id = events_table.user_id +WINDOW + my_win AS (PARTITION BY events_table.value_2, uref.k_no ORDER BY events_table.time DESC) +ORDER BY + rnk DESC, 1 DESC +LIMIT 10; + user_id | rnk +--------------------------------------------------------------------- + 3 | 7 + 2 | 7 + 3 | 6 + 2 | 6 + 4 | 5 + 3 | 5 + 2 | 5 + 1 | 5 + 6 | 4 + 5 | 4 +(10 rows) + +-- ORDER BY in the window function is an aggregate +SELECT + user_id, rank() OVER my_win as rnk, avg(value_2) as avg_val_2 +FROM + events_table +GROUP BY + user_id, date_trunc('day', time) +WINDOW + my_win AS (PARTITION BY user_id ORDER BY avg(event_type) DESC) +ORDER BY + 3 DESC, 2 DESC, 1 DESC; + user_id | rnk | avg_val_2 +--------------------------------------------------------------------- + 1 | 1 | 3.3750000000000000 + 3 | 2 | 3.1666666666666667 + 5 | 1 | 2.6666666666666667 + 6 | 1 | 2.5000000000000000 + 4 | 1 | 2.5000000000000000 + 2 | 1 | 2.4736842105263158 + 4 | 2 | 2.4000000000000000 + 1 | 2 | 2.1428571428571429 + 5 | 2 | 2.0909090909090909 + 6 | 2 | 2.0000000000000000 + 2 | 2 | 2.0000000000000000 + 3 | 1 | 1.8000000000000000 +(12 rows) + +-- lets push the limits of writing complex expressions aling with the window functions +SELECT + COUNT(*) OVER (PARTITION BY user_id, user_id + 1), + rank() OVER (PARTITION BY user_id) as cnt1, + COUNT(*) OVER (PARTITION BY user_id, abs(value_1 - value_2)) as cnt2, + date_trunc('min', lag(time) OVER (PARTITION BY user_id ORDER BY time)) as datee, + rank() OVER my_win as rnnk, + avg(CASE + WHEN user_id > 4 + THEN value_1 + ELSE value_2 + END) FILTER (WHERE user_id > 2) OVER my_win_2 as filtered_count, + sum(user_id * (5.0 / (value_1 + value_2 + 0.1)) * value_3) FILTER (WHERE value_1::text LIKE '%1%') OVER my_win_4 as cnt_with_filter_2 +FROM + users_table +WINDOW + my_win AS (PARTITION BY user_id, (value_1%3)::int ORDER BY time DESC), + my_win_2 AS (PARTITION BY user_id, (value_1)::int ORDER BY time DESC), + my_win_3 AS (PARTITION BY user_id, date_trunc('min', time)), + my_win_4 AS (my_win_3 ORDER BY value_2, value_3) +ORDER BY + cnt_with_filter_2 DESC NULLS LAST, filtered_count DESC NULLS LAST, datee DESC NULLS LAST, rnnk DESC, cnt2 DESC, cnt1 DESC, user_id DESC +LIMIT 5; + count | cnt1 | cnt2 | datee | rnnk | filtered_count | cnt_with_filter_2 +--------------------------------------------------------------------- + 23 | 1 | 7 | Thu Nov 23 02:14:00 2017 | 6 | 0.00000000000000000000 | 72.7272727272727 + 10 | 1 | 3 | Wed Nov 22 23:01:00 2017 | 1 | 1.00000000000000000000 | 57.1428571428571 + 17 | 1 | 5 | Wed Nov 22 23:24:00 2017 | 8 | 3.0000000000000000 | 28.5714285714286 + 17 | 1 | 5 | | 10 | 2.6666666666666667 | 28.5714285714286 + 17 | 1 | 5 | Thu Nov 23 00:15:00 2017 | 7 | 3.6666666666666667 | 24.1935483870968 +(5 rows) + +-- some tests with GROUP BY along with PARTITION BY +SELECT + user_id, + rank() OVER my_win as my_rank, + avg(avg(event_type)) OVER my_win_2 as avg, + max(time) as mx_time +FROM + events_table +GROUP BY + user_id, + value_2 +WINDOW + my_win AS (PARTITION BY user_id, max(event_type) ORDER BY count(*) DESC), + my_win_2 AS (PARTITION BY user_id, avg(user_id) ORDER BY count(*) DESC) +ORDER BY + avg DESC, + mx_time DESC, + my_rank DESC, + user_id DESC; + user_id | my_rank | avg | mx_time +--------------------------------------------------------------------- + 6 | 1 | 3.0000000000000000 | Thu Nov 23 14:00:13.20013 2017 + 6 | 2 | 3.0000000000000000 | Thu Nov 23 11:16:13.106691 2017 + 6 | 1 | 3.0000000000000000 | Thu Nov 23 07:27:32.822068 2017 + 3 | 1 | 2.9857142857142857 | Thu Nov 23 16:31:56.219594 2017 + 4 | 2 | 2.9555555555555556 | Thu Nov 23 14:19:25.765876 2017 + 4 | 1 | 2.9555555555555556 | Thu Nov 23 08:36:53.871919 2017 + 1 | 4 | 2.8633333333333333 | Wed Nov 22 21:06:57.457147 2017 + 1 | 1 | 2.8250000000000000 | Thu Nov 23 21:54:46.924477 2017 + 2 | 2 | 2.7738095238095238 | Thu Nov 23 13:27:37.441959 2017 + 1 | 2 | 2.7722222222222222 | Thu Nov 23 09:23:30.994345 2017 + 3 | 1 | 2.7682539682539682 | Thu Nov 23 01:17:49.040685 2017 + 2 | 1 | 2.7142857142857143 | Thu Nov 23 15:58:49.273421 2017 + 1 | 3 | 2.5791666666666667 | Thu Nov 23 11:09:38.074595 2017 + 3 | 1 | 2.5714285714285714 | Thu Nov 23 16:44:41.903713 2017 + 2 | 1 | 2.5158730158730159 | Thu Nov 23 14:02:47.738901 2017 + 4 | 1 | 2.47777777777777778333 | Thu Nov 23 16:20:33.264457 2017 + 4 | 3 | 2.47777777777777778333 | Thu Nov 23 08:14:18.231273 2017 + 4 | 3 | 2.47777777777777778333 | Thu Nov 23 07:32:45.521278 2017 + 1 | 1 | 2.4000000000000000 | Thu Nov 23 10:23:27.617726 2017 + 2 | 1 | 2.3869047619047619 | Thu Nov 23 17:26:14.563216 2017 + 3 | 1 | 2.3841269841269841 | Thu Nov 23 18:08:26.550729 2017 + 3 | 1 | 2.3841269841269841 | Thu Nov 23 09:38:45.338008 2017 + 3 | 2 | 2.3841269841269841 | Thu Nov 23 06:44:50.887182 2017 + 2 | 2 | 2.3095238095238095 | Thu Nov 23 04:05:16.217731 2017 + 5 | 2 | 2.3000000000000000 | Thu Nov 23 14:28:51.833214 2017 + 5 | 2 | 2.3000000000000000 | Thu Nov 23 14:23:09.889786 2017 + 4 | 1 | 2.2000000000000000 | Thu Nov 23 18:10:21.338399 2017 + 2 | 1 | 2.09126984126984126667 | Thu Nov 23 03:35:04.321504 2017 + 5 | 1 | 2.0000000000000000 | Thu Nov 23 16:11:02.929469 2017 + 5 | 1 | 2.0000000000000000 | Thu Nov 23 14:40:40.467511 2017 + 5 | 1 | 2.0000000000000000 | Thu Nov 23 13:26:45.571108 2017 +(31 rows) + +-- test for range and rows mode and different window functions +-- mostly to make sure that deparsing works fine +SELECT + user_id, + rank() OVER (PARTITION BY user_id ROWS BETWEEN + UNBOUNDED PRECEDING AND CURRENT ROW), + dense_rank() OVER (PARTITION BY user_id RANGE BETWEEN + UNBOUNDED PRECEDING AND CURRENT ROW), + CUME_DIST() OVER (PARTITION BY user_id RANGE BETWEEN + UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING), + PERCENT_RANK() OVER (PARTITION BY user_id ORDER BY avg(value_1) RANGE BETWEEN + UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) +FROM + users_table +GROUP BY + 1 +ORDER BY + 4 DESC,3 DESC,2 DESC ,1 DESC; + user_id | rank | dense_rank | cume_dist | percent_rank +--------------------------------------------------------------------- + 6 | 1 | 1 | 1 | 0 + 5 | 1 | 1 | 1 | 0 + 4 | 1 | 1 | 1 | 0 + 3 | 1 | 1 | 1 | 0 + 2 | 1 | 1 | 1 | 0 + 1 | 1 | 1 | 1 | 0 +(6 rows) + +-- test exclude supported +SELECT + user_id, + value_1, + array_agg(value_1) OVER (PARTITION BY user_id ORDER BY value_1 RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW), + array_agg(value_1) OVER (PARTITION BY user_id ORDER BY value_1 RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW EXCLUDE CURRENT ROW) +FROM + users_table +WHERE + user_id > 2 AND user_id < 6 +ORDER BY + user_id, value_1, 3, 4; + user_id | value_1 | array_agg | array_agg +--------------------------------------------------------------------- + 3 | 0 | {0} | + 3 | 1 | {0,1,1,1,1,1,1} | {0,1,1,1,1,1} + 3 | 1 | {0,1,1,1,1,1,1} | {0,1,1,1,1,1} + 3 | 1 | {0,1,1,1,1,1,1} | {0,1,1,1,1,1} + 3 | 1 | {0,1,1,1,1,1,1} | {0,1,1,1,1,1} + 3 | 1 | {0,1,1,1,1,1,1} | {0,1,1,1,1,1} + 3 | 1 | {0,1,1,1,1,1,1} | {0,1,1,1,1,1} + 3 | 2 | {0,1,1,1,1,1,1,2,2} | {0,1,1,1,1,1,1,2} + 3 | 2 | {0,1,1,1,1,1,1,2,2} | {0,1,1,1,1,1,1,2} + 3 | 3 | {0,1,1,1,1,1,1,2,2,3,3,3} | {0,1,1,1,1,1,1,2,2,3,3} + 3 | 3 | {0,1,1,1,1,1,1,2,2,3,3,3} | {0,1,1,1,1,1,1,2,2,3,3} + 3 | 3 | {0,1,1,1,1,1,1,2,2,3,3,3} | {0,1,1,1,1,1,1,2,2,3,3} + 3 | 4 | {0,1,1,1,1,1,1,2,2,3,3,3,4,4,4,4} | {0,1,1,1,1,1,1,2,2,3,3,3,4,4,4} + 3 | 4 | {0,1,1,1,1,1,1,2,2,3,3,3,4,4,4,4} | {0,1,1,1,1,1,1,2,2,3,3,3,4,4,4} + 3 | 4 | {0,1,1,1,1,1,1,2,2,3,3,3,4,4,4,4} | {0,1,1,1,1,1,1,2,2,3,3,3,4,4,4} + 3 | 4 | {0,1,1,1,1,1,1,2,2,3,3,3,4,4,4,4} | {0,1,1,1,1,1,1,2,2,3,3,3,4,4,4} + 3 | 5 | {0,1,1,1,1,1,1,2,2,3,3,3,4,4,4,4,5} | {0,1,1,1,1,1,1,2,2,3,3,3,4,4,4,4} + 4 | 0 | {0,0,0,0} | {0,0,0} + 4 | 0 | {0,0,0,0} | {0,0,0} + 4 | 0 | {0,0,0,0} | {0,0,0} + 4 | 0 | {0,0,0,0} | {0,0,0} + 4 | 1 | {0,0,0,0,1} | {0,0,0,0} + 4 | 2 | {0,0,0,0,1,2,2,2} | {0,0,0,0,1,2,2} + 4 | 2 | {0,0,0,0,1,2,2,2} | {0,0,0,0,1,2,2} + 4 | 2 | {0,0,0,0,1,2,2,2} | {0,0,0,0,1,2,2} + 4 | 3 | {0,0,0,0,1,2,2,2,3,3,3,3,3,3} | {0,0,0,0,1,2,2,2,3,3,3,3,3} + 4 | 3 | {0,0,0,0,1,2,2,2,3,3,3,3,3,3} | {0,0,0,0,1,2,2,2,3,3,3,3,3} + 4 | 3 | {0,0,0,0,1,2,2,2,3,3,3,3,3,3} | {0,0,0,0,1,2,2,2,3,3,3,3,3} + 4 | 3 | {0,0,0,0,1,2,2,2,3,3,3,3,3,3} | {0,0,0,0,1,2,2,2,3,3,3,3,3} + 4 | 3 | {0,0,0,0,1,2,2,2,3,3,3,3,3,3} | {0,0,0,0,1,2,2,2,3,3,3,3,3} + 4 | 3 | {0,0,0,0,1,2,2,2,3,3,3,3,3,3} | {0,0,0,0,1,2,2,2,3,3,3,3,3} + 4 | 4 | {0,0,0,0,1,2,2,2,3,3,3,3,3,3,4,4,4,4,4,4,4} | {0,0,0,0,1,2,2,2,3,3,3,3,3,3,4,4,4,4,4,4} + 4 | 4 | {0,0,0,0,1,2,2,2,3,3,3,3,3,3,4,4,4,4,4,4,4} | {0,0,0,0,1,2,2,2,3,3,3,3,3,3,4,4,4,4,4,4} + 4 | 4 | {0,0,0,0,1,2,2,2,3,3,3,3,3,3,4,4,4,4,4,4,4} | {0,0,0,0,1,2,2,2,3,3,3,3,3,3,4,4,4,4,4,4} + 4 | 4 | {0,0,0,0,1,2,2,2,3,3,3,3,3,3,4,4,4,4,4,4,4} | {0,0,0,0,1,2,2,2,3,3,3,3,3,3,4,4,4,4,4,4} + 4 | 4 | {0,0,0,0,1,2,2,2,3,3,3,3,3,3,4,4,4,4,4,4,4} | {0,0,0,0,1,2,2,2,3,3,3,3,3,3,4,4,4,4,4,4} + 4 | 4 | {0,0,0,0,1,2,2,2,3,3,3,3,3,3,4,4,4,4,4,4,4} | {0,0,0,0,1,2,2,2,3,3,3,3,3,3,4,4,4,4,4,4} + 4 | 4 | {0,0,0,0,1,2,2,2,3,3,3,3,3,3,4,4,4,4,4,4,4} | {0,0,0,0,1,2,2,2,3,3,3,3,3,3,4,4,4,4,4,4} + 4 | 5 | {0,0,0,0,1,2,2,2,3,3,3,3,3,3,4,4,4,4,4,4,4,5,5} | {0,0,0,0,1,2,2,2,3,3,3,3,3,3,4,4,4,4,4,4,4,5} + 4 | 5 | {0,0,0,0,1,2,2,2,3,3,3,3,3,3,4,4,4,4,4,4,4,5,5} | {0,0,0,0,1,2,2,2,3,3,3,3,3,3,4,4,4,4,4,4,4,5} + 5 | 0 | {0,0} | {0} + 5 | 0 | {0,0} | {0} + 5 | 1 | {0,0,1,1,1} | {0,0,1,1} + 5 | 1 | {0,0,1,1,1} | {0,0,1,1} + 5 | 1 | {0,0,1,1,1} | {0,0,1,1} + 5 | 2 | {0,0,1,1,1,2,2,2,2,2,2} | {0,0,1,1,1,2,2,2,2,2} + 5 | 2 | {0,0,1,1,1,2,2,2,2,2,2} | {0,0,1,1,1,2,2,2,2,2} + 5 | 2 | {0,0,1,1,1,2,2,2,2,2,2} | {0,0,1,1,1,2,2,2,2,2} + 5 | 2 | {0,0,1,1,1,2,2,2,2,2,2} | {0,0,1,1,1,2,2,2,2,2} + 5 | 2 | {0,0,1,1,1,2,2,2,2,2,2} | {0,0,1,1,1,2,2,2,2,2} + 5 | 2 | {0,0,1,1,1,2,2,2,2,2,2} | {0,0,1,1,1,2,2,2,2,2} + 5 | 3 | {0,0,1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3} | {0,0,1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,3,3} + 5 | 3 | {0,0,1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3} | {0,0,1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,3,3} + 5 | 3 | {0,0,1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3} | {0,0,1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,3,3} + 5 | 3 | {0,0,1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3} | {0,0,1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,3,3} + 5 | 3 | {0,0,1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3} | {0,0,1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,3,3} + 5 | 3 | {0,0,1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3} | {0,0,1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,3,3} + 5 | 3 | {0,0,1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3} | {0,0,1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,3,3} + 5 | 3 | {0,0,1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3} | {0,0,1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,3,3} + 5 | 3 | {0,0,1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3} | {0,0,1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,3,3} + 5 | 4 | {0,0,1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,4,4,4} | {0,0,1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,4,4} + 5 | 4 | {0,0,1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,4,4,4} | {0,0,1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,4,4} + 5 | 4 | {0,0,1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,4,4,4} | {0,0,1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,4,4} + 5 | 5 | {0,0,1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,4,4,4,5,5,5} | {0,0,1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,4,4,4,5,5} + 5 | 5 | {0,0,1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,4,4,4,5,5,5} | {0,0,1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,4,4,4,5,5} + 5 | 5 | {0,0,1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,4,4,4,5,5,5} | {0,0,1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,4,4,4,5,5} +(66 rows) + +-- test preceding and following on RANGE window +SELECT + user_id, + value_1, + array_agg(value_1) OVER range_window, + array_agg(value_1) OVER range_window_exclude +FROM + users_table +WHERE + user_id > 2 AND user_id < 6 +WINDOW + range_window as (PARTITION BY user_id ORDER BY value_1 RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING), + range_window_exclude as (PARTITION BY user_id ORDER BY value_1 RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING EXCLUDE CURRENT ROW) +ORDER BY + user_id, value_1, 3, 4; + user_id | value_1 | array_agg | array_agg +--------------------------------------------------------------------- + 3 | 0 | {0,1,1,1,1,1,1} | {1,1,1,1,1,1} + 3 | 1 | {0,1,1,1,1,1,1,2,2} | {0,1,1,1,1,1,2,2} + 3 | 1 | {0,1,1,1,1,1,1,2,2} | {0,1,1,1,1,1,2,2} + 3 | 1 | {0,1,1,1,1,1,1,2,2} | {0,1,1,1,1,1,2,2} + 3 | 1 | {0,1,1,1,1,1,1,2,2} | {0,1,1,1,1,1,2,2} + 3 | 1 | {0,1,1,1,1,1,1,2,2} | {0,1,1,1,1,1,2,2} + 3 | 1 | {0,1,1,1,1,1,1,2,2} | {0,1,1,1,1,1,2,2} + 3 | 2 | {1,1,1,1,1,1,2,2,3,3,3} | {1,1,1,1,1,1,2,3,3,3} + 3 | 2 | {1,1,1,1,1,1,2,2,3,3,3} | {1,1,1,1,1,1,2,3,3,3} + 3 | 3 | {2,2,3,3,3,4,4,4,4} | {2,2,3,3,4,4,4,4} + 3 | 3 | {2,2,3,3,3,4,4,4,4} | {2,2,3,3,4,4,4,4} + 3 | 3 | {2,2,3,3,3,4,4,4,4} | {2,2,3,3,4,4,4,4} + 3 | 4 | {3,3,3,4,4,4,4,5} | {3,3,3,4,4,4,5} + 3 | 4 | {3,3,3,4,4,4,4,5} | {3,3,3,4,4,4,5} + 3 | 4 | {3,3,3,4,4,4,4,5} | {3,3,3,4,4,4,5} + 3 | 4 | {3,3,3,4,4,4,4,5} | {3,3,3,4,4,4,5} + 3 | 5 | {4,4,4,4,5} | {4,4,4,4} + 4 | 0 | {0,0,0,0,1} | {0,0,0,1} + 4 | 0 | {0,0,0,0,1} | {0,0,0,1} + 4 | 0 | {0,0,0,0,1} | {0,0,0,1} + 4 | 0 | {0,0,0,0,1} | {0,0,0,1} + 4 | 1 | {0,0,0,0,1,2,2,2} | {0,0,0,0,2,2,2} + 4 | 2 | {1,2,2,2,3,3,3,3,3,3} | {1,2,2,3,3,3,3,3,3} + 4 | 2 | {1,2,2,2,3,3,3,3,3,3} | {1,2,2,3,3,3,3,3,3} + 4 | 2 | {1,2,2,2,3,3,3,3,3,3} | {1,2,2,3,3,3,3,3,3} + 4 | 3 | {2,2,2,3,3,3,3,3,3,4,4,4,4,4,4,4} | {2,2,2,3,3,3,3,3,4,4,4,4,4,4,4} + 4 | 3 | {2,2,2,3,3,3,3,3,3,4,4,4,4,4,4,4} | {2,2,2,3,3,3,3,3,4,4,4,4,4,4,4} + 4 | 3 | {2,2,2,3,3,3,3,3,3,4,4,4,4,4,4,4} | {2,2,2,3,3,3,3,3,4,4,4,4,4,4,4} + 4 | 3 | {2,2,2,3,3,3,3,3,3,4,4,4,4,4,4,4} | {2,2,2,3,3,3,3,3,4,4,4,4,4,4,4} + 4 | 3 | {2,2,2,3,3,3,3,3,3,4,4,4,4,4,4,4} | {2,2,2,3,3,3,3,3,4,4,4,4,4,4,4} + 4 | 3 | {2,2,2,3,3,3,3,3,3,4,4,4,4,4,4,4} | {2,2,2,3,3,3,3,3,4,4,4,4,4,4,4} + 4 | 4 | {3,3,3,3,3,3,4,4,4,4,4,4,4,5,5} | {3,3,3,3,3,3,4,4,4,4,4,4,5,5} + 4 | 4 | {3,3,3,3,3,3,4,4,4,4,4,4,4,5,5} | {3,3,3,3,3,3,4,4,4,4,4,4,5,5} + 4 | 4 | {3,3,3,3,3,3,4,4,4,4,4,4,4,5,5} | {3,3,3,3,3,3,4,4,4,4,4,4,5,5} + 4 | 4 | {3,3,3,3,3,3,4,4,4,4,4,4,4,5,5} | {3,3,3,3,3,3,4,4,4,4,4,4,5,5} + 4 | 4 | {3,3,3,3,3,3,4,4,4,4,4,4,4,5,5} | {3,3,3,3,3,3,4,4,4,4,4,4,5,5} + 4 | 4 | {3,3,3,3,3,3,4,4,4,4,4,4,4,5,5} | {3,3,3,3,3,3,4,4,4,4,4,4,5,5} + 4 | 4 | {3,3,3,3,3,3,4,4,4,4,4,4,4,5,5} | {3,3,3,3,3,3,4,4,4,4,4,4,5,5} + 4 | 5 | {4,4,4,4,4,4,4,5,5} | {4,4,4,4,4,4,4,5} + 4 | 5 | {4,4,4,4,4,4,4,5,5} | {4,4,4,4,4,4,4,5} + 5 | 0 | {0,0,1,1,1} | {0,1,1,1} + 5 | 0 | {0,0,1,1,1} | {0,1,1,1} + 5 | 1 | {0,0,1,1,1,2,2,2,2,2,2} | {0,0,1,1,2,2,2,2,2,2} + 5 | 1 | {0,0,1,1,1,2,2,2,2,2,2} | {0,0,1,1,2,2,2,2,2,2} + 5 | 1 | {0,0,1,1,1,2,2,2,2,2,2} | {0,0,1,1,2,2,2,2,2,2} + 5 | 2 | {1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3} | {1,1,1,2,2,2,2,2,3,3,3,3,3,3,3,3,3} + 5 | 2 | {1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3} | {1,1,1,2,2,2,2,2,3,3,3,3,3,3,3,3,3} + 5 | 2 | {1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3} | {1,1,1,2,2,2,2,2,3,3,3,3,3,3,3,3,3} + 5 | 2 | {1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3} | {1,1,1,2,2,2,2,2,3,3,3,3,3,3,3,3,3} + 5 | 2 | {1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3} | {1,1,1,2,2,2,2,2,3,3,3,3,3,3,3,3,3} + 5 | 2 | {1,1,1,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3} | {1,1,1,2,2,2,2,2,3,3,3,3,3,3,3,3,3} + 5 | 3 | {2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,4,4,4} | {2,2,2,2,2,2,3,3,3,3,3,3,3,3,4,4,4} + 5 | 3 | {2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,4,4,4} | {2,2,2,2,2,2,3,3,3,3,3,3,3,3,4,4,4} + 5 | 3 | {2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,4,4,4} | {2,2,2,2,2,2,3,3,3,3,3,3,3,3,4,4,4} + 5 | 3 | {2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,4,4,4} | {2,2,2,2,2,2,3,3,3,3,3,3,3,3,4,4,4} + 5 | 3 | {2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,4,4,4} | {2,2,2,2,2,2,3,3,3,3,3,3,3,3,4,4,4} + 5 | 3 | {2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,4,4,4} | {2,2,2,2,2,2,3,3,3,3,3,3,3,3,4,4,4} + 5 | 3 | {2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,4,4,4} | {2,2,2,2,2,2,3,3,3,3,3,3,3,3,4,4,4} + 5 | 3 | {2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,4,4,4} | {2,2,2,2,2,2,3,3,3,3,3,3,3,3,4,4,4} + 5 | 3 | {2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,4,4,4} | {2,2,2,2,2,2,3,3,3,3,3,3,3,3,4,4,4} + 5 | 4 | {3,3,3,3,3,3,3,3,3,4,4,4,5,5,5} | {3,3,3,3,3,3,3,3,3,4,4,5,5,5} + 5 | 4 | {3,3,3,3,3,3,3,3,3,4,4,4,5,5,5} | {3,3,3,3,3,3,3,3,3,4,4,5,5,5} + 5 | 4 | {3,3,3,3,3,3,3,3,3,4,4,4,5,5,5} | {3,3,3,3,3,3,3,3,3,4,4,5,5,5} + 5 | 5 | {4,4,4,5,5,5} | {4,4,4,5,5} + 5 | 5 | {4,4,4,5,5,5} | {4,4,4,5,5} + 5 | 5 | {4,4,4,5,5,5} | {4,4,4,5,5} +(66 rows) + +-- test preceding and following on ROW window +SELECT + user_id, + value_1, + array_agg(value_1) OVER row_window, + array_agg(value_1) OVER row_window_exclude +FROM + users_table +WHERE + user_id > 2 and user_id < 6 +WINDOW + row_window as (PARTITION BY user_id ORDER BY value_1 ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING), + row_window_exclude as (PARTITION BY user_id ORDER BY value_1 ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING EXCLUDE CURRENT ROW) +ORDER BY + user_id, value_1, 3, 4; + user_id | value_1 | array_agg | array_agg +--------------------------------------------------------------------- + 3 | 0 | {0,1} | {1} + 3 | 1 | {0,1,1} | {0,1} + 3 | 1 | {1,1,1} | {1,1} + 3 | 1 | {1,1,1} | {1,1} + 3 | 1 | {1,1,1} | {1,1} + 3 | 1 | {1,1,1} | {1,1} + 3 | 1 | {1,1,2} | {1,2} + 3 | 2 | {1,2,2} | {1,2} + 3 | 2 | {2,2,3} | {2,3} + 3 | 3 | {2,3,3} | {2,3} + 3 | 3 | {3,3,3} | {3,3} + 3 | 3 | {3,3,4} | {3,4} + 3 | 4 | {3,4,4} | {3,4} + 3 | 4 | {4,4,4} | {4,4} + 3 | 4 | {4,4,4} | {4,4} + 3 | 4 | {4,4,5} | {4,5} + 3 | 5 | {4,5} | {4} + 4 | 0 | {0,0} | {0} + 4 | 0 | {0,0,0} | {0,0} + 4 | 0 | {0,0,0} | {0,0} + 4 | 0 | {0,0,1} | {0,1} + 4 | 1 | {0,1,2} | {0,2} + 4 | 2 | {1,2,2} | {1,2} + 4 | 2 | {2,2,2} | {2,2} + 4 | 2 | {2,2,3} | {2,3} + 4 | 3 | {2,3,3} | {2,3} + 4 | 3 | {3,3,3} | {3,3} + 4 | 3 | {3,3,3} | {3,3} + 4 | 3 | {3,3,3} | {3,3} + 4 | 3 | {3,3,3} | {3,3} + 4 | 3 | {3,3,4} | {3,4} + 4 | 4 | {3,4,4} | {3,4} + 4 | 4 | {4,4,4} | {4,4} + 4 | 4 | {4,4,4} | {4,4} + 4 | 4 | {4,4,4} | {4,4} + 4 | 4 | {4,4,4} | {4,4} + 4 | 4 | {4,4,4} | {4,4} + 4 | 4 | {4,4,5} | {4,5} + 4 | 5 | {4,5,5} | {4,5} + 4 | 5 | {5,5} | {5} + 5 | 0 | {0,0} | {0} + 5 | 0 | {0,0,1} | {0,1} + 5 | 1 | {0,1,1} | {0,1} + 5 | 1 | {1,1,1} | {1,1} + 5 | 1 | {1,1,2} | {1,2} + 5 | 2 | {1,2,2} | {1,2} + 5 | 2 | {2,2,2} | {2,2} + 5 | 2 | {2,2,2} | {2,2} + 5 | 2 | {2,2,2} | {2,2} + 5 | 2 | {2,2,2} | {2,2} + 5 | 2 | {2,2,3} | {2,3} + 5 | 3 | {2,3,3} | {2,3} + 5 | 3 | {3,3,3} | {3,3} + 5 | 3 | {3,3,3} | {3,3} + 5 | 3 | {3,3,3} | {3,3} + 5 | 3 | {3,3,3} | {3,3} + 5 | 3 | {3,3,3} | {3,3} + 5 | 3 | {3,3,3} | {3,3} + 5 | 3 | {3,3,3} | {3,3} + 5 | 3 | {3,3,4} | {3,4} + 5 | 4 | {3,4,4} | {3,4} + 5 | 4 | {4,4,4} | {4,4} + 5 | 4 | {4,4,5} | {4,5} + 5 | 5 | {4,5,5} | {4,5} + 5 | 5 | {5,5} | {5} + 5 | 5 | {5,5,5} | {5,5} +(66 rows) + +-- repeat above 3 tests without grouping by distribution column +SELECT + value_2, + rank() OVER (PARTITION BY value_2 ROWS BETWEEN + UNBOUNDED PRECEDING AND CURRENT ROW), + dense_rank() OVER (PARTITION BY value_2 RANGE BETWEEN + UNBOUNDED PRECEDING AND CURRENT ROW), + CUME_DIST() OVER (PARTITION BY value_2 RANGE BETWEEN + UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING), + PERCENT_RANK() OVER (PARTITION BY value_2 ORDER BY avg(value_1) RANGE BETWEEN + UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) +FROM + users_table +GROUP BY + 1 +ORDER BY + 4 DESC,3 DESC,2 DESC ,1 DESC; + value_2 | rank | dense_rank | cume_dist | percent_rank +--------------------------------------------------------------------- + 5 | 1 | 1 | 1 | 0 + 4 | 1 | 1 | 1 | 0 + 3 | 1 | 1 | 1 | 0 + 2 | 1 | 1 | 1 | 0 + 1 | 1 | 1 | 1 | 0 + 0 | 1 | 1 | 1 | 0 +(6 rows) + +-- test exclude supported +SELECT + value_2, + value_1, + array_agg(value_1) OVER (PARTITION BY value_2 ORDER BY value_1 RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW), + array_agg(value_1) OVER (PARTITION BY value_2 ORDER BY value_1 RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW EXCLUDE CURRENT ROW) +FROM + users_table +WHERE + value_2 > 2 AND value_2 < 6 +ORDER BY + value_2, value_1, 3, 4; + value_2 | value_1 | array_agg | array_agg +--------------------------------------------------------------------- + 3 | 0 | {0,0,0} | {0,0} + 3 | 0 | {0,0,0} | {0,0} + 3 | 0 | {0,0,0} | {0,0} + 3 | 1 | {0,0,0,1,1,1,1} | {0,0,0,1,1,1} + 3 | 1 | {0,0,0,1,1,1,1} | {0,0,0,1,1,1} + 3 | 1 | {0,0,0,1,1,1,1} | {0,0,0,1,1,1} + 3 | 1 | {0,0,0,1,1,1,1} | {0,0,0,1,1,1} + 3 | 2 | {0,0,0,1,1,1,1,2,2} | {0,0,0,1,1,1,1,2} + 3 | 2 | {0,0,0,1,1,1,1,2,2} | {0,0,0,1,1,1,1,2} + 3 | 3 | {0,0,0,1,1,1,1,2,2,3,3} | {0,0,0,1,1,1,1,2,2,3} + 3 | 3 | {0,0,0,1,1,1,1,2,2,3,3} | {0,0,0,1,1,1,1,2,2,3} + 3 | 4 | {0,0,0,1,1,1,1,2,2,3,3,4,4,4,4,4} | {0,0,0,1,1,1,1,2,2,3,3,4,4,4,4} + 3 | 4 | {0,0,0,1,1,1,1,2,2,3,3,4,4,4,4,4} | {0,0,0,1,1,1,1,2,2,3,3,4,4,4,4} + 3 | 4 | {0,0,0,1,1,1,1,2,2,3,3,4,4,4,4,4} | {0,0,0,1,1,1,1,2,2,3,3,4,4,4,4} + 3 | 4 | {0,0,0,1,1,1,1,2,2,3,3,4,4,4,4,4} | {0,0,0,1,1,1,1,2,2,3,3,4,4,4,4} + 3 | 4 | {0,0,0,1,1,1,1,2,2,3,3,4,4,4,4,4} | {0,0,0,1,1,1,1,2,2,3,3,4,4,4,4} + 3 | 5 | {0,0,0,1,1,1,1,2,2,3,3,4,4,4,4,4,5} | {0,0,0,1,1,1,1,2,2,3,3,4,4,4,4,4} + 4 | 0 | {0,0} | {0} + 4 | 0 | {0,0} | {0} + 4 | 1 | {0,0,1,1} | {0,0,1} + 4 | 1 | {0,0,1,1} | {0,0,1} + 4 | 2 | {0,0,1,1,2,2,2} | {0,0,1,1,2,2} + 4 | 2 | {0,0,1,1,2,2,2} | {0,0,1,1,2,2} + 4 | 2 | {0,0,1,1,2,2,2} | {0,0,1,1,2,2} + 4 | 3 | {0,0,1,1,2,2,2,3,3,3,3,3,3,3} | {0,0,1,1,2,2,2,3,3,3,3,3,3} + 4 | 3 | {0,0,1,1,2,2,2,3,3,3,3,3,3,3} | {0,0,1,1,2,2,2,3,3,3,3,3,3} + 4 | 3 | {0,0,1,1,2,2,2,3,3,3,3,3,3,3} | {0,0,1,1,2,2,2,3,3,3,3,3,3} + 4 | 3 | {0,0,1,1,2,2,2,3,3,3,3,3,3,3} | {0,0,1,1,2,2,2,3,3,3,3,3,3} + 4 | 3 | {0,0,1,1,2,2,2,3,3,3,3,3,3,3} | {0,0,1,1,2,2,2,3,3,3,3,3,3} + 4 | 3 | {0,0,1,1,2,2,2,3,3,3,3,3,3,3} | {0,0,1,1,2,2,2,3,3,3,3,3,3} + 4 | 3 | {0,0,1,1,2,2,2,3,3,3,3,3,3,3} | {0,0,1,1,2,2,2,3,3,3,3,3,3} + 4 | 4 | {0,0,1,1,2,2,2,3,3,3,3,3,3,3,4,4,4,4} | {0,0,1,1,2,2,2,3,3,3,3,3,3,3,4,4,4} + 4 | 4 | {0,0,1,1,2,2,2,3,3,3,3,3,3,3,4,4,4,4} | {0,0,1,1,2,2,2,3,3,3,3,3,3,3,4,4,4} + 4 | 4 | {0,0,1,1,2,2,2,3,3,3,3,3,3,3,4,4,4,4} | {0,0,1,1,2,2,2,3,3,3,3,3,3,3,4,4,4} + 4 | 4 | {0,0,1,1,2,2,2,3,3,3,3,3,3,3,4,4,4,4} | {0,0,1,1,2,2,2,3,3,3,3,3,3,3,4,4,4} + 4 | 5 | {0,0,1,1,2,2,2,3,3,3,3,3,3,3,4,4,4,4,5,5} | {0,0,1,1,2,2,2,3,3,3,3,3,3,3,4,4,4,4,5} + 4 | 5 | {0,0,1,1,2,2,2,3,3,3,3,3,3,3,4,4,4,4,5,5} | {0,0,1,1,2,2,2,3,3,3,3,3,3,3,4,4,4,4,5} + 5 | 0 | {0,0} | {0} + 5 | 0 | {0,0} | {0} + 5 | 1 | {0,0,1} | {0,0} + 5 | 2 | {0,0,1,2,2} | {0,0,1,2} + 5 | 2 | {0,0,1,2,2} | {0,0,1,2} + 5 | 3 | {0,0,1,2,2,3,3,3,3} | {0,0,1,2,2,3,3,3} + 5 | 3 | {0,0,1,2,2,3,3,3,3} | {0,0,1,2,2,3,3,3} + 5 | 3 | {0,0,1,2,2,3,3,3,3} | {0,0,1,2,2,3,3,3} + 5 | 3 | {0,0,1,2,2,3,3,3,3} | {0,0,1,2,2,3,3,3} + 5 | 4 | {0,0,1,2,2,3,3,3,3,4,4} | {0,0,1,2,2,3,3,3,3,4} + 5 | 4 | {0,0,1,2,2,3,3,3,3,4,4} | {0,0,1,2,2,3,3,3,3,4} + 5 | 5 | {0,0,1,2,2,3,3,3,3,4,4,5,5} | {0,0,1,2,2,3,3,3,3,4,4,5} + 5 | 5 | {0,0,1,2,2,3,3,3,3,4,4,5,5} | {0,0,1,2,2,3,3,3,3,4,4,5} +(50 rows) + +-- test preceding and following on RANGE window +SELECT + value_2, + value_1, + array_agg(value_1) OVER range_window, + array_agg(value_1) OVER range_window_exclude +FROM + users_table +WHERE + value_2 > 2 AND value_2 < 6 +WINDOW + range_window as (PARTITION BY value_2 ORDER BY value_1 RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING), + range_window_exclude as (PARTITION BY value_2 ORDER BY value_1 RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING EXCLUDE CURRENT ROW) +ORDER BY + value_2, value_1, 3, 4; + value_2 | value_1 | array_agg | array_agg +--------------------------------------------------------------------- + 3 | 0 | {0,0,0,1,1,1,1} | {0,0,1,1,1,1} + 3 | 0 | {0,0,0,1,1,1,1} | {0,0,1,1,1,1} + 3 | 0 | {0,0,0,1,1,1,1} | {0,0,1,1,1,1} + 3 | 1 | {0,0,0,1,1,1,1,2,2} | {0,0,0,1,1,1,2,2} + 3 | 1 | {0,0,0,1,1,1,1,2,2} | {0,0,0,1,1,1,2,2} + 3 | 1 | {0,0,0,1,1,1,1,2,2} | {0,0,0,1,1,1,2,2} + 3 | 1 | {0,0,0,1,1,1,1,2,2} | {0,0,0,1,1,1,2,2} + 3 | 2 | {1,1,1,1,2,2,3,3} | {1,1,1,1,2,3,3} + 3 | 2 | {1,1,1,1,2,2,3,3} | {1,1,1,1,2,3,3} + 3 | 3 | {2,2,3,3,4,4,4,4,4} | {2,2,3,4,4,4,4,4} + 3 | 3 | {2,2,3,3,4,4,4,4,4} | {2,2,3,4,4,4,4,4} + 3 | 4 | {3,3,4,4,4,4,4,5} | {3,3,4,4,4,4,5} + 3 | 4 | {3,3,4,4,4,4,4,5} | {3,3,4,4,4,4,5} + 3 | 4 | {3,3,4,4,4,4,4,5} | {3,3,4,4,4,4,5} + 3 | 4 | {3,3,4,4,4,4,4,5} | {3,3,4,4,4,4,5} + 3 | 4 | {3,3,4,4,4,4,4,5} | {3,3,4,4,4,4,5} + 3 | 5 | {4,4,4,4,4,5} | {4,4,4,4,4} + 4 | 0 | {0,0,1,1} | {0,1,1} + 4 | 0 | {0,0,1,1} | {0,1,1} + 4 | 1 | {0,0,1,1,2,2,2} | {0,0,1,2,2,2} + 4 | 1 | {0,0,1,1,2,2,2} | {0,0,1,2,2,2} + 4 | 2 | {1,1,2,2,2,3,3,3,3,3,3,3} | {1,1,2,2,3,3,3,3,3,3,3} + 4 | 2 | {1,1,2,2,2,3,3,3,3,3,3,3} | {1,1,2,2,3,3,3,3,3,3,3} + 4 | 2 | {1,1,2,2,2,3,3,3,3,3,3,3} | {1,1,2,2,3,3,3,3,3,3,3} + 4 | 3 | {2,2,2,3,3,3,3,3,3,3,4,4,4,4} | {2,2,2,3,3,3,3,3,3,4,4,4,4} + 4 | 3 | {2,2,2,3,3,3,3,3,3,3,4,4,4,4} | {2,2,2,3,3,3,3,3,3,4,4,4,4} + 4 | 3 | {2,2,2,3,3,3,3,3,3,3,4,4,4,4} | {2,2,2,3,3,3,3,3,3,4,4,4,4} + 4 | 3 | {2,2,2,3,3,3,3,3,3,3,4,4,4,4} | {2,2,2,3,3,3,3,3,3,4,4,4,4} + 4 | 3 | {2,2,2,3,3,3,3,3,3,3,4,4,4,4} | {2,2,2,3,3,3,3,3,3,4,4,4,4} + 4 | 3 | {2,2,2,3,3,3,3,3,3,3,4,4,4,4} | {2,2,2,3,3,3,3,3,3,4,4,4,4} + 4 | 3 | {2,2,2,3,3,3,3,3,3,3,4,4,4,4} | {2,2,2,3,3,3,3,3,3,4,4,4,4} + 4 | 4 | {3,3,3,3,3,3,3,4,4,4,4,5,5} | {3,3,3,3,3,3,3,4,4,4,5,5} + 4 | 4 | {3,3,3,3,3,3,3,4,4,4,4,5,5} | {3,3,3,3,3,3,3,4,4,4,5,5} + 4 | 4 | {3,3,3,3,3,3,3,4,4,4,4,5,5} | {3,3,3,3,3,3,3,4,4,4,5,5} + 4 | 4 | {3,3,3,3,3,3,3,4,4,4,4,5,5} | {3,3,3,3,3,3,3,4,4,4,5,5} + 4 | 5 | {4,4,4,4,5,5} | {4,4,4,4,5} + 4 | 5 | {4,4,4,4,5,5} | {4,4,4,4,5} + 5 | 0 | {0,0,1} | {0,1} + 5 | 0 | {0,0,1} | {0,1} + 5 | 1 | {0,0,1,2,2} | {0,0,2,2} + 5 | 2 | {1,2,2,3,3,3,3} | {1,2,3,3,3,3} + 5 | 2 | {1,2,2,3,3,3,3} | {1,2,3,3,3,3} + 5 | 3 | {2,2,3,3,3,3,4,4} | {2,2,3,3,3,4,4} + 5 | 3 | {2,2,3,3,3,3,4,4} | {2,2,3,3,3,4,4} + 5 | 3 | {2,2,3,3,3,3,4,4} | {2,2,3,3,3,4,4} + 5 | 3 | {2,2,3,3,3,3,4,4} | {2,2,3,3,3,4,4} + 5 | 4 | {3,3,3,3,4,4,5,5} | {3,3,3,3,4,5,5} + 5 | 4 | {3,3,3,3,4,4,5,5} | {3,3,3,3,4,5,5} + 5 | 5 | {4,4,5,5} | {4,4,5} + 5 | 5 | {4,4,5,5} | {4,4,5} +(50 rows) + +-- test preceding and following on ROW window +SELECT + value_2, + value_1, + array_agg(value_1) OVER row_window, + array_agg(value_1) OVER row_window_exclude +FROM + users_table +WHERE + value_2 > 2 and value_2 < 6 +WINDOW + row_window as (PARTITION BY value_2 ORDER BY value_1 ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING), + row_window_exclude as (PARTITION BY value_2 ORDER BY value_1 ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING EXCLUDE CURRENT ROW) +ORDER BY + value_2, value_1, 3, 4; + value_2 | value_1 | array_agg | array_agg +--------------------------------------------------------------------- + 3 | 0 | {0,0} | {0} + 3 | 0 | {0,0,0} | {0,0} + 3 | 0 | {0,0,1} | {0,1} + 3 | 1 | {0,1,1} | {0,1} + 3 | 1 | {1,1,1} | {1,1} + 3 | 1 | {1,1,1} | {1,1} + 3 | 1 | {1,1,2} | {1,2} + 3 | 2 | {1,2,2} | {1,2} + 3 | 2 | {2,2,3} | {2,3} + 3 | 3 | {2,3,3} | {2,3} + 3 | 3 | {3,3,4} | {3,4} + 3 | 4 | {3,4,4} | {3,4} + 3 | 4 | {4,4,4} | {4,4} + 3 | 4 | {4,4,4} | {4,4} + 3 | 4 | {4,4,4} | {4,4} + 3 | 4 | {4,4,5} | {4,5} + 3 | 5 | {4,5} | {4} + 4 | 0 | {0,0} | {0} + 4 | 0 | {0,0,1} | {0,1} + 4 | 1 | {0,1,1} | {0,1} + 4 | 1 | {1,1,2} | {1,2} + 4 | 2 | {1,2,2} | {1,2} + 4 | 2 | {2,2,2} | {2,2} + 4 | 2 | {2,2,3} | {2,3} + 4 | 3 | {2,3,3} | {2,3} + 4 | 3 | {3,3,3} | {3,3} + 4 | 3 | {3,3,3} | {3,3} + 4 | 3 | {3,3,3} | {3,3} + 4 | 3 | {3,3,3} | {3,3} + 4 | 3 | {3,3,3} | {3,3} + 4 | 3 | {3,3,4} | {3,4} + 4 | 4 | {3,4,4} | {3,4} + 4 | 4 | {4,4,4} | {4,4} + 4 | 4 | {4,4,4} | {4,4} + 4 | 4 | {4,4,5} | {4,5} + 4 | 5 | {4,5,5} | {4,5} + 4 | 5 | {5,5} | {5} + 5 | 0 | {0,0} | {0} + 5 | 0 | {0,0,1} | {0,1} + 5 | 1 | {0,1,2} | {0,2} + 5 | 2 | {1,2,2} | {1,2} + 5 | 2 | {2,2,3} | {2,3} + 5 | 3 | {2,3,3} | {2,3} + 5 | 3 | {3,3,3} | {3,3} + 5 | 3 | {3,3,3} | {3,3} + 5 | 3 | {3,3,4} | {3,4} + 5 | 4 | {3,4,4} | {3,4} + 5 | 4 | {4,4,5} | {4,5} + 5 | 5 | {4,5,5} | {4,5} + 5 | 5 | {5,5} | {5} +(50 rows) + +-- some tests with GROUP BY, HAVING and LIMIT +SELECT + user_id, sum(event_type) OVER my_win , event_type +FROM + events_table +GROUP BY + user_id, event_type +HAVING count(*) > 2 + WINDOW my_win AS (PARTITION BY user_id, max(event_type) ORDER BY count(*) DESC) +ORDER BY + 2 DESC, 3 DESC, 1 DESC +LIMIT + 5; + user_id | sum | event_type +--------------------------------------------------------------------- + 4 | 4 | 4 + 3 | 4 | 4 + 2 | 4 | 4 + 1 | 4 | 4 + 5 | 3 | 3 +(5 rows) + +-- test PARTITION BY avg(...) ORDER BY avg(...) +SELECT + value_1, + avg(value_3), + dense_rank() OVER (PARTITION BY avg(value_3) ORDER BY avg(value_2)) +FROM + users_table +GROUP BY + 1 +ORDER BY + 1; + value_1 | avg | dense_rank +--------------------------------------------------------------------- + 0 | 3.08333333333333 | 1 + 1 | 2.93333333333333 | 1 + 2 | 2.22222222222222 | 1 + 3 | 2.73076923076923 | 1 + 4 | 2.9047619047619 | 1 + 5 | 2.22222222222222 | 2 +(6 rows) + +-- Group by has more columns than partition by +SELECT + DISTINCT user_id, SUM(value_2) OVER (PARTITION BY user_id) +FROM + users_table +GROUP BY + user_id, value_1, value_2 +HAVING count(*) > 2 +ORDER BY + 2 DESC, 1 +LIMIT + 10; + user_id | sum +--------------------------------------------------------------------- + 5 | 3 + 4 | 2 +(2 rows) + +SELECT + DISTINCT ON (user_id) user_id, SUM(value_2) OVER (PARTITION BY user_id) +FROM + users_table +GROUP BY + user_id, value_1, value_2 +HAVING count(*) > 2 +ORDER BY + 1, 2 DESC +LIMIT + 10; + user_id | sum +--------------------------------------------------------------------- + 4 | 2 + 5 | 3 +(2 rows) + +SELECT + DISTINCT ON (SUM(value_1) OVER (PARTITION BY user_id)) user_id, SUM(value_2) OVER (PARTITION BY user_id) +FROM + users_table +GROUP BY + user_id, value_1, value_2 +HAVING count(*) > 2 +ORDER BY + (SUM(value_1) OVER (PARTITION BY user_id)) , 2 DESC, 1 +LIMIT + 10; + user_id | sum +--------------------------------------------------------------------- + 5 | 3 + 4 | 2 +(2 rows) + +-- not a meaningful query, with interesting syntax +SELECT + user_id, + AVG(avg(value_1)) OVER (PARTITION BY user_id, max(user_id), MIN(value_2)), + AVG(avg(user_id)) OVER (PARTITION BY user_id, min(user_id), AVG(value_1)) +FROM + users_table +GROUP BY + 1 +ORDER BY + 3 DESC, 2 DESC, 1 DESC; + user_id | avg | avg +--------------------------------------------------------------------- + 6 | 2.1000000000000000 | 6.0000000000000000 + 5 | 2.6538461538461538 | 5.0000000000000000 + 4 | 2.7391304347826087 | 4.0000000000000000 + 3 | 2.3529411764705882 | 3.0000000000000000 + 2 | 2.3333333333333333 | 2.0000000000000000 + 1 | 3.2857142857142857 | 1.00000000000000000000 +(6 rows) + +SELECT coordinator_plan($Q$ +EXPLAIN (COSTS FALSE) +SELECT + user_id, + AVG(avg(value_1)) OVER (PARTITION BY user_id, max(user_id), MIN(value_2)), + AVG(avg(user_id)) OVER (PARTITION BY user_id, min(user_id), AVG(value_1)) +FROM + users_table +GROUP BY + 1 +ORDER BY + 3 DESC, 2 DESC, 1 DESC; +$Q$); + coordinator_plan +--------------------------------------------------------------------- + Sort + Sort Key: remote_scan.avg_1 DESC, remote_scan.avg DESC, remote_scan.user_id DESC + -> Custom Scan (Citus Adaptive) + Task Count: 4 +(4 rows) + +SELECT + value_2, + AVG(avg(value_1)) OVER (PARTITION BY value_2, max(value_2), MIN(value_2)), + AVG(avg(value_2)) OVER (PARTITION BY value_2, min(value_2), AVG(value_1)) +FROM + users_table +GROUP BY + 1 +ORDER BY + 3 DESC, 2 DESC, 1 DESC; + value_2 | avg | avg +--------------------------------------------------------------------- + 5 | 2.6923076923076923 | 5.0000000000000000 + 4 | 2.7500000000000000 | 4.0000000000000000 + 3 | 2.2941176470588235 | 3.0000000000000000 + 2 | 2.7619047619047619 | 2.0000000000000000 + 1 | 2.4285714285714286 | 1.00000000000000000000 + 0 | 2.2222222222222222 | 0.00000000000000000000 +(6 rows) + +SELECT + value_2, user_id, + AVG(avg(value_1)) OVER (PARTITION BY value_2, max(value_2), MIN(value_2)), + AVG(avg(value_2)) OVER (PARTITION BY user_id, min(value_2), AVG(value_1)) +FROM + users_table +GROUP BY + 1, 2 +ORDER BY + 3 DESC, 2 DESC, 1 DESC; + value_2 | user_id | avg | avg +--------------------------------------------------------------------- + 5 | 5 | 2.6666666666666667 | 5.0000000000000000 + 5 | 4 | 2.6666666666666667 | 5.0000000000000000 + 5 | 3 | 2.6666666666666667 | 5.0000000000000000 + 5 | 2 | 2.6666666666666667 | 5.0000000000000000 + 2 | 6 | 2.54583333333333333333 | 2.0000000000000000 + 2 | 5 | 2.54583333333333333333 | 2.0000000000000000 + 2 | 4 | 2.54583333333333333333 | 2.0000000000000000 + 2 | 3 | 2.54583333333333333333 | 2.0000000000000000 + 2 | 2 | 2.54583333333333333333 | 2.0000000000000000 + 2 | 1 | 2.54583333333333333333 | 2.0000000000000000 + 0 | 6 | 2.50000000000000000000 | 0.00000000000000000000 + 0 | 5 | 2.50000000000000000000 | 0.00000000000000000000 + 0 | 4 | 2.50000000000000000000 | 0.00000000000000000000 + 0 | 2 | 2.50000000000000000000 | 0.00000000000000000000 + 0 | 1 | 2.50000000000000000000 | 0.00000000000000000000 + 4 | 6 | 2.45555555555555555000 | 4.0000000000000000 + 4 | 5 | 2.45555555555555555000 | 4.0000000000000000 + 4 | 4 | 2.45555555555555555000 | 4.0000000000000000 + 4 | 3 | 2.45555555555555555000 | 4.0000000000000000 + 4 | 2 | 2.45555555555555555000 | 4.0000000000000000 + 4 | 1 | 2.45555555555555555000 | 4.0000000000000000 + 3 | 6 | 2.3500000000000000 | 3.0000000000000000 + 3 | 5 | 2.3500000000000000 | 3.0000000000000000 + 3 | 4 | 2.3500000000000000 | 3.0000000000000000 + 3 | 3 | 2.3500000000000000 | 3.0000000000000000 + 3 | 2 | 2.3500000000000000 | 3.0000000000000000 + 3 | 1 | 2.3500000000000000 | 3.0000000000000000 + 1 | 6 | 1.90666666666666666000 | 1.00000000000000000000 + 1 | 5 | 1.90666666666666666000 | 1.00000000000000000000 + 1 | 4 | 1.90666666666666666000 | 1.00000000000000000000 + 1 | 3 | 1.90666666666666666000 | 1.00000000000000000000 + 1 | 2 | 1.90666666666666666000 | 1.00000000000000000000 +(32 rows) + +SELECT user_id, sum(avg(user_id)) OVER () +FROM users_table +GROUP BY user_id +ORDER BY 1 +LIMIT 10; + user_id | sum +--------------------------------------------------------------------- + 1 | 21.00000000000000000000 + 2 | 21.00000000000000000000 + 3 | 21.00000000000000000000 + 4 | 21.00000000000000000000 + 5 | 21.00000000000000000000 + 6 | 21.00000000000000000000 +(6 rows) + +SELECT + user_id, + 1 + sum(value_1), + 1 + AVG(value_2) OVER (partition by user_id) +FROM + users_table +GROUP BY + user_id, value_2 +ORDER BY + user_id, value_2; + user_id | ?column? | ?column? +--------------------------------------------------------------------- + 1 | 5 | 3.2500000000000000 + 1 | 4 | 3.2500000000000000 + 1 | 6 | 3.2500000000000000 + 1 | 12 | 3.2500000000000000 + 2 | 3 | 3.5000000000000000 + 2 | 5 | 3.5000000000000000 + 2 | 13 | 3.5000000000000000 + 2 | 6 | 3.5000000000000000 + 2 | 17 | 3.5000000000000000 + 2 | 4 | 3.5000000000000000 + 3 | 3 | 4.0000000000000000 + 3 | 13 | 4.0000000000000000 + 3 | 10 | 4.0000000000000000 + 3 | 2 | 4.0000000000000000 + 3 | 17 | 4.0000000000000000 + 4 | 4 | 3.5000000000000000 + 4 | 28 | 3.5000000000000000 + 4 | 1 | 3.5000000000000000 + 4 | 11 | 3.5000000000000000 + 4 | 17 | 3.5000000000000000 + 4 | 8 | 3.5000000000000000 + 5 | 7 | 3.5000000000000000 + 5 | 17 | 3.5000000000000000 + 5 | 24 | 3.5000000000000000 + 5 | 9 | 3.5000000000000000 + 5 | 8 | 3.5000000000000000 + 5 | 10 | 3.5000000000000000 + 6 | 6 | 3.0000000000000000 + 6 | 3 | 3.0000000000000000 + 6 | 9 | 3.0000000000000000 + 6 | 3 | 3.0000000000000000 + 6 | 5 | 3.0000000000000000 +(32 rows) + +SELECT + user_id, + 1 + sum(value_1), + 1 + AVG(value_2) OVER (partition by user_id) +FROM + users_table +GROUP BY + user_id, value_2 +ORDER BY + 2 DESC, 1 +LIMIT 5; + user_id | ?column? | ?column? +--------------------------------------------------------------------- + 4 | 28 | 3.5000000000000000 + 5 | 24 | 3.5000000000000000 + 2 | 17 | 3.5000000000000000 + 3 | 17 | 4.0000000000000000 + 4 | 17 | 3.5000000000000000 +(5 rows) + +-- rank and ordering in the reverse order +SELECT + user_id, + avg(value_1), + RANK() OVER (partition by user_id order by value_2) +FROM + users_table +GROUP BY user_id, value_2 +ORDER BY user_id, value_2 DESC; + user_id | avg | rank +--------------------------------------------------------------------- + 1 | 3.6666666666666667 | 4 + 1 | 2.5000000000000000 | 3 + 1 | 3.0000000000000000 | 2 + 1 | 4.0000000000000000 | 1 + 2 | 1.5000000000000000 | 6 + 2 | 3.2000000000000000 | 5 + 2 | 1.6666666666666667 | 4 + 2 | 3.0000000000000000 | 3 + 2 | 1.3333333333333333 | 2 + 2 | 2.0000000000000000 | 1 + 3 | 2.6666666666666667 | 5 + 3 | 1.00000000000000000000 | 4 + 3 | 3.0000000000000000 | 3 + 3 | 2.4000000000000000 | 2 + 3 | 1.00000000000000000000 | 1 + 4 | 3.5000000000000000 | 6 + 4 | 3.2000000000000000 | 5 + 4 | 3.3333333333333333 | 4 + 4 | 0.00000000000000000000 | 3 + 4 | 3.0000000000000000 | 2 + 4 | 1.00000000000000000000 | 1 + 5 | 3.0000000000000000 | 6 + 5 | 2.3333333333333333 | 5 + 5 | 1.6000000000000000 | 4 + 5 | 2.8750000000000000 | 3 + 5 | 3.2000000000000000 | 2 + 5 | 3.0000000000000000 | 1 + 6 | 1.3333333333333333 | 5 + 6 | 2.0000000000000000 | 4 + 6 | 4.0000000000000000 | 3 + 6 | 1.00000000000000000000 | 2 + 6 | 2.5000000000000000 | 1 +(32 rows) + +-- order by in the window function is same as avg(value_1) DESC +SELECT + user_id, + avg(value_1), + RANK() OVER (partition by user_id order by 1 / (1 + avg(value_1))) +FROM + users_table +GROUP BY user_id, value_2 +ORDER BY user_id, avg(value_1) DESC; + user_id | avg | rank +--------------------------------------------------------------------- + 1 | 4.0000000000000000 | 1 + 1 | 3.6666666666666667 | 2 + 1 | 3.0000000000000000 | 3 + 1 | 2.5000000000000000 | 4 + 2 | 3.2000000000000000 | 1 + 2 | 3.0000000000000000 | 2 + 2 | 2.0000000000000000 | 3 + 2 | 1.6666666666666667 | 4 + 2 | 1.5000000000000000 | 5 + 2 | 1.3333333333333333 | 6 + 3 | 3.0000000000000000 | 1 + 3 | 2.6666666666666667 | 2 + 3 | 2.4000000000000000 | 3 + 3 | 1.00000000000000000000 | 4 + 3 | 1.00000000000000000000 | 4 + 4 | 3.5000000000000000 | 1 + 4 | 3.3333333333333333 | 2 + 4 | 3.2000000000000000 | 3 + 4 | 3.0000000000000000 | 4 + 4 | 1.00000000000000000000 | 5 + 4 | 0.00000000000000000000 | 6 + 5 | 3.2000000000000000 | 1 + 5 | 3.0000000000000000 | 2 + 5 | 3.0000000000000000 | 2 + 5 | 2.8750000000000000 | 4 + 5 | 2.3333333333333333 | 5 + 5 | 1.6000000000000000 | 6 + 6 | 4.0000000000000000 | 1 + 6 | 2.5000000000000000 | 2 + 6 | 2.0000000000000000 | 3 + 6 | 1.3333333333333333 | 4 + 6 | 1.00000000000000000000 | 5 +(32 rows) + +EXPLAIN (COSTS FALSE) +SELECT + user_id, + avg(value_1), + RANK() OVER (partition by user_id order by 1 / (1 + avg(value_1))) +FROM + users_table +GROUP BY user_id, value_2 +ORDER BY user_id, avg(value_1) DESC; + QUERY PLAN +--------------------------------------------------------------------- + Sort + Sort Key: remote_scan.user_id, remote_scan.avg DESC + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> WindowAgg + -> Sort + Sort Key: users_table.user_id, (('1'::numeric / ('1'::numeric + avg(users_table.value_1)))) + -> HashAggregate + Group Key: users_table.user_id, users_table.value_2 + -> Seq Scan on users_table_1400256 users_table +(13 rows) + +-- order by in the window function is same as avg(value_1) DESC +SELECT + user_id, + avg(value_1), + RANK() OVER (partition by user_id order by 1 / (1 + avg(value_1))) +FROM + users_table +GROUP BY user_id, value_2 +ORDER BY user_id, avg(value_1) DESC; + user_id | avg | rank +--------------------------------------------------------------------- + 1 | 4.0000000000000000 | 1 + 1 | 3.6666666666666667 | 2 + 1 | 3.0000000000000000 | 3 + 1 | 2.5000000000000000 | 4 + 2 | 3.2000000000000000 | 1 + 2 | 3.0000000000000000 | 2 + 2 | 2.0000000000000000 | 3 + 2 | 1.6666666666666667 | 4 + 2 | 1.5000000000000000 | 5 + 2 | 1.3333333333333333 | 6 + 3 | 3.0000000000000000 | 1 + 3 | 2.6666666666666667 | 2 + 3 | 2.4000000000000000 | 3 + 3 | 1.00000000000000000000 | 4 + 3 | 1.00000000000000000000 | 4 + 4 | 3.5000000000000000 | 1 + 4 | 3.3333333333333333 | 2 + 4 | 3.2000000000000000 | 3 + 4 | 3.0000000000000000 | 4 + 4 | 1.00000000000000000000 | 5 + 4 | 0.00000000000000000000 | 6 + 5 | 3.2000000000000000 | 1 + 5 | 3.0000000000000000 | 2 + 5 | 3.0000000000000000 | 2 + 5 | 2.8750000000000000 | 4 + 5 | 2.3333333333333333 | 5 + 5 | 1.6000000000000000 | 6 + 6 | 4.0000000000000000 | 1 + 6 | 2.5000000000000000 | 2 + 6 | 2.0000000000000000 | 3 + 6 | 1.3333333333333333 | 4 + 6 | 1.00000000000000000000 | 5 +(32 rows) + +-- limit is not pushed down to worker !! +EXPLAIN (COSTS FALSE) +SELECT + user_id, + avg(value_1), + RANK() OVER (partition by user_id order by 1 / (1 + avg(value_1))) +FROM + users_table +GROUP BY user_id, value_2 +ORDER BY user_id, avg(value_1) DESC +LIMIT 5; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Sort + Sort Key: remote_scan.user_id, remote_scan.avg DESC + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Limit + -> Sort + Sort Key: users_table.user_id, (avg(users_table.value_1)) DESC + -> WindowAgg + -> Sort + Sort Key: users_table.user_id, (('1'::numeric / ('1'::numeric + avg(users_table.value_1)))) + -> HashAggregate + Group Key: users_table.user_id, users_table.value_2 + -> Seq Scan on users_table_1400256 users_table +(17 rows) + +EXPLAIN (COSTS FALSE) +SELECT + user_id, + avg(value_1), + RANK() OVER (partition by user_id order by 1 / (1 + avg(value_1))) +FROM + users_table +GROUP BY user_id, value_2 +ORDER BY user_id, avg(value_1) DESC +LIMIT 5; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Sort + Sort Key: remote_scan.user_id, remote_scan.avg DESC + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Limit + -> Sort + Sort Key: users_table.user_id, (avg(users_table.value_1)) DESC + -> WindowAgg + -> Sort + Sort Key: users_table.user_id, (('1'::numeric / ('1'::numeric + avg(users_table.value_1)))) + -> HashAggregate + Group Key: users_table.user_id, users_table.value_2 + -> Seq Scan on users_table_1400256 users_table +(17 rows) + +EXPLAIN (COSTS FALSE) +SELECT + user_id, + avg(value_1), + RANK() OVER (partition by user_id order by 1 / (1 + sum(value_2))) +FROM + users_table +GROUP BY user_id, value_2 +ORDER BY user_id, avg(value_1) DESC +LIMIT 5; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Sort + Sort Key: remote_scan.user_id, remote_scan.avg DESC + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Limit + -> Sort + Sort Key: users_table.user_id, (avg(users_table.value_1)) DESC + -> WindowAgg + -> Sort + Sort Key: users_table.user_id, ((1 / (1 + sum(users_table.value_2)))) + -> HashAggregate + Group Key: users_table.user_id, users_table.value_2 + -> Seq Scan on users_table_1400256 users_table +(17 rows) + +EXPLAIN (COSTS FALSE) +SELECT + user_id, + avg(value_1), + RANK() OVER (partition by user_id order by sum(value_2)) +FROM + users_table +GROUP BY user_id, value_2 +ORDER BY user_id, avg(value_1) DESC +LIMIT 5; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Sort + Sort Key: remote_scan.user_id, remote_scan.avg DESC + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Limit + -> Sort + Sort Key: users_table.user_id, (avg(users_table.value_1)) DESC + -> WindowAgg + -> Sort + Sort Key: users_table.user_id, (sum(users_table.value_2)) + -> HashAggregate + Group Key: users_table.user_id, users_table.value_2 + -> Seq Scan on users_table_1400256 users_table +(17 rows) + +-- Grouping can be pushed down with aggregates even when window function can't +EXPLAIN (COSTS FALSE) +SELECT user_id, count(value_1), stddev(value_1), count(user_id) OVER (PARTITION BY random()) +FROM users_table GROUP BY user_id HAVING avg(value_1) > 2 LIMIT 1; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> WindowAgg + -> Sort + Sort Key: remote_scan.worker_column_5 + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Group Key: user_id + Filter: (avg(value_1) > '2'::numeric) + -> Seq Scan on users_table_1400256 users_table +(13 rows) + +-- Window function with inlined CTE +WITH cte as ( + SELECT uref.id user_id, events_table.value_2, count(*) c + FROM events_table + JOIN users_ref_test_table uref ON uref.id = events_table.user_id + GROUP BY 1, 2 +) +SELECT DISTINCT cte.value_2, cte.c, sum(cte.value_2) OVER (PARTITION BY cte.c) +FROM cte JOIN events_table et ON et.value_2 = cte.value_2 and et.value_2 = cte.c +ORDER BY 1; + value_2 | c | sum +--------------------------------------------------------------------- + 3 | 3 | 108 + 4 | 4 | 56 +(2 rows) + +-- There was a strange bug where this wouldn't have window functions being pushed down +-- Bug dependent on column ordering +CREATE TABLE daily_uniques (value_2 float, user_id bigint); +SELECT create_distributed_table('daily_uniques', 'user_id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +EXPLAIN (COSTS FALSE) SELECT + user_id, + sum(value_2) AS commits, + RANK () OVER ( + PARTITION BY user_id + ORDER BY + sum(value_2) DESC + ) +FROM daily_uniques +GROUP BY user_id +HAVING + sum(value_2) > 0 +ORDER BY commits DESC +LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Sort + Sort Key: remote_scan.commits DESC + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Limit + -> Sort + Sort Key: (sum(daily_uniques.value_2)) DESC + -> WindowAgg + -> Sort + Sort Key: daily_uniques.user_id, (sum(daily_uniques.value_2)) DESC + -> HashAggregate + Group Key: daily_uniques.user_id + Filter: (sum(daily_uniques.value_2) > '0'::double precision) + -> Seq Scan on daily_uniques_xxxxxxx daily_uniques +(18 rows) + +DROP TABLE daily_uniques; +-- Partition by reference table column joined to distribution column +SELECT DISTINCT value_2, array_agg(rnk ORDER BY rnk) FROM ( +SELECT events_table.value_2, sum(uref.k_no) OVER (PARTITION BY uref.id) AS rnk +FROM events_table +JOIN users_ref_test_table uref ON uref.id = events_table.user_id) sq +GROUP BY 1 ORDER BY 1; + value_2 | array_agg +--------------------------------------------------------------------- + 0 | {686,686,816,816,987,987,1104} + 1 | {500,500,675,675,675,686,686,816,816,816,987,987,987,987,987,1104,1104,1104,1104,1104,1104,1104} + 2 | {500,500,500,500,675,675,675,675,675,686,686,686,686,816,816,816,816,816,987,987,987,987,987,987,987,1104,1104,1104,1104,1104,1104} + 3 | {500,500,500,500,675,686,686,686,816,816,987,987,987,1104,1104,1104,1104,1104} + 4 | {675,675,675,675,686,686,686,816,816,816,987,987,1104,1104} + 5 | {675,675,816,816,987,987,1104,1104,1104} +(6 rows) + +-- https://github.com/citusdata/citus/issues/3754 +select null = sum(null::int2) over () +from public.users_table as ut limit 1; + ?column? +--------------------------------------------------------------------- + +(1 row) + +-- verify that this doesn't crash with DEBUG4 +SET log_min_messages TO DEBUG4; +SELECT + user_id, max(value_1) OVER (PARTITION BY user_id, MIN(value_2)) +FROM ( + SELECT + DISTINCT us.user_id, us.value_2, value_1, random() as r1 + FROM + users_table as us, events_table + WHERE + us.user_id = events_table.user_id AND event_type IN (1,2) + ORDER BY + user_id, value_2 + ) s +GROUP BY + 1, value_1 +ORDER BY + 2 DESC, 1; + user_id | max +--------------------------------------------------------------------- + 1 | 5 + 3 | 5 + 3 | 5 + 4 | 5 + 5 | 5 + 5 | 5 + 6 | 5 + 6 | 5 + 1 | 4 + 2 | 4 + 3 | 4 + 3 | 4 + 3 | 4 + 4 | 4 + 4 | 4 + 5 | 4 + 5 | 4 + 1 | 3 + 2 | 3 + 2 | 3 + 2 | 3 + 6 | 3 + 2 | 2 + 4 | 2 + 4 | 2 + 4 | 2 + 6 | 2 + 1 | 1 + 3 | 1 + 5 | 1 + 6 | 1 + 5 | 0 +(32 rows) + From 7c0389a7a1d369f585b0ee192ad413762f5b70e4 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Sat, 21 Aug 2021 21:33:14 +0300 Subject: [PATCH 075/104] Update propagate extension commands test for pg12 The test file was changes slightly to avoid adding an alternative output. We update the existing alternative output for pg12 with the new changes. --- .../propagate_extension_commands_1.out | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/test/regress/expected/propagate_extension_commands_1.out b/src/test/regress/expected/propagate_extension_commands_1.out index 36f7254c9..99b8ef3a2 100644 --- a/src/test/regress/expected/propagate_extension_commands_1.out +++ b/src/test/regress/expected/propagate_extension_commands_1.out @@ -161,10 +161,11 @@ SELECT run_command_on_workers($$SELECT count(extnamespace) FROM pg_extension WHE (localhost,57637,t,1) (1 row) -SELECT run_command_on_workers($$SELECT extversion FROM pg_extension WHERE extname = 'seg'$$); - run_command_on_workers +SELECT workers.result = pg_extension.extversion AS same_version + FROM run_command_on_workers($$SELECT extversion FROM pg_extension WHERE extname = 'seg'$$) workers, pg_extension WHERE extname = 'seg'; + same_version --------------------------------------------------------------------- - (localhost,57637,t,1.3) + t (1 row) -- now create the reference table @@ -253,11 +254,12 @@ SELECT run_command_on_workers($$SELECT count(extnamespace) FROM pg_extension WHE (localhost,57638,t,1) (2 rows) -SELECT run_command_on_workers($$SELECT extversion FROM pg_extension WHERE extname = 'seg'$$); - run_command_on_workers +SELECT workers.result = pg_extension.extversion AS same_version + FROM run_command_on_workers($$SELECT extversion FROM pg_extension WHERE extname = 'seg'$$) workers, pg_extension WHERE extname = 'seg'; + same_version --------------------------------------------------------------------- - (localhost,57637,t,1.3) - (localhost,57638,t,1.3) + t + t (2 rows) -- check for the unpackaged extension to be created correctly @@ -375,12 +377,12 @@ BEGIN; ROLLBACK; -- show that the CREATE EXTENSION command propagated even if the transaction -- block is rollbacked, that's a shortcoming of dependency creation logic -SELECT run_command_on_workers($$SELECT extversion FROM pg_extension WHERE extname = 'seg'$$); - run_command_on_workers +SELECT COUNT(DISTINCT workers.result) + FROM run_command_on_workers($$SELECT extversion FROM pg_extension WHERE extname = 'seg'$$) workers; + count --------------------------------------------------------------------- - (localhost,57637,t,1.3) - (localhost,57638,t,1.3) -(2 rows) + 1 +(1 row) -- drop the schema and all the objects DROP SCHEMA "extension'test" CASCADE; From d1c040305536c55679706b476a33cba9d2cbf8d4 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Sat, 21 Aug 2021 23:03:54 +0300 Subject: [PATCH 076/104] Disable Query Idenfifier calculation in tests When queryId is not 0 and verbose is true, the query identifier is emitted to the explain output. This is breaking Postgres outputs. We disable de query identifier calculation in the tests. Commit on PG that introduced the query identifier in the explain output: 4f0b0966c866ae9f0e15d7cc73ccf7ce4e1af84b --- src/backend/distributed/planner/multi_explain.c | 4 ++-- src/test/regress/bin/normalize.sed | 1 - src/test/regress/pg_regress_multi.pl | 6 ++++++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/backend/distributed/planner/multi_explain.c b/src/backend/distributed/planner/multi_explain.c index 0443523e0..a3bd6b0aa 100644 --- a/src/backend/distributed/planner/multi_explain.c +++ b/src/backend/distributed/planner/multi_explain.c @@ -11,6 +11,8 @@ #include "libpq-fe.h" #include "miscadmin.h" +#include "distributed/pg_version_constants.h" + #include "access/htup_details.h" #include "access/xact.h" #include "catalog/namespace.h" @@ -1251,10 +1253,8 @@ CitusExplainOneQuery(Query *query, int cursorOptions, IntoClause *into, /* plan the query */ PlannedStmt *plan = pg_plan_query_compat(query, NULL, cursorOptions, params); - INSTR_TIME_SET_CURRENT(planduration); INSTR_TIME_SUBTRACT(planduration, planstart); - #if PG_VERSION_NUM >= PG_VERSION_13 /* calc differences of buffer counters. */ diff --git a/src/test/regress/bin/normalize.sed b/src/test/regress/bin/normalize.sed index d15107c5e..5ede38e35 100644 --- a/src/test/regress/bin/normalize.sed +++ b/src/test/regress/bin/normalize.sed @@ -235,7 +235,6 @@ s/ERROR: parallel workers for vacuum must be between/ERROR: parallel vacuum de s/ERROR: fake_fetch_row_version not implemented/ERROR: fake_tuple_update not implemented/g s/ERROR: COMMIT is not allowed in an SQL function/ERROR: COMMIT is not allowed in a SQL function/g s/ERROR: ROLLBACK is not allowed in an SQL function/ERROR: ROLLBACK is not allowed in a SQL function/g -/.*Query Identifier.*/d /.*Async-Capable.*/d /.*Async Capable.*/d /Parent Relationship/d diff --git a/src/test/regress/pg_regress_multi.pl b/src/test/regress/pg_regress_multi.pl index 09500007b..29195285f 100755 --- a/src/test/regress/pg_regress_multi.pl +++ b/src/test/regress/pg_regress_multi.pl @@ -438,6 +438,12 @@ push(@pgOptions, "wal_receiver_status_interval=1"); # src/backend/replication/logical/launcher.c. push(@pgOptions, "wal_retrieve_retry_interval=1000"); +# disable compute_query_id so that we don't get Query Identifiers +# in explain outputs +if ($majorversion >= "14") { + push(@pgOptions, "compute_query_id=off"); +} + # Citus options set for the tests push(@pgOptions, "citus.shard_count=4"); push(@pgOptions, "citus.max_adaptive_executor_pool_size=4"); From 2656d885f94b9a249937ede8d09024b570cd72fb Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Mon, 23 Aug 2021 11:30:33 +0300 Subject: [PATCH 077/104] Rewrite AppendColumnNames for Pg14 Postgres changed stats expression types as of PG14. Hence we needed to write the AppendColumnNames method. Also they removed the error on PG side so we remove it as well. Relevant commits on pg14: a4d75c86bf15220df22de0a92c819ecef9db3849 388e75ad33489b77cfb9a8590a91e9287d8fb960 --- .../deparser/deparse_statistics_stmts.c | 22 ++++++++++++++++++- .../regress/expected/propagate_statistics.out | 5 ----- src/test/regress/sql/propagate_statistics.sql | 3 --- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/src/backend/distributed/deparser/deparse_statistics_stmts.c b/src/backend/distributed/deparser/deparse_statistics_stmts.c index 4da69a041..e6f8fc262 100644 --- a/src/backend/distributed/deparser/deparse_statistics_stmts.c +++ b/src/backend/distributed/deparser/deparse_statistics_stmts.c @@ -12,6 +12,8 @@ */ #include "postgres.h" +#include "distributed/pg_version_constants.h" + #include "distributed/citus_ruleutils.h" #include "distributed/deparser.h" #include "distributed/listutils.h" @@ -231,7 +233,25 @@ AppendStatTypes(StringInfo buf, CreateStatsStmt *stmt) appendStringInfoString(buf, ")"); } +#if PG_VERSION_NUM >= PG_VERSION_14 +static void +AppendColumnNames(StringInfo buf, CreateStatsStmt *stmt) +{ + StatsElem *column = NULL; + foreach_ptr(column, stmt->exprs) + { + const char *columnName = quote_identifier(column->name); + + appendStringInfoString(buf, columnName); + + if (column != llast(stmt->exprs)) + { + appendStringInfoString(buf, ", "); + } + } +} +#else static void AppendColumnNames(StringInfo buf, CreateStatsStmt *stmt) { @@ -257,7 +277,7 @@ AppendColumnNames(StringInfo buf, CreateStatsStmt *stmt) } } } - +#endif static void AppendTableName(StringInfo buf, CreateStatsStmt *stmt) diff --git a/src/test/regress/expected/propagate_statistics.out b/src/test/regress/expected/propagate_statistics.out index 1c798c526..fd4f8e6e2 100644 --- a/src/test/regress/expected/propagate_statistics.out +++ b/src/test/regress/expected/propagate_statistics.out @@ -99,12 +99,7 @@ SELECT create_distributed_table('test','x'); (1 row) -CREATE STATISTICS stats_xy ON (x, y) FROM test; -ERROR: only simple column references are allowed in CREATE STATISTICS -CREATE STATISTICS stats_xy ON x+y FROM test; -ERROR: only simple column references are allowed in CREATE STATISTICS CREATE STATISTICS stats_xy ON x,y FROM test; -CREATE STATISTICS IF NOT EXISTS stats_xy ON x+y FROM test; \c - - - :worker_1_port SELECT stxname FROM pg_statistic_ext diff --git a/src/test/regress/sql/propagate_statistics.sql b/src/test/regress/sql/propagate_statistics.sql index 51d3076f2..898387eaa 100644 --- a/src/test/regress/sql/propagate_statistics.sql +++ b/src/test/regress/sql/propagate_statistics.sql @@ -79,10 +79,7 @@ SELECT create_distributed_table('ownertest','a'); CREATE TABLE test (x int, y int); SELECT create_distributed_table('test','x'); -CREATE STATISTICS stats_xy ON (x, y) FROM test; -CREATE STATISTICS stats_xy ON x+y FROM test; CREATE STATISTICS stats_xy ON x,y FROM test; -CREATE STATISTICS IF NOT EXISTS stats_xy ON x+y FROM test; \c - - - :worker_1_port SELECT stxname From e63302d012142badbc777382d06628eef1ddae82 Mon Sep 17 00:00:00 2001 From: Nils Dijk Date: Mon, 23 Aug 2021 16:21:59 +0200 Subject: [PATCH 078/104] update error messages for libpq 14beta3 --- src/test/regress/expected/failure_1pc_copy_hash.out | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/regress/expected/failure_1pc_copy_hash.out b/src/test/regress/expected/failure_1pc_copy_hash.out index fe521c729..e99c6f560 100644 --- a/src/test/regress/expected/failure_1pc_copy_hash.out +++ b/src/test/regress/expected/failure_1pc_copy_hash.out @@ -36,7 +36,7 @@ SELECT citus.dump_network_traffic(); dump_network_traffic --------------------------------------------------------------------- (0,coordinator,"[initial message]") - (0,worker,"['AuthenticationOk()', 'ParameterStatus(application_name=citus)', 'ParameterStatus(client_encoding=UTF8)', 'ParameterStatus(DateStyle=ISO, MDY)', 'ParameterStatus(integer_datetimes=on)', 'ParameterStatus(IntervalStyle=postgres)', 'ParameterStatus(is_superuser=on)', 'ParameterStatus(server_encoding=UTF8)', 'ParameterStatus(server_version=XXX)', 'ParameterStatus(session_authorization=postgres)', 'ParameterStatus(standard_conforming_strings=on)', 'ParameterStatus(TimeZone=XXX)', 'BackendKeyData(XXX)', 'ReadyForQuery(state=idle)']") + (0,worker,"['AuthenticationOk()', 'ParameterStatus(application_name=citus)', 'ParameterStatus(client_encoding=UTF8)', 'ParameterStatus(DateStyle=ISO, MDY)', 'ParameterStatus(default_transaction_read_only=off)', 'ParameterStatus(in_hot_standby=off)', 'ParameterStatus(integer_datetimes=on)', 'ParameterStatus(IntervalStyle=postgres)', 'ParameterStatus(is_superuser=on)', 'ParameterStatus(server_encoding=UTF8)', 'ParameterStatus(server_version=XXX)', 'ParameterStatus(session_authorization=postgres)', 'ParameterStatus(standard_conforming_strings=on)', 'ParameterStatus(TimeZone=XXX)', 'BackendKeyData(XXX)', 'ReadyForQuery(state=idle)']") (0,coordinator,"[""Query(query=BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(0, XX, 'XXXX-XX-XX XX:XX:XX.XXXXXX-XX');)""]") (0,worker,"['CommandComplete(command=BEGIN)', ""RowDescription(fieldcount=1,fields=['F(name=assign_distributed_transaction_id,tableoid=0,colattrnum=0,typoid=2278,typlen=4,typmod=-1,format_code=0)'])"", 'DataRow(columncount=1,columns=[""C(length=0,value=b\\'\\')""])', 'CommandComplete(command=SELECT 1)', 'ReadyForQuery(state=in_transaction_block)']") (0,coordinator,"[""Query(query=COPY public.copy_test_XXXXXX (key, value) FROM STDIN WITH (format 'binary'))""]") @@ -46,7 +46,7 @@ SELECT citus.dump_network_traffic(); (0,coordinator,"['Query(query=COMMIT)']") (0,worker,"['CommandComplete(command=COMMIT)', 'ReadyForQuery(state=idle)']") (1,coordinator,"[initial message]") - (1,worker,"['AuthenticationOk()', 'ParameterStatus(application_name=citus)', 'ParameterStatus(client_encoding=UTF8)', 'ParameterStatus(DateStyle=ISO, MDY)', 'ParameterStatus(integer_datetimes=on)', 'ParameterStatus(IntervalStyle=postgres)', 'ParameterStatus(is_superuser=on)', 'ParameterStatus(server_encoding=UTF8)', 'ParameterStatus(server_version=XXX)', 'ParameterStatus(session_authorization=postgres)', 'ParameterStatus(standard_conforming_strings=on)', 'ParameterStatus(TimeZone=XXX)', 'BackendKeyData(XXX)', 'ReadyForQuery(state=idle)']") + (1,worker,"['AuthenticationOk()', 'ParameterStatus(application_name=citus)', 'ParameterStatus(client_encoding=UTF8)', 'ParameterStatus(DateStyle=ISO, MDY)', 'ParameterStatus(default_transaction_read_only=off)', 'ParameterStatus(in_hot_standby=off)', 'ParameterStatus(integer_datetimes=on)', 'ParameterStatus(IntervalStyle=postgres)', 'ParameterStatus(is_superuser=on)', 'ParameterStatus(server_encoding=UTF8)', 'ParameterStatus(server_version=XXX)', 'ParameterStatus(session_authorization=postgres)', 'ParameterStatus(standard_conforming_strings=on)', 'ParameterStatus(TimeZone=XXX)', 'BackendKeyData(XXX)', 'ReadyForQuery(state=idle)']") (1,coordinator,"['Query(query=SELECT count(1) AS count FROM public.copy_test_XXXXXX copy_test)']") (1,worker,"[""RowDescription(fieldcount=1,fields=['F(name=count,tableoid=0,colattrnum=0,typoid=20,typlen=8,typmod=-1,format_code=0)'])"", 'DataRow(columncount=1,columns=[""C(length=0,value=b\\'\\')""])', 'CommandComplete(command=SELECT 1)', 'ReadyForQuery(state=idle)']") (14 rows) From e0faf344170cb088939cbe39c656dc59cf8f35b1 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Tue, 24 Aug 2021 08:52:36 +0300 Subject: [PATCH 079/104] turn off costs in columnar_indexes explain query --- src/test/regress/expected/columnar_indexes.out | 16 ++++++++-------- src/test/regress/sql/columnar_indexes.sql | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/test/regress/expected/columnar_indexes.out b/src/test/regress/expected/columnar_indexes.out index 3789a8403..ac61c6c73 100644 --- a/src/test/regress/expected/columnar_indexes.out +++ b/src/test/regress/expected/columnar_indexes.out @@ -20,11 +20,11 @@ REINDEX INDEX CONCURRENTLY t_idx; Indexes: "t_idx" btree (a, b) -explain insert into t values (1, 2); - QUERY PLAN +explain (COSTS OFF) insert into t values (1, 2); + QUERY PLAN --------------------------------------------------------------------- - Insert on t (cost=0.00..0.01 rows=1 width=8) - -> Result (cost=0.00..0.01 rows=1 width=8) + Insert on t + -> Result (2 rows) insert into t values (1, 2); @@ -34,11 +34,11 @@ SELECT * FROM t; 1 | 2 (1 row) -explain insert into t values (1, 2); - QUERY PLAN +explain (COSTS OFF) insert into t values (1, 2); + QUERY PLAN --------------------------------------------------------------------- - Insert on t (cost=0.00..0.01 rows=1 width=8) - -> Result (cost=0.00..0.01 rows=1 width=8) + Insert on t + -> Result (2 rows) insert into t values (3, 4); diff --git a/src/test/regress/sql/columnar_indexes.sql b/src/test/regress/sql/columnar_indexes.sql index 897181645..5c27812a2 100644 --- a/src/test/regress/sql/columnar_indexes.sql +++ b/src/test/regress/sql/columnar_indexes.sql @@ -14,11 +14,11 @@ create table t(a int, b int) using columnar; create index CONCURRENTLY t_idx on t(a, b); REINDEX INDEX CONCURRENTLY t_idx; \d t -explain insert into t values (1, 2); +explain (COSTS OFF) insert into t values (1, 2); insert into t values (1, 2); SELECT * FROM t; -explain insert into t values (1, 2); +explain (COSTS OFF) insert into t values (1, 2); insert into t values (3, 4); SELECT * FROM t; From e7607b6bed9244b8abb5b346e0870fbbddae324f Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Tue, 24 Aug 2021 09:06:29 +0300 Subject: [PATCH 080/104] Add a helper function to check explain has a single task In order to avoid adding an alternative output, a function to check if a given explan plan has a single task added. This doesn't change what the changed tests intend to do. --- .../drop_column_partitioned_table.out | 111 +++++++----------- .../regress/expected/multi_test_helpers.out | 14 +++ .../sql/drop_column_partitioned_table.sql | 31 ++++- src/test/regress/sql/multi_test_helpers.sql | 15 +++ 4 files changed, 101 insertions(+), 70 deletions(-) diff --git a/src/test/regress/expected/drop_column_partitioned_table.out b/src/test/regress/expected/drop_column_partitioned_table.out index a32602511..57ef66d7c 100644 --- a/src/test/regress/expected/drop_column_partitioned_table.out +++ b/src/test/regress/expected/drop_column_partitioned_table.out @@ -226,78 +226,55 @@ EXPLAIN (COSTS FALSE) INSERT INTO sensors_2003 VALUES (3, '2003-01-01', row_to_j -> Result (7 rows) +SELECT public.explain_has_single_task( + $$ EXPLAIN (COSTS FALSE) SELECT count(*) FROM sensors WHERE measureid = 3 AND eventdatetime = '2000-02-02'; - QUERY PLAN + $$ +); + explain_has_single_task --------------------------------------------------------------------- - Custom Scan (Citus Adaptive) - Task Count: 1 - Tasks Shown: All - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Aggregate - -> Index Only Scan using sensors_2000_pkey_2580005 on sensors_2000_2580005 sensors - Index Cond: ((measureid = 3) AND (eventdatetime = '2000-02-02'::date)) -(8 rows) - -EXPLAIN (COSTS FALSE) SELECT count(*) FROM sensors_2000 WHERE measureid = 3; - QUERY PLAN ---------------------------------------------------------------------- - Custom Scan (Citus Adaptive) - Task Count: 1 - Tasks Shown: All - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Aggregate - -> Bitmap Heap Scan on sensors_2000_2580005 sensors_xxx - Recheck Cond: (measureid = 3) - -> Bitmap Index Scan on sensors_2000_pkey_2580005 - Index Cond: (measureid = 3) -(10 rows) - -EXPLAIN (COSTS FALSE) SELECT count(*) FROM sensors_2001 WHERE measureid = 3; - QUERY PLAN ---------------------------------------------------------------------- - Custom Scan (Citus Adaptive) - Task Count: 1 - Tasks Shown: All - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Aggregate - -> Bitmap Heap Scan on sensors_2001_2580009 sensors_xxx - Recheck Cond: (measureid = 3) - -> Bitmap Index Scan on sensors_2001_pkey_2580009 - Index Cond: (measureid = 3) -(10 rows) - -EXPLAIN (COSTS FALSE) SELECT count(*) FROM sensors_2002 WHERE measureid = 3; - QUERY PLAN ---------------------------------------------------------------------- - Custom Scan (Citus Adaptive) - Task Count: 1 - Tasks Shown: All - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Aggregate - -> Bitmap Heap Scan on sensors_2002_2580013 sensors_xxx - Recheck Cond: (measureid = 3) - -> Bitmap Index Scan on sensors_2002_pkey_2580013 - Index Cond: (measureid = 3) -(10 rows) + t +(1 row) +SELECT public.explain_has_single_task( + $$ EXPLAIN (COSTS FALSE) SELECT count(*) FROM sensors_2003 WHERE measureid = 3; - QUERY PLAN + $$ +); + explain_has_single_task --------------------------------------------------------------------- - Custom Scan (Citus Adaptive) - Task Count: 1 - Tasks Shown: All - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Aggregate - -> Bitmap Heap Scan on sensors_2003_2580017 sensors_xxx - Recheck Cond: (measureid = 3) - -> Bitmap Index Scan on sensors_2003_pkey_2580017 - Index Cond: (measureid = 3) -(10 rows) + t +(1 row) + +SELECT public.explain_has_single_task( + $$ +EXPLAIN (COSTS FALSE) SELECT count(*) FROM sensors_2000 WHERE measureid = 3; + $$ +); + explain_has_single_task +--------------------------------------------------------------------- + t +(1 row) + +SELECT public.explain_has_single_task( + $$ +EXPLAIN (COSTS FALSE) SELECT count(*) FROM sensors_2001 WHERE measureid = 3; + $$ +); + explain_has_single_task +--------------------------------------------------------------------- + t +(1 row) + +SELECT public.explain_has_single_task( + $$ +EXPLAIN (COSTS FALSE) SELECT count(*) FROM sensors_2002 WHERE measureid = 3; + $$ +); + explain_has_single_task +--------------------------------------------------------------------- + t +(1 row) -- execute 7 times to make sure it is re-cached EXECUTE drop_col_prepare_insert(3, '2000-10-01', row_to_json(row(1))); diff --git a/src/test/regress/expected/multi_test_helpers.out b/src/test/regress/expected/multi_test_helpers.out index 34f44361f..f46e6bcc8 100644 --- a/src/test/regress/expected/multi_test_helpers.out +++ b/src/test/regress/expected/multi_test_helpers.out @@ -57,6 +57,20 @@ BEGIN END LOOP; RETURN false; END; $$ language plpgsql; +--helper function to check there is a single task +CREATE OR REPLACE FUNCTION explain_has_single_task(explain_command text) +RETURNS BOOLEAN AS $$ +DECLARE + query_plan text; +BEGIN + FOR query_plan IN EXECUTE explain_command LOOP + IF query_plan ILIKE '%Task Count: 1%' + THEN + RETURN true; + END IF; + END LOOP; + RETURN false; +END; $$ language plpgsql; -- helper function to quickly run SQL on the whole cluster CREATE OR REPLACE FUNCTION run_command_on_coordinator_and_workers(p_sql text) RETURNS void LANGUAGE plpgsql AS $$ diff --git a/src/test/regress/sql/drop_column_partitioned_table.sql b/src/test/regress/sql/drop_column_partitioned_table.sql index 719c6c0bb..d57ad3805 100644 --- a/src/test/regress/sql/drop_column_partitioned_table.sql +++ b/src/test/regress/sql/drop_column_partitioned_table.sql @@ -111,11 +111,36 @@ EXPLAIN (COSTS FALSE) INSERT INTO sensors_2001 VALUES (3, '2001-01-01', row_to_j EXPLAIN (COSTS FALSE) INSERT INTO sensors_2002 VALUES (3, '2002-01-01', row_to_json(row(1))); EXPLAIN (COSTS FALSE) INSERT INTO sensors_2003 VALUES (3, '2003-01-01', row_to_json(row(1))); +SELECT public.explain_has_single_task( + $$ EXPLAIN (COSTS FALSE) SELECT count(*) FROM sensors WHERE measureid = 3 AND eventdatetime = '2000-02-02'; -EXPLAIN (COSTS FALSE) SELECT count(*) FROM sensors_2000 WHERE measureid = 3; -EXPLAIN (COSTS FALSE) SELECT count(*) FROM sensors_2001 WHERE measureid = 3; -EXPLAIN (COSTS FALSE) SELECT count(*) FROM sensors_2002 WHERE measureid = 3; + $$ +); + +SELECT public.explain_has_single_task( + $$ EXPLAIN (COSTS FALSE) SELECT count(*) FROM sensors_2003 WHERE measureid = 3; + $$ +); + +SELECT public.explain_has_single_task( + $$ +EXPLAIN (COSTS FALSE) SELECT count(*) FROM sensors_2000 WHERE measureid = 3; + $$ +); + +SELECT public.explain_has_single_task( + $$ +EXPLAIN (COSTS FALSE) SELECT count(*) FROM sensors_2001 WHERE measureid = 3; + $$ +); + +SELECT public.explain_has_single_task( + $$ +EXPLAIN (COSTS FALSE) SELECT count(*) FROM sensors_2002 WHERE measureid = 3; + $$ +); + -- execute 7 times to make sure it is re-cached EXECUTE drop_col_prepare_insert(3, '2000-10-01', row_to_json(row(1))); diff --git a/src/test/regress/sql/multi_test_helpers.sql b/src/test/regress/sql/multi_test_helpers.sql index 2cf3bc98a..8c035c8e2 100644 --- a/src/test/regress/sql/multi_test_helpers.sql +++ b/src/test/regress/sql/multi_test_helpers.sql @@ -63,6 +63,21 @@ BEGIN RETURN false; END; $$ language plpgsql; +--helper function to check there is a single task +CREATE OR REPLACE FUNCTION explain_has_single_task(explain_command text) +RETURNS BOOLEAN AS $$ +DECLARE + query_plan text; +BEGIN + FOR query_plan IN EXECUTE explain_command LOOP + IF query_plan ILIKE '%Task Count: 1%' + THEN + RETURN true; + END IF; + END LOOP; + RETURN false; +END; $$ language plpgsql; + -- helper function to quickly run SQL on the whole cluster CREATE OR REPLACE FUNCTION run_command_on_coordinator_and_workers(p_sql text) RETURNS void LANGUAGE plpgsql AS $$ From 96964aeee5c0bfbd79e68e00742a3cbff04619f9 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Tue, 24 Aug 2021 09:17:24 +0300 Subject: [PATCH 081/104] Turn off debug for one query to avoid adding an alternative output --- .../expected/subquery_complex_target_list.out | 13 ++++--------- .../regress/sql/subquery_complex_target_list.sql | 5 +++++ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/test/regress/expected/subquery_complex_target_list.out b/src/test/regress/expected/subquery_complex_target_list.out index d53434a76..8ac4fac2e 100644 --- a/src/test/regress/expected/subquery_complex_target_list.out +++ b/src/test/regress/expected/subquery_complex_target_list.out @@ -126,6 +126,9 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT foo."?column? 2 | 3 | 376 | 101 | 4 | 0 | 2.5 | 273 | 101 | 2.7029702970297 | Wed Nov 22 18:19:49.944985 2017 | Thu Nov 23 17:30:34.635085 2017 | 101 | 17 | 17 (1 row) +-- we reset the client min_messages here to avoid adding an alternative output +-- for pg14 as the output slightly differs. +RESET client_min_messages; -- Expressions inside the aggregates -- parts of the query is inspired by TPCH queries SELECT @@ -167,20 +170,12 @@ FROM events_table WHERE foo.avg != bar.cnt_1 AND baz.cnt_2 != events_table.event_type ORDER BY 1 DESC; -DEBUG: push down of limit count: 3 -DEBUG: generating subplan XXX_1 for subquery SELECT avg(((user_id)::numeric OPERATOR(pg_catalog.*) (5.0 OPERATOR(pg_catalog./) ((value_1)::numeric OPERATOR(pg_catalog.+) 0.1)))) AS avg FROM public.users_table ORDER BY (avg(((user_id)::numeric OPERATOR(pg_catalog.*) (5.0 OPERATOR(pg_catalog./) ((value_1)::numeric OPERATOR(pg_catalog.+) 0.1))))) DESC LIMIT 3 -DEBUG: push down of limit count: 3 -DEBUG: generating subplan XXX_2 for subquery SELECT sum(((((user_id)::numeric OPERATOR(pg_catalog.*) (5.0 OPERATOR(pg_catalog./) (((value_1 OPERATOR(pg_catalog.+) value_2))::numeric OPERATOR(pg_catalog.+) 0.1))))::double precision OPERATOR(pg_catalog.*) value_3)) AS cnt_1 FROM public.users_table ORDER BY (sum(((((user_id)::numeric OPERATOR(pg_catalog.*) (5.0 OPERATOR(pg_catalog./) (((value_1 OPERATOR(pg_catalog.+) value_2))::numeric OPERATOR(pg_catalog.+) 0.1))))::double precision OPERATOR(pg_catalog.*) value_3))) DESC LIMIT 3 -DEBUG: push down of limit count: 4 -DEBUG: generating subplan XXX_3 for subquery SELECT avg(CASE WHEN (user_id OPERATOR(pg_catalog.>) 4) THEN value_1 ELSE NULL::integer END) AS cnt_2, avg(CASE WHEN (user_id OPERATOR(pg_catalog.>) 500) THEN value_1 ELSE NULL::integer END) AS cnt_3, sum(CASE WHEN ((value_1 OPERATOR(pg_catalog.=) 1) OR (value_2 OPERATOR(pg_catalog.=) 1)) THEN 1 ELSE 0 END) AS sum_1, date_part('year'::text, max("time")) AS l_year, strpos((max(user_id))::text, '1'::text) AS pos FROM public.users_table ORDER BY (avg(CASE WHEN (user_id OPERATOR(pg_catalog.>) 4) THEN value_1 ELSE NULL::integer END)) DESC LIMIT 4 -DEBUG: push down of limit count: 25 -DEBUG: generating subplan XXX_4 for subquery SELECT COALESCE(value_3, (20)::double precision) AS count_pay FROM public.users_table ORDER BY COALESCE(value_3, (20)::double precision) OFFSET 20 LIMIT 5 -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT DISTINCT ON (foo.avg) foo.avg, bar.cnt_1, baz.cnt_2, baz.cnt_3, baz.sum_1, baz.l_year, baz.pos, tar.count_pay FROM (SELECT intermediate_result.avg FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(avg numeric)) foo, (SELECT intermediate_result.cnt_1 FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(cnt_1 double precision)) bar, (SELECT intermediate_result.cnt_2, intermediate_result.cnt_3, intermediate_result.sum_1, intermediate_result.l_year, intermediate_result.pos FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(cnt_2 numeric, cnt_3 numeric, sum_1 bigint, l_year double precision, pos integer)) baz, (SELECT intermediate_result.count_pay FROM read_intermediate_result('XXX_4'::text, 'binary'::citus_copy_format) intermediate_result(count_pay double precision)) tar, public.events_table WHERE (((foo.avg)::double precision OPERATOR(pg_catalog.<>) bar.cnt_1) AND (baz.cnt_2 OPERATOR(pg_catalog.<>) (events_table.event_type)::numeric)) ORDER BY foo.avg DESC avg | cnt_1 | cnt_2 | cnt_3 | sum_1 | l_year | pos | count_pay --------------------------------------------------------------------- 30.14666771571734992301 | 3308.14619815793 | 2.5000000000000000 | | 31 | 2017 | 0 | 1 (1 row) +SET client_min_messages TO DEBUG1; -- Multiple columns in GROUP BYs -- foo needs to be recursively planned, bar can be pushded down SELECT diff --git a/src/test/regress/sql/subquery_complex_target_list.sql b/src/test/regress/sql/subquery_complex_target_list.sql index d2d3a2c5e..1579ff017 100644 --- a/src/test/regress/sql/subquery_complex_target_list.sql +++ b/src/test/regress/sql/subquery_complex_target_list.sql @@ -87,6 +87,9 @@ FROM ) as baz ORDER BY 1 DESC; +-- we reset the client min_messages here to avoid adding an alternative output +-- for pg14 as the output slightly differs. +RESET client_min_messages; -- Expressions inside the aggregates -- parts of the query is inspired by TPCH queries SELECT @@ -128,6 +131,8 @@ FROM events_table WHERE foo.avg != bar.cnt_1 AND baz.cnt_2 != events_table.event_type ORDER BY 1 DESC; +SET client_min_messages TO DEBUG1; + -- Multiple columns in GROUP BYs -- foo needs to be recursively planned, bar can be pushded down From 4b951a2ed96b752b54db2907b8eba0004e5490a7 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Tue, 24 Aug 2021 09:32:24 +0300 Subject: [PATCH 082/104] Add alternative output for multi-mx --- src/test/regress/bin/normalize.sed | 1 + src/test/regress/expected/multi_mx_call.out | 48 +- src/test/regress/expected/multi_mx_call_0.out | 533 +++++++++++++ .../regress/expected/multi_mx_explain.out | 42 - .../multi_mx_function_call_delegation.out | 89 ++- .../multi_mx_function_call_delegation_0.out | 724 ++++++++++++++++++ .../sql/multi_mx_function_call_delegation.sql | 5 +- 7 files changed, 1332 insertions(+), 110 deletions(-) create mode 100644 src/test/regress/expected/multi_mx_call_0.out create mode 100644 src/test/regress/expected/multi_mx_function_call_delegation_0.out diff --git a/src/test/regress/bin/normalize.sed b/src/test/regress/bin/normalize.sed index 5ede38e35..ae4b63ac7 100644 --- a/src/test/regress/bin/normalize.sed +++ b/src/test/regress/bin/normalize.sed @@ -240,3 +240,4 @@ s/ERROR: ROLLBACK is not allowed in an SQL function/ERROR: ROLLBACK is not all /Parent Relationship/d /Parent-Relationship/d s/function array_cat_agg\(anycompatiblearray\)/function array_cat_agg\(anyarray\)/g +s/TRIM\(BOTH FROM value\)/btrim\(value\)/g diff --git a/src/test/regress/expected/multi_mx_call.out b/src/test/regress/expected/multi_mx_call.out index 8511ed143..c493fdcd4 100644 --- a/src/test/regress/expected/multi_mx_call.out +++ b/src/test/regress/expected/multi_mx_call.out @@ -182,10 +182,10 @@ SET client_min_messages TO DEBUG1; call multi_mx_call.mx_call_proc(2, 0); DEBUG: stored procedure does not have co-located tables DEBUG: generating subplan XXX_1 for subquery SELECT sum((t1.val OPERATOR(pg_catalog.+) t2.val)) AS sum FROM (multi_mx_call.mx_call_dist_table_1 t1 JOIN multi_mx_call.mx_call_dist_table_2 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.id))) -CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" +CONTEXT: PL/pgSQL assignment "y := y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" PL/pgSQL function mx_call_proc(integer,integer) line 8 at assignment -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))) -CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT ((3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))))::integer +CONTEXT: PL/pgSQL assignment "y := y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" PL/pgSQL function mx_call_proc(integer,integer) line 8 at assignment y --------------------------------------------------------------------- @@ -266,10 +266,10 @@ begin; call multi_mx_call.mx_call_proc(2, 0); DEBUG: cannot push down CALL in multi-statement transaction DEBUG: generating subplan XXX_1 for subquery SELECT sum((t1.val OPERATOR(pg_catalog.+) t2.val)) AS sum FROM (multi_mx_call.mx_call_dist_table_1 t1 JOIN multi_mx_call.mx_call_dist_table_2 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.id))) -CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" +CONTEXT: PL/pgSQL assignment "y := y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" PL/pgSQL function mx_call_proc(integer,integer) line 8 at assignment -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))) -CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT ((3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))))::integer +CONTEXT: PL/pgSQL assignment "y := y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" PL/pgSQL function mx_call_proc(integer,integer) line 8 at assignment y --------------------------------------------------------------------- @@ -300,10 +300,10 @@ select colocate_proc_with_table('mx_call_proc', 'mx_call_dist_table_1'::regclass call multi_mx_call.mx_call_proc(2, 0); DEBUG: cannot push down invalid distribution_argument_index DEBUG: generating subplan XXX_1 for subquery SELECT sum((t1.val OPERATOR(pg_catalog.+) t2.val)) AS sum FROM (multi_mx_call.mx_call_dist_table_1 t1 JOIN multi_mx_call.mx_call_dist_table_2 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.id))) -CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" +CONTEXT: PL/pgSQL assignment "y := y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" PL/pgSQL function mx_call_proc(integer,integer) line 8 at assignment -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))) -CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT ((3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))))::integer +CONTEXT: PL/pgSQL assignment "y := y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" PL/pgSQL function mx_call_proc(integer,integer) line 8 at assignment y --------------------------------------------------------------------- @@ -319,10 +319,10 @@ select colocate_proc_with_table('mx_call_proc', 'mx_call_dist_table_1'::regclass call multi_mx_call.mx_call_proc(2, 0); DEBUG: cannot push down invalid distribution_argument_index DEBUG: generating subplan XXX_1 for subquery SELECT sum((t1.val OPERATOR(pg_catalog.+) t2.val)) AS sum FROM (multi_mx_call.mx_call_dist_table_1 t1 JOIN multi_mx_call.mx_call_dist_table_2 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.id))) -CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" +CONTEXT: PL/pgSQL assignment "y := y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" PL/pgSQL function mx_call_proc(integer,integer) line 8 at assignment -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))) -CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT ((3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))))::integer +CONTEXT: PL/pgSQL assignment "y := y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" PL/pgSQL function mx_call_proc(integer,integer) line 8 at assignment y --------------------------------------------------------------------- @@ -354,10 +354,10 @@ select colocate_proc_with_table('mx_call_proc', 'mx_call_dist_table_replica'::re call multi_mx_call.mx_call_proc(2, 0); DEBUG: cannot push down function call for replicated distributed tables DEBUG: generating subplan XXX_1 for subquery SELECT sum((t1.val OPERATOR(pg_catalog.+) t2.val)) AS sum FROM (multi_mx_call.mx_call_dist_table_1 t1 JOIN multi_mx_call.mx_call_dist_table_2 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.id))) -CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" +CONTEXT: PL/pgSQL assignment "y := y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" PL/pgSQL function mx_call_proc(integer,integer) line 8 at assignment -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))) -CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT ((3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))))::integer +CONTEXT: PL/pgSQL assignment "y := y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" PL/pgSQL function mx_call_proc(integer,integer) line 8 at assignment y --------------------------------------------------------------------- @@ -449,10 +449,10 @@ SET client_min_messages TO DEBUG1; call multi_mx_call.mx_call_proc(2, 0); DEBUG: there is no worker node with metadata DEBUG: generating subplan XXX_1 for subquery SELECT sum((t1.val OPERATOR(pg_catalog.+) t2.val)) AS sum FROM (multi_mx_call.mx_call_dist_table_1 t1 JOIN multi_mx_call.mx_call_dist_table_2 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.id))) -CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" +CONTEXT: PL/pgSQL assignment "y := y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" PL/pgSQL function mx_call_proc(integer,integer) line 8 at assignment -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))) -CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT ((3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))))::integer +CONTEXT: PL/pgSQL assignment "y := y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" PL/pgSQL function mx_call_proc(integer,integer) line 8 at assignment y --------------------------------------------------------------------- @@ -495,10 +495,10 @@ DETAIL: A distributed function is created. To make sure subsequent commands see call multi_mx_call.mx_call_proc(2, mx_call_add(3, 4)); DEBUG: distribution argument value must be a constant DEBUG: generating subplan XXX_1 for subquery SELECT sum((t1.val OPERATOR(pg_catalog.+) t2.val)) AS sum FROM (multi_mx_call.mx_call_dist_table_1 t1 JOIN multi_mx_call.mx_call_dist_table_2 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.id))) -CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" +CONTEXT: PL/pgSQL assignment "y := y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" PL/pgSQL function mx_call_proc(integer,integer) line 8 at assignment -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))) -CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT ((3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))))::integer +CONTEXT: PL/pgSQL assignment "y := y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" PL/pgSQL function mx_call_proc(integer,integer) line 8 at assignment y --------------------------------------------------------------------- @@ -517,10 +517,10 @@ DEBUG: pushing down the procedure call multi_mx_call.mx_call_proc(floor(random())::int, 2); DEBUG: arguments in a distributed stored procedure must be constant expressions DEBUG: generating subplan XXX_1 for subquery SELECT sum((t1.val OPERATOR(pg_catalog.+) t2.val)) AS sum FROM (multi_mx_call.mx_call_dist_table_1 t1 JOIN multi_mx_call.mx_call_dist_table_2 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.id))) -CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" +CONTEXT: PL/pgSQL assignment "y := y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" PL/pgSQL function mx_call_proc(integer,integer) line 8 at assignment -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (1 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))) -CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT ((1 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))))::integer +CONTEXT: PL/pgSQL assignment "y := y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" PL/pgSQL function mx_call_proc(integer,integer) line 8 at assignment y --------------------------------------------------------------------- diff --git a/src/test/regress/expected/multi_mx_call_0.out b/src/test/regress/expected/multi_mx_call_0.out new file mode 100644 index 000000000..8511ed143 --- /dev/null +++ b/src/test/regress/expected/multi_mx_call_0.out @@ -0,0 +1,533 @@ +-- Test passing off CALL to mx workers +create schema multi_mx_call; +set search_path to multi_mx_call, public; +-- Create worker-local tables to test procedure calls were routed +set citus.shard_replication_factor to 2; +-- This table requires specific settings, create before getting into things +create table mx_call_dist_table_replica(id int, val int); +select create_distributed_table('mx_call_dist_table_replica', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +insert into mx_call_dist_table_replica values (9,1),(8,2),(7,3),(6,4),(5,5); +set citus.shard_replication_factor to 1; +-- +-- Create tables and procedures we want to use in tests +-- +create table mx_call_dist_table_1(id int, val int); +select create_distributed_table('mx_call_dist_table_1', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +insert into mx_call_dist_table_1 values (3,1),(4,5),(9,2),(6,5),(3,5); +create table mx_call_dist_table_2(id int, val int); +select create_distributed_table('mx_call_dist_table_2', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +insert into mx_call_dist_table_2 values (1,1),(1,2),(2,2),(3,3),(3,4); +create table mx_call_dist_table_bigint(id bigint, val bigint); +select create_distributed_table('mx_call_dist_table_bigint', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +insert into mx_call_dist_table_bigint values (1,1),(1,2),(2,2),(3,3),(3,4); +create table mx_call_dist_table_ref(id int, val int); +select create_reference_table('mx_call_dist_table_ref'); + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +insert into mx_call_dist_table_ref values (2,7),(1,8),(2,8),(1,8),(2,8); +create type mx_call_enum as enum ('A', 'S', 'D', 'F'); +create table mx_call_dist_table_enum(id int, key mx_call_enum); +select create_distributed_table('mx_call_dist_table_enum', 'key'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +insert into mx_call_dist_table_enum values (1,'S'),(2,'A'),(3,'D'),(4,'F'); +-- test that a distributed function can be colocated with a reference table +CREATE TABLE ref(groupid int); +SELECT create_reference_table('ref'); + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +CREATE OR REPLACE PROCEDURE my_group_id_proc() +LANGUAGE plpgsql +SET search_path FROM CURRENT +AS $$ +DECLARE + gid int; +BEGIN + SELECT groupid INTO gid + FROM pg_dist_local_group; + + INSERT INTO ref(groupid) VALUES (gid); +END; +$$; +SELECT create_distributed_function('my_group_id_proc()', colocate_with := 'ref'); + create_distributed_function +--------------------------------------------------------------------- + +(1 row) + +CALL my_group_id_proc(); +CALL my_group_id_proc(); +SELECT DISTINCT(groupid) FROM ref ORDER BY 1; + groupid +--------------------------------------------------------------------- + 14 +(1 row) + +TRUNCATE TABLE ref; +-- test round robin task assignment policy uses different workers on consecutive procedure calls. +SET citus.task_assignment_policy TO 'round-robin'; +CALL my_group_id_proc(); +CALL my_group_id_proc(); +CALL my_group_id_proc(); +SELECT DISTINCT(groupid) FROM ref ORDER BY 1; + groupid +--------------------------------------------------------------------- + 14 + 18 +(2 rows) + +TRUNCATE TABLE ref; +RESET citus.task_assignment_policy; +CREATE PROCEDURE mx_call_proc(x int, INOUT y int) +LANGUAGE plpgsql AS $$ +BEGIN + -- groupid is 0 in coordinator and non-zero in workers, so by using it here + -- we make sure the procedure is being executed in the worker. + y := x + (select case groupid when 0 then 1 else 0 end from pg_dist_local_group); + -- we also make sure that we can run distributed queries in the procedures + -- that are routed to the workers. + y := y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id); +END;$$; +CREATE PROCEDURE mx_call_proc_bigint(x bigint, INOUT y bigint) +LANGUAGE plpgsql AS $$ +BEGIN + y := x + y * 2; +END;$$; +-- create another procedure which verifies: +-- 1. we work fine with multiple return columns +-- 2. we work fine in combination with custom types +CREATE PROCEDURE mx_call_proc_custom_types(INOUT x mx_call_enum, INOUT y mx_call_enum) +LANGUAGE plpgsql AS $$ +BEGIN + y := x; + x := (select case groupid when 0 then 'F' else 'S' end from pg_dist_local_group); +END;$$; +-- Test that undistributed procedures have no issue executing +call multi_mx_call.mx_call_proc(2, 0); + y +--------------------------------------------------------------------- + 29 +(1 row) + +call multi_mx_call.mx_call_proc_custom_types('S', 'A'); + x | y +--------------------------------------------------------------------- + F | S +(1 row) + +-- Same for unqualified names +call mx_call_proc(2, 0); + y +--------------------------------------------------------------------- + 29 +(1 row) + +call mx_call_proc_custom_types('S', 'A'); + x | y +--------------------------------------------------------------------- + F | S +(1 row) + +-- Mark both procedures as distributed ... +select create_distributed_function('mx_call_proc(int,int)'); + create_distributed_function +--------------------------------------------------------------------- + +(1 row) + +select create_distributed_function('mx_call_proc_bigint(bigint,bigint)'); + create_distributed_function +--------------------------------------------------------------------- + +(1 row) + +select create_distributed_function('mx_call_proc_custom_types(mx_call_enum,mx_call_enum)'); + create_distributed_function +--------------------------------------------------------------------- + +(1 row) + +-- We still don't route them to the workers, because they aren't +-- colocated with any distributed tables. +SET client_min_messages TO DEBUG1; +call multi_mx_call.mx_call_proc(2, 0); +DEBUG: stored procedure does not have co-located tables +DEBUG: generating subplan XXX_1 for subquery SELECT sum((t1.val OPERATOR(pg_catalog.+) t2.val)) AS sum FROM (multi_mx_call.mx_call_dist_table_1 t1 JOIN multi_mx_call.mx_call_dist_table_2 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.id))) +CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" +PL/pgSQL function mx_call_proc(integer,integer) line 8 at assignment +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))) +CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" +PL/pgSQL function mx_call_proc(integer,integer) line 8 at assignment + y +--------------------------------------------------------------------- + 29 +(1 row) + +call mx_call_proc_bigint(4, 2); +DEBUG: stored procedure does not have co-located tables + y +--------------------------------------------------------------------- + 8 +(1 row) + +call multi_mx_call.mx_call_proc_custom_types('S', 'A'); +DEBUG: stored procedure does not have co-located tables + x | y +--------------------------------------------------------------------- + F | S +(1 row) + +-- Mark them as colocated with a table. Now we should route them to workers. +select colocate_proc_with_table('mx_call_proc', 'mx_call_dist_table_1'::regclass, 1); + colocate_proc_with_table +--------------------------------------------------------------------- + +(1 row) + +select colocate_proc_with_table('mx_call_proc_bigint', 'mx_call_dist_table_bigint'::regclass, 1); + colocate_proc_with_table +--------------------------------------------------------------------- + +(1 row) + +select colocate_proc_with_table('mx_call_proc_custom_types', 'mx_call_dist_table_enum'::regclass, 1); + colocate_proc_with_table +--------------------------------------------------------------------- + +(1 row) + +call multi_mx_call.mx_call_proc(2, 0); +DEBUG: pushing down the procedure + y +--------------------------------------------------------------------- + 28 +(1 row) + +call multi_mx_call.mx_call_proc_custom_types('S', 'A'); +DEBUG: pushing down the procedure + x | y +--------------------------------------------------------------------- + S | S +(1 row) + +call mx_call_proc(2, 0); +DEBUG: pushing down the procedure + y +--------------------------------------------------------------------- + 28 +(1 row) + +call mx_call_proc_custom_types('S', 'A'); +DEBUG: pushing down the procedure + x | y +--------------------------------------------------------------------- + S | S +(1 row) + +-- Test implicit cast of int to bigint +call mx_call_proc_bigint(4, 2); +DEBUG: pushing down the procedure + y +--------------------------------------------------------------------- + 8 +(1 row) + +-- We don't allow distributing calls inside transactions +begin; +call multi_mx_call.mx_call_proc(2, 0); +DEBUG: cannot push down CALL in multi-statement transaction +DEBUG: generating subplan XXX_1 for subquery SELECT sum((t1.val OPERATOR(pg_catalog.+) t2.val)) AS sum FROM (multi_mx_call.mx_call_dist_table_1 t1 JOIN multi_mx_call.mx_call_dist_table_2 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.id))) +CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" +PL/pgSQL function mx_call_proc(integer,integer) line 8 at assignment +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))) +CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" +PL/pgSQL function mx_call_proc(integer,integer) line 8 at assignment + y +--------------------------------------------------------------------- + 29 +(1 row) + +commit; +-- Drop the table colocated with mx_call_proc_custom_types. Now it shouldn't +-- be routed to workers anymore. +SET client_min_messages TO NOTICE; +drop table mx_call_dist_table_enum; +SET client_min_messages TO DEBUG1; +call multi_mx_call.mx_call_proc_custom_types('S', 'A'); +DEBUG: stored procedure does not have co-located tables + x | y +--------------------------------------------------------------------- + F | S +(1 row) + +-- Make sure we do bounds checking on distributed argument index +-- This also tests that we have cache invalidation for pg_dist_object updates +select colocate_proc_with_table('mx_call_proc', 'mx_call_dist_table_1'::regclass, -1); + colocate_proc_with_table +--------------------------------------------------------------------- + +(1 row) + +call multi_mx_call.mx_call_proc(2, 0); +DEBUG: cannot push down invalid distribution_argument_index +DEBUG: generating subplan XXX_1 for subquery SELECT sum((t1.val OPERATOR(pg_catalog.+) t2.val)) AS sum FROM (multi_mx_call.mx_call_dist_table_1 t1 JOIN multi_mx_call.mx_call_dist_table_2 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.id))) +CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" +PL/pgSQL function mx_call_proc(integer,integer) line 8 at assignment +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))) +CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" +PL/pgSQL function mx_call_proc(integer,integer) line 8 at assignment + y +--------------------------------------------------------------------- + 29 +(1 row) + +select colocate_proc_with_table('mx_call_proc', 'mx_call_dist_table_1'::regclass, 2); + colocate_proc_with_table +--------------------------------------------------------------------- + +(1 row) + +call multi_mx_call.mx_call_proc(2, 0); +DEBUG: cannot push down invalid distribution_argument_index +DEBUG: generating subplan XXX_1 for subquery SELECT sum((t1.val OPERATOR(pg_catalog.+) t2.val)) AS sum FROM (multi_mx_call.mx_call_dist_table_1 t1 JOIN multi_mx_call.mx_call_dist_table_2 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.id))) +CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" +PL/pgSQL function mx_call_proc(integer,integer) line 8 at assignment +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))) +CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" +PL/pgSQL function mx_call_proc(integer,integer) line 8 at assignment + y +--------------------------------------------------------------------- + 29 +(1 row) + +-- We support colocating with reference tables +select colocate_proc_with_table('mx_call_proc', 'mx_call_dist_table_ref'::regclass, NULL); + colocate_proc_with_table +--------------------------------------------------------------------- + +(1 row) + +call multi_mx_call.mx_call_proc(2, 0); +DEBUG: will push down CALL for reference tables +DEBUG: pushing down the procedure + y +--------------------------------------------------------------------- + 28 +(1 row) + +-- We don't currently support colocating with replicated tables +select colocate_proc_with_table('mx_call_proc', 'mx_call_dist_table_replica'::regclass, 1); + colocate_proc_with_table +--------------------------------------------------------------------- + +(1 row) + +call multi_mx_call.mx_call_proc(2, 0); +DEBUG: cannot push down function call for replicated distributed tables +DEBUG: generating subplan XXX_1 for subquery SELECT sum((t1.val OPERATOR(pg_catalog.+) t2.val)) AS sum FROM (multi_mx_call.mx_call_dist_table_1 t1 JOIN multi_mx_call.mx_call_dist_table_2 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.id))) +CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" +PL/pgSQL function mx_call_proc(integer,integer) line 8 at assignment +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))) +CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" +PL/pgSQL function mx_call_proc(integer,integer) line 8 at assignment + y +--------------------------------------------------------------------- + 29 +(1 row) + +SET client_min_messages TO NOTICE; +drop table mx_call_dist_table_replica; +SET client_min_messages TO DEBUG1; +select colocate_proc_with_table('mx_call_proc', 'mx_call_dist_table_1'::regclass, 1); + colocate_proc_with_table +--------------------------------------------------------------------- + +(1 row) + +-- Test that we handle transactional constructs correctly inside a procedure +-- that is routed to the workers. +CREATE PROCEDURE mx_call_proc_tx(x int) LANGUAGE plpgsql AS $$ +BEGIN + INSERT INTO multi_mx_call.mx_call_dist_table_1 VALUES (x, -1), (x+1, 4); + COMMIT; + UPDATE multi_mx_call.mx_call_dist_table_1 SET val = val+1 WHERE id >= x; + ROLLBACK; + -- Now do the final update! + UPDATE multi_mx_call.mx_call_dist_table_1 SET val = val-1 WHERE id >= x; +END;$$; +-- before distribution ... +CALL multi_mx_call.mx_call_proc_tx(10); +-- after distribution ... +select create_distributed_function('mx_call_proc_tx(int)', '$1', 'mx_call_dist_table_1'); +DEBUG: switching to sequential query execution mode +DETAIL: A distributed function is created. To make sure subsequent commands see the type correctly we need to make sure to use only one connection for all future commands + create_distributed_function +--------------------------------------------------------------------- + +(1 row) + +CALL multi_mx_call.mx_call_proc_tx(20); +DEBUG: pushing down the procedure +SELECT id, val FROM mx_call_dist_table_1 ORDER BY id, val; + id | val +--------------------------------------------------------------------- + 3 | 1 + 3 | 5 + 4 | 5 + 6 | 5 + 9 | 2 + 10 | -2 + 11 | 3 + 20 | -2 + 21 | 3 +(9 rows) + +-- Test that we properly propagate errors raised from procedures. +CREATE PROCEDURE mx_call_proc_raise(x int) LANGUAGE plpgsql AS $$ +BEGIN + RAISE WARNING 'warning'; + RAISE EXCEPTION 'error'; +END;$$; +select create_distributed_function('mx_call_proc_raise(int)', '$1', 'mx_call_dist_table_1'); +DEBUG: switching to sequential query execution mode +DETAIL: A distributed function is created. To make sure subsequent commands see the type correctly we need to make sure to use only one connection for all future commands + create_distributed_function +--------------------------------------------------------------------- + +(1 row) + +\set VERBOSITY terse +call multi_mx_call.mx_call_proc_raise(2); +DEBUG: pushing down the procedure +WARNING: warning +ERROR: error +\set VERBOSITY default +-- Test that we don't propagate to non-metadata worker nodes +SET client_min_messages TO WARNING; +select stop_metadata_sync_to_node('localhost', :worker_1_port); + stop_metadata_sync_to_node +--------------------------------------------------------------------- + +(1 row) + +select stop_metadata_sync_to_node('localhost', :worker_2_port); + stop_metadata_sync_to_node +--------------------------------------------------------------------- + +(1 row) + +SET client_min_messages TO DEBUG1; +call multi_mx_call.mx_call_proc(2, 0); +DEBUG: there is no worker node with metadata +DEBUG: generating subplan XXX_1 for subquery SELECT sum((t1.val OPERATOR(pg_catalog.+) t2.val)) AS sum FROM (multi_mx_call.mx_call_dist_table_1 t1 JOIN multi_mx_call.mx_call_dist_table_2 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.id))) +CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" +PL/pgSQL function mx_call_proc(integer,integer) line 8 at assignment +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))) +CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" +PL/pgSQL function mx_call_proc(integer,integer) line 8 at assignment + y +--------------------------------------------------------------------- + 29 +(1 row) + +SET client_min_messages TO NOTICE; +select start_metadata_sync_to_node('localhost', :worker_1_port); + start_metadata_sync_to_node +--------------------------------------------------------------------- + +(1 row) + +select start_metadata_sync_to_node('localhost', :worker_2_port); + start_metadata_sync_to_node +--------------------------------------------------------------------- + +(1 row) + +-- stop_metadata_sync_to_node()/start_metadata_sync_to_node() might make +-- worker backend caches inconsistent. Reconnect to coordinator to use +-- new worker connections, hence new backends. +\c - - - :master_port +SET search_path to multi_mx_call, public; +SET client_min_messages TO DEBUG1; +-- +-- Test non-const parameter values +-- +CREATE FUNCTION mx_call_add(int, int) RETURNS int + AS 'select $1 + $2;' LANGUAGE SQL IMMUTABLE; +SELECT create_distributed_function('mx_call_add(int,int)'); +DEBUG: switching to sequential query execution mode +DETAIL: A distributed function is created. To make sure subsequent commands see the type correctly we need to make sure to use only one connection for all future commands + create_distributed_function +--------------------------------------------------------------------- + +(1 row) + +-- non-const distribution parameters cannot be pushed down +call multi_mx_call.mx_call_proc(2, mx_call_add(3, 4)); +DEBUG: distribution argument value must be a constant +DEBUG: generating subplan XXX_1 for subquery SELECT sum((t1.val OPERATOR(pg_catalog.+) t2.val)) AS sum FROM (multi_mx_call.mx_call_dist_table_1 t1 JOIN multi_mx_call.mx_call_dist_table_2 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.id))) +CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" +PL/pgSQL function mx_call_proc(integer,integer) line 8 at assignment +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))) +CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" +PL/pgSQL function mx_call_proc(integer,integer) line 8 at assignment + y +--------------------------------------------------------------------- + 29 +(1 row) + +-- non-const parameter can be pushed down +call multi_mx_call.mx_call_proc(multi_mx_call.mx_call_add(3, 4), 2); +DEBUG: pushing down the procedure + y +--------------------------------------------------------------------- + 33 +(1 row) + +-- volatile parameter cannot be pushed down +call multi_mx_call.mx_call_proc(floor(random())::int, 2); +DEBUG: arguments in a distributed stored procedure must be constant expressions +DEBUG: generating subplan XXX_1 for subquery SELECT sum((t1.val OPERATOR(pg_catalog.+) t2.val)) AS sum FROM (multi_mx_call.mx_call_dist_table_1 t1 JOIN multi_mx_call.mx_call_dist_table_2 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.id))) +CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" +PL/pgSQL function mx_call_proc(integer,integer) line 8 at assignment +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (1 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))) +CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_call.mx_call_dist_table_1 t1 join multi_mx_call.mx_call_dist_table_2 t2 on t1.id = t2.id)" +PL/pgSQL function mx_call_proc(integer,integer) line 8 at assignment + y +--------------------------------------------------------------------- + 27 +(1 row) + +reset client_min_messages; +\set VERBOSITY terse +drop schema multi_mx_call cascade; +NOTICE: drop cascades to 13 other objects diff --git a/src/test/regress/expected/multi_mx_explain.out b/src/test/regress/expected/multi_mx_explain.out index ab996275c..2c58dd003 100644 --- a/src/test/regress/expected/multi_mx_explain.out +++ b/src/test/regress/expected/multi_mx_explain.out @@ -88,13 +88,11 @@ EXPLAIN (COSTS FALSE, FORMAT JSON) "Node Type": "Aggregate", "Strategy": "Hashed", "Partial Mode": "Simple", - "Parent Relationship": "Outer", "Parallel Aware": false, "Group Key": ["remote_scan.l_quantity"], "Plans": [ { "Node Type": "Custom Scan", - "Parent Relationship": "Outer", "Custom Plan Provider": "Citus Adaptive", "Parallel Aware": false, "Distributed Query": { @@ -116,7 +114,6 @@ EXPLAIN (COSTS FALSE, FORMAT JSON) "Plans": [ { "Node Type": "Seq Scan", - "Parent Relationship": "Outer", "Parallel Aware": false, "Relation Name": "lineitem_mx_1220052", "Alias": "lineitem_mx" @@ -162,7 +159,6 @@ EXPLAIN (COSTS FALSE, FORMAT XML) Aggregate Hashed Simple - Outer false remote_scan.l_quantity @@ -170,7 +166,6 @@ EXPLAIN (COSTS FALSE, FORMAT XML) Custom Scan - Outer Citus Adaptive false @@ -194,7 +189,6 @@ EXPLAIN (COSTS FALSE, FORMAT XML) Seq Scan - Outer false lineitem_mx_1220052 lineitem_mx @@ -234,13 +228,11 @@ EXPLAIN (COSTS FALSE, FORMAT YAML) - Node Type: "Aggregate" Strategy: "Hashed" Partial Mode: "Simple" - Parent Relationship: "Outer" Parallel Aware: false Group Key: - "remote_scan.l_quantity" Plans: - Node Type: "Custom Scan" - Parent Relationship: "Outer" Custom Plan Provider: "Citus Adaptive" Parallel Aware: false Distributed Query: @@ -259,7 +251,6 @@ EXPLAIN (COSTS FALSE, FORMAT YAML) - "l_quantity" Plans: - Node Type: "Seq Scan" - Parent Relationship: "Outer" Parallel Aware: false Relation Name: "lineitem_mx_1220052" Alias: "lineitem_mx" @@ -537,7 +528,6 @@ EXPLAIN (COSTS FALSE, FORMAT JSON) "Plans": [ { "Node Type": "Custom Scan", - "Parent Relationship": "Outer", "Custom Plan Provider": "Citus Adaptive", "Parallel Aware": false, "Distributed Query": { @@ -558,7 +548,6 @@ EXPLAIN (COSTS FALSE, FORMAT JSON) "Plans": [ { "Node Type": "Hash Join", - "Parent Relationship": "Outer", "Parallel Aware": false, "Join Type": "Inner", "Inner Unique": false, @@ -566,7 +555,6 @@ EXPLAIN (COSTS FALSE, FORMAT JSON) "Plans": [ { "Node Type": "Hash Join", - "Parent Relationship": "Outer", "Parallel Aware": false, "Join Type": "Inner", "Inner Unique": false, @@ -574,19 +562,16 @@ EXPLAIN (COSTS FALSE, FORMAT JSON) "Plans": [ { "Node Type": "Seq Scan", - "Parent Relationship": "Outer", "Parallel Aware": false, "Relation Name": "supplier_mx_1220087", "Alias": "supplier_mx" }, { "Node Type": "Hash", - "Parent Relationship": "Inner", "Parallel Aware": false, "Plans": [ { "Node Type": "Seq Scan", - "Parent Relationship": "Outer", "Parallel Aware": false, "Relation Name": "lineitem_mx_1220052", "Alias": "lineitem_mx" @@ -597,12 +582,10 @@ EXPLAIN (COSTS FALSE, FORMAT JSON) }, { "Node Type": "Hash", - "Parent Relationship": "Inner", "Parallel Aware": false, "Plans": [ { "Node Type": "Hash Join", - "Parent Relationship": "Outer", "Parallel Aware": false, "Join Type": "Inner", "Inner Unique": false, @@ -610,19 +593,16 @@ EXPLAIN (COSTS FALSE, FORMAT JSON) "Plans": [ { "Node Type": "Seq Scan", - "Parent Relationship": "Outer", "Parallel Aware": false, "Relation Name": "customer_mx_1220084", "Alias": "customer_mx" }, { "Node Type": "Hash", - "Parent Relationship": "Inner", "Parallel Aware": false, "Plans": [ { "Node Type": "Seq Scan", - "Parent Relationship": "Outer", "Parallel Aware": false, "Relation Name": "orders_mx_1220068", "Alias": "orders_mx" @@ -673,7 +653,6 @@ EXPLAIN (COSTS FALSE, FORMAT XML) Custom Scan - Outer Citus Adaptive false @@ -694,7 +673,6 @@ EXPLAIN (COSTS FALSE, FORMAT XML) Hash Join - Outer false Inner false @@ -702,7 +680,6 @@ EXPLAIN (COSTS FALSE, FORMAT XML) Hash Join - Outer false Inner false @@ -710,19 +687,16 @@ EXPLAIN (COSTS FALSE, FORMAT XML) Seq Scan - Outer false supplier_mx_1220087 supplier_mx Hash - Inner false Seq Scan - Outer false lineitem_mx_1220052 lineitem_mx @@ -733,12 +707,10 @@ EXPLAIN (COSTS FALSE, FORMAT XML) Hash - Inner false Hash Join - Outer false Inner false @@ -746,19 +718,16 @@ EXPLAIN (COSTS FALSE, FORMAT XML) Seq Scan - Outer false customer_mx_1220084 customer_mx Hash - Inner false Seq Scan - Outer false orders_mx_1220068 orders_mx @@ -805,7 +774,6 @@ EXPLAIN (COSTS FALSE, FORMAT YAML) Parallel Aware: false Plans: - Node Type: "Custom Scan" - Parent Relationship: "Outer" Custom Plan Provider: "Citus Adaptive" Parallel Aware: false Distributed Query: @@ -822,55 +790,45 @@ EXPLAIN (COSTS FALSE, FORMAT YAML) Parallel Aware: false Plans: - Node Type: "Hash Join" - Parent Relationship: "Outer" Parallel Aware: false Join Type: "Inner" Inner Unique: false Hash Cond: "(lineitem_mx.l_orderkey = orders_mx.o_orderkey)" Plans: - Node Type: "Hash Join" - Parent Relationship: "Outer" Parallel Aware: false Join Type: "Inner" Inner Unique: false Hash Cond: "(supplier_mx.s_suppkey = lineitem_mx.l_suppkey)" Plans: - Node Type: "Seq Scan" - Parent Relationship: "Outer" Parallel Aware: false Relation Name: "supplier_mx_1220087" Alias: "supplier_mx" - Node Type: "Hash" - Parent Relationship: "Inner" Parallel Aware: false Plans: - Node Type: "Seq Scan" - Parent Relationship: "Outer" Parallel Aware: false Relation Name: "lineitem_mx_1220052" Alias: "lineitem_mx" - Node Type: "Hash" - Parent Relationship: "Inner" Parallel Aware: false Plans: - Node Type: "Hash Join" - Parent Relationship: "Outer" Parallel Aware: false Join Type: "Inner" Inner Unique: false Hash Cond: "(customer_mx.c_custkey = orders_mx.o_custkey)" Plans: - Node Type: "Seq Scan" - Parent Relationship: "Outer" Parallel Aware: false Relation Name: "customer_mx_1220084" Alias: "customer_mx" - Node Type: "Hash" - Parent Relationship: "Inner" Parallel Aware: false Plans: - Node Type: "Seq Scan" - Parent Relationship: "Outer" Parallel Aware: false Relation Name: "orders_mx_1220068" Alias: "orders_mx" diff --git a/src/test/regress/expected/multi_mx_function_call_delegation.out b/src/test/regress/expected/multi_mx_function_call_delegation.out index 817cc92a7..dd5bfdbfc 100644 --- a/src/test/regress/expected/multi_mx_function_call_delegation.out +++ b/src/test/regress/expected/multi_mx_function_call_delegation.out @@ -143,10 +143,10 @@ SET client_min_messages TO DEBUG1; select mx_call_func(2, 0); DEBUG: function does not have co-located tables DEBUG: generating subplan XXX_1 for subquery SELECT sum((t1.val OPERATOR(pg_catalog.+) t2.val)) AS sum FROM (multi_mx_function_call_delegation.mx_call_dist_table_1 t1 JOIN multi_mx_function_call_delegation.mx_call_dist_table_2 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.id))) -CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +CONTEXT: PL/pgSQL assignment "y := y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))) -CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT ((3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))))::integer +CONTEXT: PL/pgSQL assignment "y := y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment mx_call_func --------------------------------------------------------------------- @@ -230,26 +230,33 @@ DEBUG: pushing down the function call (S,S) (1 row) --- This is currently an undetected failure when using the binary protocol --- It should not be enabled by default until this is resolved. The tests above --- will fail too, when changing the default to TRUE; +-- this is fixed with pg14 and this will fail prior to +-- pg 14 SET citus.enable_binary_protocol = TRUE; select mx_call_func_custom_types('S', 'A'); DEBUG: pushing down the function call -ERROR: wrong data type: XXXX, expected XXXX + mx_call_func_custom_types +--------------------------------------------------------------------- + (S,S) +(1 row) + select multi_mx_function_call_delegation.mx_call_func_custom_types('S', 'A'); DEBUG: pushing down the function call -ERROR: wrong data type: XXXX, expected XXXX + mx_call_func_custom_types +--------------------------------------------------------------------- + (S,S) +(1 row) + RESET citus.enable_binary_protocol; -- We don't allow distributing calls inside transactions begin; select mx_call_func(2, 0); DEBUG: not pushing down function calls in a multi-statement transaction DEBUG: generating subplan XXX_1 for subquery SELECT sum((t1.val OPERATOR(pg_catalog.+) t2.val)) AS sum FROM (multi_mx_function_call_delegation.mx_call_dist_table_1 t1 JOIN multi_mx_function_call_delegation.mx_call_dist_table_2 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.id))) -CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +CONTEXT: PL/pgSQL assignment "y := y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))) -CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT ((3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))))::integer +CONTEXT: PL/pgSQL assignment "y := y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment mx_call_func --------------------------------------------------------------------- @@ -280,10 +287,10 @@ select colocate_proc_with_table('mx_call_func', 'mx_call_dist_table_1'::regclass select mx_call_func(2, 0); DEBUG: cannot push down invalid distribution_argument_index DEBUG: generating subplan XXX_1 for subquery SELECT sum((t1.val OPERATOR(pg_catalog.+) t2.val)) AS sum FROM (multi_mx_function_call_delegation.mx_call_dist_table_1 t1 JOIN multi_mx_function_call_delegation.mx_call_dist_table_2 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.id))) -CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +CONTEXT: PL/pgSQL assignment "y := y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))) -CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT ((3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))))::integer +CONTEXT: PL/pgSQL assignment "y := y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment mx_call_func --------------------------------------------------------------------- @@ -299,10 +306,10 @@ select colocate_proc_with_table('mx_call_func', 'mx_call_dist_table_1'::regclass select mx_call_func(2, 0); DEBUG: cannot push down invalid distribution_argument_index DEBUG: generating subplan XXX_1 for subquery SELECT sum((t1.val OPERATOR(pg_catalog.+) t2.val)) AS sum FROM (multi_mx_function_call_delegation.mx_call_dist_table_1 t1 JOIN multi_mx_function_call_delegation.mx_call_dist_table_2 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.id))) -CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +CONTEXT: PL/pgSQL assignment "y := y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))) -CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT ((3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))))::integer +CONTEXT: PL/pgSQL assignment "y := y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment mx_call_func --------------------------------------------------------------------- @@ -333,10 +340,10 @@ select colocate_proc_with_table('mx_call_func', 'mx_call_dist_table_replica'::re select mx_call_func(2, 0); DEBUG: cannot push down function call for replicated distributed tables DEBUG: generating subplan XXX_1 for subquery SELECT sum((t1.val OPERATOR(pg_catalog.+) t2.val)) AS sum FROM (multi_mx_function_call_delegation.mx_call_dist_table_1 t1 JOIN multi_mx_function_call_delegation.mx_call_dist_table_2 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.id))) -CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +CONTEXT: PL/pgSQL assignment "y := y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))) -CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT ((3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))))::integer +CONTEXT: PL/pgSQL assignment "y := y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment mx_call_func --------------------------------------------------------------------- @@ -515,10 +522,10 @@ SET client_min_messages TO DEBUG1; select mx_call_func(2, 0); DEBUG: the worker node does not have metadata DEBUG: generating subplan XXX_1 for subquery SELECT sum((t1.val OPERATOR(pg_catalog.+) t2.val)) AS sum FROM (multi_mx_function_call_delegation.mx_call_dist_table_1 t1 JOIN multi_mx_function_call_delegation.mx_call_dist_table_2 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.id))) -CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +CONTEXT: PL/pgSQL assignment "y := y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))) -CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT ((3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))))::integer +CONTEXT: PL/pgSQL assignment "y := y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment mx_call_func --------------------------------------------------------------------- @@ -562,10 +569,10 @@ DETAIL: A distributed function is created. To make sure subsequent commands see select mx_call_func((select x + 1 from mx_call_add(3, 4) x), 2); DEBUG: arguments in a distributed function must not contain subqueries DEBUG: generating subplan XXX_1 for subquery SELECT sum((t1.val OPERATOR(pg_catalog.+) t2.val)) AS sum FROM (multi_mx_function_call_delegation.mx_call_dist_table_1 t1 JOIN multi_mx_function_call_delegation.mx_call_dist_table_2 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.id))) -CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +CONTEXT: PL/pgSQL assignment "y := y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (9 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))) -CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT ((9 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))))::integer +CONTEXT: PL/pgSQL assignment "y := y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment mx_call_func --------------------------------------------------------------------- @@ -576,10 +583,10 @@ PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment select mx_call_func(floor(random())::int, 2); DEBUG: arguments in a distributed function must be constant expressions DEBUG: generating subplan XXX_1 for subquery SELECT sum((t1.val OPERATOR(pg_catalog.+) t2.val)) AS sum FROM (multi_mx_function_call_delegation.mx_call_dist_table_1 t1 JOIN multi_mx_function_call_delegation.mx_call_dist_table_2 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.id))) -CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +CONTEXT: PL/pgSQL assignment "y := y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (1 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))) -CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT ((1 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))))::integer +CONTEXT: PL/pgSQL assignment "y := y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment mx_call_func --------------------------------------------------------------------- @@ -589,10 +596,10 @@ PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment -- test forms we don't distribute select * from mx_call_func(2, 0); DEBUG: generating subplan XXX_1 for subquery SELECT sum((t1.val OPERATOR(pg_catalog.+) t2.val)) AS sum FROM (multi_mx_function_call_delegation.mx_call_dist_table_1 t1 JOIN multi_mx_function_call_delegation.mx_call_dist_table_2 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.id))) -CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +CONTEXT: PL/pgSQL assignment "y := y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))) -CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT ((3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))))::integer +CONTEXT: PL/pgSQL assignment "y := y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment y --------------------------------------------------------------------- @@ -615,10 +622,10 @@ select mx_call_func(2, 0) from mx_call_dist_table_1; select mx_call_func(2, 0) where mx_call_func(0, 2) = 0; DEBUG: generating subplan XXX_1 for subquery SELECT sum((t1.val OPERATOR(pg_catalog.+) t2.val)) AS sum FROM (multi_mx_function_call_delegation.mx_call_dist_table_1 t1 JOIN multi_mx_function_call_delegation.mx_call_dist_table_2 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.id))) -CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +CONTEXT: PL/pgSQL assignment "y := y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (1 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))) -CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT ((1 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))))::integer +CONTEXT: PL/pgSQL assignment "y := y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment mx_call_func --------------------------------------------------------------------- @@ -626,16 +633,16 @@ PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment select mx_call_func(2, 0), mx_call_func(0, 2); DEBUG: generating subplan XXX_1 for subquery SELECT sum((t1.val OPERATOR(pg_catalog.+) t2.val)) AS sum FROM (multi_mx_function_call_delegation.mx_call_dist_table_1 t1 JOIN multi_mx_function_call_delegation.mx_call_dist_table_2 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.id))) -CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +CONTEXT: PL/pgSQL assignment "y := y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))) -CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT ((3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))))::integer +CONTEXT: PL/pgSQL assignment "y := y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment DEBUG: generating subplan XXX_1 for subquery SELECT sum((t1.val OPERATOR(pg_catalog.+) t2.val)) AS sum FROM (multi_mx_function_call_delegation.mx_call_dist_table_1 t1 JOIN multi_mx_function_call_delegation.mx_call_dist_table_2 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.id))) -CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +CONTEXT: PL/pgSQL assignment "y := y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (1 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))) -CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT ((1 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))))::integer +CONTEXT: PL/pgSQL assignment "y := y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment mx_call_func | mx_call_func --------------------------------------------------------------------- diff --git a/src/test/regress/expected/multi_mx_function_call_delegation_0.out b/src/test/regress/expected/multi_mx_function_call_delegation_0.out new file mode 100644 index 000000000..e16ba2922 --- /dev/null +++ b/src/test/regress/expected/multi_mx_function_call_delegation_0.out @@ -0,0 +1,724 @@ +-- Test passing off function call to mx workers +CREATE SCHEMA multi_mx_function_call_delegation; +SET search_path TO multi_mx_function_call_delegation, public; +SET citus.shard_replication_factor TO 2; +-- This table requires specific settings, create before getting into things +create table mx_call_dist_table_replica(id int, val int); +select create_distributed_table('mx_call_dist_table_replica', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +insert into mx_call_dist_table_replica values (9,1),(8,2),(7,3),(6,4),(5,5); +SET citus.shard_replication_factor TO 1; +-- +-- Create tables and functions we want to use in tests +-- +create table mx_call_dist_table_1(id int, val int); +select create_distributed_table('mx_call_dist_table_1', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +insert into mx_call_dist_table_1 values (3,1),(4,5),(9,2),(6,5),(3,5); +create table mx_call_dist_table_2(id int, val int); +select create_distributed_table('mx_call_dist_table_2', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +insert into mx_call_dist_table_2 values (1,1),(1,2),(2,2),(3,3),(3,4); +create table mx_call_dist_table_bigint(id bigint, val bigint); +select create_distributed_table('mx_call_dist_table_bigint', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +insert into mx_call_dist_table_bigint values (1,1),(1,2),(2,2),(3,3),(3,4); +create table mx_call_dist_table_ref(id int, val int); +select create_reference_table('mx_call_dist_table_ref'); + create_reference_table +--------------------------------------------------------------------- + +(1 row) + +insert into mx_call_dist_table_ref values (2,7),(1,8),(2,8),(1,8),(2,8); +create type mx_call_enum as enum ('A', 'S', 'D', 'F'); +create table mx_call_dist_table_enum(id int, key mx_call_enum); +select create_distributed_table('mx_call_dist_table_enum', 'key'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +insert into mx_call_dist_table_enum values (1,'S'),(2,'A'),(3,'D'),(4,'F'); +CREATE FUNCTION squares(int) RETURNS SETOF RECORD + AS $$ SELECT i, i * i FROM generate_series(1, $1) i $$ + LANGUAGE SQL; +CREATE FUNCTION mx_call_func(x int, INOUT y int) +LANGUAGE plpgsql AS $$ +BEGIN + -- groupid is 0 in coordinator and non-zero in workers, so by using it here + -- we make sure the function is being executed in the worker. + y := x + (select case groupid when 0 then 1 else 0 end from pg_dist_local_group); + -- we also make sure that we can run distributed queries in the functions + -- that are routed to the workers. + y := y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id); +END;$$; +CREATE FUNCTION mx_call_func_bigint(x bigint, INOUT y bigint) +LANGUAGE plpgsql AS $$ +BEGIN + y := x + y * 2; +END;$$; +-- create another function which verifies: +-- 1. we work fine with multiple return columns +-- 2. we work fine in combination with custom types +CREATE FUNCTION mx_call_func_custom_types(INOUT x mx_call_enum, INOUT y mx_call_enum) +LANGUAGE plpgsql AS $$ +BEGIN + y := x; + x := (select case groupid when 0 then 'F' else 'S' end from pg_dist_local_group); +END;$$; +-- Test that undistributed functions have no issue executing +select multi_mx_function_call_delegation.mx_call_func(2, 0); + mx_call_func +--------------------------------------------------------------------- + 29 +(1 row) + +select multi_mx_function_call_delegation.mx_call_func_custom_types('S', 'A'); + mx_call_func_custom_types +--------------------------------------------------------------------- + (F,S) +(1 row) + +select squares(4); + squares +--------------------------------------------------------------------- + (1,1) + (2,4) + (3,9) + (4,16) +(4 rows) + +-- Same for unqualified name +select mx_call_func(2, 0); + mx_call_func +--------------------------------------------------------------------- + 29 +(1 row) + +-- Mark both functions as distributed ... +select create_distributed_function('mx_call_func(int,int)'); + create_distributed_function +--------------------------------------------------------------------- + +(1 row) + +select create_distributed_function('mx_call_func_bigint(bigint,bigint)'); + create_distributed_function +--------------------------------------------------------------------- + +(1 row) + +select create_distributed_function('mx_call_func_custom_types(mx_call_enum,mx_call_enum)'); + create_distributed_function +--------------------------------------------------------------------- + +(1 row) + +select create_distributed_function('squares(int)'); + create_distributed_function +--------------------------------------------------------------------- + +(1 row) + +-- We still don't route them to the workers, because they aren't +-- colocated with any distributed tables. +SET client_min_messages TO DEBUG1; +select mx_call_func(2, 0); +DEBUG: function does not have co-located tables +DEBUG: generating subplan XXX_1 for subquery SELECT sum((t1.val OPERATOR(pg_catalog.+) t2.val)) AS sum FROM (multi_mx_function_call_delegation.mx_call_dist_table_1 t1 JOIN multi_mx_function_call_delegation.mx_call_dist_table_2 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.id))) +CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))) +CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment + mx_call_func +--------------------------------------------------------------------- + 29 +(1 row) + +select multi_mx_function_call_delegation.mx_call_func_bigint(4, 2); +DEBUG: function does not have co-located tables + mx_call_func_bigint +--------------------------------------------------------------------- + 8 +(1 row) + +select mx_call_func_custom_types('S', 'A'); +DEBUG: function does not have co-located tables + mx_call_func_custom_types +--------------------------------------------------------------------- + (F,S) +(1 row) + +-- Mark them as colocated with a table. Now we should route them to workers. +select colocate_proc_with_table('mx_call_func', 'mx_call_dist_table_1'::regclass, 1); + colocate_proc_with_table +--------------------------------------------------------------------- + +(1 row) + +select colocate_proc_with_table('mx_call_func_bigint', 'mx_call_dist_table_bigint'::regclass, 1); + colocate_proc_with_table +--------------------------------------------------------------------- + +(1 row) + +select colocate_proc_with_table('mx_call_func_custom_types', 'mx_call_dist_table_enum'::regclass, 1); + colocate_proc_with_table +--------------------------------------------------------------------- + +(1 row) + +select colocate_proc_with_table('squares', 'mx_call_dist_table_2'::regclass, 0); + colocate_proc_with_table +--------------------------------------------------------------------- + +(1 row) + +select mx_call_func(2, 0); +DEBUG: pushing down the function call + mx_call_func +--------------------------------------------------------------------- + 28 +(1 row) + +select mx_call_func_bigint(4, 2); +DEBUG: pushing down the function call + mx_call_func_bigint +--------------------------------------------------------------------- + 8 +(1 row) + +select mx_call_func_custom_types('S', 'A'); +DEBUG: pushing down the function call + mx_call_func_custom_types +--------------------------------------------------------------------- + (S,S) +(1 row) + +select squares(4); +DEBUG: pushing down the function call +ERROR: input of anonymous composite types is not implemented +select multi_mx_function_call_delegation.mx_call_func(2, 0); +DEBUG: pushing down the function call + mx_call_func +--------------------------------------------------------------------- + 28 +(1 row) + +select multi_mx_function_call_delegation.mx_call_func_custom_types('S', 'A'); +DEBUG: pushing down the function call + mx_call_func_custom_types +--------------------------------------------------------------------- + (S,S) +(1 row) + +-- this is fixed with pg14 and this will fail prior to +-- pg 14 +SET citus.enable_binary_protocol = TRUE; +select mx_call_func_custom_types('S', 'A'); +DEBUG: pushing down the function call +ERROR: wrong data type: XXXX, expected XXXX +select multi_mx_function_call_delegation.mx_call_func_custom_types('S', 'A'); +DEBUG: pushing down the function call +ERROR: wrong data type: XXXX, expected XXXX +RESET citus.enable_binary_protocol; +-- We don't allow distributing calls inside transactions +begin; +select mx_call_func(2, 0); +DEBUG: not pushing down function calls in a multi-statement transaction +DEBUG: generating subplan XXX_1 for subquery SELECT sum((t1.val OPERATOR(pg_catalog.+) t2.val)) AS sum FROM (multi_mx_function_call_delegation.mx_call_dist_table_1 t1 JOIN multi_mx_function_call_delegation.mx_call_dist_table_2 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.id))) +CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))) +CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment + mx_call_func +--------------------------------------------------------------------- + 29 +(1 row) + +commit; +-- Drop the table colocated with mx_call_func_custom_types. Now it shouldn't +-- be routed to workers anymore. +SET client_min_messages TO NOTICE; +drop table mx_call_dist_table_enum; +SET client_min_messages TO DEBUG1; +select mx_call_func_custom_types('S', 'A'); +DEBUG: function does not have co-located tables + mx_call_func_custom_types +--------------------------------------------------------------------- + (F,S) +(1 row) + +-- Make sure we do bounds checking on distributed argument index +-- This also tests that we have cache invalidation for pg_dist_object updates +select colocate_proc_with_table('mx_call_func', 'mx_call_dist_table_1'::regclass, -1); + colocate_proc_with_table +--------------------------------------------------------------------- + +(1 row) + +select mx_call_func(2, 0); +DEBUG: cannot push down invalid distribution_argument_index +DEBUG: generating subplan XXX_1 for subquery SELECT sum((t1.val OPERATOR(pg_catalog.+) t2.val)) AS sum FROM (multi_mx_function_call_delegation.mx_call_dist_table_1 t1 JOIN multi_mx_function_call_delegation.mx_call_dist_table_2 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.id))) +CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))) +CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment + mx_call_func +--------------------------------------------------------------------- + 29 +(1 row) + +select colocate_proc_with_table('mx_call_func', 'mx_call_dist_table_1'::regclass, 2); + colocate_proc_with_table +--------------------------------------------------------------------- + +(1 row) + +select mx_call_func(2, 0); +DEBUG: cannot push down invalid distribution_argument_index +DEBUG: generating subplan XXX_1 for subquery SELECT sum((t1.val OPERATOR(pg_catalog.+) t2.val)) AS sum FROM (multi_mx_function_call_delegation.mx_call_dist_table_1 t1 JOIN multi_mx_function_call_delegation.mx_call_dist_table_2 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.id))) +CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))) +CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment + mx_call_func +--------------------------------------------------------------------- + 29 +(1 row) + +-- We don't currently support colocating with reference tables +select colocate_proc_with_table('mx_call_func', 'mx_call_dist_table_ref'::regclass, 1); + colocate_proc_with_table +--------------------------------------------------------------------- + +(1 row) + +select mx_call_func(2, 0); +DEBUG: pushing down the function call + mx_call_func +--------------------------------------------------------------------- + 28 +(1 row) + +-- We don't currently support colocating with replicated tables +select colocate_proc_with_table('mx_call_func', 'mx_call_dist_table_replica'::regclass, 1); + colocate_proc_with_table +--------------------------------------------------------------------- + +(1 row) + +select mx_call_func(2, 0); +DEBUG: cannot push down function call for replicated distributed tables +DEBUG: generating subplan XXX_1 for subquery SELECT sum((t1.val OPERATOR(pg_catalog.+) t2.val)) AS sum FROM (multi_mx_function_call_delegation.mx_call_dist_table_1 t1 JOIN multi_mx_function_call_delegation.mx_call_dist_table_2 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.id))) +CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))) +CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment + mx_call_func +--------------------------------------------------------------------- + 29 +(1 row) + +SET client_min_messages TO NOTICE; +drop table mx_call_dist_table_replica; +SET client_min_messages TO DEBUG1; +select colocate_proc_with_table('mx_call_func', 'mx_call_dist_table_1'::regclass, 1); + colocate_proc_with_table +--------------------------------------------------------------------- + +(1 row) + +-- Test table returning functions. +CREATE FUNCTION mx_call_func_tbl(x int) +RETURNS TABLE (p0 int, p1 int) +LANGUAGE plpgsql AS $$ +BEGIN + INSERT INTO multi_mx_function_call_delegation.mx_call_dist_table_1 VALUES (x, -1), (x+1, 4); + UPDATE multi_mx_function_call_delegation.mx_call_dist_table_1 SET val = val+1 WHERE id >= x; + UPDATE multi_mx_function_call_delegation.mx_call_dist_table_1 SET val = val-1 WHERE id >= x; + RETURN QUERY + SELECT id, val + FROM multi_mx_function_call_delegation.mx_call_dist_table_1 t + WHERE id >= x + ORDER BY 1, 2; +END;$$; +-- before distribution ... +select mx_call_func_tbl(10); + mx_call_func_tbl +--------------------------------------------------------------------- + (10,-1) + (11,4) +(2 rows) + +-- after distribution ... +select create_distributed_function('mx_call_func_tbl(int)', '$1', 'mx_call_dist_table_1'); +DEBUG: switching to sequential query execution mode +DETAIL: A distributed function is created. To make sure subsequent commands see the type correctly we need to make sure to use only one connection for all future commands + create_distributed_function +--------------------------------------------------------------------- + +(1 row) + +select mx_call_func_tbl(20); +DEBUG: pushing down the function call + mx_call_func_tbl +--------------------------------------------------------------------- + (20,-1) + (21,4) +(2 rows) + +-- Test that we properly propagate errors raised from procedures. +CREATE FUNCTION mx_call_func_raise(x int) +RETURNS void LANGUAGE plpgsql AS $$ +BEGIN + RAISE WARNING 'warning'; + RAISE EXCEPTION 'error'; +END;$$; +select create_distributed_function('mx_call_func_raise(int)', '$1', 'mx_call_dist_table_1'); +DEBUG: switching to sequential query execution mode +DETAIL: A distributed function is created. To make sure subsequent commands see the type correctly we need to make sure to use only one connection for all future commands + create_distributed_function +--------------------------------------------------------------------- + +(1 row) + +\set VERBOSITY terse +select mx_call_func_raise(2); +DEBUG: pushing down the function call +WARNING: warning +ERROR: error +\set VERBOSITY default +-- Don't push-down when doing INSERT INTO ... SELECT func(); +SET client_min_messages TO ERROR; +CREATE TABLE test (x int primary key); +SELECT create_distributed_table('test','x'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +CREATE OR REPLACE FUNCTION delegated_function(a int) +RETURNS int +LANGUAGE plpgsql +AS $function$ +DECLARE +BEGIN + INSERT INTO multi_mx_function_call_delegation.test VALUES (a); + INSERT INTO multi_mx_function_call_delegation.test VALUES (a + 1); + RETURN a+2; +END; +$function$; +SELECT create_distributed_function('delegated_function(int)', 'a'); + create_distributed_function +--------------------------------------------------------------------- + +(1 row) + +SET client_min_messages TO DEBUG1; +INSERT INTO test SELECT delegated_function(1); +DEBUG: distributed INSERT ... SELECT can only select from distributed tables +DEBUG: not pushing down function calls in INSERT ... SELECT +DEBUG: Collecting INSERT ... SELECT results on coordinator +-- Don't push down in subqueries or CTEs. +SELECT * FROM test WHERE not exists( + SELECT delegated_function(4) +); +DEBUG: not pushing down function calls in CTEs or Subqueries +DEBUG: generating subplan XXX_1 for subquery SELECT multi_mx_function_call_delegation.delegated_function(4) AS delegated_function +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT x FROM multi_mx_function_call_delegation.test WHERE (NOT (EXISTS (SELECT intermediate_result.delegated_function FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(delegated_function integer)))) + x +--------------------------------------------------------------------- +(0 rows) + +WITH r AS ( + SELECT delegated_function(7) +) SELECT * FROM test WHERE (SELECT count(*)=0 FROM r); +DEBUG: generating subplan XXX_1 for CTE r: SELECT multi_mx_function_call_delegation.delegated_function(7) AS delegated_function +DEBUG: not pushing down function calls in CTEs or Subqueries +DEBUG: generating subplan XXX_2 for subquery SELECT (count(*) OPERATOR(pg_catalog.=) 0) FROM (SELECT intermediate_result.delegated_function FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(delegated_function integer)) r +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT x FROM multi_mx_function_call_delegation.test WHERE (SELECT intermediate_result."?column?" FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result("?column?" boolean)) + x +--------------------------------------------------------------------- +(0 rows) + +WITH r AS ( + SELECT delegated_function(10) +), t AS ( + SELECT count(*) c FROM r +) SELECT * FROM test, t WHERE t.c=0; +DEBUG: CTE t is going to be inlined via distributed planning +DEBUG: generating subplan XXX_1 for CTE r: SELECT multi_mx_function_call_delegation.delegated_function(10) AS delegated_function +DEBUG: not pushing down function calls in CTEs or Subqueries +DEBUG: generating subplan XXX_2 for subquery SELECT count(*) AS c FROM (SELECT intermediate_result.delegated_function FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(delegated_function integer)) r +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT test.x, t.c FROM multi_mx_function_call_delegation.test, (SELECT intermediate_result.c FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(c bigint)) t WHERE (t.c OPERATOR(pg_catalog.=) 0) + x | c +--------------------------------------------------------------------- +(0 rows) + +WITH r AS ( + SELECT count(*) FROM test +), s AS ( + SELECT delegated_function(13) +), t AS ( + SELECT count(*) c FROM s +) SELECT * FROM test, r, t WHERE t.c=0; +DEBUG: CTE r is going to be inlined via distributed planning +DEBUG: CTE t is going to be inlined via distributed planning +DEBUG: generating subplan XXX_1 for CTE s: SELECT multi_mx_function_call_delegation.delegated_function(13) AS delegated_function +DEBUG: not pushing down function calls in CTEs or Subqueries +DEBUG: generating subplan XXX_2 for subquery SELECT count(*) AS count FROM multi_mx_function_call_delegation.test +DEBUG: generating subplan XXX_3 for subquery SELECT count(*) AS c FROM (SELECT intermediate_result.delegated_function FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(delegated_function integer)) s +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT test.x, r.count, t.c FROM multi_mx_function_call_delegation.test, (SELECT intermediate_result.count FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(count bigint)) r, (SELECT intermediate_result.c FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(c bigint)) t WHERE (t.c OPERATOR(pg_catalog.=) 0) + x | count | c +--------------------------------------------------------------------- +(0 rows) + +-- Test that we don't propagate to non-metadata worker nodes +SET client_min_messages TO WARNING; +select stop_metadata_sync_to_node('localhost', :worker_1_port); + stop_metadata_sync_to_node +--------------------------------------------------------------------- + +(1 row) + +select stop_metadata_sync_to_node('localhost', :worker_2_port); + stop_metadata_sync_to_node +--------------------------------------------------------------------- + +(1 row) + +SET client_min_messages TO DEBUG1; +select mx_call_func(2, 0); +DEBUG: the worker node does not have metadata +DEBUG: generating subplan XXX_1 for subquery SELECT sum((t1.val OPERATOR(pg_catalog.+) t2.val)) AS sum FROM (multi_mx_function_call_delegation.mx_call_dist_table_1 t1 JOIN multi_mx_function_call_delegation.mx_call_dist_table_2 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.id))) +CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))) +CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment + mx_call_func +--------------------------------------------------------------------- + 29 +(1 row) + +SET client_min_messages TO NOTICE; +select start_metadata_sync_to_node('localhost', :worker_1_port); + start_metadata_sync_to_node +--------------------------------------------------------------------- + +(1 row) + +select start_metadata_sync_to_node('localhost', :worker_2_port); + start_metadata_sync_to_node +--------------------------------------------------------------------- + +(1 row) + +-- stop_metadata_sync_to_node()/start_metadata_sync_to_node() might make +-- worker backend caches inconsistent. Reconnect to coordinator to use +-- new worker connections, hence new backends. +\c - - - :master_port +SET search_path to multi_mx_function_call_delegation, public; +SET client_min_messages TO DEBUG1; +SET citus.shard_replication_factor = 1; +-- +-- Test non-const parameter values +-- +CREATE FUNCTION mx_call_add(int, int) RETURNS int + AS 'select $1 + $2;' LANGUAGE SQL IMMUTABLE; +SELECT create_distributed_function('mx_call_add(int,int)', '$1'); +DEBUG: switching to sequential query execution mode +DETAIL: A distributed function is created. To make sure subsequent commands see the type correctly we need to make sure to use only one connection for all future commands + create_distributed_function +--------------------------------------------------------------------- + +(1 row) + +-- subquery parameters cannot be pushed down +select mx_call_func((select x + 1 from mx_call_add(3, 4) x), 2); +DEBUG: arguments in a distributed function must not contain subqueries +DEBUG: generating subplan XXX_1 for subquery SELECT sum((t1.val OPERATOR(pg_catalog.+) t2.val)) AS sum FROM (multi_mx_function_call_delegation.mx_call_dist_table_1 t1 JOIN multi_mx_function_call_delegation.mx_call_dist_table_2 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.id))) +CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (9 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))) +CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment + mx_call_func +--------------------------------------------------------------------- + 35 +(1 row) + +-- volatile parameter cannot be pushed down +select mx_call_func(floor(random())::int, 2); +DEBUG: arguments in a distributed function must be constant expressions +DEBUG: generating subplan XXX_1 for subquery SELECT sum((t1.val OPERATOR(pg_catalog.+) t2.val)) AS sum FROM (multi_mx_function_call_delegation.mx_call_dist_table_1 t1 JOIN multi_mx_function_call_delegation.mx_call_dist_table_2 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.id))) +CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (1 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))) +CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment + mx_call_func +--------------------------------------------------------------------- + 27 +(1 row) + +-- test forms we don't distribute +select * from mx_call_func(2, 0); +DEBUG: generating subplan XXX_1 for subquery SELECT sum((t1.val OPERATOR(pg_catalog.+) t2.val)) AS sum FROM (multi_mx_function_call_delegation.mx_call_dist_table_1 t1 JOIN multi_mx_function_call_delegation.mx_call_dist_table_2 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.id))) +CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))) +CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment + y +--------------------------------------------------------------------- + 29 +(1 row) + +select mx_call_func(2, 0) from mx_call_dist_table_1; + mx_call_func +--------------------------------------------------------------------- + 28 + 28 + 28 + 28 + 28 + 28 + 28 + 28 + 28 +(9 rows) + +select mx_call_func(2, 0) where mx_call_func(0, 2) = 0; +DEBUG: generating subplan XXX_1 for subquery SELECT sum((t1.val OPERATOR(pg_catalog.+) t2.val)) AS sum FROM (multi_mx_function_call_delegation.mx_call_dist_table_1 t1 JOIN multi_mx_function_call_delegation.mx_call_dist_table_2 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.id))) +CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (1 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))) +CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment + mx_call_func +--------------------------------------------------------------------- +(0 rows) + +select mx_call_func(2, 0), mx_call_func(0, 2); +DEBUG: generating subplan XXX_1 for subquery SELECT sum((t1.val OPERATOR(pg_catalog.+) t2.val)) AS sum FROM (multi_mx_function_call_delegation.mx_call_dist_table_1 t1 JOIN multi_mx_function_call_delegation.mx_call_dist_table_2 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.id))) +CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (3 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))) +CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment +DEBUG: generating subplan XXX_1 for subquery SELECT sum((t1.val OPERATOR(pg_catalog.+) t2.val)) AS sum FROM (multi_mx_function_call_delegation.mx_call_dist_table_1 t1 JOIN multi_mx_function_call_delegation.mx_call_dist_table_2 t2 ON ((t1.id OPERATOR(pg_catalog.=) t2.id))) +CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (1 OPERATOR(pg_catalog.+) (SELECT intermediate_result.sum FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(sum bigint))) +CONTEXT: SQL statement "SELECT y + (select sum(t1.val + t2.val) from multi_mx_function_call_delegation.mx_call_dist_table_1 t1 join multi_mx_function_call_delegation.mx_call_dist_table_2 t2 on t1.id = t2.id)" +PL/pgSQL function mx_call_func(integer,integer) line 8 at assignment + mx_call_func | mx_call_func +--------------------------------------------------------------------- + 29 | 27 +(1 row) + +DO $$ BEGIN perform mx_call_func_tbl(40); END; $$; +DEBUG: not pushing down function calls in a multi-statement transaction +CONTEXT: SQL statement "SELECT mx_call_func_tbl(40)" +PL/pgSQL function inline_code_block line 1 at PERFORM +SELECT * FROM mx_call_dist_table_1 WHERE id >= 40 ORDER BY id, val; + id | val +--------------------------------------------------------------------- + 40 | -1 + 41 | 4 +(2 rows) + +-- Prepared statements. Repeat 8 times to test for generic plans +PREPARE call_plan (int, int) AS SELECT mx_call_func($1, $2); +EXECUTE call_plan(2, 0); +DEBUG: pushing down the function call + mx_call_func +--------------------------------------------------------------------- + 28 +(1 row) + +EXECUTE call_plan(2, 0); +DEBUG: pushing down the function call + mx_call_func +--------------------------------------------------------------------- + 28 +(1 row) + +EXECUTE call_plan(2, 0); +DEBUG: pushing down the function call + mx_call_func +--------------------------------------------------------------------- + 28 +(1 row) + +EXECUTE call_plan(2, 0); +DEBUG: pushing down the function call + mx_call_func +--------------------------------------------------------------------- + 28 +(1 row) + +EXECUTE call_plan(2, 0); +DEBUG: pushing down the function call + mx_call_func +--------------------------------------------------------------------- + 28 +(1 row) + +EXECUTE call_plan(2, 0); +DEBUG: pushing down the function call + mx_call_func +--------------------------------------------------------------------- + 28 +(1 row) + +EXECUTE call_plan(2, 0); +DEBUG: pushing down the function call + mx_call_func +--------------------------------------------------------------------- + 28 +(1 row) + +EXECUTE call_plan(2, 0); +DEBUG: pushing down the function call + mx_call_func +--------------------------------------------------------------------- + 28 +(1 row) + +\c - - - :worker_1_port +SET search_path TO multi_mx_function_call_delegation, public; +-- create_distributed_function is disallowed from worker nodes +select create_distributed_function('mx_call_func(int,int)'); +ERROR: operation is not allowed on this node +HINT: Connect to the coordinator and run it again. +\c - - - :master_port +SET search_path TO multi_mx_function_call_delegation, public; +RESET client_min_messages; +\set VERBOSITY terse +DROP SCHEMA multi_mx_function_call_delegation CASCADE; +NOTICE: drop cascades to 14 other objects diff --git a/src/test/regress/sql/multi_mx_function_call_delegation.sql b/src/test/regress/sql/multi_mx_function_call_delegation.sql index 4f7de1d92..513146a4e 100644 --- a/src/test/regress/sql/multi_mx_function_call_delegation.sql +++ b/src/test/regress/sql/multi_mx_function_call_delegation.sql @@ -103,9 +103,8 @@ select multi_mx_function_call_delegation.mx_call_func(2, 0); select multi_mx_function_call_delegation.mx_call_func_custom_types('S', 'A'); --- This is currently an undetected failure when using the binary protocol --- It should not be enabled by default until this is resolved. The tests above --- will fail too, when changing the default to TRUE; +-- this is fixed with pg14 and this will fail prior to +-- pg 14 SET citus.enable_binary_protocol = TRUE; select mx_call_func_custom_types('S', 'A'); select multi_mx_function_call_delegation.mx_call_func_custom_types('S', 'A'); From 2fa1e5ffe36992889c35c08e59f34faa57d00f9a Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Tue, 24 Aug 2021 10:05:20 +0300 Subject: [PATCH 083/104] Use the default max_parallel_workers_per_gather for vanilla --- src/test/regress/pg_regress_multi.pl | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/test/regress/pg_regress_multi.pl b/src/test/regress/pg_regress_multi.pl index 29195285f..5fd945320 100755 --- a/src/test/regress/pg_regress_multi.pl +++ b/src/test/regress/pg_regress_multi.pl @@ -420,8 +420,13 @@ if (-e $hll_control) } push(@pgOptions, "shared_preload_libraries='${sharedPreloadLibraries}'"); -# Avoid parallelism to stabilize explain plans -push(@pgOptions, "max_parallel_workers_per_gather=0"); +if ($vanillatest) { + # use the default used in vanilla tests + push(@pgOptions, "max_parallel_workers_per_gather=2"); +}else { + # Avoid parallelism to stabilize explain plans + push(@pgOptions, "max_parallel_workers_per_gather=0"); +} # Help with debugging push(@pgOptions, "log_error_verbosity = 'verbose'"); From 6ff609fa862db9df149ffac9688db029d85f5742 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Tue, 24 Aug 2021 11:55:43 +0300 Subject: [PATCH 084/104] Add alternative output for data_types It seems like there is a problem with Postgres14 with SELECT DISTINCT COUNT. The issue is reported to Postgres and an alternative output is added. We can remove the alternative output when the issue is fixed on PG. If this is not an issue on PG(which is unlikely) we should consider some other solution. --- src/test/regress/expected/data_types.out | 3 + src/test/regress/expected/data_types_0.out | 182 +++++++++++++++++++++ src/test/regress/sql/data_types.sql | 3 + 3 files changed, 188 insertions(+) create mode 100644 src/test/regress/expected/data_types_0.out diff --git a/src/test/regress/expected/data_types.out b/src/test/regress/expected/data_types.out index 4c8539758..8b82c9f52 100644 --- a/src/test/regress/expected/data_types.out +++ b/src/test/regress/expected/data_types.out @@ -130,6 +130,9 @@ FROM (2 rows) -- DISTINCT w/wout distribution key +-- there seems to be an issue with SELECT DISTINCT ROW with PG14 +-- so we add an alternative output that gives an error, this should +-- be removed after the issue is fixed on PG14. SELECT DISTINCT(col1, col2, col3, col4, col5, col6, col70, col7, col8, col9, col10, col11, col12, col13, col14, col15, col16, col17, col18, col19, col20, col21, col22, col23, col24, col25, col26, col27, col28, col29, col32, col33, col34, col35, col36, col37, col38) FROM data_types_table diff --git a/src/test/regress/expected/data_types_0.out b/src/test/regress/expected/data_types_0.out new file mode 100644 index 000000000..3e73fce39 --- /dev/null +++ b/src/test/regress/expected/data_types_0.out @@ -0,0 +1,182 @@ +CREATE SCHEMA data_types; +SET search_path TO data_types; +Create or replace function test_jsonb() returns jsonb as +$$ +begin + return '{"test_json": "test"}'; +end; +$$ language plpgsql; +CREATE TABLE data_types_table +( + dist_key bigint PRIMARY KEY, + col1 int[], col2 int[][], col3 int [][][], + col4 varchar[], col5 varchar[][], col6 varchar [][][], + col70 bit, col7 bit[], col8 bit[][], col9 bit [][][], + col10 bit varying(10), + col11 bit varying(10)[], col12 bit varying(10)[][], col13 bit varying(10)[][][], + col14 bytea, col15 bytea[], col16 bytea[][], col17 bytea[][][], + col18 boolean, col19 boolean[], col20 boolean[][], col21 boolean[][][], + col22 inet, col23 inet[], col24 inet[][], col25 inet[][][], + col26 macaddr, col27 macaddr[], col28 macaddr[][], col29 macaddr[][][], + col30 numeric, col32 numeric[], col33 numeric[][], col34 numeric[][][], + col35 jsonb, col36 jsonb[], col37 jsonb[][], col38 jsonb[][][] +); +CREATE TABLE data_types_table_local AS SELECT * FROM data_types_table; +SELECT create_distributed_table('data_types_table', 'dist_key'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO data_types_table (dist_key,col1, col2, col3, col4, col5, col6, col70, col7, col8, col9, col10, col11, col12, col13, col14, col15, col16, col17, col18, col19, col20, col21, col22, col23, col24, col25, col26, col27, col28, col29, col30, col32, col33, col34, col35, col36, col37, col38) +VALUES (1,ARRAY[1], ARRAY[ARRAY[0,0,0]], ARRAY[ARRAY[ARRAY[0,0,0]]], ARRAY['1'], ARRAY[ARRAY['0','0','0']], ARRAY[ARRAY[ARRAY['0','0','0']]], '1', ARRAY[b'1'], ARRAY[ARRAY[b'0',b'0',b'0']], ARRAY[ARRAY[ARRAY[b'0',b'0',b'0']]], '11101',ARRAY[b'1'], ARRAY[ARRAY[b'01',b'01',b'01']], ARRAY[ARRAY[ARRAY[b'011',b'110',b'0000']]], '\xb4a8e04c0b', ARRAY['\xb4a8e04c0b'::BYTEA], ARRAY[ARRAY['\xb4a8e04c0b'::BYTEA, '\xb4a8e04c0b'::BYTEA, '\xb4a8e04c0b'::BYTEA]], ARRAY[ARRAY[ARRAY['\xb4a8e04c0b'::BYTEA,'\x18a232a678'::BYTEA,'\x38b2697632'::BYTEA]]], '1', ARRAY[TRUE], ARRAY[ARRAY[1::boolean,TRUE,FALSE]], ARRAY[ARRAY[ARRAY[1::boolean,TRUE,FALSE]]], INET '192.168.1/24', ARRAY[INET '192.168.1.1'], ARRAY[ARRAY[INET '0.0.0.0', '0.0.0.0/32', '::ffff:fff0:1', '192.168.1/24']], ARRAY[ARRAY[ARRAY[INET '0.0.0.0', '0.0.0.0/32', '::ffff:fff0:1', '192.168.1/24']]],MACADDR '08:00:2b:01:02:03', ARRAY[MACADDR '08:00:2b:01:02:03'], ARRAY[ARRAY[MACADDR '08002b-010203', MACADDR '08002b-010203', '08002b010203']], ARRAY[ARRAY[ARRAY[MACADDR '08002b-010203', MACADDR '08002b-010203', '08002b010203']]], 690, ARRAY[1.1], ARRAY[ARRAY[0,0.111,0.15]], ARRAY[ARRAY[ARRAY[0,0,0]]], test_jsonb(), ARRAY[test_jsonb()], ARRAY[ARRAY[test_jsonb(),test_jsonb(),test_jsonb(),test_jsonb()]], ARRAY[ARRAY[ARRAY[test_jsonb(),test_jsonb(),test_jsonb(),test_jsonb()]]]), + (2,ARRAY[1,2,3], ARRAY[ARRAY[1,2,3], ARRAY[5,6,7]], ARRAY[ARRAY[ARRAY[1,2,3]], ARRAY[ARRAY[5,6,7]], ARRAY[ARRAY[1,2,3]], ARRAY[ARRAY[5,6,7]]], ARRAY['1','2','3'], ARRAY[ARRAY['1','2','3'], ARRAY['5','6','7']], ARRAY[ARRAY[ARRAY['1','2','3']], ARRAY[ARRAY['5','6','7']], ARRAY[ARRAY['1','2','3']], ARRAY[ARRAY['5','6','7']]], '0', ARRAY[b'1',b'0',b'0'], ARRAY[ARRAY[b'1',b'1',b'0'], ARRAY[b'0',b'0',b'1']], ARRAY[ARRAY[ARRAY[b'1',b'1',b'1']], ARRAY[ARRAY[b'1','0','0']], ARRAY[ARRAY[b'1','1','1']], ARRAY[ARRAY[b'0','0','0']]], '00010', ARRAY[b'11',b'10',b'01'], ARRAY[ARRAY[b'11',b'010',b'101'], ARRAY[b'101',b'01111',b'1000001']], ARRAY[ARRAY[ARRAY[b'10000',b'111111',b'1101010101']], ARRAY[ARRAY[b'1101010','0','1']], ARRAY[ARRAY[b'1','1','11111111']], ARRAY[ARRAY[b'0000000','0','0']]], '\xb4a8e04c0b', ARRAY['\xb4a8e04c0b'::BYTEA,'\x18a232a678'::BYTEA,'\x38b2697632'::BYTEA], ARRAY[ARRAY['\xb4a8e04c0b'::BYTEA,'\x18a232a678'::BYTEA,'\x38b2697632'::BYTEA], ARRAY['\xb4a8e04c0b'::BYTEA,'\x18a232a678'::BYTEA,'\x38b2697632'::BYTEA]], ARRAY[ARRAY[ARRAY['\xb4a8e04c0b'::BYTEA,'\x18a232a678'::BYTEA,'\x38b2697632'::BYTEA]], ARRAY[ARRAY['\xb4a8e04c0b'::BYTEA,'\x18a232a678'::BYTEA,'\x38b2697632'::BYTEA]], ARRAY[ARRAY['\xb4a8e04c0b'::BYTEA,'\x18a232a678'::BYTEA,'\x38b2697632'::BYTEA]], ARRAY[ARRAY['\xb4a8e04c0b'::BYTEA,'\x18a232a678'::BYTEA,'\x38b2697632'::BYTEA]]], 'true', ARRAY[1::boolean,TRUE,FALSE], ARRAY[ARRAY[1::boolean,TRUE,FALSE], ARRAY[1::boolean,TRUE,FALSE]], ARRAY[ARRAY[ARRAY[1::boolean,TRUE,FALSE]], ARRAY[ARRAY[1::boolean,TRUE,FALSE]], ARRAY[ARRAY[1::boolean,TRUE,FALSE]], ARRAY[ARRAY[1::boolean,TRUE,FALSE]]],'0.0.0.0/32', ARRAY[INET '0.0.0.0', '0.0.0.0/32', '::ffff:fff0:1', '192.168.1/24'], ARRAY[ARRAY[INET '0.0.0.0', '0.0.0.0/32', '::ffff:fff0:1', '192.168.1/24']], ARRAY[ARRAY[ARRAY[INET '0.0.0.0', '0.0.0.0/32', '::ffff:fff0:1', '192.168.1/24']], ARRAY[ARRAY[INET '0.0.0.0', '0.0.0.0/32', '::ffff:fff0:1', '192.168.1/24']], ARRAY[ARRAY[INET '0.0.0.0', '0.0.0.0/32', '::ffff:fff0:1', '192.168.1/24']], ARRAY[ARRAY[INET '0.0.0.0', '0.0.0.0/32', '::ffff:fff0:1', '192.168.1/24']]], '0800.2b01.0203', ARRAY[MACADDR '08002b-010203', MACADDR '08002b-010203', '08002b010203'], ARRAY[ARRAY[MACADDR '08002b-010203', MACADDR '08002b-010203', '08002b010203']], ARRAY[ARRAY[ARRAY[MACADDR '08002b-010203', MACADDR '08002b-010203', '08002b010203']], ARRAY[ARRAY[MACADDR '08002b-010203', MACADDR '08002b-010203', '08002b010203']], ARRAY[ARRAY[MACADDR '08002b-010203', MACADDR '08002b-010203', '08002b010203']], ARRAY[ARRAY[MACADDR '08002b-010203', MACADDR '08002b-010203', '08002b010203']]], 0.99, ARRAY[1.1,2.22,3.33], ARRAY[ARRAY[1.55,2.66,3.88], ARRAY[11.5,10101.6,7111.1]], ARRAY[ARRAY[ARRAY[1,2,3]], ARRAY[ARRAY[5,6,7]], ARRAY[ARRAY[1.1,2.1,3]], ARRAY[ARRAY[5.0,6.0,7.0]]],test_jsonb(), ARRAY[test_jsonb(),test_jsonb(),test_jsonb(),test_jsonb()], ARRAY[ARRAY[test_jsonb(),test_jsonb(),test_jsonb(),test_jsonb()], ARRAY[test_jsonb(),test_jsonb(),test_jsonb(),test_jsonb()]], ARRAY[ARRAY[ARRAY[test_jsonb(),test_jsonb(),test_jsonb(),test_jsonb()]], ARRAY[ARRAY[test_jsonb(),test_jsonb(),test_jsonb(),test_jsonb()]], ARRAY[ARRAY[test_jsonb(),test_jsonb(),test_jsonb(),test_jsonb()]], ARRAY[ARRAY[test_jsonb(),test_jsonb(),test_jsonb(),test_jsonb()]]]); +-- insert the same data to the local node as well +INSERT INTO data_types_table_local (dist_key,col1, col2, col3, col4, col5, col6, col70, col7, col8, col9, col10, col11, col12, col13, col14, col15, col16, col17, col18, col19, col20, col21, col22, col23, col24, col25, col26, col27, col28, col29, col30, col32, col33, col34, col35, col36, col37, col38) +VALUES (1,ARRAY[1], ARRAY[ARRAY[0,0,0]], ARRAY[ARRAY[ARRAY[0,0,0]]], ARRAY['1'], ARRAY[ARRAY['0','0','0']], ARRAY[ARRAY[ARRAY['0','0','0']]], '1', ARRAY[b'1'], ARRAY[ARRAY[b'0',b'0',b'0']], ARRAY[ARRAY[ARRAY[b'0',b'0',b'0']]], '11101',ARRAY[b'1'], ARRAY[ARRAY[b'01',b'01',b'01']], ARRAY[ARRAY[ARRAY[b'011',b'110',b'0000']]], '\xb4a8e04c0b', ARRAY['\xb4a8e04c0b'::BYTEA], ARRAY[ARRAY['\xb4a8e04c0b'::BYTEA, '\xb4a8e04c0b'::BYTEA, '\xb4a8e04c0b'::BYTEA]], ARRAY[ARRAY[ARRAY['\xb4a8e04c0b'::BYTEA,'\x18a232a678'::BYTEA,'\x38b2697632'::BYTEA]]], '1', ARRAY[TRUE], ARRAY[ARRAY[1::boolean,TRUE,FALSE]], ARRAY[ARRAY[ARRAY[1::boolean,TRUE,FALSE]]], INET '192.168.1/24', ARRAY[INET '192.168.1.1'], ARRAY[ARRAY[INET '0.0.0.0', '0.0.0.0/32', '::ffff:fff0:1', '192.168.1/24']], ARRAY[ARRAY[ARRAY[INET '0.0.0.0', '0.0.0.0/32', '::ffff:fff0:1', '192.168.1/24']]],MACADDR '08:00:2b:01:02:03', ARRAY[MACADDR '08:00:2b:01:02:03'], ARRAY[ARRAY[MACADDR '08002b-010203', MACADDR '08002b-010203', '08002b010203']], ARRAY[ARRAY[ARRAY[MACADDR '08002b-010203', MACADDR '08002b-010203', '08002b010203']]], 690, ARRAY[1.1], ARRAY[ARRAY[0,0.111,0.15]], ARRAY[ARRAY[ARRAY[0,0,0]]], test_jsonb(), ARRAY[test_jsonb()], ARRAY[ARRAY[test_jsonb(),test_jsonb(),test_jsonb(),test_jsonb()]], ARRAY[ARRAY[ARRAY[test_jsonb(),test_jsonb(),test_jsonb(),test_jsonb()]]]), + (2,ARRAY[1,2,3], ARRAY[ARRAY[1,2,3], ARRAY[5,6,7]], ARRAY[ARRAY[ARRAY[1,2,3]], ARRAY[ARRAY[5,6,7]], ARRAY[ARRAY[1,2,3]], ARRAY[ARRAY[5,6,7]]], ARRAY['1','2','3'], ARRAY[ARRAY['1','2','3'], ARRAY['5','6','7']], ARRAY[ARRAY[ARRAY['1','2','3']], ARRAY[ARRAY['5','6','7']], ARRAY[ARRAY['1','2','3']], ARRAY[ARRAY['5','6','7']]], '0', ARRAY[b'1',b'0',b'0'], ARRAY[ARRAY[b'1',b'1',b'0'], ARRAY[b'0',b'0',b'1']], ARRAY[ARRAY[ARRAY[b'1',b'1',b'1']], ARRAY[ARRAY[b'1','0','0']], ARRAY[ARRAY[b'1','1','1']], ARRAY[ARRAY[b'0','0','0']]], '00010', ARRAY[b'11',b'10',b'01'], ARRAY[ARRAY[b'11',b'010',b'101'], ARRAY[b'101',b'01111',b'1000001']], ARRAY[ARRAY[ARRAY[b'10000',b'111111',b'1101010101']], ARRAY[ARRAY[b'1101010','0','1']], ARRAY[ARRAY[b'1','1','11111111']], ARRAY[ARRAY[b'0000000','0','0']]], '\xb4a8e04c0b', ARRAY['\xb4a8e04c0b'::BYTEA,'\x18a232a678'::BYTEA,'\x38b2697632'::BYTEA], ARRAY[ARRAY['\xb4a8e04c0b'::BYTEA,'\x18a232a678'::BYTEA,'\x38b2697632'::BYTEA], ARRAY['\xb4a8e04c0b'::BYTEA,'\x18a232a678'::BYTEA,'\x38b2697632'::BYTEA]], ARRAY[ARRAY[ARRAY['\xb4a8e04c0b'::BYTEA,'\x18a232a678'::BYTEA,'\x38b2697632'::BYTEA]], ARRAY[ARRAY['\xb4a8e04c0b'::BYTEA,'\x18a232a678'::BYTEA,'\x38b2697632'::BYTEA]], ARRAY[ARRAY['\xb4a8e04c0b'::BYTEA,'\x18a232a678'::BYTEA,'\x38b2697632'::BYTEA]], ARRAY[ARRAY['\xb4a8e04c0b'::BYTEA,'\x18a232a678'::BYTEA,'\x38b2697632'::BYTEA]]], 'true', ARRAY[1::boolean,TRUE,FALSE], ARRAY[ARRAY[1::boolean,TRUE,FALSE], ARRAY[1::boolean,TRUE,FALSE]], ARRAY[ARRAY[ARRAY[1::boolean,TRUE,FALSE]], ARRAY[ARRAY[1::boolean,TRUE,FALSE]], ARRAY[ARRAY[1::boolean,TRUE,FALSE]], ARRAY[ARRAY[1::boolean,TRUE,FALSE]]],'0.0.0.0/32', ARRAY[INET '0.0.0.0', '0.0.0.0/32', '::ffff:fff0:1', '192.168.1/24'], ARRAY[ARRAY[INET '0.0.0.0', '0.0.0.0/32', '::ffff:fff0:1', '192.168.1/24']], ARRAY[ARRAY[ARRAY[INET '0.0.0.0', '0.0.0.0/32', '::ffff:fff0:1', '192.168.1/24']], ARRAY[ARRAY[INET '0.0.0.0', '0.0.0.0/32', '::ffff:fff0:1', '192.168.1/24']], ARRAY[ARRAY[INET '0.0.0.0', '0.0.0.0/32', '::ffff:fff0:1', '192.168.1/24']], ARRAY[ARRAY[INET '0.0.0.0', '0.0.0.0/32', '::ffff:fff0:1', '192.168.1/24']]], '0800.2b01.0203', ARRAY[MACADDR '08002b-010203', MACADDR '08002b-010203', '08002b010203'], ARRAY[ARRAY[MACADDR '08002b-010203', MACADDR '08002b-010203', '08002b010203']], ARRAY[ARRAY[ARRAY[MACADDR '08002b-010203', MACADDR '08002b-010203', '08002b010203']], ARRAY[ARRAY[MACADDR '08002b-010203', MACADDR '08002b-010203', '08002b010203']], ARRAY[ARRAY[MACADDR '08002b-010203', MACADDR '08002b-010203', '08002b010203']], ARRAY[ARRAY[MACADDR '08002b-010203', MACADDR '08002b-010203', '08002b010203']]], 0.99, ARRAY[1.1,2.22,3.33], ARRAY[ARRAY[1.55,2.66,3.88], ARRAY[11.5,10101.6,7111.1]], ARRAY[ARRAY[ARRAY[1,2,3]], ARRAY[ARRAY[5,6,7]], ARRAY[ARRAY[1.1,2.1,3]], ARRAY[ARRAY[5.0,6.0,7.0]]],test_jsonb(), ARRAY[test_jsonb(),test_jsonb(),test_jsonb(),test_jsonb()], ARRAY[ARRAY[test_jsonb(),test_jsonb(),test_jsonb(),test_jsonb()], ARRAY[test_jsonb(),test_jsonb(),test_jsonb(),test_jsonb()]], ARRAY[ARRAY[ARRAY[test_jsonb(),test_jsonb(),test_jsonb(),test_jsonb()]], ARRAY[ARRAY[test_jsonb(),test_jsonb(),test_jsonb(),test_jsonb()]], ARRAY[ARRAY[test_jsonb(),test_jsonb(),test_jsonb(),test_jsonb()]], ARRAY[ARRAY[test_jsonb(),test_jsonb(),test_jsonb(),test_jsonb()]]]); +-- different query/planning executiom types +-- compare results with Postgres +SELECT * FROM data_types_table + EXCEPT +SELECT * FROM data_types_table_local; + dist_key | col1 | col2 | col3 | col4 | col5 | col6 | col70 | col7 | col8 | col9 | col10 | col11 | col12 | col13 | col14 | col15 | col16 | col17 | col18 | col19 | col20 | col21 | col22 | col23 | col24 | col25 | col26 | col27 | col28 | col29 | col30 | col32 | col33 | col34 | col35 | col36 | col37 | col38 +--------------------------------------------------------------------- +(0 rows) + +SELECT * FROM data_types_table_local + EXCEPT +SELECT * FROM data_types_table; + dist_key | col1 | col2 | col3 | col4 | col5 | col6 | col70 | col7 | col8 | col9 | col10 | col11 | col12 | col13 | col14 | col15 | col16 | col17 | col18 | col19 | col20 | col21 | col22 | col23 | col24 | col25 | col26 | col27 | col28 | col29 | col30 | col32 | col33 | col34 | col35 | col36 | col37 | col38 +--------------------------------------------------------------------- +(0 rows) + +WITH cte_1 AS (SELECT * FROM data_types_table LIMIT 100000), +cte_2 AS (SELECT * FROM data_types_table_local LIMIT 100000) + SELECT * FROM cte_1 + EXCEPT + SELECT * FROM cte_2; + dist_key | col1 | col2 | col3 | col4 | col5 | col6 | col70 | col7 | col8 | col9 | col10 | col11 | col12 | col13 | col14 | col15 | col16 | col17 | col18 | col19 | col20 | col21 | col22 | col23 | col24 | col25 | col26 | col27 | col28 | col29 | col30 | col32 | col33 | col34 | col35 | col36 | col37 | col38 +--------------------------------------------------------------------- +(0 rows) + +WITH cte_1 AS (SELECT * FROM data_types_table LIMIT 100000), +cte_2 AS (SELECT * FROM data_types_table_local LIMIT 100000) + SELECT * FROM cte_2 + EXCEPT + SELECT * FROM cte_1; + dist_key | col1 | col2 | col3 | col4 | col5 | col6 | col70 | col7 | col8 | col9 | col10 | col11 | col12 | col13 | col14 | col15 | col16 | col17 | col18 | col19 | col20 | col21 | col22 | col23 | col24 | col25 | col26 | col27 | col28 | col29 | col30 | col32 | col33 | col34 | col35 | col36 | col37 | col38 +--------------------------------------------------------------------- +(0 rows) + +SELECT * FROM (SELECT *, random() > 100 FROM data_types_table) as foo +EXCEPT +SELECT * FROM (SELECT *, random() > 100 FROM data_types_table_local) as bar; + dist_key | col1 | col2 | col3 | col4 | col5 | col6 | col70 | col7 | col8 | col9 | col10 | col11 | col12 | col13 | col14 | col15 | col16 | col17 | col18 | col19 | col20 | col21 | col22 | col23 | col24 | col25 | col26 | col27 | col28 | col29 | col30 | col32 | col33 | col34 | col35 | col36 | col37 | col38 | ?column? +--------------------------------------------------------------------- +(0 rows) + +SELECT * FROM (SELECT *, random() > 100 FROM data_types_table_local) as bar +EXCEPT +SELECT * FROM (SELECT *, random() > 100 FROM data_types_table) as foo; + dist_key | col1 | col2 | col3 | col4 | col5 | col6 | col70 | col7 | col8 | col9 | col10 | col11 | col12 | col13 | col14 | col15 | col16 | col17 | col18 | col19 | col20 | col21 | col22 | col23 | col24 | col25 | col26 | col27 | col28 | col29 | col30 | col32 | col33 | col34 | col35 | col36 | col37 | col38 | ?column? +--------------------------------------------------------------------- +(0 rows) + +-- GROUP BY w/wout the dist key +SELECT + count(*) +FROM + data_types_table +GROUP BY + col1, col2, col3, col4, col5, col6, col70, col7, col8, col9, col10, col11, col12, col13, col14, col15, col16, col17, col18, col19, col20, col21, col22, col23, col24, col25, col26, col27, col28, col29, col32, col33, col34, col35, col36, col37, col38; + count +--------------------------------------------------------------------- + 1 + 1 +(2 rows) + +SELECT + count(*) +FROM + data_types_table +GROUP BY + dist_key, col1, col2, col3, col4, col5, col6, col70, col7, col8, col9, col10, col11, col12, col13, col14, col15, col16, col17, col18, col19, col20, col21, col22, col23, col24, col25, col26, col27, col28, col29, col32, col33, col34, col35, col36, col37, col38; + count +--------------------------------------------------------------------- + 1 + 1 +(2 rows) + +-- window function w/wout distribution key +SELECT + count(*) OVER (PARTITION BY col1, col2, col3, col4, col5, col6, col70, col7, col8, col9, col10, col11, col12, col13, col14, col15, col16, col17, col18, col19, col20, col21, col22, col23, col24, col25, col26, col27, col28, col29, col32, col33, col34, col35, col36, col37, col38) +FROM + data_types_table; + count +--------------------------------------------------------------------- + 1 + 1 +(2 rows) + +SELECT + count(*) OVER (PARTITION BY dist_key, col1, col2, col3, col4, col5, col6, col70, col7, col8, col9, col10, col11, col12, col13, col14, col15, col16, col17, col18, col19, col20, col21, col22, col23, col24, col25, col26, col27, col28, col29, col32, col33, col34, col35, col36, col37, col38) +FROM + data_types_table; + count +--------------------------------------------------------------------- + 1 + 1 +(2 rows) + +-- DISTINCT w/wout distribution key +-- there seems to be an issue with SELECT DISTINCT ROW with PG14 +-- so we add an alternative output that gives an error, this should +-- be removed after the issue is fixed on PG14. +SELECT DISTINCT(col1, col2, col3, col4, col5, col6, col70, col7, col8, col9, col10, col11, col12, col13, col14, col15, col16, col17, col18, col19, col20, col21, col22, col23, col24, col25, col26, col27, col28, col29, col32, col33, col34, col35, col36, col37, col38) +FROM + data_types_table +ORDER BY 1 DESC; +ERROR: could not identify a hash function for type bit +CONTEXT: while executing command on localhost:xxxxx +SELECT DISTINCT(dist_key, col1, col2, col3, col4, col5, col6, col70, col7, col8, col9, col10, col11, col12, col13, col14, col15, col16, col17, col18, col19, col20, col21, col22, col23, col24, col25, col26, col27, col28, col29, col32, col33, col34, col35, col36, col37, col38) +FROM + data_types_table +ORDER BY 1 DESC; +ERROR: could not identify a hash function for type bit +CONTEXT: while executing command on localhost:xxxxx +-- count DISTINCT w/wout dist key +SELECT count(DISTINCT(col1, col2, col3, col4, col5, col6, col70, col7, col8, col9, col10, col11, col12, col13, col14, col15, col16, col17, col18, col19, col20, col21, col22, col23, col24, col25, col26, col27, col28, col29, col32, col33, col34, col35, col36, col37, col38)) +FROM + data_types_table +ORDER BY 1 DESC; + count +--------------------------------------------------------------------- + 2 +(1 row) + +SELECT count(DISTINCT(dist_key, col1, col2, col3, col4, col5, col6, col70, col7, col8, col9, col10, col11, col12, col13, col14, col15, col16, col17, col18, col19, col20, col21, col22, col23, col24, col25, col26, col27, col28, col29, col32, col33, col34, col35, col36, col37, col38)) +FROM + data_types_table +ORDER BY 1 DESC; + count +--------------------------------------------------------------------- + 2 +(1 row) + +-- also test with RETURNING +ALTER TABLE data_types_table ADD COLUMN useless_column INT; +UPDATE data_types_table SET useless_column = 1 RETURNING *; + dist_key | col1 | col2 | col3 | col4 | col5 | col6 | col70 | col7 | col8 | col9 | col10 | col11 | col12 | col13 | col14 | col15 | col16 | col17 | col18 | col19 | col20 | col21 | col22 | col23 | col24 | col25 | col26 | col27 | col28 | col29 | col30 | col32 | col33 | col34 | col35 | col36 | col37 | col38 | useless_column +--------------------------------------------------------------------- + 1 | {1} | {{0,0,0}} | {{{0,0,0}}} | {1} | {{0,0,0}} | {{{0,0,0}}} | 1 | {1} | {{0,0,0}} | {{{0,0,0}}} | 11101 | {1} | {{01,01,01}} | {{{011,110,0000}}} | \xb4a8e04c0b | {"\\xb4a8e04c0b"} | {{"\\xb4a8e04c0b","\\xb4a8e04c0b","\\xb4a8e04c0b"}} | {{{"\\xb4a8e04c0b","\\x18a232a678","\\x38b2697632"}}} | t | {t} | {{t,t,f}} | {{{t,t,f}}} | 192.168.1.0/24 | {192.168.1.1} | {{0.0.0.0,0.0.0.0,::ffff:255.240.0.1,192.168.1.0/24}} | {{{0.0.0.0,0.0.0.0,::ffff:255.240.0.1,192.168.1.0/24}}} | 08:00:2b:01:02:03 | {08:00:2b:01:02:03} | {{08:00:2b:01:02:03,08:00:2b:01:02:03,08:00:2b:01:02:03}} | {{{08:00:2b:01:02:03,08:00:2b:01:02:03,08:00:2b:01:02:03}}} | 690 | {1.1} | {{0,0.111,0.15}} | {{{0,0,0}}} | {"test_json": "test"} | {"{\"test_json\": \"test\"}"} | {{"{\"test_json\": \"test\"}","{\"test_json\": \"test\"}","{\"test_json\": \"test\"}","{\"test_json\": \"test\"}"}} | {{{"{\"test_json\": \"test\"}","{\"test_json\": \"test\"}","{\"test_json\": \"test\"}","{\"test_json\": \"test\"}"}}} | 1 + 2 | {1,2,3} | {{1,2,3},{5,6,7}} | {{{1,2,3}},{{5,6,7}},{{1,2,3}},{{5,6,7}}} | {1,2,3} | {{1,2,3},{5,6,7}} | {{{1,2,3}},{{5,6,7}},{{1,2,3}},{{5,6,7}}} | 0 | {1,0,0} | {{1,1,0},{0,0,1}} | {{{1,1,1}},{{1,0,0}},{{1,1,1}},{{0,0,0}}} | 00010 | {11,10,01} | {{11,010,101},{101,01111,1000001}} | {{{10000,111111,1101010101}},{{1101010,0,1}},{{1,1,11111111}},{{0000000,0,0}}} | \xb4a8e04c0b | {"\\xb4a8e04c0b","\\x18a232a678","\\x38b2697632"} | {{"\\xb4a8e04c0b","\\x18a232a678","\\x38b2697632"},{"\\xb4a8e04c0b","\\x18a232a678","\\x38b2697632"}} | {{{"\\xb4a8e04c0b","\\x18a232a678","\\x38b2697632"}},{{"\\xb4a8e04c0b","\\x18a232a678","\\x38b2697632"}},{{"\\xb4a8e04c0b","\\x18a232a678","\\x38b2697632"}},{{"\\xb4a8e04c0b","\\x18a232a678","\\x38b2697632"}}} | t | {t,t,f} | {{t,t,f},{t,t,f}} | {{{t,t,f}},{{t,t,f}},{{t,t,f}},{{t,t,f}}} | 0.0.0.0 | {0.0.0.0,0.0.0.0,::ffff:255.240.0.1,192.168.1.0/24} | {{0.0.0.0,0.0.0.0,::ffff:255.240.0.1,192.168.1.0/24}} | {{{0.0.0.0,0.0.0.0,::ffff:255.240.0.1,192.168.1.0/24}},{{0.0.0.0,0.0.0.0,::ffff:255.240.0.1,192.168.1.0/24}},{{0.0.0.0,0.0.0.0,::ffff:255.240.0.1,192.168.1.0/24}},{{0.0.0.0,0.0.0.0,::ffff:255.240.0.1,192.168.1.0/24}}} | 08:00:2b:01:02:03 | {08:00:2b:01:02:03,08:00:2b:01:02:03,08:00:2b:01:02:03} | {{08:00:2b:01:02:03,08:00:2b:01:02:03,08:00:2b:01:02:03}} | {{{08:00:2b:01:02:03,08:00:2b:01:02:03,08:00:2b:01:02:03}},{{08:00:2b:01:02:03,08:00:2b:01:02:03,08:00:2b:01:02:03}},{{08:00:2b:01:02:03,08:00:2b:01:02:03,08:00:2b:01:02:03}},{{08:00:2b:01:02:03,08:00:2b:01:02:03,08:00:2b:01:02:03}}} | 0.99 | {1.1,2.22,3.33} | {{1.55,2.66,3.88},{11.5,10101.6,7111.1}} | {{{1,2,3}},{{5,6,7}},{{1.1,2.1,3}},{{5.0,6.0,7.0}}} | {"test_json": "test"} | {"{\"test_json\": \"test\"}","{\"test_json\": \"test\"}","{\"test_json\": \"test\"}","{\"test_json\": \"test\"}"} | {{"{\"test_json\": \"test\"}","{\"test_json\": \"test\"}","{\"test_json\": \"test\"}","{\"test_json\": \"test\"}"},{"{\"test_json\": \"test\"}","{\"test_json\": \"test\"}","{\"test_json\": \"test\"}","{\"test_json\": \"test\"}"}} | {{{"{\"test_json\": \"test\"}","{\"test_json\": \"test\"}","{\"test_json\": \"test\"}","{\"test_json\": \"test\"}"}},{{"{\"test_json\": \"test\"}","{\"test_json\": \"test\"}","{\"test_json\": \"test\"}","{\"test_json\": \"test\"}"}},{{"{\"test_json\": \"test\"}","{\"test_json\": \"test\"}","{\"test_json\": \"test\"}","{\"test_json\": \"test\"}"}},{{"{\"test_json\": \"test\"}","{\"test_json\": \"test\"}","{\"test_json\": \"test\"}","{\"test_json\": \"test\"}"}}} | 1 +(2 rows) + +-- three methods of INSERT .. SELECT +INSERT INTO data_types_table SELECT * FROM data_types_table ON CONFLICT (dist_key) DO UPDATE SET useless_column = 10; +INSERT INTO data_types_table SELECT * FROM data_types_table LIMIT 100000 ON CONFLICT (dist_key) DO UPDATE SET useless_column = 10; +INSERT INTO data_types_table (dist_key, col1, col2, col3, col4, col5, col6, col70, col7, col8, col9, col10, col11, col12, col13, col14, col15, col16, col17, col18, col19, col20, col21, col22, col23, col24, col25, col26, col27, col28, col29, col32, col33, col34, col35, col36, col37, col38) + SELECT dist_key+1, col1, col2, col3, col4, col5, col6, col70, col7, col8, col9, col10, col11, col12, col13, col14, col15, col16, col17, col18, col19, col20, col21, col22, col23, col24, col25, col26, col27, col28, col29, col32, col33, col34, col35, col36, col37, col38 FROM data_types_table ON CONFLICT (dist_key) DO UPDATE SET useless_column = 10; +SET client_min_messages TO ERROR; +DROP SCHEMA data_types CASCADE; diff --git a/src/test/regress/sql/data_types.sql b/src/test/regress/sql/data_types.sql index 5dd9b1496..5f9651567 100644 --- a/src/test/regress/sql/data_types.sql +++ b/src/test/regress/sql/data_types.sql @@ -99,6 +99,9 @@ FROM data_types_table; -- DISTINCT w/wout distribution key +-- there seems to be an issue with SELECT DISTINCT ROW with PG14 +-- so we add an alternative output that gives an error, this should +-- be removed after the issue is fixed on PG14. SELECT DISTINCT(col1, col2, col3, col4, col5, col6, col70, col7, col8, col9, col10, col11, col12, col13, col14, col15, col16, col17, col18, col19, col20, col21, col22, col23, col24, col25, col26, col27, col28, col29, col32, col33, col34, col35, col36, col37, col38) FROM data_types_table From b16dadbe7c5c73fc10bcdd0623b2bf555b18aefe Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Tue, 24 Aug 2021 12:05:24 +0300 Subject: [PATCH 085/104] Avoid NOTICE message to avoid an alternative output with pg14 --- .../regress/expected/citus_local_tables.out | 18 +++++++++--------- src/test/regress/sql/citus_local_tables.sql | 3 ++- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/test/regress/expected/citus_local_tables.out b/src/test/regress/expected/citus_local_tables.out index 517c1271b..6ed2c98b1 100644 --- a/src/test/regress/expected/citus_local_tables.out +++ b/src/test/regress/expected/citus_local_tables.out @@ -86,9 +86,9 @@ SELECT citus_add_local_table_to_metadata('citus_local_table_2'); -- also create indexes on them CREATE INDEX citus_local_table_1_idx ON citus_local_table_1(a); -NOTICE: executing the command locally: CREATE INDEX citus_local_table_1_idx_1504001 ON citus_local_tables_test_schema.citus_local_table_1_1504001 USING btree (a ) +NOTICE: executing the command locally: CREATE INDEX citus_local_table_1_idx_1504001 ON citus_local_tables_test_schema.citus_local_table_1_1504001 USING btree (a ) CREATE INDEX citus_local_table_2_idx ON citus_local_table_2(a); -NOTICE: executing the command locally: CREATE INDEX citus_local_table_2_idx_1504002 ON citus_local_tables_test_schema.citus_local_table_2_1504002 USING btree (a ) +NOTICE: executing the command locally: CREATE INDEX citus_local_table_2_idx_1504002 ON citus_local_tables_test_schema.citus_local_table_2_1504002 USING btree (a ) -- drop them for next tests DROP TABLE citus_local_table_1, citus_local_table_2; NOTICE: executing the command locally: DROP TABLE IF EXISTS citus_local_tables_test_schema.citus_local_table_2_xxxxx CASCADE @@ -416,10 +416,10 @@ NOTICE: identifier "LocalTabLE.1!?!90123456789012345678901234567890123456789012 CREATE UNIQUE INDEX uniqueIndex ON "LocalTabLE.1!?!9012345678901234567890123456789012345678901234567890123456789" (id); NOTICE: identifier "LocalTabLE.1!?!9012345678901234567890123456789012345678901234567890123456789" will be truncated to "LocalTabLE.1!?!901234567890123456789012345678901234567890123456" -- ingest some data before citus_add_local_table_to_metadata +set client_min_messages to ERROR; INSERT INTO "LocalTabLE.1!?!9012345678901234567890123456789012345678901234567890123456789" VALUES (1, 1, (1, row_to_json(row(1,1)))::local_type, row_to_json(row(1,1), true)), (2, 1, (2, row_to_json(row(2,2)))::local_type, row_to_json(row(2,2), 'false')); -NOTICE: identifier "LocalTabLE.1!?!9012345678901234567890123456789012345678901234567890123456789" will be truncated to "LocalTabLE.1!?!901234567890123456789012345678901234567890123456" -NOTICE: identifier "LocalTabLE.1!?!9012345678901234567890123456789012345678901234567890123456789" will be truncated to "LocalTabLE.1!?!901234567890123456789012345678901234567890123456" +reset client_min_messages; -- create a replica identity before citus_add_local_table_to_metadata ALTER TABLE "LocalTabLE.1!?!9012345678901234567890123456789012345678901234567890123456789" REPLICA IDENTITY USING INDEX uniqueIndex; NOTICE: identifier "LocalTabLE.1!?!9012345678901234567890123456789012345678901234567890123456789" will be truncated to "LocalTabLE.1!?!901234567890123456789012345678901234567890123456" @@ -445,10 +445,10 @@ SELECT citus_add_local_table_to_metadata('"LocalTabLE.1!?!9012345678901234567890 -- create some objects after citus_add_local_table_to_metadata CREATE INDEX "my!Index2" ON "LocalTabLE.1!?!9012345678901234567890123456789012345678901234567890123456789"(id) WITH ( fillfactor = 90 ) WHERE id < 20; NOTICE: identifier "LocalTabLE.1!?!9012345678901234567890123456789012345678901234567890123456789" will be truncated to "LocalTabLE.1!?!901234567890123456789012345678901234567890123456" -NOTICE: executing the command locally: CREATE INDEX "my!Index2_1504036" ON "CiTUS!LocalTables"."LocalTabLE.1!?!9012345678901234567890123456789_7e923997_1504036" USING btree (id ) WITH (fillfactor = '90' )WHERE (id < 20) +NOTICE: executing the command locally: CREATE INDEX "my!Index2_1504036" ON "CiTUS!LocalTables"."LocalTabLE.1!?!9012345678901234567890123456789_7e923997_1504036" USING btree (id ) WITH (fillfactor = '90' )WHERE (id < 20) CREATE UNIQUE INDEX uniqueIndex2 ON "LocalTabLE.1!?!9012345678901234567890123456789012345678901234567890123456789"(id); NOTICE: identifier "LocalTabLE.1!?!9012345678901234567890123456789012345678901234567890123456789" will be truncated to "LocalTabLE.1!?!901234567890123456789012345678901234567890123456" -NOTICE: executing the command locally: CREATE UNIQUE INDEX uniqueindex2_1504036 ON "CiTUS!LocalTables"."LocalTabLE.1!?!9012345678901234567890123456789_7e923997_1504036" USING btree (id ) +NOTICE: executing the command locally: CREATE UNIQUE INDEX uniqueindex2_1504036 ON "CiTUS!LocalTables"."LocalTabLE.1!?!9012345678901234567890123456789_7e923997_1504036" USING btree (id ) --------------------------------------------------------------------- ---- utility command execution ---- --------------------------------------------------------------------- @@ -563,7 +563,7 @@ ORDER BY 1; (2 rows) CREATE UNIQUE INDEX citus_local_table_1_idx ON citus_local_table_1(b); -NOTICE: executing the command locally: CREATE UNIQUE INDEX citus_local_table_1_idx_1504027 ON citus_local_tables_test_schema.citus_local_table_1_1504027 USING btree (b ) +NOTICE: executing the command locally: CREATE UNIQUE INDEX citus_local_table_1_idx_1504027 ON citus_local_tables_test_schema.citus_local_table_1_1504027 USING btree (b ) -- show that we successfully defined the unique index SELECT indexrelid::regclass, indrelid::regclass, indkey FROM pg_index @@ -621,7 +621,7 @@ SELECT citus_add_local_table_to_metadata('citus_local_table_4'); INSERT INTO citus_local_table_4 VALUES (1), (2), (3); NOTICE: executing the command locally: INSERT INTO citus_local_tables_test_schema.citus_local_table_4_xxxx AS citus_table_alias (a) VALUES (1), (2), (3) CREATE INDEX citus_local_table_4_idx ON citus_local_table_4(a); -NOTICE: executing the command locally: CREATE INDEX citus_local_table_4_idx_xxxxxx ON citus_local_tables_test_schema.citus_local_table_4_xxxx USING btree (a ) +NOTICE: executing the command locally: CREATE INDEX citus_local_table_4_idx_xxxxxx ON citus_local_tables_test_schema.citus_local_table_4_xxxx USING btree (a ) SELECT citus_table_size('citus_local_table_4'); citus_table_size --------------------------------------------------------------------- @@ -677,7 +677,7 @@ FROM pg_dist_partition WHERE logicalrelid = 'citus_local_table_4'::regclass; (1 row) SELECT column_name_to_column('citus_local_table_4', 'a'); - column_name_to_column + column_name_to_column --------------------------------------------------------------------- {VAR :varno 1 :varattno 1 :vartype 23 :vartypmod -1 :varcollid 0 :varlevelsup 0 :varnoold 1 :varoattno 1 :location -1} (1 row) diff --git a/src/test/regress/sql/citus_local_tables.sql b/src/test/regress/sql/citus_local_tables.sql index a4b235463..e7c495614 100644 --- a/src/test/regress/sql/citus_local_tables.sql +++ b/src/test/regress/sql/citus_local_tables.sql @@ -315,9 +315,10 @@ CREATE INDEX "my!Index1" ON "LocalTabLE.1!?!901234567890123456789012345678901234 CREATE UNIQUE INDEX uniqueIndex ON "LocalTabLE.1!?!9012345678901234567890123456789012345678901234567890123456789" (id); -- ingest some data before citus_add_local_table_to_metadata +set client_min_messages to ERROR; INSERT INTO "LocalTabLE.1!?!9012345678901234567890123456789012345678901234567890123456789" VALUES (1, 1, (1, row_to_json(row(1,1)))::local_type, row_to_json(row(1,1), true)), (2, 1, (2, row_to_json(row(2,2)))::local_type, row_to_json(row(2,2), 'false')); - +reset client_min_messages; -- create a replica identity before citus_add_local_table_to_metadata ALTER TABLE "LocalTabLE.1!?!9012345678901234567890123456789012345678901234567890123456789" REPLICA IDENTITY USING INDEX uniqueIndex; From bd501b4d80ed800cc5805ed70c479af623a227a0 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Tue, 24 Aug 2021 21:33:11 +0300 Subject: [PATCH 086/104] Enable pg12-pg14 upgrade test --- .circleci/config.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 35f700a04..bd2b9b22d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -681,6 +681,13 @@ workflows: image_tag: '12.8-13.4' requires: [build-12,build-13] + - test-pg-upgrade: + name: 'test-12-14_check-pg-upgrade' + old_pg_major: 12 + new_pg_major: 14 + image_tag: '12-13-14-dev202108191715' + requires: [build-12,build-14] + - test-pg-upgrade: name: 'test-13-14_check-pg-upgrade' old_pg_major: 13 From a6c40ebd141d1769f21b1529d3565e7d576ebf22 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Wed, 25 Aug 2021 11:33:42 +0300 Subject: [PATCH 087/104] Fix multi_follower_dml When the_table is emtpy, we don't get an error with pg14 anymore so we replace it generate_series so that we get the error. --- src/test/regress/expected/multi_follower_dml.out | 2 +- src/test/regress/sql/multi_follower_dml.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/regress/expected/multi_follower_dml.out b/src/test/regress/expected/multi_follower_dml.out index 0d1d6fd68..395c5157c 100644 --- a/src/test/regress/expected/multi_follower_dml.out +++ b/src/test/regress/expected/multi_follower_dml.out @@ -243,7 +243,7 @@ SELECT * FROM citus_local_table ORDER BY a; -- we should still disallow writes to local tables INSERT INTO local VALUES (1, 1); ERROR: cannot execute INSERT in a read-only transaction -INSERT INTO local SELECT a, b FROM the_table; +INSERT INTO local SELECT i,i FROM generate_series(0,100)i; ERROR: cannot execute INSERT in a read-only transaction -- we shouldn't be able to create local tables CREATE TEMP TABLE local_copy_of_the_table AS SELECT * FROM the_table; diff --git a/src/test/regress/sql/multi_follower_dml.sql b/src/test/regress/sql/multi_follower_dml.sql index d8cb17bb6..bca04d0a7 100644 --- a/src/test/regress/sql/multi_follower_dml.sql +++ b/src/test/regress/sql/multi_follower_dml.sql @@ -135,7 +135,7 @@ SELECT * FROM citus_local_table ORDER BY a; -- we should still disallow writes to local tables INSERT INTO local VALUES (1, 1); -INSERT INTO local SELECT a, b FROM the_table; +INSERT INTO local SELECT i,i FROM generate_series(0,100)i; -- we shouldn't be able to create local tables CREATE TEMP TABLE local_copy_of_the_table AS SELECT * FROM the_table; From c799d8cad8ca7027e3f805782de794246e737b2d Mon Sep 17 00:00:00 2001 From: Nils Dijk Date: Thu, 19 Aug 2021 17:25:07 +0200 Subject: [PATCH 088/104] add 14beta3 to CI --- .circleci/config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index bd2b9b22d..726942e25 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -448,6 +448,10 @@ workflows: name: build-13 pg_major: 13 image_tag: '13.4' + - build: + name: build-14 + pg_major: 14 + image_tag: '14beta3-dev202108191715' - check-style - check-sql-snapshots From 307eb812787420c09836c4b5e33188457c3cd1ce Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Wed, 25 Aug 2021 17:05:59 +0300 Subject: [PATCH 089/104] Fix failure for 1pc_copy_hash --- .../expected/failure_1pc_copy_append.out | 25 ------------------- .../expected/failure_1pc_copy_hash.out | 19 -------------- .../regress/sql/failure_1pc_copy_append.sql | 2 -- .../regress/sql/failure_1pc_copy_hash.sql | 2 -- 4 files changed, 48 deletions(-) diff --git a/src/test/regress/expected/failure_1pc_copy_append.out b/src/test/regress/expected/failure_1pc_copy_append.out index 7ccbbb4f1..9e86d7098 100644 --- a/src/test/regress/expected/failure_1pc_copy_append.out +++ b/src/test/regress/expected/failure_1pc_copy_append.out @@ -31,31 +31,6 @@ SELECT count(1) FROM copy_test; 4 (1 row) -SELECT citus.dump_network_traffic(); - dump_network_traffic ---------------------------------------------------------------------- - (0,coordinator,"[initial message]") - (0,worker,"['AuthenticationOk()', 'ParameterStatus(application_name=citus)', 'ParameterStatus(client_encoding=UTF8)', 'ParameterStatus(DateStyle=ISO, MDY)', 'ParameterStatus(integer_datetimes=on)', 'ParameterStatus(IntervalStyle=postgres)', 'ParameterStatus(is_superuser=on)', 'ParameterStatus(server_encoding=UTF8)', 'ParameterStatus(server_version=XXX)', 'ParameterStatus(session_authorization=postgres)', 'ParameterStatus(standard_conforming_strings=on)', 'ParameterStatus(TimeZone=XXX)', 'BackendKeyData(XXX)', 'ReadyForQuery(state=idle)']") - (0,coordinator,"[""Query(query=SELECT worker_apply_shard_ddl_command (100400, 'CREATE TABLE public.copy_test (key integer, value integer) '))""]") - (0,worker,"[""RowDescription(fieldcount=1,fields=['F(name=worker_apply_shard_ddl_command,tableoid=0,colattrnum=0,typoid=2278,typlen=4,typmod=-1,format_code=0)'])"", 'DataRow(columncount=1,columns=[""C(length=0,value=b\\'\\')""])', 'CommandComplete(command=SELECT 1)', 'ReadyForQuery(state=idle)']") - (0,coordinator,"[""Query(query=SELECT worker_apply_shard_ddl_command (100400, 'ALTER TABLE public.copy_test OWNER TO postgres'))""]") - (0,worker,"[""RowDescription(fieldcount=1,fields=['F(name=worker_apply_shard_ddl_command,tableoid=0,colattrnum=0,typoid=2278,typlen=4,typmod=-1,format_code=0)'])"", 'DataRow(columncount=1,columns=[""C(length=0,value=b\\'\\')""])', 'CommandComplete(command=SELECT 1)', 'ReadyForQuery(state=idle)']") - (0,coordinator,"[""Query(query=BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(0, XX, 'XXXX-XX-XX XX:XX:XX.XXXXXX-XX');)""]") - (0,worker,"['CommandComplete(command=BEGIN)', ""RowDescription(fieldcount=1,fields=['F(name=assign_distributed_transaction_id,tableoid=0,colattrnum=0,typoid=2278,typlen=4,typmod=-1,format_code=0)'])"", 'DataRow(columncount=1,columns=[""C(length=0,value=b\\'\\')""])', 'CommandComplete(command=SELECT 1)', 'ReadyForQuery(state=in_transaction_block)']") - (0,coordinator,"[""Query(query=COPY public.copy_test_XXXXXX FROM STDIN WITH (format 'binary'))""]") - (0,worker,"[""Backend(type=G,body=b'\\\\x01\\\\x00\\\\x02\\\\x00\\\\x01\\\\x00\\\\x01')""]") - (0,coordinator,"[""CopyData(data=b'PGCOPY\\\\n\\\\xff\\\\r\\\\n\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00')"", ""CopyData(data=b'\\\\x00\\\\x02\\\\x00\\\\x00\\\\x00\\\\x04\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x04\\\\x00\\\\x00\\\\x00\\\\x00')"", ""CopyData(data=b'\\\\x00\\\\x02\\\\x00\\\\x00\\\\x00\\\\x04\\\\x00\\\\x00\\\\x00\\\\x01\\\\x00\\\\x00\\\\x00\\\\x04\\\\x00\\\\x00\\\\x00\\\\x01')"", ""CopyData(data=b'\\\\x00\\\\x02\\\\x00\\\\x00\\\\x00\\\\x04\\\\x00\\\\x00\\\\x00\\\\x02\\\\x00\\\\x00\\\\x00\\\\x04\\\\x00\\\\x00\\\\x00\\\\x04')"", ""CopyData(data=b'\\\\x00\\\\x02\\\\x00\\\\x00\\\\x00\\\\x04\\\\x00\\\\x00\\\\x00\\\\x03\\\\x00\\\\x00\\\\x00\\\\x04\\\\x00\\\\x00\\\\x00\\\\t')"", ""CopyData(data=b'\\\\xff\\\\xff')"", 'CopyDone()']") - (0,worker,"['CommandComplete(command=COPY 4)', 'ReadyForQuery(state=in_transaction_block)']") - (0,coordinator,"[""Query(query=SELECT pg_table_size('public.copy_test_XXXXXX'))""]") - (0,worker,"[""RowDescription(fieldcount=1,fields=['F(name=pg_table_size,tableoid=0,colattrnum=0,typoid=20,typlen=8,typmod=-1,format_code=0)'])"", 'DataRow(columncount=1,columns=[""C(length=0,value=b\\'\\')""])', 'CommandComplete(command=SELECT 1)', 'ReadyForQuery(state=in_transaction_block)']") - (0,coordinator,"['Query(query=SELECT min(key), max(key) FROM public.copy_test_XXXXXX)']") - (0,worker,"[""RowDescription(fieldcount=2,fields=['F(name=min,tableoid=0,colattrnum=0,typoid=23,typlen=4,typmod=-1,format_code=0)', 'F(name=max,tableoid=0,colattrnum=0,typoid=23,typlen=4,typmod=-1,format_code=0)'])"", 'DataRow(columncount=2,columns=[""C(length=0,value=b\\'\\')"", ""C(length=1,value=b\\'0\\')""])', 'CommandComplete(command=SELECT 1)', 'ReadyForQuery(state=in_transaction_block)']") - (0,coordinator,"['Query(query=COMMIT)']") - (0,worker,"['CommandComplete(command=COMMIT)', 'ReadyForQuery(state=idle)']") - (0,coordinator,"['Query(query=SELECT count(1) AS count FROM public.copy_test_XXXXXX copy_test WHERE true)']") - (0,worker,"[""RowDescription(fieldcount=1,fields=['F(name=count,tableoid=0,colattrnum=0,typoid=20,typlen=8,typmod=-1,format_code=0)'])"", 'DataRow(columncount=1,columns=[""C(length=0,value=b\\'\\')""])', 'CommandComplete(command=SELECT 1)', 'ReadyForQuery(state=idle)']") -(20 rows) - ---- all of the following tests test behavior with 2 shard placements ---- SHOW citus.shard_replication_factor; citus.shard_replication_factor diff --git a/src/test/regress/expected/failure_1pc_copy_hash.out b/src/test/regress/expected/failure_1pc_copy_hash.out index e99c6f560..227d420e3 100644 --- a/src/test/regress/expected/failure_1pc_copy_hash.out +++ b/src/test/regress/expected/failure_1pc_copy_hash.out @@ -32,25 +32,6 @@ SELECT count(1) FROM copy_test; 4 (1 row) -SELECT citus.dump_network_traffic(); - dump_network_traffic ---------------------------------------------------------------------- - (0,coordinator,"[initial message]") - (0,worker,"['AuthenticationOk()', 'ParameterStatus(application_name=citus)', 'ParameterStatus(client_encoding=UTF8)', 'ParameterStatus(DateStyle=ISO, MDY)', 'ParameterStatus(default_transaction_read_only=off)', 'ParameterStatus(in_hot_standby=off)', 'ParameterStatus(integer_datetimes=on)', 'ParameterStatus(IntervalStyle=postgres)', 'ParameterStatus(is_superuser=on)', 'ParameterStatus(server_encoding=UTF8)', 'ParameterStatus(server_version=XXX)', 'ParameterStatus(session_authorization=postgres)', 'ParameterStatus(standard_conforming_strings=on)', 'ParameterStatus(TimeZone=XXX)', 'BackendKeyData(XXX)', 'ReadyForQuery(state=idle)']") - (0,coordinator,"[""Query(query=BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(0, XX, 'XXXX-XX-XX XX:XX:XX.XXXXXX-XX');)""]") - (0,worker,"['CommandComplete(command=BEGIN)', ""RowDescription(fieldcount=1,fields=['F(name=assign_distributed_transaction_id,tableoid=0,colattrnum=0,typoid=2278,typlen=4,typmod=-1,format_code=0)'])"", 'DataRow(columncount=1,columns=[""C(length=0,value=b\\'\\')""])', 'CommandComplete(command=SELECT 1)', 'ReadyForQuery(state=in_transaction_block)']") - (0,coordinator,"[""Query(query=COPY public.copy_test_XXXXXX (key, value) FROM STDIN WITH (format 'binary'))""]") - (0,worker,"[""Backend(type=G,body=b'\\\\x01\\\\x00\\\\x02\\\\x00\\\\x01\\\\x00\\\\x01')""]") - (0,coordinator,"[""CopyData(data=b'PGCOPY\\\\n\\\\xff\\\\r\\\\n\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00')"", ""CopyData(data=b'\\\\x00\\\\x02\\\\x00\\\\x00\\\\x00\\\\x04\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x04\\\\x00\\\\x00\\\\x00\\\\x00')"", ""CopyData(data=b'\\\\x00\\\\x02\\\\x00\\\\x00\\\\x00\\\\x04\\\\x00\\\\x00\\\\x00\\\\x01\\\\x00\\\\x00\\\\x00\\\\x04\\\\x00\\\\x00\\\\x00\\\\x01')"", ""CopyData(data=b'\\\\x00\\\\x02\\\\x00\\\\x00\\\\x00\\\\x04\\\\x00\\\\x00\\\\x00\\\\x02\\\\x00\\\\x00\\\\x00\\\\x04\\\\x00\\\\x00\\\\x00\\\\x04')"", ""CopyData(data=b'\\\\x00\\\\x02\\\\x00\\\\x00\\\\x00\\\\x04\\\\x00\\\\x00\\\\x00\\\\x03\\\\x00\\\\x00\\\\x00\\\\x04\\\\x00\\\\x00\\\\x00\\\\t')"", ""CopyData(data=b'\\\\xff\\\\xff')"", 'CopyDone()']") - (0,worker,"['CommandComplete(command=COPY 4)', 'ReadyForQuery(state=in_transaction_block)']") - (0,coordinator,"['Query(query=COMMIT)']") - (0,worker,"['CommandComplete(command=COMMIT)', 'ReadyForQuery(state=idle)']") - (1,coordinator,"[initial message]") - (1,worker,"['AuthenticationOk()', 'ParameterStatus(application_name=citus)', 'ParameterStatus(client_encoding=UTF8)', 'ParameterStatus(DateStyle=ISO, MDY)', 'ParameterStatus(default_transaction_read_only=off)', 'ParameterStatus(in_hot_standby=off)', 'ParameterStatus(integer_datetimes=on)', 'ParameterStatus(IntervalStyle=postgres)', 'ParameterStatus(is_superuser=on)', 'ParameterStatus(server_encoding=UTF8)', 'ParameterStatus(server_version=XXX)', 'ParameterStatus(session_authorization=postgres)', 'ParameterStatus(standard_conforming_strings=on)', 'ParameterStatus(TimeZone=XXX)', 'BackendKeyData(XXX)', 'ReadyForQuery(state=idle)']") - (1,coordinator,"['Query(query=SELECT count(1) AS count FROM public.copy_test_XXXXXX copy_test)']") - (1,worker,"[""RowDescription(fieldcount=1,fields=['F(name=count,tableoid=0,colattrnum=0,typoid=20,typlen=8,typmod=-1,format_code=0)'])"", 'DataRow(columncount=1,columns=[""C(length=0,value=b\\'\\')""])', 'CommandComplete(command=SELECT 1)', 'ReadyForQuery(state=idle)']") -(14 rows) - -- ==== kill the connection when we try to start a transaction ==== -- the query should abort SELECT citus.mitmproxy('conn.onQuery(query="assign_distributed_transaction").killall()'); diff --git a/src/test/regress/sql/failure_1pc_copy_append.sql b/src/test/regress/sql/failure_1pc_copy_append.sql index b2740a8b9..a97e21058 100644 --- a/src/test/regress/sql/failure_1pc_copy_append.sql +++ b/src/test/regress/sql/failure_1pc_copy_append.sql @@ -17,8 +17,6 @@ SELECT citus.clear_network_traffic(); COPY copy_test FROM PROGRAM 'echo 0, 0 && echo 1, 1 && echo 2, 4 && echo 3, 9' WITH CSV; SELECT count(1) FROM copy_test; -SELECT citus.dump_network_traffic(); - ---- all of the following tests test behavior with 2 shard placements ---- SHOW citus.shard_replication_factor; diff --git a/src/test/regress/sql/failure_1pc_copy_hash.sql b/src/test/regress/sql/failure_1pc_copy_hash.sql index 6f32ce6cc..34c3d15c4 100644 --- a/src/test/regress/sql/failure_1pc_copy_hash.sql +++ b/src/test/regress/sql/failure_1pc_copy_hash.sql @@ -18,8 +18,6 @@ SELECT citus.clear_network_traffic(); COPY copy_test FROM PROGRAM 'echo 0, 0 && echo 1, 1 && echo 2, 4 && echo 3, 9' WITH CSV; SELECT count(1) FROM copy_test; -SELECT citus.dump_network_traffic(); - -- ==== kill the connection when we try to start a transaction ==== -- the query should abort From 4e85d9ffce9919c213eb5ef239e33538a333b535 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Thu, 26 Aug 2021 09:37:26 +0300 Subject: [PATCH 090/104] Add empty pg14 sql file --- src/test/regress/expected/pg14.out | 7 +++++++ src/test/regress/expected/pg14_0.out | 6 ++++++ src/test/regress/multi_schedule | 2 +- src/test/regress/sql/pg14.sql | 7 +++++++ 4 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 src/test/regress/expected/pg14.out create mode 100644 src/test/regress/expected/pg14_0.out create mode 100644 src/test/regress/sql/pg14.sql diff --git a/src/test/regress/expected/pg14.out b/src/test/regress/expected/pg14.out new file mode 100644 index 000000000..35b3a5676 --- /dev/null +++ b/src/test/regress/expected/pg14.out @@ -0,0 +1,7 @@ +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int > 13 AS server_version_above_thirteen +\gset +\if :server_version_above_thirteen +\else +\q +\endif diff --git a/src/test/regress/expected/pg14_0.out b/src/test/regress/expected/pg14_0.out new file mode 100644 index 000000000..65b2376bc --- /dev/null +++ b/src/test/regress/expected/pg14_0.out @@ -0,0 +1,6 @@ +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int > 13 AS server_version_above_thirteen +\gset +\if :server_version_above_thirteen +\else +\q diff --git a/src/test/regress/multi_schedule b/src/test/regress/multi_schedule index 87ce839c3..6b31d2673 100644 --- a/src/test/regress/multi_schedule +++ b/src/test/regress/multi_schedule @@ -51,7 +51,7 @@ test: subquery_in_targetlist subquery_in_where subquery_complex_target_list test: subquery_prepared_statements test: non_colocated_leaf_subquery_joins non_colocated_subquery_joins non_colocated_join_order test: cte_inline recursive_view_local_table values -test: pg13 pg12 +test: pg13 pg12 pg14 test: tableam drop_column_partitioned_table # ---------- diff --git a/src/test/regress/sql/pg14.sql b/src/test/regress/sql/pg14.sql new file mode 100644 index 000000000..35b3a5676 --- /dev/null +++ b/src/test/regress/sql/pg14.sql @@ -0,0 +1,7 @@ +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int > 13 AS server_version_above_thirteen +\gset +\if :server_version_above_thirteen +\else +\q +\endif From 35a3f7240dd92992df3ca824bab2435d702676d7 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Thu, 26 Aug 2021 13:11:12 +0300 Subject: [PATCH 091/104] CHANGELOG: Allow REINDEX to change the tablespace of the new index --- .../distributed/deparser/citus_ruleutils.c | 50 ++++++- src/test/regress/Makefile | 13 +- src/test/regress/base_schedule | 2 +- src/test/regress/bin/normalize.sed | 3 + src/test/regress/expected/pg14.out | 122 ++++++++++++++++++ src/test/regress/expected/tablespace.out | 5 + src/test/regress/input/tablespace.source | 5 + src/test/regress/minimal_schedule | 1 + src/test/regress/multi_schedule | 1 + src/test/regress/output/tablespace.source | 5 + src/test/regress/sql/pg14.sql | 26 ++++ src/test/regress/sql/tablespace.sql | 5 + 12 files changed, 230 insertions(+), 8 deletions(-) create mode 100644 src/test/regress/expected/tablespace.out create mode 100644 src/test/regress/input/tablespace.source create mode 100644 src/test/regress/output/tablespace.source create mode 100644 src/test/regress/sql/tablespace.sql diff --git a/src/backend/distributed/deparser/citus_ruleutils.c b/src/backend/distributed/deparser/citus_ruleutils.c index 70bacd204..86584245b 100644 --- a/src/backend/distributed/deparser/citus_ruleutils.c +++ b/src/backend/distributed/deparser/citus_ruleutils.c @@ -78,6 +78,7 @@ static void AppendStorageParametersToString(StringInfo stringBuffer, List *optionList); static void simple_quote_literal(StringInfo buf, const char *val); static char * flatten_reloptions(Oid relid); +static void AddVacuumParams(ReindexStmt *reindexStmt, StringInfo buffer); /* @@ -755,11 +756,7 @@ deparse_shard_reindex_statement(ReindexStmt *origStmt, Oid distrelid, int64 shar } appendStringInfoString(buffer, "REINDEX "); - - if (IsReindexWithParam_compat(reindexStmt, "verbose")) - { - appendStringInfoString(buffer, "(VERBOSE) "); - } + AddVacuumParams(reindexStmt, buffer); switch (reindexStmt->kind) { @@ -829,6 +826,49 @@ bool IsReindexWithParam_compat(ReindexStmt* reindexStmt, char* param) { } +/* + * AddVacuumParams adds vacuum params to the given buffer. + */ +static void +AddVacuumParams(ReindexStmt *reindexStmt, StringInfo buffer) +{ + StringInfo temp = makeStringInfo(); + if (IsReindexWithParam_compat(reindexStmt, "verbose")) + { + appendStringInfoString(temp, "VERBOSE"); + } +#if PG_VERSION_NUM >= PG_VERSION_14 + char *tableSpaceName = NULL; + DefElem *opt = NULL; + foreach_ptr(opt, reindexStmt->params) + { + if (strcmp(opt->defname, "tablespace") == 0) + { + tableSpaceName = defGetString(opt); + break; + } + } + + if (tableSpaceName) + { + if (temp->len > 0) + { + appendStringInfo(temp, ", TABLESPACE %s", tableSpaceName); + } + else + { + appendStringInfo(temp, "TABLESPACE %s", tableSpaceName); + } + } +#endif + + if (temp->len > 0) + { + appendStringInfo(buffer, "(%s) ", temp->data); + } +} + + /* deparse_index_columns appends index or include parameters to the provided buffer */ static void deparse_index_columns(StringInfo buffer, List *indexParameterList, List *deparseContext) diff --git a/src/test/regress/Makefile b/src/test/regress/Makefile index 39f82df62..670ca4d79 100644 --- a/src/test/regress/Makefile +++ b/src/test/regress/Makefile @@ -52,12 +52,19 @@ ISOLATION_BUILDDIR=build/specs # ex: make print-generated_isolation_files print-% : ; @echo $* = $($*) -.PHONY: create-symbolic-link +.PHONY: create-symbolic-link create-tablespaces create-symbolic-link: mkdir -p $(citus_abs_srcdir)/build ln -fsn $(citus_abs_srcdir)/expected $(citus_abs_srcdir)/build/ +create-tablespaces: + rm -rf $(citus_abs_srcdir)/tmp_check/ts1 + mkdir -p $(citus_abs_srcdir)/tmp_check/ts1 + rm -rf $(citus_abs_srcdir)/tmp_check/ts0 + mkdir -p $(citus_abs_srcdir)/tmp_check/ts0 + rm -rf $(citus_abs_srcdir)/tmp_check/ts2 + mkdir -p $(citus_abs_srcdir)/tmp_check/ts2 # How this target works: # cpp is used before running isolation tests to preprocess spec files. @@ -106,7 +113,7 @@ check-base: all -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/base_schedule $(EXTRA_TESTS) # check-minimal only sets up the cluster -check-minimal: all +check-minimal: all create-tablespaces $(pg_regress_multi_check) --load-extension=citus \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/minimal_schedule $(EXTRA_TESTS) @@ -235,3 +242,5 @@ clean-upgrade-artifacts: clean distclean maintainer-clean: rm -f $(output_files) $(input_files) rm -rf tmp_check/ + +all: create-tablespaces \ No newline at end of file diff --git a/src/test/regress/base_schedule b/src/test/regress/base_schedule index 34e770b5a..e3ea4de24 100644 --- a/src/test/regress/base_schedule +++ b/src/test/regress/base_schedule @@ -6,4 +6,4 @@ test: multi_test_helpers multi_test_helpers_superuser multi_create_fdw columnar_ test: multi_test_catalog_views test: multi_create_table multi_behavioral_analytics_create_table test: multi_create_table_superuser multi_behavioral_analytics_create_table_superuser -test: multi_load_data multi_load_data_superuser +test: multi_load_data multi_load_data_superuser tablespace diff --git a/src/test/regress/bin/normalize.sed b/src/test/regress/bin/normalize.sed index ae4b63ac7..468d81810 100644 --- a/src/test/regress/bin/normalize.sed +++ b/src/test/regress/bin/normalize.sed @@ -241,3 +241,6 @@ s/ERROR: ROLLBACK is not allowed in an SQL function/ERROR: ROLLBACK is not all /Parent-Relationship/d s/function array_cat_agg\(anycompatiblearray\)/function array_cat_agg\(anyarray\)/g s/TRIM\(BOTH FROM value\)/btrim\(value\)/g +s/pg14\.idx.*/pg14\.xxxxx/g + +s/CREATE TABLESPACE test_tablespace LOCATION.*/CREATE TABLESPACE test_tablespace LOCATION XXXX/g diff --git a/src/test/regress/expected/pg14.out b/src/test/regress/expected/pg14.out index 35b3a5676..6e42f4fbc 100644 --- a/src/test/regress/expected/pg14.out +++ b/src/test/regress/expected/pg14.out @@ -5,3 +5,125 @@ SELECT substring(:'server_version', '\d+')::int > 13 AS server_version_above_thi \else \q \endif +create schema pg14; +set search_path to pg14; +create table dist(a int, b int); +select create_distributed_table('dist','a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +create index idx on dist(a); +set citus.log_remote_commands to on; +-- make sure that we send the tablespace option +SET citus.multi_shard_commit_protocol TO '1pc'; +SET citus.multi_shard_modify_mode TO 'sequential'; +reindex(TABLESPACE test_tablespace) index idx; +NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx'); +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx'); +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing REINDEX (TABLESPACE test_tablespace) INDEX pg14.xxxxx +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing REINDEX (TABLESPACE test_tablespace) INDEX pg14.xxxxx +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing REINDEX (TABLESPACE test_tablespace) INDEX pg14.xxxxx +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing REINDEX (TABLESPACE test_tablespace) INDEX pg14.xxxxx +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing REINDEX (TABLESPACE test_tablespace) INDEX pg14.xxxxx +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing REINDEX (TABLESPACE test_tablespace) INDEX pg14.xxxxx +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing REINDEX (TABLESPACE test_tablespace) INDEX pg14.xxxxx +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing REINDEX (TABLESPACE test_tablespace) INDEX pg14.xxxxx +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing COMMIT +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing COMMIT +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +reindex(TABLESPACE test_tablespace, verbose) index idx; +INFO: index "idx" was reindexed +NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx'); +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx'); +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing REINDEX (VERBOSE, TABLESPACE test_tablespace) INDEX pg14.xxxxx +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing REINDEX (VERBOSE, TABLESPACE test_tablespace) INDEX pg14.xxxxx +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing REINDEX (VERBOSE, TABLESPACE test_tablespace) INDEX pg14.xxxxx +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing REINDEX (VERBOSE, TABLESPACE test_tablespace) INDEX pg14.xxxxx +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing REINDEX (VERBOSE, TABLESPACE test_tablespace) INDEX pg14.xxxxx +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing REINDEX (VERBOSE, TABLESPACE test_tablespace) INDEX pg14.xxxxx +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing REINDEX (VERBOSE, TABLESPACE test_tablespace) INDEX pg14.xxxxx +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing REINDEX (VERBOSE, TABLESPACE test_tablespace) INDEX pg14.xxxxx +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing COMMIT +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing COMMIT +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +reindex(TABLESPACE test_tablespace, verbose false) index idx ; +NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx'); +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx'); +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing REINDEX (TABLESPACE test_tablespace) INDEX pg14.xxxxx +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing REINDEX (TABLESPACE test_tablespace) INDEX pg14.xxxxx +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing REINDEX (TABLESPACE test_tablespace) INDEX pg14.xxxxx +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing REINDEX (TABLESPACE test_tablespace) INDEX pg14.xxxxx +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing REINDEX (TABLESPACE test_tablespace) INDEX pg14.xxxxx +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing REINDEX (TABLESPACE test_tablespace) INDEX pg14.xxxxx +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing REINDEX (TABLESPACE test_tablespace) INDEX pg14.xxxxx +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing REINDEX (TABLESPACE test_tablespace) INDEX pg14.xxxxx +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing COMMIT +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing COMMIT +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +reindex(verbose, TABLESPACE test_tablespace) index idx ; +INFO: index "idx" was reindexed +NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx'); +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx'); +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing REINDEX (VERBOSE, TABLESPACE test_tablespace) INDEX pg14.xxxxx +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing REINDEX (VERBOSE, TABLESPACE test_tablespace) INDEX pg14.xxxxx +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing REINDEX (VERBOSE, TABLESPACE test_tablespace) INDEX pg14.xxxxx +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing REINDEX (VERBOSE, TABLESPACE test_tablespace) INDEX pg14.xxxxx +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing REINDEX (VERBOSE, TABLESPACE test_tablespace) INDEX pg14.xxxxx +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing REINDEX (VERBOSE, TABLESPACE test_tablespace) INDEX pg14.xxxxx +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing REINDEX (VERBOSE, TABLESPACE test_tablespace) INDEX pg14.xxxxx +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing REINDEX (VERBOSE, TABLESPACE test_tablespace) INDEX pg14.xxxxx +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing COMMIT +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing COMMIT +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +-- should error saying table space doesn't exist +reindex(TABLESPACE test_tablespace1) index idx; +ERROR: tablespace "test_tablespace1" does not exist +set citus.log_remote_commands to off; +set client_min_messages to error; +drop schema pg14 cascade; diff --git a/src/test/regress/expected/tablespace.out b/src/test/regress/expected/tablespace.out new file mode 100644 index 000000000..2aba6ffdd --- /dev/null +++ b/src/test/regress/expected/tablespace.out @@ -0,0 +1,5 @@ +CREATE TABLESPACE test_tablespace LOCATION '/home/talha/citus/src/test/regress/data/ts0'; +\c - - - :worker_1_port +CREATE TABLESPACE test_tablespace LOCATION '/home/talha/citus/src/test/regress/data/ts1'; +\c - - - :worker_2_port +CREATE TABLESPACE test_tablespace LOCATION '/home/talha/citus/src/test/regress/data/ts2'; diff --git a/src/test/regress/input/tablespace.source b/src/test/regress/input/tablespace.source new file mode 100644 index 000000000..653c0dba0 --- /dev/null +++ b/src/test/regress/input/tablespace.source @@ -0,0 +1,5 @@ +CREATE TABLESPACE test_tablespace LOCATION '@abs_srcdir@/tmp_check/ts0'; +\c - - - :worker_1_port +CREATE TABLESPACE test_tablespace LOCATION '@abs_srcdir@/tmp_check/ts1'; +\c - - - :worker_2_port +CREATE TABLESPACE test_tablespace LOCATION '@abs_srcdir@/tmp_check/ts2'; diff --git a/src/test/regress/minimal_schedule b/src/test/regress/minimal_schedule index 982059e75..c5d15ff52 100644 --- a/src/test/regress/minimal_schedule +++ b/src/test/regress/minimal_schedule @@ -1,3 +1,4 @@ test: multi_cluster_management test: multi_test_helpers multi_test_helpers_superuser columnar_test_helpers test: multi_test_catalog_views +test: tablespace diff --git a/src/test/regress/multi_schedule b/src/test/regress/multi_schedule index 6b31d2673..405089a70 100644 --- a/src/test/regress/multi_schedule +++ b/src/test/regress/multi_schedule @@ -15,6 +15,7 @@ test: insert_select_repartition window_functions dml_recursive multi_insert_sele test: multi_insert_select_conflict citus_table_triggers test: multi_row_insert insert_select_into_local_table multi_create_table_new_features test: multi_agg_approximate_distinct +test: tablespace # following should not run in parallel because it relies on connection counts to workers test: insert_select_connection_leak diff --git a/src/test/regress/output/tablespace.source b/src/test/regress/output/tablespace.source new file mode 100644 index 000000000..653c0dba0 --- /dev/null +++ b/src/test/regress/output/tablespace.source @@ -0,0 +1,5 @@ +CREATE TABLESPACE test_tablespace LOCATION '@abs_srcdir@/tmp_check/ts0'; +\c - - - :worker_1_port +CREATE TABLESPACE test_tablespace LOCATION '@abs_srcdir@/tmp_check/ts1'; +\c - - - :worker_2_port +CREATE TABLESPACE test_tablespace LOCATION '@abs_srcdir@/tmp_check/ts2'; diff --git a/src/test/regress/sql/pg14.sql b/src/test/regress/sql/pg14.sql index 35b3a5676..84107791a 100644 --- a/src/test/regress/sql/pg14.sql +++ b/src/test/regress/sql/pg14.sql @@ -5,3 +5,29 @@ SELECT substring(:'server_version', '\d+')::int > 13 AS server_version_above_thi \else \q \endif + +create schema pg14; +set search_path to pg14; + +create table dist(a int, b int); +select create_distributed_table('dist','a'); +create index idx on dist(a); + +set citus.log_remote_commands to on; +-- make sure that we send the tablespace option +SET citus.multi_shard_commit_protocol TO '1pc'; +SET citus.multi_shard_modify_mode TO 'sequential'; +reindex(TABLESPACE test_tablespace) index idx; +reindex(TABLESPACE test_tablespace, verbose) index idx; +reindex(TABLESPACE test_tablespace, verbose false) index idx ; +reindex(verbose, TABLESPACE test_tablespace) index idx ; +-- should error saying table space doesn't exist +reindex(TABLESPACE test_tablespace1) index idx; +set citus.log_remote_commands to off; + +set client_min_messages to error; +drop schema pg14 cascade; + + + + diff --git a/src/test/regress/sql/tablespace.sql b/src/test/regress/sql/tablespace.sql new file mode 100644 index 000000000..427039892 --- /dev/null +++ b/src/test/regress/sql/tablespace.sql @@ -0,0 +1,5 @@ +CREATE TABLESPACE test_tablespace LOCATION '/home/talha/citus/src/test/regress/data/ts0'; +\c - - - :worker_1_port +CREATE TABLESPACE test_tablespace LOCATION '/home/talha/citus/src/test/regress/data/ts1'; +\c - - - :worker_2_port +CREATE TABLESPACE test_tablespace LOCATION '/home/talha/citus/src/test/regress/data/ts2'; \ No newline at end of file From 66303785f3a08e625789011d68edf9fd579f5db2 Mon Sep 17 00:00:00 2001 From: Ahmet Gedemenli Date: Tue, 31 Aug 2021 16:44:12 +0300 Subject: [PATCH 092/104] Add option PROCESS_TOAST to VACUUM - PG14 #7cb3048 (#5219) (cherry picked from commit e63bdfc49f9203db14ef77313c1d5e3461a84a32) --- src/backend/distributed/commands/vacuum.c | 19 +++++- src/test/regress/expected/pg14.out | 82 ++++++++++++++--------- src/test/regress/sql/pg14.sql | 17 ++++- 3 files changed, 82 insertions(+), 36 deletions(-) diff --git a/src/backend/distributed/commands/vacuum.c b/src/backend/distributed/commands/vacuum.c index f6fabfe2b..37acfd6ba 100644 --- a/src/backend/distributed/commands/vacuum.c +++ b/src/backend/distributed/commands/vacuum.c @@ -388,7 +388,12 @@ DeparseVacuumStmtPrefix(CitusVacuumParams vacuumParams) { appendStringInfoString(vacuumPrefix, "SKIP_LOCKED,"); } - + #if PG_VERSION_NUM >= PG_VERSION_14 + if (vacuumFlags & VACOPT_PROCESS_TOAST) + { + appendStringInfoString(vacuumPrefix, "PROCESS_TOAST,"); + } + #endif if (vacuumParams.truncate != VACOPTVALUE_UNSPECIFIED) { appendStringInfoString(vacuumPrefix, @@ -504,6 +509,9 @@ VacuumStmtParams(VacuumStmt *vacstmt) bool freeze = false; bool full = false; bool disable_page_skipping = false; + #if PG_VERSION_NUM >= PG_VERSION_14 + bool process_toast = false; + #endif /* Set default value */ params.index_cleanup = VACOPTVALUE_UNSPECIFIED; @@ -549,6 +557,12 @@ VacuumStmtParams(VacuumStmt *vacstmt) { disable_page_skipping = defGetBoolean(opt); } + #if PG_VERSION_NUM >= PG_VERSION_14 + else if (strcmp(opt->defname, "process_toast") == 0) + { + process_toast = defGetBoolean(opt); + } + #endif else if (strcmp(opt->defname, "index_cleanup") == 0) { params.index_cleanup = defGetBoolean(opt) ? VACOPTVALUE_ENABLED : @@ -599,6 +613,9 @@ VacuumStmtParams(VacuumStmt *vacstmt) (analyze ? VACOPT_ANALYZE : 0) | (freeze ? VACOPT_FREEZE : 0) | (full ? VACOPT_FULL : 0) | + #if PG_VERSION_NUM >= PG_VERSION_14 + (process_toast ? VACOPT_PROCESS_TOAST : 0) | + #endif (disable_page_skipping ? VACOPT_DISABLE_PAGE_SKIPPING : 0); return params; } diff --git a/src/test/regress/expected/pg14.out b/src/test/regress/expected/pg14.out index 6e42f4fbc..954680ccd 100644 --- a/src/test/regress/expected/pg14.out +++ b/src/test/regress/expected/pg14.out @@ -7,6 +7,56 @@ SELECT substring(:'server_version', '\d+')::int > 13 AS server_version_above_thi \endif create schema pg14; set search_path to pg14; +SET citus.next_shard_id TO 980000; +SET citus.shard_count TO 2; +-- test the new vacuum option, process_toast +CREATE TABLE t1 (a int); +SELECT create_distributed_table('t1','a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SET citus.log_remote_commands TO ON; +VACUUM (FULL) t1; +NOTICE: issuing VACUUM (FULL) pg14.t1_980000 +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing VACUUM (FULL) pg14.t1_980000 +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing VACUUM (FULL) pg14.t1_980001 +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing VACUUM (FULL) pg14.t1_980001 +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +VACUUM (FULL, PROCESS_TOAST) t1; +NOTICE: issuing VACUUM (FULL,PROCESS_TOAST) pg14.t1_980000 +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing VACUUM (FULL,PROCESS_TOAST) pg14.t1_980000 +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing VACUUM (FULL,PROCESS_TOAST) pg14.t1_980001 +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing VACUUM (FULL,PROCESS_TOAST) pg14.t1_980001 +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +VACUUM (FULL, PROCESS_TOAST true) t1; +NOTICE: issuing VACUUM (FULL,PROCESS_TOAST) pg14.t1_980000 +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing VACUUM (FULL,PROCESS_TOAST) pg14.t1_980000 +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing VACUUM (FULL,PROCESS_TOAST) pg14.t1_980001 +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing VACUUM (FULL,PROCESS_TOAST) pg14.t1_980001 +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +VACUUM (FULL, PROCESS_TOAST false) t1; +ERROR: PROCESS_TOAST required with VACUUM FULL +VACUUM (PROCESS_TOAST false) t1; +NOTICE: issuing VACUUM pg14.t1_980000 +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing VACUUM pg14.t1_980000 +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing VACUUM pg14.t1_980001 +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +NOTICE: issuing VACUUM pg14.t1_980001 +DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx +SET citus.log_remote_commands TO OFF; create table dist(a int, b int); select create_distributed_table('dist','a'); create_distributed_table @@ -32,14 +82,6 @@ NOTICE: issuing REINDEX (TABLESPACE test_tablespace) INDEX pg14.xxxxx DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx NOTICE: issuing REINDEX (TABLESPACE test_tablespace) INDEX pg14.xxxxx DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing REINDEX (TABLESPACE test_tablespace) INDEX pg14.xxxxx -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing REINDEX (TABLESPACE test_tablespace) INDEX pg14.xxxxx -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing REINDEX (TABLESPACE test_tablespace) INDEX pg14.xxxxx -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing REINDEX (TABLESPACE test_tablespace) INDEX pg14.xxxxx -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx NOTICE: issuing COMMIT DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx NOTICE: issuing COMMIT @@ -58,14 +100,6 @@ NOTICE: issuing REINDEX (VERBOSE, TABLESPACE test_tablespace) INDEX pg14.xxxxx DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx NOTICE: issuing REINDEX (VERBOSE, TABLESPACE test_tablespace) INDEX pg14.xxxxx DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing REINDEX (VERBOSE, TABLESPACE test_tablespace) INDEX pg14.xxxxx -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing REINDEX (VERBOSE, TABLESPACE test_tablespace) INDEX pg14.xxxxx -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing REINDEX (VERBOSE, TABLESPACE test_tablespace) INDEX pg14.xxxxx -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing REINDEX (VERBOSE, TABLESPACE test_tablespace) INDEX pg14.xxxxx -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx NOTICE: issuing COMMIT DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx NOTICE: issuing COMMIT @@ -83,14 +117,6 @@ NOTICE: issuing REINDEX (TABLESPACE test_tablespace) INDEX pg14.xxxxx DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx NOTICE: issuing REINDEX (TABLESPACE test_tablespace) INDEX pg14.xxxxx DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing REINDEX (TABLESPACE test_tablespace) INDEX pg14.xxxxx -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing REINDEX (TABLESPACE test_tablespace) INDEX pg14.xxxxx -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing REINDEX (TABLESPACE test_tablespace) INDEX pg14.xxxxx -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing REINDEX (TABLESPACE test_tablespace) INDEX pg14.xxxxx -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx NOTICE: issuing COMMIT DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx NOTICE: issuing COMMIT @@ -109,14 +135,6 @@ NOTICE: issuing REINDEX (VERBOSE, TABLESPACE test_tablespace) INDEX pg14.xxxxx DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx NOTICE: issuing REINDEX (VERBOSE, TABLESPACE test_tablespace) INDEX pg14.xxxxx DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing REINDEX (VERBOSE, TABLESPACE test_tablespace) INDEX pg14.xxxxx -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing REINDEX (VERBOSE, TABLESPACE test_tablespace) INDEX pg14.xxxxx -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing REINDEX (VERBOSE, TABLESPACE test_tablespace) INDEX pg14.xxxxx -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -NOTICE: issuing REINDEX (VERBOSE, TABLESPACE test_tablespace) INDEX pg14.xxxxx -DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx NOTICE: issuing COMMIT DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx NOTICE: issuing COMMIT diff --git a/src/test/regress/sql/pg14.sql b/src/test/regress/sql/pg14.sql index 84107791a..24a2dac66 100644 --- a/src/test/regress/sql/pg14.sql +++ b/src/test/regress/sql/pg14.sql @@ -9,6 +9,20 @@ SELECT substring(:'server_version', '\d+')::int > 13 AS server_version_above_thi create schema pg14; set search_path to pg14; +SET citus.next_shard_id TO 980000; +SET citus.shard_count TO 2; + +-- test the new vacuum option, process_toast +CREATE TABLE t1 (a int); +SELECT create_distributed_table('t1','a'); +SET citus.log_remote_commands TO ON; +VACUUM (FULL) t1; +VACUUM (FULL, PROCESS_TOAST) t1; +VACUUM (FULL, PROCESS_TOAST true) t1; +VACUUM (FULL, PROCESS_TOAST false) t1; +VACUUM (PROCESS_TOAST false) t1; +SET citus.log_remote_commands TO OFF; + create table dist(a int, b int); select create_distributed_table('dist','a'); create index idx on dist(a); @@ -28,6 +42,3 @@ set citus.log_remote_commands to off; set client_min_messages to error; drop schema pg14 cascade; - - - From c431bb2e45c284aac87d2252a86d833b748018c2 Mon Sep 17 00:00:00 2001 From: Onder Kalaci Date: Tue, 31 Aug 2021 11:02:42 +0200 Subject: [PATCH 093/104] Add support for "COPY dist/ref tables FROM" progress report Simply call Postgres' function to report the progress on each row recieved. Note that we currently do not support "COPY dist/ref TO .." progress report nicely. Citus has some specialized logic to support "COPY dist/ref TO .." such that it either converts the underlying command into "COPY (SELECT * FROM dist/ref ) ..." or sends COPY command to shards directly. In the former case, "tuples_processed" is only updated when the executor returns all the tuples, so the progress is not accurate. In the latter case, Citus can actually implement the progress report. But, for the sake of consistency, we prefer to not implement at all. Added to PG 14 with https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=8a4f618e7ae3cb11b0b37d0f06f05c8ff905833f --- src/backend/distributed/commands/multi_copy.c | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/backend/distributed/commands/multi_copy.c b/src/backend/distributed/commands/multi_copy.c index ed616780c..6e32de61e 100644 --- a/src/backend/distributed/commands/multi_copy.c +++ b/src/backend/distributed/commands/multi_copy.c @@ -50,6 +50,7 @@ #include "postgres.h" #include "libpq-fe.h" #include "miscadmin.h" +#include "pgstat.h" #include /* for htons */ #include /* for htons */ @@ -560,7 +561,11 @@ CopyToExistingShards(CopyStmt *copyStatement, QueryCompletionCompat *completionT dest->receiveSlot(tupleTableSlot, dest); - processedRowCount += 1; + ++processedRowCount; + +#if PG_VERSION_NUM >= PG_VERSION_14 + pgstat_progress_update_param(PROGRESS_COPY_TUPLES_PROCESSED, processedRowCount); +#endif } EndCopyFrom(copyState); @@ -741,6 +746,10 @@ CopyToNewShards(CopyStmt *copyStatement, QueryCompletionCompat *completionTag, O } processedRowCount += 1; + +#if PG_VERSION_NUM >= PG_VERSION_14 + pgstat_progress_update_param(PROGRESS_COPY_TUPLES_PROCESSED, processedRowCount); +#endif } /* @@ -2984,6 +2993,13 @@ ProcessCopyStmt(CopyStmt *copyStatement, QueryCompletionCompat *completionTag, c { if (copyStatement->whereClause) { + /* + * Update progress reporting for tuples progressed so that the + * progress is reflected on pg_stat_progress_copy. Citus currently + * does not support COPY .. WHERE clause so TUPLES_EXCLUDED is not + * handled. When we remove this check, we should implement progress + * reporting as well. + */ ereport(ERROR, (errmsg( "Citus does not support COPY FROM with WHERE"))); } @@ -3148,6 +3164,7 @@ CitusCopyTo(CopyStmt *copyStatement, QueryCompletionCompat *completionTag) PQclear(result); tuplesSent += ForwardCopyDataFromConnection(copyOutState, connection); + break; } From 6fbdeb38a8bebdd2f2ad016941e42f9b5168adea Mon Sep 17 00:00:00 2001 From: Ahmet Gedemenli Date: Wed, 1 Sep 2021 10:25:56 +0300 Subject: [PATCH 094/104] ALTER TABLE ... DETACH PARTITION ... CONCURRENTLY - PG14 #71f4c8c (#5223) --- src/backend/distributed/commands/table.c | 18 ++++++++++++++++++ src/test/regress/expected/pg14.out | 18 ++++++++++++++++++ src/test/regress/sql/pg14.sql | 12 ++++++++++++ 3 files changed, 48 insertions(+) diff --git a/src/backend/distributed/commands/table.c b/src/backend/distributed/commands/table.c index 1442c6c4c..f15017f4e 100644 --- a/src/backend/distributed/commands/table.c +++ b/src/backend/distributed/commands/table.c @@ -2380,6 +2380,15 @@ ErrorIfUnsupportedAlterTableStmt(AlterTableStmt *alterTableStatement) break; } +#if PG_VERSION_NUM >= PG_VERSION_14 + case AT_DetachPartitionFinalize: + { + ereport(ERROR, (errmsg("ALTER TABLE .. DETACH PARTITION .. FINALIZE " + "commands are currently unsupported."))); + break; + } + +#endif case AT_DetachPartition: { /* we only allow partitioning commands if they are only subcommand */ @@ -2391,7 +2400,16 @@ ErrorIfUnsupportedAlterTableStmt(AlterTableStmt *alterTableStatement) errhint("You can issue each subcommand " "separately."))); } + #if PG_VERSION_NUM >= PG_VERSION_14 + PartitionCmd *partitionCommand = (PartitionCmd *) command->def; + if (partitionCommand->concurrent) + { + ereport(ERROR, (errmsg("ALTER TABLE .. DETACH PARTITION .. " + "CONCURRENTLY commands are currently " + "unsupported."))); + } + #endif ErrorIfCitusLocalTablePartitionCommand(command, relationId); break; diff --git a/src/test/regress/expected/pg14.out b/src/test/regress/expected/pg14.out index 954680ccd..2c67f0c13 100644 --- a/src/test/regress/expected/pg14.out +++ b/src/test/regress/expected/pg14.out @@ -143,5 +143,23 @@ DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx reindex(TABLESPACE test_tablespace1) index idx; ERROR: tablespace "test_tablespace1" does not exist set citus.log_remote_commands to off; +-- error out in case of ALTER TABLE .. DETACH PARTITION .. CONCURRENTLY/FINALIZE +-- only if it's a distributed partitioned table +CREATE TABLE par (a INT UNIQUE) PARTITION BY RANGE(a); +CREATE TABLE par_1 PARTITION OF par FOR VALUES FROM (1) TO (4); +CREATE TABLE par_2 PARTITION OF par FOR VALUES FROM (5) TO (8); +-- works as it's not distributed +ALTER TABLE par DETACH PARTITION par_1 CONCURRENTLY; +-- errors out +SELECT create_distributed_table('par','a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +ALTER TABLE par DETACH PARTITION par_2 CONCURRENTLY; +ERROR: ALTER TABLE .. DETACH PARTITION .. CONCURRENTLY commands are currently unsupported. +ALTER TABLE par DETACH PARTITION par_2 FINALIZE; +ERROR: ALTER TABLE .. DETACH PARTITION .. FINALIZE commands are currently unsupported. set client_min_messages to error; drop schema pg14 cascade; diff --git a/src/test/regress/sql/pg14.sql b/src/test/regress/sql/pg14.sql index 24a2dac66..141d931e7 100644 --- a/src/test/regress/sql/pg14.sql +++ b/src/test/regress/sql/pg14.sql @@ -39,6 +39,18 @@ reindex(verbose, TABLESPACE test_tablespace) index idx ; reindex(TABLESPACE test_tablespace1) index idx; set citus.log_remote_commands to off; +-- error out in case of ALTER TABLE .. DETACH PARTITION .. CONCURRENTLY/FINALIZE +-- only if it's a distributed partitioned table +CREATE TABLE par (a INT UNIQUE) PARTITION BY RANGE(a); +CREATE TABLE par_1 PARTITION OF par FOR VALUES FROM (1) TO (4); +CREATE TABLE par_2 PARTITION OF par FOR VALUES FROM (5) TO (8); +-- works as it's not distributed +ALTER TABLE par DETACH PARTITION par_1 CONCURRENTLY; +-- errors out +SELECT create_distributed_table('par','a'); +ALTER TABLE par DETACH PARTITION par_2 CONCURRENTLY; +ALTER TABLE par DETACH PARTITION par_2 FINALIZE; + set client_min_messages to error; drop schema pg14 cascade; From 113d5d66159e040a3f5113b8c490b5e79361277b Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Tue, 31 Aug 2021 16:19:53 +0300 Subject: [PATCH 095/104] Adds support for column compression in table distribution --- .../distributed/deparser/citus_ruleutils.c | 11 ++++ src/test/regress/expected/pg14.out | 51 +++++++++++++++++++ src/test/regress/sql/pg14.sql | 20 ++++++++ 3 files changed, 82 insertions(+) diff --git a/src/backend/distributed/deparser/citus_ruleutils.c b/src/backend/distributed/deparser/citus_ruleutils.c index 86584245b..12bdafeb2 100644 --- a/src/backend/distributed/deparser/citus_ruleutils.c +++ b/src/backend/distributed/deparser/citus_ruleutils.c @@ -22,6 +22,9 @@ #include "access/skey.h" #include "access/stratnum.h" #include "access/sysattr.h" +#if PG_VERSION_NUM >= PG_VERSION_14 +#include "access/toast_compression.h" +#endif #include "access/tupdesc.h" #include "catalog/dependency.h" #include "catalog/indexing.h" @@ -382,6 +385,14 @@ pg_get_tableschemadef_string(Oid tableRelationId, bool includeSequenceDefaults, appendStringInfoString(&buffer, " NOT NULL"); } +#if PG_VERSION_NUM >= PG_VERSION_14 + if (CompressionMethodIsValid(attributeForm->attcompression)) + { + appendStringInfo(&buffer, " COMPRESSION %s", + GetCompressionMethodName(attributeForm->attcompression)); + } +#endif + if (attributeForm->attcollation != InvalidOid && attributeForm->attcollation != DEFAULT_COLLATION_OID) { diff --git a/src/test/regress/expected/pg14.out b/src/test/regress/expected/pg14.out index 2c67f0c13..8c820cbab 100644 --- a/src/test/regress/expected/pg14.out +++ b/src/test/regress/expected/pg14.out @@ -161,5 +161,56 @@ ALTER TABLE par DETACH PARTITION par_2 CONCURRENTLY; ERROR: ALTER TABLE .. DETACH PARTITION .. CONCURRENTLY commands are currently unsupported. ALTER TABLE par DETACH PARTITION par_2 FINALIZE; ERROR: ALTER TABLE .. DETACH PARTITION .. FINALIZE commands are currently unsupported. +-- test column compression propagation in distribution +SET citus.shard_replication_factor TO 1; +CREATE TABLE col_compression (a TEXT COMPRESSION pglz, b TEXT); +SELECT create_distributed_table('col_compression', 'a', shard_count:=4); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT attname || ' ' || attcompression AS column_compression FROM pg_attribute WHERE attrelid::regclass::text LIKE 'col\_compression%' AND attnum > 0 ORDER BY 1; + column_compression +--------------------------------------------------------------------- + a p + b +(2 rows) + +SELECT result AS column_compression FROM run_command_on_workers($$SELECT ARRAY( +SELECT attname || ' ' || attcompression FROM pg_attribute WHERE attrelid::regclass::text LIKE 'pg14.col\_compression%' AND attnum > 0 ORDER BY 1 +)$$); + column_compression +--------------------------------------------------------------------- + {"a p","a p","b ","b "} + {"a p","a p","b ","b "} +(2 rows) + +-- test column compression propagation in rebalance +SELECT shardid INTO moving_shard FROM citus_shards WHERE table_name='col_compression'::regclass AND nodeport=:worker_1_port LIMIT 1; +SELECT citus_move_shard_placement((SELECT * FROM moving_shard), :'public_worker_1_host', :worker_1_port, :'public_worker_2_host', :worker_2_port); + citus_move_shard_placement +--------------------------------------------------------------------- + +(1 row) + +SELECT rebalance_table_shards('col_compression', rebalance_strategy := 'by_shard_count'); +NOTICE: Moving shard xxxxx from localhost:xxxxx to localhost:xxxxx ... + rebalance_table_shards +--------------------------------------------------------------------- + +(1 row) + +CALL citus_cleanup_orphaned_shards(); +NOTICE: cleaned up 1 orphaned shards +SELECT result AS column_compression FROM run_command_on_workers($$SELECT ARRAY( +SELECT attname || ' ' || attcompression FROM pg_attribute WHERE attrelid::regclass::text LIKE 'pg14.col\_compression%' AND attnum > 0 ORDER BY 1 +)$$) ORDER BY length(result); + column_compression +--------------------------------------------------------------------- + {"a p","a p","b ","b "} + {"a p","a p","b ","b "} +(2 rows) + set client_min_messages to error; drop schema pg14 cascade; diff --git a/src/test/regress/sql/pg14.sql b/src/test/regress/sql/pg14.sql index 141d931e7..12efe99b9 100644 --- a/src/test/regress/sql/pg14.sql +++ b/src/test/regress/sql/pg14.sql @@ -51,6 +51,26 @@ SELECT create_distributed_table('par','a'); ALTER TABLE par DETACH PARTITION par_2 CONCURRENTLY; ALTER TABLE par DETACH PARTITION par_2 FINALIZE; + +-- test column compression propagation in distribution +SET citus.shard_replication_factor TO 1; +CREATE TABLE col_compression (a TEXT COMPRESSION pglz, b TEXT); +SELECT create_distributed_table('col_compression', 'a', shard_count:=4); + +SELECT attname || ' ' || attcompression AS column_compression FROM pg_attribute WHERE attrelid::regclass::text LIKE 'col\_compression%' AND attnum > 0 ORDER BY 1; +SELECT result AS column_compression FROM run_command_on_workers($$SELECT ARRAY( +SELECT attname || ' ' || attcompression FROM pg_attribute WHERE attrelid::regclass::text LIKE 'pg14.col\_compression%' AND attnum > 0 ORDER BY 1 +)$$); + +-- test column compression propagation in rebalance +SELECT shardid INTO moving_shard FROM citus_shards WHERE table_name='col_compression'::regclass AND nodeport=:worker_1_port LIMIT 1; +SELECT citus_move_shard_placement((SELECT * FROM moving_shard), :'public_worker_1_host', :worker_1_port, :'public_worker_2_host', :worker_2_port); +SELECT rebalance_table_shards('col_compression', rebalance_strategy := 'by_shard_count'); +CALL citus_cleanup_orphaned_shards(); +SELECT result AS column_compression FROM run_command_on_workers($$SELECT ARRAY( +SELECT attname || ' ' || attcompression FROM pg_attribute WHERE attrelid::regclass::text LIKE 'pg14.col\_compression%' AND attnum > 0 ORDER BY 1 +)$$) ORDER BY length(result); + set client_min_messages to error; drop schema pg14 cascade; From 1ff7186d20cd9e11128b1e0548646ad5ed719117 Mon Sep 17 00:00:00 2001 From: Ahmet Gedemenli Date: Wed, 1 Sep 2021 12:15:13 +0300 Subject: [PATCH 096/104] Extended statistics on expressions - PG14 a4d75c8 (#5224) (cherry picked from commit 1268415f123b5d99cfacfe207c8670240efc1c00) --- .../deparser/deparse_statistics_stmts.c | 8 ++++++++ src/test/regress/expected/pg14.out | 14 ++++++++++++++ src/test/regress/sql/pg14.sql | 9 ++++++++- 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/backend/distributed/deparser/deparse_statistics_stmts.c b/src/backend/distributed/deparser/deparse_statistics_stmts.c index e6f8fc262..732341c28 100644 --- a/src/backend/distributed/deparser/deparse_statistics_stmts.c +++ b/src/backend/distributed/deparser/deparse_statistics_stmts.c @@ -241,6 +241,14 @@ AppendColumnNames(StringInfo buf, CreateStatsStmt *stmt) foreach_ptr(column, stmt->exprs) { + if (!column->name) + { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg( + "only simple column references are allowed in CREATE STATISTICS"))); + } + const char *columnName = quote_identifier(column->name); appendStringInfoString(buf, columnName); diff --git a/src/test/regress/expected/pg14.out b/src/test/regress/expected/pg14.out index 8c820cbab..2d7dd89c1 100644 --- a/src/test/regress/expected/pg14.out +++ b/src/test/regress/expected/pg14.out @@ -142,6 +142,20 @@ DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx -- should error saying table space doesn't exist reindex(TABLESPACE test_tablespace1) index idx; ERROR: tablespace "test_tablespace1" does not exist +reset citus.log_remote_commands; +-- CREATE STATISTICS only allow simple column references +CREATE TABLE tbl1(a timestamp, b int); +SELECT create_distributed_table('tbl1','a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- the last one should error out +CREATE STATISTICS s1 (dependencies) ON a, b FROM tbl1; +CREATE STATISTICS s2 (mcv) ON a, b FROM tbl1; +CREATE STATISTICS s3 (ndistinct) ON date_trunc('month', a), date_trunc('day', a) FROM tbl1; +ERROR: only simple column references are allowed in CREATE STATISTICS set citus.log_remote_commands to off; -- error out in case of ALTER TABLE .. DETACH PARTITION .. CONCURRENTLY/FINALIZE -- only if it's a distributed partitioned table diff --git a/src/test/regress/sql/pg14.sql b/src/test/regress/sql/pg14.sql index 12efe99b9..66c39482f 100644 --- a/src/test/regress/sql/pg14.sql +++ b/src/test/regress/sql/pg14.sql @@ -37,6 +37,14 @@ reindex(TABLESPACE test_tablespace, verbose false) index idx ; reindex(verbose, TABLESPACE test_tablespace) index idx ; -- should error saying table space doesn't exist reindex(TABLESPACE test_tablespace1) index idx; +reset citus.log_remote_commands; +-- CREATE STATISTICS only allow simple column references +CREATE TABLE tbl1(a timestamp, b int); +SELECT create_distributed_table('tbl1','a'); +-- the last one should error out +CREATE STATISTICS s1 (dependencies) ON a, b FROM tbl1; +CREATE STATISTICS s2 (mcv) ON a, b FROM tbl1; +CREATE STATISTICS s3 (ndistinct) ON date_trunc('month', a), date_trunc('day', a) FROM tbl1; set citus.log_remote_commands to off; -- error out in case of ALTER TABLE .. DETACH PARTITION .. CONCURRENTLY/FINALIZE @@ -73,4 +81,3 @@ SELECT attname || ' ' || attcompression FROM pg_attribute WHERE attrelid::regcla set client_min_messages to error; drop schema pg14 cascade; - From 5844ab286c6ff29c8026f8c1c505d5d9c29b5dc2 Mon Sep 17 00:00:00 2001 From: Onder Kalaci Date: Wed, 1 Sep 2021 14:35:41 +0200 Subject: [PATCH 097/104] Support OUT parameters in procedure pushdown delegation In PG 14, procedures can have OUT parameters. In Citus' procedure delegation framework, we need to adjust the function expression to get the outargs parameters. Releven PG change: https://github.com/postgres/postgres/commit/e56bce5d43789cce95d099554ae9593ada92b3b7 --- src/backend/distributed/commands/call.c | 72 ++++++++++++++++++- src/test/regress/expected/pg14.out | 94 +++++++++++++++++++++++++ src/test/regress/multi_schedule | 4 +- src/test/regress/sql/pg14.sql | 44 ++++++++++++ 4 files changed, 212 insertions(+), 2 deletions(-) diff --git a/src/backend/distributed/commands/call.c b/src/backend/distributed/commands/call.c index ff547f1cb..98add3f28 100644 --- a/src/backend/distributed/commands/call.c +++ b/src/backend/distributed/commands/call.c @@ -11,6 +11,9 @@ */ #include "postgres.h" +#include "funcapi.h" + +#include "distributed/pg_version_constants.h" #include "catalog/pg_proc.h" #include "commands/defrem.h" @@ -46,6 +49,9 @@ static bool CallFuncExprRemotely(CallStmt *callStmt, DistObjectCacheEntry *procedure, FuncExpr *funcExpr, DestReceiver *dest); +#if PG_VERSION_NUM >= PG_VERSION_14 +static bool FunctionHasOutOnlyParameter(Oid functionOid); +#endif /* * CallDistributedProcedureRemotely calls a stored procedure on the worker if possible. @@ -142,12 +148,26 @@ CallFuncExprRemotely(CallStmt *callStmt, DistObjectCacheEntry *procedure, return false; } + +#if PG_VERSION_NUM >= PG_VERSION_14 + + /* + * We might need to add outargs to the funcExpr->args so that they can + * be pushed down. We can implement in the future. + */ + if (FunctionHasOutOnlyParameter(funcExpr->funcid)) + { + ereport(DEBUG1, (errmsg("not pushing down procedures with OUT parameters"))); + return false; + } +#endif + ereport(DEBUG1, (errmsg("pushing down the procedure"))); /* build remote command with fully qualified names */ StringInfo callCommand = makeStringInfo(); - appendStringInfo(callCommand, "CALL %s", pg_get_rule_expr((Node *) funcExpr)); + appendStringInfo(callCommand, "CALL %s", pg_get_rule_expr((Node *) funcExpr)); { Tuplestorestate *tupleStore = tuplestore_begin_heap(true, false, work_mem); TupleDesc tupleDesc = CallStmtResultDesc(callStmt); @@ -207,3 +227,53 @@ CallFuncExprRemotely(CallStmt *callStmt, DistObjectCacheEntry *procedure, return true; } + + +#if PG_VERSION_NUM >= PG_VERSION_14 + +/* + * FunctionHasOutOnlyParameter is a helper function which takes + * a function Oid and returns true if the input function has at least + * one OUT parameter. + */ +static bool +FunctionHasOutOnlyParameter(Oid functionOid) +{ + Oid *argTypes = NULL; + char **argNames = NULL; + char *argModes = NULL; + + HeapTuple proctup = SearchSysCache1(PROCOID, ObjectIdGetDatum(functionOid)); + if (!HeapTupleIsValid(proctup)) + { + elog(ERROR, "cache lookup failed for function %u", functionOid); + } + + int numberOfArgs = get_func_arg_info(proctup, &argTypes, &argNames, &argModes); + + if (argModes == NULL) + { + /* short circuit, all arguments are IN */ + ReleaseSysCache(proctup); + + return false; + } + + int argIndex = 0; + for (; argIndex < numberOfArgs; ++argIndex) + { + if (argModes[argIndex] == PROARGMODE_OUT) + { + ReleaseSysCache(proctup); + + return true; + } + } + + ReleaseSysCache(proctup); + + return false; +} + + +#endif diff --git a/src/test/regress/expected/pg14.out b/src/test/regress/expected/pg14.out index 2d7dd89c1..16fb3e8f9 100644 --- a/src/test/regress/expected/pg14.out +++ b/src/test/regress/expected/pg14.out @@ -226,5 +226,99 @@ SELECT attname || ' ' || attcompression FROM pg_attribute WHERE attrelid::regcla {"a p","a p","b ","b "} (2 rows) +RESET citus.multi_shard_modify_mode; +-- test procedure OUT parameters with procedure pushdown +CREATE TABLE test_proc_table (a int); +create or replace procedure proc_pushdown(dist_key integer, OUT created int4[], OUT res_out text) +language plpgsql +as $$ +DECLARE + res INT := 0; +begin + INSERT INTO pg14.test_proc_table VALUES (dist_key); + SELECT count(*) INTO res FROM pg14.test_proc_table; + created := created || res; + PERFORM array_prepend(res, created); + res_out := res::text; + commit; +end;$$; +-- show the behaviour before distributing +CALL proc_pushdown(1, NULL, NULL); + created | res_out +--------------------------------------------------------------------- + {1} | 1 +(1 row) + +CALL proc_pushdown(1, ARRAY[2000,1], 'AAAA'); + created | res_out +--------------------------------------------------------------------- + {2} | 2 +(1 row) + +SELECT create_distributed_table('test_proc_table', 'a'); +NOTICE: Copying data from local table... +NOTICE: copying the data has completed +DETAIL: The local data in the table is no longer visible, but is still on disk. +HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$pg14.test_proc_table$$) + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_function('proc_pushdown(integer)', 'dist_key', 'test_proc_table' ); + create_distributed_function +--------------------------------------------------------------------- + +(1 row) + +-- make sure that metadata is synced, it may take few seconds +CREATE OR REPLACE FUNCTION wait_until_metadata_sync(timeout INTEGER DEFAULT 15000) + RETURNS void + LANGUAGE C STRICT + AS 'citus'; +SELECT wait_until_metadata_sync(30000); + wait_until_metadata_sync +--------------------------------------------------------------------- + +(1 row) + +SELECT bool_and(hasmetadata) FROM pg_dist_node WHERE nodeport IN (:worker_1_port, :worker_2_port); + bool_and +--------------------------------------------------------------------- + t +(1 row) + +-- still, we do not pushdown procedures with OUT parameters +SET client_min_messages TO DEBUG1; +CALL proc_pushdown(1, NULL, NULL); +DEBUG: not pushing down procedures with OUT parameters + created | res_out +--------------------------------------------------------------------- + {3} | 3 +(1 row) + +CALL proc_pushdown(1, ARRAY[2000,1], 'AAAA'); +DEBUG: not pushing down procedures with OUT parameters + created | res_out +--------------------------------------------------------------------- + {4} | 4 +(1 row) + +RESET client_min_messages; +-- we don't need metadata syncing anymore +SELECT stop_metadata_sync_to_node('localhost', :worker_1_port); +NOTICE: dropping metadata on the node (localhost,57637) + stop_metadata_sync_to_node +--------------------------------------------------------------------- + +(1 row) + +SELECT stop_metadata_sync_to_node('localhost', :worker_2_port); +NOTICE: dropping metadata on the node (localhost,57638) + stop_metadata_sync_to_node +--------------------------------------------------------------------- + +(1 row) + set client_min_messages to error; drop schema pg14 cascade; diff --git a/src/test/regress/multi_schedule b/src/test/regress/multi_schedule index 405089a70..e5f33cdbf 100644 --- a/src/test/regress/multi_schedule +++ b/src/test/regress/multi_schedule @@ -52,7 +52,9 @@ test: subquery_in_targetlist subquery_in_where subquery_complex_target_list test: subquery_prepared_statements test: non_colocated_leaf_subquery_joins non_colocated_subquery_joins non_colocated_join_order test: cte_inline recursive_view_local_table values -test: pg13 pg12 pg14 +test: pg13 pg12 +# run pg14 sequentially as it syncs metadata +test: pg14 test: tableam drop_column_partitioned_table # ---------- diff --git a/src/test/regress/sql/pg14.sql b/src/test/regress/sql/pg14.sql index 66c39482f..ce36c6922 100644 --- a/src/test/regress/sql/pg14.sql +++ b/src/test/regress/sql/pg14.sql @@ -79,5 +79,49 @@ SELECT result AS column_compression FROM run_command_on_workers($$SELECT ARRAY( SELECT attname || ' ' || attcompression FROM pg_attribute WHERE attrelid::regclass::text LIKE 'pg14.col\_compression%' AND attnum > 0 ORDER BY 1 )$$) ORDER BY length(result); +RESET citus.multi_shard_modify_mode; + +-- test procedure OUT parameters with procedure pushdown +CREATE TABLE test_proc_table (a int); + +create or replace procedure proc_pushdown(dist_key integer, OUT created int4[], OUT res_out text) +language plpgsql +as $$ +DECLARE + res INT := 0; +begin + INSERT INTO pg14.test_proc_table VALUES (dist_key); + SELECT count(*) INTO res FROM pg14.test_proc_table; + created := created || res; + PERFORM array_prepend(res, created); + res_out := res::text; + commit; +end;$$; + +-- show the behaviour before distributing +CALL proc_pushdown(1, NULL, NULL); +CALL proc_pushdown(1, ARRAY[2000,1], 'AAAA'); + +SELECT create_distributed_table('test_proc_table', 'a'); +SELECT create_distributed_function('proc_pushdown(integer)', 'dist_key', 'test_proc_table' ); + +-- make sure that metadata is synced, it may take few seconds +CREATE OR REPLACE FUNCTION wait_until_metadata_sync(timeout INTEGER DEFAULT 15000) + RETURNS void + LANGUAGE C STRICT + AS 'citus'; +SELECT wait_until_metadata_sync(30000); +SELECT bool_and(hasmetadata) FROM pg_dist_node WHERE nodeport IN (:worker_1_port, :worker_2_port); + +-- still, we do not pushdown procedures with OUT parameters +SET client_min_messages TO DEBUG1; +CALL proc_pushdown(1, NULL, NULL); +CALL proc_pushdown(1, ARRAY[2000,1], 'AAAA'); +RESET client_min_messages; + +-- we don't need metadata syncing anymore +SELECT stop_metadata_sync_to_node('localhost', :worker_1_port); +SELECT stop_metadata_sync_to_node('localhost', :worker_2_port); + set client_min_messages to error; drop schema pg14 cascade; From 82a3b20fb3d99fa7a07313d0f607692f765bdf3a Mon Sep 17 00:00:00 2001 From: Onder Kalaci Date: Thu, 2 Sep 2021 09:27:14 +0200 Subject: [PATCH 098/104] Fix flaky test --- src/test/regress/expected/alter_distributed_table.out | 8 ++++---- src/test/regress/sql/alter_distributed_table.sql | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/test/regress/expected/alter_distributed_table.out b/src/test/regress/expected/alter_distributed_table.out index de87dade4..20742aa0a 100644 --- a/src/test/regress/expected/alter_distributed_table.out +++ b/src/test/regress/expected/alter_distributed_table.out @@ -360,7 +360,7 @@ SELECT create_distributed_table('referencing_dist_table', 'a', colocate_with:='r SET client_min_messages TO WARNING; SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint - WHERE (conrelid::regclass::text = 'table_with_references' OR confrelid::regclass::text = 'table_with_references') AND contype = 'f' ORDER BY 1; + WHERE (conrelid::regclass::text = 'table_with_references' OR confrelid::regclass::text = 'table_with_references') AND contype = 'f' ORDER BY 1,2; Referencing Table | Definition --------------------------------------------------------------------- referencing_dist_table | FOREIGN KEY (a) REFERENCES table_with_references(a1) @@ -375,12 +375,12 @@ SELECT alter_distributed_table('table_with_references', shard_count := 12, casca (1 row) SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint - WHERE (conrelid::regclass::text = 'table_with_references' OR confrelid::regclass::text = 'table_with_references') AND contype = 'f' ORDER BY 1; + WHERE (conrelid::regclass::text = 'table_with_references' OR confrelid::regclass::text = 'table_with_references') AND contype = 'f' ORDER BY 1,2; Referencing Table | Definition --------------------------------------------------------------------- referencing_dist_table | FOREIGN KEY (a) REFERENCES table_with_references(a1) - table_with_references | FOREIGN KEY (a2) REFERENCES referenced_ref_table(a) table_with_references | FOREIGN KEY (a1) REFERENCES referenced_dist_table(a) + table_with_references | FOREIGN KEY (a2) REFERENCES referenced_ref_table(a) (3 rows) SELECT alter_distributed_table('table_with_references', shard_count := 10, cascade_to_colocated := false); @@ -392,7 +392,7 @@ WARNING: foreign key referencing_dist_table_a_fkey will be dropped (1 row) SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint - WHERE (conrelid::regclass::text = 'table_with_references' OR confrelid::regclass::text = 'table_with_references') AND contype = 'f' ORDER BY 1; + WHERE (conrelid::regclass::text = 'table_with_references' OR confrelid::regclass::text = 'table_with_references') AND contype = 'f' ORDER BY 1,2; Referencing Table | Definition --------------------------------------------------------------------- table_with_references | FOREIGN KEY (a2) REFERENCES referenced_ref_table(a) diff --git a/src/test/regress/sql/alter_distributed_table.sql b/src/test/regress/sql/alter_distributed_table.sql index 5df68c4ed..c6698ee7a 100644 --- a/src/test/regress/sql/alter_distributed_table.sql +++ b/src/test/regress/sql/alter_distributed_table.sql @@ -102,13 +102,13 @@ SELECT create_distributed_table('referencing_dist_table', 'a', colocate_with:='r SET client_min_messages TO WARNING; SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint - WHERE (conrelid::regclass::text = 'table_with_references' OR confrelid::regclass::text = 'table_with_references') AND contype = 'f' ORDER BY 1; + WHERE (conrelid::regclass::text = 'table_with_references' OR confrelid::regclass::text = 'table_with_references') AND contype = 'f' ORDER BY 1,2; SELECT alter_distributed_table('table_with_references', shard_count := 12, cascade_to_colocated := true); SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint - WHERE (conrelid::regclass::text = 'table_with_references' OR confrelid::regclass::text = 'table_with_references') AND contype = 'f' ORDER BY 1; + WHERE (conrelid::regclass::text = 'table_with_references' OR confrelid::regclass::text = 'table_with_references') AND contype = 'f' ORDER BY 1,2; SELECT alter_distributed_table('table_with_references', shard_count := 10, cascade_to_colocated := false); SELECT conrelid::regclass::text AS "Referencing Table", pg_get_constraintdef(oid, true) AS "Definition" FROM pg_constraint - WHERE (conrelid::regclass::text = 'table_with_references' OR confrelid::regclass::text = 'table_with_references') AND contype = 'f' ORDER BY 1; + WHERE (conrelid::regclass::text = 'table_with_references' OR confrelid::regclass::text = 'table_with_references') AND contype = 'f' ORDER BY 1,2; -- check when multi shard modify mode is set to sequential From 2b263f9a2ab55f90ae4ba8477c15fbcc4ac85a12 Mon Sep 17 00:00:00 2001 From: Ahmet Gedemenli Date: Thu, 2 Sep 2021 11:58:40 +0300 Subject: [PATCH 099/104] ALTER STATISTICS .. OWNER TO CURRENT_ROLE (#5225) (cherry picked from commit 42322caf90ca094777aa01376e02d1187afc1560) --- .../distributed/deparser/citus_ruleutils.c | 6 +- src/test/regress/expected/pg14.out | 57 +++++++++++++++++++ src/test/regress/sql/pg14.sql | 20 +++++++ 3 files changed, 82 insertions(+), 1 deletion(-) diff --git a/src/backend/distributed/deparser/citus_ruleutils.c b/src/backend/distributed/deparser/citus_ruleutils.c index 12bdafeb2..91343af8f 100644 --- a/src/backend/distributed/deparser/citus_ruleutils.c +++ b/src/backend/distributed/deparser/citus_ruleutils.c @@ -1275,6 +1275,8 @@ simple_quote_literal(StringInfo buf, const char *val) * * CURRENT_USER - resolved to the user name of the current role being used * SESSION_USER - resolved to the user name of the user that opened the session + * CURRENT_ROLE - same as CURRENT_USER, resolved to the user name of the current role being used + * Postgres treats CURRENT_ROLE is equivalent to CURRENT_USER, and we follow the same approach. * * withQuoteIdentifier is used, because if the results will be used in a query the quotes are needed but if not there * should not be extra quotes. @@ -1290,7 +1292,9 @@ RoleSpecString(RoleSpec *spec, bool withQuoteIdentifier) quote_identifier(spec->rolename) : spec->rolename; } - + #if PG_VERSION_NUM >= PG_VERSION_14 + case ROLESPEC_CURRENT_ROLE: + #endif case ROLESPEC_CURRENT_USER: { return withQuoteIdentifier ? diff --git a/src/test/regress/expected/pg14.out b/src/test/regress/expected/pg14.out index 16fb3e8f9..a8b74c209 100644 --- a/src/test/regress/expected/pg14.out +++ b/src/test/regress/expected/pg14.out @@ -320,5 +320,62 @@ NOTICE: dropping metadata on the node (localhost,57638) (1 row) +-- ALTER STATISTICS .. OWNER TO CURRENT_ROLE +CREATE TABLE st1 (a int, b int); +CREATE STATISTICS role_s1 ON a, b FROM st1; +SELECT create_distributed_table('st1','a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +ALTER STATISTICS role_s1 OWNER TO CURRENT_ROLE; +SET citus.enable_ddl_propagation TO off; -- for enterprise +CREATE ROLE role_1 WITH LOGIN SUPERUSER; +NOTICE: not propagating CREATE ROLE/USER commands to worker nodes +HINT: Connect to worker nodes directly to manually create all necessary users and roles. +SET citus.enable_ddl_propagation TO on; +SELECT run_command_on_workers($$CREATE ROLE role_1 WITH LOGIN SUPERUSER;$$); + run_command_on_workers +--------------------------------------------------------------------- + (localhost,57637,t,"CREATE ROLE") + (localhost,57638,t,"CREATE ROLE") +(2 rows) + +ALTER STATISTICS role_s1 OWNER TO CURRENT_ROLE; +SELECT run_command_on_workers($$SELECT rolname FROM pg_roles WHERE oid IN (SELECT stxowner FROM pg_statistic_ext WHERE stxname LIKE 'role\_s1%');$$); + run_command_on_workers +--------------------------------------------------------------------- + (localhost,57637,t,postgres) + (localhost,57638,t,postgres) +(2 rows) + +SET ROLE role_1; +ALTER STATISTICS role_s1 OWNER TO CURRENT_ROLE; +SELECT run_command_on_workers($$SELECT rolname FROM pg_roles WHERE oid IN (SELECT stxowner FROM pg_statistic_ext WHERE stxname LIKE 'role\_s1%');$$); + run_command_on_workers +--------------------------------------------------------------------- + (localhost,57637,t,role_1) + (localhost,57638,t,role_1) +(2 rows) + +SET ROLE postgres; +ALTER STATISTICS role_s1 OWNER TO CURRENT_USER; +SELECT run_command_on_workers($$SELECT rolname FROM pg_roles WHERE oid IN (SELECT stxowner FROM pg_statistic_ext WHERE stxname LIKE 'role\_s1%');$$); + run_command_on_workers +--------------------------------------------------------------------- + (localhost,57637,t,postgres) + (localhost,57638,t,postgres) +(2 rows) + +SET ROLE to NONE; +ALTER STATISTICS role_s1 OWNER TO CURRENT_ROLE; +SELECT run_command_on_workers($$SELECT rolname FROM pg_roles WHERE oid IN (SELECT stxowner FROM pg_statistic_ext WHERE stxname LIKE 'role\_s1%');$$); + run_command_on_workers +--------------------------------------------------------------------- + (localhost,57637,t,postgres) + (localhost,57638,t,postgres) +(2 rows) + set client_min_messages to error; drop schema pg14 cascade; diff --git a/src/test/regress/sql/pg14.sql b/src/test/regress/sql/pg14.sql index ce36c6922..f93167766 100644 --- a/src/test/regress/sql/pg14.sql +++ b/src/test/regress/sql/pg14.sql @@ -123,5 +123,25 @@ RESET client_min_messages; SELECT stop_metadata_sync_to_node('localhost', :worker_1_port); SELECT stop_metadata_sync_to_node('localhost', :worker_2_port); +-- ALTER STATISTICS .. OWNER TO CURRENT_ROLE +CREATE TABLE st1 (a int, b int); +CREATE STATISTICS role_s1 ON a, b FROM st1; +SELECT create_distributed_table('st1','a'); +ALTER STATISTICS role_s1 OWNER TO CURRENT_ROLE; +SET citus.enable_ddl_propagation TO off; -- for enterprise +CREATE ROLE role_1 WITH LOGIN SUPERUSER; +SET citus.enable_ddl_propagation TO on; +SELECT run_command_on_workers($$CREATE ROLE role_1 WITH LOGIN SUPERUSER;$$); +ALTER STATISTICS role_s1 OWNER TO CURRENT_ROLE; +SELECT run_command_on_workers($$SELECT rolname FROM pg_roles WHERE oid IN (SELECT stxowner FROM pg_statistic_ext WHERE stxname LIKE 'role\_s1%');$$); +SET ROLE role_1; +ALTER STATISTICS role_s1 OWNER TO CURRENT_ROLE; +SELECT run_command_on_workers($$SELECT rolname FROM pg_roles WHERE oid IN (SELECT stxowner FROM pg_statistic_ext WHERE stxname LIKE 'role\_s1%');$$); +SET ROLE postgres; +ALTER STATISTICS role_s1 OWNER TO CURRENT_USER; +SELECT run_command_on_workers($$SELECT rolname FROM pg_roles WHERE oid IN (SELECT stxowner FROM pg_statistic_ext WHERE stxname LIKE 'role\_s1%');$$); +SET ROLE to NONE; +ALTER STATISTICS role_s1 OWNER TO CURRENT_ROLE; +SELECT run_command_on_workers($$SELECT rolname FROM pg_roles WHERE oid IN (SELECT stxowner FROM pg_statistic_ext WHERE stxname LIKE 'role\_s1%');$$); set client_min_messages to error; drop schema pg14 cascade; From 2a2ebab1fa73e6a8cd25d8d70a77f0cd4743a5cb Mon Sep 17 00:00:00 2001 From: SaitTalhaNisanci Date: Thu, 2 Sep 2021 12:40:14 +0300 Subject: [PATCH 100/104] Add tests for jsonb subscripting (#5232) PG commit: 676887a3b0b8e3c0348ac3f82ab0d16e9a24bd43 --- src/test/regress/expected/pg14.out | 108 +++++++++++++++++++++++++++++ src/test/regress/sql/pg14.sql | 50 +++++++++++++ 2 files changed, 158 insertions(+) diff --git a/src/test/regress/expected/pg14.out b/src/test/regress/expected/pg14.out index a8b74c209..0f55dd350 100644 --- a/src/test/regress/expected/pg14.out +++ b/src/test/regress/expected/pg14.out @@ -377,5 +377,113 @@ SELECT run_command_on_workers($$SELECT rolname FROM pg_roles WHERE oid IN (SELEC (localhost,57638,t,postgres) (2 rows) +create TABLE test_jsonb_subscript ( + id int, + test_json jsonb +); +SELECT create_distributed_table('test_jsonb_subscript', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +insert into test_jsonb_subscript values +(1, '{}'), -- empty jsonb +(2, '{"key": "value"}'); -- jsonb with data +-- update empty jsonb +update test_jsonb_subscript set test_json['a'] = '1' where id = 1; +select * from test_jsonb_subscript ORDER BY 1,2; + id | test_json +--------------------------------------------------------------------- + 1 | {"a": 1} + 2 | {"key": "value"} +(2 rows) + +-- update jsonb with some data +update test_jsonb_subscript set test_json['a'] = '1' where id = 2; +select * from test_jsonb_subscript ORDER BY 1,2; + id | test_json +--------------------------------------------------------------------- + 1 | {"a": 1} + 2 | {"a": 1, "key": "value"} +(2 rows) + +-- replace jsonb +update test_jsonb_subscript set test_json['a'] = '"test"'; +select * from test_jsonb_subscript ORDER BY 1,2; + id | test_json +--------------------------------------------------------------------- + 1 | {"a": "test"} + 2 | {"a": "test", "key": "value"} +(2 rows) + +-- replace by object +update test_jsonb_subscript set test_json['a'] = '{"b": 1}'::jsonb; +select * from test_jsonb_subscript ORDER BY 1,2; + id | test_json +--------------------------------------------------------------------- + 1 | {"a": {"b": 1}} + 2 | {"a": {"b": 1}, "key": "value"} +(2 rows) + +-- replace by array +update test_jsonb_subscript set test_json['a'] = '[1, 2, 3]'::jsonb; +select * from test_jsonb_subscript ORDER BY 1,2; + id | test_json +--------------------------------------------------------------------- + 1 | {"a": [1, 2, 3]} + 2 | {"a": [1, 2, 3], "key": "value"} +(2 rows) + +-- use jsonb subscription in where clause +select * from test_jsonb_subscript where test_json['key'] = '"value"' ORDER BY 1,2; + id | test_json +--------------------------------------------------------------------- + 2 | {"a": [1, 2, 3], "key": "value"} +(1 row) + +select * from test_jsonb_subscript where test_json['key_doesnt_exists'] = '"value"' ORDER BY 1,2; + id | test_json +--------------------------------------------------------------------- +(0 rows) + +select * from test_jsonb_subscript where test_json['key'] = '"wrong_value"' ORDER BY 1,2; + id | test_json +--------------------------------------------------------------------- +(0 rows) + +-- NULL +update test_jsonb_subscript set test_json[NULL] = '1'; +ERROR: jsonb subscript in assignment must not be null +CONTEXT: while executing command on localhost:xxxxx +update test_jsonb_subscript set test_json['another_key'] = NULL; +select * from test_jsonb_subscript ORDER BY 1,2; + id | test_json +--------------------------------------------------------------------- + 1 | {"a": [1, 2, 3], "another_key": null} + 2 | {"a": [1, 2, 3], "key": "value", "another_key": null} +(2 rows) + +-- NULL as jsonb source +insert into test_jsonb_subscript values (3, NULL); +update test_jsonb_subscript set test_json['a'] = '1' where id = 3; +select * from test_jsonb_subscript ORDER BY 1,2; + id | test_json +--------------------------------------------------------------------- + 1 | {"a": [1, 2, 3], "another_key": null} + 2 | {"a": [1, 2, 3], "key": "value", "another_key": null} + 3 | {"a": 1} +(3 rows) + +update test_jsonb_subscript set test_json = NULL where id = 3; +update test_jsonb_subscript set test_json[0] = '1'; +select * from test_jsonb_subscript ORDER BY 1,2; + id | test_json +--------------------------------------------------------------------- + 1 | {"0": 1, "a": [1, 2, 3], "another_key": null} + 2 | {"0": 1, "a": [1, 2, 3], "key": "value", "another_key": null} + 3 | [1] +(3 rows) + set client_min_messages to error; drop schema pg14 cascade; diff --git a/src/test/regress/sql/pg14.sql b/src/test/regress/sql/pg14.sql index f93167766..d7f31292a 100644 --- a/src/test/regress/sql/pg14.sql +++ b/src/test/regress/sql/pg14.sql @@ -143,5 +143,55 @@ SELECT run_command_on_workers($$SELECT rolname FROM pg_roles WHERE oid IN (SELEC SET ROLE to NONE; ALTER STATISTICS role_s1 OWNER TO CURRENT_ROLE; SELECT run_command_on_workers($$SELECT rolname FROM pg_roles WHERE oid IN (SELECT stxowner FROM pg_statistic_ext WHERE stxname LIKE 'role\_s1%');$$); +create TABLE test_jsonb_subscript ( + id int, + test_json jsonb +); + +SELECT create_distributed_table('test_jsonb_subscript', 'id'); + +insert into test_jsonb_subscript values +(1, '{}'), -- empty jsonb +(2, '{"key": "value"}'); -- jsonb with data + +-- update empty jsonb +update test_jsonb_subscript set test_json['a'] = '1' where id = 1; +select * from test_jsonb_subscript ORDER BY 1,2; + +-- update jsonb with some data +update test_jsonb_subscript set test_json['a'] = '1' where id = 2; +select * from test_jsonb_subscript ORDER BY 1,2; + +-- replace jsonb +update test_jsonb_subscript set test_json['a'] = '"test"'; +select * from test_jsonb_subscript ORDER BY 1,2; + +-- replace by object +update test_jsonb_subscript set test_json['a'] = '{"b": 1}'::jsonb; +select * from test_jsonb_subscript ORDER BY 1,2; + +-- replace by array +update test_jsonb_subscript set test_json['a'] = '[1, 2, 3]'::jsonb; +select * from test_jsonb_subscript ORDER BY 1,2; + +-- use jsonb subscription in where clause +select * from test_jsonb_subscript where test_json['key'] = '"value"' ORDER BY 1,2; +select * from test_jsonb_subscript where test_json['key_doesnt_exists'] = '"value"' ORDER BY 1,2; +select * from test_jsonb_subscript where test_json['key'] = '"wrong_value"' ORDER BY 1,2; + +-- NULL +update test_jsonb_subscript set test_json[NULL] = '1'; +update test_jsonb_subscript set test_json['another_key'] = NULL; +select * from test_jsonb_subscript ORDER BY 1,2; + +-- NULL as jsonb source +insert into test_jsonb_subscript values (3, NULL); +update test_jsonb_subscript set test_json['a'] = '1' where id = 3; +select * from test_jsonb_subscript ORDER BY 1,2; + +update test_jsonb_subscript set test_json = NULL where id = 3; +update test_jsonb_subscript set test_json[0] = '1'; +select * from test_jsonb_subscript ORDER BY 1,2; + set client_min_messages to error; drop schema pg14 cascade; From 902af39a04f009b0644be07cd5dc406e7cb51848 Mon Sep 17 00:00:00 2001 From: SaitTalhaNisanci Date: Thu, 2 Sep 2021 16:58:35 +0300 Subject: [PATCH 101/104] Add join alias tests (#5233) PG COMMIT: 055fee7eb4dcc78e58672aef146334275e1cc40d --- src/test/regress/expected/pg14.out | 103 +++++++++++++++++++++++++++++ src/test/regress/sql/pg14.sql | 44 ++++++++++++ 2 files changed, 147 insertions(+) diff --git a/src/test/regress/expected/pg14.out b/src/test/regress/expected/pg14.out index 0f55dd350..ea4cd716d 100644 --- a/src/test/regress/expected/pg14.out +++ b/src/test/regress/expected/pg14.out @@ -485,5 +485,108 @@ select * from test_jsonb_subscript ORDER BY 1,2; 3 | [1] (3 rows) +-- JOIN ALIAS +CREATE TABLE J1_TBL ( + i integer, + j integer, + t text +); +CREATE TABLE J2_TBL ( + i integer, + k integer +); +INSERT INTO J1_TBL VALUES (1, 4, 'one'); +INSERT INTO J1_TBL VALUES (2, 3, 'two'); +INSERT INTO J1_TBL VALUES (3, 2, 'three'); +INSERT INTO J1_TBL VALUES (4, 1, 'four'); +INSERT INTO J1_TBL VALUES (5, 0, 'five'); +INSERT INTO J1_TBL VALUES (6, 6, 'six'); +INSERT INTO J1_TBL VALUES (7, 7, 'seven'); +INSERT INTO J1_TBL VALUES (8, 8, 'eight'); +INSERT INTO J1_TBL VALUES (0, NULL, 'zero'); +INSERT INTO J2_TBL VALUES (1, -1); +INSERT INTO J2_TBL VALUES (2, 2); +INSERT INTO J2_TBL VALUES (3, -3); +INSERT INTO J2_TBL VALUES (2, 4); +INSERT INTO J2_TBL VALUES (5, -5); +INSERT INTO J2_TBL VALUES (5, -5); +INSERT INTO J2_TBL VALUES (0, NULL); +SELECT create_distributed_table('J1_TBL','i'); +NOTICE: Copying data from local table... +NOTICE: copying the data has completed +DETAIL: The local data in the table is no longer visible, but is still on disk. +HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$pg14.j1_tbl$$) + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT create_distributed_table('J2_TBL','i'); +NOTICE: Copying data from local table... +NOTICE: copying the data has completed +DETAIL: The local data in the table is no longer visible, but is still on disk. +HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$pg14.j2_tbl$$) + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- test join using aliases +SELECT * FROM J1_TBL JOIN J2_TBL USING (i) WHERE J1_TBL.t = 'one' ORDER BY 1,2,3,4; -- ok + i | j | t | k +--------------------------------------------------------------------- + 1 | 4 | one | -1 +(1 row) + +SELECT * FROM J1_TBL JOIN J2_TBL USING (i) AS x WHERE J1_TBL.t = 'one' ORDER BY 1,2,3,4; -- ok + i | j | t | k +--------------------------------------------------------------------- + 1 | 4 | one | -1 +(1 row) + +SELECT * FROM (J1_TBL JOIN J2_TBL USING (i)) AS x WHERE J1_TBL.t = 'one' ORDER BY 1,2,3,4; -- error +ERROR: invalid reference to FROM-clause entry for table "j1_tbl" +HINT: There is an entry for table "j1_tbl", but it cannot be referenced from this part of the query. +SELECT * FROM J1_TBL JOIN J2_TBL USING (i) AS x WHERE x.i = 1 ORDER BY 1,2,3,4; -- ok + i | j | t | k +--------------------------------------------------------------------- + 1 | 4 | one | -1 +(1 row) + +SELECT * FROM J1_TBL JOIN J2_TBL USING (i) AS x WHERE x.t = 'one' ORDER BY 1,2,3,4; -- error +ERROR: column x.t does not exist +SELECT * FROM (J1_TBL JOIN J2_TBL USING (i) AS x) AS xx WHERE x.i = 1 ORDER BY 1,2,3,4; -- error (XXX could use better hint) +ERROR: missing FROM-clause entry for table "x" +SELECT * FROM J1_TBL a1 JOIN J2_TBL a2 USING (i) AS a1 ORDER BY 1,2,3,4; -- error +ERROR: table name "a1" specified more than once +SELECT x.* FROM J1_TBL JOIN J2_TBL USING (i) AS x WHERE J1_TBL.t = 'one' ORDER BY 1; + i +--------------------------------------------------------------------- + 1 +(1 row) + +SELECT ROW(x.*) FROM J1_TBL JOIN J2_TBL USING (i) AS x WHERE J1_TBL.t = 'one' ORDER BY 1; + row +--------------------------------------------------------------------- + (1) +(1 row) + +SELECT * FROM J1_TBL JOIN J2_TBL USING (i) AS x WHERE x.i > 1 ORDER BY 1,2,3,4; + i | j | t | k +--------------------------------------------------------------------- + 2 | 3 | two | 2 + 2 | 3 | two | 4 + 3 | 2 | three | -3 + 5 | 0 | five | -5 + 5 | 0 | five | -5 +(5 rows) + +-- ORDER BY is not supported for json and this returns 1 row, so it is okay. +SELECT row_to_json(x.*) FROM J1_TBL JOIN J2_TBL USING (i) AS x WHERE J1_TBL.t = 'one'; + row_to_json +--------------------------------------------------------------------- + {"f1":1} +(1 row) + set client_min_messages to error; drop schema pg14 cascade; diff --git a/src/test/regress/sql/pg14.sql b/src/test/regress/sql/pg14.sql index d7f31292a..97ff71e43 100644 --- a/src/test/regress/sql/pg14.sql +++ b/src/test/regress/sql/pg14.sql @@ -193,5 +193,49 @@ update test_jsonb_subscript set test_json = NULL where id = 3; update test_jsonb_subscript set test_json[0] = '1'; select * from test_jsonb_subscript ORDER BY 1,2; +-- JOIN ALIAS +CREATE TABLE J1_TBL ( + i integer, + j integer, + t text +); +CREATE TABLE J2_TBL ( + i integer, + k integer +); +INSERT INTO J1_TBL VALUES (1, 4, 'one'); +INSERT INTO J1_TBL VALUES (2, 3, 'two'); +INSERT INTO J1_TBL VALUES (3, 2, 'three'); +INSERT INTO J1_TBL VALUES (4, 1, 'four'); +INSERT INTO J1_TBL VALUES (5, 0, 'five'); +INSERT INTO J1_TBL VALUES (6, 6, 'six'); +INSERT INTO J1_TBL VALUES (7, 7, 'seven'); +INSERT INTO J1_TBL VALUES (8, 8, 'eight'); +INSERT INTO J1_TBL VALUES (0, NULL, 'zero'); +INSERT INTO J2_TBL VALUES (1, -1); +INSERT INTO J2_TBL VALUES (2, 2); +INSERT INTO J2_TBL VALUES (3, -3); +INSERT INTO J2_TBL VALUES (2, 4); +INSERT INTO J2_TBL VALUES (5, -5); +INSERT INTO J2_TBL VALUES (5, -5); +INSERT INTO J2_TBL VALUES (0, NULL); + +SELECT create_distributed_table('J1_TBL','i'); +SELECT create_distributed_table('J2_TBL','i'); + +-- test join using aliases +SELECT * FROM J1_TBL JOIN J2_TBL USING (i) WHERE J1_TBL.t = 'one' ORDER BY 1,2,3,4; -- ok +SELECT * FROM J1_TBL JOIN J2_TBL USING (i) AS x WHERE J1_TBL.t = 'one' ORDER BY 1,2,3,4; -- ok +SELECT * FROM (J1_TBL JOIN J2_TBL USING (i)) AS x WHERE J1_TBL.t = 'one' ORDER BY 1,2,3,4; -- error +SELECT * FROM J1_TBL JOIN J2_TBL USING (i) AS x WHERE x.i = 1 ORDER BY 1,2,3,4; -- ok +SELECT * FROM J1_TBL JOIN J2_TBL USING (i) AS x WHERE x.t = 'one' ORDER BY 1,2,3,4; -- error +SELECT * FROM (J1_TBL JOIN J2_TBL USING (i) AS x) AS xx WHERE x.i = 1 ORDER BY 1,2,3,4; -- error (XXX could use better hint) +SELECT * FROM J1_TBL a1 JOIN J2_TBL a2 USING (i) AS a1 ORDER BY 1,2,3,4; -- error +SELECT x.* FROM J1_TBL JOIN J2_TBL USING (i) AS x WHERE J1_TBL.t = 'one' ORDER BY 1; +SELECT ROW(x.*) FROM J1_TBL JOIN J2_TBL USING (i) AS x WHERE J1_TBL.t = 'one' ORDER BY 1; +SELECT * FROM J1_TBL JOIN J2_TBL USING (i) AS x WHERE x.i > 1 ORDER BY 1,2,3,4; +-- ORDER BY is not supported for json and this returns 1 row, so it is okay. +SELECT row_to_json(x.*) FROM J1_TBL JOIN J2_TBL USING (i) AS x WHERE J1_TBL.t = 'one'; + set client_min_messages to error; drop schema pg14 cascade; From e1f5520e1ad0e79a570ae5e1ad117a611d093385 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Wed, 1 Sep 2021 14:51:18 +0300 Subject: [PATCH 102/104] Adds propagation of ALTER TABLE .. ALTER COLUMN .. SET COMPRESSION .. --- src/backend/distributed/commands/table.c | 4 +++ src/test/regress/expected/pg14.out | 43 +++++++++++++++++++++++- src/test/regress/sql/pg14.sql | 25 +++++++++++++- 3 files changed, 70 insertions(+), 2 deletions(-) diff --git a/src/backend/distributed/commands/table.c b/src/backend/distributed/commands/table.c index f15017f4e..d2a880b41 100644 --- a/src/backend/distributed/commands/table.c +++ b/src/backend/distributed/commands/table.c @@ -2445,12 +2445,16 @@ ErrorIfUnsupportedAlterTableStmt(AlterTableStmt *alterTableStatement) case AT_ReplicaIdentity: case AT_ValidateConstraint: case AT_DropConstraint: /* we do the check for invalidation in AlterTableDropsForeignKey */ +#if PG_VERSION_NUM >= PG_VERSION_14 + case AT_SetCompression: +#endif { /* * We will not perform any special check for: * ALTER TABLE .. ALTER COLUMN .. SET NOT NULL * ALTER TABLE .. REPLICA IDENTITY .. * ALTER TABLE .. VALIDATE CONSTRAINT .. + * ALTER TABLE .. ALTER COLUMN .. SET COMPRESSION .. */ break; } diff --git a/src/test/regress/expected/pg14.out b/src/test/regress/expected/pg14.out index ea4cd716d..a59179991 100644 --- a/src/test/regress/expected/pg14.out +++ b/src/test/regress/expected/pg14.out @@ -219,13 +219,54 @@ CALL citus_cleanup_orphaned_shards(); NOTICE: cleaned up 1 orphaned shards SELECT result AS column_compression FROM run_command_on_workers($$SELECT ARRAY( SELECT attname || ' ' || attcompression FROM pg_attribute WHERE attrelid::regclass::text LIKE 'pg14.col\_compression%' AND attnum > 0 ORDER BY 1 -)$$) ORDER BY length(result); +)$$); column_compression --------------------------------------------------------------------- {"a p","a p","b ","b "} {"a p","a p","b ","b "} (2 rows) +-- test propagation of ALTER TABLE .. ALTER COLUMN .. SET COMPRESSION .. +ALTER TABLE col_compression ALTER COLUMN b SET COMPRESSION pglz; +ALTER TABLE col_compression ALTER COLUMN a SET COMPRESSION default; +SELECT result AS column_compression FROM run_command_on_workers($$SELECT ARRAY( +SELECT attname || ' ' || attcompression FROM pg_attribute WHERE attrelid::regclass::text LIKE 'pg14.col\_compression%' AND attnum > 0 ORDER BY 1 +)$$); + column_compression +--------------------------------------------------------------------- + {"a ","a ","b p","b p"} + {"a ","a ","b p","b p"} +(2 rows) + +-- test propagation of ALTER TABLE .. ADD COLUMN .. COMPRESSION .. +ALTER TABLE col_compression ADD COLUMN c TEXT COMPRESSION pglz; +SELECT result AS column_compression FROM run_command_on_workers($$SELECT ARRAY( +SELECT attname || ' ' || attcompression FROM pg_attribute WHERE attrelid::regclass::text LIKE 'pg14.col\_compression%' AND attnum > 0 ORDER BY 1 +)$$); + column_compression +--------------------------------------------------------------------- + {"a ","a ","b p","b p","c p","c p"} + {"a ","a ","b p","b p","c p","c p"} +(2 rows) + +-- test attaching to a partitioned table with column compression +CREATE TABLE col_comp_par (a TEXT COMPRESSION pglz, b TEXT) PARTITION BY RANGE (a); +SELECT create_distributed_table('col_comp_par', 'a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +CREATE TABLE col_comp_par_1 PARTITION OF col_comp_par FOR VALUES FROM ('abc') TO ('def'); +SELECT result AS column_compression FROM run_command_on_workers($$SELECT ARRAY( +SELECT attname || ' ' || attcompression FROM pg_attribute WHERE attrelid::regclass::text LIKE 'pg14.col\_comp\_par\_1\_%' AND attnum > 0 ORDER BY 1 +)$$); + column_compression +--------------------------------------------------------------------- + {"a p","b "} + {"a p","b "} +(2 rows) + RESET citus.multi_shard_modify_mode; -- test procedure OUT parameters with procedure pushdown CREATE TABLE test_proc_table (a int); diff --git a/src/test/regress/sql/pg14.sql b/src/test/regress/sql/pg14.sql index 97ff71e43..967314206 100644 --- a/src/test/regress/sql/pg14.sql +++ b/src/test/regress/sql/pg14.sql @@ -77,7 +77,30 @@ SELECT rebalance_table_shards('col_compression', rebalance_strategy := 'by_shard CALL citus_cleanup_orphaned_shards(); SELECT result AS column_compression FROM run_command_on_workers($$SELECT ARRAY( SELECT attname || ' ' || attcompression FROM pg_attribute WHERE attrelid::regclass::text LIKE 'pg14.col\_compression%' AND attnum > 0 ORDER BY 1 -)$$) ORDER BY length(result); +)$$); + +-- test propagation of ALTER TABLE .. ALTER COLUMN .. SET COMPRESSION .. +ALTER TABLE col_compression ALTER COLUMN b SET COMPRESSION pglz; +ALTER TABLE col_compression ALTER COLUMN a SET COMPRESSION default; +SELECT result AS column_compression FROM run_command_on_workers($$SELECT ARRAY( +SELECT attname || ' ' || attcompression FROM pg_attribute WHERE attrelid::regclass::text LIKE 'pg14.col\_compression%' AND attnum > 0 ORDER BY 1 +)$$); + +-- test propagation of ALTER TABLE .. ADD COLUMN .. COMPRESSION .. +ALTER TABLE col_compression ADD COLUMN c TEXT COMPRESSION pglz; +SELECT result AS column_compression FROM run_command_on_workers($$SELECT ARRAY( +SELECT attname || ' ' || attcompression FROM pg_attribute WHERE attrelid::regclass::text LIKE 'pg14.col\_compression%' AND attnum > 0 ORDER BY 1 +)$$); + +-- test attaching to a partitioned table with column compression +CREATE TABLE col_comp_par (a TEXT COMPRESSION pglz, b TEXT) PARTITION BY RANGE (a); +SELECT create_distributed_table('col_comp_par', 'a'); + +CREATE TABLE col_comp_par_1 PARTITION OF col_comp_par FOR VALUES FROM ('abc') TO ('def'); + +SELECT result AS column_compression FROM run_command_on_workers($$SELECT ARRAY( +SELECT attname || ' ' || attcompression FROM pg_attribute WHERE attrelid::regclass::text LIKE 'pg14.col\_comp\_par\_1\_%' AND attnum > 0 ORDER BY 1 +)$$); RESET citus.multi_shard_modify_mode; From 0b67fcf81d63c16af746c3f92bc7d384bd8b2263 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Wed, 25 Aug 2021 17:17:16 +0300 Subject: [PATCH 103/104] Fix style --- .circleci/config.yml | 2 +- src/backend/columnar/columnar_tableam.c | 1 - .../distributed/commands/dependencies.c | 4 ++- src/backend/distributed/commands/function.c | 6 ++-- .../distributed/commands/local_multi_copy.c | 6 ++-- src/backend/distributed/commands/multi_copy.c | 28 +++++++++---------- src/backend/distributed/commands/table.c | 20 ++++++------- .../distributed/deparser/citus_ruleutils.c | 18 ++++++++---- .../deparser/deparse_statistics_stmts.c | 5 ++++ .../distributed/deparser/ruleutils_14.c | 6 ++-- .../distributed/executor/multi_executor.c | 4 +-- src/backend/distributed/metadata/distobject.c | 8 ++++-- .../distributed/operations/delete_protocol.c | 1 - .../distributed/planner/multi_explain.c | 3 +- .../planner/multi_logical_optimizer.c | 3 +- .../relation_restriction_equivalence.c | 3 +- .../udfs/citus_finish_pg_upgrade/10.2-1.sql | 2 +- .../udfs/citus_finish_pg_upgrade/latest.sql | 2 +- .../udfs/citus_prepare_pg_upgrade/10.2-1.sql | 6 ++-- .../udfs/citus_prepare_pg_upgrade/latest.sql | 6 ++-- .../distributed/utils/function_utils.c | 2 +- src/test/regress/Makefile | 2 +- src/test/regress/bin/normalize.sed | 1 + src/test/regress/expected/.gitignore | 1 + src/test/regress/expected/multi_extension.out | 2 +- src/test/regress/expected/tablespace.out | 5 ---- .../expected/upgrade_list_citus_objects.out | 2 +- .../expected/upgrade_list_citus_objects_0.out | 2 +- src/test/regress/sql/.gitignore | 1 + src/test/regress/sql/data_types.sql | 2 +- src/test/regress/sql/partition_wise_join.sql | 2 +- src/test/regress/sql/pg13.sql | 2 +- .../sql/propagate_extension_commands.sql | 6 ++-- src/test/regress/sql/tablespace.sql | 5 ---- 34 files changed, 90 insertions(+), 79 deletions(-) delete mode 100644 src/test/regress/expected/tablespace.out delete mode 100644 src/test/regress/sql/tablespace.sql diff --git a/.circleci/config.yml b/.circleci/config.yml index 726942e25..dc344729b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -451,7 +451,7 @@ workflows: - build: name: build-14 pg_major: 14 - image_tag: '14beta3-dev202108191715' + image_tag: '14beta3-dev202108191715' - check-style - check-sql-snapshots diff --git a/src/backend/columnar/columnar_tableam.c b/src/backend/columnar/columnar_tableam.c index e50a855fe..a57274b39 100644 --- a/src/backend/columnar/columnar_tableam.c +++ b/src/backend/columnar/columnar_tableam.c @@ -2019,7 +2019,6 @@ ColumnarProcessUtility(PlannedStmt *pstmt, DestReceiver *dest, QueryCompletionCompat *completionTag) { - #if PG_VERSION_NUM >= PG_VERSION_14 if (readOnlyTree) { diff --git a/src/backend/distributed/commands/dependencies.c b/src/backend/distributed/commands/dependencies.c index 50e459512..258f5ec51 100644 --- a/src/backend/distributed/commands/dependencies.c +++ b/src/backend/distributed/commands/dependencies.c @@ -251,7 +251,9 @@ GetDependencyCreateDDLCommands(const ObjectAddress *dependency) */ Assert(false); ereport(ERROR, (errmsg("unsupported object %s for distribution by citus", - getObjectTypeDescription_compat(dependency, /* missingOk: */ false)), + getObjectTypeDescription_compat(dependency, + + /* missingOk: */ false)), errdetail( "citus tries to recreate an unsupported object on its workers"), errhint("please report a bug as this should not be happening"))); diff --git a/src/backend/distributed/commands/function.c b/src/backend/distributed/commands/function.c index e6a646735..c6e9a3519 100644 --- a/src/backend/distributed/commands/function.c +++ b/src/backend/distributed/commands/function.c @@ -1613,7 +1613,7 @@ PreprocessAlterFunctionDependsStmt(Node *node, const char *queryString, * workers */ - const char *functionName = + const char *functionName = getObjectIdentity_compat(&address, /* missingOk: */ false); ereport(ERROR, (errmsg("distrtibuted functions are not allowed to depend on an " "extension"), @@ -1933,9 +1933,9 @@ ErrorIfFunctionDependsOnExtension(const ObjectAddress *functionAddress) if (IsObjectAddressOwnedByExtension(functionAddress, &extensionAddress)) { - char *functionName = + char *functionName = getObjectIdentity_compat(functionAddress, /* missingOk: */ false); - char *extensionName = + char *extensionName = getObjectIdentity_compat(&extensionAddress, /* missingOk: */ false); ereport(ERROR, (errmsg("unable to create a distributed function from functions " "owned by an extension"), diff --git a/src/backend/distributed/commands/local_multi_copy.c b/src/backend/distributed/commands/local_multi_copy.c index 12b53e90a..fbfce7119 100644 --- a/src/backend/distributed/commands/local_multi_copy.c +++ b/src/backend/distributed/commands/local_multi_copy.c @@ -212,9 +212,9 @@ DoLocalCopy(StringInfo buffer, Oid relationId, int64 shardId, CopyStmt *copyStat (void) addRangeTableEntryForRelation(pState, shard, AccessShareLock, NULL, false, false); CopyFromState cstate = BeginCopyFrom_compat(pState, shard, NULL, NULL, false, - ReadFromLocalBufferCallback, - copyStatement->attlist, - copyStatement->options); + ReadFromLocalBufferCallback, + copyStatement->attlist, + copyStatement->options); CopyFrom(cstate); EndCopyFrom(cstate); diff --git a/src/backend/distributed/commands/multi_copy.c b/src/backend/distributed/commands/multi_copy.c index 6e32de61e..993934d3d 100644 --- a/src/backend/distributed/commands/multi_copy.c +++ b/src/backend/distributed/commands/multi_copy.c @@ -525,13 +525,13 @@ CopyToExistingShards(CopyStmt *copyStatement, QueryCompletionCompat *completionT /* initialize copy state to read from COPY data source */ CopyFromState copyState = BeginCopyFrom_compat(NULL, - copiedDistributedRelation, - NULL, - copyStatement->filename, - copyStatement->is_program, - NULL, - copyStatement->attlist, - copyStatement->options); + copiedDistributedRelation, + NULL, + copyStatement->filename, + copyStatement->is_program, + NULL, + copyStatement->attlist, + copyStatement->options); /* set up callback to identify error line number */ errorCallback.callback = CopyFromErrorCallback; @@ -627,13 +627,13 @@ CopyToNewShards(CopyStmt *copyStatement, QueryCompletionCompat *completionTag, O /* initialize copy state to read from COPY data source */ CopyFromState copyState = BeginCopyFrom_compat(NULL, - distributedRelation, - NULL, - copyStatement->filename, - copyStatement->is_program, - NULL, - copyStatement->attlist, - copyStatement->options); + distributedRelation, + NULL, + copyStatement->filename, + copyStatement->is_program, + NULL, + copyStatement->attlist, + copyStatement->options); CopyOutState copyOutState = (CopyOutState) palloc0(sizeof(CopyOutStateData)); copyOutState->delim = (char *) delimiterCharacter; diff --git a/src/backend/distributed/commands/table.c b/src/backend/distributed/commands/table.c index d2a880b41..81fe8935e 100644 --- a/src/backend/distributed/commands/table.c +++ b/src/backend/distributed/commands/table.c @@ -2448,16 +2448,16 @@ ErrorIfUnsupportedAlterTableStmt(AlterTableStmt *alterTableStatement) #if PG_VERSION_NUM >= PG_VERSION_14 case AT_SetCompression: #endif - { - /* - * We will not perform any special check for: - * ALTER TABLE .. ALTER COLUMN .. SET NOT NULL - * ALTER TABLE .. REPLICA IDENTITY .. - * ALTER TABLE .. VALIDATE CONSTRAINT .. - * ALTER TABLE .. ALTER COLUMN .. SET COMPRESSION .. - */ - break; - } + { + /* + * We will not perform any special check for: + * ALTER TABLE .. ALTER COLUMN .. SET NOT NULL + * ALTER TABLE .. REPLICA IDENTITY .. + * ALTER TABLE .. VALIDATE CONSTRAINT .. + * ALTER TABLE .. ALTER COLUMN .. SET COMPRESSION .. + */ + break; + } case AT_SetRelOptions: /* SET (...) */ case AT_ResetRelOptions: /* RESET (...) */ diff --git a/src/backend/distributed/deparser/citus_ruleutils.c b/src/backend/distributed/deparser/citus_ruleutils.c index 91343af8f..6e185eda1 100644 --- a/src/backend/distributed/deparser/citus_ruleutils.c +++ b/src/backend/distributed/deparser/citus_ruleutils.c @@ -810,15 +810,21 @@ deparse_shard_reindex_statement(ReindexStmt *origStmt, Oid distrelid, int64 shar } } + /* * IsReindexWithParam_compat returns true if the given parameter * exists for the given reindexStmt. */ -bool IsReindexWithParam_compat(ReindexStmt* reindexStmt, char* param) { +bool +IsReindexWithParam_compat(ReindexStmt *reindexStmt, char *param) +{ #if PG_VERSION_NUM < PG_VERSION_14 - if (strcmp(param, "concurrently") == 0) { + if (strcmp(param, "concurrently") == 0) + { return reindexStmt->concurrent; - }else if (strcmp(param, "verbose") == 0) { + } + else if (strcmp(param, "verbose") == 0) + { return reindexStmt->options & REINDEXOPT_VERBOSE; } return false; @@ -831,10 +837,9 @@ bool IsReindexWithParam_compat(ReindexStmt* reindexStmt, char* param) { return defGetBoolean(opt); } } - return false; + return false; #endif - -} +} /* @@ -1292,6 +1297,7 @@ RoleSpecString(RoleSpec *spec, bool withQuoteIdentifier) quote_identifier(spec->rolename) : spec->rolename; } + #if PG_VERSION_NUM >= PG_VERSION_14 case ROLESPEC_CURRENT_ROLE: #endif diff --git a/src/backend/distributed/deparser/deparse_statistics_stmts.c b/src/backend/distributed/deparser/deparse_statistics_stmts.c index 732341c28..558c242df 100644 --- a/src/backend/distributed/deparser/deparse_statistics_stmts.c +++ b/src/backend/distributed/deparser/deparse_statistics_stmts.c @@ -233,6 +233,7 @@ AppendStatTypes(StringInfo buf, CreateStatsStmt *stmt) appendStringInfoString(buf, ")"); } + #if PG_VERSION_NUM >= PG_VERSION_14 static void AppendColumnNames(StringInfo buf, CreateStatsStmt *stmt) @@ -259,6 +260,8 @@ AppendColumnNames(StringInfo buf, CreateStatsStmt *stmt) } } } + + #else static void AppendColumnNames(StringInfo buf, CreateStatsStmt *stmt) @@ -285,6 +288,8 @@ AppendColumnNames(StringInfo buf, CreateStatsStmt *stmt) } } } + + #endif static void diff --git a/src/backend/distributed/deparser/ruleutils_14.c b/src/backend/distributed/deparser/ruleutils_14.c index 7d8213155..aa1fffe94 100644 --- a/src/backend/distributed/deparser/ruleutils_14.c +++ b/src/backend/distributed/deparser/ruleutils_14.c @@ -2060,7 +2060,7 @@ get_with_clause(Query *query, deparse_context *context) if (PRETTY_INDENT(context)) appendContextKeyword(context, "", 0, 0, 0); appendStringInfoChar(buf, ')'); - + if (cte->search_clause) { bool first = true; @@ -2100,7 +2100,7 @@ get_with_clause(Query *query, deparse_context *context) } appendStringInfo(buf, " SET %s", quote_identifier(cte->cycle_clause->cycle_mark_column)); - + { Const *cmv = castNode(Const, cte->cycle_clause->cycle_mark_value); Const *cmd = castNode(Const, cte->cycle_clause->cycle_mark_default); @@ -7771,7 +7771,7 @@ get_from_clause_item(Node *jtnode, Query *query, deparse_context *context) appendStringInfoString(buf, quote_identifier(colname)); } appendStringInfoChar(buf, ')'); - + if (j->join_using_alias) appendStringInfo(buf, " AS %s", quote_identifier(j->join_using_alias->aliasname)); diff --git a/src/backend/distributed/executor/multi_executor.c b/src/backend/distributed/executor/multi_executor.c index cdbafacc0..f41553cb6 100644 --- a/src/backend/distributed/executor/multi_executor.c +++ b/src/backend/distributed/executor/multi_executor.c @@ -411,8 +411,8 @@ ReadFileIntoTupleStore(char *fileName, char *copyFormat, TupleDesc tupleDescript copyOptions = lappend(copyOptions, copyOption); CopyFromState copyState = BeginCopyFrom_compat(NULL, stubRelation, NULL, - fileName, false, NULL, - NULL, copyOptions); + fileName, false, NULL, + NULL, copyOptions); while (true) { diff --git a/src/backend/distributed/metadata/distobject.c b/src/backend/distributed/metadata/distobject.c index b96db6ed0..6eaf00f15 100644 --- a/src/backend/distributed/metadata/distobject.c +++ b/src/backend/distributed/metadata/distobject.c @@ -75,8 +75,12 @@ citus_unmark_object_distributed(PG_FUNCTION_ARGS) { ereport(ERROR, (errmsg("object still exists"), errdetail("the %s \"%s\" still exists", - getObjectTypeDescription_compat(&address, /* missingOk: */ false), - getObjectIdentity_compat(&address, /* missingOk: */ false)), + getObjectTypeDescription_compat(&address, + + /* missingOk: */ false), + getObjectIdentity_compat(&address, + + /* missingOk: */ false)), errhint("drop the object via a DROP command"))); } diff --git a/src/backend/distributed/operations/delete_protocol.c b/src/backend/distributed/operations/delete_protocol.c index b5254b545..abac684a9 100644 --- a/src/backend/distributed/operations/delete_protocol.c +++ b/src/backend/distributed/operations/delete_protocol.c @@ -688,7 +688,6 @@ ShardsMatchingDeleteCriteria(Oid relationId, List *shardIntervalList, Assert(deleteCriteria != NULL); List *deleteCriteriaList = list_make1(deleteCriteria); - /* walk over shard list and check if shards can be dropped */ ShardInterval *shardInterval = NULL; diff --git a/src/backend/distributed/planner/multi_explain.c b/src/backend/distributed/planner/multi_explain.c index a3bd6b0aa..7fdb6d8e7 100644 --- a/src/backend/distributed/planner/multi_explain.c +++ b/src/backend/distributed/planner/multi_explain.c @@ -286,9 +286,10 @@ ExplainSubPlans(DistributedPlan *distributedPlan, ExplainState *es) PlannedStmt *plan = subPlan->plan; IntoClause *into = NULL; ParamListInfo params = NULL; + /* * With PG14, we need to provide a string here, - * for now we put an empty string, which is valid according to postgres. + * for now we put an empty string, which is valid according to postgres. */ char *queryString = pstrdup(""); instr_time planduration; diff --git a/src/backend/distributed/planner/multi_logical_optimizer.c b/src/backend/distributed/planner/multi_logical_optimizer.c index 5d464f94a..87834fbab 100644 --- a/src/backend/distributed/planner/multi_logical_optimizer.c +++ b/src/backend/distributed/planner/multi_logical_optimizer.c @@ -1890,9 +1890,10 @@ MasterAggregateExpression(Aggref *originalAggregate, if (aggregateType == AGGREGATE_ARRAY_AGG) { #if PG_VERSION_NUM >= PG_VERSION_14 + /* * Postgres expects the type of the array here such as INT4ARRAYOID. - * Hence we set it to workerReturnType. If we set this to + * Hence we set it to workerReturnType. If we set this to * ANYCOMPATIBLEARRAYOID then we will get the following error: * "argument declared anycompatiblearray is not an array but type anycompatiblearray" */ diff --git a/src/backend/distributed/planner/relation_restriction_equivalence.c b/src/backend/distributed/planner/relation_restriction_equivalence.c index 1d13102fb..eb4394256 100644 --- a/src/backend/distributed/planner/relation_restriction_equivalence.c +++ b/src/backend/distributed/planner/relation_restriction_equivalence.c @@ -2054,7 +2054,8 @@ GetRestrictInfoListForRelation(RangeTblEntry *rangeTblEntry, * If the restriction involves multiple tables, we cannot add it to * input relation's expression list. */ - Relids varnos = pull_varnos_compat(relationRestriction->plannerInfo, (Node *) restrictionClause); + Relids varnos = pull_varnos_compat(relationRestriction->plannerInfo, + (Node *) restrictionClause); if (bms_num_members(varnos) != 1) { continue; diff --git a/src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/10.2-1.sql b/src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/10.2-1.sql index ae3b0b900..b285dc93a 100644 --- a/src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/10.2-1.sql +++ b/src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/10.2-1.sql @@ -31,7 +31,7 @@ BEGIN * Citus extension, so we create that dependency here. * We are not using: * ALTER EXENSION citus DROP/CREATE AGGREGATE array_cat_agg - * because we don't have an easy way to check if the aggregate + * because we don't have an easy way to check if the aggregate * exists with anyarray type or anycompatiblearray type. */ INSERT INTO pg_depend diff --git a/src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/latest.sql b/src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/latest.sql index ae3b0b900..b285dc93a 100644 --- a/src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/latest.sql +++ b/src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/latest.sql @@ -31,7 +31,7 @@ BEGIN * Citus extension, so we create that dependency here. * We are not using: * ALTER EXENSION citus DROP/CREATE AGGREGATE array_cat_agg - * because we don't have an easy way to check if the aggregate + * because we don't have an easy way to check if the aggregate * exists with anyarray type or anycompatiblearray type. */ INSERT INTO pg_depend diff --git a/src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/10.2-1.sql b/src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/10.2-1.sql index 8b5e13bc0..48a54289f 100644 --- a/src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/10.2-1.sql +++ b/src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/10.2-1.sql @@ -5,8 +5,8 @@ CREATE OR REPLACE FUNCTION pg_catalog.citus_prepare_pg_upgrade() AS $cppu$ BEGIN - DELETE FROM pg_depend WHERE - objid IN (SELECT oid FROM pg_proc WHERE proname = 'array_cat_agg') AND + DELETE FROM pg_depend WHERE + objid IN (SELECT oid FROM pg_proc WHERE proname = 'array_cat_agg') AND refobjid IN (select oid from pg_extension where extname = 'citus'); /* * We are dropping the aggregates because postgres 14 changed @@ -14,7 +14,7 @@ BEGIN * upgrading to pg14, spegifically when running pg_restore on * array_cat_agg we would get an error. So we drop the aggregate * and create the right one on citus_finish_pg_upgrade. - */ + */ DROP AGGREGATE IF EXISTS array_cat_agg(anyarray); DROP AGGREGATE IF EXISTS array_cat_agg(anycompatiblearray); -- diff --git a/src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/latest.sql b/src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/latest.sql index 8b5e13bc0..48a54289f 100644 --- a/src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/latest.sql +++ b/src/backend/distributed/sql/udfs/citus_prepare_pg_upgrade/latest.sql @@ -5,8 +5,8 @@ CREATE OR REPLACE FUNCTION pg_catalog.citus_prepare_pg_upgrade() AS $cppu$ BEGIN - DELETE FROM pg_depend WHERE - objid IN (SELECT oid FROM pg_proc WHERE proname = 'array_cat_agg') AND + DELETE FROM pg_depend WHERE + objid IN (SELECT oid FROM pg_proc WHERE proname = 'array_cat_agg') AND refobjid IN (select oid from pg_extension where extname = 'citus'); /* * We are dropping the aggregates because postgres 14 changed @@ -14,7 +14,7 @@ BEGIN * upgrading to pg14, spegifically when running pg_restore on * array_cat_agg we would get an error. So we drop the aggregate * and create the right one on citus_finish_pg_upgrade. - */ + */ DROP AGGREGATE IF EXISTS array_cat_agg(anyarray); DROP AGGREGATE IF EXISTS array_cat_agg(anycompatiblearray); -- diff --git a/src/backend/distributed/utils/function_utils.c b/src/backend/distributed/utils/function_utils.c index f0818d0c8..d410246ea 100644 --- a/src/backend/distributed/utils/function_utils.c +++ b/src/backend/distributed/utils/function_utils.c @@ -50,7 +50,7 @@ FunctionOidExtended(const char *schemaName, const char *functionName, int argume argumentCount, argumentList, findVariadics, - findDefaults, + findDefaults, false, true); diff --git a/src/test/regress/Makefile b/src/test/regress/Makefile index 670ca4d79..82b0b5fb5 100644 --- a/src/test/regress/Makefile +++ b/src/test/regress/Makefile @@ -243,4 +243,4 @@ clean distclean maintainer-clean: rm -f $(output_files) $(input_files) rm -rf tmp_check/ -all: create-tablespaces \ No newline at end of file +all: create-tablespaces diff --git a/src/test/regress/bin/normalize.sed b/src/test/regress/bin/normalize.sed index 468d81810..56607077e 100644 --- a/src/test/regress/bin/normalize.sed +++ b/src/test/regress/bin/normalize.sed @@ -239,6 +239,7 @@ s/ERROR: ROLLBACK is not allowed in an SQL function/ERROR: ROLLBACK is not all /.*Async Capable.*/d /Parent Relationship/d /Parent-Relationship/d +s/function array_cat_agg\(anyarray\) anyarray/function array_cat_agg\(anycompatiblearray\) anycompatiblearray/g s/function array_cat_agg\(anycompatiblearray\)/function array_cat_agg\(anyarray\)/g s/TRIM\(BOTH FROM value\)/btrim\(value\)/g s/pg14\.idx.*/pg14\.xxxxx/g diff --git a/src/test/regress/expected/.gitignore b/src/test/regress/expected/.gitignore index 086ad0ec6..0197c3b7e 100644 --- a/src/test/regress/expected/.gitignore +++ b/src/test/regress/expected/.gitignore @@ -20,4 +20,5 @@ /multi_mx_copy_data.out /multi_outer_join.out /multi_outer_join_reference.out +/tablespace.out /worker_copy.out diff --git a/src/test/regress/expected/multi_extension.out b/src/test/regress/expected/multi_extension.out index b115ecfc7..3188afd84 100644 --- a/src/test/regress/expected/multi_extension.out +++ b/src/test/regress/expected/multi_extension.out @@ -139,7 +139,7 @@ SELECT * FROM multi_extension.print_extension_changes(); | function alter_role_if_exists(text,text) boolean | function any_value(anyelement) anyelement | function any_value_agg(anyelement,anyelement) anyelement - | function array_cat_agg(anycompatiblearray) anycompatiblearray + | function array_cat_agg(anyarray) anycompatiblearray | function assign_distributed_transaction_id(integer,bigint,timestamp with time zone) void | function authinfo_valid(text) boolean | function broadcast_intermediate_result(text,text) bigint diff --git a/src/test/regress/expected/tablespace.out b/src/test/regress/expected/tablespace.out deleted file mode 100644 index 2aba6ffdd..000000000 --- a/src/test/regress/expected/tablespace.out +++ /dev/null @@ -1,5 +0,0 @@ -CREATE TABLESPACE test_tablespace LOCATION '/home/talha/citus/src/test/regress/data/ts0'; -\c - - - :worker_1_port -CREATE TABLESPACE test_tablespace LOCATION '/home/talha/citus/src/test/regress/data/ts1'; -\c - - - :worker_2_port -CREATE TABLESPACE test_tablespace LOCATION '/home/talha/citus/src/test/regress/data/ts2'; diff --git a/src/test/regress/expected/upgrade_list_citus_objects.out b/src/test/regress/expected/upgrade_list_citus_objects.out index 5983d6fbc..d1212ce27 100644 --- a/src/test/regress/expected/upgrade_list_citus_objects.out +++ b/src/test/regress/expected/upgrade_list_citus_objects.out @@ -26,7 +26,7 @@ ORDER BY 1; function alter_table_set_access_method(regclass,text) function any_value(anyelement) function any_value_agg(anyelement,anyelement) - function array_cat_agg(anycompatiblearray) + function array_cat_agg(anyarray) function assign_distributed_transaction_id(integer,bigint,timestamp with time zone) function authinfo_valid(text) function broadcast_intermediate_result(text,text) diff --git a/src/test/regress/expected/upgrade_list_citus_objects_0.out b/src/test/regress/expected/upgrade_list_citus_objects_0.out index fc2db7cd7..045b538f2 100644 --- a/src/test/regress/expected/upgrade_list_citus_objects_0.out +++ b/src/test/regress/expected/upgrade_list_citus_objects_0.out @@ -23,7 +23,7 @@ ORDER BY 1; function alter_table_set_access_method(regclass,text) function any_value(anyelement) function any_value_agg(anyelement,anyelement) - function array_cat_agg(anycompatiblearray) + function array_cat_agg(anyarray) function assign_distributed_transaction_id(integer,bigint,timestamp with time zone) function authinfo_valid(text) function broadcast_intermediate_result(text,text) diff --git a/src/test/regress/sql/.gitignore b/src/test/regress/sql/.gitignore index e5b354bbf..5b06080ed 100644 --- a/src/test/regress/sql/.gitignore +++ b/src/test/regress/sql/.gitignore @@ -20,4 +20,5 @@ /multi_mx_copy_data.sql /multi_outer_join.sql /multi_outer_join_reference.sql +/tablespace.sql /worker_copy.sql diff --git a/src/test/regress/sql/data_types.sql b/src/test/regress/sql/data_types.sql index 5f9651567..ce286feca 100644 --- a/src/test/regress/sql/data_types.sql +++ b/src/test/regress/sql/data_types.sql @@ -101,7 +101,7 @@ FROM -- DISTINCT w/wout distribution key -- there seems to be an issue with SELECT DISTINCT ROW with PG14 -- so we add an alternative output that gives an error, this should --- be removed after the issue is fixed on PG14. +-- be removed after the issue is fixed on PG14. SELECT DISTINCT(col1, col2, col3, col4, col5, col6, col70, col7, col8, col9, col10, col11, col12, col13, col14, col15, col16, col17, col18, col19, col20, col21, col22, col23, col24, col25, col26, col27, col28, col29, col32, col33, col34, col35, col36, col37, col38) FROM data_types_table diff --git a/src/test/regress/sql/partition_wise_join.sql b/src/test/regress/sql/partition_wise_join.sql index 0e91240e3..2b0610ec3 100644 --- a/src/test/regress/sql/partition_wise_join.sql +++ b/src/test/regress/sql/partition_wise_join.sql @@ -60,4 +60,4 @@ SELECT success FROM run_command_on_workers('select pg_reload_conf()'); RESET enable_partitionwise_join; -DROP SCHEMA partition_wise_join CASCADE; \ No newline at end of file +DROP SCHEMA partition_wise_join CASCADE; diff --git a/src/test/regress/sql/pg13.sql b/src/test/regress/sql/pg13.sql index 00c94a605..4145acf77 100644 --- a/src/test/regress/sql/pg13.sql +++ b/src/test/regress/sql/pg13.sql @@ -114,7 +114,7 @@ INSERT INTO test_wal VALUES(3,33),(4,44),(5,55) RETURNING *; -- this test has different output for pg14 and here we mostly test that -- we don't get an error, hence we use explain_has_distributed_subplan. SELECT public.explain_has_distributed_subplan( -$$ +$$ EXPLAIN (ANALYZE TRUE, WAL TRUE, COSTS FALSE, SUMMARY FALSE, BUFFERS FALSE, TIMING FALSE) WITH cte_1 AS (INSERT INTO test_wal VALUES(6,66),(7,77),(8,88) RETURNING *) SELECT * FROM cte_1; diff --git a/src/test/regress/sql/propagate_extension_commands.sql b/src/test/regress/sql/propagate_extension_commands.sql index 0ba17e293..500dc00b8 100644 --- a/src/test/regress/sql/propagate_extension_commands.sql +++ b/src/test/regress/sql/propagate_extension_commands.sql @@ -98,7 +98,7 @@ CREATE EXTENSION seg; -- show that the extension is created on existing worker SELECT run_command_on_workers($$SELECT count(extnamespace) FROM pg_extension WHERE extname = 'seg'$$); -SELECT workers.result = pg_extension.extversion AS same_version +SELECT workers.result = pg_extension.extversion AS same_version FROM run_command_on_workers($$SELECT extversion FROM pg_extension WHERE extname = 'seg'$$) workers, pg_extension WHERE extname = 'seg'; -- now create the reference table @@ -145,7 +145,7 @@ SELECT 1 from master_add_node('localhost', :worker_2_port); -- show that the extension is created on both existing and new node SELECT run_command_on_workers($$SELECT count(extnamespace) FROM pg_extension WHERE extname = 'seg'$$); -SELECT workers.result = pg_extension.extversion AS same_version +SELECT workers.result = pg_extension.extversion AS same_version FROM run_command_on_workers($$SELECT extversion FROM pg_extension WHERE extname = 'seg'$$) workers, pg_extension WHERE extname = 'seg'; -- check for the unpackaged extension to be created correctly @@ -212,7 +212,7 @@ ROLLBACK; -- show that the CREATE EXTENSION command propagated even if the transaction -- block is rollbacked, that's a shortcoming of dependency creation logic -SELECT COUNT(DISTINCT workers.result) +SELECT COUNT(DISTINCT workers.result) FROM run_command_on_workers($$SELECT extversion FROM pg_extension WHERE extname = 'seg'$$) workers; -- drop the schema and all the objects diff --git a/src/test/regress/sql/tablespace.sql b/src/test/regress/sql/tablespace.sql deleted file mode 100644 index 427039892..000000000 --- a/src/test/regress/sql/tablespace.sql +++ /dev/null @@ -1,5 +0,0 @@ -CREATE TABLESPACE test_tablespace LOCATION '/home/talha/citus/src/test/regress/data/ts0'; -\c - - - :worker_1_port -CREATE TABLESPACE test_tablespace LOCATION '/home/talha/citus/src/test/regress/data/ts1'; -\c - - - :worker_2_port -CREATE TABLESPACE test_tablespace LOCATION '/home/talha/citus/src/test/regress/data/ts2'; \ No newline at end of file From 3ad3bbba84fc368824b8d33a56a5063dcfa42052 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Fri, 3 Sep 2021 12:56:02 +0300 Subject: [PATCH 104/104] Apply latest version compat without conflicts --- src/include/distributed/version_compat.h | 36 ++++++++---------------- 1 file changed, 11 insertions(+), 25 deletions(-) diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index 1bbdd4334..dda310305 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -31,10 +31,7 @@ #endif #if PG_VERSION_NUM >= PG_VERSION_14 -#define AlterTableStmtObjType(a) ((a)->objtype) -#define F_NEXTVAL_COMPAT F_NEXTVAL -#define ROLE_MONITOR_COMPAT ROLE_PG_MONITOR -#define STATUS_WAITING_COMPAT PROC_WAIT_STATUS_WAITING +#define AlterTableStmtObjType_compat(a) ((a)->objtype) #define getObjectTypeDescription_compat(a, b) getObjectTypeDescription(a, b) #define getObjectIdentity_compat(a, b) getObjectIdentity(a, b) @@ -43,18 +40,11 @@ #define FuncnameGetCandidates_compat(a, b, c, d, e, f, g) \ FuncnameGetCandidates(a, b, c, d, e, f, g) #define expand_function_arguments_compat(a, b, c, d) expand_function_arguments(a, b, c, d) -#define VacOptValue_compat VacOptValue -#define VACOPTVALUE_UNSPECIFIED_COMPAT VACOPTVALUE_UNSPECIFIED -#define VACOPTVALUE_DISABLED_COMPAT VACOPTVALUE_DISABLED -#define VACOPTVALUE_ENABLED_COMPAT VACOPTVALUE_ENABLED -#define IsReindexWithParam_compat(reindex, param) IsReindexWithParam(reindex, param) -#define CopyFromState_compat CopyFromState #define BeginCopyFrom_compat(a, b, c, d, e, f, g, h) BeginCopyFrom(a, b, c, d, e, f, g, h) #define standard_ProcessUtility_compat(a, b, c, d, e, f, g, h) \ standard_ProcessUtility(a, b, c, d, e, f, g, h) #define ProcessUtility_compat(a, b, c, d, e, f, g, h) \ ProcessUtility(a, b, c, d, e, f, g, h) -#define COPY_FRONTEND_COMPAT COPY_FRONTEND #define SetTuplestoreDestReceiverParams_compat(a, b, c, d, e, f) \ SetTuplestoreDestReceiverParams(a, b, c, d, e, f) #define pgproc_statusflags_compat(pgproc) ((pgproc)->statusFlags) @@ -64,10 +54,10 @@ #define pull_varnos_compat(a, b) pull_varnos(a, b) #define pg_get_statisticsobj_worker_compat(a, b, c) pg_get_statisticsobj_worker(a, b, c) #else -#define AlterTableStmtObjType(a) ((a)->relkind) -#define F_NEXTVAL_COMPAT F_NEXTVAL_OID -#define ROLE_MONITOR_COMPAT DEFAULT_ROLE_MONITOR -#define STATUS_WAITING_COMPAT STATUS_WAITING +#define AlterTableStmtObjType_compat(a) ((a)->relkind) +#define F_NEXTVAL F_NEXTVAL_OID +#define ROLE_PG_MONITOR DEFAULT_ROLE_MONITOR +#define PROC_WAIT_STATUS_WAITING STATUS_WAITING #define getObjectTypeDescription_compat(a, b) getObjectTypeDescription(a) #define getObjectIdentity_compat(a, b) getObjectIdentity(a) @@ -76,20 +66,16 @@ #define FuncnameGetCandidates_compat(a, b, c, d, e, f, g) \ FuncnameGetCandidates(a, b, c, d, e, g) #define expand_function_arguments_compat(a, b, c, d) expand_function_arguments(a, c, d) -#define VacOptValue_compat VacOptTernaryValue -#define VACOPTVALUE_UNSPECIFIED_COMPAT VACOPT_TERNARY_DEFAULT -#define VACOPTVALUE_DISABLED_COMPAT VACOPT_TERNARY_DISABLED -#define VACOPTVALUE_ENABLED_COMPAT VACOPT_TERNARY_ENABLED -#define IsReindexWithParam_compat(reindex, param) \ - ((strcmp(param, "concurrently") == 0) ? ((reindex)->concurrent) : \ - ((strcmp(param, "verbose") == 0) ? ((reindex)->options == REINDEXOPT_VERBOSE) : \ - false)) -#define CopyFromState_compat CopyState +#define VacOptValue VacOptTernaryValue +#define VACOPTVALUE_UNSPECIFIED VACOPT_TERNARY_DEFAULT +#define VACOPTVALUE_DISABLED VACOPT_TERNARY_DISABLED +#define VACOPTVALUE_ENABLED VACOPT_TERNARY_ENABLED +#define CopyFromState CopyState #define BeginCopyFrom_compat(a, b, c, d, e, f, g, h) BeginCopyFrom(a, b, d, e, f, g, h) #define standard_ProcessUtility_compat(a, b, c, d, e, f, g, h) \ standard_ProcessUtility(a, b, d, e, f, g, h) #define ProcessUtility_compat(a, b, c, d, e, f, g, h) ProcessUtility(a, b, d, e, f, g, h) -#define COPY_FRONTEND_COMPAT COPY_NEW_FE +#define COPY_FRONTEND COPY_NEW_FE #define SetTuplestoreDestReceiverParams_compat(a, b, c, d, e, f) \ SetTuplestoreDestReceiverParams(a, b, c, d) #define pgproc_statusflags_compat(pgproc) \