From b7e2908fc2b1b702695d66e96d9546ec82f05eab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philip=20Dub=C3=A9?= Date: Thu, 8 Aug 2019 21:19:53 +0000 Subject: [PATCH 01/12] configure: don't prevent pg12 --- configure | 2 +- configure.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configure b/configure index 5aae85490..7dfa8965e 100755 --- a/configure +++ b/configure @@ -2531,7 +2531,7 @@ if test -z "$version_num"; then as_fn_error $? "Could not detect PostgreSQL version from pg_config." "$LINENO" 5 fi -if test "$version_num" != '10' -a "$version_num" != '11'; then +if test "$version_num" != '10' -a "$version_num" != '11' -a "$version_num" != '12'; then as_fn_error $? "Citus is not compatible with the detected PostgreSQL version ${version_num}." "$LINENO" 5 else { $as_echo "$as_me:${as_lineno-$LINENO}: building against PostgreSQL $version_num" >&5 diff --git a/configure.in b/configure.in index 04675e2dd..136db4d8e 100644 --- a/configure.in +++ b/configure.in @@ -74,7 +74,7 @@ if test -z "$version_num"; then AC_MSG_ERROR([Could not detect PostgreSQL version from pg_config.]) fi -if test "$version_num" != '10' -a "$version_num" != '11'; then +if test "$version_num" != '10' -a "$version_num" != '11' -a "$version_num" != '12'; then AC_MSG_ERROR([Citus is not compatible with the detected PostgreSQL version ${version_num}.]) else AC_MSG_NOTICE([building against PostgreSQL $version_num]) From fbc3e346e80dea261bae791e103f29ebe661ccd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philip=20Dub=C3=A9?= Date: Thu, 8 Aug 2019 21:22:55 +0000 Subject: [PATCH 02/12] ruleutils_12.c Produced this file by copying ruleutils_11.c, then comparing postgres ruleutils.c changes between REL_11_STABLE & REL_12_STABLE --- .gitattributes | 1 + src/backend/distributed/utils/ruleutils_11.c | 4 +- src/backend/distributed/utils/ruleutils_12.c | 7913 ++++++++++++++++++ src/include/distributed/citus_ruleutils.h | 3 - 4 files changed, 7916 insertions(+), 5 deletions(-) create mode 100644 src/backend/distributed/utils/ruleutils_12.c diff --git a/.gitattributes b/.gitattributes index 9a404a4c2..02d92880b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -28,4 +28,5 @@ configure -whitespace src/backend/distributed/utils/citus_outfuncs.c -citus-style src/backend/distributed/utils/ruleutils_10.c -citus-style src/backend/distributed/utils/ruleutils_11.c -citus-style +src/backend/distributed/utils/ruleutils_12.c -citus-style src/include/distributed/citus_nodes.h -citus-style diff --git a/src/backend/distributed/utils/ruleutils_11.c b/src/backend/distributed/utils/ruleutils_11.c index 460440134..13a7adbed 100644 --- a/src/backend/distributed/utils/ruleutils_11.c +++ b/src/backend/distributed/utils/ruleutils_11.c @@ -17,7 +17,7 @@ #include "postgres.h" -#if (PG_VERSION_NUM >= 110000) +#if (PG_VERSION_NUM >= 110000) && (PG_VERSION_NUM < 120000) #include #include @@ -7909,4 +7909,4 @@ get_range_partbound_string(List *bound_datums) return buf->data; } -#endif /* (PG_VERSION_NUM >= 110000) */ +#endif /* (PG_VERSION_NUM >= 110000) && (PG_VERSION_NUM < 120000) */ diff --git a/src/backend/distributed/utils/ruleutils_12.c b/src/backend/distributed/utils/ruleutils_12.c new file mode 100644 index 000000000..aa95cd3fa --- /dev/null +++ b/src/backend/distributed/utils/ruleutils_12.c @@ -0,0 +1,7913 @@ +/*------------------------------------------------------------------------- + * + * ruleutils_12.c + * Functions to convert stored expressions/querytrees back to + * source text + * + * Portions Copyright (c) 1996-2018, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/backend/distributed/utils/ruleutils_12.c + * + * This needs to be closely in sync with the core code. + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#if (PG_VERSION_NUM >= 120000) && (PG_VERSION_NUM < 130000) + +#include +#include +#include + +#include "access/amapi.h" +#include "access/htup_details.h" +#include "access/relation.h" +#include "access/sysattr.h" +#include "access/table.h" +#include "catalog/dependency.h" +#include "catalog/indexing.h" +#include "catalog/pg_aggregate.h" +#include "catalog/pg_am.h" +#include "catalog/pg_authid.h" +#include "catalog/pg_collation.h" +#include "catalog/pg_constraint.h" +#include "catalog/pg_depend.h" +#include "catalog/pg_extension.h" +#include "catalog/pg_foreign_data_wrapper.h" +#include "catalog/pg_language.h" +#include "catalog/pg_opclass.h" +#include "catalog/pg_operator.h" +#include "catalog/pg_partitioned_table.h" +#include "catalog/pg_proc.h" +#include "catalog/pg_statistic_ext.h" +#include "catalog/pg_trigger.h" +#include "catalog/pg_type.h" +#include "commands/defrem.h" +#include "commands/extension.h" +#include "commands/tablespace.h" +#include "common/keywords.h" +#include "distributed/citus_nodefuncs.h" +#include "distributed/citus_ruleutils.h" +#include "executor/spi.h" +#include "foreign/foreign.h" +#include "funcapi.h" +#include "mb/pg_wchar.h" +#include "miscadmin.h" +#include "nodes/makefuncs.h" +#include "nodes/nodeFuncs.h" +#include "optimizer/optimizer.h" +#include "parser/parse_node.h" +#include "parser/parse_agg.h" +#include "parser/parse_func.h" +#include "parser/parse_node.h" +#include "parser/parse_oper.h" +#include "parser/parser.h" +#include "parser/parsetree.h" +#include "rewrite/rewriteHandler.h" +#include "rewrite/rewriteManip.h" +#include "rewrite/rewriteSupport.h" +#include "utils/array.h" +#include "utils/builtins.h" +#include "utils/fmgroids.h" +#include "utils/hsearch.h" +#include "utils/lsyscache.h" +#include "utils/rel.h" +#include "utils/ruleutils.h" +#include "utils/snapmgr.h" +#include "utils/syscache.h" +#include "utils/typcache.h" +#include "utils/varlena.h" +#include "utils/xml.h" + + +/* ---------- + * Pretty formatting constants + * ---------- + */ + +/* Indent counts */ +#define PRETTYINDENT_STD 8 +#define PRETTYINDENT_JOIN 4 +#define PRETTYINDENT_VAR 4 + +#define PRETTYINDENT_LIMIT 40 /* wrap limit */ + +/* Pretty flags */ +#define PRETTYFLAG_PAREN 0x0001 +#define PRETTYFLAG_INDENT 0x0002 + +/* Default line length for pretty-print wrapping: 0 means wrap always */ +#define WRAP_COLUMN_DEFAULT 0 + +/* macros to test if pretty action needed */ +#define PRETTY_PAREN(context) ((context)->prettyFlags & PRETTYFLAG_PAREN) +#define PRETTY_INDENT(context) ((context)->prettyFlags & PRETTYFLAG_INDENT) + + +/* ---------- + * Local data types + * ---------- + */ + +/* Context info needed for invoking a recursive querytree display routine */ +typedef struct +{ + StringInfo buf; /* output buffer to append to */ + List *namespaces; /* List of deparse_namespace nodes */ + List *windowClause; /* Current query level's WINDOW clause */ + List *windowTList; /* targetlist for resolving WINDOW clause */ + int prettyFlags; /* enabling of pretty-print functions */ + int wrapColumn; /* max line length, or -1 for no limit */ + int indentLevel; /* current indent level for prettyprint */ + bool varprefix; /* true to print prefixes on Vars */ + Oid distrelid; /* the distributed table being modified, if valid */ + int64 shardid; /* a distributed table's shardid, if positive */ + ParseExprKind special_exprkind; /* set only for exprkinds needing special + * handling */ +} deparse_context; + +/* + * Each level of query context around a subtree needs a level of Var namespace. + * A Var having varlevelsup=N refers to the N'th item (counting from 0) in + * the current context's namespaces list. + * + * The rangetable is the list of actual RTEs from the query tree, and the + * cte list is the list of actual CTEs. + * + * rtable_names holds the alias name to be used for each RTE (either a C + * string, or NULL for nameless RTEs such as unnamed joins). + * rtable_columns holds the column alias names to be used for each RTE. + * + * In some cases we need to make names of merged JOIN USING columns unique + * across the whole query, not only per-RTE. If so, unique_using is true + * and using_names is a list of C strings representing names already assigned + * to USING columns. + * + * When deparsing plan trees, there is always just a single item in the + * deparse_namespace list (since a plan tree never contains Vars with + * varlevelsup > 0). We store the PlanState node that is the immediate + * parent of the expression to be deparsed, as well as a list of that + * PlanState's ancestors. In addition, we store its outer and inner subplan + * state nodes, as well as their plan nodes' targetlists, and the index tlist + * if the current plan node might contain INDEX_VAR Vars. (These fields could + * be derived on-the-fly from the current PlanState, but it seems notationally + * clearer to set them up as separate fields.) + */ +typedef struct +{ + List *rtable; /* List of RangeTblEntry nodes */ + List *rtable_names; /* Parallel list of names for RTEs */ + List *rtable_columns; /* Parallel list of deparse_columns structs */ + List *ctes; /* List of CommonTableExpr nodes */ + /* Workspace for column alias assignment: */ + bool unique_using; /* Are we making USING names globally unique */ + List *using_names; /* List of assigned names for USING columns */ + /* Remaining fields are used only when deparsing a Plan tree: */ + PlanState *planstate; /* immediate parent of current expression */ + List *ancestors; /* ancestors of planstate */ + PlanState *outer_planstate; /* outer subplan state, or NULL if none */ + PlanState *inner_planstate; /* inner subplan state, or NULL if none */ + List *outer_tlist; /* referent for OUTER_VAR Vars */ + List *inner_tlist; /* referent for INNER_VAR Vars */ + List *index_tlist; /* referent for INDEX_VAR Vars */ +} deparse_namespace; + +/* + * Per-relation data about column alias names. + * + * Selecting aliases is unreasonably complicated because of the need to dump + * rules/views whose underlying tables may have had columns added, deleted, or + * renamed since the query was parsed. We must nonetheless print the rule/view + * in a form that can be reloaded and will produce the same results as before. + * + * For each RTE used in the query, we must assign column aliases that are + * unique within that RTE. SQL does not require this of the original query, + * but due to factors such as *-expansion we need to be able to uniquely + * reference every column in a decompiled query. As long as we qualify all + * column references, per-RTE uniqueness is sufficient for that. + * + * However, we can't ensure per-column name uniqueness for unnamed join RTEs, + * since they just inherit column names from their input RTEs, and we can't + * rename the columns at the join level. Most of the time this isn't an issue + * because we don't need to reference the join's output columns as such; we + * can reference the input columns instead. That approach can fail for merged + * JOIN USING columns, however, so when we have one of those in an unnamed + * join, we have to make that column's alias globally unique across the whole + * query to ensure it can be referenced unambiguously. + * + * Another problem is that a JOIN USING clause requires the columns to be + * merged to have the same aliases in both input RTEs, and that no other + * columns in those RTEs or their children conflict with the USING names. + * To handle that, we do USING-column alias assignment in a recursive + * traversal of the query's jointree. When descending through a JOIN with + * USING, we preassign the USING column names to the child columns, overriding + * other rules for column alias assignment. We also mark each RTE with a list + * of all USING column names selected for joins containing that RTE, so that + * when we assign other columns' aliases later, we can avoid conflicts. + * + * Another problem is that if a JOIN's input tables have had columns added or + * deleted since the query was parsed, we must generate a column alias list + * for the join that matches the current set of input columns --- otherwise, a + * change in the number of columns in the left input would throw off matching + * of aliases to columns of the right input. Thus, positions in the printable + * column alias list are not necessarily one-for-one with varattnos of the + * JOIN, so we need a separate new_colnames[] array for printing purposes. + */ +typedef struct +{ + /* + * colnames is an array containing column aliases to use for columns that + * existed when the query was parsed. Dropped columns have NULL entries. + * This array can be directly indexed by varattno to get a Var's name. + * + * Non-NULL entries are guaranteed unique within the RTE, *except* when + * this is for an unnamed JOIN RTE. In that case we merely copy up names + * from the two input RTEs. + * + * During the recursive descent in set_using_names(), forcible assignment + * of a child RTE's column name is represented by pre-setting that element + * of the child's colnames array. So at that stage, NULL entries in this + * array just mean that no name has been preassigned, not necessarily that + * the column is dropped. + */ + int num_cols; /* length of colnames[] array */ + char **colnames; /* array of C strings and NULLs */ + + /* + * new_colnames is an array containing column aliases to use for columns + * that would exist if the query was re-parsed against the current + * definitions of its base tables. This is what to print as the column + * alias list for the RTE. This array does not include dropped columns, + * but it will include columns added since original parsing. Indexes in + * it therefore have little to do with current varattno values. As above, + * entries are unique unless this is for an unnamed JOIN RTE. (In such an + * RTE, we never actually print this array, but we must compute it anyway + * for possible use in computing column names of upper joins.) The + * parallel array is_new_col marks which of these columns are new since + * original parsing. Entries with is_new_col false must match the + * non-NULL colnames entries one-for-one. + */ + int num_new_cols; /* length of new_colnames[] array */ + char **new_colnames; /* array of C strings */ + bool *is_new_col; /* array of bool flags */ + + /* This flag tells whether we should actually print a column alias list */ + bool printaliases; + + /* This list has all names used as USING names in joins above this RTE */ + List *parentUsing; /* names assigned to parent merged columns */ + + /* + * If this struct is for a JOIN RTE, we fill these fields during the + * set_using_names() pass to describe its relationship to its child RTEs. + * + * leftattnos and rightattnos are arrays with one entry per existing + * output column of the join (hence, indexable by join varattno). For a + * simple reference to a column of the left child, leftattnos[i] is the + * child RTE's attno and rightattnos[i] is zero; and conversely for a + * column of the right child. But for merged columns produced by JOIN + * USING/NATURAL JOIN, both leftattnos[i] and rightattnos[i] are nonzero. + * Also, if the column has been dropped, both are zero. + * + * If it's a JOIN USING, usingNames holds the alias names selected for the + * merged columns (these might be different from the original USING list, + * if we had to modify names to achieve uniqueness). + */ + int leftrti; /* rangetable index of left child */ + int rightrti; /* rangetable index of right child */ + int *leftattnos; /* left-child varattnos of join cols, or 0 */ + int *rightattnos; /* right-child varattnos of join cols, or 0 */ + List *usingNames; /* names assigned to merged columns */ +} deparse_columns; + +/* This macro is analogous to rt_fetch(), but for deparse_columns structs */ +#define deparse_columns_fetch(rangetable_index, dpns) \ + ((deparse_columns *) list_nth((dpns)->rtable_columns, (rangetable_index)-1)) + +/* + * Entry in set_rtable_names' hash table + */ +typedef struct +{ + char name[NAMEDATALEN]; /* Hash key --- must be first */ + int counter; /* Largest addition used so far for name */ +} NameHashEntry; + + +/* ---------- + * Local functions + * + * Most of these functions used to use fixed-size buffers to build their + * results. Now, they take an (already initialized) StringInfo object + * as a parameter, and append their text output to its contents. + * ---------- + */ +static void set_rtable_names(deparse_namespace *dpns, List *parent_namespaces, + Bitmapset *rels_used); +static void set_deparse_for_query(deparse_namespace *dpns, Query *query, + List *parent_namespaces); +static bool has_dangerous_join_using(deparse_namespace *dpns, Node *jtnode); +static void set_using_names(deparse_namespace *dpns, Node *jtnode, + List *parentUsing); +static void set_relation_column_names(deparse_namespace *dpns, + RangeTblEntry *rte, + deparse_columns *colinfo); +static void set_join_column_names(deparse_namespace *dpns, RangeTblEntry *rte, + deparse_columns *colinfo); +static bool colname_is_unique(const char *colname, deparse_namespace *dpns, + deparse_columns *colinfo); +static char *make_colname_unique(char *colname, deparse_namespace *dpns, + deparse_columns *colinfo); +static void expand_colnames_array_to(deparse_columns *colinfo, int n); +static void identify_join_columns(JoinExpr *j, RangeTblEntry *jrte, + deparse_columns *colinfo); +static void flatten_join_using_qual(Node *qual, + List **leftvars, List **rightvars); +static char *get_rtable_name(int rtindex, deparse_context *context); +static void set_deparse_planstate(deparse_namespace *dpns, PlanState *ps); +static void push_child_plan(deparse_namespace *dpns, PlanState *ps, + deparse_namespace *save_dpns); +static void pop_child_plan(deparse_namespace *dpns, + deparse_namespace *save_dpns); +static void push_ancestor_plan(deparse_namespace *dpns, ListCell *ancestor_cell, + deparse_namespace *save_dpns); +static void pop_ancestor_plan(deparse_namespace *dpns, + deparse_namespace *save_dpns); +static void get_query_def(Query *query, StringInfo buf, List *parentnamespace, + TupleDesc resultDesc, + int prettyFlags, int wrapColumn, int startIndent); +static void get_query_def_extended(Query *query, StringInfo buf, + List *parentnamespace, Oid distrelid, int64 shardid, + TupleDesc resultDesc, int prettyFlags, int wrapColumn, + int startIndent); +static void get_values_def(List *values_lists, deparse_context *context); +static void get_with_clause(Query *query, deparse_context *context); +static void get_select_query_def(Query *query, deparse_context *context, + TupleDesc resultDesc); +static void get_insert_query_def(Query *query, deparse_context *context); +static void get_update_query_def(Query *query, deparse_context *context); +static void get_update_query_targetlist_def(Query *query, List *targetList, + deparse_context *context, + RangeTblEntry *rte); +static void get_delete_query_def(Query *query, deparse_context *context); +static void get_utility_query_def(Query *query, deparse_context *context); +static void get_basic_select_query(Query *query, deparse_context *context, + TupleDesc resultDesc); +static void get_target_list(List *targetList, deparse_context *context, + TupleDesc resultDesc); +static void get_setop_query(Node *setOp, Query *query, + deparse_context *context, + TupleDesc resultDesc); +static Node *get_rule_sortgroupclause(Index ref, List *tlist, + bool force_colno, + deparse_context *context); +static void get_rule_groupingset(GroupingSet *gset, List *targetlist, + bool omit_parens, deparse_context *context); +static void get_rule_orderby(List *orderList, List *targetList, + bool force_colno, deparse_context *context); +static void get_rule_windowclause(Query *query, deparse_context *context); +static void get_rule_windowspec(WindowClause *wc, List *targetList, + deparse_context *context); +static char *get_variable(Var *var, int levelsup, bool istoplevel, + deparse_context *context); +static void get_special_variable(Node *node, deparse_context *context, + void *private); +static void resolve_special_varno(Node *node, deparse_context *context, + void *private, + void (*callback) (Node *, deparse_context *, void *)); +static Node *find_param_referent(Param *param, deparse_context *context, + deparse_namespace **dpns_p, ListCell **ancestor_cell_p); +static void get_parameter(Param *param, deparse_context *context); +static const char *get_simple_binary_op_name(OpExpr *expr); +static bool isSimpleNode(Node *node, Node *parentNode, int prettyFlags); +static void appendContextKeyword(deparse_context *context, const char *str, + int indentBefore, int indentAfter, int indentPlus); +static void removeStringInfoSpaces(StringInfo str); +static void get_rule_expr(Node *node, deparse_context *context, + bool showimplicit); +static void get_rule_expr_toplevel(Node *node, deparse_context *context, + bool showimplicit); +static void get_rule_expr_funccall(Node *node, deparse_context *context, + bool showimplicit); +static bool looks_like_function(Node *node); +static void get_oper_expr(OpExpr *expr, deparse_context *context); +static void get_func_expr(FuncExpr *expr, deparse_context *context, + bool showimplicit); +static void get_agg_expr(Aggref *aggref, deparse_context *context, + Aggref *original_aggref); +static void get_agg_combine_expr(Node *node, deparse_context *context, + void *private); +static void get_windowfunc_expr(WindowFunc *wfunc, deparse_context *context); +static void get_coercion_expr(Node *arg, deparse_context *context, + Oid resulttype, int32 resulttypmod, + Node *parentNode); +static void get_const_expr(Const *constval, deparse_context *context, + int showtype); +static void get_const_collation(Const *constval, deparse_context *context); +static void simple_quote_literal(StringInfo buf, const char *val); +static void get_sublink_expr(SubLink *sublink, deparse_context *context); +static void get_tablefunc(TableFunc *tf, deparse_context *context, + bool showimplicit); +static void get_from_clause(Query *query, const char *prefix, + deparse_context *context); +static void get_from_clause_item(Node *jtnode, Query *query, + deparse_context *context); +static void get_column_alias_list(deparse_columns *colinfo, + deparse_context *context); +static void get_from_clause_coldeflist(RangeTblFunction *rtfunc, + deparse_columns *colinfo, + deparse_context *context); +static void get_tablesample_def(TableSampleClause *tablesample, + deparse_context *context); +static void get_opclass_name(Oid opclass, Oid actual_datatype, + StringInfo buf); +static Node *processIndirection(Node *node, deparse_context *context); +static void printSubscripts(SubscriptingRef *aref, deparse_context *context); +static char *get_relation_name(Oid relid); +static char *generate_relation_or_shard_name(Oid relid, Oid distrelid, + int64 shardid, List *namespaces); +static char *generate_fragment_name(char *schemaName, char *tableName); +static char *generate_function_name(Oid funcid, int nargs, + List *argnames, Oid *argtypes, + bool has_variadic, bool *use_variadic_p, + ParseExprKind special_exprkind); +static char *generate_operator_name(Oid operid, Oid arg1, Oid arg2); + +#define only_marker(rte) ((rte)->inh ? "" : "ONLY ") + + + +/* + * pg_get_query_def parses back one query tree, and outputs the resulting query + * string into given buffer. + */ +void +pg_get_query_def(Query *query, StringInfo buffer) +{ + get_query_def(query, buffer, NIL, NULL, 0, WRAP_COLUMN_DEFAULT, 0); +} + + +/* + * set_rtable_names: select RTE aliases to be used in printing a query + * + * We fill in dpns->rtable_names with a list of names that is one-for-one with + * the already-filled dpns->rtable list. Each RTE name is unique among those + * in the new namespace plus any ancestor namespaces listed in + * parent_namespaces. + * + * If rels_used isn't NULL, only RTE indexes listed in it are given aliases. + * + * Note that this function is only concerned with relation names, not column + * names. + */ +static void +set_rtable_names(deparse_namespace *dpns, List *parent_namespaces, + Bitmapset *rels_used) +{ + HASHCTL hash_ctl; + HTAB *names_hash; + NameHashEntry *hentry; + bool found; + int rtindex; + ListCell *lc; + + dpns->rtable_names = NIL; + /* nothing more to do if empty rtable */ + if (dpns->rtable == NIL) + return; + + /* + * We use a hash table to hold known names, so that this process is O(N) + * not O(N^2) for N names. + */ + MemSet(&hash_ctl, 0, sizeof(hash_ctl)); + hash_ctl.keysize = NAMEDATALEN; + hash_ctl.entrysize = sizeof(NameHashEntry); + hash_ctl.hcxt = CurrentMemoryContext; + names_hash = hash_create("set_rtable_names names", + list_length(dpns->rtable), + &hash_ctl, + HASH_ELEM | HASH_CONTEXT); + /* Preload the hash table with names appearing in parent_namespaces */ + foreach(lc, parent_namespaces) + { + deparse_namespace *olddpns = (deparse_namespace *) lfirst(lc); + ListCell *lc2; + + foreach(lc2, olddpns->rtable_names) + { + char *oldname = (char *) lfirst(lc2); + + if (oldname == NULL) + continue; + hentry = (NameHashEntry *) hash_search(names_hash, + oldname, + HASH_ENTER, + &found); + /* we do not complain about duplicate names in parent namespaces */ + hentry->counter = 0; + } + } + + /* Now we can scan the rtable */ + rtindex = 1; + foreach(lc, dpns->rtable) + { + RangeTblEntry *rte = (RangeTblEntry *) lfirst(lc); + char *refname; + + /* Just in case this takes an unreasonable amount of time ... */ + CHECK_FOR_INTERRUPTS(); + + if (rels_used && !bms_is_member(rtindex, rels_used)) + { + /* Ignore unreferenced RTE */ + refname = NULL; + } + else if (rte->alias) + { + /* If RTE has a user-defined alias, prefer that */ + refname = rte->alias->aliasname; + } + else if (rte->rtekind == RTE_RELATION) + { + /* Use the current actual name of the relation */ + refname = get_rel_name(rte->relid); + } + else if (rte->rtekind == RTE_JOIN) + { + /* Unnamed join has no refname */ + refname = NULL; + } + else + { + /* Otherwise use whatever the parser assigned */ + refname = rte->eref->aliasname; + } + + /* + * If the selected name isn't unique, append digits to make it so, and + * make a new hash entry for it once we've got a unique name. For a + * very long input name, we might have to truncate to stay within + * NAMEDATALEN. + */ + if (refname) + { + hentry = (NameHashEntry *) hash_search(names_hash, + refname, + HASH_ENTER, + &found); + if (found) + { + /* Name already in use, must choose a new one */ + int refnamelen = strlen(refname); + char *modname = (char *) palloc(refnamelen + 16); + NameHashEntry *hentry2; + + do + { + hentry->counter++; + for (;;) + { + /* + * We avoid using %.*s here because it can misbehave + * if the data is not valid in what libc thinks is the + * prevailing encoding. + */ + memcpy(modname, refname, refnamelen); + sprintf(modname + refnamelen, "_%d", hentry->counter); + if (strlen(modname) < NAMEDATALEN) + break; + /* drop chars from refname to keep all the digits */ + refnamelen = pg_mbcliplen(refname, refnamelen, + refnamelen - 1); + } + hentry2 = (NameHashEntry *) hash_search(names_hash, + modname, + HASH_ENTER, + &found); + } while (found); + hentry2->counter = 0; /* init new hash entry */ + refname = modname; + } + else + { + /* Name not previously used, need only initialize hentry */ + hentry->counter = 0; + } + } + + dpns->rtable_names = lappend(dpns->rtable_names, refname); + rtindex++; + } + + hash_destroy(names_hash); +} + +/* + * set_deparse_for_query: set up deparse_namespace for deparsing a Query tree + * + * For convenience, this is defined to initialize the deparse_namespace struct + * from scratch. + */ +static void +set_deparse_for_query(deparse_namespace *dpns, Query *query, + List *parent_namespaces) +{ + ListCell *lc; + ListCell *lc2; + + /* Initialize *dpns and fill rtable/ctes links */ + memset(dpns, 0, sizeof(deparse_namespace)); + dpns->rtable = query->rtable; + dpns->ctes = query->cteList; + + /* Assign a unique relation alias to each RTE */ + set_rtable_names(dpns, parent_namespaces, NULL); + + /* Initialize dpns->rtable_columns to contain zeroed structs */ + dpns->rtable_columns = NIL; + while (list_length(dpns->rtable_columns) < list_length(dpns->rtable)) + dpns->rtable_columns = lappend(dpns->rtable_columns, + palloc0(sizeof(deparse_columns))); + + /* If it's a utility query, it won't have a jointree */ + if (query->jointree) + { + /* Detect whether global uniqueness of USING names is needed */ + dpns->unique_using = + has_dangerous_join_using(dpns, (Node *) query->jointree); + + /* + * Select names for columns merged by USING, via a recursive pass over + * the query jointree. + */ + set_using_names(dpns, (Node *) query->jointree, NIL); + } + + /* + * Now assign remaining column aliases for each RTE. We do this in a + * linear scan of the rtable, so as to process RTEs whether or not they + * are in the jointree (we mustn't miss NEW.*, INSERT target relations, + * etc). JOIN RTEs must be processed after their children, but this is + * okay because they appear later in the rtable list than their children + * (cf Asserts in identify_join_columns()). + */ + forboth(lc, dpns->rtable, lc2, dpns->rtable_columns) + { + RangeTblEntry *rte = (RangeTblEntry *) lfirst(lc); + deparse_columns *colinfo = (deparse_columns *) lfirst(lc2); + + if (rte->rtekind == RTE_JOIN) + set_join_column_names(dpns, rte, colinfo); + else + set_relation_column_names(dpns, rte, colinfo); + } +} + +/* + * has_dangerous_join_using: search jointree for unnamed JOIN USING + * + * Merged columns of a JOIN USING may act differently from either of the input + * columns, either because they are merged with COALESCE (in a FULL JOIN) or + * because an implicit coercion of the underlying input column is required. + * In such a case the column must be referenced as a column of the JOIN not as + * a column of either input. And this is problematic if the join is unnamed + * (alias-less): we cannot qualify the column's name with an RTE name, since + * there is none. (Forcibly assigning an alias to the join is not a solution, + * since that will prevent legal references to tables below the join.) + * To ensure that every column in the query is unambiguously referenceable, + * we must assign such merged columns names that are globally unique across + * the whole query, aliasing other columns out of the way as necessary. + * + * Because the ensuing re-aliasing is fairly damaging to the readability of + * the query, we don't do this unless we have to. So, we must pre-scan + * the join tree to see if we have to, before starting set_using_names(). + */ +static bool +has_dangerous_join_using(deparse_namespace *dpns, Node *jtnode) +{ + if (IsA(jtnode, RangeTblRef)) + { + /* nothing to do here */ + } + else if (IsA(jtnode, FromExpr)) + { + FromExpr *f = (FromExpr *) jtnode; + ListCell *lc; + + foreach(lc, f->fromlist) + { + if (has_dangerous_join_using(dpns, (Node *) lfirst(lc))) + return true; + } + } + else if (IsA(jtnode, JoinExpr)) + { + JoinExpr *j = (JoinExpr *) jtnode; + + /* Is it an unnamed JOIN with USING? */ + if (j->alias == NULL && j->usingClause) + { + /* + * Yes, so check each join alias var to see if any of them are not + * simple references to underlying columns. If so, we have a + * dangerous situation and must pick unique aliases. + */ + RangeTblEntry *jrte = rt_fetch(j->rtindex, dpns->rtable); + ListCell *lc; + + foreach(lc, jrte->joinaliasvars) + { + Var *aliasvar = (Var *) lfirst(lc); + + if (aliasvar != NULL && !IsA(aliasvar, Var)) + return true; + } + } + + /* Nope, but inspect children */ + if (has_dangerous_join_using(dpns, j->larg)) + return true; + if (has_dangerous_join_using(dpns, j->rarg)) + return true; + } + else + elog(ERROR, "unrecognized node type: %d", + (int) nodeTag(jtnode)); + return false; +} + +/* + * set_using_names: select column aliases to be used for merged USING columns + * + * We do this during a recursive descent of the query jointree. + * dpns->unique_using must already be set to determine the global strategy. + * + * Column alias info is saved in the dpns->rtable_columns list, which is + * assumed to be filled with pre-zeroed deparse_columns structs. + * + * parentUsing is a list of all USING aliases assigned in parent joins of + * the current jointree node. (The passed-in list must not be modified.) + */ +static void +set_using_names(deparse_namespace *dpns, Node *jtnode, List *parentUsing) +{ + if (IsA(jtnode, RangeTblRef)) + { + /* nothing to do now */ + } + else if (IsA(jtnode, FromExpr)) + { + FromExpr *f = (FromExpr *) jtnode; + ListCell *lc; + + foreach(lc, f->fromlist) + set_using_names(dpns, (Node *) lfirst(lc), parentUsing); + } + else if (IsA(jtnode, JoinExpr)) + { + JoinExpr *j = (JoinExpr *) jtnode; + RangeTblEntry *rte = rt_fetch(j->rtindex, dpns->rtable); + deparse_columns *colinfo = deparse_columns_fetch(j->rtindex, dpns); + int *leftattnos; + int *rightattnos; + deparse_columns *leftcolinfo; + deparse_columns *rightcolinfo; + int i; + ListCell *lc; + + /* Get info about the shape of the join */ + identify_join_columns(j, rte, colinfo); + leftattnos = colinfo->leftattnos; + rightattnos = colinfo->rightattnos; + + /* Look up the not-yet-filled-in child deparse_columns structs */ + leftcolinfo = deparse_columns_fetch(colinfo->leftrti, dpns); + rightcolinfo = deparse_columns_fetch(colinfo->rightrti, dpns); + + /* + * If this join is unnamed, then we cannot substitute new aliases at + * this level, so any name requirements pushed down to here must be + * pushed down again to the children. + */ + if (rte->alias == NULL) + { + for (i = 0; i < colinfo->num_cols; i++) + { + char *colname = colinfo->colnames[i]; + + if (colname == NULL) + continue; + + /* Push down to left column, unless it's a system column */ + if (leftattnos[i] > 0) + { + expand_colnames_array_to(leftcolinfo, leftattnos[i]); + leftcolinfo->colnames[leftattnos[i] - 1] = colname; + } + + /* Same on the righthand side */ + if (rightattnos[i] > 0) + { + expand_colnames_array_to(rightcolinfo, rightattnos[i]); + rightcolinfo->colnames[rightattnos[i] - 1] = colname; + } + } + } + + /* + * If there's a USING clause, select the USING column names and push + * those names down to the children. We have two strategies: + * + * If dpns->unique_using is true, we force all USING names to be + * unique across the whole query level. In principle we'd only need + * the names of dangerous USING columns to be globally unique, but to + * safely assign all USING names in a single pass, we have to enforce + * the same uniqueness rule for all of them. However, if a USING + * column's name has been pushed down from the parent, we should use + * it as-is rather than making a uniqueness adjustment. This is + * necessary when we're at an unnamed join, and it creates no risk of + * ambiguity. Also, if there's a user-written output alias for a + * merged column, we prefer to use that rather than the input name; + * this simplifies the logic and seems likely to lead to less aliasing + * overall. + * + * If dpns->unique_using is false, we only need USING names to be + * unique within their own join RTE. We still need to honor + * pushed-down names, though. + * + * Though significantly different in results, these two strategies are + * implemented by the same code, with only the difference of whether + * to put assigned names into dpns->using_names. + */ + if (j->usingClause) + { + /* Copy the input parentUsing list so we don't modify it */ + parentUsing = list_copy(parentUsing); + + /* USING names must correspond to the first join output columns */ + expand_colnames_array_to(colinfo, list_length(j->usingClause)); + i = 0; + foreach(lc, j->usingClause) + { + char *colname = strVal(lfirst(lc)); + + /* Assert it's a merged column */ + Assert(leftattnos[i] != 0 && rightattnos[i] != 0); + + /* Adopt passed-down name if any, else select unique name */ + if (colinfo->colnames[i] != NULL) + colname = colinfo->colnames[i]; + else + { + /* Prefer user-written output alias if any */ + if (rte->alias && i < list_length(rte->alias->colnames)) + colname = strVal(list_nth(rte->alias->colnames, i)); + /* Make it appropriately unique */ + colname = make_colname_unique(colname, dpns, colinfo); + if (dpns->unique_using) + dpns->using_names = lappend(dpns->using_names, + colname); + /* Save it as output column name, too */ + colinfo->colnames[i] = colname; + } + + /* Remember selected names for use later */ + colinfo->usingNames = lappend(colinfo->usingNames, colname); + parentUsing = lappend(parentUsing, colname); + + /* Push down to left column, unless it's a system column */ + if (leftattnos[i] > 0) + { + expand_colnames_array_to(leftcolinfo, leftattnos[i]); + leftcolinfo->colnames[leftattnos[i] - 1] = colname; + } + + /* Same on the righthand side */ + if (rightattnos[i] > 0) + { + expand_colnames_array_to(rightcolinfo, rightattnos[i]); + rightcolinfo->colnames[rightattnos[i] - 1] = colname; + } + + i++; + } + } + + /* Mark child deparse_columns structs with correct parentUsing info */ + leftcolinfo->parentUsing = parentUsing; + rightcolinfo->parentUsing = parentUsing; + + /* Now recursively assign USING column names in children */ + set_using_names(dpns, j->larg, parentUsing); + set_using_names(dpns, j->rarg, parentUsing); + } + else + elog(ERROR, "unrecognized node type: %d", + (int) nodeTag(jtnode)); +} + +/* + * set_relation_column_names: select column aliases for a non-join RTE + * + * Column alias info is saved in *colinfo, which is assumed to be pre-zeroed. + * If any colnames entries are already filled in, those override local + * choices. + */ +static void +set_relation_column_names(deparse_namespace *dpns, RangeTblEntry *rte, + deparse_columns *colinfo) +{ + int ncolumns; + char **real_colnames; + bool changed_any; + int noldcolumns; + int i; + int j; + + /* + * Extract the RTE's "real" column names. This is comparable to + * get_rte_attribute_name, except that it's important to disregard dropped + * columns. We put NULL into the array for a dropped column. + */ + if (rte->rtekind == RTE_RELATION) + { + /* Relation --- look to the system catalogs for up-to-date info */ + Relation rel; + TupleDesc tupdesc; + + rel = relation_open(rte->relid, AccessShareLock); + tupdesc = RelationGetDescr(rel); + + ncolumns = tupdesc->natts; + real_colnames = (char **) palloc(ncolumns * sizeof(char *)); + + for (i = 0; i < ncolumns; i++) + { + Form_pg_attribute attr = TupleDescAttr(tupdesc, i); + + if (attr->attisdropped) + real_colnames[i] = NULL; + else + real_colnames[i] = pstrdup(NameStr(attr->attname)); + } + relation_close(rel, AccessShareLock); + } + else + { + /* Otherwise use the column names from eref */ + ListCell *lc; + + ncolumns = list_length(rte->eref->colnames); + real_colnames = (char **) palloc(ncolumns * sizeof(char *)); + + i = 0; + foreach(lc, rte->eref->colnames) + { + /* + * If the column name shown in eref is an empty string, then it's + * a column that was dropped at the time of parsing the query, so + * treat it as dropped. + */ + char *cname = strVal(lfirst(lc)); + + if (cname[0] == '\0') + cname = NULL; + real_colnames[i] = cname; + i++; + } + } + + /* + * Ensure colinfo->colnames has a slot for each column. (It could be long + * enough already, if we pushed down a name for the last column.) Note: + * it's possible that there are now more columns than there were when the + * query was parsed, ie colnames could be longer than rte->eref->colnames. + * We must assign unique aliases to the new columns too, else there could + * be unresolved conflicts when the view/rule is reloaded. + */ + expand_colnames_array_to(colinfo, ncolumns); + Assert(colinfo->num_cols == ncolumns); + + /* + * Make sufficiently large new_colnames and is_new_col arrays, too. + * + * Note: because we leave colinfo->num_new_cols zero until after the loop, + * colname_is_unique will not consult that array, which is fine because it + * would only be duplicate effort. + */ + colinfo->new_colnames = (char **) palloc(ncolumns * sizeof(char *)); + colinfo->is_new_col = (bool *) palloc(ncolumns * sizeof(bool)); + + /* + * Scan the columns, select a unique alias for each one, and store it in + * colinfo->colnames and colinfo->new_colnames. The former array has NULL + * entries for dropped columns, the latter omits them. Also mark + * new_colnames entries as to whether they are new since parse time; this + * is the case for entries beyond the length of rte->eref->colnames. + */ + noldcolumns = list_length(rte->eref->colnames); + changed_any = false; + j = 0; + for (i = 0; i < ncolumns; i++) + { + char *real_colname = real_colnames[i]; + char *colname = colinfo->colnames[i]; + + /* Skip dropped columns */ + if (real_colname == NULL) + { + Assert(colname == NULL); /* colnames[i] is already NULL */ + continue; + } + + /* If alias already assigned, that's what to use */ + if (colname == NULL) + { + /* If user wrote an alias, prefer that over real column name */ + if (rte->alias && i < list_length(rte->alias->colnames)) + colname = strVal(list_nth(rte->alias->colnames, i)); + else + colname = real_colname; + + /* Unique-ify and insert into colinfo */ + colname = make_colname_unique(colname, dpns, colinfo); + + colinfo->colnames[i] = colname; + } + + /* Put names of non-dropped columns in new_colnames[] too */ + colinfo->new_colnames[j] = colname; + /* And mark them as new or not */ + colinfo->is_new_col[j] = (i >= noldcolumns); + j++; + + /* Remember if any assigned aliases differ from "real" name */ + if (!changed_any && strcmp(colname, real_colname) != 0) + changed_any = true; + } + + /* + * Set correct length for new_colnames[] array. (Note: if columns have + * been added, colinfo->num_cols includes them, which is not really quite + * right but is harmless, since any new columns must be at the end where + * they won't affect varattnos of pre-existing columns.) + */ + colinfo->num_new_cols = j; + + /* + * For a relation RTE, we need only print the alias column names if any + * are different from the underlying "real" names. For a function RTE, + * always emit a complete column alias list; this is to protect against + * possible instability of the default column names (eg, from altering + * parameter names). For tablefunc RTEs, we never print aliases, because + * the column names are part of the clause itself. For other RTE types, + * print if we changed anything OR if there were user-written column + * aliases (since the latter would be part of the underlying "reality"). + */ + if (rte->rtekind == RTE_RELATION) + colinfo->printaliases = changed_any; + else if (rte->rtekind == RTE_FUNCTION) + colinfo->printaliases = true; + else if (rte->rtekind == RTE_TABLEFUNC) + colinfo->printaliases = false; + else if (rte->alias && rte->alias->colnames != NIL) + colinfo->printaliases = true; + else + colinfo->printaliases = changed_any; +} + +/* + * set_join_column_names: select column aliases for a join RTE + * + * Column alias info is saved in *colinfo, which is assumed to be pre-zeroed. + * If any colnames entries are already filled in, those override local + * choices. Also, names for USING columns were already chosen by + * set_using_names(). We further expect that column alias selection has been + * completed for both input RTEs. + */ +static void +set_join_column_names(deparse_namespace *dpns, RangeTblEntry *rte, + deparse_columns *colinfo) +{ + deparse_columns *leftcolinfo; + deparse_columns *rightcolinfo; + bool changed_any; + int noldcolumns; + int nnewcolumns; + Bitmapset *leftmerged = NULL; + Bitmapset *rightmerged = NULL; + int i; + int j; + int ic; + int jc; + + /* Look up the previously-filled-in child deparse_columns structs */ + leftcolinfo = deparse_columns_fetch(colinfo->leftrti, dpns); + rightcolinfo = deparse_columns_fetch(colinfo->rightrti, dpns); + + /* + * Ensure colinfo->colnames has a slot for each column. (It could be long + * enough already, if we pushed down a name for the last column.) Note: + * it's possible that one or both inputs now have more columns than there + * were when the query was parsed, but we'll deal with that below. We + * only need entries in colnames for pre-existing columns. + */ + noldcolumns = list_length(rte->eref->colnames); + expand_colnames_array_to(colinfo, noldcolumns); + Assert(colinfo->num_cols == noldcolumns); + + /* + * Scan the join output columns, select an alias for each one, and store + * it in colinfo->colnames. If there are USING columns, set_using_names() + * already selected their names, so we can start the loop at the first + * non-merged column. + */ + changed_any = false; + for (i = list_length(colinfo->usingNames); i < noldcolumns; i++) + { + char *colname = colinfo->colnames[i]; + char *real_colname; + + /* Ignore dropped column (only possible for non-merged column) */ + if (colinfo->leftattnos[i] == 0 && colinfo->rightattnos[i] == 0) + { + Assert(colname == NULL); + continue; + } + + /* Get the child column name */ + if (colinfo->leftattnos[i] > 0) + real_colname = leftcolinfo->colnames[colinfo->leftattnos[i] - 1]; + else if (colinfo->rightattnos[i] > 0) + real_colname = rightcolinfo->colnames[colinfo->rightattnos[i] - 1]; + else + { + /* We're joining system columns --- use eref name */ + real_colname = strVal(list_nth(rte->eref->colnames, i)); + } + Assert(real_colname != NULL); + + /* In an unnamed join, just report child column names as-is */ + if (rte->alias == NULL) + { + colinfo->colnames[i] = real_colname; + continue; + } + + /* If alias already assigned, that's what to use */ + if (colname == NULL) + { + /* If user wrote an alias, prefer that over real column name */ + if (rte->alias && i < list_length(rte->alias->colnames)) + colname = strVal(list_nth(rte->alias->colnames, i)); + else + colname = real_colname; + + /* Unique-ify and insert into colinfo */ + colname = make_colname_unique(colname, dpns, colinfo); + + colinfo->colnames[i] = colname; + } + + /* Remember if any assigned aliases differ from "real" name */ + if (!changed_any && strcmp(colname, real_colname) != 0) + changed_any = true; + } + + /* + * Calculate number of columns the join would have if it were re-parsed + * now, and create storage for the new_colnames and is_new_col arrays. + * + * Note: colname_is_unique will be consulting new_colnames[] during the + * loops below, so its not-yet-filled entries must be zeroes. + */ + nnewcolumns = leftcolinfo->num_new_cols + rightcolinfo->num_new_cols - + list_length(colinfo->usingNames); + colinfo->num_new_cols = nnewcolumns; + colinfo->new_colnames = (char **) palloc0(nnewcolumns * sizeof(char *)); + colinfo->is_new_col = (bool *) palloc0(nnewcolumns * sizeof(bool)); + + /* + * Generating the new_colnames array is a bit tricky since any new columns + * added since parse time must be inserted in the right places. This code + * must match the parser, which will order a join's columns as merged + * columns first (in USING-clause order), then non-merged columns from the + * left input (in attnum order), then non-merged columns from the right + * input (ditto). If one of the inputs is itself a join, its columns will + * be ordered according to the same rule, which means newly-added columns + * might not be at the end. We can figure out what's what by consulting + * the leftattnos and rightattnos arrays plus the input is_new_col arrays. + * + * In these loops, i indexes leftattnos/rightattnos (so it's join varattno + * less one), j indexes new_colnames/is_new_col, and ic/jc have similar + * meanings for the current child RTE. + */ + + /* Handle merged columns; they are first and can't be new */ + i = j = 0; + while (i < noldcolumns && + colinfo->leftattnos[i] != 0 && + colinfo->rightattnos[i] != 0) + { + /* column name is already determined and known unique */ + colinfo->new_colnames[j] = colinfo->colnames[i]; + colinfo->is_new_col[j] = false; + + /* build bitmapsets of child attnums of merged columns */ + if (colinfo->leftattnos[i] > 0) + leftmerged = bms_add_member(leftmerged, colinfo->leftattnos[i]); + if (colinfo->rightattnos[i] > 0) + rightmerged = bms_add_member(rightmerged, colinfo->rightattnos[i]); + + i++, j++; + } + + /* Handle non-merged left-child columns */ + ic = 0; + for (jc = 0; jc < leftcolinfo->num_new_cols; jc++) + { + char *child_colname = leftcolinfo->new_colnames[jc]; + + if (!leftcolinfo->is_new_col[jc]) + { + /* Advance ic to next non-dropped old column of left child */ + while (ic < leftcolinfo->num_cols && + leftcolinfo->colnames[ic] == NULL) + ic++; + Assert(ic < leftcolinfo->num_cols); + ic++; + /* If it is a merged column, we already processed it */ + if (bms_is_member(ic, leftmerged)) + continue; + /* Else, advance i to the corresponding existing join column */ + while (i < colinfo->num_cols && + colinfo->colnames[i] == NULL) + i++; + Assert(i < colinfo->num_cols); + Assert(ic == colinfo->leftattnos[i]); + /* Use the already-assigned name of this column */ + colinfo->new_colnames[j] = colinfo->colnames[i]; + i++; + } + else + { + /* + * Unique-ify the new child column name and assign, unless we're + * in an unnamed join, in which case just copy + */ + if (rte->alias != NULL) + { + colinfo->new_colnames[j] = + make_colname_unique(child_colname, dpns, colinfo); + if (!changed_any && + strcmp(colinfo->new_colnames[j], child_colname) != 0) + changed_any = true; + } + else + colinfo->new_colnames[j] = child_colname; + } + + colinfo->is_new_col[j] = leftcolinfo->is_new_col[jc]; + j++; + } + + /* Handle non-merged right-child columns in exactly the same way */ + ic = 0; + for (jc = 0; jc < rightcolinfo->num_new_cols; jc++) + { + char *child_colname = rightcolinfo->new_colnames[jc]; + + if (!rightcolinfo->is_new_col[jc]) + { + /* Advance ic to next non-dropped old column of right child */ + while (ic < rightcolinfo->num_cols && + rightcolinfo->colnames[ic] == NULL) + ic++; + Assert(ic < rightcolinfo->num_cols); + ic++; + /* If it is a merged column, we already processed it */ + if (bms_is_member(ic, rightmerged)) + continue; + /* Else, advance i to the corresponding existing join column */ + while (i < colinfo->num_cols && + colinfo->colnames[i] == NULL) + i++; + Assert(i < colinfo->num_cols); + Assert(ic == colinfo->rightattnos[i]); + /* Use the already-assigned name of this column */ + colinfo->new_colnames[j] = colinfo->colnames[i]; + i++; + } + else + { + /* + * Unique-ify the new child column name and assign, unless we're + * in an unnamed join, in which case just copy + */ + if (rte->alias != NULL) + { + colinfo->new_colnames[j] = + make_colname_unique(child_colname, dpns, colinfo); + if (!changed_any && + strcmp(colinfo->new_colnames[j], child_colname) != 0) + changed_any = true; + } + else + colinfo->new_colnames[j] = child_colname; + } + + colinfo->is_new_col[j] = rightcolinfo->is_new_col[jc]; + j++; + } + + /* Assert we processed the right number of columns */ +#ifdef USE_ASSERT_CHECKING + while (i < colinfo->num_cols && colinfo->colnames[i] == NULL) + i++; + Assert(i == colinfo->num_cols); + Assert(j == nnewcolumns); +#endif + + /* + * For a named join, print column aliases if we changed any from the child + * names. Unnamed joins cannot print aliases. + */ + if (rte->alias != NULL) + colinfo->printaliases = changed_any; + else + colinfo->printaliases = false; +} + +/* + * colname_is_unique: is colname distinct from already-chosen column names? + * + * dpns is query-wide info, colinfo is for the column's RTE + */ +static bool +colname_is_unique(const char *colname, deparse_namespace *dpns, + deparse_columns *colinfo) +{ + int i; + ListCell *lc; + + /* Check against already-assigned column aliases within RTE */ + for (i = 0; i < colinfo->num_cols; i++) + { + char *oldname = colinfo->colnames[i]; + + if (oldname && strcmp(oldname, colname) == 0) + return false; + } + + /* + * If we're building a new_colnames array, check that too (this will be + * partially but not completely redundant with the previous checks) + */ + for (i = 0; i < colinfo->num_new_cols; i++) + { + char *oldname = colinfo->new_colnames[i]; + + if (oldname && strcmp(oldname, colname) == 0) + return false; + } + + /* Also check against USING-column names that must be globally unique */ + foreach(lc, dpns->using_names) + { + char *oldname = (char *) lfirst(lc); + + if (strcmp(oldname, colname) == 0) + return false; + } + + /* Also check against names already assigned for parent-join USING cols */ + foreach(lc, colinfo->parentUsing) + { + char *oldname = (char *) lfirst(lc); + + if (strcmp(oldname, colname) == 0) + return false; + } + + return true; +} + +/* + * make_colname_unique: modify colname if necessary to make it unique + * + * dpns is query-wide info, colinfo is for the column's RTE + */ +static char * +make_colname_unique(char *colname, deparse_namespace *dpns, + deparse_columns *colinfo) +{ + /* + * If the selected name isn't unique, append digits to make it so. For a + * very long input name, we might have to truncate to stay within + * NAMEDATALEN. + */ + if (!colname_is_unique(colname, dpns, colinfo)) + { + int colnamelen = strlen(colname); + char *modname = (char *) palloc(colnamelen + 16); + int i = 0; + + do + { + i++; + for (;;) + { + /* + * We avoid using %.*s here because it can misbehave if the + * data is not valid in what libc thinks is the prevailing + * encoding. + */ + memcpy(modname, colname, colnamelen); + sprintf(modname + colnamelen, "_%d", i); + if (strlen(modname) < NAMEDATALEN) + break; + /* drop chars from colname to keep all the digits */ + colnamelen = pg_mbcliplen(colname, colnamelen, + colnamelen - 1); + } + } while (!colname_is_unique(modname, dpns, colinfo)); + colname = modname; + } + return colname; +} + +/* + * expand_colnames_array_to: make colinfo->colnames at least n items long + * + * Any added array entries are initialized to zero. + */ +static void +expand_colnames_array_to(deparse_columns *colinfo, int n) +{ + if (n > colinfo->num_cols) + { + if (colinfo->colnames == NULL) + colinfo->colnames = (char **) palloc0(n * sizeof(char *)); + else + { + colinfo->colnames = (char **) repalloc(colinfo->colnames, + n * sizeof(char *)); + memset(colinfo->colnames + colinfo->num_cols, 0, + (n - colinfo->num_cols) * sizeof(char *)); + } + colinfo->num_cols = n; + } +} + +/* + * identify_join_columns: figure out where columns of a join come from + * + * Fills the join-specific fields of the colinfo struct, except for + * usingNames which is filled later. + */ +static void +identify_join_columns(JoinExpr *j, RangeTblEntry *jrte, + deparse_columns *colinfo) +{ + int numjoincols; + int i; + ListCell *lc; + + /* Extract left/right child RT indexes */ + if (IsA(j->larg, RangeTblRef)) + colinfo->leftrti = ((RangeTblRef *) j->larg)->rtindex; + else if (IsA(j->larg, JoinExpr)) + colinfo->leftrti = ((JoinExpr *) j->larg)->rtindex; + else + elog(ERROR, "unrecognized node type in jointree: %d", + (int) nodeTag(j->larg)); + if (IsA(j->rarg, RangeTblRef)) + colinfo->rightrti = ((RangeTblRef *) j->rarg)->rtindex; + else if (IsA(j->rarg, JoinExpr)) + colinfo->rightrti = ((JoinExpr *) j->rarg)->rtindex; + else + elog(ERROR, "unrecognized node type in jointree: %d", + (int) nodeTag(j->rarg)); + + /* Assert children will be processed earlier than join in second pass */ + Assert(colinfo->leftrti < j->rtindex); + Assert(colinfo->rightrti < j->rtindex); + + /* Initialize result arrays with zeroes */ + numjoincols = list_length(jrte->joinaliasvars); + Assert(numjoincols == list_length(jrte->eref->colnames)); + colinfo->leftattnos = (int *) palloc0(numjoincols * sizeof(int)); + colinfo->rightattnos = (int *) palloc0(numjoincols * sizeof(int)); + + /* Scan the joinaliasvars list to identify simple column references */ + i = 0; + foreach(lc, jrte->joinaliasvars) + { + Var *aliasvar = (Var *) lfirst(lc); + + /* get rid of any implicit coercion above the Var */ + aliasvar = (Var *) strip_implicit_coercions((Node *) aliasvar); + + if (aliasvar == NULL) + { + /* It's a dropped column; nothing to do here */ + } + else if (IsA(aliasvar, Var)) + { + Assert(aliasvar->varlevelsup == 0); + Assert(aliasvar->varattno != 0); + if (aliasvar->varno == colinfo->leftrti) + colinfo->leftattnos[i] = aliasvar->varattno; + else if (aliasvar->varno == colinfo->rightrti) + colinfo->rightattnos[i] = aliasvar->varattno; + else + elog(ERROR, "unexpected varno %d in JOIN RTE", + aliasvar->varno); + } + else if (IsA(aliasvar, CoalesceExpr)) + { + /* + * It's a merged column in FULL JOIN USING. Ignore it for now and + * let the code below identify the merged columns. + */ + } + else + elog(ERROR, "unrecognized node type in join alias vars: %d", + (int) nodeTag(aliasvar)); + + i++; + } + + /* + * If there's a USING clause, deconstruct the join quals to identify the + * merged columns. This is a tad painful but if we cannot rely on the + * column names, there is no other representation of which columns were + * joined by USING. (Unless the join type is FULL, we can't tell from the + * joinaliasvars list which columns are merged.) Note: we assume that the + * merged columns are the first output column(s) of the join. + */ + if (j->usingClause) + { + List *leftvars = NIL; + List *rightvars = NIL; + ListCell *lc2; + + /* Extract left- and right-side Vars from the qual expression */ + flatten_join_using_qual(j->quals, &leftvars, &rightvars); + Assert(list_length(leftvars) == list_length(j->usingClause)); + Assert(list_length(rightvars) == list_length(j->usingClause)); + + /* Mark the output columns accordingly */ + i = 0; + forboth(lc, leftvars, lc2, rightvars) + { + Var *leftvar = (Var *) lfirst(lc); + Var *rightvar = (Var *) lfirst(lc2); + + Assert(leftvar->varlevelsup == 0); + Assert(leftvar->varattno != 0); + if (leftvar->varno != colinfo->leftrti) + elog(ERROR, "unexpected varno %d in JOIN USING qual", + leftvar->varno); + colinfo->leftattnos[i] = leftvar->varattno; + + Assert(rightvar->varlevelsup == 0); + Assert(rightvar->varattno != 0); + if (rightvar->varno != colinfo->rightrti) + elog(ERROR, "unexpected varno %d in JOIN USING qual", + rightvar->varno); + colinfo->rightattnos[i] = rightvar->varattno; + + i++; + } + } +} + +/* + * flatten_join_using_qual: extract Vars being joined from a JOIN/USING qual + * + * We assume that transformJoinUsingClause won't have produced anything except + * AND nodes, equality operator nodes, and possibly implicit coercions, and + * that the AND node inputs match left-to-right with the original USING list. + * + * Caller must initialize the result lists to NIL. + */ +static void +flatten_join_using_qual(Node *qual, List **leftvars, List **rightvars) +{ + if (IsA(qual, BoolExpr)) + { + /* Handle AND nodes by recursion */ + BoolExpr *b = (BoolExpr *) qual; + ListCell *lc; + + Assert(b->boolop == AND_EXPR); + foreach(lc, b->args) + { + flatten_join_using_qual((Node *) lfirst(lc), + leftvars, rightvars); + } + } + else if (IsA(qual, OpExpr)) + { + /* Otherwise we should have an equality operator */ + OpExpr *op = (OpExpr *) qual; + Var *var; + + if (list_length(op->args) != 2) + elog(ERROR, "unexpected unary operator in JOIN/USING qual"); + /* Arguments should be Vars with perhaps implicit coercions */ + var = (Var *) strip_implicit_coercions((Node *) linitial(op->args)); + if (!IsA(var, Var)) + elog(ERROR, "unexpected node type in JOIN/USING qual: %d", + (int) nodeTag(var)); + *leftvars = lappend(*leftvars, var); + var = (Var *) strip_implicit_coercions((Node *) lsecond(op->args)); + if (!IsA(var, Var)) + elog(ERROR, "unexpected node type in JOIN/USING qual: %d", + (int) nodeTag(var)); + *rightvars = lappend(*rightvars, var); + } + else + { + /* Perhaps we have an implicit coercion to boolean? */ + Node *q = strip_implicit_coercions(qual); + + if (q != qual) + flatten_join_using_qual(q, leftvars, rightvars); + else + elog(ERROR, "unexpected node type in JOIN/USING qual: %d", + (int) nodeTag(qual)); + } +} + +/* + * get_rtable_name: convenience function to get a previously assigned RTE alias + * + * The RTE must belong to the topmost namespace level in "context". + */ +static char * +get_rtable_name(int rtindex, deparse_context *context) +{ + deparse_namespace *dpns = (deparse_namespace *) linitial(context->namespaces); + + Assert(rtindex > 0 && rtindex <= list_length(dpns->rtable_names)); + return (char *) list_nth(dpns->rtable_names, rtindex - 1); +} + +/* + * set_deparse_planstate: set up deparse_namespace to parse subexpressions + * of a given PlanState node + * + * This sets the planstate, outer_planstate, inner_planstate, outer_tlist, + * inner_tlist, and index_tlist fields. Caller is responsible for adjusting + * the ancestors list if necessary. Note that the rtable and ctes fields do + * not need to change when shifting attention to different plan nodes in a + * single plan tree. + */ +static void +set_deparse_planstate(deparse_namespace *dpns, PlanState *ps) +{ + dpns->planstate = ps; + + /* + * We special-case Append and MergeAppend to pretend that the first child + * plan is the OUTER referent; we have to interpret OUTER Vars in their + * tlists according to one of the children, and the first one is the most + * natural choice. Likewise special-case ModifyTable to pretend that the + * first child plan is the OUTER referent; this is to support RETURNING + * lists containing references to non-target relations. + */ + if (IsA(ps, AppendState)) + dpns->outer_planstate = ((AppendState *) ps)->appendplans[0]; + else if (IsA(ps, MergeAppendState)) + dpns->outer_planstate = ((MergeAppendState *) ps)->mergeplans[0]; + else if (IsA(ps, ModifyTableState)) + dpns->outer_planstate = ((ModifyTableState *) ps)->mt_plans[0]; + else + dpns->outer_planstate = outerPlanState(ps); + + if (dpns->outer_planstate) + dpns->outer_tlist = dpns->outer_planstate->plan->targetlist; + else + dpns->outer_tlist = NIL; + + /* + * For a SubqueryScan, pretend the subplan is INNER referent. (We don't + * use OUTER because that could someday conflict with the normal meaning.) + * Likewise, for a CteScan, pretend the subquery's plan is INNER referent. + * For ON CONFLICT .. UPDATE we just need the inner tlist to point to the + * excluded expression's tlist. (Similar to the SubqueryScan we don't want + * to reuse OUTER, it's used for RETURNING in some modify table cases, + * although not INSERT .. CONFLICT). + */ + if (IsA(ps, SubqueryScanState)) + dpns->inner_planstate = ((SubqueryScanState *) ps)->subplan; + else if (IsA(ps, CteScanState)) + dpns->inner_planstate = ((CteScanState *) ps)->cteplanstate; + else if (IsA(ps, ModifyTableState)) + dpns->inner_planstate = ps; + else + dpns->inner_planstate = innerPlanState(ps); + + if (IsA(ps, ModifyTableState)) + dpns->inner_tlist = ((ModifyTableState *) ps)->mt_excludedtlist; + else if (dpns->inner_planstate) + dpns->inner_tlist = dpns->inner_planstate->plan->targetlist; + else + dpns->inner_tlist = NIL; + + /* Set up referent for INDEX_VAR Vars, if needed */ + if (IsA(ps->plan, IndexOnlyScan)) + dpns->index_tlist = ((IndexOnlyScan *) ps->plan)->indextlist; + else if (IsA(ps->plan, ForeignScan)) + dpns->index_tlist = ((ForeignScan *) ps->plan)->fdw_scan_tlist; + else if (IsA(ps->plan, CustomScan)) + dpns->index_tlist = ((CustomScan *) ps->plan)->custom_scan_tlist; + else + dpns->index_tlist = NIL; +} + +/* + * push_child_plan: temporarily transfer deparsing attention to a child plan + * + * When expanding an OUTER_VAR or INNER_VAR reference, we must adjust the + * deparse context in case the referenced expression itself uses + * OUTER_VAR/INNER_VAR. We modify the top stack entry in-place to avoid + * affecting levelsup issues (although in a Plan tree there really shouldn't + * be any). + * + * Caller must provide a local deparse_namespace variable to save the + * previous state for pop_child_plan. + */ +static void +push_child_plan(deparse_namespace *dpns, PlanState *ps, + deparse_namespace *save_dpns) +{ + /* Save state for restoration later */ + *save_dpns = *dpns; + + /* Link current plan node into ancestors list */ + dpns->ancestors = lcons(dpns->planstate, dpns->ancestors); + + /* Set attention on selected child */ + set_deparse_planstate(dpns, ps); +} + +/* + * pop_child_plan: undo the effects of push_child_plan + */ +static void +pop_child_plan(deparse_namespace *dpns, deparse_namespace *save_dpns) +{ + List *ancestors; + + /* Get rid of ancestors list cell added by push_child_plan */ + ancestors = list_delete_first(dpns->ancestors); + + /* Restore fields changed by push_child_plan */ + *dpns = *save_dpns; + + /* Make sure dpns->ancestors is right (may be unnecessary) */ + dpns->ancestors = ancestors; +} + +/* + * push_ancestor_plan: temporarily transfer deparsing attention to an + * ancestor plan + * + * When expanding a Param reference, we must adjust the deparse context + * to match the plan node that contains the expression being printed; + * otherwise we'd fail if that expression itself contains a Param or + * OUTER_VAR/INNER_VAR/INDEX_VAR variable. + * + * The target ancestor is conveniently identified by the ListCell holding it + * in dpns->ancestors. + * + * Caller must provide a local deparse_namespace variable to save the + * previous state for pop_ancestor_plan. + */ +static void +push_ancestor_plan(deparse_namespace *dpns, ListCell *ancestor_cell, + deparse_namespace *save_dpns) +{ + PlanState *ps = (PlanState *) lfirst(ancestor_cell); + List *ancestors; + + /* Save state for restoration later */ + *save_dpns = *dpns; + + /* Build a new ancestor list with just this node's ancestors */ + ancestors = NIL; + while ((ancestor_cell = lnext(ancestor_cell)) != NULL) + ancestors = lappend(ancestors, lfirst(ancestor_cell)); + dpns->ancestors = ancestors; + + /* Set attention on selected ancestor */ + set_deparse_planstate(dpns, ps); +} + +/* + * pop_ancestor_plan: undo the effects of push_ancestor_plan + */ +static void +pop_ancestor_plan(deparse_namespace *dpns, deparse_namespace *save_dpns) +{ + /* Free the ancestor list made in push_ancestor_plan */ + list_free(dpns->ancestors); + + /* Restore fields changed by push_ancestor_plan */ + *dpns = *save_dpns; +} + + +/* ---------- + * deparse_shard_query - Parse back a query for execution on a shard + * + * Builds an SQL string to perform the provided query on a specific shard and + * places this string into the provided buffer. + * ---------- + */ +void +deparse_shard_query(Query *query, Oid distrelid, int64 shardid, + StringInfo buffer) +{ + get_query_def_extended(query, buffer, NIL, distrelid, shardid, NULL, 0, + WRAP_COLUMN_DEFAULT, 0); +} + + +/* ---------- + * get_query_def - Parse back one query parsetree + * + * If resultDesc is not NULL, then it is the output tuple descriptor for + * the view represented by a SELECT query. + * ---------- + */ +static void +get_query_def(Query *query, StringInfo buf, List *parentnamespace, + TupleDesc resultDesc, + int prettyFlags, int wrapColumn, int startIndent) +{ + get_query_def_extended(query, buf, parentnamespace, InvalidOid, 0, resultDesc, + prettyFlags, wrapColumn, startIndent); +} + + +/* ---------- + * get_query_def_extended - Parse back one query parsetree, optionally + * with extension using a shard identifier. + * + * If distrelid is valid and shardid is positive, the provided shardid is added + * any time the provided relid is deparsed, so that the query may be executed + * on a placement for the given shard. + * ---------- + */ +static void +get_query_def_extended(Query *query, StringInfo buf, List *parentnamespace, + Oid distrelid, int64 shardid, TupleDesc resultDesc, + int prettyFlags, int wrapColumn, int startIndent) +{ + deparse_context context; + deparse_namespace dpns; + + OverrideSearchPath *overridePath = NULL; + + /* Guard against excessively long or deeply-nested queries */ + CHECK_FOR_INTERRUPTS(); + check_stack_depth(); + + /* + * Before we begin to examine the query, acquire locks on referenced + * relations, and fix up deleted columns in JOIN RTEs. This ensures + * consistent results. Note we assume it's OK to scribble on the passed + * querytree! + * + * We are only deparsing the query (we are not about to execute it), so we + * only need AccessShareLock on the relations it mentions. + */ + AcquireRewriteLocks(query, false, false); + + /* + * Set search_path to NIL so that all objects outside of pg_catalog will be + * schema-prefixed. pg_catalog will be added automatically when we call + * PushOverrideSearchPath(), since we set addCatalog to true; + */ + overridePath = GetOverrideSearchPath(CurrentMemoryContext); + overridePath->schemas = NIL; + overridePath->addCatalog = true; + PushOverrideSearchPath(overridePath); + + context.buf = buf; + context.namespaces = lcons(&dpns, list_copy(parentnamespace)); + context.windowClause = NIL; + context.windowTList = NIL; + context.varprefix = (parentnamespace != NIL || + list_length(query->rtable) != 1); + context.prettyFlags = prettyFlags; + context.wrapColumn = wrapColumn; + context.indentLevel = startIndent; + context.special_exprkind = EXPR_KIND_NONE; + context.distrelid = distrelid; + context.shardid = shardid; + + set_deparse_for_query(&dpns, query, parentnamespace); + + switch (query->commandType) + { + case CMD_SELECT: + get_select_query_def(query, &context, resultDesc); + break; + + case CMD_UPDATE: + get_update_query_def(query, &context); + break; + + case CMD_INSERT: + get_insert_query_def(query, &context); + break; + + case CMD_DELETE: + get_delete_query_def(query, &context); + break; + + case CMD_NOTHING: + appendStringInfoString(buf, "NOTHING"); + break; + + case CMD_UTILITY: + get_utility_query_def(query, &context); + break; + + default: + elog(ERROR, "unrecognized query command type: %d", + query->commandType); + break; + } + + /* revert back to original search_path */ + PopOverrideSearchPath(); +} + +/* ---------- + * get_values_def - Parse back a VALUES list + * ---------- + */ +static void +get_values_def(List *values_lists, deparse_context *context) +{ + StringInfo buf = context->buf; + bool first_list = true; + ListCell *vtl; + + appendStringInfoString(buf, "VALUES "); + + foreach(vtl, values_lists) + { + List *sublist = (List *) lfirst(vtl); + bool first_col = true; + ListCell *lc; + + if (first_list) + first_list = false; + else + appendStringInfoString(buf, ", "); + + appendStringInfoChar(buf, '('); + foreach(lc, sublist) + { + Node *col = (Node *) lfirst(lc); + + if (first_col) + first_col = false; + else + appendStringInfoChar(buf, ','); + + /* + * Print the value. Whole-row Vars need special treatment. + */ + get_rule_expr_toplevel(col, context, false); + } + appendStringInfoChar(buf, ')'); + } +} + +/* ---------- + * get_with_clause - Parse back a WITH clause + * ---------- + */ +static void +get_with_clause(Query *query, deparse_context *context) +{ + StringInfo buf = context->buf; + const char *sep; + ListCell *l; + + if (query->cteList == NIL) + return; + + if (PRETTY_INDENT(context)) + { + context->indentLevel += PRETTYINDENT_STD; + appendStringInfoChar(buf, ' '); + } + + if (query->hasRecursive) + sep = "WITH RECURSIVE "; + else + sep = "WITH "; + foreach(l, query->cteList) + { + CommonTableExpr *cte = (CommonTableExpr *) lfirst(l); + + appendStringInfoString(buf, sep); + appendStringInfoString(buf, quote_identifier(cte->ctename)); + if (cte->aliascolnames) + { + bool first = true; + ListCell *col; + + appendStringInfoChar(buf, '('); + foreach(col, cte->aliascolnames) + { + if (first) + first = false; + else + appendStringInfoString(buf, ", "); + appendStringInfoString(buf, + quote_identifier(strVal(lfirst(col)))); + } + appendStringInfoChar(buf, ')'); + } + appendStringInfoString(buf, " AS "); + switch (cte->ctematerialized) + { + case CTEMaterializeDefault: + break; + case CTEMaterializeAlways: + appendStringInfoString(buf, "MATERIALIZED "); + break; + case CTEMaterializeNever: + appendStringInfoString(buf, "NOT MATERIALIZED "); + break; + } + appendStringInfoChar(buf, '('); + if (PRETTY_INDENT(context)) + appendContextKeyword(context, "", 0, 0, 0); + get_query_def((Query *) cte->ctequery, buf, context->namespaces, NULL, + context->prettyFlags, context->wrapColumn, + context->indentLevel); + if (PRETTY_INDENT(context)) + appendContextKeyword(context, "", 0, 0, 0); + appendStringInfoChar(buf, ')'); + sep = ", "; + } + + if (PRETTY_INDENT(context)) + { + context->indentLevel -= PRETTYINDENT_STD; + appendContextKeyword(context, "", 0, 0, 0); + } + else + appendStringInfoChar(buf, ' '); +} + +/* ---------- + * get_select_query_def - Parse back a SELECT parsetree + * ---------- + */ +static void +get_select_query_def(Query *query, deparse_context *context, + TupleDesc resultDesc) +{ + StringInfo buf = context->buf; + List *save_windowclause; + List *save_windowtlist; + bool force_colno; + ListCell *l; + + /* Insert the WITH clause if given */ + get_with_clause(query, context); + + /* Set up context for possible window functions */ + save_windowclause = context->windowClause; + context->windowClause = query->windowClause; + save_windowtlist = context->windowTList; + context->windowTList = query->targetList; + + /* + * If the Query node has a setOperations tree, then it's the top level of + * a UNION/INTERSECT/EXCEPT query; only the WITH, ORDER BY and LIMIT + * fields are interesting in the top query itself. + */ + if (query->setOperations) + { + get_setop_query(query->setOperations, query, context, resultDesc); + /* ORDER BY clauses must be simple in this case */ + force_colno = true; + } + else + { + get_basic_select_query(query, context, resultDesc); + force_colno = false; + } + + /* Add the ORDER BY clause if given */ + if (query->sortClause != NIL) + { + appendContextKeyword(context, " ORDER BY ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_rule_orderby(query->sortClause, query->targetList, + force_colno, context); + } + + /* Add the LIMIT clause if given */ + if (query->limitOffset != NULL) + { + appendContextKeyword(context, " OFFSET ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); + get_rule_expr(query->limitOffset, context, false); + } + if (query->limitCount != NULL) + { + appendContextKeyword(context, " LIMIT ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); + if (IsA(query->limitCount, Const) && + ((Const *) query->limitCount)->constisnull) + appendStringInfoString(buf, "ALL"); + else + get_rule_expr(query->limitCount, context, false); + } + + /* Add FOR [KEY] UPDATE/SHARE clauses if present */ + if (query->hasForUpdate) + { + foreach(l, query->rowMarks) + { + RowMarkClause *rc = (RowMarkClause *) lfirst(l); + + /* don't print implicit clauses */ + if (rc->pushedDown) + continue; + + switch (rc->strength) + { + case LCS_NONE: + /* we intentionally throw an error for LCS_NONE */ + elog(ERROR, "unrecognized LockClauseStrength %d", + (int) rc->strength); + break; + case LCS_FORKEYSHARE: + appendContextKeyword(context, " FOR KEY SHARE", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); + break; + case LCS_FORSHARE: + appendContextKeyword(context, " FOR SHARE", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); + break; + case LCS_FORNOKEYUPDATE: + appendContextKeyword(context, " FOR NO KEY UPDATE", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); + break; + case LCS_FORUPDATE: + appendContextKeyword(context, " FOR UPDATE", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); + break; + } + + appendStringInfo(buf, " OF %s", + quote_identifier(get_rtable_name(rc->rti, + context))); + if (rc->waitPolicy == LockWaitError) + appendStringInfoString(buf, " NOWAIT"); + else if (rc->waitPolicy == LockWaitSkip) + appendStringInfoString(buf, " SKIP LOCKED"); + } + } + + context->windowClause = save_windowclause; + context->windowTList = save_windowtlist; +} + +/* + * Detect whether query looks like SELECT ... FROM VALUES(); + * if so, return the VALUES RTE. Otherwise return NULL. + */ +static RangeTblEntry * +get_simple_values_rte(Query *query) +{ + RangeTblEntry *result = NULL; + ListCell *lc; + + /* + * We want to return true even if the Query also contains OLD or NEW rule + * RTEs. So the idea is to scan the rtable and see if there is only one + * inFromCl RTE that is a VALUES RTE. + */ + foreach(lc, query->rtable) + { + RangeTblEntry *rte = (RangeTblEntry *) lfirst(lc); + + if (rte->rtekind == RTE_VALUES && rte->inFromCl) + { + if (result) + return NULL; /* multiple VALUES (probably not possible) */ + result = rte; + } + else if (rte->rtekind == RTE_RELATION && !rte->inFromCl) + continue; /* ignore rule entries */ + else + return NULL; /* something else -> not simple VALUES */ + } + + /* + * We don't need to check the targetlist in any great detail, because + * parser/analyze.c will never generate a "bare" VALUES RTE --- they only + * appear inside auto-generated sub-queries with very restricted + * structure. However, DefineView might have modified the tlist by + * injecting new column aliases; so compare tlist resnames against the + * RTE's names to detect that. + */ + if (result) + { + ListCell *lcn; + + if (list_length(query->targetList) != list_length(result->eref->colnames)) + return NULL; /* this probably cannot happen */ + forboth(lc, query->targetList, lcn, result->eref->colnames) + { + TargetEntry *tle = (TargetEntry *) lfirst(lc); + char *cname = strVal(lfirst(lcn)); + + if (tle->resjunk) + return NULL; /* this probably cannot happen */ + if (tle->resname == NULL || strcmp(tle->resname, cname) != 0) + return NULL; /* column name has been changed */ + } + } + + return result; +} + +static void +get_basic_select_query(Query *query, deparse_context *context, + TupleDesc resultDesc) +{ + StringInfo buf = context->buf; + RangeTblEntry *values_rte; + char *sep; + ListCell *l; + + if (PRETTY_INDENT(context)) + { + context->indentLevel += PRETTYINDENT_STD; + appendStringInfoChar(buf, ' '); + } + + /* + * If the query looks like SELECT * FROM (VALUES ...), then print just the + * VALUES part. This reverses what transformValuesClause() did at parse + * time. + */ + values_rte = get_simple_values_rte(query); + if (values_rte) + { + get_values_def(values_rte->values_lists, context); + return; + } + + /* + * Build up the query string - first we say SELECT + */ + appendStringInfoString(buf, "SELECT"); + + /* Add the DISTINCT clause if given */ + if (query->distinctClause != NIL) + { + if (query->hasDistinctOn) + { + appendStringInfoString(buf, " DISTINCT ON ("); + sep = ""; + foreach(l, query->distinctClause) + { + SortGroupClause *srt = (SortGroupClause *) lfirst(l); + + appendStringInfoString(buf, sep); + get_rule_sortgroupclause(srt->tleSortGroupRef, query->targetList, + false, context); + sep = ", "; + } + appendStringInfoChar(buf, ')'); + } + else + appendStringInfoString(buf, " DISTINCT"); + } + + /* Then we tell what to select (the targetlist) */ + get_target_list(query->targetList, context, resultDesc); + + /* Add the FROM clause if needed */ + get_from_clause(query, " FROM ", context); + + /* Add the WHERE clause if given */ + if (query->jointree->quals != NULL) + { + appendContextKeyword(context, " WHERE ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_rule_expr(query->jointree->quals, context, false); + } + + /* Add the GROUP BY clause if given */ + if (query->groupClause != NULL || query->groupingSets != NULL) + { + ParseExprKind save_exprkind; + + appendContextKeyword(context, " GROUP BY ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + + save_exprkind = context->special_exprkind; + context->special_exprkind = EXPR_KIND_GROUP_BY; + + if (query->groupingSets == NIL) + { + sep = ""; + foreach(l, query->groupClause) + { + SortGroupClause *grp = (SortGroupClause *) lfirst(l); + + appendStringInfoString(buf, sep); + get_rule_sortgroupclause(grp->tleSortGroupRef, query->targetList, + false, context); + sep = ", "; + } + } + else + { + sep = ""; + foreach(l, query->groupingSets) + { + GroupingSet *grp = lfirst(l); + + appendStringInfoString(buf, sep); + get_rule_groupingset(grp, query->targetList, true, context); + sep = ", "; + } + } + + context->special_exprkind = save_exprkind; + } + + /* Add the HAVING clause if given */ + if (query->havingQual != NULL) + { + appendContextKeyword(context, " HAVING ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 0); + get_rule_expr(query->havingQual, context, false); + } + + /* Add the WINDOW clause if needed */ + if (query->windowClause != NIL) + get_rule_windowclause(query, context); +} + +/* ---------- + * get_target_list - Parse back a SELECT target list + * + * This is also used for RETURNING lists in INSERT/UPDATE/DELETE. + * ---------- + */ +static void +get_target_list(List *targetList, deparse_context *context, + TupleDesc resultDesc) +{ + StringInfo buf = context->buf; + StringInfoData targetbuf; + bool last_was_multiline = false; + char *sep; + int colno; + ListCell *l; + + /* we use targetbuf to hold each TLE's text temporarily */ + initStringInfo(&targetbuf); + + sep = " "; + colno = 0; + foreach(l, targetList) + { + TargetEntry *tle = (TargetEntry *) lfirst(l); + char *colname; + char *attname; + + if (tle->resjunk) + continue; /* ignore junk entries */ + + appendStringInfoString(buf, sep); + sep = ", "; + colno++; + + /* + * Put the new field text into targetbuf so we can decide after we've + * got it whether or not it needs to go on a new line. + */ + resetStringInfo(&targetbuf); + context->buf = &targetbuf; + + /* + * We special-case Var nodes rather than using get_rule_expr. This is + * needed because get_rule_expr will display a whole-row Var as + * "foo.*", which is the preferred notation in most contexts, but at + * the top level of a SELECT list it's not right (the parser will + * expand that notation into multiple columns, yielding behavior + * different from a whole-row Var). We need to call get_variable + * directly so that we can tell it to do the right thing, and so that + * we can get the attribute name which is the default AS label. + */ + if (tle->expr && (IsA(tle->expr, Var))) + { + attname = get_variable((Var *) tle->expr, 0, true, context); + } + else + { + get_rule_expr((Node *) tle->expr, context, true); + /* We'll show the AS name unless it's this: */ + attname = "?column?"; + } + + /* + * Figure out what the result column should be called. In the context + * of a view, use the view's tuple descriptor (so as to pick up the + * effects of any column RENAME that's been done on the view). + * Otherwise, just use what we can find in the TLE. + */ + if (resultDesc && colno <= resultDesc->natts) + colname = NameStr(TupleDescAttr(resultDesc, colno - 1)->attname); + else + colname = tle->resname; + + /* Show AS unless the column's name is correct as-is */ + if (colname) /* resname could be NULL */ + { + if (attname == NULL || strcmp(attname, colname) != 0) + appendStringInfo(&targetbuf, " AS %s", quote_identifier(colname)); + } + + /* Restore context's output buffer */ + context->buf = buf; + + /* Consider line-wrapping if enabled */ + if (PRETTY_INDENT(context) && context->wrapColumn >= 0) + { + int leading_nl_pos; + + /* Does the new field start with a new line? */ + if (targetbuf.len > 0 && targetbuf.data[0] == '\n') + leading_nl_pos = 0; + else + leading_nl_pos = -1; + + /* If so, we shouldn't add anything */ + if (leading_nl_pos >= 0) + { + /* instead, remove any trailing spaces currently in buf */ + removeStringInfoSpaces(buf); + } + else + { + char *trailing_nl; + + /* Locate the start of the current line in the output buffer */ + trailing_nl = strrchr(buf->data, '\n'); + if (trailing_nl == NULL) + trailing_nl = buf->data; + else + trailing_nl++; + + /* + * Add a newline, plus some indentation, if the new field is + * not the first and either the new field would cause an + * overflow or the last field used more than one line. + */ + if (colno > 1 && + ((strlen(trailing_nl) + targetbuf.len > context->wrapColumn) || + last_was_multiline)) + appendContextKeyword(context, "", -PRETTYINDENT_STD, + PRETTYINDENT_STD, PRETTYINDENT_VAR); + } + + /* Remember this field's multiline status for next iteration */ + last_was_multiline = + (strchr(targetbuf.data + leading_nl_pos + 1, '\n') != NULL); + } + + /* Add the new field */ + appendStringInfoString(buf, targetbuf.data); + } + + /* clean up */ + pfree(targetbuf.data); +} + +static void +get_setop_query(Node *setOp, Query *query, deparse_context *context, + TupleDesc resultDesc) +{ + StringInfo buf = context->buf; + bool need_paren; + + /* Guard against excessively long or deeply-nested queries */ + CHECK_FOR_INTERRUPTS(); + check_stack_depth(); + + if (IsA(setOp, RangeTblRef)) + { + RangeTblRef *rtr = (RangeTblRef *) setOp; + RangeTblEntry *rte = rt_fetch(rtr->rtindex, query->rtable); + Query *subquery = rte->subquery; + + Assert(subquery != NULL); + Assert(subquery->setOperations == NULL); + /* Need parens if WITH, ORDER BY, FOR UPDATE, or LIMIT; see gram.y */ + need_paren = (subquery->cteList || + subquery->sortClause || + subquery->rowMarks || + subquery->limitOffset || + subquery->limitCount); + if (need_paren) + appendStringInfoChar(buf, '('); + get_query_def(subquery, buf, context->namespaces, resultDesc, + context->prettyFlags, context->wrapColumn, + context->indentLevel); + if (need_paren) + appendStringInfoChar(buf, ')'); + } + else if (IsA(setOp, SetOperationStmt)) + { + SetOperationStmt *op = (SetOperationStmt *) setOp; + int subindent; + + /* + * We force parens when nesting two SetOperationStmts, except when the + * lefthand input is another setop of the same kind. Syntactically, + * we could omit parens in rather more cases, but it seems best to use + * parens to flag cases where the setop operator changes. If we use + * parens, we also increase the indentation level for the child query. + * + * There are some cases in which parens are needed around a leaf query + * too, but those are more easily handled at the next level down (see + * code above). + */ + if (IsA(op->larg, SetOperationStmt)) + { + SetOperationStmt *lop = (SetOperationStmt *) op->larg; + + if (op->op == lop->op && op->all == lop->all) + need_paren = false; + else + need_paren = true; + } + else + need_paren = false; + + if (need_paren) + { + appendStringInfoChar(buf, '('); + subindent = PRETTYINDENT_STD; + appendContextKeyword(context, "", subindent, 0, 0); + } + else + subindent = 0; + + get_setop_query(op->larg, query, context, resultDesc); + + if (need_paren) + appendContextKeyword(context, ") ", -subindent, 0, 0); + else if (PRETTY_INDENT(context)) + appendContextKeyword(context, "", -subindent, 0, 0); + else + appendStringInfoChar(buf, ' '); + + switch (op->op) + { + case SETOP_UNION: + appendStringInfoString(buf, "UNION "); + break; + case SETOP_INTERSECT: + appendStringInfoString(buf, "INTERSECT "); + break; + case SETOP_EXCEPT: + appendStringInfoString(buf, "EXCEPT "); + break; + default: + elog(ERROR, "unrecognized set op: %d", + (int) op->op); + } + if (op->all) + appendStringInfoString(buf, "ALL "); + + /* Always parenthesize if RHS is another setop */ + need_paren = IsA(op->rarg, SetOperationStmt); + + /* + * The indentation code here is deliberately a bit different from that + * for the lefthand input, because we want the line breaks in + * different places. + */ + if (need_paren) + { + appendStringInfoChar(buf, '('); + subindent = PRETTYINDENT_STD; + } + else + subindent = 0; + appendContextKeyword(context, "", subindent, 0, 0); + + get_setop_query(op->rarg, query, context, resultDesc); + + if (PRETTY_INDENT(context)) + context->indentLevel -= subindent; + if (need_paren) + appendContextKeyword(context, ")", 0, 0, 0); + } + else + { + elog(ERROR, "unrecognized node type: %d", + (int) nodeTag(setOp)); + } +} + +/* + * Display a sort/group clause. + * + * Also returns the expression tree, so caller need not find it again. + */ +static Node * +get_rule_sortgroupclause(Index ref, List *tlist, bool force_colno, + deparse_context *context) +{ + StringInfo buf = context->buf; + TargetEntry *tle; + Node *expr; + + tle = get_sortgroupref_tle(ref, tlist); + expr = (Node *) tle->expr; + + /* + * Use column-number form if requested by caller. Otherwise, if + * expression is a constant, force it to be dumped with an explicit cast + * as decoration --- this is because a simple integer constant is + * ambiguous (and will be misinterpreted by findTargetlistEntry()) if we + * dump it without any decoration. If it's anything more complex than a + * simple Var, then force extra parens around it, to ensure it can't be + * misinterpreted as a cube() or rollup() construct. + */ + if (force_colno) + { + Assert(!tle->resjunk); + appendStringInfo(buf, "%d", tle->resno); + } + else if (expr && IsA(expr, Const)) + get_const_expr((Const *) expr, context, 1); + else if (!expr || IsA(expr, Var)) + get_rule_expr(expr, context, true); + else + { + /* + * We must force parens for function-like expressions even if + * PRETTY_PAREN is off, since those are the ones in danger of + * misparsing. For other expressions we need to force them only if + * PRETTY_PAREN is on, since otherwise the expression will output them + * itself. (We can't skip the parens.) + */ + bool need_paren = (PRETTY_PAREN(context) + || IsA(expr, FuncExpr) + ||IsA(expr, Aggref) + ||IsA(expr, WindowFunc)); + + if (need_paren) + appendStringInfoChar(context->buf, '('); + get_rule_expr(expr, context, true); + if (need_paren) + appendStringInfoChar(context->buf, ')'); + } + + return expr; +} + +/* + * Display a GroupingSet + */ +static void +get_rule_groupingset(GroupingSet *gset, List *targetlist, + bool omit_parens, deparse_context *context) +{ + ListCell *l; + StringInfo buf = context->buf; + bool omit_child_parens = true; + char *sep = ""; + + switch (gset->kind) + { + case GROUPING_SET_EMPTY: + appendStringInfoString(buf, "()"); + return; + + case GROUPING_SET_SIMPLE: + { + if (!omit_parens || list_length(gset->content) != 1) + appendStringInfoChar(buf, '('); + + foreach(l, gset->content) + { + Index ref = lfirst_int(l); + + appendStringInfoString(buf, sep); + get_rule_sortgroupclause(ref, targetlist, + false, context); + sep = ", "; + } + + if (!omit_parens || list_length(gset->content) != 1) + appendStringInfoChar(buf, ')'); + } + return; + + case GROUPING_SET_ROLLUP: + appendStringInfoString(buf, "ROLLUP("); + break; + case GROUPING_SET_CUBE: + appendStringInfoString(buf, "CUBE("); + break; + case GROUPING_SET_SETS: + appendStringInfoString(buf, "GROUPING SETS ("); + omit_child_parens = false; + break; + } + + foreach(l, gset->content) + { + appendStringInfoString(buf, sep); + get_rule_groupingset(lfirst(l), targetlist, omit_child_parens, context); + sep = ", "; + } + + appendStringInfoChar(buf, ')'); +} + +/* + * Display an ORDER BY list. + */ +static void +get_rule_orderby(List *orderList, List *targetList, + bool force_colno, deparse_context *context) +{ + StringInfo buf = context->buf; + const char *sep; + ListCell *l; + + sep = ""; + foreach(l, orderList) + { + SortGroupClause *srt = (SortGroupClause *) lfirst(l); + Node *sortexpr; + Oid sortcoltype; + TypeCacheEntry *typentry; + + appendStringInfoString(buf, sep); + sortexpr = get_rule_sortgroupclause(srt->tleSortGroupRef, targetList, + force_colno, context); + sortcoltype = exprType(sortexpr); + /* See whether operator is default < or > for datatype */ + typentry = lookup_type_cache(sortcoltype, + TYPECACHE_LT_OPR | TYPECACHE_GT_OPR); + if (srt->sortop == typentry->lt_opr) + { + /* ASC is default, so emit nothing for it */ + if (srt->nulls_first) + appendStringInfoString(buf, " NULLS FIRST"); + } + else if (srt->sortop == typentry->gt_opr) + { + appendStringInfoString(buf, " DESC"); + /* DESC defaults to NULLS FIRST */ + if (!srt->nulls_first) + appendStringInfoString(buf, " NULLS LAST"); + } + else + { + appendStringInfo(buf, " USING %s", + generate_operator_name(srt->sortop, + sortcoltype, + sortcoltype)); + /* be specific to eliminate ambiguity */ + if (srt->nulls_first) + appendStringInfoString(buf, " NULLS FIRST"); + else + appendStringInfoString(buf, " NULLS LAST"); + } + sep = ", "; + } +} + +/* + * Display a WINDOW clause. + * + * Note that the windowClause list might contain only anonymous window + * specifications, in which case we should print nothing here. + */ +static void +get_rule_windowclause(Query *query, deparse_context *context) +{ + StringInfo buf = context->buf; + const char *sep; + ListCell *l; + + sep = NULL; + foreach(l, query->windowClause) + { + WindowClause *wc = (WindowClause *) lfirst(l); + + if (wc->name == NULL) + continue; /* ignore anonymous windows */ + + if (sep == NULL) + appendContextKeyword(context, " WINDOW ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + else + appendStringInfoString(buf, sep); + + appendStringInfo(buf, "%s AS ", quote_identifier(wc->name)); + + get_rule_windowspec(wc, query->targetList, context); + + sep = ", "; + } +} + +/* + * Display a window definition + */ +static void +get_rule_windowspec(WindowClause *wc, List *targetList, + deparse_context *context) +{ + StringInfo buf = context->buf; + bool needspace = false; + const char *sep; + ListCell *l; + + appendStringInfoChar(buf, '('); + if (wc->refname) + { + appendStringInfoString(buf, quote_identifier(wc->refname)); + needspace = true; + } + /* partition clauses are always inherited, so only print if no refname */ + if (wc->partitionClause && !wc->refname) + { + if (needspace) + appendStringInfoChar(buf, ' '); + appendStringInfoString(buf, "PARTITION BY "); + sep = ""; + foreach(l, wc->partitionClause) + { + SortGroupClause *grp = (SortGroupClause *) lfirst(l); + + appendStringInfoString(buf, sep); + get_rule_sortgroupclause(grp->tleSortGroupRef, targetList, + false, context); + sep = ", "; + } + needspace = true; + } + /* print ordering clause only if not inherited */ + if (wc->orderClause && !wc->copiedOrder) + { + if (needspace) + appendStringInfoChar(buf, ' '); + appendStringInfoString(buf, "ORDER BY "); + get_rule_orderby(wc->orderClause, targetList, false, context); + needspace = true; + } + /* framing clause is never inherited, so print unless it's default */ + if (wc->frameOptions & FRAMEOPTION_NONDEFAULT) + { + if (needspace) + appendStringInfoChar(buf, ' '); + if (wc->frameOptions & FRAMEOPTION_RANGE) + appendStringInfoString(buf, "RANGE "); + else if (wc->frameOptions & FRAMEOPTION_ROWS) + appendStringInfoString(buf, "ROWS "); + else if (wc->frameOptions & FRAMEOPTION_GROUPS) + appendStringInfoString(buf, "GROUPS "); + else + Assert(false); + if (wc->frameOptions & FRAMEOPTION_BETWEEN) + appendStringInfoString(buf, "BETWEEN "); + if (wc->frameOptions & FRAMEOPTION_START_UNBOUNDED_PRECEDING) + appendStringInfoString(buf, "UNBOUNDED PRECEDING "); + else if (wc->frameOptions & FRAMEOPTION_START_CURRENT_ROW) + appendStringInfoString(buf, "CURRENT ROW "); + else if (wc->frameOptions & FRAMEOPTION_START_OFFSET) + { + get_rule_expr(wc->startOffset, context, false); + if (wc->frameOptions & FRAMEOPTION_START_OFFSET_PRECEDING) + appendStringInfoString(buf, " PRECEDING "); + else if (wc->frameOptions & FRAMEOPTION_START_OFFSET_FOLLOWING) + appendStringInfoString(buf, " FOLLOWING "); + else + Assert(false); + } + else + Assert(false); + if (wc->frameOptions & FRAMEOPTION_BETWEEN) + { + appendStringInfoString(buf, "AND "); + if (wc->frameOptions & FRAMEOPTION_END_UNBOUNDED_FOLLOWING) + appendStringInfoString(buf, "UNBOUNDED FOLLOWING "); + else if (wc->frameOptions & FRAMEOPTION_END_CURRENT_ROW) + appendStringInfoString(buf, "CURRENT ROW "); + else if (wc->frameOptions & FRAMEOPTION_END_OFFSET) + { + get_rule_expr(wc->endOffset, context, false); + if (wc->frameOptions & FRAMEOPTION_END_OFFSET_PRECEDING) + appendStringInfoString(buf, " PRECEDING "); + else if (wc->frameOptions & FRAMEOPTION_END_OFFSET_FOLLOWING) + appendStringInfoString(buf, " FOLLOWING "); + else + Assert(false); + } + else + Assert(false); + } + if (wc->frameOptions & FRAMEOPTION_EXCLUDE_CURRENT_ROW) + appendStringInfoString(buf, "EXCLUDE CURRENT ROW "); + else if (wc->frameOptions & FRAMEOPTION_EXCLUDE_GROUP) + appendStringInfoString(buf, "EXCLUDE GROUP "); + else if (wc->frameOptions & FRAMEOPTION_EXCLUDE_TIES) + appendStringInfoString(buf, "EXCLUDE TIES "); + /* we will now have a trailing space; remove it */ + buf->len--; + } + appendStringInfoChar(buf, ')'); +} + +/* ---------- + * get_insert_query_def - Parse back an INSERT parsetree + * ---------- + */ +static void +get_insert_query_def(Query *query, deparse_context *context) +{ + StringInfo buf = context->buf; + RangeTblEntry *select_rte = NULL; + RangeTblEntry *values_rte = NULL; + RangeTblEntry *rte; + char *sep; + ListCell *l; + List *strippedexprs; + + /* Insert the WITH clause if given */ + get_with_clause(query, context); + + /* + * If it's an INSERT ... SELECT or multi-row VALUES, there will be a + * single RTE for the SELECT or VALUES. Plain VALUES has neither. + */ + foreach(l, query->rtable) + { + rte = (RangeTblEntry *) lfirst(l); + + if (rte->rtekind == RTE_SUBQUERY) + { + if (select_rte) + elog(ERROR, "too many subquery RTEs in INSERT"); + select_rte = rte; + } + + if (rte->rtekind == RTE_VALUES) + { + if (values_rte) + elog(ERROR, "too many values RTEs in INSERT"); + values_rte = rte; + } + } + if (select_rte && values_rte) + elog(ERROR, "both subquery and values RTEs in INSERT"); + + /* + * Start the query with INSERT INTO relname + */ + rte = rt_fetch(query->resultRelation, query->rtable); + Assert(rte->rtekind == RTE_RELATION); + + if (PRETTY_INDENT(context)) + { + context->indentLevel += PRETTYINDENT_STD; + appendStringInfoChar(buf, ' '); + } + appendStringInfo(buf, "INSERT INTO %s ", + generate_relation_or_shard_name(rte->relid, + context->distrelid, + context->shardid, NIL)); + /* INSERT requires AS keyword for target alias */ + if (rte->alias != NULL) + appendStringInfo(buf, "AS %s ", + quote_identifier(rte->alias->aliasname)); + + /* + * Add the insert-column-names list. Any indirection decoration needed on + * the column names can be inferred from the top targetlist. + */ + strippedexprs = NIL; + sep = ""; + if (query->targetList) + appendStringInfoChar(buf, '('); + foreach(l, query->targetList) + { + TargetEntry *tle = (TargetEntry *) lfirst(l); + + if (tle->resjunk) + continue; /* ignore junk entries */ + + appendStringInfoString(buf, sep); + sep = ", "; + + /* + * Put out name of target column; look in the catalogs, not at + * tle->resname, since resname will fail to track RENAME. + */ + appendStringInfoString(buf, + quote_identifier(get_attname(rte->relid, + tle->resno, + false))); + + /* + * Print any indirection needed (subfields or subscripts), and strip + * off the top-level nodes representing the indirection assignments. + * Add the stripped expressions to strippedexprs. (If it's a + * single-VALUES statement, the stripped expressions are the VALUES to + * print below. Otherwise they're just Vars and not really + * interesting.) + */ + strippedexprs = lappend(strippedexprs, + processIndirection((Node *) tle->expr, + context)); + } + if (query->targetList) + appendStringInfoString(buf, ") "); + + if (query->override) + { + if (query->override == OVERRIDING_SYSTEM_VALUE) + appendStringInfoString(buf, "OVERRIDING SYSTEM VALUE "); + else if (query->override == OVERRIDING_USER_VALUE) + appendStringInfoString(buf, "OVERRIDING USER VALUE "); + } + + if (select_rte) + { + /* Add the SELECT */ + get_query_def(select_rte->subquery, buf, NIL, NULL, + context->prettyFlags, context->wrapColumn, + context->indentLevel); + } + else if (values_rte) + { + /* Add the multi-VALUES expression lists */ + get_values_def(values_rte->values_lists, context); + } + else if (strippedexprs) + { + /* Add the single-VALUES expression list */ + appendContextKeyword(context, "VALUES (", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 2); + get_rule_expr((Node *) strippedexprs, context, false); + appendStringInfoChar(buf, ')'); + } + else + { + /* No expressions, so it must be DEFAULT VALUES */ + appendStringInfoString(buf, "DEFAULT VALUES"); + } + + /* Add ON CONFLICT if present */ + if (query->onConflict) + { + OnConflictExpr *confl = query->onConflict; + + appendStringInfoString(buf, " ON CONFLICT"); + + if (confl->arbiterElems) + { + /* Add the single-VALUES expression list */ + appendStringInfoChar(buf, '('); + get_rule_expr((Node *) confl->arbiterElems, context, false); + appendStringInfoChar(buf, ')'); + + /* Add a WHERE clause (for partial indexes) if given */ + if (confl->arbiterWhere != NULL) + { + bool save_varprefix; + + /* + * Force non-prefixing of Vars, since parser assumes that they + * belong to target relation. WHERE clause does not use + * InferenceElem, so this is separately required. + */ + save_varprefix = context->varprefix; + context->varprefix = false; + + appendContextKeyword(context, " WHERE ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_rule_expr(confl->arbiterWhere, context, false); + + context->varprefix = save_varprefix; + } + } + else if (OidIsValid(confl->constraint)) + { + char *constraint = get_constraint_name(confl->constraint); + int64 shardId = context->shardid; + + if (shardId > 0) + { + AppendShardIdToName(&constraint, shardId); + } + + if (!constraint) + elog(ERROR, "cache lookup failed for constraint %u", + confl->constraint); + appendStringInfo(buf, " ON CONSTRAINT %s", + quote_identifier(constraint)); + } + + if (confl->action == ONCONFLICT_NOTHING) + { + appendStringInfoString(buf, " DO NOTHING"); + } + else + { + appendStringInfoString(buf, " DO UPDATE SET "); + /* Deparse targetlist */ + get_update_query_targetlist_def(query, confl->onConflictSet, + context, rte); + + /* Add a WHERE clause if given */ + if (confl->onConflictWhere != NULL) + { + appendContextKeyword(context, " WHERE ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_rule_expr(confl->onConflictWhere, context, false); + } + } + } + + /* Add RETURNING if present */ + if (query->returningList) + { + appendContextKeyword(context, " RETURNING", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_target_list(query->returningList, context, NULL); + } +} + + +/* ---------- + * get_update_query_def - Parse back an UPDATE parsetree + * ---------- + */ +static void +get_update_query_def(Query *query, deparse_context *context) +{ + StringInfo buf = context->buf; + RangeTblEntry *rte; + + /* Insert the WITH clause if given */ + get_with_clause(query, context); + + /* + * Start the query with UPDATE relname SET + */ + rte = rt_fetch(query->resultRelation, query->rtable); + + if (PRETTY_INDENT(context)) + { + appendStringInfoChar(buf, ' '); + context->indentLevel += PRETTYINDENT_STD; + } + + /* if it's a shard, do differently */ + if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) + { + char *fragmentSchemaName = NULL; + char *fragmentTableName = NULL; + + ExtractRangeTblExtraData(rte, NULL, &fragmentSchemaName, &fragmentTableName, NULL); + + /* use schema and table name from the remote alias */ + appendStringInfo(buf, "UPDATE %s%s", + only_marker(rte), + generate_fragment_name(fragmentSchemaName, fragmentTableName)); + + if(rte->eref != NULL) + appendStringInfo(buf, " %s", + quote_identifier(rte->eref->aliasname)); + } + else + { + appendStringInfo(buf, "UPDATE %s%s", + only_marker(rte), + generate_relation_or_shard_name(rte->relid, + context->distrelid, + context->shardid, NIL)); + + if (rte->alias != NULL) + appendStringInfo(buf, " %s", + quote_identifier(rte->alias->aliasname)); + } + + appendStringInfoString(buf, " SET "); + + /* Deparse targetlist */ + get_update_query_targetlist_def(query, query->targetList, context, rte); + + /* Add the FROM clause if needed */ + get_from_clause(query, " FROM ", context); + + /* Add a WHERE clause if given */ + if (query->jointree->quals != NULL) + { + appendContextKeyword(context, " WHERE ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_rule_expr(query->jointree->quals, context, false); + } + + /* Add RETURNING if present */ + if (query->returningList) + { + appendContextKeyword(context, " RETURNING", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_target_list(query->returningList, context, NULL); + } +} + + +/* ---------- + * get_update_query_targetlist_def - Parse back an UPDATE targetlist + * ---------- + */ +static void +get_update_query_targetlist_def(Query *query, List *targetList, + deparse_context *context, RangeTblEntry *rte) +{ + StringInfo buf = context->buf; + ListCell *l; + ListCell *next_ma_cell; + int remaining_ma_columns; + const char *sep; + SubLink *cur_ma_sublink; + List *ma_sublinks; + + /* + * Prepare to deal with MULTIEXPR assignments: collect the source SubLinks + * into a list. We expect them to appear, in ID order, in resjunk tlist + * entries. + */ + ma_sublinks = NIL; + if (query->hasSubLinks) /* else there can't be any */ + { + foreach(l, targetList) + { + TargetEntry *tle = (TargetEntry *) lfirst(l); + + if (tle->resjunk && IsA(tle->expr, SubLink)) + { + SubLink *sl = (SubLink *) tle->expr; + + if (sl->subLinkType == MULTIEXPR_SUBLINK) + { + ma_sublinks = lappend(ma_sublinks, sl); + Assert(sl->subLinkId == list_length(ma_sublinks)); + } + } + } + } + next_ma_cell = list_head(ma_sublinks); + cur_ma_sublink = NULL; + remaining_ma_columns = 0; + + /* Add the comma separated list of 'attname = value' */ + sep = ""; + foreach(l, targetList) + { + TargetEntry *tle = (TargetEntry *) lfirst(l); + Node *expr; + + if (tle->resjunk) + continue; /* ignore junk entries */ + + /* Emit separator (OK whether we're in multiassignment or not) */ + appendStringInfoString(buf, sep); + sep = ", "; + + /* + * Check to see if we're starting a multiassignment group: if so, + * output a left paren. + */ + if (next_ma_cell != NULL && cur_ma_sublink == NULL) + { + /* + * We must dig down into the expr to see if it's a PARAM_MULTIEXPR + * Param. That could be buried under FieldStores and + * SubscriptingRefs and CoerceToDomains (cf processIndirection()), + * and underneath those there could be an implicit type coercion. + * Because we would ignore implicit type coercions anyway, we + * don't need to be as careful as processIndirection() is about + * descending past implicit CoerceToDomains. + */ + expr = (Node *) tle->expr; + while (expr) + { + if (IsA(expr, FieldStore)) + { + FieldStore *fstore = (FieldStore *) expr; + + expr = (Node *) linitial(fstore->newvals); + } + else if (IsA(expr, SubscriptingRef)) + { + SubscriptingRef *sbsref = (SubscriptingRef *) expr; + + if (sbsref->refassgnexpr == NULL) + break; + expr = (Node *) sbsref->refassgnexpr; + } + else if (IsA(expr, CoerceToDomain)) + { + CoerceToDomain *cdomain = (CoerceToDomain *) expr; + + if (cdomain->coercionformat != COERCE_IMPLICIT_CAST) + break; + expr = (Node *) cdomain->arg; + } + else + break; + } + expr = strip_implicit_coercions(expr); + + if (expr && IsA(expr, Param) && + ((Param *) expr)->paramkind == PARAM_MULTIEXPR) + { + cur_ma_sublink = (SubLink *) lfirst(next_ma_cell); + next_ma_cell = lnext(next_ma_cell); + remaining_ma_columns = count_nonjunk_tlist_entries( + ((Query *) cur_ma_sublink->subselect)->targetList); + Assert(((Param *) expr)->paramid == + ((cur_ma_sublink->subLinkId << 16) | 1)); + appendStringInfoChar(buf, '('); + } + } + + /* + * Put out name of target column; look in the catalogs, not at + * tle->resname, since resname will fail to track RENAME. + */ + appendStringInfoString(buf, + quote_identifier(get_attname(rte->relid, + tle->resno, + false))); + + /* + * Print any indirection needed (subfields or subscripts), and strip + * off the top-level nodes representing the indirection assignments. + */ + expr = processIndirection((Node *) tle->expr, context); + + /* + * If we're in a multiassignment, skip printing anything more, unless + * this is the last column; in which case, what we print should be the + * sublink, not the Param. + */ + if (cur_ma_sublink != NULL) + { + if (--remaining_ma_columns > 0) + continue; /* not the last column of multiassignment */ + appendStringInfoChar(buf, ')'); + expr = (Node *) cur_ma_sublink; + cur_ma_sublink = NULL; + } + + appendStringInfoString(buf, " = "); + + get_rule_expr(expr, context, false); + } +} + + +/* ---------- + * get_delete_query_def - Parse back a DELETE parsetree + * ---------- + */ +static void +get_delete_query_def(Query *query, deparse_context *context) +{ + StringInfo buf = context->buf; + RangeTblEntry *rte; + + /* Insert the WITH clause if given */ + get_with_clause(query, context); + + /* + * Start the query with DELETE FROM relname + */ + rte = rt_fetch(query->resultRelation, query->rtable); + + if (PRETTY_INDENT(context)) + { + appendStringInfoChar(buf, ' '); + context->indentLevel += PRETTYINDENT_STD; + } + + /* if it's a shard, do differently */ + if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) + { + char *fragmentSchemaName = NULL; + char *fragmentTableName = NULL; + + ExtractRangeTblExtraData(rte, NULL, &fragmentSchemaName, &fragmentTableName, NULL); + + /* use schema and table name from the remote alias */ + appendStringInfo(buf, "DELETE FROM %s%s", + only_marker(rte), + generate_fragment_name(fragmentSchemaName, fragmentTableName)); + + if(rte->eref != NULL) + appendStringInfo(buf, " %s", + quote_identifier(rte->eref->aliasname)); + } + else + { + appendStringInfo(buf, "DELETE FROM %s%s", + only_marker(rte), + generate_relation_or_shard_name(rte->relid, + context->distrelid, + context->shardid, NIL)); + + if (rte->alias != NULL) + appendStringInfo(buf, " %s", + quote_identifier(rte->alias->aliasname)); + } + + /* Add the USING clause if given */ + get_from_clause(query, " USING ", context); + + /* Add a WHERE clause if given */ + if (query->jointree->quals != NULL) + { + appendContextKeyword(context, " WHERE ", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_rule_expr(query->jointree->quals, context, false); + } + + /* Add RETURNING if present */ + if (query->returningList) + { + appendContextKeyword(context, " RETURNING", + -PRETTYINDENT_STD, PRETTYINDENT_STD, 1); + get_target_list(query->returningList, context, NULL); + } +} + + +/* ---------- + * get_utility_query_def - Parse back a UTILITY parsetree + * ---------- + */ +static void +get_utility_query_def(Query *query, deparse_context *context) +{ + StringInfo buf = context->buf; + + if (query->utilityStmt && IsA(query->utilityStmt, NotifyStmt)) + { + NotifyStmt *stmt = (NotifyStmt *) query->utilityStmt; + + appendContextKeyword(context, "", + 0, PRETTYINDENT_STD, 1); + appendStringInfo(buf, "NOTIFY %s", + quote_identifier(stmt->conditionname)); + if (stmt->payload) + { + appendStringInfoString(buf, ", "); + simple_quote_literal(buf, stmt->payload); + } + } + else if (query->utilityStmt && IsA(query->utilityStmt, TruncateStmt)) + { + TruncateStmt *stmt = (TruncateStmt *) query->utilityStmt; + List *relationList = stmt->relations; + ListCell *relationCell = NULL; + + appendContextKeyword(context, "", + 0, PRETTYINDENT_STD, 1); + + appendStringInfo(buf, "TRUNCATE TABLE"); + + foreach(relationCell, relationList) + { + RangeVar *relationVar = (RangeVar *) lfirst(relationCell); + Oid relationId = RangeVarGetRelid(relationVar, NoLock, false); + char *relationName = generate_relation_or_shard_name(relationId, + context->distrelid, + context->shardid, NIL); + appendStringInfo(buf, " %s", relationName); + + if (lnext(relationCell) != NULL) + { + appendStringInfo(buf, ","); + } + } + + if (stmt->restart_seqs) + { + appendStringInfo(buf, " RESTART IDENTITY"); + } + + if (stmt->behavior == DROP_CASCADE) + { + appendStringInfo(buf, " CASCADE"); + } + } + else + { + /* Currently only NOTIFY utility commands can appear in rules */ + elog(ERROR, "unexpected utility statement type"); + } +} + +/* + * Display a Var appropriately. + * + * In some cases (currently only when recursing into an unnamed join) + * the Var's varlevelsup has to be interpreted with respect to a context + * above the current one; levelsup indicates the offset. + * + * If istoplevel is true, the Var is at the top level of a SELECT's + * targetlist, which means we need special treatment of whole-row Vars. + * Instead of the normal "tab.*", we'll print "tab.*::typename", which is a + * dirty hack to prevent "tab.*" from being expanded into multiple columns. + * (The parser will strip the useless coercion, so no inefficiency is added in + * dump and reload.) We used to print just "tab" in such cases, but that is + * ambiguous and will yield the wrong result if "tab" is also a plain column + * name in the query. + * + * Returns the attname of the Var, or NULL if the Var has no attname (because + * it is a whole-row Var or a subplan output reference). + */ +static char * +get_variable(Var *var, int levelsup, bool istoplevel, deparse_context *context) +{ + StringInfo buf = context->buf; + RangeTblEntry *rte; + AttrNumber attnum; + int netlevelsup; + deparse_namespace *dpns; + deparse_columns *colinfo; + char *refname; + char *attname; + + /* Find appropriate nesting depth */ + netlevelsup = var->varlevelsup + levelsup; + if (netlevelsup >= list_length(context->namespaces)) + elog(ERROR, "bogus varlevelsup: %d offset %d", + var->varlevelsup, levelsup); + dpns = (deparse_namespace *) list_nth(context->namespaces, + netlevelsup); + + /* + * Try to find the relevant RTE in this rtable. In a plan tree, it's + * likely that varno is OUTER_VAR or INNER_VAR, in which case we must dig + * down into the subplans, or INDEX_VAR, which is resolved similarly. Also + * find the aliases previously assigned for this RTE. + */ + if (var->varno >= 1 && var->varno <= list_length(dpns->rtable)) + { + rte = rt_fetch(var->varno, dpns->rtable); + refname = (char *) list_nth(dpns->rtable_names, var->varno - 1); + colinfo = deparse_columns_fetch(var->varno, dpns); + attnum = var->varattno; + } + else + { + resolve_special_varno((Node *) var, context, NULL, + get_special_variable); + return NULL; + } + + /* + * The planner will sometimes emit Vars referencing resjunk elements of a + * subquery's target list (this is currently only possible if it chooses + * to generate a "physical tlist" for a SubqueryScan or CteScan node). + * Although we prefer to print subquery-referencing Vars using the + * subquery's alias, that's not possible for resjunk items since they have + * no alias. So in that case, drill down to the subplan and print the + * contents of the referenced tlist item. This works because in a plan + * tree, such Vars can only occur in a SubqueryScan or CteScan node, and + * we'll have set dpns->inner_planstate to reference the child plan node. + */ + if ((rte->rtekind == RTE_SUBQUERY || rte->rtekind == RTE_CTE) && + attnum > list_length(rte->eref->colnames) && + dpns->inner_planstate) + { + TargetEntry *tle; + deparse_namespace save_dpns; + + tle = get_tle_by_resno(dpns->inner_tlist, var->varattno); + if (!tle) + elog(ERROR, "invalid attnum %d for relation \"%s\"", + var->varattno, rte->eref->aliasname); + + Assert(netlevelsup == 0); + push_child_plan(dpns, dpns->inner_planstate, &save_dpns); + + /* + * Force parentheses because our caller probably assumed a Var is a + * simple expression. + */ + if (!IsA(tle->expr, Var)) + appendStringInfoChar(buf, '('); + get_rule_expr((Node *) tle->expr, context, true); + if (!IsA(tle->expr, Var)) + appendStringInfoChar(buf, ')'); + + pop_child_plan(dpns, &save_dpns); + return NULL; + } + + /* + * If it's an unnamed join, look at the expansion of the alias variable. + * If it's a simple reference to one of the input vars, then recursively + * print the name of that var instead. When it's not a simple reference, + * we have to just print the unqualified join column name. (This can only + * happen with "dangerous" merged columns in a JOIN USING; we took pains + * previously to make the unqualified column name unique in such cases.) + * + * This wouldn't work in decompiling plan trees, because we don't store + * joinaliasvars lists after planning; but a plan tree should never + * contain a join alias variable. + */ + if (rte->rtekind == RTE_JOIN && rte->alias == NULL) + { + if (rte->joinaliasvars == NIL) + elog(ERROR, "cannot decompile join alias var in plan tree"); + if (attnum > 0) + { + Var *aliasvar; + + aliasvar = (Var *) list_nth(rte->joinaliasvars, attnum - 1); + /* we intentionally don't strip implicit coercions here */ + if (aliasvar && IsA(aliasvar, Var)) + { + return get_variable(aliasvar, var->varlevelsup + levelsup, + istoplevel, context); + } + } + + /* + * Unnamed join has no refname. (Note: since it's unnamed, there is + * no way the user could have referenced it to create a whole-row Var + * for it. So we don't have to cover that case below.) + */ + Assert(refname == NULL); + } + + if (attnum == InvalidAttrNumber) + attname = NULL; + else if (attnum > 0) + { + /* Get column name to use from the colinfo struct */ + if (attnum > colinfo->num_cols) + elog(ERROR, "invalid attnum %d for relation \"%s\"", + attnum, rte->eref->aliasname); + attname = colinfo->colnames[attnum - 1]; + if (attname == NULL) /* dropped column? */ + elog(ERROR, "invalid attnum %d for relation \"%s\"", + attnum, rte->eref->aliasname); + } + else if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) + { + /* System column on a Citus shard */ + attname = get_attname(rte->relid, attnum, false); + } + else + { + /* System column - name is fixed, get it from the catalog */ + attname = get_rte_attribute_name(rte, attnum); + } + + if (refname && (context->varprefix || attname == NULL)) + { + appendStringInfoString(buf, quote_identifier(refname)); + appendStringInfoChar(buf, '.'); + } + if (attname) + appendStringInfoString(buf, quote_identifier(attname)); + else + { + appendStringInfoChar(buf, '*'); + if (istoplevel) + appendStringInfo(buf, "::%s", + format_type_with_typemod(var->vartype, + var->vartypmod)); + } + + return attname; +} + +/* + * Deparse a Var which references OUTER_VAR, INNER_VAR, or INDEX_VAR. This + * routine is actually a callback for get_special_varno, which handles finding + * the correct TargetEntry. We get the expression contained in that + * TargetEntry and just need to deparse it, a job we can throw back on + * get_rule_expr. + */ +static void +get_special_variable(Node *node, deparse_context *context, void *private) +{ + StringInfo buf = context->buf; + + /* + * Force parentheses because our caller probably assumed a Var is a simple + * expression. + */ + if (!IsA(node, Var)) + appendStringInfoChar(buf, '('); + get_rule_expr(node, context, true); + if (!IsA(node, Var)) + appendStringInfoChar(buf, ')'); +} + +/* + * Chase through plan references to special varnos (OUTER_VAR, INNER_VAR, + * INDEX_VAR) until we find a real Var or some kind of non-Var node; then, + * invoke the callback provided. + */ +static void +resolve_special_varno(Node *node, deparse_context *context, void *private, + void (*callback) (Node *, deparse_context *, void *)) +{ + Var *var; + deparse_namespace *dpns; + + /* If it's not a Var, invoke the callback. */ + if (!IsA(node, Var)) + { + callback(node, context, private); + return; + } + + /* Find appropriate nesting depth */ + var = (Var *) node; + dpns = (deparse_namespace *) list_nth(context->namespaces, + var->varlevelsup); + + /* + * It's a special RTE, so recurse. + */ + if (var->varno == OUTER_VAR && dpns->outer_tlist) + { + TargetEntry *tle; + deparse_namespace save_dpns; + + tle = get_tle_by_resno(dpns->outer_tlist, var->varattno); + if (!tle) + elog(ERROR, "bogus varattno for OUTER_VAR var: %d", var->varattno); + + push_child_plan(dpns, dpns->outer_planstate, &save_dpns); + resolve_special_varno((Node *) tle->expr, context, private, callback); + pop_child_plan(dpns, &save_dpns); + return; + } + else if (var->varno == INNER_VAR && dpns->inner_tlist) + { + TargetEntry *tle; + deparse_namespace save_dpns; + + tle = get_tle_by_resno(dpns->inner_tlist, var->varattno); + if (!tle) + elog(ERROR, "bogus varattno for INNER_VAR var: %d", var->varattno); + + push_child_plan(dpns, dpns->inner_planstate, &save_dpns); + resolve_special_varno((Node *) tle->expr, context, private, callback); + pop_child_plan(dpns, &save_dpns); + return; + } + else if (var->varno == INDEX_VAR && dpns->index_tlist) + { + TargetEntry *tle; + + tle = get_tle_by_resno(dpns->index_tlist, var->varattno); + if (!tle) + elog(ERROR, "bogus varattno for INDEX_VAR var: %d", var->varattno); + + resolve_special_varno((Node *) tle->expr, context, private, callback); + return; + } + else if (var->varno < 1 || var->varno > list_length(dpns->rtable)) + elog(ERROR, "bogus varno: %d", var->varno); + + /* Not special. Just invoke the callback. */ + callback(node, context, private); +} + +/* + * Get the name of a field of an expression of composite type. The + * expression is usually a Var, but we handle other cases too. + * + * levelsup is an extra offset to interpret the Var's varlevelsup correctly. + * + * This is fairly straightforward when the expression has a named composite + * type; we need only look up the type in the catalogs. However, the type + * could also be RECORD. Since no actual table or view column is allowed to + * have type RECORD, a Var of type RECORD must refer to a JOIN or FUNCTION RTE + * or to a subquery output. We drill down to find the ultimate defining + * expression and attempt to infer the field name from it. We ereport if we + * can't determine the name. + * + * Similarly, a PARAM of type RECORD has to refer to some expression of + * a determinable composite type. + */ +static const char * +get_name_for_var_field(Var *var, int fieldno, + int levelsup, deparse_context *context) +{ + RangeTblEntry *rte; + AttrNumber attnum; + int netlevelsup; + deparse_namespace *dpns; + TupleDesc tupleDesc; + Node *expr; + + /* + * If it's a RowExpr that was expanded from a whole-row Var, use the + * column names attached to it. + */ + if (IsA(var, RowExpr)) + { + RowExpr *r = (RowExpr *) var; + + if (fieldno > 0 && fieldno <= list_length(r->colnames)) + return strVal(list_nth(r->colnames, fieldno - 1)); + } + + /* + * If it's a Param of type RECORD, try to find what the Param refers to. + */ + if (IsA(var, Param)) + { + Param *param = (Param *) var; + ListCell *ancestor_cell; + + expr = find_param_referent(param, context, &dpns, &ancestor_cell); + if (expr) + { + /* Found a match, so recurse to decipher the field name */ + deparse_namespace save_dpns; + const char *result; + + push_ancestor_plan(dpns, ancestor_cell, &save_dpns); + result = get_name_for_var_field((Var *) expr, fieldno, + 0, context); + pop_ancestor_plan(dpns, &save_dpns); + return result; + } + } + + /* + * If it's a Var of type RECORD, we have to find what the Var refers to; + * if not, we can use get_expr_result_tupdesc(). + */ + if (!IsA(var, Var) || + var->vartype != RECORDOID) + { + tupleDesc = get_expr_result_tupdesc((Node *) var, false); + /* Got the tupdesc, so we can extract the field name */ + Assert(fieldno >= 1 && fieldno <= tupleDesc->natts); + return NameStr(TupleDescAttr(tupleDesc, fieldno - 1)->attname); + } + + /* Find appropriate nesting depth */ + netlevelsup = var->varlevelsup + levelsup; + if (netlevelsup >= list_length(context->namespaces)) + elog(ERROR, "bogus varlevelsup: %d offset %d", + var->varlevelsup, levelsup); + dpns = (deparse_namespace *) list_nth(context->namespaces, + netlevelsup); + + /* + * Try to find the relevant RTE in this rtable. In a plan tree, it's + * likely that varno is OUTER_VAR or INNER_VAR, in which case we must dig + * down into the subplans, or INDEX_VAR, which is resolved similarly. + */ + if (var->varno >= 1 && var->varno <= list_length(dpns->rtable)) + { + rte = rt_fetch(var->varno, dpns->rtable); + attnum = var->varattno; + } + else if (var->varno == OUTER_VAR && dpns->outer_tlist) + { + TargetEntry *tle; + deparse_namespace save_dpns; + const char *result; + + tle = get_tle_by_resno(dpns->outer_tlist, var->varattno); + if (!tle) + elog(ERROR, "bogus varattno for OUTER_VAR var: %d", var->varattno); + + Assert(netlevelsup == 0); + push_child_plan(dpns, dpns->outer_planstate, &save_dpns); + + result = get_name_for_var_field((Var *) tle->expr, fieldno, + levelsup, context); + + pop_child_plan(dpns, &save_dpns); + return result; + } + else if (var->varno == INNER_VAR && dpns->inner_tlist) + { + TargetEntry *tle; + deparse_namespace save_dpns; + const char *result; + + tle = get_tle_by_resno(dpns->inner_tlist, var->varattno); + if (!tle) + elog(ERROR, "bogus varattno for INNER_VAR var: %d", var->varattno); + + Assert(netlevelsup == 0); + push_child_plan(dpns, dpns->inner_planstate, &save_dpns); + + result = get_name_for_var_field((Var *) tle->expr, fieldno, + levelsup, context); + + pop_child_plan(dpns, &save_dpns); + return result; + } + else if (var->varno == INDEX_VAR && dpns->index_tlist) + { + TargetEntry *tle; + const char *result; + + tle = get_tle_by_resno(dpns->index_tlist, var->varattno); + if (!tle) + elog(ERROR, "bogus varattno for INDEX_VAR var: %d", var->varattno); + + Assert(netlevelsup == 0); + + result = get_name_for_var_field((Var *) tle->expr, fieldno, + levelsup, context); + + return result; + } + else + { + elog(ERROR, "bogus varno: %d", var->varno); + return NULL; /* keep compiler quiet */ + } + + if (attnum == InvalidAttrNumber) + { + /* Var is whole-row reference to RTE, so select the right field */ + return get_rte_attribute_name(rte, fieldno); + } + + /* + * This part has essentially the same logic as the parser's + * expandRecordVariable() function, but we are dealing with a different + * representation of the input context, and we only need one field name + * not a TupleDesc. Also, we need special cases for finding subquery and + * CTE subplans when deparsing Plan trees. + */ + expr = (Node *) var; /* default if we can't drill down */ + + switch (rte->rtekind) + { + case RTE_RELATION: + case RTE_VALUES: + case RTE_NAMEDTUPLESTORE: + case RTE_RESULT: + + /* + * This case should not occur: a column of a table or values list + * shouldn't have type RECORD. Fall through and fail (most + * likely) at the bottom. + */ + break; + case RTE_SUBQUERY: + /* Subselect-in-FROM: examine sub-select's output expr */ + { + if (rte->subquery) + { + TargetEntry *ste = get_tle_by_resno(rte->subquery->targetList, + attnum); + + if (ste == NULL || ste->resjunk) + elog(ERROR, "subquery %s does not have attribute %d", + rte->eref->aliasname, attnum); + expr = (Node *) ste->expr; + if (IsA(expr, Var)) + { + /* + * Recurse into the sub-select to see what its Var + * refers to. We have to build an additional level of + * namespace to keep in step with varlevelsup in the + * subselect. + */ + deparse_namespace mydpns; + const char *result; + + set_deparse_for_query(&mydpns, rte->subquery, + context->namespaces); + + context->namespaces = lcons(&mydpns, + context->namespaces); + + result = get_name_for_var_field((Var *) expr, fieldno, + 0, context); + + context->namespaces = + list_delete_first(context->namespaces); + + return result; + } + /* else fall through to inspect the expression */ + } + else + { + /* + * We're deparsing a Plan tree so we don't have complete + * RTE entries (in particular, rte->subquery is NULL). But + * the only place we'd see a Var directly referencing a + * SUBQUERY RTE is in a SubqueryScan plan node, and we can + * look into the child plan's tlist instead. + */ + TargetEntry *tle; + deparse_namespace save_dpns; + const char *result; + + if (!dpns->inner_planstate) + elog(ERROR, "failed to find plan for subquery %s", + rte->eref->aliasname); + tle = get_tle_by_resno(dpns->inner_tlist, attnum); + if (!tle) + elog(ERROR, "bogus varattno for subquery var: %d", + attnum); + Assert(netlevelsup == 0); + push_child_plan(dpns, dpns->inner_planstate, &save_dpns); + + result = get_name_for_var_field((Var *) tle->expr, fieldno, + levelsup, context); + + pop_child_plan(dpns, &save_dpns); + return result; + } + } + break; + case RTE_JOIN: + /* Join RTE --- recursively inspect the alias variable */ + if (rte->joinaliasvars == NIL) + elog(ERROR, "cannot decompile join alias var in plan tree"); + Assert(attnum > 0 && attnum <= list_length(rte->joinaliasvars)); + expr = (Node *) list_nth(rte->joinaliasvars, attnum - 1); + Assert(expr != NULL); + /* we intentionally don't strip implicit coercions here */ + if (IsA(expr, Var)) + return get_name_for_var_field((Var *) expr, fieldno, + var->varlevelsup + levelsup, + context); + /* else fall through to inspect the expression */ + break; + case RTE_FUNCTION: + case RTE_TABLEFUNC: + + /* + * We couldn't get here unless a function is declared with one of + * its result columns as RECORD, which is not allowed. + */ + break; + case RTE_CTE: + /* CTE reference: examine subquery's output expr */ + { + CommonTableExpr *cte = NULL; + Index ctelevelsup; + ListCell *lc; + + /* + * Try to find the referenced CTE using the namespace stack. + */ + ctelevelsup = rte->ctelevelsup + netlevelsup; + if (ctelevelsup >= list_length(context->namespaces)) + lc = NULL; + else + { + deparse_namespace *ctedpns; + + ctedpns = (deparse_namespace *) + list_nth(context->namespaces, ctelevelsup); + foreach(lc, ctedpns->ctes) + { + cte = (CommonTableExpr *) lfirst(lc); + if (strcmp(cte->ctename, rte->ctename) == 0) + break; + } + } + if (lc != NULL) + { + Query *ctequery = (Query *) cte->ctequery; + TargetEntry *ste = get_tle_by_resno(GetCTETargetList(cte), + attnum); + + if (ste == NULL || ste->resjunk) + elog(ERROR, "subquery %s does not have attribute %d", + rte->eref->aliasname, attnum); + expr = (Node *) ste->expr; + if (IsA(expr, Var)) + { + /* + * Recurse into the CTE to see what its Var refers to. + * We have to build an additional level of namespace + * to keep in step with varlevelsup in the CTE. + * Furthermore it could be an outer CTE, so we may + * have to delete some levels of namespace. + */ + List *save_nslist = context->namespaces; + List *new_nslist; + deparse_namespace mydpns; + const char *result; + + set_deparse_for_query(&mydpns, ctequery, + context->namespaces); + + new_nslist = list_copy_tail(context->namespaces, + ctelevelsup); + context->namespaces = lcons(&mydpns, new_nslist); + + result = get_name_for_var_field((Var *) expr, fieldno, + 0, context); + + context->namespaces = save_nslist; + + return result; + } + /* else fall through to inspect the expression */ + } + else + { + /* + * We're deparsing a Plan tree so we don't have a CTE + * list. But the only place we'd see a Var directly + * referencing a CTE RTE is in a CteScan plan node, and we + * can look into the subplan's tlist instead. + */ + TargetEntry *tle; + deparse_namespace save_dpns; + const char *result; + + if (!dpns->inner_planstate) + elog(ERROR, "failed to find plan for CTE %s", + rte->eref->aliasname); + tle = get_tle_by_resno(dpns->inner_tlist, attnum); + if (!tle) + elog(ERROR, "bogus varattno for subquery var: %d", + attnum); + Assert(netlevelsup == 0); + push_child_plan(dpns, dpns->inner_planstate, &save_dpns); + + result = get_name_for_var_field((Var *) tle->expr, fieldno, + levelsup, context); + + pop_child_plan(dpns, &save_dpns); + return result; + } + } + break; + } + + /* + * We now have an expression we can't expand any more, so see if + * get_expr_result_tupdesc() can do anything with it. + */ + tupleDesc = get_expr_result_tupdesc(expr, false); + /* Got the tupdesc, so we can extract the field name */ + Assert(fieldno >= 1 && fieldno <= tupleDesc->natts); + return NameStr(TupleDescAttr(tupleDesc, fieldno - 1)->attname); +} + +/* + * Try to find the referenced expression for a PARAM_EXEC Param that might + * reference a parameter supplied by an upper NestLoop or SubPlan plan node. + * + * If successful, return the expression and set *dpns_p and *ancestor_cell_p + * appropriately for calling push_ancestor_plan(). If no referent can be + * found, return NULL. + */ +static Node * +find_param_referent(Param *param, deparse_context *context, + deparse_namespace **dpns_p, ListCell **ancestor_cell_p) +{ + /* Initialize output parameters to prevent compiler warnings */ + *dpns_p = NULL; + *ancestor_cell_p = NULL; + + /* + * If it's a PARAM_EXEC parameter, look for a matching NestLoopParam or + * SubPlan argument. This will necessarily be in some ancestor of the + * current expression's PlanState. + */ + if (param->paramkind == PARAM_EXEC) + { + deparse_namespace *dpns; + PlanState *child_ps; + bool in_same_plan_level; + ListCell *lc; + + dpns = (deparse_namespace *) linitial(context->namespaces); + child_ps = dpns->planstate; + in_same_plan_level = true; + + foreach(lc, dpns->ancestors) + { + PlanState *ps = (PlanState *) lfirst(lc); + ListCell *lc2; + + /* + * NestLoops transmit params to their inner child only; also, once + * we've crawled up out of a subplan, this couldn't possibly be + * the right match. + */ + if (IsA(ps, NestLoopState) && + child_ps == innerPlanState(ps) && + in_same_plan_level) + { + NestLoop *nl = (NestLoop *) ps->plan; + + foreach(lc2, nl->nestParams) + { + NestLoopParam *nlp = (NestLoopParam *) lfirst(lc2); + + if (nlp->paramno == param->paramid) + { + /* Found a match, so return it */ + *dpns_p = dpns; + *ancestor_cell_p = lc; + return (Node *) nlp->paramval; + } + } + } + + /* + * Check to see if we're crawling up from a subplan. + */ + foreach(lc2, ps->subPlan) + { + SubPlanState *sstate = (SubPlanState *) lfirst(lc2); + SubPlan *subplan = sstate->subplan; + ListCell *lc3; + ListCell *lc4; + + if (child_ps != sstate->planstate) + continue; + + /* Matched subplan, so check its arguments */ + forboth(lc3, subplan->parParam, lc4, subplan->args) + { + int paramid = lfirst_int(lc3); + Node *arg = (Node *) lfirst(lc4); + + if (paramid == param->paramid) + { + /* Found a match, so return it */ + *dpns_p = dpns; + *ancestor_cell_p = lc; + return arg; + } + } + + /* Keep looking, but we are emerging from a subplan. */ + in_same_plan_level = false; + break; + } + + /* + * Likewise check to see if we're emerging from an initplan. + * Initplans never have any parParams, so no need to search that + * list, but we need to know if we should reset + * in_same_plan_level. + */ + foreach(lc2, ps->initPlan) + { + SubPlanState *sstate = (SubPlanState *) lfirst(lc2); + + if (child_ps != sstate->planstate) + continue; + + /* No parameters to be had here. */ + Assert(sstate->subplan->parParam == NIL); + + /* Keep looking, but we are emerging from an initplan. */ + in_same_plan_level = false; + break; + } + + /* No luck, crawl up to next ancestor */ + child_ps = ps; + } + } + + /* No referent found */ + return NULL; +} + +/* + * Display a Param appropriately. + */ +static void +get_parameter(Param *param, deparse_context *context) +{ + Node *expr; + deparse_namespace *dpns; + ListCell *ancestor_cell; + + /* + * If it's a PARAM_EXEC parameter, try to locate the expression from which + * the parameter was computed. Note that failing to find a referent isn't + * an error, since the Param might well be a subplan output rather than an + * input. + */ + expr = find_param_referent(param, context, &dpns, &ancestor_cell); + if (expr) + { + /* Found a match, so print it */ + deparse_namespace save_dpns; + bool save_varprefix; + bool need_paren; + + /* Switch attention to the ancestor plan node */ + push_ancestor_plan(dpns, ancestor_cell, &save_dpns); + + /* + * Force prefixing of Vars, since they won't belong to the relation + * being scanned in the original plan node. + */ + save_varprefix = context->varprefix; + context->varprefix = true; + + /* + * A Param's expansion is typically a Var, Aggref, or upper-level + * Param, which wouldn't need extra parentheses. Otherwise, insert + * parens to ensure the expression looks atomic. + */ + need_paren = !(IsA(expr, Var) || + IsA(expr, Aggref) || + IsA(expr, Param)); + if (need_paren) + appendStringInfoChar(context->buf, '('); + + get_rule_expr(expr, context, false); + + if (need_paren) + appendStringInfoChar(context->buf, ')'); + + context->varprefix = save_varprefix; + + pop_ancestor_plan(dpns, &save_dpns); + + return; + } + + /* + * Not PARAM_EXEC, or couldn't find referent: just print $N. + */ + appendStringInfo(context->buf, "$%d", param->paramid); +} + +/* + * get_simple_binary_op_name + * + * helper function for isSimpleNode + * will return single char binary operator name, or NULL if it's not + */ +static const char * +get_simple_binary_op_name(OpExpr *expr) +{ + List *args = expr->args; + + if (list_length(args) == 2) + { + /* binary operator */ + Node *arg1 = (Node *) linitial(args); + Node *arg2 = (Node *) lsecond(args); + const char *op; + + op = generate_operator_name(expr->opno, exprType(arg1), exprType(arg2)); + if (strlen(op) == 1) + return op; + } + return NULL; +} + + +/* + * isSimpleNode - check if given node is simple (doesn't need parenthesizing) + * + * true : simple in the context of parent node's type + * false : not simple + */ +static bool +isSimpleNode(Node *node, Node *parentNode, int prettyFlags) +{ + if (!node) + return false; + + switch (nodeTag(node)) + { + case T_Var: + case T_Const: + case T_Param: + case T_CoerceToDomainValue: + case T_SetToDefault: + case T_CurrentOfExpr: + /* single words: always simple */ + return true; + + case T_SubscriptingRef: + case T_ArrayExpr: + case T_RowExpr: + case T_CoalesceExpr: + case T_MinMaxExpr: + case T_SQLValueFunction: + case T_XmlExpr: + case T_NextValueExpr: + case T_NullIfExpr: + case T_Aggref: + case T_WindowFunc: + case T_FuncExpr: + /* function-like: name(..) or name[..] */ + return true; + + /* CASE keywords act as parentheses */ + case T_CaseExpr: + return true; + + case T_FieldSelect: + + /* + * appears simple since . has top precedence, unless parent is + * T_FieldSelect itself! + */ + return (IsA(parentNode, FieldSelect) ? false : true); + + case T_FieldStore: + + /* + * treat like FieldSelect (probably doesn't matter) + */ + return (IsA(parentNode, FieldStore) ? false : true); + + case T_CoerceToDomain: + /* maybe simple, check args */ + return isSimpleNode((Node *) ((CoerceToDomain *) node)->arg, + node, prettyFlags); + case T_RelabelType: + return isSimpleNode((Node *) ((RelabelType *) node)->arg, + node, prettyFlags); + case T_CoerceViaIO: + return isSimpleNode((Node *) ((CoerceViaIO *) node)->arg, + node, prettyFlags); + case T_ArrayCoerceExpr: + return isSimpleNode((Node *) ((ArrayCoerceExpr *) node)->arg, + node, prettyFlags); + case T_ConvertRowtypeExpr: + return isSimpleNode((Node *) ((ConvertRowtypeExpr *) node)->arg, + node, prettyFlags); + + case T_OpExpr: + { + /* depends on parent node type; needs further checking */ + if (prettyFlags & PRETTYFLAG_PAREN && IsA(parentNode, OpExpr)) + { + const char *op; + const char *parentOp; + bool is_lopriop; + bool is_hipriop; + bool is_lopriparent; + bool is_hipriparent; + + op = get_simple_binary_op_name((OpExpr *) node); + if (!op) + return false; + + /* We know only the basic operators + - and * / % */ + is_lopriop = (strchr("+-", *op) != NULL); + is_hipriop = (strchr("*/%", *op) != NULL); + if (!(is_lopriop || is_hipriop)) + return false; + + parentOp = get_simple_binary_op_name((OpExpr *) parentNode); + if (!parentOp) + return false; + + is_lopriparent = (strchr("+-", *parentOp) != NULL); + is_hipriparent = (strchr("*/%", *parentOp) != NULL); + if (!(is_lopriparent || is_hipriparent)) + return false; + + if (is_hipriop && is_lopriparent) + return true; /* op binds tighter than parent */ + + if (is_lopriop && is_hipriparent) + return false; + + /* + * Operators are same priority --- can skip parens only if + * we have (a - b) - c, not a - (b - c). + */ + if (node == (Node *) linitial(((OpExpr *) parentNode)->args)) + return true; + + return false; + } + /* else do the same stuff as for T_SubLink et al. */ + } + /* FALLTHROUGH */ + + case T_SubLink: + case T_NullTest: + case T_BooleanTest: + case T_DistinctExpr: + switch (nodeTag(parentNode)) + { + case T_FuncExpr: + { + /* special handling for casts */ + CoercionForm type = ((FuncExpr *) parentNode)->funcformat; + + if (type == COERCE_EXPLICIT_CAST || + type == COERCE_IMPLICIT_CAST) + return false; + return true; /* own parentheses */ + } + case T_BoolExpr: /* lower precedence */ + case T_SubscriptingRef: /* other separators */ + case T_ArrayExpr: /* other separators */ + case T_RowExpr: /* other separators */ + case T_CoalesceExpr: /* own parentheses */ + case T_MinMaxExpr: /* own parentheses */ + case T_XmlExpr: /* own parentheses */ + case T_NullIfExpr: /* other separators */ + case T_Aggref: /* own parentheses */ + case T_WindowFunc: /* own parentheses */ + case T_CaseExpr: /* other separators */ + return true; + default: + return false; + } + + case T_BoolExpr: + switch (nodeTag(parentNode)) + { + case T_BoolExpr: + if (prettyFlags & PRETTYFLAG_PAREN) + { + BoolExprType type; + BoolExprType parentType; + + type = ((BoolExpr *) node)->boolop; + parentType = ((BoolExpr *) parentNode)->boolop; + switch (type) + { + case NOT_EXPR: + case AND_EXPR: + if (parentType == AND_EXPR || parentType == OR_EXPR) + return true; + break; + case OR_EXPR: + if (parentType == OR_EXPR) + return true; + break; + } + } + return false; + case T_FuncExpr: + { + /* special handling for casts */ + CoercionForm type = ((FuncExpr *) parentNode)->funcformat; + + if (type == COERCE_EXPLICIT_CAST || + type == COERCE_IMPLICIT_CAST) + return false; + return true; /* own parentheses */ + } + case T_SubscriptingRef: /* other separators */ + case T_ArrayExpr: /* other separators */ + case T_RowExpr: /* other separators */ + case T_CoalesceExpr: /* own parentheses */ + case T_MinMaxExpr: /* own parentheses */ + case T_XmlExpr: /* own parentheses */ + case T_NullIfExpr: /* other separators */ + case T_Aggref: /* own parentheses */ + case T_WindowFunc: /* own parentheses */ + case T_CaseExpr: /* other separators */ + return true; + default: + return false; + } + + default: + break; + } + /* those we don't know: in dubio complexo */ + return false; +} + + +/* + * appendContextKeyword - append a keyword to buffer + * + * If prettyPrint is enabled, perform a line break, and adjust indentation. + * Otherwise, just append the keyword. + */ +static void +appendContextKeyword(deparse_context *context, const char *str, + int indentBefore, int indentAfter, int indentPlus) +{ + StringInfo buf = context->buf; + + if (PRETTY_INDENT(context)) + { + int indentAmount; + + context->indentLevel += indentBefore; + + /* remove any trailing spaces currently in the buffer ... */ + removeStringInfoSpaces(buf); + /* ... then add a newline and some spaces */ + appendStringInfoChar(buf, '\n'); + + if (context->indentLevel < PRETTYINDENT_LIMIT) + indentAmount = Max(context->indentLevel, 0) + indentPlus; + else + { + /* + * If we're indented more than PRETTYINDENT_LIMIT characters, try + * to conserve horizontal space by reducing the per-level + * indentation. For best results the scale factor here should + * divide all the indent amounts that get added to indentLevel + * (PRETTYINDENT_STD, etc). It's important that the indentation + * not grow unboundedly, else deeply-nested trees use O(N^2) + * whitespace; so we also wrap modulo PRETTYINDENT_LIMIT. + */ + indentAmount = PRETTYINDENT_LIMIT + + (context->indentLevel - PRETTYINDENT_LIMIT) / + (PRETTYINDENT_STD / 2); + indentAmount %= PRETTYINDENT_LIMIT; + /* scale/wrap logic affects indentLevel, but not indentPlus */ + indentAmount += indentPlus; + } + appendStringInfoSpaces(buf, indentAmount); + + appendStringInfoString(buf, str); + + context->indentLevel += indentAfter; + if (context->indentLevel < 0) + context->indentLevel = 0; + } + else + appendStringInfoString(buf, str); +} + +/* + * removeStringInfoSpaces - delete trailing spaces from a buffer. + * + * Possibly this should move to stringinfo.c at some point. + */ +static void +removeStringInfoSpaces(StringInfo str) +{ + while (str->len > 0 && str->data[str->len - 1] == ' ') + str->data[--(str->len)] = '\0'; +} + + +/* + * get_rule_expr_paren - deparse expr using get_rule_expr, + * embracing the string with parentheses if necessary for prettyPrint. + * + * Never embrace if prettyFlags=0, because it's done in the calling node. + * + * Any node that does *not* embrace its argument node by sql syntax (with + * parentheses, non-operator keywords like CASE/WHEN/ON, or comma etc) should + * use get_rule_expr_paren instead of get_rule_expr so parentheses can be + * added. + */ +static void +get_rule_expr_paren(Node *node, deparse_context *context, + bool showimplicit, Node *parentNode) +{ + bool need_paren; + + need_paren = PRETTY_PAREN(context) && + !isSimpleNode(node, parentNode, context->prettyFlags); + + if (need_paren) + appendStringInfoChar(context->buf, '('); + + get_rule_expr(node, context, showimplicit); + + if (need_paren) + appendStringInfoChar(context->buf, ')'); +} + + +/* ---------- + * get_rule_expr - Parse back an expression + * + * Note: showimplicit determines whether we display any implicit cast that + * is present at the top of the expression tree. It is a passed argument, + * not a field of the context struct, because we change the value as we + * recurse down into the expression. In general we suppress implicit casts + * when the result type is known with certainty (eg, the arguments of an + * OR must be boolean). We display implicit casts for arguments of functions + * and operators, since this is needed to be certain that the same function + * or operator will be chosen when the expression is re-parsed. + * ---------- + */ +static void +get_rule_expr(Node *node, deparse_context *context, + bool showimplicit) +{ + StringInfo buf = context->buf; + + if (node == NULL) + return; + + /* Guard against excessively long or deeply-nested queries */ + CHECK_FOR_INTERRUPTS(); + check_stack_depth(); + + /* + * Each level of get_rule_expr must emit an indivisible term + * (parenthesized if necessary) to ensure result is reparsed into the same + * expression tree. The only exception is that when the input is a List, + * we emit the component items comma-separated with no surrounding + * decoration; this is convenient for most callers. + */ + switch (nodeTag(node)) + { + case T_Var: + (void) get_variable((Var *) node, 0, false, context); + break; + + case T_Const: + get_const_expr((Const *) node, context, 0); + break; + + case T_Param: + get_parameter((Param *) node, context); + break; + + case T_Aggref: + get_agg_expr((Aggref *) node, context, (Aggref *) node); + break; + + case T_GroupingFunc: + { + GroupingFunc *gexpr = (GroupingFunc *) node; + + appendStringInfoString(buf, "GROUPING("); + get_rule_expr((Node *) gexpr->args, context, true); + appendStringInfoChar(buf, ')'); + } + break; + + case T_WindowFunc: + get_windowfunc_expr((WindowFunc *) node, context); + break; + + case T_SubscriptingRef: + { + SubscriptingRef *sbsref = (SubscriptingRef *) node; + bool need_parens; + + /* + * If the argument is a CaseTestExpr, we must be inside a + * FieldStore, ie, we are assigning to an element of an array + * within a composite column. Since we already punted on + * displaying the FieldStore's target information, just punt + * here too, and display only the assignment source + * expression. + */ + if (IsA(sbsref->refexpr, CaseTestExpr)) + { + Assert(sbsref->refassgnexpr); + get_rule_expr((Node *) sbsref->refassgnexpr, + context, showimplicit); + break; + } + + /* + * Parenthesize the argument unless it's a simple Var or a + * FieldSelect. (In particular, if it's another + * SubscriptingRef, we *must* parenthesize to avoid + * confusion.) + */ + need_parens = !IsA(sbsref->refexpr, Var) && + !IsA(sbsref->refexpr, FieldSelect); + if (need_parens) + appendStringInfoChar(buf, '('); + get_rule_expr((Node *) sbsref->refexpr, context, showimplicit); + if (need_parens) + appendStringInfoChar(buf, ')'); + + /* + * If there's a refassgnexpr, we want to print the node in the + * format "container[subscripts] := refassgnexpr". This is + * not legal SQL, so decompilation of INSERT or UPDATE + * statements should always use processIndirection as part of + * the statement-level syntax. We should only see this when + * EXPLAIN tries to print the targetlist of a plan resulting + * from such a statement. + */ + if (sbsref->refassgnexpr) + { + Node *refassgnexpr; + + /* + * Use processIndirection to print this node's subscripts + * as well as any additional field selections or + * subscripting in immediate descendants. It returns the + * RHS expr that is actually being "assigned". + */ + refassgnexpr = processIndirection(node, context); + appendStringInfoString(buf, " := "); + get_rule_expr(refassgnexpr, context, showimplicit); + } + else + { + /* Just an ordinary container fetch, so print subscripts */ + printSubscripts(sbsref, context); + } + } + break; + + case T_FuncExpr: + get_func_expr((FuncExpr *) node, context, showimplicit); + break; + + case T_NamedArgExpr: + { + NamedArgExpr *na = (NamedArgExpr *) node; + + appendStringInfo(buf, "%s => ", quote_identifier(na->name)); + get_rule_expr((Node *) na->arg, context, showimplicit); + } + break; + + case T_OpExpr: + get_oper_expr((OpExpr *) node, context); + break; + + case T_DistinctExpr: + { + DistinctExpr *expr = (DistinctExpr *) node; + List *args = expr->args; + Node *arg1 = (Node *) linitial(args); + Node *arg2 = (Node *) lsecond(args); + + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren(arg1, context, true, node); + appendStringInfoString(buf, " IS DISTINCT FROM "); + get_rule_expr_paren(arg2, context, true, node); + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + } + break; + + case T_NullIfExpr: + { + NullIfExpr *nullifexpr = (NullIfExpr *) node; + + appendStringInfoString(buf, "NULLIF("); + get_rule_expr((Node *) nullifexpr->args, context, true); + appendStringInfoChar(buf, ')'); + } + break; + + case T_ScalarArrayOpExpr: + { + ScalarArrayOpExpr *expr = (ScalarArrayOpExpr *) node; + List *args = expr->args; + Node *arg1 = (Node *) linitial(args); + Node *arg2 = (Node *) lsecond(args); + + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren(arg1, context, true, node); + appendStringInfo(buf, " %s %s (", + generate_operator_name(expr->opno, + exprType(arg1), + get_base_element_type(exprType(arg2))), + expr->useOr ? "ANY" : "ALL"); + get_rule_expr_paren(arg2, context, true, node); + + /* + * There's inherent ambiguity in "x op ANY/ALL (y)" when y is + * a bare sub-SELECT. Since we're here, the sub-SELECT must + * be meant as a scalar sub-SELECT yielding an array value to + * be used in ScalarArrayOpExpr; but the grammar will + * preferentially interpret such a construct as an ANY/ALL + * SubLink. To prevent misparsing the output that way, insert + * a dummy coercion (which will be stripped by parse analysis, + * so no inefficiency is added in dump and reload). This is + * indeed most likely what the user wrote to get the construct + * accepted in the first place. + */ + if (IsA(arg2, SubLink) && + ((SubLink *) arg2)->subLinkType == EXPR_SUBLINK) + appendStringInfo(buf, "::%s", + format_type_with_typemod(exprType(arg2), + exprTypmod(arg2))); + appendStringInfoChar(buf, ')'); + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + } + break; + + case T_BoolExpr: + { + BoolExpr *expr = (BoolExpr *) node; + Node *first_arg = linitial(expr->args); + ListCell *arg = lnext(list_head(expr->args)); + + switch (expr->boolop) + { + case AND_EXPR: + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren(first_arg, context, + false, node); + while (arg) + { + appendStringInfoString(buf, " AND "); + get_rule_expr_paren((Node *) lfirst(arg), context, + false, node); + arg = lnext(arg); + } + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + break; + + case OR_EXPR: + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren(first_arg, context, + false, node); + while (arg) + { + appendStringInfoString(buf, " OR "); + get_rule_expr_paren((Node *) lfirst(arg), context, + false, node); + arg = lnext(arg); + } + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + break; + + case NOT_EXPR: + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + appendStringInfoString(buf, "NOT "); + get_rule_expr_paren(first_arg, context, + false, node); + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + break; + + default: + elog(ERROR, "unrecognized boolop: %d", + (int) expr->boolop); + } + } + break; + + case T_SubLink: + get_sublink_expr((SubLink *) node, context); + break; + + case T_SubPlan: + { + SubPlan *subplan = (SubPlan *) node; + + /* + * We cannot see an already-planned subplan in rule deparsing, + * only while EXPLAINing a query plan. We don't try to + * reconstruct the original SQL, just reference the subplan + * that appears elsewhere in EXPLAIN's result. + */ + if (subplan->useHashTable) + appendStringInfo(buf, "(hashed %s)", subplan->plan_name); + else + appendStringInfo(buf, "(%s)", subplan->plan_name); + } + break; + + case T_AlternativeSubPlan: + { + AlternativeSubPlan *asplan = (AlternativeSubPlan *) node; + ListCell *lc; + + /* As above, this can only happen during EXPLAIN */ + appendStringInfoString(buf, "(alternatives: "); + foreach(lc, asplan->subplans) + { + SubPlan *splan = lfirst_node(SubPlan, lc); + + if (splan->useHashTable) + appendStringInfo(buf, "hashed %s", splan->plan_name); + else + appendStringInfoString(buf, splan->plan_name); + if (lnext(lc)) + appendStringInfoString(buf, " or "); + } + appendStringInfoChar(buf, ')'); + } + break; + + case T_FieldSelect: + { + FieldSelect *fselect = (FieldSelect *) node; + Node *arg = (Node *) fselect->arg; + int fno = fselect->fieldnum; + const char *fieldname; + bool need_parens; + + /* + * Parenthesize the argument unless it's an SubscriptingRef or + * another FieldSelect. Note in particular that it would be + * WRONG to not parenthesize a Var argument; simplicity is not + * the issue here, having the right number of names is. + */ + need_parens = !IsA(arg, SubscriptingRef) && + !IsA(arg, FieldSelect); + if (need_parens) + appendStringInfoChar(buf, '('); + get_rule_expr(arg, context, true); + if (need_parens) + appendStringInfoChar(buf, ')'); + + /* + * Get and print the field name. + */ + fieldname = get_name_for_var_field((Var *) arg, fno, + 0, context); + appendStringInfo(buf, ".%s", quote_identifier(fieldname)); + } + break; + + case T_FieldStore: + { + FieldStore *fstore = (FieldStore *) node; + bool need_parens; + + /* + * There is no good way to represent a FieldStore as real SQL, + * so decompilation of INSERT or UPDATE statements should + * always use processIndirection as part of the + * statement-level syntax. We should only get here when + * EXPLAIN tries to print the targetlist of a plan resulting + * from such a statement. The plan case is even harder than + * ordinary rules would be, because the planner tries to + * collapse multiple assignments to the same field or subfield + * into one FieldStore; so we can see a list of target fields + * not just one, and the arguments could be FieldStores + * themselves. We don't bother to try to print the target + * field names; we just print the source arguments, with a + * ROW() around them if there's more than one. This isn't + * terribly complete, but it's probably good enough for + * EXPLAIN's purposes; especially since anything more would be + * either hopelessly confusing or an even poorer + * representation of what the plan is actually doing. + */ + need_parens = (list_length(fstore->newvals) != 1); + if (need_parens) + appendStringInfoString(buf, "ROW("); + get_rule_expr((Node *) fstore->newvals, context, showimplicit); + if (need_parens) + appendStringInfoChar(buf, ')'); + } + break; + + case T_RelabelType: + { + RelabelType *relabel = (RelabelType *) node; + Node *arg = (Node *) relabel->arg; + + if (relabel->relabelformat == COERCE_IMPLICIT_CAST && + !showimplicit) + { + /* don't show the implicit cast */ + get_rule_expr_paren(arg, context, false, node); + } + else + { + get_coercion_expr(arg, context, + relabel->resulttype, + relabel->resulttypmod, + node); + } + } + break; + + case T_CoerceViaIO: + { + CoerceViaIO *iocoerce = (CoerceViaIO *) node; + Node *arg = (Node *) iocoerce->arg; + + if (iocoerce->coerceformat == COERCE_IMPLICIT_CAST && + !showimplicit) + { + /* don't show the implicit cast */ + get_rule_expr_paren(arg, context, false, node); + } + else + { + get_coercion_expr(arg, context, + iocoerce->resulttype, + -1, + node); + } + } + break; + + case T_ArrayCoerceExpr: + { + ArrayCoerceExpr *acoerce = (ArrayCoerceExpr *) node; + Node *arg = (Node *) acoerce->arg; + + if (acoerce->coerceformat == COERCE_IMPLICIT_CAST && + !showimplicit) + { + /* don't show the implicit cast */ + get_rule_expr_paren(arg, context, false, node); + } + else + { + get_coercion_expr(arg, context, + acoerce->resulttype, + acoerce->resulttypmod, + node); + } + } + break; + + case T_ConvertRowtypeExpr: + { + ConvertRowtypeExpr *convert = (ConvertRowtypeExpr *) node; + Node *arg = (Node *) convert->arg; + + if (convert->convertformat == COERCE_IMPLICIT_CAST && + !showimplicit) + { + /* don't show the implicit cast */ + get_rule_expr_paren(arg, context, false, node); + } + else + { + get_coercion_expr(arg, context, + convert->resulttype, -1, + node); + } + } + break; + + case T_CollateExpr: + { + CollateExpr *collate = (CollateExpr *) node; + Node *arg = (Node *) collate->arg; + + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren(arg, context, showimplicit, node); + appendStringInfo(buf, " COLLATE %s", + generate_collation_name(collate->collOid)); + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + } + break; + + case T_CaseExpr: + { + CaseExpr *caseexpr = (CaseExpr *) node; + ListCell *temp; + + appendContextKeyword(context, "CASE", + 0, PRETTYINDENT_VAR, 0); + if (caseexpr->arg) + { + appendStringInfoChar(buf, ' '); + get_rule_expr((Node *) caseexpr->arg, context, true); + } + foreach(temp, caseexpr->args) + { + CaseWhen *when = (CaseWhen *) lfirst(temp); + Node *w = (Node *) when->expr; + + if (caseexpr->arg) + { + /* + * The parser should have produced WHEN clauses of the + * form "CaseTestExpr = RHS", possibly with an + * implicit coercion inserted above the CaseTestExpr. + * For accurate decompilation of rules it's essential + * that we show just the RHS. However in an + * expression that's been through the optimizer, the + * WHEN clause could be almost anything (since the + * equality operator could have been expanded into an + * inline function). If we don't recognize the form + * of the WHEN clause, just punt and display it as-is. + */ + if (IsA(w, OpExpr)) + { + List *args = ((OpExpr *) w)->args; + + if (list_length(args) == 2 && + IsA(strip_implicit_coercions(linitial(args)), + CaseTestExpr)) + w = (Node *) lsecond(args); + } + } + + if (!PRETTY_INDENT(context)) + appendStringInfoChar(buf, ' '); + appendContextKeyword(context, "WHEN ", + 0, 0, 0); + get_rule_expr(w, context, false); + appendStringInfoString(buf, " THEN "); + get_rule_expr((Node *) when->result, context, true); + } + if (!PRETTY_INDENT(context)) + appendStringInfoChar(buf, ' '); + appendContextKeyword(context, "ELSE ", + 0, 0, 0); + get_rule_expr((Node *) caseexpr->defresult, context, true); + if (!PRETTY_INDENT(context)) + appendStringInfoChar(buf, ' '); + appendContextKeyword(context, "END", + -PRETTYINDENT_VAR, 0, 0); + } + break; + + case T_CaseTestExpr: + { + /* + * Normally we should never get here, since for expressions + * that can contain this node type we attempt to avoid + * recursing to it. But in an optimized expression we might + * be unable to avoid that (see comments for CaseExpr). If we + * do see one, print it as CASE_TEST_EXPR. + */ + appendStringInfoString(buf, "CASE_TEST_EXPR"); + } + break; + + case T_ArrayExpr: + { + ArrayExpr *arrayexpr = (ArrayExpr *) node; + + appendStringInfoString(buf, "ARRAY["); + get_rule_expr((Node *) arrayexpr->elements, context, true); + appendStringInfoChar(buf, ']'); + + /* + * If the array isn't empty, we assume its elements are + * coerced to the desired type. If it's empty, though, we + * need an explicit coercion to the array type. + */ + if (arrayexpr->elements == NIL) + appendStringInfo(buf, "::%s", + format_type_with_typemod(arrayexpr->array_typeid, -1)); + } + break; + + case T_RowExpr: + { + RowExpr *rowexpr = (RowExpr *) node; + TupleDesc tupdesc = NULL; + ListCell *arg; + int i; + char *sep; + + /* + * If it's a named type and not RECORD, we may have to skip + * dropped columns and/or claim there are NULLs for added + * columns. + */ + if (rowexpr->row_typeid != RECORDOID) + { + tupdesc = lookup_rowtype_tupdesc(rowexpr->row_typeid, -1); + Assert(list_length(rowexpr->args) <= tupdesc->natts); + } + + /* + * SQL99 allows "ROW" to be omitted when there is more than + * one column, but for simplicity we always print it. + */ + appendStringInfoString(buf, "ROW("); + sep = ""; + i = 0; + foreach(arg, rowexpr->args) + { + Node *e = (Node *) lfirst(arg); + + if (tupdesc == NULL || + !TupleDescAttr(tupdesc, i)->attisdropped) + { + appendStringInfoString(buf, sep); + /* Whole-row Vars need special treatment here */ + get_rule_expr_toplevel(e, context, true); + sep = ", "; + } + i++; + } + if (tupdesc != NULL) + { + while (i < tupdesc->natts) + { + if (!TupleDescAttr(tupdesc, i)->attisdropped) + { + appendStringInfoString(buf, sep); + appendStringInfoString(buf, "NULL"); + sep = ", "; + } + i++; + } + + ReleaseTupleDesc(tupdesc); + } + appendStringInfoChar(buf, ')'); + if (rowexpr->row_format == COERCE_EXPLICIT_CAST) + appendStringInfo(buf, "::%s", + format_type_with_typemod(rowexpr->row_typeid, -1)); + } + break; + + case T_RowCompareExpr: + { + RowCompareExpr *rcexpr = (RowCompareExpr *) node; + ListCell *arg; + char *sep; + + /* + * SQL99 allows "ROW" to be omitted when there is more than + * one column, but for simplicity we always print it. + */ + appendStringInfoString(buf, "(ROW("); + sep = ""; + foreach(arg, rcexpr->largs) + { + Node *e = (Node *) lfirst(arg); + + appendStringInfoString(buf, sep); + get_rule_expr(e, context, true); + sep = ", "; + } + + /* + * We assume that the name of the first-column operator will + * do for all the rest too. This is definitely open to + * failure, eg if some but not all operators were renamed + * since the construct was parsed, but there seems no way to + * be perfect. + */ + appendStringInfo(buf, ") %s ROW(", + generate_operator_name(linitial_oid(rcexpr->opnos), + exprType(linitial(rcexpr->largs)), + exprType(linitial(rcexpr->rargs)))); + sep = ""; + foreach(arg, rcexpr->rargs) + { + Node *e = (Node *) lfirst(arg); + + appendStringInfoString(buf, sep); + get_rule_expr(e, context, true); + sep = ", "; + } + appendStringInfoString(buf, "))"); + } + break; + + case T_CoalesceExpr: + { + CoalesceExpr *coalesceexpr = (CoalesceExpr *) node; + + appendStringInfoString(buf, "COALESCE("); + get_rule_expr((Node *) coalesceexpr->args, context, true); + appendStringInfoChar(buf, ')'); + } + break; + + case T_MinMaxExpr: + { + MinMaxExpr *minmaxexpr = (MinMaxExpr *) node; + + switch (minmaxexpr->op) + { + case IS_GREATEST: + appendStringInfoString(buf, "GREATEST("); + break; + case IS_LEAST: + appendStringInfoString(buf, "LEAST("); + break; + } + get_rule_expr((Node *) minmaxexpr->args, context, true); + appendStringInfoChar(buf, ')'); + } + break; + + case T_SQLValueFunction: + { + SQLValueFunction *svf = (SQLValueFunction *) node; + + /* + * Note: this code knows that typmod for time, timestamp, and + * timestamptz just prints as integer. + */ + switch (svf->op) + { + case SVFOP_CURRENT_DATE: + appendStringInfoString(buf, "CURRENT_DATE"); + break; + case SVFOP_CURRENT_TIME: + appendStringInfoString(buf, "CURRENT_TIME"); + break; + case SVFOP_CURRENT_TIME_N: + appendStringInfo(buf, "CURRENT_TIME(%d)", svf->typmod); + break; + case SVFOP_CURRENT_TIMESTAMP: + appendStringInfoString(buf, "CURRENT_TIMESTAMP"); + break; + case SVFOP_CURRENT_TIMESTAMP_N: + appendStringInfo(buf, "CURRENT_TIMESTAMP(%d)", + svf->typmod); + break; + case SVFOP_LOCALTIME: + appendStringInfoString(buf, "LOCALTIME"); + break; + case SVFOP_LOCALTIME_N: + appendStringInfo(buf, "LOCALTIME(%d)", svf->typmod); + break; + case SVFOP_LOCALTIMESTAMP: + appendStringInfoString(buf, "LOCALTIMESTAMP"); + break; + case SVFOP_LOCALTIMESTAMP_N: + appendStringInfo(buf, "LOCALTIMESTAMP(%d)", + svf->typmod); + break; + case SVFOP_CURRENT_ROLE: + appendStringInfoString(buf, "CURRENT_ROLE"); + break; + case SVFOP_CURRENT_USER: + appendStringInfoString(buf, "CURRENT_USER"); + break; + case SVFOP_USER: + appendStringInfoString(buf, "USER"); + break; + case SVFOP_SESSION_USER: + appendStringInfoString(buf, "SESSION_USER"); + break; + case SVFOP_CURRENT_CATALOG: + appendStringInfoString(buf, "CURRENT_CATALOG"); + break; + case SVFOP_CURRENT_SCHEMA: + appendStringInfoString(buf, "CURRENT_SCHEMA"); + break; + } + } + break; + + case T_XmlExpr: + { + XmlExpr *xexpr = (XmlExpr *) node; + bool needcomma = false; + ListCell *arg; + ListCell *narg; + Const *con; + + switch (xexpr->op) + { + case IS_XMLCONCAT: + appendStringInfoString(buf, "XMLCONCAT("); + break; + case IS_XMLELEMENT: + appendStringInfoString(buf, "XMLELEMENT("); + break; + case IS_XMLFOREST: + appendStringInfoString(buf, "XMLFOREST("); + break; + case IS_XMLPARSE: + appendStringInfoString(buf, "XMLPARSE("); + break; + case IS_XMLPI: + appendStringInfoString(buf, "XMLPI("); + break; + case IS_XMLROOT: + appendStringInfoString(buf, "XMLROOT("); + break; + case IS_XMLSERIALIZE: + appendStringInfoString(buf, "XMLSERIALIZE("); + break; + case IS_DOCUMENT: + break; + } + if (xexpr->op == IS_XMLPARSE || xexpr->op == IS_XMLSERIALIZE) + { + if (xexpr->xmloption == XMLOPTION_DOCUMENT) + appendStringInfoString(buf, "DOCUMENT "); + else + appendStringInfoString(buf, "CONTENT "); + } + if (xexpr->name) + { + appendStringInfo(buf, "NAME %s", + quote_identifier(map_xml_name_to_sql_identifier(xexpr->name))); + needcomma = true; + } + if (xexpr->named_args) + { + if (xexpr->op != IS_XMLFOREST) + { + if (needcomma) + appendStringInfoString(buf, ", "); + appendStringInfoString(buf, "XMLATTRIBUTES("); + needcomma = false; + } + forboth(arg, xexpr->named_args, narg, xexpr->arg_names) + { + Node *e = (Node *) lfirst(arg); + char *argname = strVal(lfirst(narg)); + + if (needcomma) + appendStringInfoString(buf, ", "); + get_rule_expr((Node *) e, context, true); + appendStringInfo(buf, " AS %s", + quote_identifier(map_xml_name_to_sql_identifier(argname))); + needcomma = true; + } + if (xexpr->op != IS_XMLFOREST) + appendStringInfoChar(buf, ')'); + } + if (xexpr->args) + { + if (needcomma) + appendStringInfoString(buf, ", "); + switch (xexpr->op) + { + case IS_XMLCONCAT: + case IS_XMLELEMENT: + case IS_XMLFOREST: + case IS_XMLPI: + case IS_XMLSERIALIZE: + /* no extra decoration needed */ + get_rule_expr((Node *) xexpr->args, context, true); + break; + case IS_XMLPARSE: + Assert(list_length(xexpr->args) == 2); + + get_rule_expr((Node *) linitial(xexpr->args), + context, true); + + con = lsecond_node(Const, xexpr->args); + Assert(!con->constisnull); + if (DatumGetBool(con->constvalue)) + appendStringInfoString(buf, + " PRESERVE WHITESPACE"); + else + appendStringInfoString(buf, + " STRIP WHITESPACE"); + break; + case IS_XMLROOT: + Assert(list_length(xexpr->args) == 3); + + get_rule_expr((Node *) linitial(xexpr->args), + context, true); + + appendStringInfoString(buf, ", VERSION "); + con = (Const *) lsecond(xexpr->args); + if (IsA(con, Const) && + con->constisnull) + appendStringInfoString(buf, "NO VALUE"); + else + get_rule_expr((Node *) con, context, false); + + con = lthird_node(Const, xexpr->args); + if (con->constisnull) + /* suppress STANDALONE NO VALUE */ ; + else + { + switch (DatumGetInt32(con->constvalue)) + { + case XML_STANDALONE_YES: + appendStringInfoString(buf, + ", STANDALONE YES"); + break; + case XML_STANDALONE_NO: + appendStringInfoString(buf, + ", STANDALONE NO"); + break; + case XML_STANDALONE_NO_VALUE: + appendStringInfoString(buf, + ", STANDALONE NO VALUE"); + break; + default: + break; + } + } + break; + case IS_DOCUMENT: + get_rule_expr_paren((Node *) xexpr->args, context, false, node); + break; + } + + } + if (xexpr->op == IS_XMLSERIALIZE) + appendStringInfo(buf, " AS %s", + format_type_with_typemod(xexpr->type, + xexpr->typmod)); + if (xexpr->op == IS_DOCUMENT) + appendStringInfoString(buf, " IS DOCUMENT"); + else + appendStringInfoChar(buf, ')'); + } + break; + + case T_NullTest: + { + NullTest *ntest = (NullTest *) node; + + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren((Node *) ntest->arg, context, true, node); + + /* + * For scalar inputs, we prefer to print as IS [NOT] NULL, + * which is shorter and traditional. If it's a rowtype input + * but we're applying a scalar test, must print IS [NOT] + * DISTINCT FROM NULL to be semantically correct. + */ + if (ntest->argisrow || + !type_is_rowtype(exprType((Node *) ntest->arg))) + { + switch (ntest->nulltesttype) + { + case IS_NULL: + appendStringInfoString(buf, " IS NULL"); + break; + case IS_NOT_NULL: + appendStringInfoString(buf, " IS NOT NULL"); + break; + default: + elog(ERROR, "unrecognized nulltesttype: %d", + (int) ntest->nulltesttype); + } + } + else + { + switch (ntest->nulltesttype) + { + case IS_NULL: + appendStringInfoString(buf, " IS NOT DISTINCT FROM NULL"); + break; + case IS_NOT_NULL: + appendStringInfoString(buf, " IS DISTINCT FROM NULL"); + break; + default: + elog(ERROR, "unrecognized nulltesttype: %d", + (int) ntest->nulltesttype); + } + } + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + } + break; + + case T_BooleanTest: + { + BooleanTest *btest = (BooleanTest *) node; + + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren((Node *) btest->arg, context, false, node); + switch (btest->booltesttype) + { + case IS_TRUE: + appendStringInfoString(buf, " IS TRUE"); + break; + case IS_NOT_TRUE: + appendStringInfoString(buf, " IS NOT TRUE"); + break; + case IS_FALSE: + appendStringInfoString(buf, " IS FALSE"); + break; + case IS_NOT_FALSE: + appendStringInfoString(buf, " IS NOT FALSE"); + break; + case IS_UNKNOWN: + appendStringInfoString(buf, " IS UNKNOWN"); + break; + case IS_NOT_UNKNOWN: + appendStringInfoString(buf, " IS NOT UNKNOWN"); + break; + default: + elog(ERROR, "unrecognized booltesttype: %d", + (int) btest->booltesttype); + } + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + } + break; + + case T_CoerceToDomain: + { + CoerceToDomain *ctest = (CoerceToDomain *) node; + Node *arg = (Node *) ctest->arg; + + if (ctest->coercionformat == COERCE_IMPLICIT_CAST && + !showimplicit) + { + /* don't show the implicit cast */ + get_rule_expr(arg, context, false); + } + else + { + get_coercion_expr(arg, context, + ctest->resulttype, + ctest->resulttypmod, + node); + } + } + break; + + case T_CoerceToDomainValue: + appendStringInfoString(buf, "VALUE"); + break; + + case T_SetToDefault: + appendStringInfoString(buf, "DEFAULT"); + break; + + case T_CurrentOfExpr: + { + CurrentOfExpr *cexpr = (CurrentOfExpr *) node; + + if (cexpr->cursor_name) + appendStringInfo(buf, "CURRENT OF %s", + quote_identifier(cexpr->cursor_name)); + else + appendStringInfo(buf, "CURRENT OF $%d", + cexpr->cursor_param); + } + break; + + case T_NextValueExpr: + { + NextValueExpr *nvexpr = (NextValueExpr *) node; + + /* + * This isn't exactly nextval(), but that seems close enough + * for EXPLAIN's purposes. + */ + appendStringInfoString(buf, "nextval("); + simple_quote_literal(buf, + generate_relation_name(nvexpr->seqid, + NIL)); + appendStringInfoChar(buf, ')'); + } + break; + + case T_InferenceElem: + { + InferenceElem *iexpr = (InferenceElem *) node; + bool save_varprefix; + bool need_parens; + + /* + * InferenceElem can only refer to target relation, so a + * prefix is not useful, and indeed would cause parse errors. + */ + save_varprefix = context->varprefix; + context->varprefix = false; + + /* + * Parenthesize the element unless it's a simple Var or a bare + * function call. Follows pg_get_indexdef_worker(). + */ + need_parens = !IsA(iexpr->expr, Var); + if (IsA(iexpr->expr, FuncExpr) && + ((FuncExpr *) iexpr->expr)->funcformat == + COERCE_EXPLICIT_CALL) + need_parens = false; + + if (need_parens) + appendStringInfoChar(buf, '('); + get_rule_expr((Node *) iexpr->expr, + context, false); + if (need_parens) + appendStringInfoChar(buf, ')'); + + context->varprefix = save_varprefix; + + if (iexpr->infercollid) + appendStringInfo(buf, " COLLATE %s", + generate_collation_name(iexpr->infercollid)); + + /* Add the operator class name, if not default */ + if (iexpr->inferopclass) + { + Oid inferopclass = iexpr->inferopclass; + Oid inferopcinputtype = get_opclass_input_type(iexpr->inferopclass); + + get_opclass_name(inferopclass, inferopcinputtype, buf); + } + } + break; + + case T_PartitionBoundSpec: + { + PartitionBoundSpec *spec = (PartitionBoundSpec *) node; + ListCell *cell; + char *sep; + + if (spec->is_default) + { + appendStringInfoString(buf, "DEFAULT"); + break; + } + + switch (spec->strategy) + { + case PARTITION_STRATEGY_HASH: + Assert(spec->modulus > 0 && spec->remainder >= 0); + Assert(spec->modulus > spec->remainder); + + appendStringInfoString(buf, "FOR VALUES"); + appendStringInfo(buf, " WITH (modulus %d, remainder %d)", + spec->modulus, spec->remainder); + break; + + case PARTITION_STRATEGY_LIST: + Assert(spec->listdatums != NIL); + + appendStringInfoString(buf, "FOR VALUES IN ("); + sep = ""; + foreach(cell, spec->listdatums) + { + Const *val = castNode(Const, lfirst(cell)); + + appendStringInfoString(buf, sep); + get_const_expr(val, context, -1); + sep = ", "; + } + + appendStringInfoChar(buf, ')'); + break; + + case PARTITION_STRATEGY_RANGE: + Assert(spec->lowerdatums != NIL && + spec->upperdatums != NIL && + list_length(spec->lowerdatums) == + list_length(spec->upperdatums)); + + appendStringInfo(buf, "FOR VALUES FROM %s TO %s", + get_range_partbound_string(spec->lowerdatums), + get_range_partbound_string(spec->upperdatums)); + break; + + default: + elog(ERROR, "unrecognized partition strategy: %d", + (int) spec->strategy); + break; + } + } + break; + + case T_List: + { + char *sep; + ListCell *l; + + sep = ""; + foreach(l, (List *) node) + { + appendStringInfoString(buf, sep); + get_rule_expr((Node *) lfirst(l), context, showimplicit); + sep = ", "; + } + } + break; + + case T_TableFunc: + get_tablefunc((TableFunc *) node, context, showimplicit); + break; + + default: + elog(ERROR, "unrecognized node type: %d", (int) nodeTag(node)); + break; + } +} + +/* + * get_rule_expr_toplevel - Parse back a toplevel expression + * + * Same as get_rule_expr(), except that if the expr is just a Var, we pass + * istoplevel = true not false to get_variable(). This causes whole-row Vars + * to get printed with decoration that will prevent expansion of "*". + * We need to use this in contexts such as ROW() and VALUES(), where the + * parser would expand "foo.*" appearing at top level. (In principle we'd + * use this in get_target_list() too, but that has additional worries about + * whether to print AS, so it needs to invoke get_variable() directly anyway.) + */ +static void +get_rule_expr_toplevel(Node *node, deparse_context *context, + bool showimplicit) +{ + if (node && IsA(node, Var)) + (void) get_variable((Var *) node, 0, true, context); + else + get_rule_expr(node, context, showimplicit); +} + +/* + * get_rule_expr_funccall - Parse back a function-call expression + * + * Same as get_rule_expr(), except that we guarantee that the output will + * look like a function call, or like one of the things the grammar treats as + * equivalent to a function call (see the func_expr_windowless production). + * This is needed in places where the grammar uses func_expr_windowless and + * you can't substitute a parenthesized a_expr. If what we have isn't going + * to look like a function call, wrap it in a dummy CAST() expression, which + * will satisfy the grammar --- and, indeed, is likely what the user wrote to + * produce such a thing. + */ +static void +get_rule_expr_funccall(Node *node, deparse_context *context, + bool showimplicit) +{ + if (looks_like_function(node)) + get_rule_expr(node, context, showimplicit); + else + { + StringInfo buf = context->buf; + + appendStringInfoString(buf, "CAST("); + /* no point in showing any top-level implicit cast */ + get_rule_expr(node, context, false); + appendStringInfo(buf, " AS %s)", + format_type_with_typemod(exprType(node), + exprTypmod(node))); + } +} + +/* + * Helper function to identify node types that satisfy func_expr_windowless. + * If in doubt, "false" is always a safe answer. + */ +static bool +looks_like_function(Node *node) +{ + if (node == NULL) + return false; /* probably shouldn't happen */ + switch (nodeTag(node)) + { + case T_FuncExpr: + /* OK, unless it's going to deparse as a cast */ + return (((FuncExpr *) node)->funcformat == COERCE_EXPLICIT_CALL); + case T_NullIfExpr: + case T_CoalesceExpr: + case T_MinMaxExpr: + case T_SQLValueFunction: + case T_XmlExpr: + /* these are all accepted by func_expr_common_subexpr */ + return true; + default: + break; + } + return false; +} + + +/* + * get_oper_expr - Parse back an OpExpr node + */ +static void +get_oper_expr(OpExpr *expr, deparse_context *context) +{ + StringInfo buf = context->buf; + Oid opno = expr->opno; + List *args = expr->args; + + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + if (list_length(args) == 2) + { + /* binary operator */ + Node *arg1 = (Node *) linitial(args); + Node *arg2 = (Node *) lsecond(args); + + get_rule_expr_paren(arg1, context, true, (Node *) expr); + appendStringInfo(buf, " %s ", + generate_operator_name(opno, + exprType(arg1), + exprType(arg2))); + get_rule_expr_paren(arg2, context, true, (Node *) expr); + } + else + { + /* unary operator --- but which side? */ + Node *arg = (Node *) linitial(args); + HeapTuple tp; + Form_pg_operator optup; + + tp = SearchSysCache1(OPEROID, ObjectIdGetDatum(opno)); + if (!HeapTupleIsValid(tp)) + elog(ERROR, "cache lookup failed for operator %u", opno); + optup = (Form_pg_operator) GETSTRUCT(tp); + switch (optup->oprkind) + { + case 'l': + appendStringInfo(buf, "%s ", + generate_operator_name(opno, + InvalidOid, + exprType(arg))); + get_rule_expr_paren(arg, context, true, (Node *) expr); + break; + case 'r': + get_rule_expr_paren(arg, context, true, (Node *) expr); + appendStringInfo(buf, " %s", + generate_operator_name(opno, + exprType(arg), + InvalidOid)); + break; + default: + elog(ERROR, "bogus oprkind: %d", optup->oprkind); + } + ReleaseSysCache(tp); + } + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); +} + +/* + * get_func_expr - Parse back a FuncExpr node + */ +static void +get_func_expr(FuncExpr *expr, deparse_context *context, + bool showimplicit) +{ + StringInfo buf = context->buf; + Oid funcoid = expr->funcid; + Oid argtypes[FUNC_MAX_ARGS]; + int nargs; + List *argnames; + bool use_variadic; + ListCell *l; + + /* + * If the function call came from an implicit coercion, then just show the + * first argument --- unless caller wants to see implicit coercions. + */ + if (expr->funcformat == COERCE_IMPLICIT_CAST && !showimplicit) + { + get_rule_expr_paren((Node *) linitial(expr->args), context, + false, (Node *) expr); + return; + } + + /* + * If the function call came from a cast, then show the first argument + * plus an explicit cast operation. + */ + if (expr->funcformat == COERCE_EXPLICIT_CAST || + expr->funcformat == COERCE_IMPLICIT_CAST) + { + Node *arg = linitial(expr->args); + Oid rettype = expr->funcresulttype; + int32 coercedTypmod; + + /* Get the typmod if this is a length-coercion function */ + (void) exprIsLengthCoercion((Node *) expr, &coercedTypmod); + + get_coercion_expr(arg, context, + rettype, coercedTypmod, + (Node *) expr); + + return; + } + + /* + * Normal function: display as proname(args). First we need to extract + * the argument datatypes. + */ + if (list_length(expr->args) > FUNC_MAX_ARGS) + ereport(ERROR, + (errcode(ERRCODE_TOO_MANY_ARGUMENTS), + errmsg("too many arguments"))); + nargs = 0; + argnames = NIL; + foreach(l, expr->args) + { + Node *arg = (Node *) lfirst(l); + + if (IsA(arg, NamedArgExpr)) + argnames = lappend(argnames, ((NamedArgExpr *) arg)->name); + argtypes[nargs] = exprType(arg); + nargs++; + } + + appendStringInfo(buf, "%s(", + generate_function_name(funcoid, nargs, + argnames, argtypes, + expr->funcvariadic, + &use_variadic, + context->special_exprkind)); + nargs = 0; + foreach(l, expr->args) + { + if (nargs++ > 0) + appendStringInfoString(buf, ", "); + if (use_variadic && lnext(l) == NULL) + appendStringInfoString(buf, "VARIADIC "); + get_rule_expr((Node *) lfirst(l), context, true); + } + appendStringInfoChar(buf, ')'); +} + +/* + * get_agg_expr - Parse back an Aggref node + */ +static void +get_agg_expr(Aggref *aggref, deparse_context *context, + Aggref *original_aggref) +{ + StringInfo buf = context->buf; + Oid argtypes[FUNC_MAX_ARGS]; + int nargs; + bool use_variadic; + + /* + * For a combining aggregate, we look up and deparse the corresponding + * partial aggregate instead. This is necessary because our input + * argument list has been replaced; the new argument list always has just + * one element, which will point to a partial Aggref that supplies us with + * transition states to combine. + */ + if (DO_AGGSPLIT_COMBINE(aggref->aggsplit)) + { + TargetEntry *tle = linitial_node(TargetEntry, aggref->args); + + Assert(list_length(aggref->args) == 1); + resolve_special_varno((Node *) tle->expr, context, original_aggref, + get_agg_combine_expr); + return; + } + + /* + * Mark as PARTIAL, if appropriate. We look to the original aggref so as + * to avoid printing this when recursing from the code just above. + */ + if (DO_AGGSPLIT_SKIPFINAL(original_aggref->aggsplit)) + appendStringInfoString(buf, "PARTIAL "); + + /* Extract the argument types as seen by the parser */ + nargs = get_aggregate_argtypes(aggref, argtypes); + + /* Print the aggregate name, schema-qualified if needed */ + appendStringInfo(buf, "%s(%s", + generate_function_name(aggref->aggfnoid, nargs, + NIL, argtypes, + aggref->aggvariadic, + &use_variadic, + context->special_exprkind), + (aggref->aggdistinct != NIL) ? "DISTINCT " : ""); + + if (AGGKIND_IS_ORDERED_SET(aggref->aggkind)) + { + /* + * Ordered-set aggregates do not use "*" syntax. Also, we needn't + * worry about inserting VARIADIC. So we can just dump the direct + * args as-is. + */ + Assert(!aggref->aggvariadic); + get_rule_expr((Node *) aggref->aggdirectargs, context, true); + Assert(aggref->aggorder != NIL); + appendStringInfoString(buf, ") WITHIN GROUP (ORDER BY "); + get_rule_orderby(aggref->aggorder, aggref->args, false, context); + } + else + { + /* aggstar can be set only in zero-argument aggregates */ + if (aggref->aggstar) + appendStringInfoChar(buf, '*'); + else + { + ListCell *l; + int i; + + i = 0; + foreach(l, aggref->args) + { + TargetEntry *tle = (TargetEntry *) lfirst(l); + Node *arg = (Node *) tle->expr; + + Assert(!IsA(arg, NamedArgExpr)); + if (tle->resjunk) + continue; + if (i++ > 0) + appendStringInfoString(buf, ", "); + if (use_variadic && i == nargs) + appendStringInfoString(buf, "VARIADIC "); + get_rule_expr(arg, context, true); + } + } + + if (aggref->aggorder != NIL) + { + appendStringInfoString(buf, " ORDER BY "); + get_rule_orderby(aggref->aggorder, aggref->args, false, context); + } + } + + if (aggref->aggfilter != NULL) + { + appendStringInfoString(buf, ") FILTER (WHERE "); + get_rule_expr((Node *) aggref->aggfilter, context, false); + } + + appendStringInfoChar(buf, ')'); +} + +/* + * This is a helper function for get_agg_expr(). It's used when we deparse + * a combining Aggref; resolve_special_varno locates the corresponding partial + * Aggref and then calls this. + */ +static void +get_agg_combine_expr(Node *node, deparse_context *context, void *private) +{ + Aggref *aggref; + Aggref *original_aggref = private; + + if (!IsA(node, Aggref)) + elog(ERROR, "combining Aggref does not point to an Aggref"); + + aggref = (Aggref *) node; + get_agg_expr(aggref, context, original_aggref); +} + +/* + * get_windowfunc_expr - Parse back a WindowFunc node + */ +static void +get_windowfunc_expr(WindowFunc *wfunc, deparse_context *context) +{ + StringInfo buf = context->buf; + Oid argtypes[FUNC_MAX_ARGS]; + int nargs; + List *argnames; + ListCell *l; + + if (list_length(wfunc->args) > FUNC_MAX_ARGS) + ereport(ERROR, + (errcode(ERRCODE_TOO_MANY_ARGUMENTS), + errmsg("too many arguments"))); + nargs = 0; + argnames = NIL; + foreach(l, wfunc->args) + { + Node *arg = (Node *) lfirst(l); + + if (IsA(arg, NamedArgExpr)) + argnames = lappend(argnames, ((NamedArgExpr *) arg)->name); + argtypes[nargs] = exprType(arg); + nargs++; + } + + appendStringInfo(buf, "%s(", + generate_function_name(wfunc->winfnoid, nargs, + argnames, argtypes, + false, NULL, + context->special_exprkind)); + /* winstar can be set only in zero-argument aggregates */ + if (wfunc->winstar) + appendStringInfoChar(buf, '*'); + else + get_rule_expr((Node *) wfunc->args, context, true); + + if (wfunc->aggfilter != NULL) + { + appendStringInfoString(buf, ") FILTER (WHERE "); + get_rule_expr((Node *) wfunc->aggfilter, context, false); + } + + appendStringInfoString(buf, ") OVER "); + + foreach(l, context->windowClause) + { + WindowClause *wc = (WindowClause *) lfirst(l); + + if (wc->winref == wfunc->winref) + { + if (wc->name) + appendStringInfoString(buf, quote_identifier(wc->name)); + else + get_rule_windowspec(wc, context->windowTList, context); + break; + } + } + if (l == NULL) + { + if (context->windowClause) + elog(ERROR, "could not find window clause for winref %u", + wfunc->winref); + + /* + * In EXPLAIN, we don't have window context information available, so + * we have to settle for this: + */ + appendStringInfoString(buf, "(?)"); + } +} + +/* ---------- + * get_coercion_expr + * + * Make a string representation of a value coerced to a specific type + * ---------- + */ +static void +get_coercion_expr(Node *arg, deparse_context *context, + Oid resulttype, int32 resulttypmod, + Node *parentNode) +{ + StringInfo buf = context->buf; + + /* + * Since parse_coerce.c doesn't immediately collapse application of + * length-coercion functions to constants, what we'll typically see in + * such cases is a Const with typmod -1 and a length-coercion function + * right above it. Avoid generating redundant output. However, beware of + * suppressing casts when the user actually wrote something like + * 'foo'::text::char(3). + * + * Note: it might seem that we are missing the possibility of needing to + * print a COLLATE clause for such a Const. However, a Const could only + * have nondefault collation in a post-constant-folding tree, in which the + * length coercion would have been folded too. See also the special + * handling of CollateExpr in coerce_to_target_type(): any collation + * marking will be above the coercion node, not below it. + */ + if (arg && IsA(arg, Const) && + ((Const *) arg)->consttype == resulttype && + ((Const *) arg)->consttypmod == -1) + { + /* Show the constant without normal ::typename decoration */ + get_const_expr((Const *) arg, context, -1); + } + else + { + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr_paren(arg, context, false, parentNode); + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + } + appendStringInfo(buf, "::%s", + format_type_with_typemod(resulttype, resulttypmod)); +} + +/* ---------- + * get_const_expr + * + * Make a string representation of a Const + * + * showtype can be -1 to never show "::typename" decoration, or +1 to always + * show it, or 0 to show it only if the constant wouldn't be assumed to be + * the right type by default. + * + * If the Const's collation isn't default for its type, show that too. + * We mustn't do this when showtype is -1 (since that means the caller will + * print "::typename", and we can't put a COLLATE clause in between). It's + * caller's responsibility that collation isn't missed in such cases. + * ---------- + */ +static void +get_const_expr(Const *constval, deparse_context *context, int showtype) +{ + StringInfo buf = context->buf; + Oid typoutput; + bool typIsVarlena; + char *extval; + bool needlabel = false; + + if (constval->constisnull) + { + /* + * Always label the type of a NULL constant to prevent misdecisions + * about type when reparsing. + */ + appendStringInfoString(buf, "NULL"); + if (showtype >= 0) + { + appendStringInfo(buf, "::%s", + format_type_with_typemod(constval->consttype, + constval->consttypmod)); + get_const_collation(constval, context); + } + return; + } + + getTypeOutputInfo(constval->consttype, + &typoutput, &typIsVarlena); + + extval = OidOutputFunctionCall(typoutput, constval->constvalue); + + switch (constval->consttype) + { + case INT4OID: + + /* + * INT4 can be printed without any decoration, unless it is + * negative; in that case print it as '-nnn'::integer to ensure + * that the output will re-parse as a constant, not as a constant + * plus operator. In most cases we could get away with printing + * (-nnn) instead, because of the way that gram.y handles negative + * literals; but that doesn't work for INT_MIN, and it doesn't + * seem that much prettier anyway. + */ + if (extval[0] != '-') + appendStringInfoString(buf, extval); + else + { + appendStringInfo(buf, "'%s'", extval); + needlabel = true; /* we must attach a cast */ + } + break; + + case NUMERICOID: + + /* + * NUMERIC can be printed without quotes if it looks like a float + * constant (not an integer, and not Infinity or NaN) and doesn't + * have a leading sign (for the same reason as for INT4). + */ + if (isdigit((unsigned char) extval[0]) && + strcspn(extval, "eE.") != strlen(extval)) + { + appendStringInfoString(buf, extval); + } + else + { + appendStringInfo(buf, "'%s'", extval); + needlabel = true; /* we must attach a cast */ + } + break; + + case BITOID: + case VARBITOID: + appendStringInfo(buf, "B'%s'", extval); + break; + + case BOOLOID: + if (strcmp(extval, "t") == 0) + appendStringInfoString(buf, "true"); + else + appendStringInfoString(buf, "false"); + break; + + default: + simple_quote_literal(buf, extval); + break; + } + + pfree(extval); + + if (showtype < 0) + return; + + /* + * For showtype == 0, append ::typename unless the constant will be + * implicitly typed as the right type when it is read in. + * + * XXX this code has to be kept in sync with the behavior of the parser, + * especially make_const. + */ + switch (constval->consttype) + { + case BOOLOID: + case UNKNOWNOID: + /* These types can be left unlabeled */ + needlabel = false; + break; + case INT4OID: + /* We determined above whether a label is needed */ + break; + case NUMERICOID: + + /* + * Float-looking constants will be typed as numeric, which we + * checked above; but if there's a nondefault typmod we need to + * show it. + */ + needlabel |= (constval->consttypmod >= 0); + break; + default: + needlabel = true; + break; + } + if (needlabel || showtype > 0) + appendStringInfo(buf, "::%s", + format_type_with_typemod(constval->consttype, + constval->consttypmod)); + + get_const_collation(constval, context); +} + +/* + * helper for get_const_expr: append COLLATE if needed + */ +static void +get_const_collation(Const *constval, deparse_context *context) +{ + StringInfo buf = context->buf; + + if (OidIsValid(constval->constcollid)) + { + Oid typcollation = get_typcollation(constval->consttype); + + if (constval->constcollid != typcollation) + { + appendStringInfo(buf, " COLLATE %s", + generate_collation_name(constval->constcollid)); + } + } +} + +/* + * simple_quote_literal - Format a string as a SQL literal, append to buf + */ +static void +simple_quote_literal(StringInfo buf, const char *val) +{ + const char *valptr; + + /* + * We form the string literal according to the prevailing setting of + * standard_conforming_strings; we never use E''. User is responsible for + * making sure result is used correctly. + */ + appendStringInfoChar(buf, '\''); + for (valptr = val; *valptr; valptr++) + { + char ch = *valptr; + + if (SQL_STR_DOUBLE(ch, !standard_conforming_strings)) + appendStringInfoChar(buf, ch); + appendStringInfoChar(buf, ch); + } + appendStringInfoChar(buf, '\''); +} + + +/* ---------- + * get_sublink_expr - Parse back a sublink + * ---------- + */ +static void +get_sublink_expr(SubLink *sublink, deparse_context *context) +{ + StringInfo buf = context->buf; + Query *query = (Query *) (sublink->subselect); + char *opname = NULL; + bool need_paren; + + if (sublink->subLinkType == ARRAY_SUBLINK) + appendStringInfoString(buf, "ARRAY("); + else + appendStringInfoChar(buf, '('); + + /* + * Note that we print the name of only the first operator, when there are + * multiple combining operators. This is an approximation that could go + * wrong in various scenarios (operators in different schemas, renamed + * operators, etc) but there is not a whole lot we can do about it, since + * the syntax allows only one operator to be shown. + */ + if (sublink->testexpr) + { + if (IsA(sublink->testexpr, OpExpr)) + { + /* single combining operator */ + OpExpr *opexpr = (OpExpr *) sublink->testexpr; + + get_rule_expr(linitial(opexpr->args), context, true); + opname = generate_operator_name(opexpr->opno, + exprType(linitial(opexpr->args)), + exprType(lsecond(opexpr->args))); + } + else if (IsA(sublink->testexpr, BoolExpr)) + { + /* multiple combining operators, = or <> cases */ + char *sep; + ListCell *l; + + appendStringInfoChar(buf, '('); + sep = ""; + foreach(l, ((BoolExpr *) sublink->testexpr)->args) + { + OpExpr *opexpr = lfirst_node(OpExpr, l); + + appendStringInfoString(buf, sep); + get_rule_expr(linitial(opexpr->args), context, true); + if (!opname) + opname = generate_operator_name(opexpr->opno, + exprType(linitial(opexpr->args)), + exprType(lsecond(opexpr->args))); + sep = ", "; + } + appendStringInfoChar(buf, ')'); + } + else if (IsA(sublink->testexpr, RowCompareExpr)) + { + /* multiple combining operators, < <= > >= cases */ + RowCompareExpr *rcexpr = (RowCompareExpr *) sublink->testexpr; + + appendStringInfoChar(buf, '('); + get_rule_expr((Node *) rcexpr->largs, context, true); + opname = generate_operator_name(linitial_oid(rcexpr->opnos), + exprType(linitial(rcexpr->largs)), + exprType(linitial(rcexpr->rargs))); + appendStringInfoChar(buf, ')'); + } + else + elog(ERROR, "unrecognized testexpr type: %d", + (int) nodeTag(sublink->testexpr)); + } + + need_paren = true; + + switch (sublink->subLinkType) + { + case EXISTS_SUBLINK: + appendStringInfoString(buf, "EXISTS "); + break; + + case ANY_SUBLINK: + if (strcmp(opname, "=") == 0) /* Represent = ANY as IN */ + appendStringInfoString(buf, " IN "); + else + appendStringInfo(buf, " %s ANY ", opname); + break; + + case ALL_SUBLINK: + appendStringInfo(buf, " %s ALL ", opname); + break; + + case ROWCOMPARE_SUBLINK: + appendStringInfo(buf, " %s ", opname); + break; + + case EXPR_SUBLINK: + case MULTIEXPR_SUBLINK: + case ARRAY_SUBLINK: + need_paren = false; + break; + + case CTE_SUBLINK: /* shouldn't occur in a SubLink */ + default: + elog(ERROR, "unrecognized sublink type: %d", + (int) sublink->subLinkType); + break; + } + + if (need_paren) + appendStringInfoChar(buf, '('); + + get_query_def(query, buf, context->namespaces, NULL, + context->prettyFlags, context->wrapColumn, + context->indentLevel); + + if (need_paren) + appendStringInfoString(buf, "))"); + else + appendStringInfoChar(buf, ')'); +} + + +/* ---------- + * get_tablefunc - Parse back a table function + * ---------- + */ +static void +get_tablefunc(TableFunc *tf, deparse_context *context, bool showimplicit) +{ + StringInfo buf = context->buf; + + /* XMLTABLE is the only existing implementation. */ + + appendStringInfoString(buf, "XMLTABLE("); + + if (tf->ns_uris != NIL) + { + ListCell *lc1, + *lc2; + bool first = true; + + appendStringInfoString(buf, "XMLNAMESPACES ("); + forboth(lc1, tf->ns_uris, lc2, tf->ns_names) + { + Node *expr = (Node *) lfirst(lc1); + char *name = strVal(lfirst(lc2)); + + if (!first) + appendStringInfoString(buf, ", "); + else + first = false; + + if (name != NULL) + { + get_rule_expr(expr, context, showimplicit); + appendStringInfo(buf, " AS %s", name); + } + else + { + appendStringInfoString(buf, "DEFAULT "); + get_rule_expr(expr, context, showimplicit); + } + } + appendStringInfoString(buf, "), "); + } + + appendStringInfoChar(buf, '('); + get_rule_expr((Node *) tf->rowexpr, context, showimplicit); + appendStringInfoString(buf, ") PASSING ("); + get_rule_expr((Node *) tf->docexpr, context, showimplicit); + appendStringInfoChar(buf, ')'); + + if (tf->colexprs != NIL) + { + ListCell *l1; + ListCell *l2; + ListCell *l3; + ListCell *l4; + ListCell *l5; + int colnum = 0; + + appendStringInfoString(buf, " COLUMNS "); + forfive(l1, tf->colnames, l2, tf->coltypes, l3, tf->coltypmods, + l4, tf->colexprs, l5, tf->coldefexprs) + { + char *colname = strVal(lfirst(l1)); + Oid typid = lfirst_oid(l2); + int32 typmod = lfirst_int(l3); + Node *colexpr = (Node *) lfirst(l4); + Node *coldefexpr = (Node *) lfirst(l5); + bool ordinality = (tf->ordinalitycol == colnum); + bool notnull = bms_is_member(colnum, tf->notnulls); + + if (colnum > 0) + appendStringInfoString(buf, ", "); + colnum++; + + appendStringInfo(buf, "%s %s", quote_identifier(colname), + ordinality ? "FOR ORDINALITY" : + format_type_with_typemod(typid, typmod)); + if (ordinality) + continue; + + if (coldefexpr != NULL) + { + appendStringInfoString(buf, " DEFAULT ("); + get_rule_expr((Node *) coldefexpr, context, showimplicit); + appendStringInfoChar(buf, ')'); + } + if (colexpr != NULL) + { + appendStringInfoString(buf, " PATH ("); + get_rule_expr((Node *) colexpr, context, showimplicit); + appendStringInfoChar(buf, ')'); + } + if (notnull) + appendStringInfoString(buf, " NOT NULL"); + } + } + + appendStringInfoChar(buf, ')'); +} + +/* ---------- + * get_from_clause - Parse back a FROM clause + * + * "prefix" is the keyword that denotes the start of the list of FROM + * elements. It is FROM when used to parse back SELECT and UPDATE, but + * is USING when parsing back DELETE. + * ---------- + */ +static void +get_from_clause(Query *query, const char *prefix, deparse_context *context) +{ + StringInfo buf = context->buf; + bool first = true; + ListCell *l; + + /* + * We use the query's jointree as a guide to what to print. However, we + * must ignore auto-added RTEs that are marked not inFromCl. (These can + * only appear at the top level of the jointree, so it's sufficient to + * check here.) This check also ensures we ignore the rule pseudo-RTEs + * for NEW and OLD. + */ + foreach(l, query->jointree->fromlist) + { + Node *jtnode = (Node *) lfirst(l); + + if (IsA(jtnode, RangeTblRef)) + { + int varno = ((RangeTblRef *) jtnode)->rtindex; + RangeTblEntry *rte = rt_fetch(varno, query->rtable); + + if (!rte->inFromCl) + continue; + } + + if (first) + { + appendContextKeyword(context, prefix, + -PRETTYINDENT_STD, PRETTYINDENT_STD, 2); + first = false; + + get_from_clause_item(jtnode, query, context); + } + else + { + StringInfoData itembuf; + + appendStringInfoString(buf, ", "); + + /* + * Put the new FROM item's text into itembuf so we can decide + * after we've got it whether or not it needs to go on a new line. + */ + initStringInfo(&itembuf); + context->buf = &itembuf; + + get_from_clause_item(jtnode, query, context); + + /* Restore context's output buffer */ + context->buf = buf; + + /* Consider line-wrapping if enabled */ + if (PRETTY_INDENT(context) && context->wrapColumn >= 0) + { + /* Does the new item start with a new line? */ + if (itembuf.len > 0 && itembuf.data[0] == '\n') + { + /* If so, we shouldn't add anything */ + /* instead, remove any trailing spaces currently in buf */ + removeStringInfoSpaces(buf); + } + else + { + char *trailing_nl; + + /* Locate the start of the current line in the buffer */ + trailing_nl = strrchr(buf->data, '\n'); + if (trailing_nl == NULL) + trailing_nl = buf->data; + else + trailing_nl++; + + /* + * Add a newline, plus some indentation, if the new item + * would cause an overflow. + */ + if (strlen(trailing_nl) + itembuf.len > context->wrapColumn) + appendContextKeyword(context, "", -PRETTYINDENT_STD, + PRETTYINDENT_STD, + PRETTYINDENT_VAR); + } + } + + /* Add the new item */ + appendStringInfoString(buf, itembuf.data); + + /* clean up */ + pfree(itembuf.data); + } + } +} + +static void +get_from_clause_item(Node *jtnode, Query *query, deparse_context *context) +{ + StringInfo buf = context->buf; + deparse_namespace *dpns = (deparse_namespace *) linitial(context->namespaces); + + if (IsA(jtnode, RangeTblRef)) + { + int varno = ((RangeTblRef *) jtnode)->rtindex; + RangeTblEntry *rte = rt_fetch(varno, query->rtable); + char *refname = get_rtable_name(varno, context); + deparse_columns *colinfo = deparse_columns_fetch(varno, dpns); + RangeTblFunction *rtfunc1 = NULL; + bool printalias; + CitusRTEKind rteKind = GetRangeTblKind(rte); + + if (rte->lateral) + appendStringInfoString(buf, "LATERAL "); + + /* Print the FROM item proper */ + switch (rte->rtekind) + { + case RTE_RELATION: + /* Normal relation RTE */ + appendStringInfo(buf, "%s%s", + only_marker(rte), + generate_relation_or_shard_name(rte->relid, + context->distrelid, + context->shardid, + context->namespaces)); + break; + case RTE_SUBQUERY: + /* Subquery RTE */ + appendStringInfoChar(buf, '('); + get_query_def(rte->subquery, buf, context->namespaces, NULL, + context->prettyFlags, context->wrapColumn, + context->indentLevel); + appendStringInfoChar(buf, ')'); + break; + case RTE_FUNCTION: + /* if it's a shard, do differently */ + if (GetRangeTblKind(rte) == CITUS_RTE_SHARD) + { + char *fragmentSchemaName = NULL; + char *fragmentTableName = NULL; + + ExtractRangeTblExtraData(rte, NULL, &fragmentSchemaName, &fragmentTableName, NULL); + + /* use schema and table name from the remote alias */ + appendStringInfoString(buf, + generate_fragment_name(fragmentSchemaName, + fragmentTableName)); + break; + } + + /* Function RTE */ + rtfunc1 = (RangeTblFunction *) linitial(rte->functions); + + /* + * Omit ROWS FROM() syntax for just one function, unless it + * has both a coldeflist and WITH ORDINALITY. If it has both, + * we must use ROWS FROM() syntax to avoid ambiguity about + * whether the coldeflist includes the ordinality column. + */ + if (list_length(rte->functions) == 1 && + (rtfunc1->funccolnames == NIL || !rte->funcordinality)) + { + get_rule_expr_funccall(rtfunc1->funcexpr, context, true); + /* we'll print the coldeflist below, if it has one */ + } + else + { + bool all_unnest; + ListCell *lc; + + /* + * If all the function calls in the list are to unnest, + * and none need a coldeflist, then collapse the list back + * down to UNNEST(args). (If we had more than one + * built-in unnest function, this would get more + * difficult.) + * + * XXX This is pretty ugly, since it makes not-terribly- + * future-proof assumptions about what the parser would do + * with the output; but the alternative is to emit our + * nonstandard ROWS FROM() notation for what might have + * been a perfectly spec-compliant multi-argument + * UNNEST(). + */ + all_unnest = true; + foreach(lc, rte->functions) + { + RangeTblFunction *rtfunc = (RangeTblFunction *) lfirst(lc); + + if (!IsA(rtfunc->funcexpr, FuncExpr) || + ((FuncExpr *) rtfunc->funcexpr)->funcid != F_ARRAY_UNNEST || + rtfunc->funccolnames != NIL) + { + all_unnest = false; + break; + } + } + + if (all_unnest) + { + List *allargs = NIL; + + foreach(lc, rte->functions) + { + RangeTblFunction *rtfunc = (RangeTblFunction *) lfirst(lc); + List *args = ((FuncExpr *) rtfunc->funcexpr)->args; + + allargs = list_concat(allargs, list_copy(args)); + } + + appendStringInfoString(buf, "UNNEST("); + get_rule_expr((Node *) allargs, context, true); + appendStringInfoChar(buf, ')'); + } + else + { + int funcno = 0; + + appendStringInfoString(buf, "ROWS FROM("); + foreach(lc, rte->functions) + { + RangeTblFunction *rtfunc = (RangeTblFunction *) lfirst(lc); + + if (funcno > 0) + appendStringInfoString(buf, ", "); + get_rule_expr_funccall(rtfunc->funcexpr, context, true); + if (rtfunc->funccolnames != NIL) + { + /* Reconstruct the column definition list */ + appendStringInfoString(buf, " AS "); + get_from_clause_coldeflist(rtfunc, + NULL, + context); + } + funcno++; + } + appendStringInfoChar(buf, ')'); + } + /* prevent printing duplicate coldeflist below */ + rtfunc1 = NULL; + } + if (rte->funcordinality) + appendStringInfoString(buf, " WITH ORDINALITY"); + break; + case RTE_TABLEFUNC: + get_tablefunc(rte->tablefunc, context, true); + break; + case RTE_VALUES: + /* Values list RTE */ + appendStringInfoChar(buf, '('); + get_values_def(rte->values_lists, context); + appendStringInfoChar(buf, ')'); + break; + case RTE_CTE: + appendStringInfoString(buf, quote_identifier(rte->ctename)); + break; + default: + elog(ERROR, "unrecognized RTE kind: %d", (int) rte->rtekind); + break; + } + + /* Print the relation alias, if needed */ + printalias = false; + if (rte->alias != NULL) + { + /* Always print alias if user provided one */ + printalias = true; + } + else if (colinfo->printaliases) + { + /* Always print alias if we need to print column aliases */ + printalias = true; + } + else if (rte->rtekind == RTE_RELATION) + { + /* + * No need to print alias if it's same as relation name (this + * would normally be the case, but not if set_rtable_names had to + * resolve a conflict). + */ + if (strcmp(refname, get_relation_name(rte->relid)) != 0) + printalias = true; + } + else if (rte->rtekind == RTE_FUNCTION) + { + /* + * For a function RTE, always print alias. This covers possible + * renaming of the function and/or instability of the + * FigureColname rules for things that aren't simple functions. + * Note we'd need to force it anyway for the columndef list case. + */ + printalias = true; + } + else if (rte->rtekind == RTE_VALUES) + { + /* Alias is syntactically required for VALUES */ + printalias = true; + } + else if (rte->rtekind == RTE_CTE) + { + /* + * No need to print alias if it's same as CTE name (this would + * normally be the case, but not if set_rtable_names had to + * resolve a conflict). + */ + if (strcmp(refname, rte->ctename) != 0) + printalias = true; + } + else if (rte->rtekind == RTE_SUBQUERY) + { + /* subquery requires alias too */ + printalias = true; + } + if (printalias) + appendStringInfo(buf, " %s", quote_identifier(refname)); + + /* Print the column definitions or aliases, if needed */ + if (rtfunc1 && rtfunc1->funccolnames != NIL) + { + /* Reconstruct the columndef list, which is also the aliases */ + get_from_clause_coldeflist(rtfunc1, colinfo, context); + } + else if (GetRangeTblKind(rte) != CITUS_RTE_SHARD) + { + /* Else print column aliases as needed */ + get_column_alias_list(colinfo, context); + } + /* check if column's are given aliases in distributed tables */ + else if (colinfo->parentUsing != NIL) + { + Assert(colinfo->printaliases); + get_column_alias_list(colinfo, context); + } + + /* Tablesample clause must go after any alias */ + if ((rteKind == CITUS_RTE_RELATION || rteKind == CITUS_RTE_SHARD) && + rte->tablesample) + { + get_tablesample_def(rte->tablesample, context); + } + } + else if (IsA(jtnode, JoinExpr)) + { + JoinExpr *j = (JoinExpr *) jtnode; + deparse_columns *colinfo = deparse_columns_fetch(j->rtindex, dpns); + bool need_paren_on_right; + + need_paren_on_right = PRETTY_PAREN(context) && + !IsA(j->rarg, RangeTblRef) && + !(IsA(j->rarg, JoinExpr) &&((JoinExpr *) j->rarg)->alias != NULL); + + if (!PRETTY_PAREN(context) || j->alias != NULL) + appendStringInfoChar(buf, '('); + + get_from_clause_item(j->larg, query, context); + + switch (j->jointype) + { + case JOIN_INNER: + if (j->quals) + appendContextKeyword(context, " JOIN ", + -PRETTYINDENT_STD, + PRETTYINDENT_STD, + PRETTYINDENT_JOIN); + else + appendContextKeyword(context, " CROSS JOIN ", + -PRETTYINDENT_STD, + PRETTYINDENT_STD, + PRETTYINDENT_JOIN); + break; + case JOIN_LEFT: + appendContextKeyword(context, " LEFT JOIN ", + -PRETTYINDENT_STD, + PRETTYINDENT_STD, + PRETTYINDENT_JOIN); + break; + case JOIN_FULL: + appendContextKeyword(context, " FULL JOIN ", + -PRETTYINDENT_STD, + PRETTYINDENT_STD, + PRETTYINDENT_JOIN); + break; + case JOIN_RIGHT: + appendContextKeyword(context, " RIGHT JOIN ", + -PRETTYINDENT_STD, + PRETTYINDENT_STD, + PRETTYINDENT_JOIN); + break; + default: + elog(ERROR, "unrecognized join type: %d", + (int) j->jointype); + } + + if (need_paren_on_right) + appendStringInfoChar(buf, '('); + get_from_clause_item(j->rarg, query, context); + if (need_paren_on_right) + appendStringInfoChar(buf, ')'); + + if (j->usingClause) + { + ListCell *lc; + bool first = true; + + appendStringInfoString(buf, " USING ("); + /* Use the assigned names, not what's in usingClause */ + foreach(lc, colinfo->usingNames) + { + char *colname = (char *) lfirst(lc); + + if (first) + first = false; + else + appendStringInfoString(buf, ", "); + appendStringInfoString(buf, quote_identifier(colname)); + } + appendStringInfoChar(buf, ')'); + } + else if (j->quals) + { + appendStringInfoString(buf, " ON "); + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, '('); + get_rule_expr(j->quals, context, false); + if (!PRETTY_PAREN(context)) + appendStringInfoChar(buf, ')'); + } + else if (j->jointype != JOIN_INNER) + { + /* If we didn't say CROSS JOIN above, we must provide an ON */ + appendStringInfoString(buf, " ON TRUE"); + } + + if (!PRETTY_PAREN(context) || j->alias != NULL) + appendStringInfoChar(buf, ')'); + + /* Yes, it's correct to put alias after the right paren ... */ + if (j->alias != NULL) + { + /* + * Note that it's correct to emit an alias clause if and only if + * there was one originally. Otherwise we'd be converting a named + * join to unnamed or vice versa, which creates semantic + * subtleties we don't want. However, we might print a different + * alias name than was there originally. + */ + appendStringInfo(buf, " %s", + quote_identifier(get_rtable_name(j->rtindex, + context))); + get_column_alias_list(colinfo, context); + } + } + else + elog(ERROR, "unrecognized node type: %d", + (int) nodeTag(jtnode)); +} + +/* + * get_column_alias_list - print column alias list for an RTE + * + * Caller must already have printed the relation's alias name. + */ +static void +get_column_alias_list(deparse_columns *colinfo, deparse_context *context) +{ + StringInfo buf = context->buf; + int i; + bool first = true; + + /* Don't print aliases if not needed */ + if (!colinfo->printaliases) + return; + + for (i = 0; i < colinfo->num_new_cols; i++) + { + char *colname = colinfo->new_colnames[i]; + + if (first) + { + appendStringInfoChar(buf, '('); + first = false; + } + else + appendStringInfoString(buf, ", "); + appendStringInfoString(buf, quote_identifier(colname)); + } + if (!first) + appendStringInfoChar(buf, ')'); +} + +/* + * get_from_clause_coldeflist - reproduce FROM clause coldeflist + * + * When printing a top-level coldeflist (which is syntactically also the + * relation's column alias list), use column names from colinfo. But when + * printing a coldeflist embedded inside ROWS FROM(), we prefer to use the + * original coldeflist's names, which are available in rtfunc->funccolnames. + * Pass NULL for colinfo to select the latter behavior. + * + * The coldeflist is appended immediately (no space) to buf. Caller is + * responsible for ensuring that an alias or AS is present before it. + */ +static void +get_from_clause_coldeflist(RangeTblFunction *rtfunc, + deparse_columns *colinfo, + deparse_context *context) +{ + StringInfo buf = context->buf; + ListCell *l1; + ListCell *l2; + ListCell *l3; + ListCell *l4; + int i; + + appendStringInfoChar(buf, '('); + + i = 0; + forfour(l1, rtfunc->funccoltypes, + l2, rtfunc->funccoltypmods, + l3, rtfunc->funccolcollations, + l4, rtfunc->funccolnames) + { + Oid atttypid = lfirst_oid(l1); + int32 atttypmod = lfirst_int(l2); + Oid attcollation = lfirst_oid(l3); + char *attname; + + if (colinfo) + attname = colinfo->colnames[i]; + else + attname = strVal(lfirst(l4)); + + Assert(attname); /* shouldn't be any dropped columns here */ + + if (i > 0) + appendStringInfoString(buf, ", "); + appendStringInfo(buf, "%s %s", + quote_identifier(attname), + format_type_with_typemod(atttypid, atttypmod)); + if (OidIsValid(attcollation) && + attcollation != get_typcollation(atttypid)) + appendStringInfo(buf, " COLLATE %s", + generate_collation_name(attcollation)); + + i++; + } + + appendStringInfoChar(buf, ')'); +} + +/* + * get_tablesample_def - print a TableSampleClause + */ +static void +get_tablesample_def(TableSampleClause *tablesample, deparse_context *context) +{ + StringInfo buf = context->buf; + Oid argtypes[1]; + int nargs; + ListCell *l; + + /* + * We should qualify the handler's function name if it wouldn't be + * resolved by lookup in the current search path. + */ + argtypes[0] = INTERNALOID; + appendStringInfo(buf, " TABLESAMPLE %s (", + generate_function_name(tablesample->tsmhandler, 1, + NIL, argtypes, + false, NULL, EXPR_KIND_NONE)); + + nargs = 0; + foreach(l, tablesample->args) + { + if (nargs++ > 0) + appendStringInfoString(buf, ", "); + get_rule_expr((Node *) lfirst(l), context, false); + } + appendStringInfoChar(buf, ')'); + + if (tablesample->repeatable != NULL) + { + appendStringInfoString(buf, " REPEATABLE ("); + get_rule_expr((Node *) tablesample->repeatable, context, false); + appendStringInfoChar(buf, ')'); + } +} + +/* + * get_opclass_name - fetch name of an index operator class + * + * The opclass name is appended (after a space) to buf. + * + * Output is suppressed if the opclass is the default for the given + * actual_datatype. (If you don't want this behavior, just pass + * InvalidOid for actual_datatype.) + */ +static void +get_opclass_name(Oid opclass, Oid actual_datatype, + StringInfo buf) +{ + HeapTuple ht_opc; + Form_pg_opclass opcrec; + char *opcname; + char *nspname; + + ht_opc = SearchSysCache1(CLAOID, ObjectIdGetDatum(opclass)); + if (!HeapTupleIsValid(ht_opc)) + elog(ERROR, "cache lookup failed for opclass %u", opclass); + opcrec = (Form_pg_opclass) GETSTRUCT(ht_opc); + + if (!OidIsValid(actual_datatype) || + GetDefaultOpClass(actual_datatype, opcrec->opcmethod) != opclass) + { + /* Okay, we need the opclass name. Do we need to qualify it? */ + opcname = NameStr(opcrec->opcname); + if (OpclassIsVisible(opclass)) + appendStringInfo(buf, " %s", quote_identifier(opcname)); + else + { + nspname = get_namespace_name(opcrec->opcnamespace); + appendStringInfo(buf, " %s.%s", + quote_identifier(nspname), + quote_identifier(opcname)); + } + } + ReleaseSysCache(ht_opc); +} + +/* + * processIndirection - take care of array and subfield assignment + * + * We strip any top-level FieldStore or assignment SubscriptingRef nodes that + * appear in the input, printing them as decoration for the base column + * name (which we assume the caller just printed). We might also need to + * strip CoerceToDomain nodes, but only ones that appear above assignment + * nodes. + * + * Returns the subexpression that's to be assigned. + */ +static Node * +processIndirection(Node *node, deparse_context *context) +{ + StringInfo buf = context->buf; + CoerceToDomain *cdomain = NULL; + + for (;;) + { + if (node == NULL) + break; + if (IsA(node, FieldStore)) + { + FieldStore *fstore = (FieldStore *) node; + Oid typrelid; + char *fieldname; + + /* lookup tuple type */ + typrelid = get_typ_typrelid(fstore->resulttype); + if (!OidIsValid(typrelid)) + elog(ERROR, "argument type %s of FieldStore is not a tuple type", + format_type_be(fstore->resulttype)); + + /* + * Print the field name. There should only be one target field in + * stored rules. There could be more than that in executable + * target lists, but this function cannot be used for that case. + */ + Assert(list_length(fstore->fieldnums) == 1); + fieldname = get_attname(typrelid, + linitial_int(fstore->fieldnums), false); + appendStringInfo(buf, ".%s", quote_identifier(fieldname)); + + /* + * We ignore arg since it should be an uninteresting reference to + * the target column or subcolumn. + */ + node = (Node *) linitial(fstore->newvals); + } + else if (IsA(node, SubscriptingRef)) + { + SubscriptingRef *sbsref = (SubscriptingRef *) node; + + if (sbsref->refassgnexpr == NULL) + break; + printSubscripts(sbsref, context); + + /* + * We ignore refexpr since it should be an uninteresting reference + * to the target column or subcolumn. + */ + node = (Node *) sbsref->refassgnexpr; + } + else if (IsA(node, CoerceToDomain)) + { + cdomain = (CoerceToDomain *) node; + /* If it's an explicit domain coercion, we're done */ + if (cdomain->coercionformat != COERCE_IMPLICIT_CAST) + break; + /* Tentatively descend past the CoerceToDomain */ + node = (Node *) cdomain->arg; + } + else + break; + } + + /* + * If we descended past a CoerceToDomain whose argument turned out not to + * be a FieldStore or array assignment, back up to the CoerceToDomain. + * (This is not enough to be fully correct if there are nested implicit + * CoerceToDomains, but such cases shouldn't ever occur.) + */ + if (cdomain && node == (Node *) cdomain->arg) + node = (Node *) cdomain; + + return node; +} + +static void +printSubscripts(SubscriptingRef *sbsref, deparse_context *context) +{ + StringInfo buf = context->buf; + ListCell *lowlist_item; + ListCell *uplist_item; + + lowlist_item = list_head(sbsref->reflowerindexpr); /* could be NULL */ + foreach(uplist_item, sbsref->refupperindexpr) + { + appendStringInfoChar(buf, '['); + if (lowlist_item) + { + /* If subexpression is NULL, get_rule_expr prints nothing */ + get_rule_expr((Node *) lfirst(lowlist_item), context, false); + appendStringInfoChar(buf, ':'); + lowlist_item = lnext(lowlist_item); + } + /* If subexpression is NULL, get_rule_expr prints nothing */ + get_rule_expr((Node *) lfirst(uplist_item), context, false); + appendStringInfoChar(buf, ']'); + } +} + +/* + * get_relation_name + * Get the unqualified name of a relation specified by OID + * + * This differs from the underlying get_rel_name() function in that it will + * throw error instead of silently returning NULL if the OID is bad. + */ +static char * +get_relation_name(Oid relid) +{ + char *relname = get_rel_name(relid); + + if (!relname) + elog(ERROR, "cache lookup failed for relation %u", relid); + return relname; +} + +/* + * generate_relation_or_shard_name + * Compute the name to display for a relation or shard + * + * If the provided relid is equal to the provided distrelid, this function + * returns a shard-extended relation name; otherwise, it falls through to a + * simple generate_relation_name call. + */ +static char * +generate_relation_or_shard_name(Oid relid, Oid distrelid, int64 shardid, + List *namespaces) +{ + char *relname = NULL; + + if (relid == distrelid) + { + relname = get_relation_name(relid); + + if (shardid > 0) + { + Oid schemaOid = get_rel_namespace(relid); + char *schemaName = get_namespace_name(schemaOid); + + AppendShardIdToName(&relname, shardid); + + relname = quote_qualified_identifier(schemaName, relname); + } + } + else + { + relname = generate_relation_name(relid, namespaces); + } + + return relname; +} + +/* + * generate_relation_name + * Compute the name to display for a relation specified by OID + * + * The result includes all necessary quoting and schema-prefixing. + * + * If namespaces isn't NIL, it must be a list of deparse_namespace nodes. + * We will forcibly qualify the relation name if it equals any CTE name + * visible in the namespace list. + */ +char * +generate_relation_name(Oid relid, List *namespaces) +{ + HeapTuple tp; + Form_pg_class reltup; + bool need_qual; + ListCell *nslist; + char *relname; + char *nspname; + char *result; + + tp = SearchSysCache1(RELOID, ObjectIdGetDatum(relid)); + if (!HeapTupleIsValid(tp)) + elog(ERROR, "cache lookup failed for relation %u", relid); + reltup = (Form_pg_class) GETSTRUCT(tp); + relname = NameStr(reltup->relname); + + /* Check for conflicting CTE name */ + need_qual = false; + foreach(nslist, namespaces) + { + deparse_namespace *dpns = (deparse_namespace *) lfirst(nslist); + ListCell *ctlist; + + foreach(ctlist, dpns->ctes) + { + CommonTableExpr *cte = (CommonTableExpr *) lfirst(ctlist); + + if (strcmp(cte->ctename, relname) == 0) + { + need_qual = true; + break; + } + } + if (need_qual) + break; + } + + /* Otherwise, qualify the name if not visible in search path */ + if (!need_qual) + need_qual = !RelationIsVisible(relid); + + if (need_qual) + nspname = get_namespace_name(reltup->relnamespace); + else + nspname = NULL; + + result = quote_qualified_identifier(nspname, relname); + + ReleaseSysCache(tp); + + return result; +} + +/* + * generate_fragment_name + * Compute the name to display for a shard or merged table + * + * The result includes all necessary quoting and schema-prefixing. The schema + * name can be NULL for regular shards. For merged tables, they are always + * declared within a job-specific schema, and therefore can't have null schema + * names. + */ +static char * +generate_fragment_name(char *schemaName, char *tableName) +{ + StringInfo fragmentNameString = makeStringInfo(); + + if (schemaName != NULL) + { + appendStringInfo(fragmentNameString, "%s.%s", quote_identifier(schemaName), + quote_identifier(tableName)); + } + else + { + appendStringInfoString(fragmentNameString, quote_identifier(tableName)); + } + + return fragmentNameString->data; +} + +/* + * generate_function_name + * Compute the name to display for a function specified by OID, + * given that it is being called with the specified actual arg names and + * types. (Those matter because of ambiguous-function resolution rules.) + * + * If we're dealing with a potentially variadic function (in practice, this + * means a FuncExpr or Aggref, not some other way of calling a function), then + * has_variadic must specify whether variadic arguments have been merged, + * and *use_variadic_p will be set to indicate whether to print VARIADIC in + * the output. For non-FuncExpr cases, has_variadic should be false and + * use_variadic_p can be NULL. + * + * The result includes all necessary quoting and schema-prefixing. + */ +static char * +generate_function_name(Oid funcid, int nargs, List *argnames, Oid *argtypes, + bool has_variadic, bool *use_variadic_p, + ParseExprKind special_exprkind) +{ + char *result; + HeapTuple proctup; + Form_pg_proc procform; + char *proname; + bool use_variadic; + char *nspname; + FuncDetailCode p_result; + Oid p_funcid; + Oid p_rettype; + bool p_retset; + int p_nvargs; + Oid p_vatype; + Oid *p_true_typeids; + bool force_qualify = false; + + proctup = SearchSysCache1(PROCOID, ObjectIdGetDatum(funcid)); + if (!HeapTupleIsValid(proctup)) + elog(ERROR, "cache lookup failed for function %u", funcid); + procform = (Form_pg_proc) GETSTRUCT(proctup); + proname = NameStr(procform->proname); + + /* + * Due to parser hacks to avoid needing to reserve CUBE, we need to force + * qualification in some special cases. + */ + if (special_exprkind == EXPR_KIND_GROUP_BY) + { + if (strcmp(proname, "cube") == 0 || strcmp(proname, "rollup") == 0) + force_qualify = true; + } + + /* + * Determine whether VARIADIC should be printed. We must do this first + * since it affects the lookup rules in func_get_detail(). + * + * Currently, we always print VARIADIC if the function has a merged + * variadic-array argument. Note that this is always the case for + * functions taking a VARIADIC argument type other than VARIADIC ANY. + * + * In principle, if VARIADIC wasn't originally specified and the array + * actual argument is deconstructable, we could print the array elements + * separately and not print VARIADIC, thus more nearly reproducing the + * original input. For the moment that seems like too much complication + * for the benefit, and anyway we do not know whether VARIADIC was + * originally specified if it's a non-ANY type. + */ + if (use_variadic_p) + { + /* Parser should not have set funcvariadic unless fn is variadic */ + Assert(!has_variadic || OidIsValid(procform->provariadic)); + use_variadic = has_variadic; + *use_variadic_p = use_variadic; + } + else + { + Assert(!has_variadic); + use_variadic = false; + } + + /* + * The idea here is to schema-qualify only if the parser would fail to + * resolve the correct function given the unqualified func name with the + * specified argtypes and VARIADIC flag. But if we already decided to + * force qualification, then we can skip the lookup and pretend we didn't + * find it. + */ + if (!force_qualify) + p_result = func_get_detail(list_make1(makeString(proname)), + NIL, argnames, nargs, argtypes, + !use_variadic, true, + &p_funcid, &p_rettype, + &p_retset, &p_nvargs, &p_vatype, + &p_true_typeids, NULL); + else + { + p_result = FUNCDETAIL_NOTFOUND; + p_funcid = InvalidOid; + } + + if ((p_result == FUNCDETAIL_NORMAL || + p_result == FUNCDETAIL_AGGREGATE || + p_result == FUNCDETAIL_WINDOWFUNC) && + p_funcid == funcid) + nspname = NULL; + else + nspname = get_namespace_name(procform->pronamespace); + + result = quote_qualified_identifier(nspname, proname); + + ReleaseSysCache(proctup); + + return result; +} + +/* + * generate_operator_name + * Compute the name to display for an operator specified by OID, + * given that it is being called with the specified actual arg types. + * (Arg types matter because of ambiguous-operator resolution rules. + * Pass InvalidOid for unused arg of a unary operator.) + * + * The result includes all necessary quoting and schema-prefixing, + * plus the OPERATOR() decoration needed to use a qualified operator name + * in an expression. + */ +static char * +generate_operator_name(Oid operid, Oid arg1, Oid arg2) +{ + StringInfoData buf; + HeapTuple opertup; + Form_pg_operator operform; + char *oprname; + char *nspname; + + initStringInfo(&buf); + + opertup = SearchSysCache1(OPEROID, ObjectIdGetDatum(operid)); + if (!HeapTupleIsValid(opertup)) + elog(ERROR, "cache lookup failed for operator %u", operid); + operform = (Form_pg_operator) GETSTRUCT(opertup); + oprname = NameStr(operform->oprname); + + /* + * Unlike generate_operator_name() in postgres/src/backend/utils/adt/ruleutils.c, + * we don't check if the operator is in current namespace or not. This is + * because this check is costly when the operator is not in current namespace. + */ + nspname = get_namespace_name(operform->oprnamespace); + Assert(nspname != NULL); + appendStringInfo(&buf, "OPERATOR(%s.", quote_identifier(nspname)); + appendStringInfoString(&buf, oprname); + appendStringInfoChar(&buf, ')'); + + ReleaseSysCache(opertup); + + return buf.data; +} + +/* + * get_one_range_partition_bound_string + * A C string representation of one range partition bound + */ +char * +get_range_partbound_string(List *bound_datums) +{ + deparse_context context; + StringInfo buf = makeStringInfo(); + ListCell *cell; + char *sep; + + memset(&context, 0, sizeof(deparse_context)); + context.buf = buf; + + appendStringInfoString(buf, "("); + sep = ""; + foreach(cell, bound_datums) + { + PartitionRangeDatum *datum = + castNode(PartitionRangeDatum, lfirst(cell)); + + appendStringInfoString(buf, sep); + if (datum->kind == PARTITION_RANGE_DATUM_MINVALUE) + appendStringInfoString(buf, "MINVALUE"); + else if (datum->kind == PARTITION_RANGE_DATUM_MAXVALUE) + appendStringInfoString(buf, "MAXVALUE"); + else + { + Const *val = castNode(Const, datum->value); + + get_const_expr(val, &context, -1); + } + sep = ", "; + } + appendStringInfoChar(buf, ')'); + + return buf->data; +} + +#endif /* (PG_VERSION_NUM >= 120000) && (PG_VERSION_NUM < 130000) */ diff --git a/src/include/distributed/citus_ruleutils.h b/src/include/distributed/citus_ruleutils.h index c6653c43f..19b4b4118 100644 --- a/src/include/distributed/citus_ruleutils.h +++ b/src/include/distributed/citus_ruleutils.h @@ -12,11 +12,8 @@ #define CITUS_RULEUTILS_H #include "postgres.h" /* IWYU pragma: keep */ -#include "c.h" -#if (PG_VERSION_NUM >= 100000) #include "catalog/pg_sequence.h" -#endif #include "commands/sequence.h" #include "lib/stringinfo.h" #include "nodes/parsenodes.h" From 68c4b71f933e3c40b223cb6fa1c2d9170b0320ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philip=20Dub=C3=A9?= Date: Thu, 8 Aug 2019 21:34:05 +0000 Subject: [PATCH 03/12] Fix up includes with pg12 changes --- .../distributed/commands/foreign_constraint.c | 3 +++ src/backend/distributed/commands/index.c | 3 +++ src/backend/distributed/commands/sequence.c | 4 ---- .../distributed/commands/subscription.c | 5 ----- src/backend/distributed/commands/table.c | 3 +++ .../distributed/executor/adaptive_executor.c | 5 ----- .../master/master_delete_protocol.c | 10 ++++++++- .../master/master_metadata_utility.c | 4 +++- .../master/master_modify_multiple_shards.c | 6 +++++- .../master/master_stage_protocol.c | 1 - .../distributed/master/worker_node_manager.c | 2 -- .../distributed/metadata/metadata_sync.c | 1 - .../planner/extended_op_node_utils.c | 5 +++++ .../planner/fast_path_router_planner.c | 7 +++++++ .../planner/insert_select_planner.c | 4 ++++ .../distributed/planner/multi_join_order.c | 5 +++++ .../planner/multi_logical_optimizer.c | 4 ++++ .../planner/multi_logical_planner.c | 7 ++++++- .../planner/multi_master_planner.c | 4 ++++ .../planner/multi_physical_planner.c | 9 ++++++-- .../planner/multi_router_planner.c | 9 +++++--- .../planner/query_pushdown_planning.c | 7 ++++++- .../distributed/planner/recursive_planning.c | 4 ++++ .../relation_restriction_equivalence.c | 4 ++++ src/backend/distributed/test/fake_fdw.c | 4 ++++ .../distributed/test/prune_shard_list.c | 6 +++++- .../transaction/transaction_recovery.c | 3 +++ .../distributed/utils/citus_outfuncs.c | 4 ++++ .../utils/foreign_key_relationship.c | 6 ++++++ .../utils/multi_partitioning_utils.c | 3 +++ src/include/distributed/commands.h | 3 --- src/include/distributed/distributed_planner.h | 5 +++++ src/include/distributed/hash_helpers.h | 6 ++++++ .../distributed/master_metadata_utility.h | 21 ------------------- .../distributed/multi_physical_planner.h | 5 ++--- src/include/distributed/recursive_planning.h | 4 ++++ src/include/distributed/task_tracker.h | 2 -- 37 files changed, 130 insertions(+), 58 deletions(-) diff --git a/src/backend/distributed/commands/foreign_constraint.c b/src/backend/distributed/commands/foreign_constraint.c index 31ef9747f..85a556c63 100644 --- a/src/backend/distributed/commands/foreign_constraint.c +++ b/src/backend/distributed/commands/foreign_constraint.c @@ -15,6 +15,9 @@ #include "access/htup_details.h" #include "catalog/namespace.h" #include "catalog/pg_constraint.h" +#if (PG_VERSION_NUM >= 120000) +#include "access/genam.h" +#endif #if (PG_VERSION_NUM < 110000) #include "catalog/pg_constraint_fn.h" #endif diff --git a/src/backend/distributed/commands/index.c b/src/backend/distributed/commands/index.c index b53c5d3f9..d49014ec5 100644 --- a/src/backend/distributed/commands/index.c +++ b/src/backend/distributed/commands/index.c @@ -10,6 +10,9 @@ #include "postgres.h" +#if PG_VERSION_NUM > 12000 +#include "access/genam.h" +#endif #include "access/htup_details.h" #include "access/xact.h" #include "catalog/catalog.h" diff --git a/src/backend/distributed/commands/sequence.c b/src/backend/distributed/commands/sequence.c index 2b5a2a0ca..d9ba5cdfc 100644 --- a/src/backend/distributed/commands/sequence.c +++ b/src/backend/distributed/commands/sequence.c @@ -68,7 +68,6 @@ ErrorIfDistributedAlterSeqOwnedBy(AlterSeqStmt *alterSeqStmt) return; } -#if (PG_VERSION_NUM >= 100000) sequenceOwned = sequenceIsOwned(sequenceId, DEPENDENCY_AUTO, &ownedByTableId, &ownedByColumnId); if (!sequenceOwned) @@ -76,9 +75,6 @@ ErrorIfDistributedAlterSeqOwnedBy(AlterSeqStmt *alterSeqStmt) sequenceOwned = sequenceIsOwned(sequenceId, DEPENDENCY_INTERNAL, &ownedByTableId, &ownedByColumnId); } -#else - sequenceOwned = sequenceIsOwned(sequenceId, &ownedByTableId, &ownedByColumnId); -#endif /* see whether the sequence is already owned by a distributed table */ if (sequenceOwned) diff --git a/src/backend/distributed/commands/subscription.c b/src/backend/distributed/commands/subscription.c index 3b5495e3a..03a25aae6 100644 --- a/src/backend/distributed/commands/subscription.c +++ b/src/backend/distributed/commands/subscription.c @@ -10,8 +10,6 @@ #include "postgres.h" -#if (PG_VERSION_NUM >= 100000) - #include "distributed/commands.h" #include "nodes/parsenodes.h" @@ -22,6 +20,3 @@ ProcessCreateSubscriptionStmt(CreateSubscriptionStmt *createSubStmt) { return (Node *) createSubStmt; } - - -#endif /* PG_VERSION_NUM >= 100000 */ diff --git a/src/backend/distributed/commands/table.c b/src/backend/distributed/commands/table.c index dfd02f40f..8093a0c9a 100644 --- a/src/backend/distributed/commands/table.c +++ b/src/backend/distributed/commands/table.c @@ -10,6 +10,9 @@ #include "postgres.h" +#if PG_VERSION_NUM >= 120000 +#include "access/genam.h" +#endif #include "access/htup_details.h" #include "access/xact.h" #include "catalog/index.h" diff --git a/src/backend/distributed/executor/adaptive_executor.c b/src/backend/distributed/executor/adaptive_executor.c index 72d11710f..ae3f23ae0 100644 --- a/src/backend/distributed/executor/adaptive_executor.c +++ b/src/backend/distributed/executor/adaptive_executor.c @@ -1695,13 +1695,8 @@ RunDistributedExecution(DistributedExecution *execution) } /* wait for I/O events */ -#if (PG_VERSION_NUM >= 100000) eventCount = WaitEventSetWait(execution->waitEventSet, timeout, events, eventSetSize, WAIT_EVENT_CLIENT_READ); -#else - eventCount = WaitEventSetWait(execution->waitEventSet, timeout, events, - eventSetSize); -#endif /* process I/O events */ for (; eventIndex < eventCount; eventIndex++) diff --git a/src/backend/distributed/master/master_delete_protocol.c b/src/backend/distributed/master/master_delete_protocol.c index 797404afa..7d1c01490 100644 --- a/src/backend/distributed/master/master_delete_protocol.c +++ b/src/backend/distributed/master/master_delete_protocol.c @@ -42,13 +42,21 @@ #include "distributed/worker_protocol.h" #include "distributed/worker_transaction.h" #include "lib/stringinfo.h" +#if PG_VERSION_NUM >= 120000 +#include "nodes/nodeFuncs.h" +#endif #include "nodes/nodes.h" #include "nodes/parsenodes.h" #include "nodes/pg_list.h" #include "nodes/primnodes.h" -#include "nodes/relation.h" #include "optimizer/clauses.h" +#if PG_VERSION_NUM >= 120000 +#include "nodes/pathnodes.h" +#include "optimizer/optimizer.h" +#else +#include "nodes/relation.h" #include "optimizer/predtest.h" +#endif #include "optimizer/restrictinfo.h" #include "storage/lock.h" #include "storage/lmgr.h" diff --git a/src/backend/distributed/master/master_metadata_utility.c b/src/backend/distributed/master/master_metadata_utility.c index 5bbc8603e..eeaf8b4ed 100644 --- a/src/backend/distributed/master/master_metadata_utility.c +++ b/src/backend/distributed/master/master_metadata_utility.c @@ -15,6 +15,9 @@ #include "libpq-fe.h" #include "miscadmin.h" +#if PG_VERSION_NUM >= 120000 +#include "access/genam.h" +#endif #include "access/htup_details.h" #include "access/sysattr.h" #include "access/xact.h" @@ -53,7 +56,6 @@ #include "utils/lsyscache.h" #include "utils/rel.h" #include "utils/syscache.h" -#include "utils/tqual.h" /* Local functions forward declarations */ diff --git a/src/backend/distributed/master/master_modify_multiple_shards.c b/src/backend/distributed/master/master_modify_multiple_shards.c index d10c3cb0b..f7615ffbb 100644 --- a/src/backend/distributed/master/master_modify_multiple_shards.c +++ b/src/backend/distributed/master/master_modify_multiple_shards.c @@ -47,9 +47,13 @@ #include "distributed/worker_protocol.h" #include "distributed/worker_transaction.h" #include "optimizer/clauses.h" +#if PG_VERSION_NUM >= 120000 +#include "optimizer/optimizer.h" +#else #include "optimizer/predtest.h" -#include "optimizer/restrictinfo.h" #include "optimizer/var.h" +#endif +#include "optimizer/restrictinfo.h" #include "nodes/makefuncs.h" #include "tcop/tcopprot.h" #include "utils/builtins.h" diff --git a/src/backend/distributed/master/master_stage_protocol.c b/src/backend/distributed/master/master_stage_protocol.c index 698a584fc..58dfda15a 100644 --- a/src/backend/distributed/master/master_stage_protocol.c +++ b/src/backend/distributed/master/master_stage_protocol.c @@ -55,7 +55,6 @@ #include "utils/lsyscache.h" #include "utils/syscache.h" #include "utils/rel.h" -#include "utils/tqual.h" /* Local functions forward declarations */ diff --git a/src/backend/distributed/master/worker_node_manager.c b/src/backend/distributed/master/worker_node_manager.c index b8fc43e6f..98c79ef9d 100644 --- a/src/backend/distributed/master/worker_node_manager.c +++ b/src/backend/distributed/master/worker_node_manager.c @@ -19,9 +19,7 @@ #include "distributed/metadata_cache.h" #include "distributed/multi_client_executor.h" #include "libpq/hba.h" -#if (PG_VERSION_NUM >= 100000) #include "common/ip.h" -#endif #include "libpq/libpq-be.h" #include "postmaster/postmaster.h" #include "storage/fd.h" diff --git a/src/backend/distributed/metadata/metadata_sync.c b/src/backend/distributed/metadata/metadata_sync.c index 61301c63b..64be88d1c 100644 --- a/src/backend/distributed/metadata/metadata_sync.c +++ b/src/backend/distributed/metadata/metadata_sync.c @@ -48,7 +48,6 @@ #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/syscache.h" -#include "utils/tqual.h" static char * LocalGroupIdUpdateCommand(int32 groupId); diff --git a/src/backend/distributed/planner/extended_op_node_utils.c b/src/backend/distributed/planner/extended_op_node_utils.c index 9ed16aea7..b277432c6 100644 --- a/src/backend/distributed/planner/extended_op_node_utils.c +++ b/src/backend/distributed/planner/extended_op_node_utils.c @@ -14,7 +14,12 @@ #include "distributed/metadata_cache.h" #include "distributed/multi_logical_optimizer.h" #include "distributed/pg_dist_partition.h" +#if PG_VERSION_NUM >= 120000 +#include "optimizer/optimizer.h" +#else #include "optimizer/var.h" +#endif +#include "optimizer/restrictinfo.h" #include "nodes/nodeFuncs.h" #include "nodes/pg_list.h" diff --git a/src/backend/distributed/planner/fast_path_router_planner.c b/src/backend/distributed/planner/fast_path_router_planner.c index ebba58214..06d3f9d63 100644 --- a/src/backend/distributed/planner/fast_path_router_planner.c +++ b/src/backend/distributed/planner/fast_path_router_planner.c @@ -41,10 +41,17 @@ #include "distributed/pg_dist_partition.h" #include "distributed/shardinterval_utils.h" #include "distributed/shard_pruning.h" +#if PG_VERSION_NUM >= 120000 +#include "nodes/makefuncs.h" +#endif #include "nodes/nodeFuncs.h" #include "nodes/parsenodes.h" #include "nodes/pg_list.h" +#if PG_VERSION_NUM >= 120000 +#include "optimizer/optimizer.h" +#else #include "optimizer/clauses.h" +#endif bool EnableFastPathRouterPlanner = true; diff --git a/src/backend/distributed/planner/insert_select_planner.c b/src/backend/distributed/planner/insert_select_planner.c index ff1899a54..0b1cc18f1 100644 --- a/src/backend/distributed/planner/insert_select_planner.c +++ b/src/backend/distributed/planner/insert_select_planner.c @@ -33,7 +33,11 @@ #include "optimizer/planner.h" #include "optimizer/restrictinfo.h" #include "optimizer/tlist.h" +#if PG_VERSION_NUM >= 120000 +#include "optimizer/optimizer.h" +#else #include "optimizer/var.h" +#endif #include "parser/parsetree.h" #include "parser/parse_coerce.h" #include "parser/parse_relation.h" diff --git a/src/backend/distributed/planner/multi_join_order.c b/src/backend/distributed/planner/multi_join_order.c index dd219d6ed..7de41d44b 100644 --- a/src/backend/distributed/planner/multi_join_order.c +++ b/src/backend/distributed/planner/multi_join_order.c @@ -24,7 +24,12 @@ #include "distributed/pg_dist_partition.h" #include "distributed/worker_protocol.h" #include "lib/stringinfo.h" +#if PG_VERSION_NUM >= 120000 +#include "optimizer/optimizer.h" +#else #include "optimizer/var.h" +#endif +#include "utils/builtins.h" #include "nodes/nodeFuncs.h" #include "utils/builtins.h" #include "utils/datum.h" diff --git a/src/backend/distributed/planner/multi_logical_optimizer.c b/src/backend/distributed/planner/multi_logical_optimizer.c index f59f522f6..eba8a9d34 100644 --- a/src/backend/distributed/planner/multi_logical_optimizer.c +++ b/src/backend/distributed/planner/multi_logical_optimizer.c @@ -40,7 +40,11 @@ #include "nodes/print.h" #include "optimizer/clauses.h" #include "optimizer/tlist.h" +#if PG_VERSION_NUM >= 120000 +#include "optimizer/optimizer.h" +#else #include "optimizer/var.h" +#endif #include "parser/parse_agg.h" #include "parser/parse_coerce.h" #include "parser/parse_oper.h" diff --git a/src/backend/distributed/planner/multi_logical_planner.c b/src/backend/distributed/planner/multi_logical_planner.c index d874e8899..db404942a 100644 --- a/src/backend/distributed/planner/multi_logical_planner.c +++ b/src/backend/distributed/planner/multi_logical_planner.c @@ -33,11 +33,16 @@ #include "distributed/version_compat.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" +#if PG_VERSION_NUM >= 120000 +#include "nodes/pathnodes.h" +#include "optimizer/optimizer.h" +#else #include "nodes/relation.h" +#include "optimizer/var.h" +#endif #include "optimizer/clauses.h" #include "optimizer/prep.h" #include "optimizer/tlist.h" -#include "optimizer/var.h" #include "parser/parsetree.h" #include "utils/datum.h" #include "utils/lsyscache.h" diff --git a/src/backend/distributed/planner/multi_master_planner.c b/src/backend/distributed/planner/multi_master_planner.c index 40d0dade1..eabf113ab 100644 --- a/src/backend/distributed/planner/multi_master_planner.c +++ b/src/backend/distributed/planner/multi_master_planner.c @@ -29,7 +29,11 @@ #include "optimizer/cost.h" #include "optimizer/planmain.h" #include "optimizer/tlist.h" +#if PG_VERSION_NUM >= 120000 +#include "optimizer/optimizer.h" +#else #include "optimizer/var.h" +#endif #include "utils/builtins.h" #include "utils/guc.h" #include "utils/memutils.h" diff --git a/src/backend/distributed/planner/multi_physical_planner.c b/src/backend/distributed/planner/multi_physical_planner.c index 80fbc1d98..40bd3cd11 100644 --- a/src/backend/distributed/planner/multi_physical_planner.c +++ b/src/backend/distributed/planner/multi_physical_planner.c @@ -54,9 +54,14 @@ #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "optimizer/clauses.h" -#include "optimizer/predtest.h" -#include "optimizer/restrictinfo.h" +#if PG_VERSION_NUM >= 120000 +#include "nodes/pathnodes.h" +#include "optimizer/optimizer.h" +#else +#include "nodes/relation.h" #include "optimizer/var.h" +#endif +#include "optimizer/restrictinfo.h" #include "parser/parse_relation.h" #include "parser/parsetree.h" #include "utils/builtins.h" diff --git a/src/backend/distributed/planner/multi_router_planner.c b/src/backend/distributed/planner/multi_router_planner.c index aea7c7714..86445a5df 100644 --- a/src/backend/distributed/planner/multi_router_planner.c +++ b/src/backend/distributed/planner/multi_router_planner.c @@ -11,7 +11,6 @@ */ #include "postgres.h" -#include "c.h" #include @@ -57,9 +56,13 @@ #include "optimizer/joininfo.h" #include "optimizer/pathnode.h" #include "optimizer/paths.h" -#include "optimizer/predtest.h" -#include "optimizer/restrictinfo.h" +#if PG_VERSION_NUM >= 120000 +#include "optimizer/optimizer.h" +#else #include "optimizer/var.h" +#include "optimizer/predtest.h" +#endif +#include "optimizer/restrictinfo.h" #include "parser/parsetree.h" #include "parser/parse_oper.h" #include "storage/lock.h" diff --git a/src/backend/distributed/planner/query_pushdown_planning.c b/src/backend/distributed/planner/query_pushdown_planning.c index 5aef6fde5..a3ccbb6b3 100644 --- a/src/backend/distributed/planner/query_pushdown_planning.c +++ b/src/backend/distributed/planner/query_pushdown_planning.c @@ -32,9 +32,14 @@ #include "distributed/query_pushdown_planning.h" #include "distributed/relation_restriction_equivalence.h" #include "nodes/nodeFuncs.h" +#if PG_VERSION_NUM >= 120000 +#include "nodes/makefuncs.h" +#include "optimizer/optimizer.h" +#else +#include "optimizer/var.h" +#endif #include "nodes/pg_list.h" #include "optimizer/clauses.h" -#include "optimizer/var.h" #include "parser/parsetree.h" diff --git a/src/backend/distributed/planner/recursive_planning.c b/src/backend/distributed/planner/recursive_planning.c index 573fa314b..56c55c810 100644 --- a/src/backend/distributed/planner/recursive_planning.c +++ b/src/backend/distributed/planner/recursive_planning.c @@ -78,7 +78,11 @@ #include "nodes/nodes.h" #include "nodes/pg_list.h" #include "nodes/primnodes.h" +#if PG_VERSION_NUM >= 120000 +#include "nodes/pathnodes.h" +#else #include "nodes/relation.h" +#endif #include "utils/builtins.h" #include "utils/guc.h" diff --git a/src/backend/distributed/planner/relation_restriction_equivalence.c b/src/backend/distributed/planner/relation_restriction_equivalence.c index 827880e2d..f40d308f0 100644 --- a/src/backend/distributed/planner/relation_restriction_equivalence.c +++ b/src/backend/distributed/planner/relation_restriction_equivalence.c @@ -19,7 +19,11 @@ #include "nodes/nodeFuncs.h" #include "nodes/pg_list.h" #include "nodes/primnodes.h" +#if PG_VERSION_NUM >= 120000 +#include "nodes/pathnodes.h" +#else #include "nodes/relation.h" +#endif #include "parser/parsetree.h" #include "optimizer/pathnode.h" diff --git a/src/backend/distributed/test/fake_fdw.c b/src/backend/distributed/test/fake_fdw.c index 7fa966da4..1155bc89b 100644 --- a/src/backend/distributed/test/fake_fdw.c +++ b/src/backend/distributed/test/fake_fdw.c @@ -22,7 +22,11 @@ #include "nodes/nodes.h" #include "nodes/pg_list.h" #include "nodes/plannodes.h" +#if PG_VERSION_NUM >= 120000 +#include "nodes/pathnodes.h" +#else #include "nodes/relation.h" +#endif #include "optimizer/pathnode.h" #include "optimizer/planmain.h" #include "optimizer/restrictinfo.h" diff --git a/src/backend/distributed/test/prune_shard_list.c b/src/backend/distributed/test/prune_shard_list.c index adfac2f41..44bd5c33e 100644 --- a/src/backend/distributed/test/prune_shard_list.c +++ b/src/backend/distributed/test/prune_shard_list.c @@ -25,9 +25,13 @@ #include "distributed/multi_physical_planner.h" #include "distributed/resource_lock.h" #include "distributed/shard_pruning.h" +#if PG_VERSION_NUM >= 120000 +#include "nodes/makefuncs.h" +#include "nodes/nodeFuncs.h" +#endif +#include "nodes/nodes.h" #include "nodes/pg_list.h" #include "nodes/primnodes.h" -#include "nodes/nodes.h" #include "optimizer/clauses.h" #include "utils/array.h" #include "utils/palloc.h" diff --git a/src/backend/distributed/transaction/transaction_recovery.c b/src/backend/distributed/transaction/transaction_recovery.c index 8d28dd0b1..73e9dee75 100644 --- a/src/backend/distributed/transaction/transaction_recovery.c +++ b/src/backend/distributed/transaction/transaction_recovery.c @@ -19,6 +19,9 @@ #include #include +#if PG_VERSION_NUM >= 120000 +#include "access/genam.h" +#endif #include "access/heapam.h" #include "access/htup_details.h" #include "access/relscan.h" diff --git a/src/backend/distributed/utils/citus_outfuncs.c b/src/backend/distributed/utils/citus_outfuncs.c index bc92940b9..71afc0b58 100644 --- a/src/backend/distributed/utils/citus_outfuncs.c +++ b/src/backend/distributed/utils/citus_outfuncs.c @@ -30,7 +30,11 @@ #include "distributed/master_metadata_utility.h" #include "lib/stringinfo.h" #include "nodes/plannodes.h" +#if PG_VERSION_NUM >= 120000 +#include "nodes/pathnodes.h" +#else #include "nodes/relation.h" +#endif #include "utils/datum.h" diff --git a/src/backend/distributed/utils/foreign_key_relationship.c b/src/backend/distributed/utils/foreign_key_relationship.c index 6ddb58f09..a4ce23458 100644 --- a/src/backend/distributed/utils/foreign_key_relationship.c +++ b/src/backend/distributed/utils/foreign_key_relationship.c @@ -12,8 +12,14 @@ #include "postgres.h" +#if PG_VERSION_NUM >= 120000 +#include "access/genam.h" +#endif #include "access/htup_details.h" #include "access/stratnum.h" +#if PG_VERSION_NUM >= 120000 +#include "access/table.h" +#endif #include "catalog/pg_constraint.h" #include "distributed/foreign_key_relationship.h" #include "distributed/hash_helpers.h" diff --git a/src/backend/distributed/utils/multi_partitioning_utils.c b/src/backend/distributed/utils/multi_partitioning_utils.c index b505e9f0f..41cc08965 100644 --- a/src/backend/distributed/utils/multi_partitioning_utils.c +++ b/src/backend/distributed/utils/multi_partitioning_utils.c @@ -25,6 +25,9 @@ #include "lib/stringinfo.h" #include "nodes/pg_list.h" #include "pgstat.h" +#if PG_VERSION_NUM >= 120000 +#include "partitioning/partdesc.h" +#endif #include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" diff --git a/src/include/distributed/commands.h b/src/include/distributed/commands.h index 127e33aa3..883b58439 100644 --- a/src/include/distributed/commands.h +++ b/src/include/distributed/commands.h @@ -93,11 +93,8 @@ extern void ErrorIfUnsupportedSeqStmt(CreateSeqStmt *createSeqStmt); extern void ErrorIfDistributedAlterSeqOwnedBy(AlterSeqStmt *alterSeqStmt); -#if (PG_VERSION_NUM >= 100000) - /* subscription.c - forward declarations */ extern Node * ProcessCreateSubscriptionStmt(CreateSubscriptionStmt *createSubStmt); -#endif /* PG_VERSION_NUM >= 100000 */ /* table.c - forward declarations */ diff --git a/src/include/distributed/distributed_planner.h b/src/include/distributed/distributed_planner.h index 927cef48b..963a4929d 100644 --- a/src/include/distributed/distributed_planner.h +++ b/src/include/distributed/distributed_planner.h @@ -11,7 +11,12 @@ #define DISTRIBUTED_PLANNER_H #include "nodes/plannodes.h" + +#if PG_VERSION_NUM >= 120000 +#include "nodes/pathnodes.h" +#else #include "nodes/relation.h" +#endif #include "distributed/citus_nodes.h" #include "distributed/errormessage.h" diff --git a/src/include/distributed/hash_helpers.h b/src/include/distributed/hash_helpers.h index 0fecf0c03..a22281a77 100644 --- a/src/include/distributed/hash_helpers.h +++ b/src/include/distributed/hash_helpers.h @@ -11,6 +11,9 @@ #include "utils/hsearch.h" +/* pg12 includes this exact implementation of hash_combine */ +#if PG_VERSION_NUM < 120000 + /* * Combine two hash values, resulting in another hash value, with decent bit * mixing. @@ -25,6 +28,9 @@ hash_combine(uint32 a, uint32 b) } +#endif + + extern void hash_delete_all(HTAB *htab); #endif diff --git a/src/include/distributed/master_metadata_utility.h b/src/include/distributed/master_metadata_utility.h index bd6a9ce57..9a714c3b0 100644 --- a/src/include/distributed/master_metadata_utility.h +++ b/src/include/distributed/master_metadata_utility.h @@ -32,27 +32,6 @@ #define PG_TOTAL_RELATION_SIZE_FUNCTION "pg_total_relation_size(%s)" #define CSTORE_TABLE_SIZE_FUNCTION "cstore_table_size(%s)" -#if (PG_VERSION_NUM < 100000) -static inline void -CatalogTupleUpdate(Relation heapRel, ItemPointer otid, HeapTuple tup) -{ - simple_heap_update(heapRel, otid, tup); - CatalogUpdateIndexes(heapRel, tup); -} - - -static inline Oid -CatalogTupleInsert(Relation heapRel, HeapTuple tup) -{ - Oid oid = simple_heap_insert(heapRel, tup); - CatalogUpdateIndexes(heapRel, tup); - - return oid; -} - - -#endif - /* In-memory representation of a typed tuple in pg_dist_shard. */ typedef struct ShardInterval { diff --git a/src/include/distributed/multi_physical_planner.h b/src/include/distributed/multi_physical_planner.h index 35685f3bb..b033ccf86 100644 --- a/src/include/distributed/multi_physical_planner.h +++ b/src/include/distributed/multi_physical_planner.h @@ -51,13 +51,12 @@ typedef enum CitusRTEKind CITUS_RTE_SUBQUERY = RTE_SUBQUERY, /* subquery in FROM */ CITUS_RTE_JOIN = RTE_JOIN, /* join */ CITUS_RTE_FUNCTION = RTE_FUNCTION, /* function in FROM */ -#if (PG_VERSION_NUM >= 100000) CITUS_RTE_TABLEFUNC = RTE_TABLEFUNC, /* TableFunc(.., column list) */ -#endif CITUS_RTE_VALUES = RTE_VALUES, /* VALUES (), (), ... */ CITUS_RTE_CTE = RTE_CTE, /* common table expr (WITH list element) */ -#if (PG_VERSION_NUM >= 100000) CITUS_RTE_NAMEDTUPLESTORE = RTE_NAMEDTUPLESTORE, /* tuplestore, e.g. for triggers */ +#if (PG_VERSION_NUM >= 120000) + CITUS_RTE_RESULT = RTE_RESULT, /* RTE represents an empty FROM clause */ #endif CITUS_RTE_SHARD, CITUS_RTE_REMOTE_QUERY diff --git a/src/include/distributed/recursive_planning.h b/src/include/distributed/recursive_planning.h index 777e0e9db..f0c8ca2ed 100644 --- a/src/include/distributed/recursive_planning.h +++ b/src/include/distributed/recursive_planning.h @@ -15,7 +15,11 @@ #include "distributed/relation_restriction_equivalence.h" #include "nodes/pg_list.h" #include "nodes/primnodes.h" +#if PG_VERSION_NUM >= 120000 +#include "nodes/pathnodes.h" +#else #include "nodes/relation.h" +#endif extern List * GenerateSubplansForSubqueriesAndCTEs(uint64 planId, Query *originalQuery, diff --git a/src/include/distributed/task_tracker.h b/src/include/distributed/task_tracker.h index 23cb2c15b..1a1f608e3 100644 --- a/src/include/distributed/task_tracker.h +++ b/src/include/distributed/task_tracker.h @@ -103,9 +103,7 @@ typedef struct WorkerTasksSharedStateData { /* Lock protecting workerNodesHash */ int taskHashTrancheId; -#if (PG_VERSION_NUM >= 100000) char *taskHashTrancheName; -#endif LWLock taskHashLock; bool conninfosValid; } WorkerTasksSharedStateData; From 9643ff580ed3960bd73b5e51c0554a6ffce5639e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philip=20Dub=C3=A9?= Date: Thu, 8 Aug 2019 21:46:06 +0000 Subject: [PATCH 04/12] Update commands/vacuum.c with pg12 changes Adds support for SKIP_LOCKED, INDEX_CLEANUP, TRUNCATE Removes broken assert --- src/backend/distributed/commands/vacuum.c | 195 +++++++++++++++++++--- 1 file changed, 169 insertions(+), 26 deletions(-) diff --git a/src/backend/distributed/commands/vacuum.c b/src/backend/distributed/commands/vacuum.c index 3eb20f58a..e779242e6 100644 --- a/src/backend/distributed/commands/vacuum.c +++ b/src/backend/distributed/commands/vacuum.c @@ -9,8 +9,11 @@ */ #include "postgres.h" -#include "c.h" +#if PG_VERSION_NUM >= 120000 +#include "commands/defrem.h" +#endif +#include "commands/vacuum.h" #include "distributed/commands.h" #include "distributed/commands/utility_hook.h" #include "distributed/metadata_cache.h" @@ -22,13 +25,26 @@ #include "utils/builtins.h" #include "utils/lsyscache.h" +/* + * Subset of VacuumParams we care about + */ +typedef struct CitusVacuumParams +{ + int options; +#if PG_VERSION_NUM >= 120000 + VacOptTernaryValue truncate; + VacOptTernaryValue index_cleanup; +#endif +} CitusVacuumParams; + /* Local functions forward declarations for processing distributed table commands */ -static bool IsDistributedVacuumStmt(VacuumStmt *vacuumStmt, List *vacuumRelationIdList); -static List * VacuumTaskList(Oid relationId, int vacuumOptions, List *vacuumColumnList); -static StringInfo DeparseVacuumStmtPrefix(int vacuumFlags); +static bool IsDistributedVacuumStmt(int vacuumOptions, List *vacuumRelationIdList); +static List * VacuumTaskList(Oid relationId, CitusVacuumParams vacuumParams, + List *vacuumColumnList); +static StringInfo DeparseVacuumStmtPrefix(CitusVacuumParams vacuumParams); static char * DeparseVacuumColumnNames(List *columnNameList); - +static CitusVacuumParams VacuumStmtParams(VacuumStmt *vacstmt); /* * ProcessVacuumStmt processes vacuum statements that may need propagation to @@ -49,7 +65,8 @@ ProcessVacuumStmt(VacuumStmt *vacuumStmt, const char *vacuumCommand) ListCell *vacuumRelationCell = NULL; List *relationIdList = NIL; ListCell *relationIdCell = NULL; - LOCKMODE lockMode = (vacuumStmt->options & VACOPT_FULL) ? AccessExclusiveLock : + CitusVacuumParams vacuumParams = VacuumStmtParams(vacuumStmt); + LOCKMODE lockMode = (vacuumParams.options & VACOPT_FULL) ? AccessExclusiveLock : ShareUpdateExclusiveLock; int executedVacuumCount = 0; @@ -60,7 +77,7 @@ ProcessVacuumStmt(VacuumStmt *vacuumStmt, const char *vacuumCommand) relationIdList = lappend_oid(relationIdList, relationId); } - distributedVacuumStmt = IsDistributedVacuumStmt(vacuumStmt, relationIdList); + distributedVacuumStmt = IsDistributedVacuumStmt(vacuumParams.options, relationIdList); if (!distributedVacuumStmt) { return; @@ -81,7 +98,7 @@ ProcessVacuumStmt(VacuumStmt *vacuumStmt, const char *vacuumCommand) * commands can run inside a transaction block. Notice that we do this * once even if there are multiple distributed tables to be vacuumed. */ - if (executedVacuumCount == 0 && (vacuumStmt->options & VACOPT_VACUUM) != 0) + if (executedVacuumCount == 0 && (vacuumParams.options & VACOPT_VACUUM) != 0) { /* save old commit protocol to restore at xact end */ Assert(SavedMultiShardCommitProtocol == COMMIT_PROTOCOL_BARE); @@ -90,7 +107,7 @@ ProcessVacuumStmt(VacuumStmt *vacuumStmt, const char *vacuumCommand) } vacuumColumnList = VacuumColumnList(vacuumStmt, relationIndex); - taskList = VacuumTaskList(relationId, vacuumStmt->options, vacuumColumnList); + taskList = VacuumTaskList(relationId, vacuumParams, vacuumColumnList); /* use adaptive executor when enabled */ ExecuteUtilityTaskListWithoutResults(taskList); @@ -110,9 +127,9 @@ ProcessVacuumStmt(VacuumStmt *vacuumStmt, const char *vacuumCommand) * false otherwise. */ static bool -IsDistributedVacuumStmt(VacuumStmt *vacuumStmt, List *vacuumRelationIdList) +IsDistributedVacuumStmt(int vacuumOptions, List *vacuumRelationIdList) { - const char *stmtName = (vacuumStmt->options & VACOPT_VACUUM) ? "VACUUM" : "ANALYZE"; + const char *stmtName = (vacuumOptions & VACOPT_VACUUM) ? "VACUUM" : "ANALYZE"; bool distributeStmt = false; ListCell *relationIdCell = NULL; int distributedRelationCount = 0; @@ -166,14 +183,14 @@ IsDistributedVacuumStmt(VacuumStmt *vacuumStmt, List *vacuumRelationIdList) * a VacuumStmt which targets a distributed relation. */ static List * -VacuumTaskList(Oid relationId, int vacuumOptions, List *vacuumColumnList) +VacuumTaskList(Oid relationId, CitusVacuumParams vacuumParams, List *vacuumColumnList) { List *taskList = NIL; List *shardIntervalList = NIL; ListCell *shardIntervalCell = NULL; uint64 jobId = INVALID_JOB_ID; int taskId = 1; - StringInfo vacuumString = DeparseVacuumStmtPrefix(vacuumOptions); + StringInfo vacuumString = DeparseVacuumStmtPrefix(vacuumParams); const char *columnNames = NULL; const int vacuumPrefixLen = vacuumString->len; Oid schemaId = get_rel_namespace(relationId); @@ -233,18 +250,12 @@ VacuumTaskList(Oid relationId, int vacuumOptions, List *vacuumColumnList) * statements. */ static StringInfo -DeparseVacuumStmtPrefix(int vacuumFlags) +DeparseVacuumStmtPrefix(CitusVacuumParams vacuumParams) { + int vacuumFlags = vacuumParams.options; StringInfo vacuumPrefix = makeStringInfo(); - const int unsupportedFlags PG_USED_FOR_ASSERTS_ONLY = ~( - VACOPT_ANALYZE | - VACOPT_DISABLE_PAGE_SKIPPING | - VACOPT_FREEZE | - VACOPT_FULL | - VACOPT_VERBOSE - ); - /* determine actual command and block out its bit */ + /* determine actual command and block out its bits */ if (vacuumFlags & VACOPT_VACUUM) { appendStringInfoString(vacuumPrefix, "VACUUM "); @@ -252,6 +263,8 @@ DeparseVacuumStmtPrefix(int vacuumFlags) } else { + Assert((vacuumFlags & VACOPT_ANALYZE) != 0); + appendStringInfoString(vacuumPrefix, "ANALYZE "); vacuumFlags &= ~VACOPT_ANALYZE; @@ -262,11 +275,13 @@ DeparseVacuumStmtPrefix(int vacuumFlags) } } - /* unsupported flags should have already been rejected */ - Assert((vacuumFlags & unsupportedFlags) == 0); - /* if no flags remain, exit early */ - if (vacuumFlags == 0) + if (vacuumFlags == 0 +#if PG_VERSION_NUM >= 120000 + && vacuumParams.truncate == VACOPT_TERNARY_DEFAULT && + vacuumParams.index_cleanup == VACOPT_TERNARY_DEFAULT +#endif + ) { return vacuumPrefix; } @@ -299,6 +314,29 @@ DeparseVacuumStmtPrefix(int vacuumFlags) appendStringInfoString(vacuumPrefix, "VERBOSE,"); } +#if PG_VERSION_NUM >= 120000 + if (vacuumFlags & VACOPT_SKIP_LOCKED) + { + appendStringInfoString(vacuumPrefix, "SKIP_LOCKED,"); + } + + if (vacuumParams.truncate != VACOPT_TERNARY_DEFAULT) + { + appendStringInfoString(vacuumPrefix, + vacuumParams.truncate == VACOPT_TERNARY_ENABLED ? + "TRUNCATE," : "TRUNCATE false," + ); + } + + if (vacuumParams.index_cleanup != VACOPT_TERNARY_DEFAULT) + { + appendStringInfoString(vacuumPrefix, + vacuumParams.index_cleanup == VACOPT_TERNARY_ENABLED ? + "INDEX_CLEANUP," : "INDEX_CLEANUP false," + ); + } +#endif + vacuumPrefix->data[vacuumPrefix->len - 1] = ')'; appendStringInfoChar(vacuumPrefix, ' '); @@ -339,3 +377,108 @@ DeparseVacuumColumnNames(List *columnNameList) return columnNames->data; } + + +/* + * VacuumStmtParams returns a CitusVacuumParams based on the supplied VacuumStmt. + */ +#if PG_VERSION_NUM >= 120000 + +/* + * This is mostly ExecVacuum from Postgres's commands/vacuum.c + */ +static CitusVacuumParams +VacuumStmtParams(VacuumStmt *vacstmt) +{ + CitusVacuumParams params; + bool verbose = false; + bool skip_locked = false; + bool analyze = false; + bool freeze = false; + bool full = false; + bool disable_page_skipping = false; + ListCell *lc; + + /* Set default value */ + params.index_cleanup = VACOPT_TERNARY_DEFAULT; + params.truncate = VACOPT_TERNARY_DEFAULT; + + /* Parse options list */ + foreach(lc, vacstmt->options) + { + DefElem *opt = (DefElem *) lfirst(lc); + + /* Parse common options for VACUUM and ANALYZE */ + if (strcmp(opt->defname, "verbose") == 0) + { + verbose = defGetBoolean(opt); + } + else if (strcmp(opt->defname, "skip_locked") == 0) + { + skip_locked = defGetBoolean(opt); + } + else if (!vacstmt->is_vacuumcmd) + { + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("unrecognized ANALYZE option \"%s\"", opt->defname))); + } + + /* Parse options available on VACUUM */ + else if (strcmp(opt->defname, "analyze") == 0) + { + analyze = defGetBoolean(opt); + } + else if (strcmp(opt->defname, "freeze") == 0) + { + freeze = defGetBoolean(opt); + } + else if (strcmp(opt->defname, "full") == 0) + { + full = defGetBoolean(opt); + } + else if (strcmp(opt->defname, "disable_page_skipping") == 0) + { + disable_page_skipping = defGetBoolean(opt); + } + else if (strcmp(opt->defname, "index_cleanup") == 0) + { + params.index_cleanup = defGetBoolean(opt) ? VACOPT_TERNARY_ENABLED : + VACOPT_TERNARY_DISABLED; + } + else if (strcmp(opt->defname, "truncate") == 0) + { + params.truncate = defGetBoolean(opt) ? VACOPT_TERNARY_ENABLED : + VACOPT_TERNARY_DISABLED; + } + else + { + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("unrecognized VACUUM option \"%s\"", opt->defname) + )); + } + } + + params.options = (vacstmt->is_vacuumcmd ? VACOPT_VACUUM : VACOPT_ANALYZE) | + (verbose ? VACOPT_VERBOSE : 0) | + (skip_locked ? VACOPT_SKIP_LOCKED : 0) | + (analyze ? VACOPT_ANALYZE : 0) | + (freeze ? VACOPT_FREEZE : 0) | + (full ? VACOPT_FULL : 0) | + (disable_page_skipping ? VACOPT_DISABLE_PAGE_SKIPPING : 0); + return params; +} + + +#else +static CitusVacuumParams +VacuumStmtParams(VacuumStmt *vacuumStmt) +{ + CitusVacuumParams params; + params.options = vacuumStmt->options; + return params; +} + + +#endif From 018ad1c58e9c38af43ce49fc7b7cf4af1a36168b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philip=20Dub=C3=A9?= Date: Thu, 8 Aug 2019 21:55:52 +0000 Subject: [PATCH 05/12] pg12: version_compat.h, tuples, oids, misc --- .../commands/create_distributed_table.c | 21 +++++++- src/backend/distributed/commands/multi_copy.c | 10 ++-- .../distributed/executor/citus_custom_scan.c | 5 ++ .../distributed/executor/multi_executor.c | 6 ++- .../executor/multi_router_executor.c | 3 +- .../distributed/master/master_node_protocol.c | 8 +-- .../distributed/planner/deparse_shard_query.c | 2 +- .../planner/insert_select_planner.c | 3 +- .../planner/multi_logical_optimizer.c | 11 ++-- .../planner/multi_logical_planner.c | 14 +++--- .../planner/multi_router_planner.c | 6 ++- .../planner/query_pushdown_planning.c | 50 ++++++++++++++++--- .../distributed/planner/recursive_planning.c | 3 +- .../distributed/progress/multi_progress.c | 4 +- .../distributed/relay/relay_event_utility.c | 2 +- .../distributed/utils/citus_nodefuncs.c | 11 ++-- .../distributed/utils/citus_readfuncs.c | 8 ++- .../distributed/utils/citus_ruleutils.c | 4 ++ .../distributed/utils/statistics_collection.c | 7 ++- .../worker/worker_merge_protocol.c | 16 +++++- .../distributed/query_pushdown_planning.h | 1 + src/include/distributed/version_compat.h | 40 ++++++++++++++- 22 files changed, 190 insertions(+), 45 deletions(-) diff --git a/src/backend/distributed/commands/create_distributed_table.c b/src/backend/distributed/commands/create_distributed_table.c index 69b6621fe..b1bb140aa 100644 --- a/src/backend/distributed/commands/create_distributed_table.c +++ b/src/backend/distributed/commands/create_distributed_table.c @@ -645,6 +645,8 @@ EnsureRelationCanBeDistributed(Oid relationId, Var *distributionColumn, relationDesc = RelationGetDescr(relation); relationName = RelationGetRelationName(relation); +#if PG_VERSION_NUM < 120000 + /* verify target relation does not use WITH (OIDS) PostgreSQL feature */ if (relationDesc->tdhasoid) { @@ -653,6 +655,7 @@ EnsureRelationCanBeDistributed(Oid relationId, Var *distributionColumn, errdetail("Distributed relations must not specify the WITH " "(OIDS) option in their definitions."))); } +#endif /* verify target relation does not use identity columns */ if (RelationUsesIdentityColumns(relationDesc)) @@ -1196,7 +1199,11 @@ CopyLocalDataIntoShards(Oid distributedRelationId) bool stopOnFailure = true; EState *estate = NULL; +#if PG_VERSION_NUM >= 120000 + TableScanDesc scan = NULL; +#else HeapScanDesc scan = NULL; +#endif HeapTuple tuple = NULL; ExprContext *econtext = NULL; MemoryContext oldContext = NULL; @@ -1230,7 +1237,7 @@ CopyLocalDataIntoShards(Oid distributedRelationId) /* get the table columns */ tupleDescriptor = RelationGetDescr(distributedRelation); - slot = MakeSingleTupleTableSlot(tupleDescriptor); + slot = MakeSingleTupleTableSlotCompat(tupleDescriptor, &TTSOpsHeapTuple); columnNameList = TupleDescColumnNameList(tupleDescriptor); /* determine the partition column in the tuple descriptor */ @@ -1256,14 +1263,22 @@ CopyLocalDataIntoShards(Oid distributedRelationId) copyDest->rStartup(copyDest, 0, tupleDescriptor); /* begin reading from local table */ +#if PG_VERSION_NUM >= 120000 + scan = table_beginscan(distributedRelation, GetActiveSnapshot(), 0, NULL); +#else scan = heap_beginscan(distributedRelation, GetActiveSnapshot(), 0, NULL); +#endif oldContext = MemoryContextSwitchTo(GetPerTupleMemoryContext(estate)); while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL) { /* materialize tuple and send it to a shard */ +#if PG_VERSION_NUM >= 120000 + ExecStoreHeapTuple(tuple, slot, false); +#else ExecStoreTuple(tuple, slot, InvalidBuffer, false); +#endif copyDest->receiveSlot(slot, copyDest); /* clear tuple memory */ @@ -1293,7 +1308,11 @@ CopyLocalDataIntoShards(Oid distributedRelationId) MemoryContextSwitchTo(oldContext); /* finish reading from the local table */ +#if PG_VERSION_NUM >= 120000 + table_endscan(scan); +#else heap_endscan(scan); +#endif /* finish writing into the shards */ copyDest->rShutdown(copyDest); diff --git a/src/backend/distributed/commands/multi_copy.c b/src/backend/distributed/commands/multi_copy.c index 5b5464680..904fab1dc 100644 --- a/src/backend/distributed/commands/multi_copy.c +++ b/src/backend/distributed/commands/multi_copy.c @@ -463,7 +463,7 @@ CopyToExistingShards(CopyStmt *copyStatement, char *completionTag) columnNulls = palloc0(columnCount * sizeof(bool)); /* set up a virtual tuple table slot */ - tupleTableSlot = MakeSingleTupleTableSlot(tupleDescriptor); + tupleTableSlot = MakeSingleTupleTableSlotCompat(tupleDescriptor, &TTSOpsVirtual); tupleTableSlot->tts_nvalid = columnCount; tupleTableSlot->tts_values = columnValues; tupleTableSlot->tts_isnull = columnNulls; @@ -561,8 +561,8 @@ CopyToExistingShards(CopyStmt *copyStatement, char *completionTag) oldContext = MemoryContextSwitchTo(executorTupleContext); /* parse a row from the input */ - nextRowFound = NextCopyFrom(copyState, executorExpressionContext, - columnValues, columnNulls, NULL); + nextRowFound = NextCopyFromCompat(copyState, executorExpressionContext, + columnValues, columnNulls); if (!nextRowFound) { @@ -681,8 +681,8 @@ CopyToNewShards(CopyStmt *copyStatement, char *completionTag, Oid relationId) oldContext = MemoryContextSwitchTo(executorTupleContext); /* parse a row from the input */ - nextRowFound = NextCopyFrom(copyState, executorExpressionContext, - columnValues, columnNulls, NULL); + nextRowFound = NextCopyFromCompat(copyState, executorExpressionContext, + columnValues, columnNulls); if (!nextRowFound) { diff --git a/src/backend/distributed/executor/citus_custom_scan.c b/src/backend/distributed/executor/citus_custom_scan.c index e320dd2a3..e98de4391 100644 --- a/src/backend/distributed/executor/citus_custom_scan.c +++ b/src/backend/distributed/executor/citus_custom_scan.c @@ -162,6 +162,11 @@ CitusBeginScan(CustomScanState *node, EState *estate, int eflags) MarkCitusInitiatedCoordinatorBackend(); scanState = (CitusScanState *) node; + +#if PG_VERSION_NUM >= 120000 + ExecInitResultSlot(&scanState->customScanState.ss.ps, &TTSOpsMinimalTuple); +#endif + distributedPlan = scanState->distributedPlan; if (distributedPlan->modLevel == ROW_MODIFY_READONLY || distributedPlan->insertSelectSubquery != NULL) diff --git a/src/backend/distributed/executor/multi_executor.c b/src/backend/distributed/executor/multi_executor.c index 066a9aa6c..5e82b6efb 100644 --- a/src/backend/distributed/executor/multi_executor.c +++ b/src/backend/distributed/executor/multi_executor.c @@ -138,7 +138,9 @@ CitusExecutorRun(QueryDesc *queryDesc, EState *estate = queryDesc->estate; estate->es_processed = 0; +#if PG_VERSION_NUM < 120000 estate->es_lastoid = InvalidOid; +#endif /* start and shutdown tuple receiver to simulate empty result */ dest->rStartup(queryDesc->dest, CMD_SELECT, queryDesc->tupDesc); @@ -351,8 +353,8 @@ ReadFileIntoTupleStore(char *fileName, char *copyFormat, TupleDesc tupleDescript ResetPerTupleExprContext(executorState); oldContext = MemoryContextSwitchTo(executorTupleContext); - nextRowFound = NextCopyFrom(copyState, executorExpressionContext, - columnValues, columnNulls, NULL); + nextRowFound = NextCopyFromCompat(copyState, executorExpressionContext, + columnValues, columnNulls); if (!nextRowFound) { MemoryContextSwitchTo(oldContext); diff --git a/src/backend/distributed/executor/multi_router_executor.c b/src/backend/distributed/executor/multi_router_executor.c index 8a2d86716..995cd945b 100644 --- a/src/backend/distributed/executor/multi_router_executor.c +++ b/src/backend/distributed/executor/multi_router_executor.c @@ -706,7 +706,8 @@ SortTupleStore(CitusScanState *scanState) /* iterate over all the sorted tuples, add them to original tuplestore */ while (true) { - TupleTableSlot *newSlot = MakeSingleTupleTableSlot(tupleDescriptor); + TupleTableSlot *newSlot = MakeSingleTupleTableSlotCompat(tupleDescriptor, + &TTSOpsMinimalTuple); bool found = tuplesort_gettupleslot(tuplesortstate, true, false, newSlot, NULL); if (!found) diff --git a/src/backend/distributed/master/master_node_protocol.c b/src/backend/distributed/master/master_node_protocol.c index 8e91fdfac..245af72f3 100644 --- a/src/backend/distributed/master/master_node_protocol.c +++ b/src/backend/distributed/master/master_node_protocol.c @@ -60,7 +60,6 @@ #include "utils/palloc.h" #include "utils/relcache.h" #include "utils/ruleutils.h" -#include "utils/tqual.h" #include "utils/varlena.h" @@ -472,7 +471,6 @@ master_get_active_worker_nodes(PG_FUNCTION_ARGS) MemoryContext oldContext = NULL; List *workerNodeList = NIL; TupleDesc tupleDescriptor = NULL; - bool hasOid = false; /* create a function context for cross-call persistence */ functionContext = SRF_FIRSTCALL_INIT(); @@ -490,7 +488,11 @@ master_get_active_worker_nodes(PG_FUNCTION_ARGS) * This tuple descriptor must match the output parameters declared for * the function in pg_proc. */ - tupleDescriptor = CreateTemplateTupleDesc(WORKER_NODE_FIELDS, hasOid); +#if PG_VERSION_NUM < 120000 + tupleDescriptor = CreateTemplateTupleDesc(WORKER_NODE_FIELDS, false); +#else + tupleDescriptor = CreateTemplateTupleDesc(WORKER_NODE_FIELDS); +#endif TupleDescInitEntry(tupleDescriptor, (AttrNumber) 1, "node_name", TEXTOID, -1, 0); TupleDescInitEntry(tupleDescriptor, (AttrNumber) 2, "node_port", diff --git a/src/backend/distributed/planner/deparse_shard_query.c b/src/backend/distributed/planner/deparse_shard_query.c index 68d84a6c6..3f3aadf00 100644 --- a/src/backend/distributed/planner/deparse_shard_query.c +++ b/src/backend/distributed/planner/deparse_shard_query.c @@ -203,7 +203,7 @@ UpdateRelationToShardNames(Node *node, List *relationShardList) if (IsA(node, Query)) { return query_tree_walker((Query *) node, UpdateRelationToShardNames, - relationShardList, QTW_EXAMINE_RTES); + relationShardList, QTW_EXAMINE_RTES_BEFORE); } if (!IsA(node, RangeTblEntry)) diff --git a/src/backend/distributed/planner/insert_select_planner.c b/src/backend/distributed/planner/insert_select_planner.c index 0b1cc18f1..27e9f5465 100644 --- a/src/backend/distributed/planner/insert_select_planner.c +++ b/src/backend/distributed/planner/insert_select_planner.c @@ -26,6 +26,7 @@ #include "distributed/query_pushdown_planning.h" #include "distributed/recursive_planning.h" #include "distributed/resource_lock.h" +#include "distributed/version_compat.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "nodes/parsenodes.h" @@ -769,7 +770,7 @@ MultiTaskRouterSelectQuerySupported(Query *query) Assert(subquery->commandType == CMD_SELECT); /* pushing down rtes without relations yields (shardCount * expectedRows) */ - if (subquery->rtable == NIL) + if (HasEmptyJoinTree(subquery)) { return DeferredError(ERRCODE_FEATURE_NOT_SUPPORTED, "Subqueries without relations are not allowed in " diff --git a/src/backend/distributed/planner/multi_logical_optimizer.c b/src/backend/distributed/planner/multi_logical_optimizer.c index eba8a9d34..f93312779 100644 --- a/src/backend/distributed/planner/multi_logical_optimizer.c +++ b/src/backend/distributed/planner/multi_logical_optimizer.c @@ -35,6 +35,7 @@ #include "distributed/multi_physical_planner.h" #include "distributed/pg_dist_partition.h" #include "distributed/worker_protocol.h" +#include "distributed/version_compat.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "nodes/print.h" @@ -53,7 +54,6 @@ #include "utils/lsyscache.h" #include "utils/rel.h" #include "utils/syscache.h" -#include "utils/tqual.h" /* Config variable managed via guc.c */ @@ -2966,7 +2966,11 @@ AggregateFunctionOid(const char *functionName, Oid inputType) /* check if input type and found value type match */ if (procForm->proargtypes.values[0] == inputType) { +#if PG_VERSION_NUM < 120000 functionOid = HeapTupleGetOid(heapTuple); +#else + functionOid = procForm->oid; +#endif break; } } @@ -2996,8 +3000,9 @@ TypeOid(Oid schemaId, const char *typeName) { Oid typeOid; - typeOid = GetSysCacheOid2(TYPENAMENSP, PointerGetDatum(typeName), - ObjectIdGetDatum(schemaId)); + typeOid = GetSysCacheOid2Compat(TYPENAMENSP, Anum_pg_type_oid, PointerGetDatum( + typeName), + ObjectIdGetDatum(schemaId)); return typeOid; } diff --git a/src/backend/distributed/planner/multi_logical_planner.c b/src/backend/distributed/planner/multi_logical_planner.c index db404942a..e04fc4dc8 100644 --- a/src/backend/distributed/planner/multi_logical_planner.c +++ b/src/backend/distributed/planner/multi_logical_planner.c @@ -155,7 +155,7 @@ MultiLogicalPlanCreate(Query *originalQuery, Query *queryTree, * FindNodeCheck finds a node for which the check function returns true. * * To call this function directly with an RTE, use: - * range_table_walker(rte, FindNodeCheck, check, QTW_EXAMINE_RTES) + * range_table_walker(rte, FindNodeCheck, check, QTW_EXAMINE_RTES_BEFORE) */ bool FindNodeCheck(Node *node, bool (*check)(Node *)) @@ -177,7 +177,8 @@ FindNodeCheck(Node *node, bool (*check)(Node *)) } else if (IsA(node, Query)) { - return query_tree_walker((Query *) node, FindNodeCheck, check, QTW_EXAMINE_RTES); + return query_tree_walker((Query *) node, FindNodeCheck, check, + QTW_EXAMINE_RTES_BEFORE); } return expression_tree_walker(node, FindNodeCheck, check); @@ -385,7 +386,7 @@ AllTargetExpressionsAreColumnReferences(List *targetEntryList) bool FindNodeCheckInRangeTableList(List *rtable, bool (*check)(Node *)) { - return range_table_walker(rtable, FindNodeCheck, check, QTW_EXAMINE_RTES); + return range_table_walker(rtable, FindNodeCheck, check, QTW_EXAMINE_RTES_BEFORE); } @@ -1997,7 +1998,8 @@ ExtractRangeTableRelationWalker(Node *node, List **rangeTableRelationList) { walkIsComplete = query_tree_walker((Query *) node, ExtractRangeTableRelationWalker, - rangeTableRelationList, QTW_EXAMINE_RTES); + rangeTableRelationList, + QTW_EXAMINE_RTES_BEFORE); } else { @@ -2045,7 +2047,7 @@ ExtractRangeTableEntryWalker(Node *node, List **rangeTableList) walkIsComplete = query_tree_walker((Query *) node, ExtractRangeTableEntryWalker, rangeTableList, - QTW_EXAMINE_RTES); + QTW_EXAMINE_RTES_BEFORE); } else { @@ -2053,7 +2055,7 @@ ExtractRangeTableEntryWalker(Node *node, List **rangeTableList) walkIsComplete = range_table_walker(query->rtable, ExtractRangeTableEntryWalker, rangeTableList, - QTW_EXAMINE_RTES); + QTW_EXAMINE_RTES_BEFORE); } } else diff --git a/src/backend/distributed/planner/multi_router_planner.c b/src/backend/distributed/planner/multi_router_planner.c index 86445a5df..8cbf960c6 100644 --- a/src/backend/distributed/planner/multi_router_planner.c +++ b/src/backend/distributed/planner/multi_router_planner.c @@ -679,7 +679,11 @@ ModifyQuerySupported(Query *queryTree, Query *originalQuery, bool multiShardQuer NULL, NULL); } } - else if (rangeTableEntry->rtekind == RTE_VALUES) + else if (rangeTableEntry->rtekind == RTE_VALUES +#if PG_VERSION_NUM >= 120000 + || rangeTableEntry->rtekind == RTE_RESULT +#endif + ) { /* do nothing, this type is supported */ } diff --git a/src/backend/distributed/planner/query_pushdown_planning.c b/src/backend/distributed/planner/query_pushdown_planning.c index a3ccbb6b3..2b3f8eeee 100644 --- a/src/backend/distributed/planner/query_pushdown_planning.c +++ b/src/backend/distributed/planner/query_pushdown_planning.c @@ -31,6 +31,7 @@ #include "distributed/pg_dist_partition.h" #include "distributed/query_pushdown_planning.h" #include "distributed/relation_restriction_equivalence.h" +#include "distributed/version_compat.h" #include "nodes/nodeFuncs.h" #if PG_VERSION_NUM >= 120000 #include "nodes/makefuncs.h" @@ -199,6 +200,32 @@ JoinTreeContainsSubquery(Query *query) } +/* + * HasEmptyJoinTree returns whether the query selects from anything. + */ +bool +HasEmptyJoinTree(Query *query) +{ + if (query->rtable == NIL) + { + return true; + } + +#if PG_VERSION_NUM >= 120000 + else if (list_length(query->rtable) == 1) + { + RangeTblEntry *rte = (RangeTblEntry *) linitial(query->rtable); + if (rte->rtekind == RTE_RESULT) + { + return true; + } + } +#endif + + return false; +} + + /* * JoinTreeContainsSubqueryWalker returns true if the input joinTreeNode * references to a subquery. Otherwise, recurses into the expression. @@ -656,7 +683,7 @@ FromClauseRecurringTupleType(Query *queryTree) { RecurringTuplesType recurType = RECURRING_TUPLES_INVALID; - if (queryTree->rtable == NIL) + if (HasEmptyJoinTree(queryTree)) { return RECURRING_TUPLES_EMPTY_JOIN_TREE; } @@ -822,7 +849,7 @@ DeferErrorIfCannotPushdownSubquery(Query *subqueryTree, bool outerMostQueryHasLi return deferredError; } - if (subqueryTree->rtable == NIL && + if (HasEmptyJoinTree(subqueryTree) && contain_mutable_functions((Node *) subqueryTree->targetList)) { preconditionsSatisfied = false; @@ -1014,7 +1041,11 @@ DeferErrorIfUnsupportedTableCombination(Query *queryTree) * subquery, or immutable function. */ if (rangeTableEntry->rtekind == RTE_RELATION || - rangeTableEntry->rtekind == RTE_SUBQUERY) + rangeTableEntry->rtekind == RTE_SUBQUERY +#if PG_VERSION_NUM >= 120000 + || rangeTableEntry->rtekind == RTE_RESULT +#endif + ) { /* accepted */ } @@ -1337,7 +1368,7 @@ static bool IsRecurringRangeTable(List *rangeTable, RecurringTuplesType *recurType) { return range_table_walker(rangeTable, HasRecurringTuples, recurType, - QTW_EXAMINE_RTES); + QTW_EXAMINE_RTES_BEFORE); } @@ -1393,6 +1424,13 @@ HasRecurringTuples(Node *node, RecurringTuplesType *recurType) */ return true; } +#if PG_VERSION_NUM >= 120000 + else if (rangeTableEntry->rtekind == RTE_RESULT) + { + *recurType = RECURRING_TUPLES_EMPTY_JOIN_TREE; + return true; + } +#endif return false; } @@ -1400,7 +1438,7 @@ HasRecurringTuples(Node *node, RecurringTuplesType *recurType) { Query *query = (Query *) node; - if (query->rtable == NIL) + if (HasEmptyJoinTree(query)) { *recurType = RECURRING_TUPLES_EMPTY_JOIN_TREE; @@ -1412,7 +1450,7 @@ HasRecurringTuples(Node *node, RecurringTuplesType *recurType) } return query_tree_walker((Query *) node, HasRecurringTuples, - recurType, QTW_EXAMINE_RTES); + recurType, QTW_EXAMINE_RTES_BEFORE); } return expression_tree_walker(node, HasRecurringTuples, recurType); diff --git a/src/backend/distributed/planner/recursive_planning.c b/src/backend/distributed/planner/recursive_planning.c index 56c55c810..8bb474638 100644 --- a/src/backend/distributed/planner/recursive_planning.c +++ b/src/backend/distributed/planner/recursive_planning.c @@ -1222,7 +1222,8 @@ CteReferenceListWalker(Node *node, CteReferenceWalkerContext *context) Query *query = (Query *) node; context->level += 1; - query_tree_walker(query, CteReferenceListWalker, context, QTW_EXAMINE_RTES); + query_tree_walker(query, CteReferenceListWalker, context, + QTW_EXAMINE_RTES_BEFORE); context->level -= 1; return false; diff --git a/src/backend/distributed/progress/multi_progress.c b/src/backend/distributed/progress/multi_progress.c index c7e6e9e06..62e0b3abd 100644 --- a/src/backend/distributed/progress/multi_progress.c +++ b/src/backend/distributed/progress/multi_progress.c @@ -13,6 +13,7 @@ #include "distributed/function_utils.h" #include "distributed/multi_progress.h" +#include "distributed/version_compat.h" #include "storage/dsm.h" #include "utils/builtins.h" @@ -155,7 +156,8 @@ ProgressMonitorList(uint64 commandTypeMagicNumber, List **attachedDSMSegments) getProgressInfoFunctionOid, commandTypeDatum); - tupleTableSlot = MakeSingleTupleTableSlot(progressResultSet->setDesc); + tupleTableSlot = MakeSingleTupleTableSlotCompat(progressResultSet->setDesc, + &TTSOpsMinimalTuple); /* iterate over tuples in tuple store, and send them to destination */ for (;;) diff --git a/src/backend/distributed/relay/relay_event_utility.c b/src/backend/distributed/relay/relay_event_utility.c index 8bbfc3b35..0fa9e080c 100644 --- a/src/backend/distributed/relay/relay_event_utility.c +++ b/src/backend/distributed/relay/relay_event_utility.c @@ -756,7 +756,7 @@ AppendShardIdToName(char **name, uint64 shardId) if (neededBytes < 0) { ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), - errmsg("out of memory: %s", strerror(errno)))); + errmsg("out of memory: %m"))); } else if (neededBytes >= NAMEDATALEN) { diff --git a/src/backend/distributed/utils/citus_nodefuncs.c b/src/backend/distributed/utils/citus_nodefuncs.c index ce0aa9f56..6ad0f6cb2 100644 --- a/src/backend/distributed/utils/citus_nodefuncs.c +++ b/src/backend/distributed/utils/citus_nodefuncs.c @@ -309,10 +309,13 @@ GetRangeTblKind(RangeTblEntry *rte) case RTE_JOIN: case RTE_VALUES: case RTE_CTE: - { - rteKind = (CitusRTEKind) rte->rtekind; - break; - } +#if PG_VERSION_NUM >= 120000 + case RTE_RESULT: +#endif + { + rteKind = (CitusRTEKind) rte->rtekind; + break; + } case RTE_FUNCTION: { diff --git a/src/backend/distributed/utils/citus_readfuncs.c b/src/backend/distributed/utils/citus_readfuncs.c index a15c5d0f6..a69764280 100644 --- a/src/backend/distributed/utils/citus_readfuncs.c +++ b/src/backend/distributed/utils/citus_readfuncs.c @@ -45,9 +45,15 @@ CitusSetTag(Node *node, int tag) nodeTypeName *local_node = (nodeTypeName *) CitusSetTag((Node *) node, T_##nodeTypeName) /* And a few guys need only the pg_strtok support fields */ +#if PG_VERSION_NUM >= 120000 #define READ_TEMP_LOCALS() \ - char *token; \ + const char *token; \ int length +#else +#define READ_TEMP_LOCALS() \ + char *token; \ + int length +#endif /* ... but most need both */ #define READ_LOCALS(nodeTypeName) \ diff --git a/src/backend/distributed/utils/citus_ruleutils.c b/src/backend/distributed/utils/citus_ruleutils.c index 7bc91c580..fa192d79c 100644 --- a/src/backend/distributed/utils/citus_ruleutils.c +++ b/src/backend/distributed/utils/citus_ruleutils.c @@ -128,7 +128,11 @@ get_extension_schema(Oid ext_oid) rel = heap_open(ExtensionRelationId, AccessShareLock); ScanKeyInit(&entry[0], +#if PG_VERSION_NUM >= 120000 + Anum_pg_extension_oid, +#else ObjectIdAttributeNumber, +#endif BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(ext_oid)); diff --git a/src/backend/distributed/utils/statistics_collection.c b/src/backend/distributed/utils/statistics_collection.c index fcdb6100e..2a30be485 100644 --- a/src/backend/distributed/utils/statistics_collection.c +++ b/src/backend/distributed/utils/statistics_collection.c @@ -13,7 +13,6 @@ #include "citus_version.h" #include "fmgr.h" #include "utils/uuid.h" -#include "utils/backend_random.h" bool EnableStatisticsCollection = true; /* send basic usage statistics to Citus */ @@ -600,11 +599,11 @@ citus_server_id(PG_FUNCTION_ARGS) uint8 *buf = (uint8 *) palloc(UUID_LEN); /* - * If pg_backend_random() fails, fall-back to using random(). In previous - * versions of postgres we don't have pg_backend_random(), so use it by + * If pg_strong_random() fails, fall-back to using random(). In previous + * versions of postgres we don't have pg_strong_random(), so use it by * default in that case. */ - if (!pg_backend_random((char *) buf, UUID_LEN)) + if (!pg_strong_random((char *) buf, UUID_LEN)) { int bufIdx = 0; for (bufIdx = 0; bufIdx < UUID_LEN; bufIdx++) diff --git a/src/backend/distributed/worker/worker_merge_protocol.c b/src/backend/distributed/worker/worker_merge_protocol.c index 07012516d..1b9c4cca0 100644 --- a/src/backend/distributed/worker/worker_merge_protocol.c +++ b/src/backend/distributed/worker/worker_merge_protocol.c @@ -17,6 +17,10 @@ #include "funcapi.h" #include "miscadmin.h" +#if PG_VERSION_NUM >= 120000 +#include "access/genam.h" +#include "access/table.h" +#endif #include "access/htup_details.h" #include "access/xact.h" #include "catalog/dependency.h" @@ -35,7 +39,6 @@ #include "utils/builtins.h" #include "utils/snapmgr.h" #include "utils/syscache.h" -#include "utils/tqual.h" /* Local functions forward declarations */ @@ -263,7 +266,11 @@ Datum worker_cleanup_job_schema_cache(PG_FUNCTION_ARGS) { Relation pgNamespace = NULL; +#if PG_VERSION_NUM >= 120000 + TableScanDesc scanDescriptor = NULL; +#else HeapScanDesc scanDescriptor = NULL; +#endif ScanKey scanKey = NULL; int scanKeyCount = 0; HeapTuple heapTuple = NULL; @@ -271,7 +278,11 @@ worker_cleanup_job_schema_cache(PG_FUNCTION_ARGS) CheckCitusVersion(ERROR); pgNamespace = heap_open(NamespaceRelationId, AccessExclusiveLock); +#if PG_VERSION_NUM >= 120000 + scanDescriptor = table_beginscan_catalog(pgNamespace, scanKeyCount, scanKey); +#else scanDescriptor = heap_beginscan_catalog(pgNamespace, scanKeyCount, scanKey); +#endif heapTuple = heap_getnext(scanDescriptor, ForwardScanDirection); while (HeapTupleIsValid(heapTuple)) @@ -362,7 +373,8 @@ RemoveJobSchema(StringInfo schemaName) Datum schemaNameDatum = CStringGetDatum(schemaName->data); Oid schemaId = InvalidOid; - schemaId = GetSysCacheOid(NAMESPACENAME, schemaNameDatum, 0, 0, 0); + schemaId = GetSysCacheOid1Compat(NAMESPACENAME, Anum_pg_namespace_oid, + schemaNameDatum); if (OidIsValid(schemaId)) { ObjectAddress schemaObject = { 0, 0, 0 }; diff --git a/src/include/distributed/query_pushdown_planning.h b/src/include/distributed/query_pushdown_planning.h index 607d63b11..412868e6e 100644 --- a/src/include/distributed/query_pushdown_planning.h +++ b/src/include/distributed/query_pushdown_planning.h @@ -24,6 +24,7 @@ extern bool SubqueryPushdown; extern bool ShouldUseSubqueryPushDown(Query *originalQuery, Query *rewrittenQuery); extern bool JoinTreeContainsSubquery(Query *query); +extern bool HasEmptyJoinTree(Query *query); extern bool WhereClauseContainsSubquery(Query *query); extern bool SafeToPushdownWindowFunction(Query *query, StringInfo *errorDetail); extern MultiNode * SubqueryMultiNodeTree(Query *originalQuery, diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index 4803313b5..c21ce8302 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -16,7 +16,11 @@ #include "catalog/namespace.h" #include "nodes/parsenodes.h" -#if (PG_VERSION_NUM >= 100000 && PG_VERSION_NUM < 110000) +#if (PG_VERSION_NUM >= 120000) +#include "optimizer/optimizer.h" +#endif + +#if (PG_VERSION_NUM < 110000) #include "access/hash.h" #include "storage/fd.h" @@ -240,5 +244,39 @@ RangeVarGetRelidInternal(const RangeVar *relation, LOCKMODE lockmode, uint32 fla #endif +#if PG_VERSION_NUM >= 120000 + +#define MakeSingleTupleTableSlotCompat MakeSingleTupleTableSlot +#define AllocSetContextCreateExtended AllocSetContextCreateInternal +#define NextCopyFromCompat NextCopyFrom +#define ArrayRef SubscriptingRef +#define T_ArrayRef T_SubscriptingRef +#define or_clause is_orclause +#define GetSysCacheOid1Compat GetSysCacheOid1 +#define GetSysCacheOid2Compat GetSysCacheOid2 +#define GetSysCacheOid3Compat GetSysCacheOid3 +#define GetSysCacheOid4Compat GetSysCacheOid4 + +#else /* pre PG12 */ +#define QTW_EXAMINE_RTES_BEFORE QTW_EXAMINE_RTES +#define MakeSingleTupleTableSlotCompat(tupleDesc, tts_opts) \ + MakeSingleTupleTableSlot(tupleDesc) +#define NextCopyFromCompat(cstate, econtext, values, nulls) \ + NextCopyFrom(cstate, econtext, values, nulls, NULL) + +/* + * In PG12 GetSysCacheOid requires an oid column, + * whereas beforehand the oid column was implicit with WITH OIDS + */ +#define GetSysCacheOid1Compat(cacheId, oidcol, key1) \ + GetSysCacheOid1(cacheId, key1) +#define GetSysCacheOid2Compat(cacheId, oidcol, key1, key2) \ + GetSysCacheOid2(cacheId, key1, key2) +#define GetSysCacheOid3Compat(cacheId, oidcol, key1, key2, key3) \ + GetSysCacheOid3(cacheId, key1, key2, key3) +#define GetSysCacheOid4Compat(cacheId, oidcol, key1, key2, key3, key4) \ + GetSysCacheOid4(cacheId, key1, key2, key3, key4) + +#endif /* PG12 */ #endif /* VERSION_COMPAT_H */ From fe10ca453d29c6c53500a849950397348725549d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philip=20Dub=C3=A9?= Date: Thu, 8 Aug 2019 21:59:58 +0000 Subject: [PATCH 06/12] Implement FileCompat to abstract pg12 requiring API consumer to track file offsets --- src/backend/distributed/commands/transmit.c | 20 +++--- .../executor/intermediate_results.c | 23 +++--- .../worker/worker_partition_protocol.c | 47 ++++++------ .../worker/worker_sql_task_protocol.c | 24 ++++--- src/include/distributed/version_compat.h | 72 +++++++++++++++++++ src/include/distributed/worker_protocol.h | 2 +- 6 files changed, 136 insertions(+), 52 deletions(-) diff --git a/src/backend/distributed/commands/transmit.c b/src/backend/distributed/commands/transmit.c index bfa434b31..9e6b0f059 100644 --- a/src/backend/distributed/commands/transmit.c +++ b/src/backend/distributed/commands/transmit.c @@ -43,11 +43,10 @@ RedirectCopyDataToRegularFile(const char *filename) { StringInfo copyData = makeStringInfo(); bool copyDone = false; - File fileDesc = -1; const int fileFlags = (O_APPEND | O_CREAT | O_RDWR | O_TRUNC | PG_BINARY); const int fileMode = (S_IRUSR | S_IWUSR); - - fileDesc = FileOpenForTransmit(filename, fileFlags, fileMode); + File fileDesc = FileOpenForTransmit(filename, fileFlags, fileMode); + FileCompat fileCompat = FileCompatFromFileStart(fileDesc); SendCopyInStart(); @@ -57,8 +56,8 @@ RedirectCopyDataToRegularFile(const char *filename) /* if received data has contents, append to regular file */ if (copyData->len > 0) { - int appended = FileWrite(fileDesc, copyData->data, copyData->len, - PG_WAIT_IO); + int appended = FileWriteCompat(&fileCompat, copyData->data, + copyData->len, PG_WAIT_IO); if (appended != copyData->len) { @@ -84,7 +83,6 @@ RedirectCopyDataToRegularFile(const char *filename) void SendRegularFile(const char *filename) { - File fileDesc = -1; StringInfo fileBuffer = NULL; int readBytes = -1; const uint32 fileBufferSize = 32768; /* 32 KB */ @@ -92,7 +90,8 @@ SendRegularFile(const char *filename) const int fileMode = 0; /* we currently do not check if the caller has permissions for this file */ - fileDesc = FileOpenForTransmit(filename, fileFlags, fileMode); + File fileDesc = FileOpenForTransmit(filename, fileFlags, fileMode); + FileCompat fileCompat = FileCompatFromFileStart(fileDesc); /* * We read file's contents into buffers of 32 KB. This buffer size is twice @@ -103,7 +102,8 @@ SendRegularFile(const char *filename) SendCopyOutStart(); - readBytes = FileRead(fileDesc, fileBuffer->data, fileBufferSize, PG_WAIT_IO); + readBytes = FileReadCompat(&fileCompat, fileBuffer->data, fileBufferSize, + PG_WAIT_IO); while (readBytes > 0) { fileBuffer->len = readBytes; @@ -111,8 +111,8 @@ SendRegularFile(const char *filename) SendCopyData(fileBuffer); resetStringInfo(fileBuffer); - readBytes = FileRead(fileDesc, fileBuffer->data, fileBufferSize, - PG_WAIT_IO); + readBytes = FileReadCompat(&fileCompat, fileBuffer->data, fileBufferSize, + PG_WAIT_IO); } SendCopyDone(); diff --git a/src/backend/distributed/executor/intermediate_results.c b/src/backend/distributed/executor/intermediate_results.c index 71da880c6..cec12ce34 100644 --- a/src/backend/distributed/executor/intermediate_results.c +++ b/src/backend/distributed/executor/intermediate_results.c @@ -27,6 +27,7 @@ #include "distributed/remote_commands.h" #include "distributed/transmit.h" #include "distributed/transaction_identifier.h" +#include "distributed/version_compat.h" #include "distributed/worker_protocol.h" #include "nodes/makefuncs.h" #include "nodes/parsenodes.h" @@ -65,7 +66,7 @@ typedef struct RemoteFileDestReceiver /* whether to write to a local file */ bool writeLocalFile; - File fileDesc; + FileCompat fileCompat; /* state on how to copy out data types */ CopyOutState copyOutState; @@ -79,7 +80,7 @@ typedef struct RemoteFileDestReceiver static void RemoteFileDestReceiverStartup(DestReceiver *dest, int operation, TupleDesc inputTupleDescriptor); static StringInfo ConstructCopyResultStatement(const char *resultId); -static void WriteToLocalFile(StringInfo copyData, File fileDesc); +static void WriteToLocalFile(StringInfo copyData, FileCompat *fileCompat); static bool RemoteFileDestReceiverReceive(TupleTableSlot *slot, DestReceiver *dest); static void BroadcastCopyData(StringInfo dataBuffer, List *connectionList); static void SendCopyDataOverConnection(StringInfo dataBuffer, @@ -263,7 +264,9 @@ RemoteFileDestReceiverStartup(DestReceiver *dest, int operation, elog(DEBUG1, "writing to local file \"%s\"", fileName); - resultDest->fileDesc = FileOpenForTransmit(fileName, fileFlags, fileMode); + resultDest->fileCompat = FileCompatFromFileStart(FileOpenForTransmit(fileName, + fileFlags, + fileMode)); } foreach(initialNodeCell, initialNodeList) @@ -329,7 +332,7 @@ RemoteFileDestReceiverStartup(DestReceiver *dest, int operation, if (resultDest->writeLocalFile) { - WriteToLocalFile(copyOutState->fe_msgbuf, resultDest->fileDesc); + WriteToLocalFile(copyOutState->fe_msgbuf, &resultDest->fileCompat); } } @@ -394,7 +397,7 @@ RemoteFileDestReceiverReceive(TupleTableSlot *slot, DestReceiver *dest) /* write to local file (if applicable) */ if (resultDest->writeLocalFile) { - WriteToLocalFile(copyOutState->fe_msgbuf, resultDest->fileDesc); + WriteToLocalFile(copyOutState->fe_msgbuf, &resultDest->fileCompat); } MemoryContextSwitchTo(oldContext); @@ -411,9 +414,11 @@ RemoteFileDestReceiverReceive(TupleTableSlot *slot, DestReceiver *dest) * WriteToLocalResultsFile writes the bytes in a StringInfo to a local file. */ static void -WriteToLocalFile(StringInfo copyData, File fileDesc) +WriteToLocalFile(StringInfo copyData, FileCompat *fileCompat) { - int bytesWritten = FileWrite(fileDesc, copyData->data, copyData->len, PG_WAIT_IO); + int bytesWritten = FileWriteCompat(fileCompat, copyData->data, + copyData->len, + PG_WAIT_IO); if (bytesWritten < 0) { ereport(ERROR, (errcode_for_file_access(), @@ -444,7 +449,7 @@ RemoteFileDestReceiverShutdown(DestReceiver *destReceiver) if (resultDest->writeLocalFile) { - WriteToLocalFile(copyOutState->fe_msgbuf, resultDest->fileDesc); + WriteToLocalFile(copyOutState->fe_msgbuf, &resultDest->fileCompat); } } @@ -453,7 +458,7 @@ RemoteFileDestReceiverShutdown(DestReceiver *destReceiver) if (resultDest->writeLocalFile) { - FileClose(resultDest->fileDesc); + FileClose(resultDest->fileCompat.fd); } } diff --git a/src/backend/distributed/worker/worker_partition_protocol.c b/src/backend/distributed/worker/worker_partition_protocol.c index d22ffe1d7..89f06a54f 100644 --- a/src/backend/distributed/worker/worker_partition_protocol.c +++ b/src/backend/distributed/worker/worker_partition_protocol.c @@ -65,8 +65,8 @@ static uint32 FileBufferSize(int partitionBufferSizeInKB, uint32 fileCount); static FileOutputStream * OpenPartitionFiles(StringInfo directoryName, uint32 fileCount); static void ClosePartitionFiles(FileOutputStream *partitionFileArray, uint32 fileCount); static void RenameDirectory(StringInfo oldDirectoryName, StringInfo newDirectoryName); -static void FileOutputStreamWrite(FileOutputStream file, StringInfo dataToWrite); -static void FileOutputStreamFlush(FileOutputStream file); +static void FileOutputStreamWrite(FileOutputStream *file, StringInfo dataToWrite); +static void FileOutputStreamFlush(FileOutputStream *file); static void FilterAndPartitionTable(const char *filterQuery, const char *columnName, Oid columnType, uint32 (*PartitionIdFunction)(Datum, const void *), @@ -221,6 +221,7 @@ worker_hash_partition_table(PG_FUNCTION_ARGS) partitionContext->hashFunction = hashFunction; partitionContext->partitionCount = partitionCount; + partitionContext->collation = PG_GET_COLLATION(); /* we'll use binary search, we need the comparison function */ if (!partitionContext->hasUniformHashDistribution) @@ -464,7 +465,7 @@ OpenPartitionFiles(StringInfo directoryName, uint32 fileCount) FileOutputStream *partitionFileArray = NULL; File fileDescriptor = 0; uint32 fileIndex = 0; - const int fileFlags = (O_APPEND | O_CREAT | O_RDWR | PG_BINARY); + const int fileFlags = (O_APPEND | O_CREAT | O_RDWR | O_TRUNC | PG_BINARY); const int fileMode = (S_IRUSR | S_IWUSR); partitionFileArray = palloc0(fileCount * sizeof(FileOutputStream)); @@ -480,7 +481,8 @@ OpenPartitionFiles(StringInfo directoryName, uint32 fileCount) errmsg("could not open file \"%s\": %m", filePath->data))); } - partitionFileArray[fileIndex].fileDescriptor = fileDescriptor; + partitionFileArray[fileIndex].fileCompat = FileCompatFromFileStart( + fileDescriptor); partitionFileArray[fileIndex].fileBuffer = makeStringInfo(); partitionFileArray[fileIndex].filePath = filePath; } @@ -500,13 +502,13 @@ ClosePartitionFiles(FileOutputStream *partitionFileArray, uint32 fileCount) uint32 fileIndex = 0; for (fileIndex = 0; fileIndex < fileCount; fileIndex++) { - FileOutputStream partitionFile = partitionFileArray[fileIndex]; + FileOutputStream *partitionFile = &partitionFileArray[fileIndex]; FileOutputStreamFlush(partitionFile); - FileClose(partitionFile.fileDescriptor); - FreeStringInfo(partitionFile.fileBuffer); - FreeStringInfo(partitionFile.filePath); + FileClose(partitionFile->fileCompat.fd); + FreeStringInfo(partitionFile->fileBuffer); + FreeStringInfo(partitionFile->filePath); } pfree(partitionFileArray); @@ -829,9 +831,9 @@ RenameDirectory(StringInfo oldDirectoryName, StringInfo newDirectoryName) * if so, the function flushes the buffer to the underlying file. */ static void -FileOutputStreamWrite(FileOutputStream file, StringInfo dataToWrite) +FileOutputStreamWrite(FileOutputStream *file, StringInfo dataToWrite) { - StringInfo fileBuffer = file.fileBuffer; + StringInfo fileBuffer = file->fileBuffer; uint32 newBufferSize = fileBuffer->len + dataToWrite->len; appendBinaryStringInfo(fileBuffer, dataToWrite->data, dataToWrite->len); @@ -847,19 +849,19 @@ FileOutputStreamWrite(FileOutputStream file, StringInfo dataToWrite) /* Flushes data buffered in the file stream object to the underlying file. */ static void -FileOutputStreamFlush(FileOutputStream file) +FileOutputStreamFlush(FileOutputStream *file) { - StringInfo fileBuffer = file.fileBuffer; + StringInfo fileBuffer = file->fileBuffer; int written = 0; errno = 0; - written = FileWrite(file.fileDescriptor, fileBuffer->data, fileBuffer->len, - PG_WAIT_IO); + written = FileWriteCompat(&file->fileCompat, fileBuffer->data, fileBuffer->len, + PG_WAIT_IO); if (written != fileBuffer->len) { ereport(ERROR, (errcode_for_file_access(), errmsg("could not write %d bytes to partition file \"%s\"", - fileBuffer->len, file.filePath->data))); + fileBuffer->len, file->filePath->data))); } } @@ -952,7 +954,7 @@ FilterAndPartitionTable(const char *filterQuery, { HeapTuple row = SPI_tuptable->vals[rowIndex]; TupleDesc rowDescriptor = SPI_tuptable->tupdesc; - FileOutputStream partitionFile = { 0, 0, 0 }; + FileOutputStream *partitionFile = NULL; StringInfo rowText = NULL; Datum partitionKey = 0; bool partitionKeyNull = false; @@ -988,7 +990,7 @@ FilterAndPartitionTable(const char *filterQuery, rowText = rowOutputState->fe_msgbuf; - partitionFile = partitionFileArray[partitionId]; + partitionFile = &partitionFileArray[partitionId]; FileOutputStreamWrite(partitionFile, rowText); resetStringInfo(rowText); @@ -1136,7 +1138,7 @@ OutputBinaryHeaders(FileOutputStream *partitionFileArray, uint32 fileCount) for (fileIndex = 0; fileIndex < fileCount; fileIndex++) { /* Generate header for a binary copy */ - FileOutputStream partitionFile = { 0, 0, 0 }; + FileOutputStream partitionFile = { }; CopyOutStateData headerOutputStateData; CopyOutState headerOutputState = (CopyOutState) & headerOutputStateData; @@ -1146,7 +1148,7 @@ OutputBinaryHeaders(FileOutputStream *partitionFileArray, uint32 fileCount) AppendCopyBinaryHeaders(headerOutputState); partitionFile = partitionFileArray[fileIndex]; - FileOutputStreamWrite(partitionFile, headerOutputState->fe_msgbuf); + FileOutputStreamWrite(&partitionFile, headerOutputState->fe_msgbuf); } } @@ -1162,7 +1164,7 @@ OutputBinaryFooters(FileOutputStream *partitionFileArray, uint32 fileCount) for (fileIndex = 0; fileIndex < fileCount; fileIndex++) { /* Generate footer for a binary copy */ - FileOutputStream partitionFile = { 0, 0, 0 }; + FileOutputStream partitionFile = { }; CopyOutStateData footerOutputStateData; CopyOutState footerOutputState = (CopyOutState) & footerOutputStateData; @@ -1172,7 +1174,7 @@ OutputBinaryFooters(FileOutputStream *partitionFileArray, uint32 fileCount) AppendCopyBinaryFooters(footerOutputState); partitionFile = partitionFileArray[fileIndex]; - FileOutputStreamWrite(partitionFile, footerOutputState->fe_msgbuf); + FileOutputStreamWrite(&partitionFile, footerOutputState->fe_msgbuf); } } @@ -1263,7 +1265,8 @@ HashPartitionId(Datum partitionValue, const void *context) ShardInterval **syntheticShardIntervalArray = hashPartitionContext->syntheticShardIntervalArray; FmgrInfo *comparisonFunction = hashPartitionContext->comparisonFunction; - Datum hashDatum = FunctionCall1(hashFunction, partitionValue); + Datum hashDatum = FunctionCall1Coll(hashFunction, hashPartitionContext->collation, + partitionValue); int32 hashResult = 0; uint32 hashPartitionId = 0; diff --git a/src/backend/distributed/worker/worker_sql_task_protocol.c b/src/backend/distributed/worker/worker_sql_task_protocol.c index f0a8a1b53..8c1cee426 100644 --- a/src/backend/distributed/worker/worker_sql_task_protocol.c +++ b/src/backend/distributed/worker/worker_sql_task_protocol.c @@ -16,6 +16,7 @@ #include "distributed/commands/multi_copy.h" #include "distributed/multi_executor.h" #include "distributed/transmit.h" +#include "distributed/version_compat.h" #include "distributed/worker_protocol.h" #include "utils/builtins.h" #include "utils/memutils.h" @@ -38,7 +39,7 @@ typedef struct TaskFileDestReceiver /* output file */ char *filePath; - File fileDesc; + FileCompat fileCompat; bool binaryCopyFormat; /* state on how to copy out data types */ @@ -55,7 +56,7 @@ static DestReceiver * CreateTaskFileDestReceiver(char *filePath, EState *executo static void TaskFileDestReceiverStartup(DestReceiver *dest, int operation, TupleDesc inputTupleDescriptor); static bool TaskFileDestReceiverReceive(TupleTableSlot *slot, DestReceiver *dest); -static void WriteToLocalFile(StringInfo copyData, File fileDesc); +static void WriteToLocalFile(StringInfo copyData, TaskFileDestReceiver *taskFileDest); static void TaskFileDestReceiverShutdown(DestReceiver *destReceiver); static void TaskFileDestReceiverDestroy(DestReceiver *destReceiver); @@ -183,8 +184,10 @@ TaskFileDestReceiverStartup(DestReceiver *dest, int operation, taskFileDest->columnOutputFunctions = ColumnOutputFunctions(inputTupleDescriptor, copyOutState->binary); - taskFileDest->fileDesc = FileOpenForTransmit(taskFileDest->filePath, fileFlags, - fileMode); + taskFileDest->fileCompat = FileCompatFromFileStart(FileOpenForTransmit( + taskFileDest->filePath, + fileFlags, + fileMode)); if (copyOutState->binary) { @@ -192,7 +195,7 @@ TaskFileDestReceiverStartup(DestReceiver *dest, int operation, resetStringInfo(copyOutState->fe_msgbuf); AppendCopyBinaryHeaders(copyOutState); - WriteToLocalFile(copyOutState->fe_msgbuf, taskFileDest->fileDesc); + WriteToLocalFile(copyOutState->fe_msgbuf, taskFileDest); } MemoryContextSwitchTo(oldContext); @@ -233,7 +236,7 @@ TaskFileDestReceiverReceive(TupleTableSlot *slot, DestReceiver *dest) AppendCopyRowData(columnValues, columnNulls, tupleDescriptor, copyOutState, columnOutputFunctions, NULL); - WriteToLocalFile(copyOutState->fe_msgbuf, taskFileDest->fileDesc); + WriteToLocalFile(copyOutState->fe_msgbuf, taskFileDest); MemoryContextSwitchTo(oldContext); @@ -249,9 +252,10 @@ TaskFileDestReceiverReceive(TupleTableSlot *slot, DestReceiver *dest) * WriteToLocalResultsFile writes the bytes in a StringInfo to a local file. */ static void -WriteToLocalFile(StringInfo copyData, File fileDesc) +WriteToLocalFile(StringInfo copyData, TaskFileDestReceiver *taskFileDest) { - int bytesWritten = FileWrite(fileDesc, copyData->data, copyData->len, PG_WAIT_IO); + int bytesWritten = FileWriteCompat(&taskFileDest->fileCompat, copyData->data, + copyData->len, PG_WAIT_IO); if (bytesWritten < 0) { ereport(ERROR, (errcode_for_file_access(), @@ -276,10 +280,10 @@ TaskFileDestReceiverShutdown(DestReceiver *destReceiver) /* write footers when using binary encoding */ resetStringInfo(copyOutState->fe_msgbuf); AppendCopyBinaryFooters(copyOutState); - WriteToLocalFile(copyOutState->fe_msgbuf, taskFileDest->fileDesc); + WriteToLocalFile(copyOutState->fe_msgbuf, taskFileDest); } - FileClose(taskFileDest->fileDesc); + FileClose(taskFileDest->fileCompat.fd); } diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index c21ce8302..7dbe2f137 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -257,6 +257,48 @@ RangeVarGetRelidInternal(const RangeVar *relation, LOCKMODE lockmode, uint32 fla #define GetSysCacheOid3Compat GetSysCacheOid3 #define GetSysCacheOid4Compat GetSysCacheOid4 +typedef struct +{ + File fd; + off_t offset; +} FileCompat; + +static inline int +FileWriteCompat(FileCompat *file, char *buffer, int amount, uint32 wait_event_info) +{ + int count = FileWrite(file->fd, buffer, amount, file->offset, wait_event_info); + if (count > 0) + { + file->offset += count; + } + return count; +} + + +static inline int +FileReadCompat(FileCompat *file, char *buffer, int amount, uint32 wait_event_info) +{ + int count = FileRead(file->fd, buffer, amount, file->offset, wait_event_info); + if (count > 0) + { + file->offset += count; + } + return count; +} + + +static inline FileCompat +FileCompatFromFileStart(File fileDesc) +{ + FileCompat fc = { + .fd = fileDesc, + .offset = 0 + }; + + return fc; +} + + #else /* pre PG12 */ #define QTW_EXAMINE_RTES_BEFORE QTW_EXAMINE_RTES #define MakeSingleTupleTableSlotCompat(tupleDesc, tts_opts) \ @@ -277,6 +319,36 @@ RangeVarGetRelidInternal(const RangeVar *relation, LOCKMODE lockmode, uint32 fla #define GetSysCacheOid4Compat(cacheId, oidcol, key1, key2, key3, key4) \ GetSysCacheOid4(cacheId, key1, key2, key3, key4) +typedef struct +{ + File fd; +} FileCompat; + +static inline int +FileWriteCompat(FileCompat *file, char *buffer, int amount, uint32 wait_event_info) +{ + return FileWrite(file->fd, buffer, amount, wait_event_info); +} + + +static inline int +FileReadCompat(FileCompat *file, char *buffer, int amount, uint32 wait_event_info) +{ + return FileRead(file->fd, buffer, amount, wait_event_info); +} + + +static inline FileCompat +FileCompatFromFileStart(File fileDesc) +{ + FileCompat fc = { + .fd = fileDesc, + }; + + return fc; +} + + #endif /* PG12 */ #endif /* VERSION_COMPAT_H */ diff --git a/src/include/distributed/worker_protocol.h b/src/include/distributed/worker_protocol.h index 5dca935cb..5ee20d68f 100644 --- a/src/include/distributed/worker_protocol.h +++ b/src/include/distributed/worker_protocol.h @@ -91,7 +91,7 @@ typedef struct HashPartitionContext */ typedef struct FileOutputStream { - File fileDescriptor; + FileCompat fileCompat; StringInfo fileBuffer; StringInfo filePath; } FileOutputStream; From be3285828f17c8841d649361edc2aa3dd094f0cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philip=20Dub=C3=A9?= Date: Wed, 14 Aug 2019 21:29:48 +0000 Subject: [PATCH 07/12] Collations matter for hashing strings in pg12 See https://www.postgresql.org/docs/12/collation.html#COLLATION-NONDETERMINISTIC --- .../distributed/master/master_split_shards.c | 2 +- .../planner/multi_master_planner.c | 62 +++++++++++------ .../planner/postgres_planning_functions.c | 69 ++++++++++++++++++- .../distributed/utils/shardinterval_utils.c | 4 +- src/include/distributed/worker_protocol.h | 2 + 5 files changed, 113 insertions(+), 26 deletions(-) diff --git a/src/backend/distributed/master/master_split_shards.c b/src/backend/distributed/master/master_split_shards.c index a1e928e5d..d7cf8f328 100644 --- a/src/backend/distributed/master/master_split_shards.c +++ b/src/backend/distributed/master/master_split_shards.c @@ -84,7 +84,7 @@ worker_hash(PG_FUNCTION_ARGS) fmgr_info_copy(hashFunction, &(typeEntry->hash_proc_finfo), CurrentMemoryContext); /* calculate hash value */ - hashedValueDatum = FunctionCall1(hashFunction, valueDatum); + hashedValueDatum = FunctionCall1Coll(hashFunction, PG_GET_COLLATION(), valueDatum); PG_RETURN_INT32(hashedValueDatum); } diff --git a/src/backend/distributed/planner/multi_master_planner.c b/src/backend/distributed/planner/multi_master_planner.c index eabf113ab..68c78367a 100644 --- a/src/backend/distributed/planner/multi_master_planner.c +++ b/src/backend/distributed/planner/multi_master_planner.c @@ -21,6 +21,7 @@ #include "distributed/multi_physical_planner.h" #include "distributed/distributed_planner.h" #include "distributed/multi_server_executor.h" +#include "distributed/version_compat.h" #include "distributed/worker_protocol.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" @@ -51,6 +52,8 @@ static bool UseGroupAggregateWithHLL(Query *masterQuery); static bool QueryContainsAggregateWithHLL(Query *query); static Plan * BuildDistinctPlan(Query *masterQuery, Plan *subPlan); static List * PrepareTargetListForNextPlan(List *targetList); +static Agg * makeAggNode(List *groupClauseList, List *havingQual, + AggStrategy aggrStrategy, List *queryTargetList, Plan *subPlan); /* @@ -267,7 +270,6 @@ BuildAggregatePlan(Query *masterQuery, Plan *subPlan) Agg *aggregatePlan = NULL; AggStrategy aggregateStrategy = AGG_PLAIN; AggClauseCosts aggregateCosts; - AttrNumber *groupColumnIdArray = NULL; List *aggregateTargetList = NIL; List *groupColumnList = NIL; List *aggregateColumnList = NIL; @@ -275,9 +277,7 @@ BuildAggregatePlan(Query *masterQuery, Plan *subPlan) List *columnList = NIL; ListCell *columnCell = NULL; Node *havingQual = NULL; - Oid *groupColumnOpArray = NULL; uint32 groupColumnCount = 0; - const long rowEstimate = 10; /* assert that we need to build an aggregate plan */ Assert(masterQuery->hasAggs || masterQuery->groupClause); @@ -353,17 +353,11 @@ BuildAggregatePlan(Query *masterQuery, Plan *subPlan) { aggregateStrategy = AGG_HASHED; } - - /* get column indexes that are being grouped */ - groupColumnIdArray = extract_grouping_cols(groupColumnList, subPlan->targetlist); - groupColumnOpArray = extract_grouping_ops(groupColumnList); } /* finally create the plan */ - aggregatePlan = make_agg(aggregateTargetList, (List *) havingQual, aggregateStrategy, - AGGSPLIT_SIMPLE, groupColumnCount, groupColumnIdArray, - groupColumnOpArray, NIL, NIL, - rowEstimate, subPlan); + aggregatePlan = makeAggNode(groupColumnList, (List *) havingQual, + aggregateStrategy, aggregateTargetList, subPlan); /* just for reproducible costs between different PostgreSQL versions */ aggregatePlan->plan.startup_cost = 0; @@ -527,17 +521,8 @@ BuildDistinctPlan(Query *masterQuery, Plan *subPlan) if (enable_hashagg && distinctClausesHashable && !hasDistinctAggregate) { - const long rowEstimate = 10; /* using the same value as BuildAggregatePlan() */ - AttrNumber *distinctColumnIdArray = extract_grouping_cols(distinctClauseList, - subPlan->targetlist); - Oid *distinctColumnOpArray = extract_grouping_ops(distinctClauseList); - uint32 distinctClauseCount = list_length(distinctClauseList); - - distinctPlan = (Plan *) make_agg(targetList, NIL, AGG_HASHED, - AGGSPLIT_SIMPLE, distinctClauseCount, - distinctColumnIdArray, - distinctColumnOpArray, NIL, NIL, - rowEstimate, subPlan); + distinctPlan = (Plan *) makeAggNode(distinctClauseList, NIL, AGG_HASHED, + targetList, subPlan); } else { @@ -581,3 +566,36 @@ PrepareTargetListForNextPlan(List *targetList) return newtargetList; } + + +/* + * makeAggNode creates a "Agg" plan node. groupClauseList is a list of + * SortGroupClause's. + */ +static Agg * +makeAggNode(List *groupClauseList, List *havingQual, AggStrategy aggrStrategy, + List *queryTargetList, Plan *subPlan) +{ + Agg *aggNode = NULL; + int groupColumnCount = list_length(groupClauseList); + AttrNumber *groupColumnIdArray = + extract_grouping_cols(groupClauseList, subPlan->targetlist); + Oid *groupColumnOpArray = extract_grouping_ops(groupClauseList); + const int rowEstimate = 10; + +#if (PG_VERSION_NUM >= 120000) + aggNode = make_agg(queryTargetList, havingQual, aggrStrategy, + AGGSPLIT_SIMPLE, groupColumnCount, groupColumnIdArray, + groupColumnOpArray, + extract_grouping_collations(groupClauseList, + subPlan->targetlist), + NIL, NIL, rowEstimate, subPlan); +#else + aggNode = make_agg(queryTargetList, havingQual, aggrStrategy, + AGGSPLIT_SIMPLE, groupColumnCount, groupColumnIdArray, + groupColumnOpArray, + NIL, NIL, rowEstimate, subPlan); +#endif + + return aggNode; +} diff --git a/src/backend/distributed/planner/postgres_planning_functions.c b/src/backend/distributed/planner/postgres_planning_functions.c index 20b05f921..c867566c8 100644 --- a/src/backend/distributed/planner/postgres_planning_functions.c +++ b/src/backend/distributed/planner/postgres_planning_functions.c @@ -15,15 +15,77 @@ #include "distributed/multi_master_planner.h" #include "nodes/plannodes.h" +#if PG_VERSION_NUM >= 120000 +#include "nodes/nodeFuncs.h" +#include "optimizer/optimizer.h" +#else #include "optimizer/tlist.h" - +#endif /* * make_unique_from_sortclauses creates and returns a unique node * from provided distinct clause list. * The functions is copied from postgresql from * src/backend/optimizer/plan/createplan.c. - * + */ + +#if PG_VERSION_NUM >= 120000 + +/* + * distinctList is a list of SortGroupClauses, identifying the targetlist items + * that should be considered by the Unique filter. The input path must + * already be sorted accordingly. + */ +Unique * +make_unique_from_sortclauses(Plan *lefttree, List *distinctList) +{ + Unique *node = makeNode(Unique); + Plan *plan = &node->plan; + int numCols = list_length(distinctList); + int keyno = 0; + AttrNumber *uniqColIdx; + Oid *uniqOperators; + Oid *uniqCollations; + ListCell *slitem; + + plan->targetlist = lefttree->targetlist; + plan->qual = NIL; + plan->lefttree = lefttree; + plan->righttree = NULL; + + /* + * convert SortGroupClause list into arrays of attr indexes and equality + * operators, as wanted by executor + */ + Assert(numCols > 0); + uniqColIdx = (AttrNumber *) palloc(sizeof(AttrNumber) * numCols); + uniqOperators = (Oid *) palloc(sizeof(Oid) * numCols); + uniqCollations = (Oid *) palloc(sizeof(Oid) * numCols); + + foreach(slitem, distinctList) + { + SortGroupClause *sortcl = (SortGroupClause *) lfirst(slitem); + TargetEntry *tle = get_sortgroupclause_tle(sortcl, plan->targetlist); + + uniqColIdx[keyno] = tle->resno; + uniqOperators[keyno] = sortcl->eqop; + uniqCollations[keyno] = exprCollation((Node *) tle->expr); + Assert(OidIsValid(uniqOperators[keyno])); + keyno++; + } + + node->numCols = numCols; + node->uniqColIdx = uniqColIdx; + node->uniqOperators = uniqOperators; + node->uniqCollations = uniqCollations; + + return node; +} + + +#else + +/* * distinctList is a list of SortGroupClauses, identifying the targetlist items * that should be considered by the Unique filter. The input path must * already be sorted accordingly. @@ -69,3 +131,6 @@ make_unique_from_sortclauses(Plan *lefttree, List *distinctList) return node; } + + +#endif diff --git a/src/backend/distributed/utils/shardinterval_utils.c b/src/backend/distributed/utils/shardinterval_utils.c index 690f6fa49..425c77970 100644 --- a/src/backend/distributed/utils/shardinterval_utils.c +++ b/src/backend/distributed/utils/shardinterval_utils.c @@ -254,7 +254,9 @@ FindShardInterval(Datum partitionColumnValue, DistTableCacheEntry *cacheEntry) if (cacheEntry->partitionMethod == DISTRIBUTE_BY_HASH) { - searchedValue = FunctionCall1(cacheEntry->hashFunction, partitionColumnValue); + searchedValue = FunctionCall1Coll(cacheEntry->hashFunction, + cacheEntry->partitionColumn->varcollid, + partitionColumnValue); } shardIndex = FindShardIntervalIndex(searchedValue, cacheEntry); diff --git a/src/include/distributed/worker_protocol.h b/src/include/distributed/worker_protocol.h index 5ee20d68f..adaea886e 100644 --- a/src/include/distributed/worker_protocol.h +++ b/src/include/distributed/worker_protocol.h @@ -20,6 +20,7 @@ #include "nodes/parsenodes.h" #include "storage/fd.h" #include "utils/array.h" +#include "distributed/version_compat.h" /* Number of rows to prefetch when reading data with a cursor */ @@ -79,6 +80,7 @@ typedef struct HashPartitionContext FmgrInfo *comparisonFunction; ShardInterval **syntheticShardIntervalArray; uint32 partitionCount; + Oid collation; bool hasUniformHashDistribution; } HashPartitionContext; From bee779e7d478d6d85ed19b6bf01c43b065365cb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philip=20Dub=C3=A9?= Date: Thu, 8 Aug 2019 22:02:59 +0000 Subject: [PATCH 08/12] planner/distributed_planner.c: get_func_cost replaced with add_function_cost in pg12 --- .../distributed/planner/distributed_planner.c | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/backend/distributed/planner/distributed_planner.c b/src/backend/distributed/planner/distributed_planner.c index a769e9b78..ddb21593e 100644 --- a/src/backend/distributed/planner/distributed_planner.c +++ b/src/backend/distributed/planner/distributed_planner.c @@ -35,7 +35,12 @@ #include "nodes/nodeFuncs.h" #include "parser/parsetree.h" #include "parser/parse_type.h" +#if PG_VERSION_NUM >= 120000 +#include "optimizer/optimizer.h" +#include "optimizer/plancat.h" +#else #include "optimizer/cost.h" +#endif #include "optimizer/pathnode.h" #include "optimizer/planner.h" #include "utils/builtins.h" @@ -1327,6 +1332,11 @@ AdjustReadIntermediateResultCost(RangeTblEntry *rangeTableEntry, RelOptInfo *rel double rowSizeEstimate = 0; double rowCountEstimate = 0.; double ioCost = 0.; +#if PG_VERSION_NUM >= 120000 + QualCost funcCost = { 0., 0. }; +#else + double funcCost = 0.; +#endif if (rangeTableEntry->rtekind != RTE_FUNCTION || list_length(rangeTableEntry->functions) != 1) @@ -1413,9 +1423,19 @@ AdjustReadIntermediateResultCost(RangeTblEntry *rangeTableEntry, RelOptInfo *rel rowSizeEstimate += 1; } + /* add the cost of parsing a column */ - rowCost += get_func_cost(inputFunctionId) * cpu_operator_cost; +#if PG_VERSION_NUM >= 120000 + add_function_cost(NULL, inputFunctionId, NULL, &funcCost); +#else + funcCost += get_func_cost(inputFunctionId); +#endif } +#if PG_VERSION_NUM >= 120000 + rowCost += funcCost.per_tuple; +#else + rowCost += funcCost * cpu_operator_cost; +#endif /* estimate the number of rows based on the file size and estimated row size */ rowCountEstimate = Max(1, (double) resultSize / rowSizeEstimate); @@ -1429,6 +1449,10 @@ AdjustReadIntermediateResultCost(RangeTblEntry *rangeTableEntry, RelOptInfo *rel path = (Path *) linitial(pathList); path->rows = rowCountEstimate; path->total_cost = rowCountEstimate * rowCost + ioCost; + +#if PG_VERSION_NUM >= 120000 + path->startup_cost = funcCost.startup + relOptInfo->baserestrictcost.startup; +#endif } From e5cd298a987fd4b1b0804e61de21863031bf2078 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philip=20Dub=C3=A9?= Date: Thu, 8 Aug 2019 22:05:16 +0000 Subject: [PATCH 09/12] pg12 revised layout of FunctionCallInfoData See https://github.com/postgres/postgres/commit/a9c35cf85ca1ff72f16f0f10d7ddee6e582b62b8 clang raises a warning due to FunctionCall2InfoData technically being variable sized This is fine, as the struct is the size we want it to be. So silence the warning --- configure | 35 ++++++++ configure.in | 1 + .../distributed/planner/shard_pruning.c | 85 +++++++++++-------- .../distributed/utils/function_utils.c | 12 ++- .../distributed/utils/metadata_cache.c | 34 ++++---- src/include/distributed/version_compat.h | 14 +++ 6 files changed, 121 insertions(+), 60 deletions(-) diff --git a/configure b/configure index 7dfa8965e..b18542566 100755 --- a/configure +++ b/configure @@ -3832,6 +3832,41 @@ if test x"$citusac_cv_prog_cc_cflags__Wno_clobbered" = x"yes"; then CITUS_CFLAGS="$CITUS_CFLAGS -Wno-clobbered" fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC supports -Wno-gnu-variable-sized-type-not-at-end" >&5 +$as_echo_n "checking whether $CC supports -Wno-gnu-variable-sized-type-not-at-end... " >&6; } +if ${citusac_cv_prog_cc_cflags__Wno_gnu_variable_sized_type_not_at_end+:} false; then : + $as_echo_n "(cached) " >&6 +else + citusac_save_CFLAGS=$CFLAGS +CFLAGS="$citusac_save_CFLAGS -Wno-gnu-variable-sized-type-not-at-end" +ac_save_c_werror_flag=$ac_c_werror_flag +ac_c_werror_flag=yes +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + citusac_cv_prog_cc_cflags__Wno_gnu_variable_sized_type_not_at_end=yes +else + citusac_cv_prog_cc_cflags__Wno_gnu_variable_sized_type_not_at_end=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +ac_c_werror_flag=$ac_save_c_werror_flag +CFLAGS="$citusac_save_CFLAGS" +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $citusac_cv_prog_cc_cflags__Wno_gnu_variable_sized_type_not_at_end" >&5 +$as_echo "$citusac_cv_prog_cc_cflags__Wno_gnu_variable_sized_type_not_at_end" >&6; } +if test x"$citusac_cv_prog_cc_cflags__Wno_gnu_variable_sized_type_not_at_end" = x"yes"; then + CITUS_CFLAGS="$CITUS_CFLAGS -Wno-gnu-variable-sized-type-not-at-end" +fi + # And add a few extra warnings { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC supports -Wdeclaration-after-statement" >&5 $as_echo_n "checking whether $CC supports -Wdeclaration-after-statement... " >&6; } diff --git a/configure.in b/configure.in index 136db4d8e..77e8326c5 100644 --- a/configure.in +++ b/configure.in @@ -157,6 +157,7 @@ CITUSAC_PROG_CC_CFLAGS_OPT([-Wno-unused-parameter]) CITUSAC_PROG_CC_CFLAGS_OPT([-Wno-sign-compare]) CITUSAC_PROG_CC_CFLAGS_OPT([-Wno-missing-field-initializers]) CITUSAC_PROG_CC_CFLAGS_OPT([-Wno-clobbered]) +CITUSAC_PROG_CC_CFLAGS_OPT([-Wno-gnu-variable-sized-type-not-at-end]) # And add a few extra warnings CITUSAC_PROG_CC_CFLAGS_OPT([-Wdeclaration-after-statement]) CITUSAC_PROG_CC_CFLAGS_OPT([-Wendif-labels]) diff --git a/src/backend/distributed/planner/shard_pruning.c b/src/backend/distributed/planner/shard_pruning.c index 328d1618c..8ad9130b0 100644 --- a/src/backend/distributed/planner/shard_pruning.c +++ b/src/backend/distributed/planner/shard_pruning.c @@ -59,6 +59,7 @@ #include "distributed/multi_physical_planner.h" #include "distributed/shardinterval_utils.h" #include "distributed/pg_dist_partition.h" +#include "distributed/version_compat.h" #include "distributed/worker_protocol.h" #include "nodes/nodeFuncs.h" #include "nodes/makefuncs.h" @@ -138,6 +139,17 @@ typedef struct PendingPruningInstance Node *continueAt; } PendingPruningInstance; +#if PG_VERSION_NUM >= 120000 +typedef union \ +{ \ + FunctionCallInfoBaseData fcinfo; \ + /* ensure enough space for nargs args is available */ \ + char fcinfo_data[SizeForFunctionCallInfo(2)]; \ +} FunctionCall2InfoData; +#else +typedef FunctionCallInfoData FunctionCall2InfoData; +typedef FunctionCallInfoData *FunctionCallInfo; +#endif /* * Data necessary to perform a single PruneShards(). @@ -161,11 +173,11 @@ typedef struct ClauseWalkerContext /* * Information about function calls we need to perform. Re-using the same - * FunctionCallInfoData, instead of using FunctionCall2Coll, is often + * FunctionCall2InfoData, instead of using FunctionCall2Coll, is often * cheaper. */ - FunctionCallInfoData compareValueFunctionCall; - FunctionCallInfoData compareIntervalFunctionCall; + FunctionCall2InfoData compareValueFunctionCall; + FunctionCall2InfoData compareIntervalFunctionCall; } ClauseWalkerContext; static void PrunableExpressions(Node *originalNode, ClauseWalkerContext *context); @@ -184,9 +196,9 @@ static void AddNewConjuction(ClauseWalkerContext *context, OpExpr *op); static PruningInstance * CopyPartialPruningInstance(PruningInstance *sourceInstance); static List * ShardArrayToList(ShardInterval **shardArray, int length); static List * DeepCopyShardIntervalList(List *originalShardIntervalList); -static int PerformValueCompare(FunctionCallInfoData *compareFunctionCall, Datum a, +static int PerformValueCompare(FunctionCallInfo compareFunctionCall, Datum a, Datum b); -static int PerformCompare(FunctionCallInfoData *compareFunctionCall); +static int PerformCompare(FunctionCallInfo compareFunctionCall); static List * PruneOne(DistTableCacheEntry *cacheEntry, ClauseWalkerContext *context, PruningInstance *prune); @@ -201,11 +213,11 @@ static bool ExhaustivePruneOne(ShardInterval *curInterval, PruningInstance *prune); static int UpperShardBoundary(Datum partitionColumnValue, ShardInterval **shardIntervalCache, - int shardCount, FunctionCallInfoData *compareFunction, + int shardCount, FunctionCallInfo compareFunction, bool includeMin); static int LowerShardBoundary(Datum partitionColumnValue, ShardInterval **shardIntervalCache, - int shardCount, FunctionCallInfoData *compareFunction, + int shardCount, FunctionCallInfo compareFunction, bool includeMax); @@ -261,7 +273,8 @@ PruneShards(Oid relationId, Index rangeTableId, List *whereClauseList, if (cacheEntry->shardIntervalCompareFunction) { /* initiate function call info once (allows comparators to cache metadata) */ - InitFunctionCallInfoData(context.compareIntervalFunctionCall, + InitFunctionCallInfoData(*(FunctionCallInfo) & + context.compareIntervalFunctionCall, cacheEntry->shardIntervalCompareFunction, 2, DEFAULT_COLLATION_OID, NULL, NULL); } @@ -274,7 +287,8 @@ PruneShards(Oid relationId, Index rangeTableId, List *whereClauseList, if (cacheEntry->shardColumnCompareFunction) { /* initiate function call info once (allows comparators to cache metadata) */ - InitFunctionCallInfoData(context.compareValueFunctionCall, + InitFunctionCallInfoData(*(FunctionCallInfo) & + context.compareValueFunctionCall, cacheEntry->shardColumnCompareFunction, 2, DEFAULT_COLLATION_OID, NULL, NULL); } @@ -753,7 +767,8 @@ AddPartitionKeyRestrictionToInstance(ClauseWalkerContext *context, OpExpr *opCla case BTLessStrategyNumber: { if (!prune->lessConsts || - PerformValueCompare(&context->compareValueFunctionCall, + PerformValueCompare((FunctionCallInfo) & + context->compareValueFunctionCall, constantClause->constvalue, prune->lessConsts->constvalue) < 0) { @@ -766,7 +781,8 @@ AddPartitionKeyRestrictionToInstance(ClauseWalkerContext *context, OpExpr *opCla case BTLessEqualStrategyNumber: { if (!prune->lessEqualConsts || - PerformValueCompare(&context->compareValueFunctionCall, + PerformValueCompare((FunctionCallInfo) & + context->compareValueFunctionCall, constantClause->constvalue, prune->lessEqualConsts->constvalue) < 0) { @@ -782,7 +798,8 @@ AddPartitionKeyRestrictionToInstance(ClauseWalkerContext *context, OpExpr *opCla { prune->equalConsts = constantClause; } - else if (PerformValueCompare(&context->compareValueFunctionCall, + else if (PerformValueCompare((FunctionCallInfo) & + context->compareValueFunctionCall, constantClause->constvalue, prune->equalConsts->constvalue) != 0) { @@ -796,7 +813,8 @@ AddPartitionKeyRestrictionToInstance(ClauseWalkerContext *context, OpExpr *opCla case BTGreaterEqualStrategyNumber: { if (!prune->greaterEqualConsts || - PerformValueCompare(&context->compareValueFunctionCall, + PerformValueCompare((FunctionCallInfo) & + context->compareValueFunctionCall, constantClause->constvalue, prune->greaterEqualConsts->constvalue) > 0 ) @@ -810,7 +828,8 @@ AddPartitionKeyRestrictionToInstance(ClauseWalkerContext *context, OpExpr *opCla case BTGreaterStrategyNumber: { if (!prune->greaterConsts || - PerformValueCompare(&context->compareValueFunctionCall, + PerformValueCompare((FunctionCallInfo) & + context->compareValueFunctionCall, constantClause->constvalue, prune->greaterConsts->constvalue) > 0) { @@ -1133,7 +1152,7 @@ PruneOne(DistTableCacheEntry *cacheEntry, ClauseWalkerContext *context, * unexpected NULL returns. */ static int -PerformCompare(FunctionCallInfoData *compareFunctionCall) +PerformCompare(FunctionCallInfo compareFunctionCall) { Datum result = FunctionCallInvoke(compareFunctionCall); @@ -1151,12 +1170,10 @@ PerformCompare(FunctionCallInfoData *compareFunctionCall) * NULL returns. */ static int -PerformValueCompare(FunctionCallInfoData *compareFunctionCall, Datum a, Datum b) +PerformValueCompare(FunctionCallInfo compareFunctionCall, Datum a, Datum b) { - compareFunctionCall->arg[0] = a; - compareFunctionCall->argnull[0] = false; - compareFunctionCall->arg[1] = b; - compareFunctionCall->argnull[1] = false; + fcSetArg(compareFunctionCall, 0, a); + fcSetArg(compareFunctionCall, 1, b); return PerformCompare(compareFunctionCall); } @@ -1168,7 +1185,7 @@ PerformValueCompare(FunctionCallInfoData *compareFunctionCall, Datum a, Datum b) */ static int LowerShardBoundary(Datum partitionColumnValue, ShardInterval **shardIntervalCache, - int shardCount, FunctionCallInfoData *compareFunction, bool includeMax) + int shardCount, FunctionCallInfo compareFunction, bool includeMax) { int lowerBoundIndex = 0; int upperBoundIndex = shardCount; @@ -1176,8 +1193,7 @@ LowerShardBoundary(Datum partitionColumnValue, ShardInterval **shardIntervalCach Assert(shardCount != 0); /* setup partitionColumnValue argument once */ - compareFunction->arg[0] = partitionColumnValue; - compareFunction->argnull[0] = false; + fcSetArg(compareFunction, 0, partitionColumnValue); while (lowerBoundIndex < upperBoundIndex) { @@ -1186,8 +1202,7 @@ LowerShardBoundary(Datum partitionColumnValue, ShardInterval **shardIntervalCach int minValueComparison = 0; /* setup minValue as argument */ - compareFunction->arg[1] = shardIntervalCache[middleIndex]->minValue; - compareFunction->argnull[1] = false; + fcSetArg(compareFunction, 1, shardIntervalCache[middleIndex]->minValue); /* execute cmp(partitionValue, lowerBound) */ minValueComparison = PerformCompare(compareFunction); @@ -1201,8 +1216,7 @@ LowerShardBoundary(Datum partitionColumnValue, ShardInterval **shardIntervalCach } /* setup maxValue as argument */ - compareFunction->arg[1] = shardIntervalCache[middleIndex]->maxValue; - compareFunction->argnull[1] = false; + fcSetArg(compareFunction, 1, shardIntervalCache[middleIndex]->maxValue); /* execute cmp(partitionValue, upperBound) */ maxValueComparison = PerformCompare(compareFunction); @@ -1249,7 +1263,7 @@ LowerShardBoundary(Datum partitionColumnValue, ShardInterval **shardIntervalCach */ static int UpperShardBoundary(Datum partitionColumnValue, ShardInterval **shardIntervalCache, - int shardCount, FunctionCallInfoData *compareFunction, bool includeMin) + int shardCount, FunctionCallInfo compareFunction, bool includeMin) { int lowerBoundIndex = 0; int upperBoundIndex = shardCount; @@ -1257,8 +1271,7 @@ UpperShardBoundary(Datum partitionColumnValue, ShardInterval **shardIntervalCach Assert(shardCount != 0); /* setup partitionColumnValue argument once */ - compareFunction->arg[0] = partitionColumnValue; - compareFunction->argnull[0] = false; + fcSetArg(compareFunction, 0, partitionColumnValue); while (lowerBoundIndex < upperBoundIndex) { @@ -1267,8 +1280,7 @@ UpperShardBoundary(Datum partitionColumnValue, ShardInterval **shardIntervalCach int minValueComparison = 0; /* setup minValue as argument */ - compareFunction->arg[1] = shardIntervalCache[middleIndex]->minValue; - compareFunction->argnull[1] = false; + fcSetArg(compareFunction, 1, shardIntervalCache[middleIndex]->minValue); /* execute cmp(partitionValue, lowerBound) */ minValueComparison = PerformCompare(compareFunction); @@ -1283,8 +1295,7 @@ UpperShardBoundary(Datum partitionColumnValue, ShardInterval **shardIntervalCach } /* setup maxValue as argument */ - compareFunction->arg[1] = shardIntervalCache[middleIndex]->maxValue; - compareFunction->argnull[1] = false; + fcSetArg(compareFunction, 1, shardIntervalCache[middleIndex]->maxValue); /* execute cmp(partitionValue, upperBound) */ maxValueComparison = PerformCompare(compareFunction); @@ -1345,7 +1356,8 @@ PruneWithBoundaries(DistTableCacheEntry *cacheEntry, ClauseWalkerContext *contex int lowerBoundIdx = -1; int upperBoundIdx = -1; int curIdx = 0; - FunctionCallInfo compareFunctionCall = &context->compareIntervalFunctionCall; + FunctionCallInfo compareFunctionCall = (FunctionCallInfo) & + context->compareIntervalFunctionCall; if (prune->greaterEqualConsts) { @@ -1476,7 +1488,8 @@ ExhaustivePruneOne(ShardInterval *curInterval, ClauseWalkerContext *context, PruningInstance *prune) { - FunctionCallInfo compareFunctionCall = &context->compareIntervalFunctionCall; + FunctionCallInfo compareFunctionCall = (FunctionCallInfo) & + context->compareIntervalFunctionCall; Datum compareWith = 0; /* NULL boundaries can't be compared to */ diff --git a/src/backend/distributed/utils/function_utils.c b/src/backend/distributed/utils/function_utils.c index 862604b02..6f8f5b0a4 100644 --- a/src/backend/distributed/utils/function_utils.c +++ b/src/backend/distributed/utils/function_utils.c @@ -11,11 +11,10 @@ #include "catalog/namespace.h" #include "distributed/function_utils.h" +#include "distributed/version_compat.h" #include "executor/executor.h" #include "utils/builtins.h" -#if (PG_VERSION_NUM >= 100000) #include "utils/regproc.h" -#endif /* * FunctionOid searches for a function that has the given name and the given @@ -83,7 +82,7 @@ FunctionOidExtended(const char *schemaName, const char *functionName, int argume ReturnSetInfo * FunctionCallGetTupleStore1(PGFunction function, Oid functionId, Datum argument) { - FunctionCallInfoData fcinfo; + LOCAL_FCINFO(fcinfo, 1); FmgrInfo flinfo; ReturnSetInfo *rsinfo = makeNode(ReturnSetInfo); EState *estate = CreateExecutorState(); @@ -91,12 +90,11 @@ FunctionCallGetTupleStore1(PGFunction function, Oid functionId, Datum argument) rsinfo->allowedModes = SFRM_Materialize; fmgr_info(functionId, &flinfo); - InitFunctionCallInfoData(fcinfo, &flinfo, 1, InvalidOid, NULL, (Node *) rsinfo); + InitFunctionCallInfoData(*fcinfo, &flinfo, 1, InvalidOid, NULL, (Node *) rsinfo); - fcinfo.arg[0] = argument; - fcinfo.argnull[0] = false; + fcSetArg(fcinfo, 0, argument); - (*function)(&fcinfo); + (*function)(fcinfo); return rsinfo; } diff --git a/src/backend/distributed/utils/metadata_cache.c b/src/backend/distributed/utils/metadata_cache.c index fd5fcb876..492df0e7e 100644 --- a/src/backend/distributed/utils/metadata_cache.c +++ b/src/backend/distributed/utils/metadata_cache.c @@ -20,6 +20,7 @@ #include "access/sysattr.h" #include "catalog/indexing.h" #include "catalog/pg_am.h" +#include "catalog/pg_enum.h" #include "catalog/pg_extension.h" #include "catalog/pg_namespace.h" #include "catalog/pg_type.h" @@ -43,6 +44,7 @@ #include "distributed/pg_dist_placement.h" #include "distributed/shared_library_init.h" #include "distributed/shardinterval_utils.h" +#include "distributed/version_compat.h" #include "distributed/worker_manager.h" #include "distributed/worker_protocol.h" #include "executor/executor.h" @@ -1259,8 +1261,6 @@ static ShardInterval ** SortShardIntervalArray(ShardInterval **shardIntervalArray, int shardCount, FmgrInfo *shardIntervalSortCompareFunction) { - ShardInterval **sortedShardIntervalArray = NULL; - /* short cut if there are no shard intervals in the array */ if (shardCount == 0) { @@ -1272,9 +1272,7 @@ SortShardIntervalArray(ShardInterval **shardIntervalArray, int shardCount, (qsort_arg_comparator) CompareShardIntervals, (void *) shardIntervalSortCompareFunction); - sortedShardIntervalArray = shardIntervalArray; - - return sortedShardIntervalArray; + return shardIntervalArray; } @@ -1624,9 +1622,8 @@ AvailableExtensionVersion(void) { ReturnSetInfo *extensionsResultSet = NULL; TupleTableSlot *tupleTableSlot = NULL; - FunctionCallInfoData *fcinfo = NULL; - FmgrInfo *flinfo = NULL; - int argumentCount = 0; + LOCAL_FCINFO(fcinfo, 0); + FmgrInfo flinfo; EState *estate = NULL; bool hasTuple = false; @@ -1641,17 +1638,15 @@ AvailableExtensionVersion(void) extensionsResultSet->econtext = GetPerTupleExprContext(estate); extensionsResultSet->allowedModes = SFRM_Materialize; - fcinfo = palloc0(sizeof(FunctionCallInfoData)); - flinfo = palloc0(sizeof(FmgrInfo)); - - fmgr_info(F_PG_AVAILABLE_EXTENSIONS, flinfo); - InitFunctionCallInfoData(*fcinfo, flinfo, argumentCount, InvalidOid, NULL, + fmgr_info(F_PG_AVAILABLE_EXTENSIONS, &flinfo); + InitFunctionCallInfoData(*fcinfo, &flinfo, 0, InvalidOid, NULL, (Node *) extensionsResultSet); /* pg_available_extensions returns result set containing all available extensions */ (*pg_available_extensions)(fcinfo); - tupleTableSlot = MakeSingleTupleTableSlot(extensionsResultSet->setDesc); + tupleTableSlot = MakeSingleTupleTableSlotCompat(extensionsResultSet->setDesc, + &TTSOpsMinimalTuple); hasTuple = tuplestore_gettupleslot(extensionsResultSet->setResult, goForward, doCopy, tupleTableSlot); while (hasTuple) @@ -1992,9 +1987,10 @@ CitusCopyFormatTypeId(void) if (MetadataCache.copyFormatTypeId == InvalidOid) { char *typeName = "citus_copy_format"; - MetadataCache.copyFormatTypeId = GetSysCacheOid2(TYPENAMENSP, - PointerGetDatum(typeName), - PG_CATALOG_NAMESPACE); + MetadataCache.copyFormatTypeId = GetSysCacheOid2Compat(TYPENAMENSP, + Anum_pg_enum_oid, + PointerGetDatum(typeName), + PG_CATALOG_NAMESPACE); } return MetadataCache.copyFormatTypeId; @@ -2256,7 +2252,11 @@ LookupNodeRoleTypeOid() return InvalidOid; } +#if PG_VERSION_NUM >= 120000 + nodeRoleTypId = ((Form_pg_type) GETSTRUCT(tup))->oid; +#else nodeRoleTypId = HeapTupleGetOid(tup); +#endif ReleaseSysCache(tup); return nodeRoleTypId; diff --git a/src/include/distributed/version_compat.h b/src/include/distributed/version_compat.h index 7dbe2f137..d4c6f7854 100644 --- a/src/include/distributed/version_compat.h +++ b/src/include/distributed/version_compat.h @@ -257,6 +257,11 @@ RangeVarGetRelidInternal(const RangeVar *relation, LOCKMODE lockmode, uint32 fla #define GetSysCacheOid3Compat GetSysCacheOid3 #define GetSysCacheOid4Compat GetSysCacheOid4 +#define fcSetArg(fc, n, argval) \ + (((fc)->args[n].isnull = false), ((fc)->args[n].value = (argval))) +#define fcSetArgNull(fc, n) \ + (((fc)->args[n].isnull = true), ((fc)->args[n].value = (Datum) 0)) + typedef struct { File fd; @@ -319,6 +324,15 @@ FileCompatFromFileStart(File fileDesc) #define GetSysCacheOid4Compat(cacheId, oidcol, key1, key2, key3, key4) \ GetSysCacheOid4(cacheId, key1, key2, key3, key4) +#define LOCAL_FCINFO(name, nargs) \ + FunctionCallInfoData name ## data; \ + FunctionCallInfoData *name = &name ## data + +#define fcSetArg(fc, n, value) \ + (((fc)->argnull[n] = false), ((fc)->arg[n] = (value))) +#define fcSetArgNull(fc, n) \ + (((fc)->argnull[n] = true), ((fc)->arg[n] = (Datum) 0)) + typedef struct { File fd; From e84fcc0b123e6f9cce932db14e99ce66a460de2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philip=20Dub=C3=A9?= Date: Wed, 7 Aug 2019 18:18:54 +0000 Subject: [PATCH 10/12] Modify tests to be consistent between versions Normalize UNION to prevent optimization Remove WITH OIDS Sort ddl events client_min_messages no longer accepts FATAL --- src/test/regress/bin/normalize.sed | 17 ++ src/test/regress/bin/normalized_tests.lst | 51 +++-- .../regress/expected/failure_multi_dml.out | 2 +- .../regress/expected/failure_multi_dml_9.out | 2 +- .../foreign_key_restriction_enforcement.out | 214 +----------------- .../foreign_key_restriction_enforcement_0.out | 212 +---------------- .../regress/expected/multi_create_shards.out | 8 +- .../expected/multi_master_protocol.out | 6 +- .../regress/expected/multi_metadata_sync.out | 92 ++++---- .../regress/expected/multi_partitioning.out | 13 +- .../regress/expected/multi_partitioning_0.out | 13 +- .../regress/expected/multi_partitioning_1.out | 15 +- .../expected/multi_reference_table.out | 4 - ...lti_subquery_in_where_reference_clause.out | 4 +- .../expected/multi_utility_statements.out | 5 +- src/test/regress/expected/sql_procedure.out | 14 +- src/test/regress/expected/sql_procedure_0.out | 2 + .../regress/expected/subquery_in_where.out | 13 +- src/test/regress/sql/failure_multi_dml.sql | 2 +- .../foreign_key_restriction_enforcement.sql | 8 + src/test/regress/sql/multi_create_shards.sql | 5 - .../regress/sql/multi_master_protocol.sql | 2 +- src/test/regress/sql/multi_metadata_sync.sql | 12 +- src/test/regress/sql/multi_partitioning.sql | 7 +- .../regress/sql/multi_reference_table.sql | 3 - ...lti_subquery_in_where_reference_clause.sql | 2 +- .../regress/sql/multi_utility_statements.sql | 5 +- src/test/regress/sql/sql_procedure.sql | 2 + src/test/regress/sql/subquery_in_where.sql | 55 +++-- 29 files changed, 197 insertions(+), 593 deletions(-) diff --git a/src/test/regress/bin/normalize.sed b/src/test/regress/bin/normalize.sed index 929eb548f..e74229049 100644 --- a/src/test/regress/bin/normalize.sed +++ b/src/test/regress/bin/normalize.sed @@ -12,6 +12,10 @@ s/ port=[0-9]+ / port=xxxxx /g s/placement [0-9]+/placement xxxxx/g s/shard [0-9]+/shard xxxxx/g s/assigned task [0-9]+ to node/assigned task to node/ +s/node group [12] (but|does)/node group \1/ + +# Differing names can have differing table column widths +s/(-+\|)+-+/---/g # In foreign_key_to_reference_table, normalize shard table names, etc in # the generated plan @@ -69,3 +73,16 @@ s/(job_[0-9]+\/task_[0-9]+\/p_[0-9]+\.)[0-9]+/\1xxxx/g # isolation_ref2ref_foreign_keys s/"(ref_table_[0-9]_|ref_table_[0-9]_value_fkey_)[0-9]+"/"\1xxxxxxx"/g + +# Line info varies between versions +/^LINE [0-9]+:.*$/d +/^ *\^$/d + +# pg12 changes +s/Partitioned table "/Table "/g +s/\) TABLESPACE pg_default$/\)/g +s/invalid input syntax for type /invalid input syntax for /g +s/_id_ref_id_fkey/_id_fkey/g +s/_ref_id_id_fkey_/_ref_id_fkey_/g +s/fk_test_2_col1_col2_fkey/fk_test_2_col1_fkey/g +s/_id_other_column_ref_fkey/_id_fkey/g diff --git a/src/test/regress/bin/normalized_tests.lst b/src/test/regress/bin/normalized_tests.lst index 39b6eff34..a893e270a 100644 --- a/src/test/regress/bin/normalized_tests.lst +++ b/src/test/regress/bin/normalized_tests.lst @@ -1,41 +1,44 @@ # List of tests whose output we want to normalize, one per line -multi_alter_table_add_constraints -multi_alter_table_statements -foreign_key_to_reference_table +custom_aggregate_support failure_copy_on_hash -failure_savepoints -foreign_key_restriction_enforcement failure_real_time_select +failure_savepoints failure_vacuum +foreign_key_restriction_enforcement +foreign_key_to_reference_table +intermediate_results isolation_citus_dist_activity isolation_ref2ref_foreign_keys +multi_alter_table_add_constraints +multi_alter_table_statements +multi_copy +multi_create_table_constraints +multi_explain +multi_foreign_key +multi_generate_ddl_commands +multi_having_pushdown multi_insert_select multi_insert_select_conflict -multi_multiuser -multi_name_lengths -multi_partition_pruning -multi_subtransactions -multi_modifying_xacts -multi_insert_select -sql_procedure -multi_reference_table -multi_create_table_constraints - -# the following tests' output are -# normalized for EXPLAIN outputs -# where the executor name is wiped out multi_join_order_tpch_small multi_join_pruning +multi_master_protocol +multi_metadata_sync +multi_modifying_xacts +multi_multiuser +multi_mx_explain +multi_name_lengths +multi_null_minmax_value_pruning multi_orderby_limit_pushdown +multi_partitioning +multi_partitioning_utils multi_partition_pruning +multi_reference_table multi_select_distinct multi_subquery_window_functions +multi_subtransactions multi_task_assignment_policy multi_view -multi_explain -multi_null_minmax_value_pruning +sql_procedure window_functions -multi_having_pushdown -multi_partitioning -multi_mx_explain -custom_aggregate_support +worker_check_invalid_arguments + diff --git a/src/test/regress/expected/failure_multi_dml.out b/src/test/regress/expected/failure_multi_dml.out index 380fab673..734412fa9 100644 --- a/src/test/regress/expected/failure_multi_dml.out +++ b/src/test/regress/expected/failure_multi_dml.out @@ -305,7 +305,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT").kill()'); -- hide the error message (it has the PID)... -- we'll test for the txn side-effects to ensure it didn't run -SET client_min_messages TO FATAL; +SET client_min_messages TO ERROR; BEGIN; DELETE FROM dml_test WHERE id = 1; DELETE FROM dml_test WHERE id = 2; diff --git a/src/test/regress/expected/failure_multi_dml_9.out b/src/test/regress/expected/failure_multi_dml_9.out index 6d8e928eb..0552e8e80 100644 --- a/src/test/regress/expected/failure_multi_dml_9.out +++ b/src/test/regress/expected/failure_multi_dml_9.out @@ -305,7 +305,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT").kill()'); -- hide the error message (it has the PID)... -- we'll test for the txn side-effects to ensure it didn't run -SET client_min_messages TO FATAL; +SET client_min_messages TO ERROR; BEGIN; DELETE FROM dml_test WHERE id = 1; DELETE FROM dml_test WHERE id = 2; diff --git a/src/test/regress/expected/foreign_key_restriction_enforcement.out b/src/test/regress/expected/foreign_key_restriction_enforcement.out index ee0b3d889..b527f5f16 100644 --- a/src/test/regress/expected/foreign_key_restriction_enforcement.out +++ b/src/test/regress/expected/foreign_key_restriction_enforcement.out @@ -532,7 +532,7 @@ BEGIN; DEBUG: switching to sequential query execution mode DETAIL: Reference relation "transitive_reference_table" is modified, which might lead to data inconsistencies or distributed deadlocks via parallel accesses to hash distributed relations due to foreign keys. Any parallel modification to those hash distributed relations in the same transaction can only be executed in sequential query execution mode COPY on_update_fkey_table FROM STDIN WITH CSV; -ERROR: insert or update on table "on_update_fkey_table_2380005" violates foreign key constraint "fkey_2380005" +ERROR: insert or update on table "on_update_fkey_table_2380004" violates foreign key constraint "fkey_2380004" DETAIL: Key (value_1)=(101) is not present in table "reference_table_2380001". ROLLBACK; -- case 2.8: UPDATE to a reference table is followed by TRUNCATE @@ -817,6 +817,8 @@ ERROR: cannot execute DDL on reference relation "transitive_reference_table" be HINT: Try re-running the transaction with "SET LOCAL citus.multi_shard_modify_mode TO 'sequential';" ROLLBACK; -- case 4.5: SELECT to a dist table is follwed by a TRUNCATE +\set VERBOSITY terse +SET client_min_messages to LOG; BEGIN; SELECT count(*) FROM on_update_fkey_table WHERE value_1 = 99; count @@ -825,11 +827,8 @@ BEGIN; (1 row) TRUNCATE reference_table CASCADE; -DEBUG: switching to sequential query execution mode -DETAIL: Reference relation "reference_table" is modified, which might lead to data inconsistencies or distributed deadlocks via parallel accesses to hash distributed relations due to foreign keys. Any parallel modification to those hash distributed relations in the same transaction can only be executed in sequential query execution mode NOTICE: truncate cascades to table "on_update_fkey_table" ERROR: cannot execute DDL on reference relation "reference_table" because there was a parallel SELECT access to distributed relation "on_update_fkey_table" in the same transaction -HINT: Try re-running the transaction with "SET LOCAL citus.multi_shard_modify_mode TO 'sequential';" ROLLBACK; BEGIN; SELECT count(*) FROM on_update_fkey_table WHERE value_1 = 99; @@ -839,12 +838,9 @@ BEGIN; (1 row) TRUNCATE transitive_reference_table CASCADE; -DEBUG: switching to sequential query execution mode -DETAIL: Reference relation "transitive_reference_table" is modified, which might lead to data inconsistencies or distributed deadlocks via parallel accesses to hash distributed relations due to foreign keys. Any parallel modification to those hash distributed relations in the same transaction can only be executed in sequential query execution mode NOTICE: truncate cascades to table "reference_table" NOTICE: truncate cascades to table "on_update_fkey_table" ERROR: cannot execute DDL on reference relation "transitive_reference_table" because there was a parallel SELECT access to distributed relation "on_update_fkey_table" in the same transaction -HINT: Try re-running the transaction with "SET LOCAL citus.multi_shard_modify_mode TO 'sequential';" ROLLBACK; -- case 4.6: Router SELECT to a dist table is followed by a TRUNCATE BEGIN; @@ -855,19 +851,7 @@ BEGIN; (1 row) TRUNCATE reference_table CASCADE; -DEBUG: switching to sequential query execution mode -DETAIL: Reference relation "reference_table" is modified, which might lead to data inconsistencies or distributed deadlocks via parallel accesses to hash distributed relations due to foreign keys. Any parallel modification to those hash distributed relations in the same transaction can only be executed in sequential query execution mode NOTICE: truncate cascades to table "on_update_fkey_table" -DEBUG: truncate cascades to table "on_update_fkey_table_2380003" -DETAIL: NOTICE from localhost:57638 -DEBUG: truncate cascades to table "on_update_fkey_table_2380005" -DETAIL: NOTICE from localhost:57638 -DEBUG: truncate cascades to table "on_update_fkey_table_2380002" -DETAIL: NOTICE from localhost:57637 -DEBUG: truncate cascades to table "on_update_fkey_table_2380004" -DETAIL: NOTICE from localhost:57637 -DEBUG: building index "reference_table_pkey" on table "reference_table" serially -DEBUG: building index "on_update_fkey_table_pkey" on table "on_update_fkey_table" serially ROLLBACK; BEGIN; SELECT count(*) FROM on_update_fkey_table WHERE id = 9; @@ -877,34 +861,11 @@ BEGIN; (1 row) TRUNCATE transitive_reference_table CASCADE; -DEBUG: switching to sequential query execution mode -DETAIL: Reference relation "transitive_reference_table" is modified, which might lead to data inconsistencies or distributed deadlocks via parallel accesses to hash distributed relations due to foreign keys. Any parallel modification to those hash distributed relations in the same transaction can only be executed in sequential query execution mode NOTICE: truncate cascades to table "reference_table" NOTICE: truncate cascades to table "on_update_fkey_table" -DEBUG: truncate cascades to table "reference_table_2380001" -DETAIL: NOTICE from localhost:57638 -DEBUG: truncate cascades to table "on_update_fkey_table_2380003" -DETAIL: NOTICE from localhost:57638 -DEBUG: truncate cascades to table "on_update_fkey_table_2380005" -DETAIL: NOTICE from localhost:57638 -DEBUG: truncate cascades to table "reference_table_2380001" -DETAIL: NOTICE from localhost:57637 -DEBUG: truncate cascades to table "on_update_fkey_table_2380002" -DETAIL: NOTICE from localhost:57637 -DEBUG: truncate cascades to table "on_update_fkey_table_2380004" -DETAIL: NOTICE from localhost:57637 -DEBUG: truncate cascades to table "on_update_fkey_table_2380002" -DETAIL: NOTICE from localhost:57637 -DEBUG: truncate cascades to table "on_update_fkey_table_2380003" -DETAIL: NOTICE from localhost:57638 -DEBUG: truncate cascades to table "on_update_fkey_table_2380004" -DETAIL: NOTICE from localhost:57637 -DEBUG: truncate cascades to table "on_update_fkey_table_2380005" -DETAIL: NOTICE from localhost:57638 -DEBUG: building index "transitive_reference_table_pkey" on table "transitive_reference_table" serially -DEBUG: building index "reference_table_pkey" on table "reference_table" serially -DEBUG: building index "on_update_fkey_table_pkey" on table "on_update_fkey_table" serially ROLLBACK; +RESET client_min_messages; +\set VERBOSITY default -- case 5.1: Parallel UPDATE on distributed table follow by a SELECT BEGIN; UPDATE on_update_fkey_table SET value_1 = 16 WHERE value_1 = 15; @@ -954,18 +915,12 @@ ROLLBACK; BEGIN; UPDATE on_update_fkey_table SET value_1 = 16 WHERE value_1 = 15; ALTER TABLE reference_table ALTER COLUMN id SET DATA TYPE smallint; -DEBUG: rewriting table "reference_table" -DEBUG: building index "reference_table_pkey" on table "reference_table" serially -DEBUG: validating foreign key constraint "fkey" ERROR: cannot execute DDL on reference relation "reference_table" because there was a parallel DML access to distributed relation "on_update_fkey_table" in the same transaction HINT: Try re-running the transaction with "SET LOCAL citus.multi_shard_modify_mode TO 'sequential';" ROLLBACK; BEGIN; UPDATE on_update_fkey_table SET value_1 = 16 WHERE value_1 = 15; ALTER TABLE transitive_reference_table ALTER COLUMN id SET DATA TYPE smallint; -DEBUG: rewriting table "transitive_reference_table" -DEBUG: building index "transitive_reference_table_pkey" on table "transitive_reference_table" serially -DEBUG: validating foreign key constraint "fkey" ERROR: cannot execute DDL on reference relation "transitive_reference_table" because there was a parallel DML access to distributed relation "on_update_fkey_table" in the same transaction HINT: Try re-running the transaction with "SET LOCAL citus.multi_shard_modify_mode TO 'sequential';" ROLLBACK; @@ -985,16 +940,10 @@ ROLLBACK; -- case 6:2: Related parallel DDL on distributed table followed by SELECT on ref. table BEGIN; ALTER TABLE on_update_fkey_table ALTER COLUMN value_1 SET DATA TYPE smallint; -DEBUG: rewriting table "on_update_fkey_table" -DEBUG: building index "on_update_fkey_table_pkey" on table "on_update_fkey_table" serially -DEBUG: validating foreign key constraint "fkey" UPDATE reference_table SET id = 160 WHERE id = 15; ROLLBACK; BEGIN; ALTER TABLE on_update_fkey_table ALTER COLUMN value_1 SET DATA TYPE smallint; -DEBUG: rewriting table "on_update_fkey_table" -DEBUG: building index "on_update_fkey_table_pkey" on table "on_update_fkey_table" serially -DEBUG: validating foreign key constraint "fkey" UPDATE transitive_reference_table SET id = 160 WHERE id = 15; ROLLBACK; -- case 6:3: Unrelated parallel DDL on distributed table followed by UPDATE on ref. table @@ -1090,26 +1039,14 @@ ERROR: current transaction is aborted, commands ignored until end of transactio ROLLBACK; BEGIN; CREATE TABLE test_table_1(id int PRIMARY KEY); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_1_pkey" for table "test_table_1" -DEBUG: building index "test_table_1_pkey" on table "test_table_1" serially SELECT create_reference_table('test_table_1'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 create_reference_table ------------------------ (1 row) CREATE TABLE test_table_2(id int PRIMARY KEY, value_1 int, FOREIGN KEY(value_1) REFERENCES test_table_1(id)); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_2_pkey" for table "test_table_2" -DEBUG: building index "test_table_2_pkey" on table "test_table_2" serially SELECT create_distributed_table('test_table_2', 'id'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 create_distributed_table -------------------------- @@ -1123,39 +1060,21 @@ ROLLBACK; -- already executed a parallel query BEGIN; CREATE TABLE test_table_1(id int PRIMARY KEY); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_1_pkey" for table "test_table_1" -DEBUG: building index "test_table_1_pkey" on table "test_table_1" serially SELECT create_reference_table('test_table_1'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 create_reference_table ------------------------ (1 row) CREATE TABLE tt4(id int PRIMARY KEY, value_1 int, FOREIGN KEY(id) REFERENCES tt4(id)); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "tt4_pkey" for table "tt4" -DEBUG: building index "tt4_pkey" on table "tt4" serially SELECT create_distributed_table('tt4', 'id'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 create_distributed_table -------------------------- (1 row) CREATE TABLE test_table_2(id int PRIMARY KEY, value_1 int, FOREIGN KEY(value_1) REFERENCES test_table_1(id), FOREIGN KEY(id) REFERENCES tt4(id)); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_2_pkey" for table "test_table_2" -DEBUG: building index "test_table_2_pkey" on table "test_table_2" serially SELECT create_distributed_table('test_table_2', 'id'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 ERROR: cannot distribute relation "test_table_2" in this transaction because it has a foreign key to a reference table DETAIL: If a hash distributed table has a foreign key to a reference table, it has to be created in sequential mode before any parallel commands have been executed in the same transaction HINT: Try re-running the transaction with "SET LOCAL citus.multi_shard_modify_mode TO 'sequential';" @@ -1170,39 +1089,21 @@ ROLLBACK; BEGIN; SET LOCAL citus.multi_shard_modify_mode TO 'sequential'; CREATE TABLE test_table_1(id int PRIMARY KEY); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_1_pkey" for table "test_table_1" -DEBUG: building index "test_table_1_pkey" on table "test_table_1" serially SELECT create_reference_table('test_table_1'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 create_reference_table ------------------------ (1 row) CREATE TABLE tt4(id int PRIMARY KEY, value_1 int, FOREIGN KEY(id) REFERENCES tt4(id)); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "tt4_pkey" for table "tt4" -DEBUG: building index "tt4_pkey" on table "tt4" serially SELECT create_distributed_table('tt4', 'id'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 create_distributed_table -------------------------- (1 row) CREATE TABLE test_table_2(id int PRIMARY KEY, value_1 int, FOREIGN KEY(value_1) REFERENCES test_table_1(id), FOREIGN KEY(id) REFERENCES tt4(id)); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_2_pkey" for table "test_table_2" -DEBUG: building index "test_table_2_pkey" on table "test_table_2" serially SELECT create_distributed_table('test_table_2', 'id'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 create_distributed_table -------------------------- @@ -1218,26 +1119,14 @@ ROLLBACK; BEGIN; CREATE TABLE test_table_1(id int PRIMARY KEY); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_1_pkey" for table "test_table_1" -DEBUG: building index "test_table_1_pkey" on table "test_table_1" serially SELECT create_reference_table('test_table_1'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 create_reference_table ------------------------ (1 row) CREATE TABLE test_table_2(id int PRIMARY KEY, value_1 int); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_2_pkey" for table "test_table_2" -DEBUG: building index "test_table_2_pkey" on table "test_table_2" serially SELECT create_distributed_table('test_table_2', 'id'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 create_distributed_table -------------------------- @@ -1258,26 +1147,14 @@ BEGIN; SET LOCAL citus.multi_shard_modify_mode TO 'sequential'; CREATE TABLE test_table_1(id int PRIMARY KEY); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_1_pkey" for table "test_table_1" -DEBUG: building index "test_table_1_pkey" on table "test_table_1" serially SELECT create_reference_table('test_table_1'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 create_reference_table ------------------------ (1 row) CREATE TABLE test_table_2(id int PRIMARY KEY, value_1 int); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_2_pkey" for table "test_table_2" -DEBUG: building index "test_table_2_pkey" on table "test_table_2" serially SELECT create_distributed_table('test_table_2', 'id'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 create_distributed_table -------------------------- @@ -1293,26 +1170,14 @@ COMMIT; -- changed BEGIN; CREATE TABLE test_table_2(id int PRIMARY KEY, value_1 int); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_2_pkey" for table "test_table_2" -DEBUG: building index "test_table_2_pkey" on table "test_table_2" serially SELECT create_distributed_table('test_table_2', 'id'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 create_distributed_table -------------------------- (1 row) CREATE TABLE test_table_1(id int PRIMARY KEY); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_1_pkey" for table "test_table_1" -DEBUG: building index "test_table_1_pkey" on table "test_table_1" serially SELECT create_reference_table('test_table_1'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 create_reference_table ------------------------ @@ -1332,26 +1197,14 @@ ROLLBACK; BEGIN; SET LOCAL citus.multi_shard_modify_mode TO 'sequential'; CREATE TABLE test_table_2(id int PRIMARY KEY, value_1 int); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_2_pkey" for table "test_table_2" -DEBUG: building index "test_table_2_pkey" on table "test_table_2" serially SELECT create_distributed_table('test_table_2', 'id'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 create_distributed_table -------------------------- (1 row) CREATE TABLE test_table_1(id int PRIMARY KEY); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_1_pkey" for table "test_table_1" -DEBUG: building index "test_table_1_pkey" on table "test_table_1" serially SELECT create_reference_table('test_table_1'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 create_reference_table ------------------------ @@ -1374,13 +1227,7 @@ BEGIN; SET LOCAL citus.multi_shard_modify_mode TO 'sequential'; CREATE TABLE test_table_2(id int PRIMARY KEY, value_1 int); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_2_pkey" for table "test_table_2" -DEBUG: building index "test_table_2_pkey" on table "test_table_2" serially SELECT create_distributed_table('test_table_2', 'id'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 ERROR: cannot distribute relation "test_table_2" in this transaction because it has a foreign key to a reference table DETAIL: If a hash distributed table has a foreign key to a reference table, it has to be created in sequential mode before any parallel commands have been executed in the same transaction HINT: Try re-running the transaction with "SET LOCAL citus.multi_shard_modify_mode TO 'sequential';" @@ -1401,32 +1248,17 @@ ROLLBACK; BEGIN; CREATE TABLE test_table_1(id int PRIMARY KEY); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_1_pkey" for table "test_table_1" -DEBUG: building index "test_table_1_pkey" on table "test_table_1" serially INSERT INTO test_table_1 SELECT i FROM generate_series(0,100) i; CREATE TABLE test_table_2(id int PRIMARY KEY, value_1 int, FOREIGN KEY(value_1) REFERENCES test_table_1(id)); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_2_pkey" for table "test_table_2" -DEBUG: building index "test_table_2_pkey" on table "test_table_2" serially INSERT INTO test_table_2 SELECT i, i FROM generate_series(0,100) i; SELECT create_reference_table('test_table_1'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 -DEBUG: switching to sequential query execution mode -DETAIL: Reference relation "test_table_1" is modified, which might lead to data inconsistencies or distributed deadlocks via parallel accesses to hash distributed relations due to foreign keys. Any parallel modification to those hash distributed relations in the same transaction can only be executed in sequential query execution mode NOTICE: Copying data from local table... -DEBUG: Copied 101 rows create_reference_table ------------------------ (1 row) SELECT create_distributed_table('test_table_2', 'id'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 ERROR: cannot distribute "test_table_2" in sequential mode because it is not empty HINT: If you have manually set citus.multi_shard_modify_mode to 'sequential', try with 'parallel' option. If that is not the case, try distributing local tables when they are empty. -- make sure that the output isn't too verbose @@ -1441,30 +1273,17 @@ BEGIN; SET LOCAL citus.multi_shard_modify_mode TO 'sequential'; CREATE TABLE test_table_1(id int PRIMARY KEY); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_1_pkey" for table "test_table_1" -DEBUG: building index "test_table_1_pkey" on table "test_table_1" serially INSERT INTO test_table_1 SELECT i FROM generate_series(0,100) i; CREATE TABLE test_table_2(id int PRIMARY KEY, value_1 int, FOREIGN KEY(value_1) REFERENCES test_table_1(id)); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_2_pkey" for table "test_table_2" -DEBUG: building index "test_table_2_pkey" on table "test_table_2" serially INSERT INTO test_table_2 SELECT i, i FROM generate_series(0,100) i; SELECT create_reference_table('test_table_1'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 NOTICE: Copying data from local table... -DEBUG: Copied 101 rows create_reference_table ------------------------ (1 row) SELECT create_distributed_table('test_table_2', 'id'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 ERROR: cannot distribute "test_table_2" in sequential mode because it is not empty HINT: If you have manually set citus.multi_shard_modify_mode to 'sequential', try with 'parallel' option. If that is not the case, try distributing local tables when they are empty. @@ -1479,28 +1298,14 @@ COMMIT; BEGIN; CREATE TABLE test_table_1(id int PRIMARY KEY); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_1_pkey" for table "test_table_1" -DEBUG: building index "test_table_1_pkey" on table "test_table_1" serially CREATE TABLE test_table_2(id int PRIMARY KEY, value_1 int, FOREIGN KEY(value_1) REFERENCES test_table_1(id)); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_2_pkey" for table "test_table_2" -DEBUG: building index "test_table_2_pkey" on table "test_table_2" serially SELECT create_reference_table('test_table_1'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 -DEBUG: switching to sequential query execution mode -DETAIL: Reference relation "test_table_1" is modified, which might lead to data inconsistencies or distributed deadlocks via parallel accesses to hash distributed relations due to foreign keys. Any parallel modification to those hash distributed relations in the same transaction can only be executed in sequential query execution mode create_reference_table ------------------------ (1 row) SELECT create_distributed_table('test_table_2', 'id'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 create_distributed_table -------------------------- @@ -1508,9 +1313,7 @@ DETAIL: NOTICE from localhost:57637 -- and maybe some other test CREATE INDEX i1 ON test_table_1(id); -DEBUG: building index "i1" on table "test_table_1" serially ALTER TABLE test_table_2 ADD CONSTRAINT check_val CHECK (id > 0); -DEBUG: verifying table "test_table_2" SELECT count(*) FROM test_table_2; count ------- @@ -1655,11 +1458,8 @@ DEBUG: Plan 184 query after replacing subqueries and CTEs: DELETE FROM test_fke ROLLBACK; RESET client_min_messages; +\set VERBOSITY terse DROP SCHEMA test_fkey_to_ref_in_tx CASCADE; NOTICE: drop cascades to 5 other objects -DETAIL: drop cascades to table transitive_reference_table -drop cascades to table on_update_fkey_table -drop cascades to table unrelated_dist_table -drop cascades to table reference_table -drop cascades to table distributed_table +\set VERBOSITY default SET search_path TO public; diff --git a/src/test/regress/expected/foreign_key_restriction_enforcement_0.out b/src/test/regress/expected/foreign_key_restriction_enforcement_0.out index 0c461a3ca..07855c836 100644 --- a/src/test/regress/expected/foreign_key_restriction_enforcement_0.out +++ b/src/test/regress/expected/foreign_key_restriction_enforcement_0.out @@ -817,6 +817,8 @@ ERROR: cannot execute DDL on reference relation "transitive_reference_table" be HINT: Try re-running the transaction with "SET LOCAL citus.multi_shard_modify_mode TO 'sequential';" ROLLBACK; -- case 4.5: SELECT to a dist table is follwed by a TRUNCATE +\set VERBOSITY terse +SET client_min_messages to LOG; BEGIN; SELECT count(*) FROM on_update_fkey_table WHERE value_1 = 99; count @@ -825,11 +827,8 @@ BEGIN; (1 row) TRUNCATE reference_table CASCADE; -DEBUG: switching to sequential query execution mode -DETAIL: Reference relation "reference_table" is modified, which might lead to data inconsistencies or distributed deadlocks via parallel accesses to hash distributed relations due to foreign keys. Any parallel modification to those hash distributed relations in the same transaction can only be executed in sequential query execution mode NOTICE: truncate cascades to table "on_update_fkey_table" ERROR: cannot execute DDL on reference relation "reference_table" because there was a parallel SELECT access to distributed relation "on_update_fkey_table" in the same transaction -HINT: Try re-running the transaction with "SET LOCAL citus.multi_shard_modify_mode TO 'sequential';" ROLLBACK; BEGIN; SELECT count(*) FROM on_update_fkey_table WHERE value_1 = 99; @@ -839,12 +838,9 @@ BEGIN; (1 row) TRUNCATE transitive_reference_table CASCADE; -DEBUG: switching to sequential query execution mode -DETAIL: Reference relation "transitive_reference_table" is modified, which might lead to data inconsistencies or distributed deadlocks via parallel accesses to hash distributed relations due to foreign keys. Any parallel modification to those hash distributed relations in the same transaction can only be executed in sequential query execution mode NOTICE: truncate cascades to table "reference_table" NOTICE: truncate cascades to table "on_update_fkey_table" ERROR: cannot execute DDL on reference relation "transitive_reference_table" because there was a parallel SELECT access to distributed relation "on_update_fkey_table" in the same transaction -HINT: Try re-running the transaction with "SET LOCAL citus.multi_shard_modify_mode TO 'sequential';" ROLLBACK; -- case 4.6: Router SELECT to a dist table is followed by a TRUNCATE BEGIN; @@ -855,19 +851,7 @@ BEGIN; (1 row) TRUNCATE reference_table CASCADE; -DEBUG: switching to sequential query execution mode -DETAIL: Reference relation "reference_table" is modified, which might lead to data inconsistencies or distributed deadlocks via parallel accesses to hash distributed relations due to foreign keys. Any parallel modification to those hash distributed relations in the same transaction can only be executed in sequential query execution mode NOTICE: truncate cascades to table "on_update_fkey_table" -DEBUG: truncate cascades to table "on_update_fkey_table_2380003" -DETAIL: NOTICE from localhost:57638 -DEBUG: truncate cascades to table "on_update_fkey_table_2380005" -DETAIL: NOTICE from localhost:57638 -DEBUG: truncate cascades to table "on_update_fkey_table_2380002" -DETAIL: NOTICE from localhost:57637 -DEBUG: truncate cascades to table "on_update_fkey_table_2380004" -DETAIL: NOTICE from localhost:57637 -DEBUG: building index "reference_table_pkey" on table "reference_table" -DEBUG: building index "on_update_fkey_table_pkey" on table "on_update_fkey_table" ROLLBACK; BEGIN; SELECT count(*) FROM on_update_fkey_table WHERE id = 9; @@ -877,34 +861,11 @@ BEGIN; (1 row) TRUNCATE transitive_reference_table CASCADE; -DEBUG: switching to sequential query execution mode -DETAIL: Reference relation "transitive_reference_table" is modified, which might lead to data inconsistencies or distributed deadlocks via parallel accesses to hash distributed relations due to foreign keys. Any parallel modification to those hash distributed relations in the same transaction can only be executed in sequential query execution mode NOTICE: truncate cascades to table "reference_table" NOTICE: truncate cascades to table "on_update_fkey_table" -DEBUG: truncate cascades to table "reference_table_2380001" -DETAIL: NOTICE from localhost:57638 -DEBUG: truncate cascades to table "on_update_fkey_table_2380003" -DETAIL: NOTICE from localhost:57638 -DEBUG: truncate cascades to table "on_update_fkey_table_2380005" -DETAIL: NOTICE from localhost:57638 -DEBUG: truncate cascades to table "reference_table_2380001" -DETAIL: NOTICE from localhost:57637 -DEBUG: truncate cascades to table "on_update_fkey_table_2380002" -DETAIL: NOTICE from localhost:57637 -DEBUG: truncate cascades to table "on_update_fkey_table_2380004" -DETAIL: NOTICE from localhost:57637 -DEBUG: truncate cascades to table "on_update_fkey_table_2380002" -DETAIL: NOTICE from localhost:57637 -DEBUG: truncate cascades to table "on_update_fkey_table_2380004" -DETAIL: NOTICE from localhost:57637 -DEBUG: truncate cascades to table "on_update_fkey_table_2380003" -DETAIL: NOTICE from localhost:57638 -DEBUG: truncate cascades to table "on_update_fkey_table_2380005" -DETAIL: NOTICE from localhost:57638 -DEBUG: building index "transitive_reference_table_pkey" on table "transitive_reference_table" -DEBUG: building index "reference_table_pkey" on table "reference_table" -DEBUG: building index "on_update_fkey_table_pkey" on table "on_update_fkey_table" ROLLBACK; +RESET client_min_messages; +\set VERBOSITY default -- case 5.1: Parallel UPDATE on distributed table follow by a SELECT BEGIN; UPDATE on_update_fkey_table SET value_1 = 16 WHERE value_1 = 15; @@ -954,18 +915,12 @@ ROLLBACK; BEGIN; UPDATE on_update_fkey_table SET value_1 = 16 WHERE value_1 = 15; ALTER TABLE reference_table ALTER COLUMN id SET DATA TYPE smallint; -DEBUG: rewriting table "reference_table" -DEBUG: building index "reference_table_pkey" on table "reference_table" -DEBUG: validating foreign key constraint "fkey" ERROR: cannot execute DDL on reference relation "reference_table" because there was a parallel DML access to distributed relation "on_update_fkey_table" in the same transaction HINT: Try re-running the transaction with "SET LOCAL citus.multi_shard_modify_mode TO 'sequential';" ROLLBACK; BEGIN; UPDATE on_update_fkey_table SET value_1 = 16 WHERE value_1 = 15; ALTER TABLE transitive_reference_table ALTER COLUMN id SET DATA TYPE smallint; -DEBUG: rewriting table "transitive_reference_table" -DEBUG: building index "transitive_reference_table_pkey" on table "transitive_reference_table" -DEBUG: validating foreign key constraint "fkey" ERROR: cannot execute DDL on reference relation "transitive_reference_table" because there was a parallel DML access to distributed relation "on_update_fkey_table" in the same transaction HINT: Try re-running the transaction with "SET LOCAL citus.multi_shard_modify_mode TO 'sequential';" ROLLBACK; @@ -985,16 +940,10 @@ ROLLBACK; -- case 6:2: Related parallel DDL on distributed table followed by SELECT on ref. table BEGIN; ALTER TABLE on_update_fkey_table ALTER COLUMN value_1 SET DATA TYPE smallint; -DEBUG: rewriting table "on_update_fkey_table" -DEBUG: building index "on_update_fkey_table_pkey" on table "on_update_fkey_table" -DEBUG: validating foreign key constraint "fkey" UPDATE reference_table SET id = 160 WHERE id = 15; ROLLBACK; BEGIN; ALTER TABLE on_update_fkey_table ALTER COLUMN value_1 SET DATA TYPE smallint; -DEBUG: rewriting table "on_update_fkey_table" -DEBUG: building index "on_update_fkey_table_pkey" on table "on_update_fkey_table" -DEBUG: validating foreign key constraint "fkey" UPDATE transitive_reference_table SET id = 160 WHERE id = 15; ROLLBACK; -- case 6:3: Unrelated parallel DDL on distributed table followed by UPDATE on ref. table @@ -1090,26 +1039,14 @@ ERROR: current transaction is aborted, commands ignored until end of transactio ROLLBACK; BEGIN; CREATE TABLE test_table_1(id int PRIMARY KEY); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_1_pkey" for table "test_table_1" -DEBUG: building index "test_table_1_pkey" on table "test_table_1" SELECT create_reference_table('test_table_1'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 create_reference_table ------------------------ (1 row) CREATE TABLE test_table_2(id int PRIMARY KEY, value_1 int, FOREIGN KEY(value_1) REFERENCES test_table_1(id)); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_2_pkey" for table "test_table_2" -DEBUG: building index "test_table_2_pkey" on table "test_table_2" SELECT create_distributed_table('test_table_2', 'id'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 create_distributed_table -------------------------- @@ -1123,39 +1060,21 @@ ROLLBACK; -- already executed a parallel query BEGIN; CREATE TABLE test_table_1(id int PRIMARY KEY); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_1_pkey" for table "test_table_1" -DEBUG: building index "test_table_1_pkey" on table "test_table_1" SELECT create_reference_table('test_table_1'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 create_reference_table ------------------------ (1 row) CREATE TABLE tt4(id int PRIMARY KEY, value_1 int, FOREIGN KEY(id) REFERENCES tt4(id)); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "tt4_pkey" for table "tt4" -DEBUG: building index "tt4_pkey" on table "tt4" SELECT create_distributed_table('tt4', 'id'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 create_distributed_table -------------------------- (1 row) CREATE TABLE test_table_2(id int PRIMARY KEY, value_1 int, FOREIGN KEY(value_1) REFERENCES test_table_1(id), FOREIGN KEY(id) REFERENCES tt4(id)); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_2_pkey" for table "test_table_2" -DEBUG: building index "test_table_2_pkey" on table "test_table_2" SELECT create_distributed_table('test_table_2', 'id'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 ERROR: cannot distribute relation "test_table_2" in this transaction because it has a foreign key to a reference table DETAIL: If a hash distributed table has a foreign key to a reference table, it has to be created in sequential mode before any parallel commands have been executed in the same transaction HINT: Try re-running the transaction with "SET LOCAL citus.multi_shard_modify_mode TO 'sequential';" @@ -1170,39 +1089,21 @@ ROLLBACK; BEGIN; SET LOCAL citus.multi_shard_modify_mode TO 'sequential'; CREATE TABLE test_table_1(id int PRIMARY KEY); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_1_pkey" for table "test_table_1" -DEBUG: building index "test_table_1_pkey" on table "test_table_1" SELECT create_reference_table('test_table_1'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 create_reference_table ------------------------ (1 row) CREATE TABLE tt4(id int PRIMARY KEY, value_1 int, FOREIGN KEY(id) REFERENCES tt4(id)); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "tt4_pkey" for table "tt4" -DEBUG: building index "tt4_pkey" on table "tt4" SELECT create_distributed_table('tt4', 'id'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 create_distributed_table -------------------------- (1 row) CREATE TABLE test_table_2(id int PRIMARY KEY, value_1 int, FOREIGN KEY(value_1) REFERENCES test_table_1(id), FOREIGN KEY(id) REFERENCES tt4(id)); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_2_pkey" for table "test_table_2" -DEBUG: building index "test_table_2_pkey" on table "test_table_2" SELECT create_distributed_table('test_table_2', 'id'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 create_distributed_table -------------------------- @@ -1218,26 +1119,14 @@ ROLLBACK; BEGIN; CREATE TABLE test_table_1(id int PRIMARY KEY); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_1_pkey" for table "test_table_1" -DEBUG: building index "test_table_1_pkey" on table "test_table_1" SELECT create_reference_table('test_table_1'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 create_reference_table ------------------------ (1 row) CREATE TABLE test_table_2(id int PRIMARY KEY, value_1 int); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_2_pkey" for table "test_table_2" -DEBUG: building index "test_table_2_pkey" on table "test_table_2" SELECT create_distributed_table('test_table_2', 'id'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 create_distributed_table -------------------------- @@ -1258,26 +1147,14 @@ BEGIN; SET LOCAL citus.multi_shard_modify_mode TO 'sequential'; CREATE TABLE test_table_1(id int PRIMARY KEY); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_1_pkey" for table "test_table_1" -DEBUG: building index "test_table_1_pkey" on table "test_table_1" SELECT create_reference_table('test_table_1'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 create_reference_table ------------------------ (1 row) CREATE TABLE test_table_2(id int PRIMARY KEY, value_1 int); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_2_pkey" for table "test_table_2" -DEBUG: building index "test_table_2_pkey" on table "test_table_2" SELECT create_distributed_table('test_table_2', 'id'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 create_distributed_table -------------------------- @@ -1293,26 +1170,14 @@ COMMIT; -- changed BEGIN; CREATE TABLE test_table_2(id int PRIMARY KEY, value_1 int); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_2_pkey" for table "test_table_2" -DEBUG: building index "test_table_2_pkey" on table "test_table_2" SELECT create_distributed_table('test_table_2', 'id'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 create_distributed_table -------------------------- (1 row) CREATE TABLE test_table_1(id int PRIMARY KEY); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_1_pkey" for table "test_table_1" -DEBUG: building index "test_table_1_pkey" on table "test_table_1" SELECT create_reference_table('test_table_1'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 create_reference_table ------------------------ @@ -1332,26 +1197,14 @@ ROLLBACK; BEGIN; SET LOCAL citus.multi_shard_modify_mode TO 'sequential'; CREATE TABLE test_table_2(id int PRIMARY KEY, value_1 int); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_2_pkey" for table "test_table_2" -DEBUG: building index "test_table_2_pkey" on table "test_table_2" SELECT create_distributed_table('test_table_2', 'id'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 create_distributed_table -------------------------- (1 row) CREATE TABLE test_table_1(id int PRIMARY KEY); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_1_pkey" for table "test_table_1" -DEBUG: building index "test_table_1_pkey" on table "test_table_1" SELECT create_reference_table('test_table_1'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 create_reference_table ------------------------ @@ -1374,13 +1227,7 @@ BEGIN; SET LOCAL citus.multi_shard_modify_mode TO 'sequential'; CREATE TABLE test_table_2(id int PRIMARY KEY, value_1 int); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_2_pkey" for table "test_table_2" -DEBUG: building index "test_table_2_pkey" on table "test_table_2" SELECT create_distributed_table('test_table_2', 'id'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 ERROR: cannot distribute relation "test_table_2" in this transaction because it has a foreign key to a reference table DETAIL: If a hash distributed table has a foreign key to a reference table, it has to be created in sequential mode before any parallel commands have been executed in the same transaction HINT: Try re-running the transaction with "SET LOCAL citus.multi_shard_modify_mode TO 'sequential';" @@ -1401,32 +1248,17 @@ ROLLBACK; BEGIN; CREATE TABLE test_table_1(id int PRIMARY KEY); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_1_pkey" for table "test_table_1" -DEBUG: building index "test_table_1_pkey" on table "test_table_1" INSERT INTO test_table_1 SELECT i FROM generate_series(0,100) i; CREATE TABLE test_table_2(id int PRIMARY KEY, value_1 int, FOREIGN KEY(value_1) REFERENCES test_table_1(id)); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_2_pkey" for table "test_table_2" -DEBUG: building index "test_table_2_pkey" on table "test_table_2" INSERT INTO test_table_2 SELECT i, i FROM generate_series(0,100) i; SELECT create_reference_table('test_table_1'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 -DEBUG: switching to sequential query execution mode -DETAIL: Reference relation "test_table_1" is modified, which might lead to data inconsistencies or distributed deadlocks via parallel accesses to hash distributed relations due to foreign keys. Any parallel modification to those hash distributed relations in the same transaction can only be executed in sequential query execution mode NOTICE: Copying data from local table... -DEBUG: Copied 101 rows create_reference_table ------------------------ (1 row) SELECT create_distributed_table('test_table_2', 'id'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 ERROR: cannot distribute "test_table_2" in sequential mode because it is not empty HINT: If you have manually set citus.multi_shard_modify_mode to 'sequential', try with 'parallel' option. If that is not the case, try distributing local tables when they are empty. -- make sure that the output isn't too verbose @@ -1441,30 +1273,17 @@ BEGIN; SET LOCAL citus.multi_shard_modify_mode TO 'sequential'; CREATE TABLE test_table_1(id int PRIMARY KEY); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_1_pkey" for table "test_table_1" -DEBUG: building index "test_table_1_pkey" on table "test_table_1" INSERT INTO test_table_1 SELECT i FROM generate_series(0,100) i; CREATE TABLE test_table_2(id int PRIMARY KEY, value_1 int, FOREIGN KEY(value_1) REFERENCES test_table_1(id)); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_2_pkey" for table "test_table_2" -DEBUG: building index "test_table_2_pkey" on table "test_table_2" INSERT INTO test_table_2 SELECT i, i FROM generate_series(0,100) i; SELECT create_reference_table('test_table_1'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 NOTICE: Copying data from local table... -DEBUG: Copied 101 rows create_reference_table ------------------------ (1 row) SELECT create_distributed_table('test_table_2', 'id'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 ERROR: cannot distribute "test_table_2" in sequential mode because it is not empty HINT: If you have manually set citus.multi_shard_modify_mode to 'sequential', try with 'parallel' option. If that is not the case, try distributing local tables when they are empty. @@ -1479,28 +1298,14 @@ COMMIT; BEGIN; CREATE TABLE test_table_1(id int PRIMARY KEY); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_1_pkey" for table "test_table_1" -DEBUG: building index "test_table_1_pkey" on table "test_table_1" CREATE TABLE test_table_2(id int PRIMARY KEY, value_1 int, FOREIGN KEY(value_1) REFERENCES test_table_1(id)); -DEBUG: CREATE TABLE / PRIMARY KEY will create implicit index "test_table_2_pkey" for table "test_table_2" -DEBUG: building index "test_table_2_pkey" on table "test_table_2" SELECT create_reference_table('test_table_1'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 -DEBUG: switching to sequential query execution mode -DETAIL: Reference relation "test_table_1" is modified, which might lead to data inconsistencies or distributed deadlocks via parallel accesses to hash distributed relations due to foreign keys. Any parallel modification to those hash distributed relations in the same transaction can only be executed in sequential query execution mode create_reference_table ------------------------ (1 row) SELECT create_distributed_table('test_table_2', 'id'); -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57638 -DEBUG: schema "test_fkey_to_ref_in_tx" already exists, skipping -DETAIL: NOTICE from localhost:57637 create_distributed_table -------------------------- @@ -1508,9 +1313,7 @@ DETAIL: NOTICE from localhost:57637 -- and maybe some other test CREATE INDEX i1 ON test_table_1(id); -DEBUG: building index "i1" on table "test_table_1" ALTER TABLE test_table_2 ADD CONSTRAINT check_val CHECK (id > 0); -DEBUG: verifying table "test_table_2" SELECT count(*) FROM test_table_2; count ------- @@ -1655,11 +1458,8 @@ DEBUG: Plan 184 query after replacing subqueries and CTEs: DELETE FROM test_fke ROLLBACK; RESET client_min_messages; +\set VERBOSITY terse DROP SCHEMA test_fkey_to_ref_in_tx CASCADE; NOTICE: drop cascades to 5 other objects -DETAIL: drop cascades to table transitive_reference_table -drop cascades to table on_update_fkey_table -drop cascades to table unrelated_dist_table -drop cascades to table reference_table -drop cascades to table distributed_table +\set VERBOSITY default SET search_path TO public; diff --git a/src/test/regress/expected/multi_create_shards.out b/src/test/regress/expected/multi_create_shards.out index fa6b7a179..254a15619 100644 --- a/src/test/regress/expected/multi_create_shards.out +++ b/src/test/regress/expected/multi_create_shards.out @@ -39,13 +39,9 @@ CREATE TABLE table_to_distribute ( json_data json, test_type_data dummy_type ); --- use the table WITH (OIDS) set -ALTER TABLE table_to_distribute SET WITH OIDS; SELECT create_distributed_table('table_to_distribute', 'id', 'hash'); -ERROR: cannot distribute relation: table_to_distribute -DETAIL: Distributed relations must not specify the WITH (OIDS) option in their definitions. --- revert WITH (OIDS) from above -ALTER TABLE table_to_distribute SET WITHOUT OIDS; +ERROR: cannot create constraint on "table_to_distribute" +DETAIL: Distributed relations cannot have UNIQUE, EXCLUDE, or PRIMARY KEY constraints that do not include the partition column (with an equality operator if EXCLUDE). -- use an index instead of table name SELECT create_distributed_table('table_to_distribute_pkey', 'id', 'hash'); ERROR: table_to_distribute_pkey is not a regular, foreign or partitioned table diff --git a/src/test/regress/expected/multi_master_protocol.out b/src/test/regress/expected/multi_master_protocol.out index c67941706..0381679aa 100644 --- a/src/test/regress/expected/multi_master_protocol.out +++ b/src/test/regress/expected/multi_master_protocol.out @@ -10,13 +10,13 @@ SELECT part_storage_type, part_key, part_replica_count, part_max_size, t | l_orderkey | 2 | 1536000 | 2 (1 row) -SELECT * FROM master_get_table_ddl_events('lineitem'); +SELECT * FROM master_get_table_ddl_events('lineitem') order by 1; master_get_table_ddl_events ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - CREATE TABLE public.lineitem (l_orderkey bigint NOT NULL, l_partkey integer NOT NULL, l_suppkey integer NOT NULL, l_linenumber integer NOT NULL, l_quantity numeric(15,2) NOT NULL, l_extendedprice numeric(15,2) NOT NULL, l_discount numeric(15,2) NOT NULL, l_tax numeric(15,2) NOT NULL, l_returnflag character(1) NOT NULL, l_linestatus character(1) NOT NULL, l_shipdate date NOT NULL, l_commitdate date NOT NULL, l_receiptdate date NOT NULL, l_shipinstruct character(25) NOT NULL, l_shipmode character(10) NOT NULL, l_comment character varying(44) NOT NULL) + ALTER TABLE public.lineitem ADD CONSTRAINT lineitem_pkey PRIMARY KEY (l_orderkey, l_linenumber) ALTER TABLE public.lineitem OWNER TO postgres CREATE INDEX lineitem_time_index ON public.lineitem USING btree (l_shipdate) TABLESPACE pg_default - ALTER TABLE public.lineitem ADD CONSTRAINT lineitem_pkey PRIMARY KEY (l_orderkey, l_linenumber) + CREATE TABLE public.lineitem (l_orderkey bigint NOT NULL, l_partkey integer NOT NULL, l_suppkey integer NOT NULL, l_linenumber integer NOT NULL, l_quantity numeric(15,2) NOT NULL, l_extendedprice numeric(15,2) NOT NULL, l_discount numeric(15,2) NOT NULL, l_tax numeric(15,2) NOT NULL, l_returnflag character(1) NOT NULL, l_linestatus character(1) NOT NULL, l_shipdate date NOT NULL, l_commitdate date NOT NULL, l_receiptdate date NOT NULL, l_shipinstruct character(25) NOT NULL, l_shipmode character(10) NOT NULL, l_comment character varying(44) NOT NULL) (4 rows) SELECT * FROM master_get_new_shardid(); diff --git a/src/test/regress/expected/multi_metadata_sync.out b/src/test/regress/expected/multi_metadata_sync.out index dd3999c02..9a68e5f1f 100644 --- a/src/test/regress/expected/multi_metadata_sync.out +++ b/src/test/regress/expected/multi_metadata_sync.out @@ -26,12 +26,12 @@ SELECT * FROM pg_dist_partition WHERE partmethod='h' AND repmodel='s'; -- Show that, with no MX tables, metadata snapshot contains only the delete commands, -- pg_dist_node entries and reference tables -SELECT unnest(master_metadata_snapshot()); +SELECT unnest(master_metadata_snapshot()) order by 1; unnest ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + INSERT INTO pg_dist_node (nodeid, groupid, nodename, nodeport, noderack, hasmetadata, isactive, noderole, nodecluster) VALUES (1, 1, 'localhost', 57637, 'default', FALSE, TRUE, 'primary'::noderole, 'default'),(2, 2, 'localhost', 57638, 'default', FALSE, TRUE, 'primary'::noderole, 'default') SELECT worker_drop_distributed_table(logicalrelid::regclass::text) FROM pg_dist_partition TRUNCATE pg_dist_node CASCADE - INSERT INTO pg_dist_node (nodeid, groupid, nodename, nodeport, noderack, hasmetadata, isactive, noderole, nodecluster) VALUES (1, 1, 'localhost', 57637, 'default', FALSE, TRUE, 'primary'::noderole, 'default'),(2, 2, 'localhost', 57638, 'default', FALSE, TRUE, 'primary'::noderole, 'default') (3 rows) -- Create a test table with constraints and SERIAL @@ -52,43 +52,43 @@ SELECT master_create_worker_shards('mx_test_table', 8, 1); -- considered as an MX table UPDATE pg_dist_partition SET repmodel='s' WHERE logicalrelid='mx_test_table'::regclass; -- Show that the created MX table is included in the metadata snapshot -SELECT unnest(master_metadata_snapshot()); +SELECT unnest(master_metadata_snapshot()) order by 1; unnest -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - SELECT worker_drop_distributed_table(logicalrelid::regclass::text) FROM pg_dist_partition - TRUNCATE pg_dist_node CASCADE - INSERT INTO pg_dist_node (nodeid, groupid, nodename, nodeport, noderack, hasmetadata, isactive, noderole, nodecluster) VALUES (1, 1, 'localhost', 57637, 'default', FALSE, TRUE, 'primary'::noderole, 'default'),(2, 2, 'localhost', 57638, 'default', FALSE, TRUE, 'primary'::noderole, 'default') - SELECT worker_apply_sequence_command ('CREATE SEQUENCE IF NOT EXISTS mx_test_table_col_3_seq INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 NO CYCLE') ALTER SEQUENCE public.mx_test_table_col_3_seq OWNER TO postgres - CREATE TABLE public.mx_test_table (col_1 integer, col_2 text NOT NULL, col_3 bigint DEFAULT nextval('public.mx_test_table_col_3_seq'::regclass) NOT NULL) - ALTER TABLE public.mx_test_table OWNER TO postgres ALTER TABLE public.mx_test_table ADD CONSTRAINT mx_test_table_col_1_key UNIQUE (col_1) ALTER TABLE public.mx_test_table OWNER TO postgres + ALTER TABLE public.mx_test_table OWNER TO postgres + CREATE TABLE public.mx_test_table (col_1 integer, col_2 text NOT NULL, col_3 bigint DEFAULT nextval('public.mx_test_table_col_3_seq'::regclass) NOT NULL) + INSERT INTO pg_dist_node (nodeid, groupid, nodename, nodeport, noderack, hasmetadata, isactive, noderole, nodecluster) VALUES (1, 1, 'localhost', 57637, 'default', FALSE, TRUE, 'primary'::noderole, 'default'),(2, 2, 'localhost', 57638, 'default', FALSE, TRUE, 'primary'::noderole, 'default') INSERT INTO pg_dist_partition (logicalrelid, partmethod, partkey, colocationid, repmodel) VALUES ('public.mx_test_table'::regclass, 'h', column_name_to_column('public.mx_test_table','col_1'), 0, 's') - SELECT worker_create_truncate_trigger('public.mx_test_table') INSERT INTO pg_dist_placement (shardid, shardstate, shardlength, groupid, placementid) VALUES (1310000, 1, 0, 1, 100000),(1310001, 1, 0, 2, 100001),(1310002, 1, 0, 1, 100002),(1310003, 1, 0, 2, 100003),(1310004, 1, 0, 1, 100004),(1310005, 1, 0, 2, 100005),(1310006, 1, 0, 1, 100006),(1310007, 1, 0, 2, 100007) INSERT INTO pg_dist_shard (logicalrelid, shardid, shardstorage, shardminvalue, shardmaxvalue) VALUES ('public.mx_test_table'::regclass, 1310000, 't', '-2147483648', '-1610612737'),('public.mx_test_table'::regclass, 1310001, 't', '-1610612736', '-1073741825'),('public.mx_test_table'::regclass, 1310002, 't', '-1073741824', '-536870913'),('public.mx_test_table'::regclass, 1310003, 't', '-536870912', '-1'),('public.mx_test_table'::regclass, 1310004, 't', '0', '536870911'),('public.mx_test_table'::regclass, 1310005, 't', '536870912', '1073741823'),('public.mx_test_table'::regclass, 1310006, 't', '1073741824', '1610612735'),('public.mx_test_table'::regclass, 1310007, 't', '1610612736', '2147483647') + SELECT worker_apply_sequence_command ('CREATE SEQUENCE IF NOT EXISTS mx_test_table_col_3_seq INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 NO CYCLE') + SELECT worker_create_truncate_trigger('public.mx_test_table') + SELECT worker_drop_distributed_table(logicalrelid::regclass::text) FROM pg_dist_partition + TRUNCATE pg_dist_node CASCADE (13 rows) -- Show that CREATE INDEX commands are included in the metadata snapshot CREATE INDEX mx_index ON mx_test_table(col_2); -SELECT unnest(master_metadata_snapshot()); +SELECT unnest(master_metadata_snapshot()) order by 1; unnest -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - SELECT worker_drop_distributed_table(logicalrelid::regclass::text) FROM pg_dist_partition - TRUNCATE pg_dist_node CASCADE - INSERT INTO pg_dist_node (nodeid, groupid, nodename, nodeport, noderack, hasmetadata, isactive, noderole, nodecluster) VALUES (1, 1, 'localhost', 57637, 'default', FALSE, TRUE, 'primary'::noderole, 'default'),(2, 2, 'localhost', 57638, 'default', FALSE, TRUE, 'primary'::noderole, 'default') - SELECT worker_apply_sequence_command ('CREATE SEQUENCE IF NOT EXISTS mx_test_table_col_3_seq INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 NO CYCLE') ALTER SEQUENCE public.mx_test_table_col_3_seq OWNER TO postgres - CREATE TABLE public.mx_test_table (col_1 integer, col_2 text NOT NULL, col_3 bigint DEFAULT nextval('public.mx_test_table_col_3_seq'::regclass) NOT NULL) - ALTER TABLE public.mx_test_table OWNER TO postgres - CREATE INDEX mx_index ON public.mx_test_table USING btree (col_2) TABLESPACE pg_default ALTER TABLE public.mx_test_table ADD CONSTRAINT mx_test_table_col_1_key UNIQUE (col_1) ALTER TABLE public.mx_test_table OWNER TO postgres + ALTER TABLE public.mx_test_table OWNER TO postgres + CREATE INDEX mx_index ON public.mx_test_table USING btree (col_2) TABLESPACE pg_default + CREATE TABLE public.mx_test_table (col_1 integer, col_2 text NOT NULL, col_3 bigint DEFAULT nextval('public.mx_test_table_col_3_seq'::regclass) NOT NULL) + INSERT INTO pg_dist_node (nodeid, groupid, nodename, nodeport, noderack, hasmetadata, isactive, noderole, nodecluster) VALUES (1, 1, 'localhost', 57637, 'default', FALSE, TRUE, 'primary'::noderole, 'default'),(2, 2, 'localhost', 57638, 'default', FALSE, TRUE, 'primary'::noderole, 'default') INSERT INTO pg_dist_partition (logicalrelid, partmethod, partkey, colocationid, repmodel) VALUES ('public.mx_test_table'::regclass, 'h', column_name_to_column('public.mx_test_table','col_1'), 0, 's') - SELECT worker_create_truncate_trigger('public.mx_test_table') INSERT INTO pg_dist_placement (shardid, shardstate, shardlength, groupid, placementid) VALUES (1310000, 1, 0, 1, 100000),(1310001, 1, 0, 2, 100001),(1310002, 1, 0, 1, 100002),(1310003, 1, 0, 2, 100003),(1310004, 1, 0, 1, 100004),(1310005, 1, 0, 2, 100005),(1310006, 1, 0, 1, 100006),(1310007, 1, 0, 2, 100007) INSERT INTO pg_dist_shard (logicalrelid, shardid, shardstorage, shardminvalue, shardmaxvalue) VALUES ('public.mx_test_table'::regclass, 1310000, 't', '-2147483648', '-1610612737'),('public.mx_test_table'::regclass, 1310001, 't', '-1610612736', '-1073741825'),('public.mx_test_table'::regclass, 1310002, 't', '-1073741824', '-536870913'),('public.mx_test_table'::regclass, 1310003, 't', '-536870912', '-1'),('public.mx_test_table'::regclass, 1310004, 't', '0', '536870911'),('public.mx_test_table'::regclass, 1310005, 't', '536870912', '1073741823'),('public.mx_test_table'::regclass, 1310006, 't', '1073741824', '1610612735'),('public.mx_test_table'::regclass, 1310007, 't', '1610612736', '2147483647') + SELECT worker_apply_sequence_command ('CREATE SEQUENCE IF NOT EXISTS mx_test_table_col_3_seq INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 NO CYCLE') + SELECT worker_create_truncate_trigger('public.mx_test_table') + SELECT worker_drop_distributed_table(logicalrelid::regclass::text) FROM pg_dist_partition + TRUNCATE pg_dist_node CASCADE (14 rows) -- Show that schema changes are included in the metadata snapshot @@ -96,23 +96,23 @@ CREATE SCHEMA mx_testing_schema; ALTER TABLE mx_test_table SET SCHEMA mx_testing_schema; WARNING: not propagating ALTER ... SET SCHEMA commands to worker nodes HINT: Connect to worker nodes directly to manually change schemas of affected objects. -SELECT unnest(master_metadata_snapshot()); +SELECT unnest(master_metadata_snapshot()) order by 1; unnest ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ - SELECT worker_drop_distributed_table(logicalrelid::regclass::text) FROM pg_dist_partition - TRUNCATE pg_dist_node CASCADE - INSERT INTO pg_dist_node (nodeid, groupid, nodename, nodeport, noderack, hasmetadata, isactive, noderole, nodecluster) VALUES (1, 1, 'localhost', 57637, 'default', FALSE, TRUE, 'primary'::noderole, 'default'),(2, 2, 'localhost', 57638, 'default', FALSE, TRUE, 'primary'::noderole, 'default') - SELECT worker_apply_sequence_command ('CREATE SEQUENCE IF NOT EXISTS mx_testing_schema.mx_test_table_col_3_seq INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 NO CYCLE') ALTER SEQUENCE mx_testing_schema.mx_test_table_col_3_seq OWNER TO postgres - CREATE TABLE mx_testing_schema.mx_test_table (col_1 integer, col_2 text NOT NULL, col_3 bigint DEFAULT nextval('mx_testing_schema.mx_test_table_col_3_seq'::regclass) NOT NULL) - ALTER TABLE mx_testing_schema.mx_test_table OWNER TO postgres - CREATE INDEX mx_index ON mx_testing_schema.mx_test_table USING btree (col_2) TABLESPACE pg_default ALTER TABLE mx_testing_schema.mx_test_table ADD CONSTRAINT mx_test_table_col_1_key UNIQUE (col_1) ALTER TABLE mx_testing_schema.mx_test_table OWNER TO postgres + ALTER TABLE mx_testing_schema.mx_test_table OWNER TO postgres + CREATE INDEX mx_index ON mx_testing_schema.mx_test_table USING btree (col_2) TABLESPACE pg_default + CREATE TABLE mx_testing_schema.mx_test_table (col_1 integer, col_2 text NOT NULL, col_3 bigint DEFAULT nextval('mx_testing_schema.mx_test_table_col_3_seq'::regclass) NOT NULL) + INSERT INTO pg_dist_node (nodeid, groupid, nodename, nodeport, noderack, hasmetadata, isactive, noderole, nodecluster) VALUES (1, 1, 'localhost', 57637, 'default', FALSE, TRUE, 'primary'::noderole, 'default'),(2, 2, 'localhost', 57638, 'default', FALSE, TRUE, 'primary'::noderole, 'default') INSERT INTO pg_dist_partition (logicalrelid, partmethod, partkey, colocationid, repmodel) VALUES ('mx_testing_schema.mx_test_table'::regclass, 'h', column_name_to_column('mx_testing_schema.mx_test_table','col_1'), 0, 's') - SELECT worker_create_truncate_trigger('mx_testing_schema.mx_test_table') INSERT INTO pg_dist_placement (shardid, shardstate, shardlength, groupid, placementid) VALUES (1310000, 1, 0, 1, 100000),(1310001, 1, 0, 2, 100001),(1310002, 1, 0, 1, 100002),(1310003, 1, 0, 2, 100003),(1310004, 1, 0, 1, 100004),(1310005, 1, 0, 2, 100005),(1310006, 1, 0, 1, 100006),(1310007, 1, 0, 2, 100007) INSERT INTO pg_dist_shard (logicalrelid, shardid, shardstorage, shardminvalue, shardmaxvalue) VALUES ('mx_testing_schema.mx_test_table'::regclass, 1310000, 't', '-2147483648', '-1610612737'),('mx_testing_schema.mx_test_table'::regclass, 1310001, 't', '-1610612736', '-1073741825'),('mx_testing_schema.mx_test_table'::regclass, 1310002, 't', '-1073741824', '-536870913'),('mx_testing_schema.mx_test_table'::regclass, 1310003, 't', '-536870912', '-1'),('mx_testing_schema.mx_test_table'::regclass, 1310004, 't', '0', '536870911'),('mx_testing_schema.mx_test_table'::regclass, 1310005, 't', '536870912', '1073741823'),('mx_testing_schema.mx_test_table'::regclass, 1310006, 't', '1073741824', '1610612735'),('mx_testing_schema.mx_test_table'::regclass, 1310007, 't', '1610612736', '2147483647') + SELECT worker_apply_sequence_command ('CREATE SEQUENCE IF NOT EXISTS mx_testing_schema.mx_test_table_col_3_seq INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 NO CYCLE') + SELECT worker_create_truncate_trigger('mx_testing_schema.mx_test_table') + SELECT worker_drop_distributed_table(logicalrelid::regclass::text) FROM pg_dist_partition + TRUNCATE pg_dist_node CASCADE (14 rows) -- Show that append distributed tables are not included in the metadata snapshot @@ -124,44 +124,44 @@ SELECT master_create_distributed_table('non_mx_test_table', 'col_1', 'append'); (1 row) UPDATE pg_dist_partition SET repmodel='s' WHERE logicalrelid='non_mx_test_table'::regclass; -SELECT unnest(master_metadata_snapshot()); +SELECT unnest(master_metadata_snapshot()) order by 1; unnest ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ - SELECT worker_drop_distributed_table(logicalrelid::regclass::text) FROM pg_dist_partition - TRUNCATE pg_dist_node CASCADE - INSERT INTO pg_dist_node (nodeid, groupid, nodename, nodeport, noderack, hasmetadata, isactive, noderole, nodecluster) VALUES (1, 1, 'localhost', 57637, 'default', FALSE, TRUE, 'primary'::noderole, 'default'),(2, 2, 'localhost', 57638, 'default', FALSE, TRUE, 'primary'::noderole, 'default') - SELECT worker_apply_sequence_command ('CREATE SEQUENCE IF NOT EXISTS mx_testing_schema.mx_test_table_col_3_seq INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 NO CYCLE') ALTER SEQUENCE mx_testing_schema.mx_test_table_col_3_seq OWNER TO postgres - CREATE TABLE mx_testing_schema.mx_test_table (col_1 integer, col_2 text NOT NULL, col_3 bigint DEFAULT nextval('mx_testing_schema.mx_test_table_col_3_seq'::regclass) NOT NULL) - ALTER TABLE mx_testing_schema.mx_test_table OWNER TO postgres - CREATE INDEX mx_index ON mx_testing_schema.mx_test_table USING btree (col_2) TABLESPACE pg_default ALTER TABLE mx_testing_schema.mx_test_table ADD CONSTRAINT mx_test_table_col_1_key UNIQUE (col_1) ALTER TABLE mx_testing_schema.mx_test_table OWNER TO postgres + ALTER TABLE mx_testing_schema.mx_test_table OWNER TO postgres + CREATE INDEX mx_index ON mx_testing_schema.mx_test_table USING btree (col_2) TABLESPACE pg_default + CREATE TABLE mx_testing_schema.mx_test_table (col_1 integer, col_2 text NOT NULL, col_3 bigint DEFAULT nextval('mx_testing_schema.mx_test_table_col_3_seq'::regclass) NOT NULL) + INSERT INTO pg_dist_node (nodeid, groupid, nodename, nodeport, noderack, hasmetadata, isactive, noderole, nodecluster) VALUES (1, 1, 'localhost', 57637, 'default', FALSE, TRUE, 'primary'::noderole, 'default'),(2, 2, 'localhost', 57638, 'default', FALSE, TRUE, 'primary'::noderole, 'default') INSERT INTO pg_dist_partition (logicalrelid, partmethod, partkey, colocationid, repmodel) VALUES ('mx_testing_schema.mx_test_table'::regclass, 'h', column_name_to_column('mx_testing_schema.mx_test_table','col_1'), 0, 's') - SELECT worker_create_truncate_trigger('mx_testing_schema.mx_test_table') INSERT INTO pg_dist_placement (shardid, shardstate, shardlength, groupid, placementid) VALUES (1310000, 1, 0, 1, 100000),(1310001, 1, 0, 2, 100001),(1310002, 1, 0, 1, 100002),(1310003, 1, 0, 2, 100003),(1310004, 1, 0, 1, 100004),(1310005, 1, 0, 2, 100005),(1310006, 1, 0, 1, 100006),(1310007, 1, 0, 2, 100007) INSERT INTO pg_dist_shard (logicalrelid, shardid, shardstorage, shardminvalue, shardmaxvalue) VALUES ('mx_testing_schema.mx_test_table'::regclass, 1310000, 't', '-2147483648', '-1610612737'),('mx_testing_schema.mx_test_table'::regclass, 1310001, 't', '-1610612736', '-1073741825'),('mx_testing_schema.mx_test_table'::regclass, 1310002, 't', '-1073741824', '-536870913'),('mx_testing_schema.mx_test_table'::regclass, 1310003, 't', '-536870912', '-1'),('mx_testing_schema.mx_test_table'::regclass, 1310004, 't', '0', '536870911'),('mx_testing_schema.mx_test_table'::regclass, 1310005, 't', '536870912', '1073741823'),('mx_testing_schema.mx_test_table'::regclass, 1310006, 't', '1073741824', '1610612735'),('mx_testing_schema.mx_test_table'::regclass, 1310007, 't', '1610612736', '2147483647') + SELECT worker_apply_sequence_command ('CREATE SEQUENCE IF NOT EXISTS mx_testing_schema.mx_test_table_col_3_seq INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 NO CYCLE') + SELECT worker_create_truncate_trigger('mx_testing_schema.mx_test_table') + SELECT worker_drop_distributed_table(logicalrelid::regclass::text) FROM pg_dist_partition + TRUNCATE pg_dist_node CASCADE (14 rows) -- Show that range distributed tables are not included in the metadata snapshot UPDATE pg_dist_partition SET partmethod='r' WHERE logicalrelid='non_mx_test_table'::regclass; -SELECT unnest(master_metadata_snapshot()); +SELECT unnest(master_metadata_snapshot()) order by 1; unnest ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ - SELECT worker_drop_distributed_table(logicalrelid::regclass::text) FROM pg_dist_partition - TRUNCATE pg_dist_node CASCADE - INSERT INTO pg_dist_node (nodeid, groupid, nodename, nodeport, noderack, hasmetadata, isactive, noderole, nodecluster) VALUES (1, 1, 'localhost', 57637, 'default', FALSE, TRUE, 'primary'::noderole, 'default'),(2, 2, 'localhost', 57638, 'default', FALSE, TRUE, 'primary'::noderole, 'default') - SELECT worker_apply_sequence_command ('CREATE SEQUENCE IF NOT EXISTS mx_testing_schema.mx_test_table_col_3_seq INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 NO CYCLE') ALTER SEQUENCE mx_testing_schema.mx_test_table_col_3_seq OWNER TO postgres - CREATE TABLE mx_testing_schema.mx_test_table (col_1 integer, col_2 text NOT NULL, col_3 bigint DEFAULT nextval('mx_testing_schema.mx_test_table_col_3_seq'::regclass) NOT NULL) - ALTER TABLE mx_testing_schema.mx_test_table OWNER TO postgres - CREATE INDEX mx_index ON mx_testing_schema.mx_test_table USING btree (col_2) TABLESPACE pg_default ALTER TABLE mx_testing_schema.mx_test_table ADD CONSTRAINT mx_test_table_col_1_key UNIQUE (col_1) ALTER TABLE mx_testing_schema.mx_test_table OWNER TO postgres + ALTER TABLE mx_testing_schema.mx_test_table OWNER TO postgres + CREATE INDEX mx_index ON mx_testing_schema.mx_test_table USING btree (col_2) TABLESPACE pg_default + CREATE TABLE mx_testing_schema.mx_test_table (col_1 integer, col_2 text NOT NULL, col_3 bigint DEFAULT nextval('mx_testing_schema.mx_test_table_col_3_seq'::regclass) NOT NULL) + INSERT INTO pg_dist_node (nodeid, groupid, nodename, nodeport, noderack, hasmetadata, isactive, noderole, nodecluster) VALUES (1, 1, 'localhost', 57637, 'default', FALSE, TRUE, 'primary'::noderole, 'default'),(2, 2, 'localhost', 57638, 'default', FALSE, TRUE, 'primary'::noderole, 'default') INSERT INTO pg_dist_partition (logicalrelid, partmethod, partkey, colocationid, repmodel) VALUES ('mx_testing_schema.mx_test_table'::regclass, 'h', column_name_to_column('mx_testing_schema.mx_test_table','col_1'), 0, 's') - SELECT worker_create_truncate_trigger('mx_testing_schema.mx_test_table') INSERT INTO pg_dist_placement (shardid, shardstate, shardlength, groupid, placementid) VALUES (1310000, 1, 0, 1, 100000),(1310001, 1, 0, 2, 100001),(1310002, 1, 0, 1, 100002),(1310003, 1, 0, 2, 100003),(1310004, 1, 0, 1, 100004),(1310005, 1, 0, 2, 100005),(1310006, 1, 0, 1, 100006),(1310007, 1, 0, 2, 100007) INSERT INTO pg_dist_shard (logicalrelid, shardid, shardstorage, shardminvalue, shardmaxvalue) VALUES ('mx_testing_schema.mx_test_table'::regclass, 1310000, 't', '-2147483648', '-1610612737'),('mx_testing_schema.mx_test_table'::regclass, 1310001, 't', '-1610612736', '-1073741825'),('mx_testing_schema.mx_test_table'::regclass, 1310002, 't', '-1073741824', '-536870913'),('mx_testing_schema.mx_test_table'::regclass, 1310003, 't', '-536870912', '-1'),('mx_testing_schema.mx_test_table'::regclass, 1310004, 't', '0', '536870911'),('mx_testing_schema.mx_test_table'::regclass, 1310005, 't', '536870912', '1073741823'),('mx_testing_schema.mx_test_table'::regclass, 1310006, 't', '1073741824', '1610612735'),('mx_testing_schema.mx_test_table'::regclass, 1310007, 't', '1610612736', '2147483647') + SELECT worker_apply_sequence_command ('CREATE SEQUENCE IF NOT EXISTS mx_testing_schema.mx_test_table_col_3_seq INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 NO CYCLE') + SELECT worker_create_truncate_trigger('mx_testing_schema.mx_test_table') + SELECT worker_drop_distributed_table(logicalrelid::regclass::text) FROM pg_dist_partition + TRUNCATE pg_dist_node CASCADE (14 rows) -- Test start_metadata_sync_to_node UDF diff --git a/src/test/regress/expected/multi_partitioning.out b/src/test/regress/expected/multi_partitioning.out index 9089d942c..6aee55f4a 100644 --- a/src/test/regress/expected/multi_partitioning.out +++ b/src/test/regress/expected/multi_partitioning.out @@ -4,12 +4,11 @@ SET citus.next_shard_id TO 1660000; SET citus.shard_count TO 4; SET citus.shard_replication_factor TO 1; --- print major version number for version-specific tests SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int AS server_version; - server_version ----------------- - 11 +SELECT substring(:'server_version', '\d+')::int > 10 AS server_version_above_ten; + server_version_above_ten +-------------------------- + t (1 row) -- @@ -540,7 +539,7 @@ CREATE INDEX partitioning_2009_index ON partitioning_test_2009(id); -- CREATE INDEX CONCURRENTLY on partition CREATE INDEX CONCURRENTLY partitioned_2010_index ON partitioning_test_2010(id); -- see index is created -SELECT tablename, indexname FROM pg_indexes WHERE tablename LIKE 'partitioning_test%' ORDER BY indexname; +SELECT tablename, indexname FROM pg_indexes WHERE tablename LIKE 'partitioning_test_%' ORDER BY indexname; tablename | indexname ---------------------------+---------------------------------- partitioning_test_2010 | partitioned_2010_index @@ -571,7 +570,7 @@ CREATE TABLE non_distributed_partitioned_table_1 PARTITION OF non_distributed_pa FOR VALUES FROM (0) TO (10); CREATE INDEX non_distributed_partitioned_table_index ON non_distributed_partitioned_table(a); -- see index is created -SELECT tablename, indexname FROM pg_indexes WHERE tablename LIKE 'non_distributed%' ORDER BY indexname; +SELECT tablename, indexname FROM pg_indexes WHERE tablename LIKE 'non_distributed_partitioned_table_%' ORDER BY indexname; tablename | indexname -------------------------------------+------------------------------------------- non_distributed_partitioned_table_1 | non_distributed_partitioned_table_1_a_idx diff --git a/src/test/regress/expected/multi_partitioning_0.out b/src/test/regress/expected/multi_partitioning_0.out index 9eaf20f3a..7ffd8edf0 100644 --- a/src/test/regress/expected/multi_partitioning_0.out +++ b/src/test/regress/expected/multi_partitioning_0.out @@ -4,12 +4,11 @@ SET citus.next_shard_id TO 1660000; SET citus.shard_count TO 4; SET citus.shard_replication_factor TO 1; --- print major version number for version-specific tests SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int AS server_version; - server_version ----------------- - 10 +SELECT substring(:'server_version', '\d+')::int > 10 AS server_version_above_ten; + server_version_above_ten +-------------------------- + f (1 row) -- @@ -543,7 +542,7 @@ CREATE INDEX partitioning_2009_index ON partitioning_test_2009(id); -- CREATE INDEX CONCURRENTLY on partition CREATE INDEX CONCURRENTLY partitioned_2010_index ON partitioning_test_2010(id); -- see index is created -SELECT tablename, indexname FROM pg_indexes WHERE tablename LIKE 'partitioning_test%' ORDER BY indexname; +SELECT tablename, indexname FROM pg_indexes WHERE tablename LIKE 'partitioning_test_%' ORDER BY indexname; tablename | indexname ------------------------+------------------------- partitioning_test_2010 | partitioned_2010_index @@ -568,7 +567,7 @@ FOR VALUES FROM (0) TO (10); CREATE INDEX non_distributed_partitioned_table_index ON non_distributed_partitioned_table(a); ERROR: cannot create index on partitioned table "non_distributed_partitioned_table" -- see index is created -SELECT tablename, indexname FROM pg_indexes WHERE tablename LIKE 'non_distributed%' ORDER BY indexname; +SELECT tablename, indexname FROM pg_indexes WHERE tablename LIKE 'non_distributed_partitioned_table_%' ORDER BY indexname; tablename | indexname -----------+----------- (0 rows) diff --git a/src/test/regress/expected/multi_partitioning_1.out b/src/test/regress/expected/multi_partitioning_1.out index 72ea95550..a2a423606 100644 --- a/src/test/regress/expected/multi_partitioning_1.out +++ b/src/test/regress/expected/multi_partitioning_1.out @@ -4,12 +4,11 @@ SET citus.next_shard_id TO 1660000; SET citus.shard_count TO 4; SET citus.shard_replication_factor TO 1; --- print major version number for version-specific tests SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int AS server_version; - server_version ----------------- - 11 +SELECT substring(:'server_version', '\d+')::int > 10 AS server_version_above_ten; + server_version_above_ten +-------------------------- + t (1 row) -- @@ -420,7 +419,7 @@ SELECT * FROM partitioning_test WHERE id = 9 OR id = 10 ORDER BY 1; -- create default partition CREATE TABLE partitioning_test_default PARTITION OF partitioning_test DEFAULT; \d+ partitioning_test - Table "public.partitioning_test" + Partitioned table "public.partitioning_test" Column | Type | Collation | Nullable | Default | Storage | Stats target | Description --------+---------+-----------+----------+---------+---------+--------------+------------- id | integer | | | | plain | | @@ -540,7 +539,7 @@ CREATE INDEX partitioning_2009_index ON partitioning_test_2009(id); -- CREATE INDEX CONCURRENTLY on partition CREATE INDEX CONCURRENTLY partitioned_2010_index ON partitioning_test_2010(id); -- see index is created -SELECT tablename, indexname FROM pg_indexes WHERE tablename LIKE 'partitioning_test%' ORDER BY indexname; +SELECT tablename, indexname FROM pg_indexes WHERE tablename LIKE 'partitioning_test_%' ORDER BY indexname; tablename | indexname ---------------------------+---------------------------------- partitioning_test_2010 | partitioned_2010_index @@ -571,7 +570,7 @@ CREATE TABLE non_distributed_partitioned_table_1 PARTITION OF non_distributed_pa FOR VALUES FROM (0) TO (10); CREATE INDEX non_distributed_partitioned_table_index ON non_distributed_partitioned_table(a); -- see index is created -SELECT tablename, indexname FROM pg_indexes WHERE tablename LIKE 'non_distributed%' ORDER BY indexname; +SELECT tablename, indexname FROM pg_indexes WHERE tablename LIKE 'non_distributed_partitioned_table_%' ORDER BY indexname; tablename | indexname -------------------------------------+------------------------------------------- non_distributed_partitioned_table_1 | non_distributed_partitioned_table_1_a_idx diff --git a/src/test/regress/expected/multi_reference_table.out b/src/test/regress/expected/multi_reference_table.out index cb4bf963e..eb390afed 100644 --- a/src/test/regress/expected/multi_reference_table.out +++ b/src/test/regress/expected/multi_reference_table.out @@ -1382,10 +1382,6 @@ SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='reference_sche (0 rows) \c - - - :master_port --- as we expect, setting WITH OIDS does not work for reference tables -ALTER TABLE reference_schema.reference_table_ddl SET WITH OIDS; -ERROR: alter table command is currently unsupported -DETAIL: Only ADD|DROP COLUMN, SET|DROP NOT NULL, SET|DROP DEFAULT, ADD|DROP|VALIDATE CONSTRAINT, SET (), RESET (), ATTACH|DETACH PARTITION and TYPE subcommands are supported. -- now test the renaming of the table, and back to the expected name ALTER TABLE reference_schema.reference_table_ddl RENAME TO reference_table_ddl_test; ALTER TABLE reference_schema.reference_table_ddl_test RENAME TO reference_table_ddl; diff --git a/src/test/regress/expected/multi_subquery_in_where_reference_clause.out b/src/test/regress/expected/multi_subquery_in_where_reference_clause.out index 0531e0dd3..377d651dc 100644 --- a/src/test/regress/expected/multi_subquery_in_where_reference_clause.out +++ b/src/test/regress/expected/multi_subquery_in_where_reference_clause.out @@ -87,7 +87,7 @@ DETAIL: Functions are not allowed in FROM clause when the query has subqueries SELECT user_id FROM - (SELECT 5 AS user_id) users_reference_table + (SELECT 5 AS user_id UNION ALL SELECT 6) users_reference_table WHERE NOT EXISTS (SELECT @@ -99,7 +99,7 @@ WHERE ) LIMIT 3; ERROR: cannot pushdown the subquery -DETAIL: Subqueries without FROM are not allowed in FROM clause when the outer query has subqueries in WHERE clause and it references a column from another query +DETAIL: Complex subqueries and CTEs are not allowed in the FROM clause when the query has subqueries in the WHERE clause and it references a column from another query -- join with distributed table prevents FROM from recurring SELECT DISTINCT user_id diff --git a/src/test/regress/expected/multi_utility_statements.out b/src/test/regress/expected/multi_utility_statements.out index 67e3d5d52..31888483a 100644 --- a/src/test/regress/expected/multi_utility_statements.out +++ b/src/test/regress/expected/multi_utility_statements.out @@ -188,10 +188,9 @@ VIETNAM RUSSIA UNITED KINGDOM UNITED STATES --- Test that we can create on-commit drop tables, and also test creating with --- oids, along with changing column names +-- Test that we can create on-commit drop tables, along with changing column names BEGIN; -CREATE TEMP TABLE customer_few (customer_key) WITH (OIDS) ON COMMIT DROP AS +CREATE TEMP TABLE customer_few (customer_key) ON COMMIT DROP AS (SELECT * FROM customer WHERE c_nationkey = 1 ORDER BY c_custkey LIMIT 10); SELECT customer_key, c_name, c_address FROM customer_few ORDER BY customer_key LIMIT 5; diff --git a/src/test/regress/expected/sql_procedure.out b/src/test/regress/expected/sql_procedure.out index 889af5b8a..c84db5611 100644 --- a/src/test/regress/expected/sql_procedure.out +++ b/src/test/regress/expected/sql_procedure.out @@ -98,7 +98,7 @@ $$; CALL test_procedure_modify_insert(2,12); ERROR: duplicate key value violates unique constraint "idx_table_100503" DETAIL: Key (id, org_id)=(2, 12) already exists. -CONTEXT: while executing command on localhost:57637 +CONTEXT: while executing command on localhost:57638 SQL statement "INSERT INTO test_table VALUES (tt_id, tt_org_id)" PL/pgSQL function test_procedure_modify_insert(integer,integer) line 5 at SQL statement SELECT * FROM test_table ORDER BY 1, 2; @@ -118,7 +118,7 @@ $$; CALL test_procedure_modify_insert_commit(2,30); ERROR: duplicate key value violates unique constraint "idx_table_100503" DETAIL: Key (id, org_id)=(2, 30) already exists. -CONTEXT: while executing command on localhost:57637 +CONTEXT: while executing command on localhost:57638 SQL statement "INSERT INTO test_table VALUES (tt_id, tt_org_id)" PL/pgSQL function test_procedure_modify_insert_commit(integer,integer) line 5 at SQL statement SELECT * FROM test_table ORDER BY 1, 2; @@ -209,14 +209,8 @@ SELECT * from test_table; ----+-------- (0 rows) +\set VERBOSITY terse DROP SCHEMA procedure_schema CASCADE; NOTICE: drop cascades to 8 other objects -DETAIL: drop cascades to table test_table -drop cascades to function test_procedure_delete_insert(integer,integer) -drop cascades to function test_procedure_modify_insert(integer,integer) -drop cascades to function test_procedure_modify_insert_commit(integer,integer) -drop cascades to function test_procedure_rollback_3(integer,integer) -drop cascades to function test_procedure_rollback(integer,integer) -drop cascades to function test_procedure_rollback_2(integer,integer) -drop cascades to function test_procedure(integer,integer) +\set VERBOSITY default RESET SEARCH_PATH; diff --git a/src/test/regress/expected/sql_procedure_0.out b/src/test/regress/expected/sql_procedure_0.out index ef63e9c1b..a8b289a63 100644 --- a/src/test/regress/expected/sql_procedure_0.out +++ b/src/test/regress/expected/sql_procedure_0.out @@ -277,6 +277,8 @@ SELECT * from test_table; ----+-------- (0 rows) +\set VERBOSITY terse DROP SCHEMA procedure_schema CASCADE; NOTICE: drop cascades to table test_table +\set VERBOSITY default RESET SEARCH_PATH; diff --git a/src/test/regress/expected/subquery_in_where.out b/src/test/regress/expected/subquery_in_where.out index ad583faa9..926c3e309 100644 --- a/src/test/regress/expected/subquery_in_where.out +++ b/src/test/regress/expected/subquery_in_where.out @@ -39,17 +39,18 @@ ERROR: cannot pushdown the subquery DETAIL: Complex subqueries and CTEs are not allowed in the FROM clause when the query has subqueries in the WHERE clause and it references a column from another query -- Recurring tuples as empty join tree SELECT * -FROM (SELECT 1 AS id, - 2 AS value_1, - 3 AS value_3) AS tt1 +FROM (SELECT 1 AS id, 2 AS value_1, 3 AS value_3 + UNION ALL SELECT 2 as id, 3 as value_1, 4 as value_3) AS tt1 WHERE id IN (SELECT user_id FROM events_table); -DEBUG: generating subplan 6_1 for subquery SELECT user_id FROM public.events_table -DEBUG: Plan 6 query after replacing subqueries and CTEs: SELECT id, value_1, value_3 FROM (SELECT 1 AS id, 2 AS value_1, 3 AS value_3) tt1 WHERE (id OPERATOR(pg_catalog.=) ANY (SELECT intermediate_result.user_id FROM read_intermediate_result('6_1'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer))) +DEBUG: generating subplan 6_1 for subquery SELECT 1 AS id, 2 AS value_1, 3 AS value_3 UNION ALL SELECT 2 AS id, 3 AS value_1, 4 AS value_3 +DEBUG: generating subplan 6_2 for subquery SELECT user_id FROM public.events_table +DEBUG: Plan 6 query after replacing subqueries and CTEs: SELECT id, value_1, value_3 FROM (SELECT intermediate_result.id, intermediate_result.value_1, intermediate_result.value_3 FROM read_intermediate_result('6_1'::text, 'binary'::citus_copy_format) intermediate_result(id integer, value_1 integer, value_3 integer)) tt1 WHERE (id OPERATOR(pg_catalog.=) ANY (SELECT intermediate_result.user_id FROM read_intermediate_result('6_2'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer))) id | value_1 | value_3 ----+---------+--------- 1 | 2 | 3 -(1 row) + 2 | 3 | 4 +(2 rows) -- Recurring tuples in from clause as CTE and SET operation in WHERE clause SELECT Count(*) diff --git a/src/test/regress/sql/failure_multi_dml.sql b/src/test/regress/sql/failure_multi_dml.sql index a317da996..d70e27f2b 100644 --- a/src/test/regress/sql/failure_multi_dml.sql +++ b/src/test/regress/sql/failure_multi_dml.sql @@ -157,7 +157,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT").kill()'); -- hide the error message (it has the PID)... -- we'll test for the txn side-effects to ensure it didn't run -SET client_min_messages TO FATAL; +SET client_min_messages TO ERROR; BEGIN; DELETE FROM dml_test WHERE id = 1; diff --git a/src/test/regress/sql/foreign_key_restriction_enforcement.sql b/src/test/regress/sql/foreign_key_restriction_enforcement.sql index 02dce96b0..1adafed74 100644 --- a/src/test/regress/sql/foreign_key_restriction_enforcement.sql +++ b/src/test/regress/sql/foreign_key_restriction_enforcement.sql @@ -467,6 +467,9 @@ BEGIN; ROLLBACK; -- case 4.5: SELECT to a dist table is follwed by a TRUNCATE +\set VERBOSITY terse +SET client_min_messages to LOG; + BEGIN; SELECT count(*) FROM on_update_fkey_table WHERE value_1 = 99; TRUNCATE reference_table CASCADE; @@ -488,6 +491,9 @@ BEGIN; TRUNCATE transitive_reference_table CASCADE; ROLLBACK; +RESET client_min_messages; +\set VERBOSITY default + -- case 5.1: Parallel UPDATE on distributed table follow by a SELECT BEGIN; UPDATE on_update_fkey_table SET value_1 = 16 WHERE value_1 = 15; @@ -899,6 +905,8 @@ ROLLBACK; RESET client_min_messages; +\set VERBOSITY terse DROP SCHEMA test_fkey_to_ref_in_tx CASCADE; +\set VERBOSITY default SET search_path TO public; diff --git a/src/test/regress/sql/multi_create_shards.sql b/src/test/regress/sql/multi_create_shards.sql index 5e6dde8b9..8e77eeac8 100644 --- a/src/test/regress/sql/multi_create_shards.sql +++ b/src/test/regress/sql/multi_create_shards.sql @@ -51,13 +51,8 @@ CREATE TABLE table_to_distribute ( test_type_data dummy_type ); --- use the table WITH (OIDS) set -ALTER TABLE table_to_distribute SET WITH OIDS; SELECT create_distributed_table('table_to_distribute', 'id', 'hash'); --- revert WITH (OIDS) from above -ALTER TABLE table_to_distribute SET WITHOUT OIDS; - -- use an index instead of table name SELECT create_distributed_table('table_to_distribute_pkey', 'id', 'hash'); diff --git a/src/test/regress/sql/multi_master_protocol.sql b/src/test/regress/sql/multi_master_protocol.sql index 4a56722ee..ce4c65c5d 100644 --- a/src/test/regress/sql/multi_master_protocol.sql +++ b/src/test/regress/sql/multi_master_protocol.sql @@ -10,7 +10,7 @@ SET citus.next_shard_id TO 740000; SELECT part_storage_type, part_key, part_replica_count, part_max_size, part_placement_policy FROM master_get_table_metadata('lineitem'); -SELECT * FROM master_get_table_ddl_events('lineitem'); +SELECT * FROM master_get_table_ddl_events('lineitem') order by 1; SELECT * FROM master_get_new_shardid(); diff --git a/src/test/regress/sql/multi_metadata_sync.sql b/src/test/regress/sql/multi_metadata_sync.sql index c1c608af4..8019395b7 100644 --- a/src/test/regress/sql/multi_metadata_sync.sql +++ b/src/test/regress/sql/multi_metadata_sync.sql @@ -29,7 +29,7 @@ SELECT * FROM pg_dist_partition WHERE partmethod='h' AND repmodel='s'; -- Show that, with no MX tables, metadata snapshot contains only the delete commands, -- pg_dist_node entries and reference tables -SELECT unnest(master_metadata_snapshot()); +SELECT unnest(master_metadata_snapshot()) order by 1; -- Create a test table with constraints and SERIAL CREATE TABLE mx_test_table (col_1 int UNIQUE, col_2 text NOT NULL, col_3 BIGSERIAL); @@ -41,26 +41,26 @@ SELECT master_create_worker_shards('mx_test_table', 8, 1); UPDATE pg_dist_partition SET repmodel='s' WHERE logicalrelid='mx_test_table'::regclass; -- Show that the created MX table is included in the metadata snapshot -SELECT unnest(master_metadata_snapshot()); +SELECT unnest(master_metadata_snapshot()) order by 1; -- Show that CREATE INDEX commands are included in the metadata snapshot CREATE INDEX mx_index ON mx_test_table(col_2); -SELECT unnest(master_metadata_snapshot()); +SELECT unnest(master_metadata_snapshot()) order by 1; -- Show that schema changes are included in the metadata snapshot CREATE SCHEMA mx_testing_schema; ALTER TABLE mx_test_table SET SCHEMA mx_testing_schema; -SELECT unnest(master_metadata_snapshot()); +SELECT unnest(master_metadata_snapshot()) order by 1; -- Show that append distributed tables are not included in the metadata snapshot CREATE TABLE non_mx_test_table (col_1 int, col_2 text); SELECT master_create_distributed_table('non_mx_test_table', 'col_1', 'append'); UPDATE pg_dist_partition SET repmodel='s' WHERE logicalrelid='non_mx_test_table'::regclass; -SELECT unnest(master_metadata_snapshot()); +SELECT unnest(master_metadata_snapshot()) order by 1; -- Show that range distributed tables are not included in the metadata snapshot UPDATE pg_dist_partition SET partmethod='r' WHERE logicalrelid='non_mx_test_table'::regclass; -SELECT unnest(master_metadata_snapshot()); +SELECT unnest(master_metadata_snapshot()) order by 1; -- Test start_metadata_sync_to_node UDF diff --git a/src/test/regress/sql/multi_partitioning.sql b/src/test/regress/sql/multi_partitioning.sql index fbe2ec280..4641ca166 100644 --- a/src/test/regress/sql/multi_partitioning.sql +++ b/src/test/regress/sql/multi_partitioning.sql @@ -6,9 +6,8 @@ SET citus.next_shard_id TO 1660000; SET citus.shard_count TO 4; SET citus.shard_replication_factor TO 1; --- print major version number for version-specific tests SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int AS server_version; +SELECT substring(:'server_version', '\d+')::int > 10 AS server_version_above_ten; -- -- Distributed Partitioned Table Creation Tests @@ -339,7 +338,7 @@ CREATE INDEX partitioning_2009_index ON partitioning_test_2009(id); CREATE INDEX CONCURRENTLY partitioned_2010_index ON partitioning_test_2010(id); -- see index is created -SELECT tablename, indexname FROM pg_indexes WHERE tablename LIKE 'partitioning_test%' ORDER BY indexname; +SELECT tablename, indexname FROM pg_indexes WHERE tablename LIKE 'partitioning_test_%' ORDER BY indexname; -- test drop -- indexes created on parent table can only be dropped on parent table @@ -360,7 +359,7 @@ FOR VALUES FROM (0) TO (10); CREATE INDEX non_distributed_partitioned_table_index ON non_distributed_partitioned_table(a); -- see index is created -SELECT tablename, indexname FROM pg_indexes WHERE tablename LIKE 'non_distributed%' ORDER BY indexname; +SELECT tablename, indexname FROM pg_indexes WHERE tablename LIKE 'non_distributed_partitioned_table_%' ORDER BY indexname; -- drop the index and see it is dropped DROP INDEX non_distributed_partitioned_table_index; diff --git a/src/test/regress/sql/multi_reference_table.sql b/src/test/regress/sql/multi_reference_table.sql index 767027d2e..bdccc4fc0 100644 --- a/src/test/regress/sql/multi_reference_table.sql +++ b/src/test/regress/sql/multi_reference_table.sql @@ -874,9 +874,6 @@ SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='reference_sche \di reference_schema.reference_index_2* \c - - - :master_port --- as we expect, setting WITH OIDS does not work for reference tables -ALTER TABLE reference_schema.reference_table_ddl SET WITH OIDS; - -- now test the renaming of the table, and back to the expected name ALTER TABLE reference_schema.reference_table_ddl RENAME TO reference_table_ddl_test; ALTER TABLE reference_schema.reference_table_ddl_test RENAME TO reference_table_ddl; diff --git a/src/test/regress/sql/multi_subquery_in_where_reference_clause.sql b/src/test/regress/sql/multi_subquery_in_where_reference_clause.sql index 6f9c74650..294cb240d 100644 --- a/src/test/regress/sql/multi_subquery_in_where_reference_clause.sql +++ b/src/test/regress/sql/multi_subquery_in_where_reference_clause.sql @@ -77,7 +77,7 @@ LIMIT 3; SELECT user_id FROM - (SELECT 5 AS user_id) users_reference_table + (SELECT 5 AS user_id UNION ALL SELECT 6) users_reference_table WHERE NOT EXISTS (SELECT diff --git a/src/test/regress/sql/multi_utility_statements.sql b/src/test/regress/sql/multi_utility_statements.sql index 1039dc849..32fb4b8cd 100644 --- a/src/test/regress/sql/multi_utility_statements.sql +++ b/src/test/regress/sql/multi_utility_statements.sql @@ -106,12 +106,11 @@ COPY nation TO STDOUT; -- ensure individual cols can be copied out, too COPY nation(n_name) TO STDOUT; --- Test that we can create on-commit drop tables, and also test creating with --- oids, along with changing column names +-- Test that we can create on-commit drop tables, along with changing column names BEGIN; -CREATE TEMP TABLE customer_few (customer_key) WITH (OIDS) ON COMMIT DROP AS +CREATE TEMP TABLE customer_few (customer_key) ON COMMIT DROP AS (SELECT * FROM customer WHERE c_nationkey = 1 ORDER BY c_custkey LIMIT 10); SELECT customer_key, c_name, c_address diff --git a/src/test/regress/sql/sql_procedure.sql b/src/test/regress/sql/sql_procedure.sql index 4771d78a8..6c665181c 100644 --- a/src/test/regress/sql/sql_procedure.sql +++ b/src/test/regress/sql/sql_procedure.sql @@ -166,6 +166,8 @@ call test_procedure(1,1); call test_procedure(20, 20); SELECT * from test_table; +\set VERBOSITY terse DROP SCHEMA procedure_schema CASCADE; +\set VERBOSITY default RESET SEARCH_PATH; diff --git a/src/test/regress/sql/subquery_in_where.sql b/src/test/regress/sql/subquery_in_where.sql index afcf6fccc..be9b45f0e 100644 --- a/src/test/regress/sql/subquery_in_where.sql +++ b/src/test/regress/sql/subquery_in_where.sql @@ -6,46 +6,45 @@ SET search_path TO subquery_in_where, public; SET client_min_messages TO DEBUG1; ---CTEs can be used as a recurring tuple with subqueries in WHERE +--CTEs can be used as a recurring tuple with subqueries in WHERE WITH event_id - AS (SELECT user_id AS events_user_id, - time AS events_time, + AS (SELECT user_id AS events_user_id, + time AS events_time, event_type - FROM events_table) -SELECT Count(*) + FROM events_table) +SELECT Count(*) FROM event_id WHERE events_user_id IN (SELECT user_id FROM users_table); ---Correlated subqueries can not be used in WHERE clause -WITH event_id - AS (SELECT user_id AS events_user_id, - time AS events_time, - event_type - FROM events_table) -SELECT Count(*) -FROM event_id -WHERE events_user_id IN (SELECT user_id - FROM users_table - WHERE users_table.time = events_time); +--Correlated subqueries can not be used in WHERE clause +WITH event_id + AS (SELECT user_id AS events_user_id, + time AS events_time, + event_type + FROM events_table) +SELECT Count(*) +FROM event_id +WHERE events_user_id IN (SELECT user_id + FROM users_table + WHERE users_table.time = events_time); --- Recurring tuples as empty join tree -SELECT * -FROM (SELECT 1 AS id, - 2 AS value_1, - 3 AS value_3) AS tt1 -WHERE id IN (SELECT user_id - FROM events_table); +-- Recurring tuples as empty join tree +SELECT * +FROM (SELECT 1 AS id, 2 AS value_1, 3 AS value_3 + UNION ALL SELECT 2 as id, 3 as value_1, 4 as value_3) AS tt1 +WHERE id IN (SELECT user_id + FROM events_table); -- Recurring tuples in from clause as CTE and SET operation in WHERE clause SELECT Count(*) FROM (WITH event_id AS - (SELECT user_id AS events_user_id, time AS events_time, event_type + (SELECT user_id AS events_user_id, time AS events_time, event_type FROM events_table) SELECT events_user_id, events_time, event_type - FROM event_id + FROM event_id ORDER BY 1,2,3 - LIMIT 10) AS sub_table + LIMIT 10) AS sub_table WHERE events_user_id IN ( (SELECT user_id FROM users_table @@ -157,7 +156,7 @@ FROM user_id as events_user_id, time as events_time, event_type FROM events_table - ORDER BY + ORDER BY 1,2,3 LIMIT 10 @@ -499,7 +498,7 @@ FROM users_table ORDER BY user_id - LIMIT + LIMIT 10) as sub_table WHERE user_id From 693d4695d7295888adba59997e312b1b3a8eb0ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philip=20Dub=C3=A9?= Date: Wed, 14 Aug 2019 07:14:01 +0000 Subject: [PATCH 11/12] Create a test 'pg12' for pg12 features & error on unsupported new features Unsupported new features: COPY FROM WHERE, GENERATED ALWAYS AS, non-heap table access methods --- .../commands/create_distributed_table.c | 62 +++ src/backend/distributed/commands/multi_copy.c | 10 +- .../distributed/commands/utility_hook.c | 2 +- src/backend/distributed/test/blackhole_am.c | 502 ++++++++++++++++++ src/test/regress/expected/pg12.out | 368 +++++++++++++ src/test/regress/expected/pg12_1.out | 6 + src/test/regress/multi_schedule | 3 +- src/test/regress/sql/pg12.sql | 262 +++++++++ 8 files changed, 1211 insertions(+), 4 deletions(-) create mode 100644 src/backend/distributed/test/blackhole_am.c create mode 100644 src/test/regress/expected/pg12.out create mode 100644 src/test/regress/expected/pg12_1.out create mode 100644 src/test/regress/sql/pg12.sql diff --git a/src/backend/distributed/commands/create_distributed_table.c b/src/backend/distributed/commands/create_distributed_table.c index b1bb140aa..d144c2ada 100644 --- a/src/backend/distributed/commands/create_distributed_table.c +++ b/src/backend/distributed/commands/create_distributed_table.c @@ -21,12 +21,16 @@ #include "catalog/dependency.h" #include "catalog/index.h" #include "catalog/pg_am.h" +#include "catalog/pg_attribute.h" #if (PG_VERSION_NUM < 110000) #include "catalog/pg_constraint_fn.h" #endif #include "catalog/pg_enum.h" #include "catalog/pg_extension.h" #include "catalog/pg_opclass.h" +#if PG_VERSION_NUM >= 12000 +#include "catalog/pg_proc.h" +#endif #include "catalog/pg_trigger.h" #include "commands/defrem.h" #include "commands/extension.h" @@ -99,6 +103,8 @@ static bool LocalTableEmpty(Oid tableId); static void CopyLocalDataIntoShards(Oid relationId); static List * TupleDescColumnNameList(TupleDesc tupleDescriptor); static bool RelationUsesIdentityColumns(TupleDesc relationDesc); +static bool RelationUsesGeneratedStoredColumns(TupleDesc relationDesc); +static bool RelationUsesHeapAccessMethodOrNone(Relation relation); static bool CanUseExclusiveConnections(Oid relationId, bool localTableEmpty); /* exports for SQL callable functions */ @@ -645,6 +651,12 @@ EnsureRelationCanBeDistributed(Oid relationId, Var *distributionColumn, relationDesc = RelationGetDescr(relation); relationName = RelationGetRelationName(relation); + if (!RelationUsesHeapAccessMethodOrNone(relation)) + { + ereport(ERROR, (errmsg( + "cannot distribute relations using non-heap access methods"))); + } + #if PG_VERSION_NUM < 120000 /* verify target relation does not use WITH (OIDS) PostgreSQL feature */ @@ -666,6 +678,15 @@ EnsureRelationCanBeDistributed(Oid relationId, Var *distributionColumn, "... AS IDENTITY."))); } + /* verify target relation does not use generated columns */ + if (RelationUsesGeneratedStoredColumns(relationDesc)) + { + ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot distribute relation: %s", relationName), + errdetail("Distributed relations must not use GENERATED ALWAYS " + "AS (...) STORED."))); + } + /* check for support function needed by specified partition method */ if (distributionMethod == DISTRIBUTE_BY_HASH) { @@ -1374,3 +1395,44 @@ RelationUsesIdentityColumns(TupleDesc relationDesc) return false; } + + +/* + * RelationUsesIdentityColumns returns whether a given relation uses the SQL + * GENERATED ... AS IDENTITY features introduced as of PostgreSQL 10. + */ +static bool +RelationUsesGeneratedStoredColumns(TupleDesc relationDesc) +{ +#if PG_VERSION_NUM >= 120000 + int attributeIndex = 0; + + for (attributeIndex = 0; attributeIndex < relationDesc->natts; attributeIndex++) + { + Form_pg_attribute attributeForm = TupleDescAttr(relationDesc, attributeIndex); + + if (attributeForm->attgenerated == ATTRIBUTE_GENERATED_STORED) + { + return true; + } + } +#endif + + return false; +} + + +/* + * Returns whether given relation uses default access method + */ +static bool +RelationUsesHeapAccessMethodOrNone(Relation relation) +{ +#if PG_VERSION_NUM >= 120000 + + return relation->rd_rel->relkind != RELKIND_RELATION || + relation->rd_amhandler == HEAP_TABLE_AM_HANDLER_OID; +#else + return true; +#endif +} diff --git a/src/backend/distributed/commands/multi_copy.c b/src/backend/distributed/commands/multi_copy.c index 904fab1dc..defc08756 100644 --- a/src/backend/distributed/commands/multi_copy.c +++ b/src/backend/distributed/commands/multi_copy.c @@ -2803,6 +2803,14 @@ ProcessCopyStmt(CopyStmt *copyStatement, char *completionTag, const char *queryS { if (copyStatement->is_from) { +#if PG_VERSION_NUM >= 120000 + if (copyStatement->whereClause) + { + ereport(ERROR, (errmsg( + "Citus does not support COPY FROM with WHERE"))); + } +#endif + /* check permissions, we're bypassing postgres' normal checks */ if (!isCopyFromWorker) { @@ -2812,7 +2820,7 @@ ProcessCopyStmt(CopyStmt *copyStatement, char *completionTag, const char *queryS CitusCopyFrom(copyStatement, completionTag); return NULL; } - else if (!copyStatement->is_from) + else { /* * The copy code only handles SELECTs in COPY ... TO on master tables, diff --git a/src/backend/distributed/commands/utility_hook.c b/src/backend/distributed/commands/utility_hook.c index a8443a2e0..aad67cbfa 100644 --- a/src/backend/distributed/commands/utility_hook.c +++ b/src/backend/distributed/commands/utility_hook.c @@ -194,7 +194,7 @@ multi_ProcessUtility(PlannedStmt *pstmt, /* * TRANSMIT used to be separate command, but to avoid patching the grammar - * it's no overlaid onto COPY, but with FORMAT = 'transmit' instead of the + * it's now overlaid onto COPY, but with FORMAT = 'transmit' instead of the * normal FORMAT options. */ if (IsTransmitStmt(parsetree)) diff --git a/src/backend/distributed/test/blackhole_am.c b/src/backend/distributed/test/blackhole_am.c new file mode 100644 index 000000000..ac0e60ae1 --- /dev/null +++ b/src/backend/distributed/test/blackhole_am.c @@ -0,0 +1,502 @@ +/*------------------------------------------------------------------------- + * + * blackhole_am.c + * blackhole table access method code + * + * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * Copied from https://github.com/michaelpq/pg_plugins/blob/master/blackhole_am/blackhole_am.c + * + * + * NOTES + * This file introduces the table access method blackhole, which can + * be used as a template for other table access methods, and guarantees + * that any data inserted into it gets sent to the void. + * + *------------------------------------------------------------------------- + */ + +/* *INDENT-OFF* */ +#include "postgres.h" + +#if PG_VERSION_NUM >= 120000 + +#include "access/tableam.h" +#include "access/heapam.h" +#include "access/amapi.h" +#include "catalog/index.h" +#include "commands/vacuum.h" +#include "executor/tuptable.h" + +PG_FUNCTION_INFO_V1(blackhole_am_handler); + +/* Base structures for scans */ +typedef struct BlackholeScanDescData +{ + TableScanDescData rs_base; /* AM independent part of the descriptor */ + + /* Add more fields here as needed by the AM. */ +} BlackholeScanDescData; +typedef struct BlackholeScanDescData *BlackholeScanDesc; + +static const TableAmRoutine blackhole_methods; + + +/* ------------------------------------------------------------------------ + * Slot related callbacks for blackhole AM + * ------------------------------------------------------------------------ + */ + +static const TupleTableSlotOps * +blackhole_slot_callbacks(Relation relation) +{ + /* + * Here you would most likely want to invent your own set of + * slot callbacks for your AM. + */ + return &TTSOpsMinimalTuple; +} + +/* ------------------------------------------------------------------------ + * Table Scan Callbacks for blackhole AM + * ------------------------------------------------------------------------ + */ + +static TableScanDesc +blackhole_scan_begin(Relation relation, Snapshot snapshot, + int nkeys, ScanKey key, + ParallelTableScanDesc parallel_scan, + uint32 flags) +{ + BlackholeScanDesc scan; + + scan = (BlackholeScanDesc) palloc(sizeof(BlackholeScanDescData)); + + scan->rs_base.rs_rd = relation; + scan->rs_base.rs_snapshot = snapshot; + scan->rs_base.rs_nkeys = nkeys; + scan->rs_base.rs_flags = flags; + scan->rs_base.rs_parallel = parallel_scan; + + return (TableScanDesc) scan; } + +static void +blackhole_scan_end(TableScanDesc sscan) +{ + BlackholeScanDesc scan = (BlackholeScanDesc) sscan; + pfree(scan); +} + +static void +blackhole_scan_rescan(TableScanDesc sscan, ScanKey key, bool set_params, + bool allow_strat, bool allow_sync, bool allow_pagemode) +{ + /* nothing to do */ +} + +static bool +blackhole_scan_getnextslot(TableScanDesc sscan, ScanDirection direction, + TupleTableSlot *slot) +{ + /* nothing to do */ + return false; +} + +/* ------------------------------------------------------------------------ + * Index Scan Callbacks for blackhole AM + * ------------------------------------------------------------------------ + */ + +static IndexFetchTableData * +blackhole_index_fetch_begin(Relation rel) +{ + return NULL; +} + +static void +blackhole_index_fetch_reset(IndexFetchTableData *scan) +{ + /* nothing to do here */ +} + +static void +blackhole_index_fetch_end(IndexFetchTableData *scan) +{ + /* nothing to do here */ +} + +static bool +blackhole_index_fetch_tuple(struct IndexFetchTableData *scan, + ItemPointer tid, + Snapshot snapshot, + TupleTableSlot *slot, + bool *call_again, bool *all_dead) +{ + /* there is no data */ + return 0; +} + + +/* ------------------------------------------------------------------------ + * Callbacks for non-modifying operations on individual tuples for + * blackhole AM. + * ------------------------------------------------------------------------ + */ + +static bool +blackhole_fetch_row_version(Relation relation, + ItemPointer tid, + Snapshot snapshot, + TupleTableSlot *slot) +{ + /* nothing to do */ + return false; +} + +static void +blackhole_get_latest_tid(TableScanDesc sscan, + ItemPointer tid) +{ + /* nothing to do */ +} + +static bool +blackhole_tuple_tid_valid(TableScanDesc scan, ItemPointer tid) +{ + return false; +} + +static bool +blackhole_tuple_satisfies_snapshot(Relation rel, TupleTableSlot *slot, + Snapshot snapshot) +{ + return false; +} + +static TransactionId +blackhole_compute_xid_horizon_for_tuples(Relation rel, + ItemPointerData *tids, + int nitems) +{ + return InvalidTransactionId; +} + +/* ---------------------------------------------------------------------------- + * Functions for manipulations of physical tuples for blackhole AM. + * ---------------------------------------------------------------------------- + */ + +static void +blackhole_tuple_insert(Relation relation, TupleTableSlot *slot, + CommandId cid, int options, BulkInsertState bistate) +{ + /* nothing to do */ +} + +static void +blackhole_tuple_insert_speculative(Relation relation, TupleTableSlot *slot, + CommandId cid, int options, + BulkInsertState bistate, + uint32 specToken) +{ + /* nothing to do */ +} + +static void +blackhole_tuple_complete_speculative(Relation relation, TupleTableSlot *slot, + uint32 spekToken, bool succeeded) +{ + /* nothing to do */ +} + +static void +blackhole_multi_insert(Relation relation, TupleTableSlot **slots, + int ntuples, CommandId cid, int options, + BulkInsertState bistate) +{ + /* nothing to do */ +} + +static TM_Result +blackhole_tuple_delete(Relation relation, ItemPointer tid, CommandId cid, + Snapshot snapshot, Snapshot crosscheck, bool wait, + TM_FailureData *tmfd, bool changingPart) +{ + /* nothing to do, so it is always OK */ + return TM_Ok; +} + + +static TM_Result +blackhole_tuple_update(Relation relation, ItemPointer otid, + TupleTableSlot *slot, CommandId cid, + Snapshot snapshot, Snapshot crosscheck, + bool wait, TM_FailureData *tmfd, + LockTupleMode *lockmode, bool *update_indexes) +{ + /* nothing to do, so it is always OK */ + return TM_Ok; +} + +static TM_Result +blackhole_tuple_lock(Relation relation, ItemPointer tid, Snapshot snapshot, + TupleTableSlot *slot, CommandId cid, LockTupleMode mode, + LockWaitPolicy wait_policy, uint8 flags, + TM_FailureData *tmfd) +{ + /* nothing to do, so it is always OK */ + return TM_Ok; +} + +static void +blackhole_finish_bulk_insert(Relation relation, int options) +{ + /* nothing to do */ +} + + +/* ------------------------------------------------------------------------ + * DDL related callbacks for blackhole AM. + * ------------------------------------------------------------------------ + */ + +static void +blackhole_relation_set_new_filenode(Relation rel, + const RelFileNode *newrnode, + char persistence, + TransactionId *freezeXid, + MultiXactId *minmulti) +{ + /* nothing to do */ +} + +static void +blackhole_relation_nontransactional_truncate(Relation rel) +{ + /* nothing to do */ +} + +static void +blackhole_copy_data(Relation rel, const RelFileNode *newrnode) +{ + /* there is no data */ +} + +static void +blackhole_copy_for_cluster(Relation OldTable, Relation NewTable, + Relation OldIndex, bool use_sort, + TransactionId OldestXmin, + TransactionId *xid_cutoff, + MultiXactId *multi_cutoff, + double *num_tuples, + double *tups_vacuumed, + double *tups_recently_dead) +{ + /* no data, so nothing to do */ +} + +static void +blackhole_vacuum(Relation onerel, VacuumParams *params, + BufferAccessStrategy bstrategy) +{ + /* no data, so nothing to do */ +} + +static bool +blackhole_scan_analyze_next_block(TableScanDesc scan, BlockNumber blockno, + BufferAccessStrategy bstrategy) +{ + /* no data, so no point to analyze next block */ + return false; +} + +static bool +blackhole_scan_analyze_next_tuple(TableScanDesc scan, TransactionId OldestXmin, + double *liverows, double *deadrows, + TupleTableSlot *slot) +{ + /* no data, so no point to analyze next tuple */ + return false; +} + +static double +blackhole_index_build_range_scan(Relation tableRelation, + Relation indexRelation, + IndexInfo *indexInfo, + bool allow_sync, + bool anyvisible, + bool progress, + BlockNumber start_blockno, + BlockNumber numblocks, + IndexBuildCallback callback, + void *callback_state, + TableScanDesc scan) +{ + /* no data, so no tuples */ + return 0; +} + +static void +blackhole_index_validate_scan(Relation tableRelation, + Relation indexRelation, + IndexInfo *indexInfo, + Snapshot snapshot, + ValidateIndexState *state) +{ + /* nothing to do */ +} + + +/* ------------------------------------------------------------------------ + * Miscellaneous callbacks for the blackhole AM + * ------------------------------------------------------------------------ + */ + +static uint64 +blackhole_relation_size(Relation rel, ForkNumber forkNumber) +{ + /* there is nothing */ + return 0; +} + +/* + * Check to see whether the table needs a TOAST table. + */ +static bool +blackhole_relation_needs_toast_table(Relation rel) +{ + /* no data, so no toast table needed */ + return false; +} + + +/* ------------------------------------------------------------------------ + * Planner related callbacks for the blackhole AM + * ------------------------------------------------------------------------ + */ + +static void +blackhole_estimate_rel_size(Relation rel, int32 *attr_widths, + BlockNumber *pages, double *tuples, + double *allvisfrac) +{ + /* no data available */ + *attr_widths = 0; + *tuples = 0; + *allvisfrac = 0; + *pages = 0; +} + + +/* ------------------------------------------------------------------------ + * Executor related callbacks for the blackhole AM + * ------------------------------------------------------------------------ + */ + +static bool +blackhole_scan_bitmap_next_block(TableScanDesc scan, + TBMIterateResult *tbmres) +{ + /* no data, so no point to scan next block */ + return false; +} + +static bool +blackhole_scan_bitmap_next_tuple(TableScanDesc scan, + TBMIterateResult *tbmres, + TupleTableSlot *slot) +{ + /* no data, so no point to scan next tuple */ + return false; +} + +static bool +blackhole_scan_sample_next_block(TableScanDesc scan, + SampleScanState *scanstate) +{ + /* no data, so no point to scan next block for sampling */ + return false; +} + +static bool +blackhole_scan_sample_next_tuple(TableScanDesc scan, + SampleScanState *scanstate, + TupleTableSlot *slot) +{ + /* no data, so no point to scan next tuple for sampling */ + return false; +} + + +/* ------------------------------------------------------------------------ + * Definition of the blackhole table access method. + * ------------------------------------------------------------------------ + */ + +static const TableAmRoutine blackhole_methods = { + .type = T_TableAmRoutine, + + .slot_callbacks = blackhole_slot_callbacks, + + .scan_begin = blackhole_scan_begin, + .scan_end = blackhole_scan_end, + .scan_rescan = blackhole_scan_rescan, + .scan_getnextslot = blackhole_scan_getnextslot, + + /* these are common helper functions */ + .parallelscan_estimate = table_block_parallelscan_estimate, + .parallelscan_initialize = table_block_parallelscan_initialize, + .parallelscan_reinitialize = table_block_parallelscan_reinitialize, + + .index_fetch_begin = blackhole_index_fetch_begin, + .index_fetch_reset = blackhole_index_fetch_reset, + .index_fetch_end = blackhole_index_fetch_end, + .index_fetch_tuple = blackhole_index_fetch_tuple, + + .tuple_insert = blackhole_tuple_insert, + .tuple_insert_speculative = blackhole_tuple_insert_speculative, + .tuple_complete_speculative = blackhole_tuple_complete_speculative, + .multi_insert = blackhole_multi_insert, + .tuple_delete = blackhole_tuple_delete, + .tuple_update = blackhole_tuple_update, + .tuple_lock = blackhole_tuple_lock, + .finish_bulk_insert = blackhole_finish_bulk_insert, + + .tuple_fetch_row_version = blackhole_fetch_row_version, + .tuple_get_latest_tid = blackhole_get_latest_tid, + .tuple_tid_valid = blackhole_tuple_tid_valid, + .tuple_satisfies_snapshot = blackhole_tuple_satisfies_snapshot, + .compute_xid_horizon_for_tuples = blackhole_compute_xid_horizon_for_tuples, + + .relation_set_new_filenode = blackhole_relation_set_new_filenode, + .relation_nontransactional_truncate = blackhole_relation_nontransactional_truncate, + .relation_copy_data = blackhole_copy_data, + .relation_copy_for_cluster = blackhole_copy_for_cluster, + .relation_vacuum = blackhole_vacuum, + .scan_analyze_next_block = blackhole_scan_analyze_next_block, + .scan_analyze_next_tuple = blackhole_scan_analyze_next_tuple, + .index_build_range_scan = blackhole_index_build_range_scan, + .index_validate_scan = blackhole_index_validate_scan, + + .relation_size = blackhole_relation_size, + .relation_needs_toast_table = blackhole_relation_needs_toast_table, + + .relation_estimate_size = blackhole_estimate_rel_size, + + .scan_bitmap_next_block = blackhole_scan_bitmap_next_block, + .scan_bitmap_next_tuple = blackhole_scan_bitmap_next_tuple, + .scan_sample_next_block = blackhole_scan_sample_next_block, + .scan_sample_next_tuple = blackhole_scan_sample_next_tuple +}; + + +Datum +blackhole_am_handler(PG_FUNCTION_ARGS) +{ + PG_RETURN_POINTER(&blackhole_methods); +} + +#endif diff --git a/src/test/regress/expected/pg12.out b/src/test/regress/expected/pg12.out new file mode 100644 index 000000000..be554a521 --- /dev/null +++ b/src/test/regress/expected/pg12.out @@ -0,0 +1,368 @@ +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int > 11 AS server_version_above_eleven +\gset +\if :server_version_above_eleven +\else +\q +\endif +SET citus.shard_replication_factor to 1; +SET citus.next_shard_id TO 60000; +SET citus.next_placement_id TO 60000; +create schema test_pg12; +set search_path to test_pg12; +CREATE FUNCTION blackhole_am_handler(internal) +RETURNS table_am_handler +AS 'citus' +LANGUAGE C; +CREATE ACCESS METHOD blackhole_am TYPE TABLE HANDLER blackhole_am_handler; +create table test_am(id int, val int) using blackhole_am; +insert into test_am values (1, 1); +-- Custom table access methods should be rejected +select create_distributed_table('test_am','id'); +ERROR: cannot distribute relations using non-heap access methods +-- Test generated columns +create table gen1 ( + id int, + val1 int, + val2 int GENERATED ALWAYS AS (val1 + 2) STORED +); +create table gen2 ( + id int, + val1 int, + val2 int GENERATED ALWAYS AS (val1 + 2) STORED +); +insert into gen1 (id, val1) values (1,4),(3,6),(5,2),(7,2); +insert into gen2 (id, val1) values (1,4),(3,6),(5,2),(7,2); +select * from create_distributed_table('gen1', 'id'); +ERROR: cannot distribute relation: gen1 +DETAIL: Distributed relations must not use GENERATED ALWAYS AS (...) STORED. +select * from create_distributed_table('gen2', 'val2'); +ERROR: cannot distribute relation: gen2 +DETAIL: Distributed relations must not use GENERATED ALWAYS AS (...) STORED. +insert into gen1 (id, val1) values (2,4),(4,6),(6,2),(8,2); +insert into gen2 (id, val1) values (2,4),(4,6),(6,2),(8,2); +select * from gen1; + id | val1 | val2 +----+------+------ + 1 | 4 | 6 + 3 | 6 | 8 + 5 | 2 | 4 + 7 | 2 | 4 + 2 | 4 | 6 + 4 | 6 | 8 + 6 | 2 | 4 + 8 | 2 | 4 +(8 rows) + +select * from gen2; + id | val1 | val2 +----+------+------ + 1 | 4 | 6 + 3 | 6 | 8 + 5 | 2 | 4 + 7 | 2 | 4 + 2 | 4 | 6 + 4 | 6 | 8 + 6 | 2 | 4 + 8 | 2 | 4 +(8 rows) + +-- Test new VACUUM/ANALYZE options +analyze (skip_locked) gen1; +vacuum (skip_locked) gen1; +vacuum (truncate 0) gen1; +vacuum (index_cleanup 1) gen1; +-- COPY FROM +create table cptest (id int, val int); +select create_distributed_table('cptest', 'id'); + create_distributed_table +-------------------------- + +(1 row) + +copy cptest from STDIN with csv where val < 4; +ERROR: Citus does not support COPY FROM with WHERE +1,6 +2,3 +3,2 +4,9 +5,4 +\. +invalid command \. +select sum(id), sum(val) from cptest; +ERROR: syntax error at or near "1" +LINE 1: 1,6 + ^ +-- CTE materialized/not materialized +CREATE TABLE single_hash_repartition_first (id int, sum int, avg float); +CREATE TABLE single_hash_repartition_second (id int primary key, sum int, avg float); +SELECT create_distributed_table('single_hash_repartition_first', 'id'); + create_distributed_table +-------------------------- + +(1 row) + +SELECT create_distributed_table('single_hash_repartition_second', 'id'); + create_distributed_table +-------------------------- + +(1 row) + +INSERT INTO single_hash_repartition_first +SELECT i, i * 3, i * 0.3 +FROM generate_series(0, 100) i; +INSERT INTO single_hash_repartition_second +SELECT i * 2, i * 5, i * 0.6 +FROM generate_series(0, 100) i; +-- a sample router query with NOT MATERIALIZED +-- which pushes down the filters to the CTE +SELECT public.coordinator_plan($Q$ +EXPLAIN (COSTS OFF) +WITH cte1 AS NOT MATERIALIZED +( + SELECT id + FROM single_hash_repartition_first t1 +) +SELECT count(*) +FROM cte1, single_hash_repartition_second +WHERE cte1.id = single_hash_repartition_second.id AND single_hash_repartition_second.id = 45; +$Q$); + coordinator_plan +------------------------------ + Custom Scan (Citus Adaptive) + Task Count: 1 +(2 rows) + +-- same query, without NOT MATERIALIZED, which is already default +-- which pushes down the filters to the CTE +SELECT public.coordinator_plan($Q$ +EXPLAIN (COSTS OFF) +WITH cte1 AS +( + SELECT id + FROM single_hash_repartition_first t1 +) +SELECT count(*) +FROM cte1, single_hash_repartition_second +WHERE cte1.id = single_hash_repartition_second.id AND single_hash_repartition_second.id = 45; +$Q$); + coordinator_plan +------------------------------ + Custom Scan (Citus Adaptive) + Task Count: 1 +(2 rows) + +-- same query with MATERIALIZED +-- which prevents pushing down filters to the CTE, +-- thus becoming a real-time query +SELECT public.coordinator_plan($Q$ +EXPLAIN (COSTS OFF) +WITH cte1 AS MATERIALIZED +( + SELECT id + FROM single_hash_repartition_first t1 +) +SELECT count(*) +FROM cte1, single_hash_repartition_second +WHERE cte1.id = single_hash_repartition_second.id AND single_hash_repartition_second.id = 45; +$Q$); + coordinator_plan +------------------------------------------ + Custom Scan (Citus Adaptive) + -> Distributed Subplan 5_1 + -> Custom Scan (Citus Adaptive) + Task Count: 4 +(4 rows) + +-- similar query with MATERIALIZED +-- now manually have the same filter in the CTE +-- thus becoming a router query again +SELECT public.coordinator_plan($Q$ +EXPLAIN (COSTS OFF) +WITH cte1 AS MATERIALIZED +( + SELECT id + FROM single_hash_repartition_first t1 + WHERE id = 45 +) +SELECT count(*) +FROM cte1, single_hash_repartition_second +WHERE cte1.id = single_hash_repartition_second.id AND single_hash_repartition_second.id = 45; +$Q$); + coordinator_plan +------------------------------ + Custom Scan (Citus Adaptive) + Task Count: 1 +(2 rows) + +-- now, have a real-time query without MATERIALIZED +-- these are sanitiy checks, because all of the CTEs are recursively +-- planned and there is no benefit that Citus can have +SELECT public.coordinator_plan($Q$ +EXPLAIN (COSTS OFF) +WITH cte1 AS MATERIALIZED +( + SELECT id + FROM single_hash_repartition_first t1 + WHERE sum = 45 +) +SELECT count(*) +FROM cte1, single_hash_repartition_second +WHERE cte1.id = single_hash_repartition_second.id AND single_hash_repartition_second.sum = 45; +$Q$); + coordinator_plan +------------------------------------------------ + Aggregate + -> Custom Scan (Citus Adaptive) + -> Distributed Subplan 8_1 + -> Custom Scan (Citus Adaptive) + Task Count: 4 +(5 rows) + +SELECT public.coordinator_plan($Q$ +EXPLAIN (COSTS OFF) +WITH cte1 AS NOT MATERIALIZED +( + SELECT id + FROM single_hash_repartition_first t1 + WHERE sum = 45 +) +SELECT count(*) +FROM cte1, single_hash_repartition_second +WHERE cte1.id = single_hash_repartition_second.id AND single_hash_repartition_second.sum = 45; +$Q$); + coordinator_plan +------------------------------------------------ + Aggregate + -> Custom Scan (Citus Adaptive) + -> Distributed Subplan 10_1 + -> Custom Scan (Citus Adaptive) + Task Count: 4 +(5 rows) + +-- Foreign keys to partition tables +CREATE TABLE collections_list ( + key bigint, + collection_id integer, + value numeric, + PRIMARY KEY(key, collection_id) +) PARTITION BY LIST (collection_id); +CREATE TABLE collections_list_0 + PARTITION OF collections_list (key, collection_id, value) + FOR VALUES IN ( 0 ); +CREATE TABLE collections_list_1 + PARTITION OF collections_list (key, collection_id, value) + FOR VALUES IN ( 1 ); +CREATE TABLE collection_users + (used_id integer, collection_id integer, key bigint); +ALTER TABLE collection_users + ADD CONSTRAINT collection_users_fkey FOREIGN KEY (key, collection_id) REFERENCES collections_list (key, collection_id); +-- sanity check for postgres +INSERT INTO collections_list VALUES (1, 0, '1.1'); +INSERT INTO collection_users VALUES (1, 0, 1); +-- should fail because of fkey +INSERT INTO collection_users VALUES (1, 1000, 1); +ERROR: insert or update on table "collection_users" violates foreign key constraint "collection_users_fkey" +DETAIL: Key (key, collection_id)=(1, 1000) is not present in table "collections_list". +SELECT create_distributed_table('collections_list', 'key'); +NOTICE: Copying data from local table... + create_distributed_table +-------------------------- + +(1 row) + +SELECT create_distributed_table('collection_users', 'key'); +NOTICE: Copying data from local table... + create_distributed_table +-------------------------- + +(1 row) + +-- should still fail because of fkey +INSERT INTO collection_users VALUES (1, 1000, 1); +ERROR: insert or update on table "collection_users_60024" violates foreign key constraint "collection_users_fkey_60024" +DETAIL: Key (key, collection_id)=(1, 1000) is not present in table "collections_list_60012". +CONTEXT: while executing command on localhost:57637 +-- whereas new record with partition should go through +INSERT INTO collections_list VALUES (2, 1, '1.2'); +INSERT INTO collection_users VALUES (5, 1, 2); +-- AND CHAIN +CREATE TABLE test (x int, y int); +INSERT INTO test (x,y) SELECT i,i*3 from generate_series(1, 100) i; +SELECT create_distributed_table('test', 'x'); +NOTICE: Copying data from local table... + create_distributed_table +-------------------------- + +(1 row) + +-- single shard queries with CHAIN +BEGIN; +UPDATE test SET y = 15 WHERE x = 1; +COMMIT AND CHAIN; +SELECT * FROM test WHERE x = 1; + x | y +---+---- + 1 | 15 +(1 row) + +COMMIT; +BEGIN; +UPDATE test SET y = 20 WHERE x = 1; +ROLLBACK AND CHAIN; +SELECT * FROM test WHERE x = 1; + x | y +---+---- + 1 | 15 +(1 row) + +COMMIT; +-- multi shard queries with CHAIN +BEGIN; +UPDATE test SET y = 25; +COMMIT AND CHAIN; +SELECT DISTINCT y FROM test; + y +---- + 25 +(1 row) + +COMMIT; +BEGIN; +UPDATE test SET y = 30; +ROLLBACK AND CHAIN; +SELECT DISTINCT y FROM test; + y +---- + 25 +(1 row) + +COMMIT; +-- does read only carry over? +BEGIN READ ONLY; +COMMIT AND CHAIN; +UPDATE test SET y = 35; +ERROR: cannot execute UPDATE in a read-only transaction +COMMIT; +SELECT DISTINCT y FROM test; + y +---- + 25 +(1 row) + +BEGIN READ ONLY; +ROLLBACK AND CHAIN; +UPDATE test SET y = 40; +ERROR: cannot execute UPDATE in a read-only transaction +COMMIT; +SELECT DISTINCT y FROM test; + y +---- + 25 +(1 row) + +\set VERBOSITY terse +drop schema test_pg12 cascade; +NOTICE: drop cascades to 11 other objects +\set VERBOSITY default +SET citus.shard_replication_factor to 2; diff --git a/src/test/regress/expected/pg12_1.out b/src/test/regress/expected/pg12_1.out new file mode 100644 index 000000000..9d92533f9 --- /dev/null +++ b/src/test/regress/expected/pg12_1.out @@ -0,0 +1,6 @@ +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int > 11 AS server_version_above_eleven +\gset +\if :server_version_above_eleven +\else +\q diff --git a/src/test/regress/multi_schedule b/src/test/regress/multi_schedule index 68b7c0957..c095f4e3f 100644 --- a/src/test/regress/multi_schedule +++ b/src/test/regress/multi_schedule @@ -56,7 +56,7 @@ test: multi_partitioning_utils multi_partitioning replicated_partitioned_table test: subquery_basics subquery_local_tables subquery_executors subquery_and_cte set_operations set_operation_and_local_tables test: subqueries_deep subquery_view subquery_partitioning subquery_complex_target_list subqueries_not_supported subquery_in_where test: non_colocated_leaf_subquery_joins non_colocated_subquery_joins non_colocated_join_order -test: subquery_prepared_statements +test: subquery_prepared_statements pg12 # ---------- # Miscellaneous tests to check our query planning behavior @@ -276,4 +276,3 @@ test: multi_task_string_size # connection encryption tests # --------- test: ssl_by_default - diff --git a/src/test/regress/sql/pg12.sql b/src/test/regress/sql/pg12.sql new file mode 100644 index 000000000..3054724e0 --- /dev/null +++ b/src/test/regress/sql/pg12.sql @@ -0,0 +1,262 @@ +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int > 11 AS server_version_above_eleven +\gset +\if :server_version_above_eleven +\else +\q +\endif + +SET citus.shard_replication_factor to 1; +SET citus.next_shard_id TO 60000; +SET citus.next_placement_id TO 60000; + +create schema test_pg12; +set search_path to test_pg12; + +CREATE FUNCTION blackhole_am_handler(internal) +RETURNS table_am_handler +AS 'citus' +LANGUAGE C; +CREATE ACCESS METHOD blackhole_am TYPE TABLE HANDLER blackhole_am_handler; + +create table test_am(id int, val int) using blackhole_am; +insert into test_am values (1, 1); +-- Custom table access methods should be rejected +select create_distributed_table('test_am','id'); + +-- Test generated columns +create table gen1 ( + id int, + val1 int, + val2 int GENERATED ALWAYS AS (val1 + 2) STORED +); +create table gen2 ( + id int, + val1 int, + val2 int GENERATED ALWAYS AS (val1 + 2) STORED +); + +insert into gen1 (id, val1) values (1,4),(3,6),(5,2),(7,2); +insert into gen2 (id, val1) values (1,4),(3,6),(5,2),(7,2); + +select * from create_distributed_table('gen1', 'id'); +select * from create_distributed_table('gen2', 'val2'); + +insert into gen1 (id, val1) values (2,4),(4,6),(6,2),(8,2); +insert into gen2 (id, val1) values (2,4),(4,6),(6,2),(8,2); + +select * from gen1; +select * from gen2; + +-- Test new VACUUM/ANALYZE options +analyze (skip_locked) gen1; +vacuum (skip_locked) gen1; +vacuum (truncate 0) gen1; +vacuum (index_cleanup 1) gen1; + +-- COPY FROM +create table cptest (id int, val int); +select create_distributed_table('cptest', 'id'); +copy cptest from STDIN with csv where val < 4; +1,6 +2,3 +3,2 +4,9 +5,4 +\. +select sum(id), sum(val) from cptest; + +-- CTE materialized/not materialized +CREATE TABLE single_hash_repartition_first (id int, sum int, avg float); +CREATE TABLE single_hash_repartition_second (id int primary key, sum int, avg float); + +SELECT create_distributed_table('single_hash_repartition_first', 'id'); +SELECT create_distributed_table('single_hash_repartition_second', 'id'); + +INSERT INTO single_hash_repartition_first +SELECT i, i * 3, i * 0.3 +FROM generate_series(0, 100) i; + +INSERT INTO single_hash_repartition_second +SELECT i * 2, i * 5, i * 0.6 +FROM generate_series(0, 100) i; + +-- a sample router query with NOT MATERIALIZED +-- which pushes down the filters to the CTE +SELECT public.coordinator_plan($Q$ +EXPLAIN (COSTS OFF) +WITH cte1 AS NOT MATERIALIZED +( + SELECT id + FROM single_hash_repartition_first t1 +) +SELECT count(*) +FROM cte1, single_hash_repartition_second +WHERE cte1.id = single_hash_repartition_second.id AND single_hash_repartition_second.id = 45; +$Q$); + +-- same query, without NOT MATERIALIZED, which is already default +-- which pushes down the filters to the CTE +SELECT public.coordinator_plan($Q$ +EXPLAIN (COSTS OFF) +WITH cte1 AS +( + SELECT id + FROM single_hash_repartition_first t1 +) +SELECT count(*) +FROM cte1, single_hash_repartition_second +WHERE cte1.id = single_hash_repartition_second.id AND single_hash_repartition_second.id = 45; +$Q$); + +-- same query with MATERIALIZED +-- which prevents pushing down filters to the CTE, +-- thus becoming a real-time query +SELECT public.coordinator_plan($Q$ +EXPLAIN (COSTS OFF) +WITH cte1 AS MATERIALIZED +( + SELECT id + FROM single_hash_repartition_first t1 +) +SELECT count(*) +FROM cte1, single_hash_repartition_second +WHERE cte1.id = single_hash_repartition_second.id AND single_hash_repartition_second.id = 45; +$Q$); + +-- similar query with MATERIALIZED +-- now manually have the same filter in the CTE +-- thus becoming a router query again +SELECT public.coordinator_plan($Q$ +EXPLAIN (COSTS OFF) +WITH cte1 AS MATERIALIZED +( + SELECT id + FROM single_hash_repartition_first t1 + WHERE id = 45 +) +SELECT count(*) +FROM cte1, single_hash_repartition_second +WHERE cte1.id = single_hash_repartition_second.id AND single_hash_repartition_second.id = 45; +$Q$); + +-- now, have a real-time query without MATERIALIZED +-- these are sanitiy checks, because all of the CTEs are recursively +-- planned and there is no benefit that Citus can have +SELECT public.coordinator_plan($Q$ +EXPLAIN (COSTS OFF) +WITH cte1 AS MATERIALIZED +( + SELECT id + FROM single_hash_repartition_first t1 + WHERE sum = 45 +) +SELECT count(*) +FROM cte1, single_hash_repartition_second +WHERE cte1.id = single_hash_repartition_second.id AND single_hash_repartition_second.sum = 45; +$Q$); + +SELECT public.coordinator_plan($Q$ +EXPLAIN (COSTS OFF) +WITH cte1 AS NOT MATERIALIZED +( + SELECT id + FROM single_hash_repartition_first t1 + WHERE sum = 45 +) +SELECT count(*) +FROM cte1, single_hash_repartition_second +WHERE cte1.id = single_hash_repartition_second.id AND single_hash_repartition_second.sum = 45; +$Q$); + +-- Foreign keys to partition tables +CREATE TABLE collections_list ( + key bigint, + collection_id integer, + value numeric, + PRIMARY KEY(key, collection_id) +) PARTITION BY LIST (collection_id); + +CREATE TABLE collections_list_0 + PARTITION OF collections_list (key, collection_id, value) + FOR VALUES IN ( 0 ); +CREATE TABLE collections_list_1 + PARTITION OF collections_list (key, collection_id, value) + FOR VALUES IN ( 1 ); + +CREATE TABLE collection_users + (used_id integer, collection_id integer, key bigint); + +ALTER TABLE collection_users + ADD CONSTRAINT collection_users_fkey FOREIGN KEY (key, collection_id) REFERENCES collections_list (key, collection_id); + +-- sanity check for postgres +INSERT INTO collections_list VALUES (1, 0, '1.1'); +INSERT INTO collection_users VALUES (1, 0, 1); + +-- should fail because of fkey +INSERT INTO collection_users VALUES (1, 1000, 1); + +SELECT create_distributed_table('collections_list', 'key'); +SELECT create_distributed_table('collection_users', 'key'); + +-- should still fail because of fkey +INSERT INTO collection_users VALUES (1, 1000, 1); + +-- whereas new record with partition should go through +INSERT INTO collections_list VALUES (2, 1, '1.2'); +INSERT INTO collection_users VALUES (5, 1, 2); + +-- AND CHAIN +CREATE TABLE test (x int, y int); +INSERT INTO test (x,y) SELECT i,i*3 from generate_series(1, 100) i; + +SELECT create_distributed_table('test', 'x'); + +-- single shard queries with CHAIN +BEGIN; +UPDATE test SET y = 15 WHERE x = 1; +COMMIT AND CHAIN; +SELECT * FROM test WHERE x = 1; +COMMIT; + +BEGIN; +UPDATE test SET y = 20 WHERE x = 1; +ROLLBACK AND CHAIN; +SELECT * FROM test WHERE x = 1; +COMMIT; + +-- multi shard queries with CHAIN +BEGIN; +UPDATE test SET y = 25; +COMMIT AND CHAIN; +SELECT DISTINCT y FROM test; +COMMIT; + +BEGIN; +UPDATE test SET y = 30; +ROLLBACK AND CHAIN; +SELECT DISTINCT y FROM test; +COMMIT; + +-- does read only carry over? + +BEGIN READ ONLY; +COMMIT AND CHAIN; +UPDATE test SET y = 35; +COMMIT; +SELECT DISTINCT y FROM test; + +BEGIN READ ONLY; +ROLLBACK AND CHAIN; +UPDATE test SET y = 40; +COMMIT; +SELECT DISTINCT y FROM test; + + +\set VERBOSITY terse +drop schema test_pg12 cascade; +\set VERBOSITY default + +SET citus.shard_replication_factor to 2; + From 6b0d8ed83d7b95d711546b712f087579ab484860 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philip=20Dub=C3=A9?= Date: Thu, 22 Aug 2019 17:30:41 +0000 Subject: [PATCH 12/12] SortList in FinalizedShardPlacementList, makes 3 failure tests consistent between 11/12 --- .../master/master_metadata_utility.c | 33 ++++++- .../distributed/test/distribution_metadata.c | 34 ------- .../distributed/master_metadata_utility.h | 2 + .../expected/failure_1pc_copy_append.out | 45 +++------ .../expected/failure_1pc_copy_append_9.out | 45 ++------- .../expected/failure_1pc_copy_hash.out | 6 +- .../expected/failure_single_select.out | 99 +++++++++---------- .../expected/failure_single_select_9.out | 99 ++++++++++--------- .../regress/sql/failure_1pc_copy_append.sql | 8 -- .../regress/sql/failure_single_select.sql | 40 ++++---- 10 files changed, 179 insertions(+), 232 deletions(-) diff --git a/src/backend/distributed/master/master_metadata_utility.c b/src/backend/distributed/master/master_metadata_utility.c index eeaf8b4ed..00a6e215f 100644 --- a/src/backend/distributed/master/master_metadata_utility.c +++ b/src/backend/distributed/master/master_metadata_utility.c @@ -30,11 +30,13 @@ #include "commands/extension.h" #include "distributed/connection_management.h" #include "distributed/citus_nodes.h" +#include "distributed/listutils.h" #include "distributed/master_metadata_utility.h" #include "distributed/master_protocol.h" #include "distributed/metadata_cache.h" #include "distributed/multi_join_order.h" #include "distributed/multi_logical_optimizer.h" +#include "distributed/multi_physical_planner.h" #include "distributed/pg_dist_colocation.h" #include "distributed/pg_dist_partition.h" #include "distributed/pg_dist_shard.h" @@ -408,6 +410,35 @@ ErrorIfNotSuitableToGetSize(Oid relationId) } +/* + * CompareShardPlacementsByWorker compares two shard placements by their + * worker node name and port. + */ +int +CompareShardPlacementsByWorker(const void *leftElement, const void *rightElement) +{ + const ShardPlacement *leftPlacement = *((const ShardPlacement **) leftElement); + const ShardPlacement *rightPlacement = *((const ShardPlacement **) rightElement); + + int nodeNameCmp = strncmp(leftPlacement->nodeName, rightPlacement->nodeName, + WORKER_LENGTH); + if (nodeNameCmp != 0) + { + return nodeNameCmp; + } + else if (leftPlacement->nodePort > rightPlacement->nodePort) + { + return 1; + } + else if (leftPlacement->nodePort < rightPlacement->nodePort) + { + return -1; + } + + return 0; +} + + /* * TableShardReplicationFactor returns the current replication factor of the * given relation by looking into shard placements. It errors out if there @@ -693,7 +724,7 @@ FinalizedShardPlacementList(uint64 shardId) } } - return finalizedPlacementList; + return SortList(finalizedPlacementList, CompareShardPlacementsByWorker); } diff --git a/src/backend/distributed/test/distribution_metadata.c b/src/backend/distributed/test/distribution_metadata.c index 8df36d56f..ad3f5aee9 100644 --- a/src/backend/distributed/test/distribution_metadata.c +++ b/src/backend/distributed/test/distribution_metadata.c @@ -40,11 +40,6 @@ #include "utils/palloc.h" -/* forward declaration of local functions */ -static int CompareShardPlacementsByWorker(const void *leftElement, - const void *rightElement); - - /* declarations for dynamic loading */ PG_FUNCTION_INFO_V1(load_shard_id_array); PG_FUNCTION_INFO_V1(load_shard_interval_array); @@ -171,35 +166,6 @@ load_shard_placement_array(PG_FUNCTION_ARGS) } -/* - * CompareShardPlacementsByWorker compares two shard placements by their - * worker node name and port. - */ -static int -CompareShardPlacementsByWorker(const void *leftElement, const void *rightElement) -{ - const ShardPlacement *leftPlacement = *((const ShardPlacement **) leftElement); - const ShardPlacement *rightPlacement = *((const ShardPlacement **) rightElement); - - int nodeNameCmp = strncmp(leftPlacement->nodeName, rightPlacement->nodeName, - WORKER_LENGTH); - if (nodeNameCmp != 0) - { - return nodeNameCmp; - } - else if (leftPlacement->nodePort > rightPlacement->nodePort) - { - return 1; - } - else if (leftPlacement->nodePort < rightPlacement->nodePort) - { - return -1; - } - - return 0; -} - - /* * partition_column_id simply finds a distributed table using the provided Oid * and returns the column_id of its partition column. If the specified table is diff --git a/src/include/distributed/master_metadata_utility.h b/src/include/distributed/master_metadata_utility.h index 9a714c3b0..c388d417e 100644 --- a/src/include/distributed/master_metadata_utility.h +++ b/src/include/distributed/master_metadata_utility.h @@ -147,6 +147,8 @@ extern char * ConstructQualifiedShardName(ShardInterval *shardInterval); extern uint64 GetFirstShardId(Oid relationId); extern Datum StringToDatum(char *inputString, Oid dataType); extern char * DatumToString(Datum datum, Oid dataType); +extern int CompareShardPlacementsByWorker(const void *leftElement, + const void *rightElement); #endif /* MASTER_METADATA_UTILITY_H */ diff --git a/src/test/regress/expected/failure_1pc_copy_append.out b/src/test/regress/expected/failure_1pc_copy_append.out index 797ea7171..44d80cc52 100644 --- a/src/test/regress/expected/failure_1pc_copy_append.out +++ b/src/test/regress/expected/failure_1pc_copy_append.out @@ -52,8 +52,8 @@ SELECT citus.dump_network_traffic(); (0,worker,"[""RowDescription(fieldcount=2,fields=['F(name=min,tableoid=0,colattrnum=0,typoid=23,typlen=4,typmod=-1,format_code=0)', 'F(name=max,tableoid=0,colattrnum=0,typoid=23,typlen=4,typmod=-1,format_code=0)'])"", 'DataRow(columncount=2,columns=[""C(length=0,value=b\\'\\')"", ""C(length=1,value=b\\'0\\')""])', 'CommandComplete(command=SELECT 1)', 'ReadyForQuery(state=in_transaction_block)']") (0,coordinator,"['Query(query=COMMIT)']") (0,worker,"['CommandComplete(command=COMMIT)', 'ReadyForQuery(state=idle)']") - (0,coordinator,"['Query(query=COPY (SELECT count(1) AS count FROM copy_test_100400 copy_test WHERE true) TO STDOUT)']") - (0,worker,"[""CopyOutResponse(format=0,columncount=1,columns=['Anonymous(format=0)'])"", ""CopyData(data=b'4\\\\n')"", 'CopyDone()', 'CommandComplete(command=COPY 1)', 'ReadyForQuery(state=idle)']") + (0,coordinator,"['Query(query=SELECT count(1) AS count FROM copy_test_100400 copy_test WHERE true)']") + (0,worker,"[""RowDescription(fieldcount=1,fields=['F(name=count,tableoid=0,colattrnum=0,typoid=20,typlen=8,typmod=-1,format_code=0)'])"", 'DataRow(columncount=1,columns=[""C(length=0,value=b\\'\\')""])', 'CommandComplete(command=SELECT 1)', 'ReadyForQuery(state=idle)']") (20 rows) ---- all of the following tests test behavior with 2 shard placements ---- @@ -168,7 +168,10 @@ SELECT citus.mitmproxy('conn.onQuery(query="SELECT|COPY").kill()'); (1 row) SELECT count(1) FROM copy_test; -WARNING: could not consume data from worker node +WARNING: connection error: localhost:9060 +DETAIL: server closed the connection unexpectedly + This probably means the server terminated abnormally + before or while processing the request. count ------- 4 @@ -224,26 +227,6 @@ SELECT count(1) FROM copy_test; 4 (1 row) --- we round-robin when picking which node to run pg_table_size on, this COPY runs it on --- the other node, so the next copy will try to run it on our node -COPY copy_test FROM PROGRAM 'echo 0, 0 && echo 1, 1 && echo 2, 4 && echo 3, 9' WITH CSV; -SELECT * FROM pg_dist_shard s, pg_dist_shard_placement p - WHERE (s.shardid = p.shardid) AND s.logicalrelid = 'copy_test'::regclass - ORDER BY p.nodeport, p.placementid; - logicalrelid | shardid | shardstorage | shardminvalue | shardmaxvalue | shardid | shardstate | shardlength | nodename | nodeport | placementid ---------------+---------+--------------+---------------+---------------+---------+------------+-------------+-----------+----------+------------- - copy_test | 100400 | t | 0 | 3 | 100400 | 1 | 8192 | localhost | 9060 | 101 - copy_test | 100407 | t | 0 | 3 | 100407 | 1 | 8192 | localhost | 9060 | 110 - copy_test | 100400 | t | 0 | 3 | 100400 | 1 | 8192 | localhost | 57637 | 100 - copy_test | 100407 | t | 0 | 3 | 100407 | 1 | 8192 | localhost | 57637 | 111 -(4 rows) - -SELECT count(1) FROM copy_test; - count -------- - 8 -(1 row) - ---- kill the connection when we try to get the min, max of the table ---- SELECT citus.mitmproxy('conn.onQuery(query="SELECT min\(key\), max\(key\)").kill()'); mitmproxy @@ -266,14 +249,12 @@ SELECT * FROM pg_dist_shard s, pg_dist_shard_placement p --------------+---------+--------------+---------------+---------------+---------+------------+-------------+-----------+----------+------------- copy_test | 100400 | t | 0 | 3 | 100400 | 1 | 8192 | localhost | 57637 | 100 copy_test | 100400 | t | 0 | 3 | 100400 | 1 | 8192 | localhost | 9060 | 101 - copy_test | 100407 | t | 0 | 3 | 100407 | 1 | 8192 | localhost | 9060 | 110 - copy_test | 100407 | t | 0 | 3 | 100407 | 1 | 8192 | localhost | 57637 | 111 -(4 rows) +(2 rows) SELECT count(1) FROM copy_test; count ------- - 8 + 4 (1 row) ---- kill the connection when we try to COMMIT ---- @@ -296,16 +277,14 @@ SELECT * FROM pg_dist_shard s, pg_dist_shard_placement p --------------+---------+--------------+---------------+---------------+---------+------------+-------------+-----------+----------+------------- copy_test | 100400 | t | 0 | 3 | 100400 | 1 | 8192 | localhost | 57637 | 100 copy_test | 100400 | t | 0 | 3 | 100400 | 1 | 8192 | localhost | 9060 | 101 - copy_test | 100407 | t | 0 | 3 | 100407 | 1 | 8192 | localhost | 9060 | 110 - copy_test | 100407 | t | 0 | 3 | 100407 | 1 | 8192 | localhost | 57637 | 111 - copy_test | 100409 | t | 0 | 3 | 100409 | 3 | 8192 | localhost | 9060 | 114 - copy_test | 100409 | t | 0 | 3 | 100409 | 1 | 8192 | localhost | 57637 | 115 -(6 rows) + copy_test | 100408 | t | 0 | 3 | 100408 | 1 | 8192 | localhost | 57637 | 112 + copy_test | 100408 | t | 0 | 3 | 100408 | 3 | 8192 | localhost | 9060 | 113 +(4 rows) SELECT count(1) FROM copy_test; count ------- - 12 + 8 (1 row) -- ==== Clean up, we're done here ==== diff --git a/src/test/regress/expected/failure_1pc_copy_append_9.out b/src/test/regress/expected/failure_1pc_copy_append_9.out index 554d8baee..c9ac2c2d7 100644 --- a/src/test/regress/expected/failure_1pc_copy_append_9.out +++ b/src/test/regress/expected/failure_1pc_copy_append_9.out @@ -52,8 +52,8 @@ SELECT citus.dump_network_traffic(); (0,worker,"[""RowDescription(fieldcount=2,fields=['F(name=min,tableoid=0,colattrnum=0,typoid=23,typlen=4,typmod=-1,format_code=0)', 'F(name=max,tableoid=0,colattrnum=0,typoid=23,typlen=4,typmod=-1,format_code=0)'])"", 'DataRow(columncount=2,columns=[""C(length=0,value=b\\'\\')"", ""C(length=1,value=b\\'0\\')""])', 'CommandComplete(command=SELECT 1)', 'ReadyForQuery(state=in_transaction_block)']") (0,coordinator,"['Query(query=COMMIT)']") (0,worker,"['CommandComplete(command=COMMIT)', 'ReadyForQuery(state=idle)']") - (0,coordinator,"['Query(query=SELECT count(1) AS count FROM copy_test_100400 copy_test WHERE true)']") - (0,worker,"[""RowDescription(fieldcount=1,fields=['F(name=count,tableoid=0,colattrnum=0,typoid=20,typlen=8,typmod=-1,format_code=0)'])"", 'DataRow(columncount=1,columns=[""C(length=0,value=b\\'\\')""])', 'CommandComplete(command=SELECT 1)', 'ReadyForQuery(state=idle)']") + (0,coordinator,"['Query(query=COPY (SELECT count(1) AS count FROM copy_test_100400 copy_test WHERE true) TO STDOUT)']") + (0,worker,"[""CopyOutResponse(format=0,columncount=1,columns=['Anonymous(format=0)'])"", ""CopyData(data=b'4\\\\n')"", 'CopyDone()', 'CommandComplete(command=COPY 1)', 'ReadyForQuery(state=idle)']") (20 rows) ---- all of the following tests test behavior with 2 shard placements ---- @@ -168,10 +168,7 @@ SELECT citus.mitmproxy('conn.onQuery(query="SELECT|COPY").kill()'); (1 row) SELECT count(1) FROM copy_test; -WARNING: connection error: localhost:9060 -DETAIL: server closed the connection unexpectedly - This probably means the server terminated abnormally - before or while processing the request. +WARNING: could not consume data from worker node count ------- 4 @@ -227,26 +224,6 @@ SELECT count(1) FROM copy_test; 4 (1 row) --- we round-robin when picking which node to run pg_table_size on, this COPY runs it on --- the other node, so the next copy will try to run it on our node -COPY copy_test FROM PROGRAM 'echo 0, 0 && echo 1, 1 && echo 2, 4 && echo 3, 9' WITH CSV; -SELECT * FROM pg_dist_shard s, pg_dist_shard_placement p - WHERE (s.shardid = p.shardid) AND s.logicalrelid = 'copy_test'::regclass - ORDER BY p.nodeport, p.placementid; - logicalrelid | shardid | shardstorage | shardminvalue | shardmaxvalue | shardid | shardstate | shardlength | nodename | nodeport | placementid ---------------+---------+--------------+---------------+---------------+---------+------------+-------------+-----------+----------+------------- - copy_test | 100400 | t | 0 | 3 | 100400 | 1 | 8192 | localhost | 9060 | 101 - copy_test | 100407 | t | 0 | 3 | 100407 | 1 | 8192 | localhost | 9060 | 110 - copy_test | 100400 | t | 0 | 3 | 100400 | 1 | 8192 | localhost | 57637 | 100 - copy_test | 100407 | t | 0 | 3 | 100407 | 1 | 8192 | localhost | 57637 | 111 -(4 rows) - -SELECT count(1) FROM copy_test; - count -------- - 8 -(1 row) - ---- kill the connection when we try to get the min, max of the table ---- SELECT citus.mitmproxy('conn.onQuery(query="SELECT min\(key\), max\(key\)").kill()'); mitmproxy @@ -269,14 +246,12 @@ SELECT * FROM pg_dist_shard s, pg_dist_shard_placement p --------------+---------+--------------+---------------+---------------+---------+------------+-------------+-----------+----------+------------- copy_test | 100400 | t | 0 | 3 | 100400 | 1 | 8192 | localhost | 57637 | 100 copy_test | 100400 | t | 0 | 3 | 100400 | 1 | 8192 | localhost | 9060 | 101 - copy_test | 100407 | t | 0 | 3 | 100407 | 1 | 8192 | localhost | 9060 | 110 - copy_test | 100407 | t | 0 | 3 | 100407 | 1 | 8192 | localhost | 57637 | 111 -(4 rows) +(2 rows) SELECT count(1) FROM copy_test; count ------- - 8 + 4 (1 row) ---- kill the connection when we try to COMMIT ---- @@ -299,16 +274,14 @@ SELECT * FROM pg_dist_shard s, pg_dist_shard_placement p --------------+---------+--------------+---------------+---------------+---------+------------+-------------+-----------+----------+------------- copy_test | 100400 | t | 0 | 3 | 100400 | 1 | 8192 | localhost | 57637 | 100 copy_test | 100400 | t | 0 | 3 | 100400 | 1 | 8192 | localhost | 9060 | 101 - copy_test | 100407 | t | 0 | 3 | 100407 | 1 | 8192 | localhost | 9060 | 110 - copy_test | 100407 | t | 0 | 3 | 100407 | 1 | 8192 | localhost | 57637 | 111 - copy_test | 100409 | t | 0 | 3 | 100409 | 3 | 8192 | localhost | 9060 | 114 - copy_test | 100409 | t | 0 | 3 | 100409 | 1 | 8192 | localhost | 57637 | 115 -(6 rows) + copy_test | 100408 | t | 0 | 3 | 100408 | 1 | 8192 | localhost | 57637 | 112 + copy_test | 100408 | t | 0 | 3 | 100408 | 3 | 8192 | localhost | 9060 | 113 +(4 rows) SELECT count(1) FROM copy_test; count ------- - 12 + 8 (1 row) -- ==== Clean up, we're done here ==== diff --git a/src/test/regress/expected/failure_1pc_copy_hash.out b/src/test/regress/expected/failure_1pc_copy_hash.out index f888b2b43..0e4d97ec2 100644 --- a/src/test/regress/expected/failure_1pc_copy_hash.out +++ b/src/test/regress/expected/failure_1pc_copy_hash.out @@ -45,7 +45,11 @@ SELECT citus.dump_network_traffic(); (0,worker,"['CommandComplete(command=COPY 4)', 'ReadyForQuery(state=in_transaction_block)']") (0,coordinator,"['Query(query=COMMIT)']") (0,worker,"['CommandComplete(command=COMMIT)', 'ReadyForQuery(state=idle)']") -(10 rows) + (1,coordinator,"[initial message]") + (1,worker,"['AuthenticationOk()', 'ParameterStatus(application_name=citus)', 'ParameterStatus(client_encoding=UTF8)', 'ParameterStatus(DateStyle=ISO, MDY)', 'ParameterStatus(integer_datetimes=on)', 'ParameterStatus(IntervalStyle=postgres)', 'ParameterStatus(is_superuser=on)', 'ParameterStatus(server_encoding=UTF8)', 'ParameterStatus(server_version=XXX)', 'ParameterStatus(session_authorization=postgres)', 'ParameterStatus(standard_conforming_strings=on)', 'ParameterStatus(TimeZone=XXX)', 'BackendKeyData(XXX)', 'ReadyForQuery(state=idle)']") + (1,coordinator,"['Query(query=SELECT count(1) AS count FROM public.copy_test_XXXXXX copy_test)']") + (1,worker,"[""RowDescription(fieldcount=1,fields=['F(name=count,tableoid=0,colattrnum=0,typoid=20,typlen=8,typmod=-1,format_code=0)'])"", 'DataRow(columncount=1,columns=[""C(length=0,value=b\\'\\')""])', 'CommandComplete(command=SELECT 1)', 'ReadyForQuery(state=idle)']") +(14 rows) -- ==== kill the connection when we try to start a transaction ==== -- the query should abort diff --git a/src/test/regress/expected/failure_single_select.out b/src/test/regress/expected/failure_single_select.out index c974e5036..2f07f7f5c 100644 --- a/src/test/regress/expected/failure_single_select.out +++ b/src/test/regress/expected/failure_single_select.out @@ -20,31 +20,31 @@ SELECT create_distributed_table('select_test', 'key'); (1 row) -- put data in shard for which mitm node is first placement -INSERT INTO select_test VALUES (2, 'test data'); +INSERT INTO select_test VALUES (3, 'test data'); SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").kill()'); mitmproxy ----------- (1 row) -SELECT * FROM select_test WHERE key = 2; -WARNING: server closed the connection unexpectedly +SELECT * FROM select_test WHERE key = 3; +WARNING: connection error: localhost:9060 +DETAIL: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. -CONTEXT: while executing command on localhost:9060 key | value -----+----------- - 2 | test data + 3 | test data (1 row) -SELECT * FROM select_test WHERE key = 2; -WARNING: server closed the connection unexpectedly +SELECT * FROM select_test WHERE key = 3; +WARNING: connection error: localhost:9060 +DETAIL: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. -CONTEXT: while executing command on localhost:9060 key | value -----+----------- - 2 | test data + 3 | test data (1 row) -- kill after first SELECT; txn should work (though placement marked bad) @@ -55,34 +55,32 @@ SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").kill()'); (1 row) BEGIN; -INSERT INTO select_test VALUES (2, 'more data'); -SELECT * FROM select_test WHERE key = 2; -WARNING: server closed the connection unexpectedly +INSERT INTO select_test VALUES (3, 'more data'); +SELECT * FROM select_test WHERE key = 3; +WARNING: connection error: localhost:9060 +DETAIL: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. -CONTEXT: while executing command on localhost:9060 key | value -----+----------- - 2 | test data - 2 | more data + 3 | test data + 3 | more data (2 rows) -INSERT INTO select_test VALUES (2, 'even more data'); -SELECT * FROM select_test WHERE key = 2; +INSERT INTO select_test VALUES (3, 'even more data'); +SELECT * FROM select_test WHERE key = 3; +WARNING: connection error: localhost:9060 +DETAIL: server closed the connection unexpectedly + This probably means the server terminated abnormally + before or while processing the request. key | value -----+---------------- - 2 | test data - 2 | more data - 2 | even more data + 3 | test data + 3 | more data + 3 | even more data (3 rows) COMMIT; -WARNING: connection not open -CONTEXT: while executing command on localhost:9060 -WARNING: connection not open -CONTEXT: while executing command on localhost:9060 -WARNING: connection not open -CONTEXT: while executing command on localhost:9060 -- some clean up UPDATE pg_dist_shard_placement SET shardstate = 1 WHERE shardid IN ( @@ -91,16 +89,16 @@ WHERE shardid IN ( TRUNCATE select_test; -- now the same tests with query cancellation -- put data in shard for which mitm node is first placement -INSERT INTO select_test VALUES (2, 'test data'); +INSERT INTO select_test VALUES (3, 'test data'); SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").cancel(' || pg_backend_pid() || ')'); mitmproxy ----------- (1 row) -SELECT * FROM select_test WHERE key = 2; +SELECT * FROM select_test WHERE key = 3; ERROR: canceling statement due to user request -SELECT * FROM select_test WHERE key = 2; +SELECT * FROM select_test WHERE key = 3; ERROR: canceling statement due to user request -- cancel after first SELECT; txn should fail and nothing should be marked as invalid SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").cancel(' || pg_backend_pid() || ')'); @@ -110,8 +108,8 @@ SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").cancel(' || pg_backend_pi (1 row) BEGIN; -INSERT INTO select_test VALUES (2, 'more data'); -SELECT * FROM select_test WHERE key = 2; +INSERT INTO select_test VALUES (3, 'more data'); +SELECT * FROM select_test WHERE key = 3; ERROR: canceling statement due to user request COMMIT; -- show that all placements are OK @@ -134,15 +132,15 @@ SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").after(1).cancel(' || pg_b (1 row) BEGIN; -INSERT INTO select_test VALUES (2, 'more data'); -SELECT * FROM select_test WHERE key = 2; +INSERT INTO select_test VALUES (3, 'more data'); +SELECT * FROM select_test WHERE key = 3; key | value -----+----------- - 2 | more data + 3 | more data (1 row) -INSERT INTO select_test VALUES (2, 'even more data'); -SELECT * FROM select_test WHERE key = 2; +INSERT INTO select_test VALUES (3, 'even more data'); +SELECT * FROM select_test WHERE key = 3; ERROR: canceling statement due to user request COMMIT; -- error after second SELECT; txn should work (though placement marked bad) @@ -153,32 +151,26 @@ SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").after(1).reset()'); (1 row) BEGIN; -INSERT INTO select_test VALUES (2, 'more data'); -SELECT * FROM select_test WHERE key = 2; +INSERT INTO select_test VALUES (3, 'more data'); +SELECT * FROM select_test WHERE key = 3; key | value -----+----------- - 2 | more data + 3 | more data (1 row) -INSERT INTO select_test VALUES (2, 'even more data'); -SELECT * FROM select_test WHERE key = 2; -WARNING: server closed the connection unexpectedly +INSERT INTO select_test VALUES (3, 'even more data'); +SELECT * FROM select_test WHERE key = 3; +WARNING: connection error: localhost:9060 +DETAIL: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. -CONTEXT: while executing command on localhost:9060 key | value -----+---------------- - 2 | more data - 2 | even more data + 3 | more data + 3 | even more data (2 rows) COMMIT; -WARNING: connection not open -CONTEXT: while executing command on localhost:9060 -WARNING: connection not open -CONTEXT: while executing command on localhost:9060 -WARNING: connection not open -CONTEXT: while executing command on localhost:9060 SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").after(2).kill()'); mitmproxy ----------- @@ -223,11 +215,10 @@ SELECT * FROM select_test WHERE key = 1; (1 row) SELECT * FROM select_test WHERE key = 1; -WARNING: server closed the connection unexpectedly +ERROR: connection error: localhost:9060 +DETAIL: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. -CONTEXT: while executing command on localhost:9060 -ERROR: could not receive query results -- now the same test with query cancellation SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").after(1).cancel(' || pg_backend_pid() || ')'); mitmproxy diff --git a/src/test/regress/expected/failure_single_select_9.out b/src/test/regress/expected/failure_single_select_9.out index 419777a00..fbe351d40 100644 --- a/src/test/regress/expected/failure_single_select_9.out +++ b/src/test/regress/expected/failure_single_select_9.out @@ -20,31 +20,31 @@ SELECT create_distributed_table('select_test', 'key'); (1 row) -- put data in shard for which mitm node is first placement -INSERT INTO select_test VALUES (2, 'test data'); +INSERT INTO select_test VALUES (3, 'test data'); SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").kill()'); mitmproxy ----------- (1 row) -SELECT * FROM select_test WHERE key = 2; -WARNING: connection error: localhost:9060 -DETAIL: server closed the connection unexpectedly +SELECT * FROM select_test WHERE key = 3; +WARNING: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. +CONTEXT: while executing command on localhost:9060 key | value -----+----------- - 2 | test data + 3 | test data (1 row) -SELECT * FROM select_test WHERE key = 2; -WARNING: connection error: localhost:9060 -DETAIL: server closed the connection unexpectedly +SELECT * FROM select_test WHERE key = 3; +WARNING: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. +CONTEXT: while executing command on localhost:9060 key | value -----+----------- - 2 | test data + 3 | test data (1 row) -- kill after first SELECT; txn should work (though placement marked bad) @@ -55,32 +55,34 @@ SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").kill()'); (1 row) BEGIN; -INSERT INTO select_test VALUES (2, 'more data'); -SELECT * FROM select_test WHERE key = 2; -WARNING: connection error: localhost:9060 -DETAIL: server closed the connection unexpectedly +INSERT INTO select_test VALUES (3, 'more data'); +SELECT * FROM select_test WHERE key = 3; +WARNING: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. +CONTEXT: while executing command on localhost:9060 key | value -----+----------- - 2 | test data - 2 | more data + 3 | test data + 3 | more data (2 rows) -INSERT INTO select_test VALUES (2, 'even more data'); -SELECT * FROM select_test WHERE key = 2; -WARNING: connection error: localhost:9060 -DETAIL: server closed the connection unexpectedly - This probably means the server terminated abnormally - before or while processing the request. +INSERT INTO select_test VALUES (3, 'even more data'); +SELECT * FROM select_test WHERE key = 3; key | value -----+---------------- - 2 | test data - 2 | more data - 2 | even more data + 3 | test data + 3 | more data + 3 | even more data (3 rows) COMMIT; +WARNING: connection not open +CONTEXT: while executing command on localhost:9060 +WARNING: connection not open +CONTEXT: while executing command on localhost:9060 +WARNING: connection not open +CONTEXT: while executing command on localhost:9060 -- some clean up UPDATE pg_dist_shard_placement SET shardstate = 1 WHERE shardid IN ( @@ -89,16 +91,16 @@ WHERE shardid IN ( TRUNCATE select_test; -- now the same tests with query cancellation -- put data in shard for which mitm node is first placement -INSERT INTO select_test VALUES (2, 'test data'); +INSERT INTO select_test VALUES (3, 'test data'); SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").cancel(' || pg_backend_pid() || ')'); mitmproxy ----------- (1 row) -SELECT * FROM select_test WHERE key = 2; +SELECT * FROM select_test WHERE key = 3; ERROR: canceling statement due to user request -SELECT * FROM select_test WHERE key = 2; +SELECT * FROM select_test WHERE key = 3; ERROR: canceling statement due to user request -- cancel after first SELECT; txn should fail and nothing should be marked as invalid SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").cancel(' || pg_backend_pid() || ')'); @@ -108,8 +110,8 @@ SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").cancel(' || pg_backend_pi (1 row) BEGIN; -INSERT INTO select_test VALUES (2, 'more data'); -SELECT * FROM select_test WHERE key = 2; +INSERT INTO select_test VALUES (3, 'more data'); +SELECT * FROM select_test WHERE key = 3; ERROR: canceling statement due to user request COMMIT; -- show that all placements are OK @@ -132,15 +134,15 @@ SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").after(1).cancel(' || pg_b (1 row) BEGIN; -INSERT INTO select_test VALUES (2, 'more data'); -SELECT * FROM select_test WHERE key = 2; +INSERT INTO select_test VALUES (3, 'more data'); +SELECT * FROM select_test WHERE key = 3; key | value -----+----------- - 2 | more data + 3 | more data (1 row) -INSERT INTO select_test VALUES (2, 'even more data'); -SELECT * FROM select_test WHERE key = 2; +INSERT INTO select_test VALUES (3, 'even more data'); +SELECT * FROM select_test WHERE key = 3; ERROR: canceling statement due to user request COMMIT; -- error after second SELECT; txn should work (though placement marked bad) @@ -151,26 +153,32 @@ SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").after(1).reset()'); (1 row) BEGIN; -INSERT INTO select_test VALUES (2, 'more data'); -SELECT * FROM select_test WHERE key = 2; +INSERT INTO select_test VALUES (3, 'more data'); +SELECT * FROM select_test WHERE key = 3; key | value -----+----------- - 2 | more data + 3 | more data (1 row) -INSERT INTO select_test VALUES (2, 'even more data'); -SELECT * FROM select_test WHERE key = 2; -WARNING: connection error: localhost:9060 -DETAIL: server closed the connection unexpectedly +INSERT INTO select_test VALUES (3, 'even more data'); +SELECT * FROM select_test WHERE key = 3; +WARNING: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. +CONTEXT: while executing command on localhost:9060 key | value -----+---------------- - 2 | more data - 2 | even more data + 3 | more data + 3 | even more data (2 rows) COMMIT; +WARNING: connection not open +CONTEXT: while executing command on localhost:9060 +WARNING: connection not open +CONTEXT: while executing command on localhost:9060 +WARNING: connection not open +CONTEXT: while executing command on localhost:9060 SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").after(2).kill()'); mitmproxy ----------- @@ -215,10 +223,11 @@ SELECT * FROM select_test WHERE key = 1; (1 row) SELECT * FROM select_test WHERE key = 1; -ERROR: connection error: localhost:9060 -DETAIL: server closed the connection unexpectedly +WARNING: server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. +CONTEXT: while executing command on localhost:9060 +ERROR: could not receive query results -- now the same test with query cancellation SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").after(1).cancel(' || pg_backend_pid() || ')'); mitmproxy diff --git a/src/test/regress/sql/failure_1pc_copy_append.sql b/src/test/regress/sql/failure_1pc_copy_append.sql index 71d698567..b2740a8b9 100644 --- a/src/test/regress/sql/failure_1pc_copy_append.sql +++ b/src/test/regress/sql/failure_1pc_copy_append.sql @@ -72,14 +72,6 @@ SELECT * FROM pg_dist_shard s, pg_dist_shard_placement p ORDER BY placementid; SELECT count(1) FROM copy_test; --- we round-robin when picking which node to run pg_table_size on, this COPY runs it on --- the other node, so the next copy will try to run it on our node -COPY copy_test FROM PROGRAM 'echo 0, 0 && echo 1, 1 && echo 2, 4 && echo 3, 9' WITH CSV; -SELECT * FROM pg_dist_shard s, pg_dist_shard_placement p - WHERE (s.shardid = p.shardid) AND s.logicalrelid = 'copy_test'::regclass - ORDER BY p.nodeport, p.placementid; -SELECT count(1) FROM copy_test; - ---- kill the connection when we try to get the min, max of the table ---- SELECT citus.mitmproxy('conn.onQuery(query="SELECT min\(key\), max\(key\)").kill()'); COPY copy_test FROM PROGRAM 'echo 0, 0 && echo 1, 1 && echo 2, 4 && echo 3, 9' WITH CSV; diff --git a/src/test/regress/sql/failure_single_select.sql b/src/test/regress/sql/failure_single_select.sql index 565df402d..d6e86b649 100644 --- a/src/test/regress/sql/failure_single_select.sql +++ b/src/test/regress/sql/failure_single_select.sql @@ -8,20 +8,20 @@ CREATE TABLE select_test (key int, value text); SELECT create_distributed_table('select_test', 'key'); -- put data in shard for which mitm node is first placement -INSERT INTO select_test VALUES (2, 'test data'); +INSERT INTO select_test VALUES (3, 'test data'); SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").kill()'); -SELECT * FROM select_test WHERE key = 2; -SELECT * FROM select_test WHERE key = 2; +SELECT * FROM select_test WHERE key = 3; +SELECT * FROM select_test WHERE key = 3; -- kill after first SELECT; txn should work (though placement marked bad) SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").kill()'); BEGIN; -INSERT INTO select_test VALUES (2, 'more data'); -SELECT * FROM select_test WHERE key = 2; -INSERT INTO select_test VALUES (2, 'even more data'); -SELECT * FROM select_test WHERE key = 2; +INSERT INTO select_test VALUES (3, 'more data'); +SELECT * FROM select_test WHERE key = 3; +INSERT INTO select_test VALUES (3, 'even more data'); +SELECT * FROM select_test WHERE key = 3; COMMIT; -- some clean up @@ -34,18 +34,18 @@ TRUNCATE select_test; -- now the same tests with query cancellation -- put data in shard for which mitm node is first placement -INSERT INTO select_test VALUES (2, 'test data'); +INSERT INTO select_test VALUES (3, 'test data'); SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").cancel(' || pg_backend_pid() || ')'); -SELECT * FROM select_test WHERE key = 2; -SELECT * FROM select_test WHERE key = 2; +SELECT * FROM select_test WHERE key = 3; +SELECT * FROM select_test WHERE key = 3; -- cancel after first SELECT; txn should fail and nothing should be marked as invalid SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").cancel(' || pg_backend_pid() || ')'); BEGIN; -INSERT INTO select_test VALUES (2, 'more data'); -SELECT * FROM select_test WHERE key = 2; +INSERT INTO select_test VALUES (3, 'more data'); +SELECT * FROM select_test WHERE key = 3; COMMIT; -- show that all placements are OK @@ -60,20 +60,20 @@ TRUNCATE select_test; SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").after(1).cancel(' || pg_backend_pid() || ')'); BEGIN; -INSERT INTO select_test VALUES (2, 'more data'); -SELECT * FROM select_test WHERE key = 2; -INSERT INTO select_test VALUES (2, 'even more data'); -SELECT * FROM select_test WHERE key = 2; +INSERT INTO select_test VALUES (3, 'more data'); +SELECT * FROM select_test WHERE key = 3; +INSERT INTO select_test VALUES (3, 'even more data'); +SELECT * FROM select_test WHERE key = 3; COMMIT; -- error after second SELECT; txn should work (though placement marked bad) SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").after(1).reset()'); BEGIN; -INSERT INTO select_test VALUES (2, 'more data'); -SELECT * FROM select_test WHERE key = 2; -INSERT INTO select_test VALUES (2, 'even more data'); -SELECT * FROM select_test WHERE key = 2; +INSERT INTO select_test VALUES (3, 'more data'); +SELECT * FROM select_test WHERE key = 3; +INSERT INTO select_test VALUES (3, 'even more data'); +SELECT * FROM select_test WHERE key = 3; COMMIT; SELECT citus.mitmproxy('conn.onQuery(query="^SELECT").after(2).kill()');