external
/
citus
mirror of https://github.com/citusdata/citus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #823 from citusdata/update_postgresql_files 

Update PostgreSQL-sourced files with latest changes

cr: @anarazel
pull/824/head
Jason Petersen 2016-09-30 16:06:28 -06:00 committed by GitHub
parent 9a5b844a81 f59cf2b818
commit d52ebffce5
7 changed files with 84 additions and 452 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
.gitattributes vendored
View File

@ -27,16 +27,6 @@ configure -whitespace
# except these exceptions...
src/backend/distributed/utils/citus_outfuncs.c -citus-style
src/backend/distributed/utils/citus_read.c -citus-style
src/backend/distributed/utils/citus_readfuncs_94.c -citus-style
src/backend/distributed/utils/citus_readfuncs_95.c -citus-style
src/backend/distributed/utils/ruleutils_94.c -citus-style
src/backend/distributed/utils/ruleutils_95.c -citus-style
src/include/distributed/citus_nodes.h -citus-style
src/include/dumputils.h -citus-style
# all csql files use PostgreSQL style...
src/bin/csql/*.[ch] -citus-style
# except these exceptions
src/bin/csql/copy_options.c citus-style
src/bin/csql/stage.[ch] citus-style

116
src/backend/distributed/utils/ruleutils_95.c
View File

@ -1,6 +1,6 @@
/*-------------------------------------------------------------------------
*
* citus_ruleutils.c
* ruleutils_95.c
* Additional, non core exposed, functions to convert stored
* expressions/querytrees back to source text
*
@ -9,7 +9,7 @@
*
*
* IDENTIFICATION
* src/backend/distributed/utils/citus_ruleutils.c
* src/backend/distributed/utils/ruleutils_95.c
*
* This needs to be closely in sync with the core code.
*-------------------------------------------------------------------------
@ -3062,25 +3062,24 @@ get_insert_query_def(Query *query, deparse_context *context)
/* Add a WHERE clause (for partial indexes) if given */
if (confl->arbiterWhere != NULL)
{
bool varprefixInitialValue = context->varprefix;
bool save_varprefix;
/*
* Force non-prefixing of Vars, since parser assumes that they
* belong to target relation. WHERE clause does not use
* InferenceElem, so this is separately required.
*/
save_varprefix = context->varprefix;
context->varprefix = false;
appendContextKeyword(context, " WHERE ",
-PRETTYINDENT_STD, PRETTYINDENT_STD, 1);
/*
* Postgres deparses arbiter WHERE clauses incorrectly. It adds
* varprefix to the arbiter WHERE clause, for which Postgres parser
* errors out. Thus, we temporarily set varprefix to false.
*/
context->varprefix = false;
get_rule_expr(confl->arbiterWhere, context, false);
/* now set the varprefix back to its initial value */
context->varprefix = varprefixInitialValue;
context->varprefix = save_varprefix;
}
}
else if (confl->constraint != InvalidOid)
else if (OidIsValid(confl->constraint))
{
char *constraint = get_constraint_name(confl->constraint);
int64 shardId = context->shardid;
@ -3090,8 +3089,11 @@ get_insert_query_def(Query *query, deparse_context *context)
AppendShardIdToName(&constraint, shardId);
}
if (!constraint)
elog(ERROR, "cache lookup failed for constraint %u",
confl->constraint);
appendStringInfo(buf, " ON CONSTRAINT %s",
quote_qualified_identifier(NULL, constraint));
quote_identifier(constraint));
}
if (confl->action == ONCONFLICT_NOTHING)
@ -3592,7 +3594,8 @@ get_variable(Var *var, int levelsup, bool istoplevel, deparse_context *context)
tle = get_tle_by_resno(dpns->inner_tlist, var->varattno);
if (!tle)
elog(ERROR, "bogus varattno for subquery var: %d", var->varattno);
elog(ERROR, "invalid attnum %d for relation \"%s\"",
var->varattno, rte->eref->aliasname);
Assert(netlevelsup == 0);
push_child_plan(dpns, dpns->inner_planstate, &save_dpns);
@ -3653,9 +3656,13 @@ get_variable(Var *var, int levelsup, bool istoplevel, deparse_context *context)
else if (attnum > 0)
{
/* Get column name to use from the colinfo struct */
Assert(attnum <= colinfo->num_cols);
if (attnum > colinfo->num_cols)
elog(ERROR, "invalid attnum %d for relation \"%s\"",
attnum, rte->eref->aliasname);
attname = colinfo->colnames[attnum - 1];
Assert(attname != NULL);
if (attname == NULL) /* dropped column? */
elog(ERROR, "invalid attnum %d for relation \"%s\"",
attnum, rte->eref->aliasname);
}
else if (GetRangeTblKind(rte) == CITUS_RTE_SHARD)
{
@ -4791,6 +4798,24 @@ get_rule_expr(Node *node, deparse_context *context,
get_base_element_type(exprType(arg2))),
expr->useOr ? "ANY" : "ALL");
get_rule_expr_paren(arg2, context, true, node);
/*
* There's inherent ambiguity in "x op ANY/ALL (y)" when y is
* a bare sub-SELECT. Since we're here, the sub-SELECT must
* be meant as a scalar sub-SELECT yielding an array value to
* be used in ScalarArrayOpExpr; but the grammar will
* preferentially interpret such a construct as an ANY/ALL
* SubLink. To prevent misparsing the output that way, insert
* a dummy coercion (which will be stripped by parse analysis,
* so no inefficiency is added in dump and reload). This is
* indeed most likely what the user wrote to get the construct
* accepted in the first place.
*/
if (IsA(arg2, SubLink) &&
((SubLink *) arg2)->subLinkType == EXPR_SUBLINK)
appendStringInfo(buf, "::%s",
format_type_with_typemod(exprType(arg2),
exprTypmod(arg2)));
appendStringInfoChar(buf, ')');
if (!PRETTY_PAREN(context))
appendStringInfoChar(buf, ')');
@ -5452,17 +5477,43 @@ get_rule_expr(Node *node, deparse_context *context,
if (!PRETTY_PAREN(context))
appendStringInfoChar(buf, '(');
get_rule_expr_paren((Node *) ntest->arg, context, true, node);
switch (ntest->nulltesttype)
/*
* For scalar inputs, we prefer to print as IS [NOT] NULL,
* which is shorter and traditional. If it's a rowtype input
* but we're applying a scalar test, must print IS [NOT]
* DISTINCT FROM NULL to be semantically correct.
*/
if (ntest->argisrow ||
!type_is_rowtype(exprType((Node *) ntest->arg)))
{
case IS_NULL:
appendStringInfoString(buf, " IS NULL");
break;
case IS_NOT_NULL:
appendStringInfoString(buf, " IS NOT NULL");
break;
default:
elog(ERROR, "unrecognized nulltesttype: %d",
(int) ntest->nulltesttype);
switch (ntest->nulltesttype)
{
case IS_NULL:
appendStringInfoString(buf, " IS NULL");
break;
case IS_NOT_NULL:
appendStringInfoString(buf, " IS NOT NULL");
break;
default:
elog(ERROR, "unrecognized nulltesttype: %d",
(int) ntest->nulltesttype);
}
}
else
{
switch (ntest->nulltesttype)
{
case IS_NULL:
appendStringInfoString(buf, " IS NOT DISTINCT FROM NULL");
break;
case IS_NOT_NULL:
appendStringInfoString(buf, " IS DISTINCT FROM NULL");
break;
default:
elog(ERROR, "unrecognized nulltesttype: %d",
(int) ntest->nulltesttype);
}
}
if (!PRETTY_PAREN(context))
appendStringInfoChar(buf, ')');
@ -5550,13 +5601,14 @@ get_rule_expr(Node *node, deparse_context *context,
case T_InferenceElem:
{
InferenceElem *iexpr = (InferenceElem *) node;
bool varprefix = context->varprefix;
bool save_varprefix;
bool need_parens;
/*
* InferenceElem can only refer to target relation, so a
* prefix is never useful.
* prefix is not useful, and indeed would cause parse errors.
*/
save_varprefix = context->varprefix;
context->varprefix = false;
/*
@ -5576,7 +5628,7 @@ get_rule_expr(Node *node, deparse_context *context,
if (need_parens)
appendStringInfoChar(buf, ')');
context->varprefix = varprefix;
context->varprefix = save_varprefix;
if (iexpr->infercollid)
appendStringInfo(buf, " COLLATE %s",
@ -7331,4 +7383,4 @@ generate_operator_name(Oid operid, Oid arg1, Oid arg2)
return buf.data;
}
#endif
#endif /* (PG_VERSION_NUM >= 90500 && PG_VERSION_NUM < 90600) */

73
src/include/dumputils.h
View File

@ -1,73 +0,0 @@
/*-------------------------------------------------------------------------
*
* Utility routines for SQL dumping
* Basically this is stuff that is useful in both pg_dump and pg_dumpall.
* Lately it's also being used by psql and bin/scripts/ ...
*
*
* Portions Copyright (c) 1996-2014, PostgreSQL Global Development Group
* Portions Copyright (c) 1994, Regents of the University of California
*
* src/bin/pg_dump/dumputils.h
*
*-------------------------------------------------------------------------
*/
#ifndef DUMPUTILS_H
#define DUMPUTILS_H
#include "libpq-fe.h"
#include "pqexpbuffer.h"
typedef struct SimpleStringListCell
{
struct SimpleStringListCell *next;
char val[1]; /* VARIABLE LENGTH FIELD */
} SimpleStringListCell;
typedef struct SimpleStringList
{
SimpleStringListCell *head;
SimpleStringListCell *tail;
} SimpleStringList;
extern int quote_all_identifiers;
extern PQExpBuffer (*getLocalPQExpBuffer) (void);
extern const char *fmtId(const char *identifier);
extern const char *fmtQualifiedId(int remoteVersion,
const char *schema, const char *id);
extern void appendStringLiteral(PQExpBuffer buf, const char *str,
int encoding, bool std_strings);
extern void appendStringLiteralConn(PQExpBuffer buf, const char *str,
PGconn *conn);
extern void appendStringLiteralDQ(PQExpBuffer buf, const char *str,
const char *dqprefix);
extern void appendByteaLiteral(PQExpBuffer buf,
const unsigned char *str, size_t length,
bool std_strings);
extern bool parsePGArray(const char *atext, char ***itemarray, int *nitems);
extern bool buildACLCommands(const char *name, const char *subname,
const char *type, const char *acls, const char *owner,
const char *prefix, int remoteVersion,
PQExpBuffer sql);
extern bool buildDefaultACLCommands(const char *type, const char *nspname,
const char *acls, const char *owner,
int remoteVersion,
PQExpBuffer sql);
extern bool processSQLNamePattern(PGconn *conn, PQExpBuffer buf,
const char *pattern,
bool have_where, bool force_escape,
const char *schemavar, const char *namevar,
const char *altnamevar, const char *visibilityrule);
extern void buildShSecLabelQuery(PGconn *conn, const char *catalog_name,
uint32 objectId, PQExpBuffer sql);
extern void emitShSecLabels(PGconn *conn, PGresult *res,
PQExpBuffer buffer, const char *target, const char *objname);
extern void set_dump_section(const char *arg, int *dumpSections);
extern void simple_string_list_append(SimpleStringList *list, const char *val);
extern bool simple_string_list_member(SimpleStringList *list, const char *val);
#endif /* DUMPUTILS_H */

329
src/test/regress/expected/multi_hash_pruning_0.out
View File

@ -1,329 +0,0 @@
--
-- MULTI_HASH_PRUNING
--
-- Tests for shard and join pruning logic on hash partitioned tables.
ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 630000;
ALTER SEQUENCE pg_catalog.pg_dist_jobid_seq RESTART 630000;
-- Create a table partitioned on integer column and update partition type to
-- hash. Then load data into this table and update shard min max values with
-- hashed ones. Hash value of 1, 2, 3 and 4 are consecutively -1905060026,
-- 1134484726, -28094569 and -1011077333.
CREATE TABLE orders_hash_partitioned (
o_orderkey integer,
o_custkey integer,
o_orderstatus char(1),
o_totalprice decimal(15,2),
o_orderdate date,
o_orderpriority char(15),
o_clerk char(15),
o_shippriority integer,
o_comment varchar(79) );
SELECT master_create_distributed_table('orders_hash_partitioned', 'o_orderkey', 'hash');
master_create_distributed_table
---------------------------------
(1 row)
SELECT master_create_worker_shards('orders_hash_partitioned', 4, 1);
master_create_worker_shards
-----------------------------
(1 row)
SET client_min_messages TO DEBUG2;
-- Check that we can prune shards for simple cases, boolean expressions and
-- immutable functions.
-- Since router plans are not triggered for task-tracker executor type,
-- we need to run the tests that triggers router planning seperately for
-- both executors. Otherwise, check-full fails on the task-tracker.
-- Later, we need to switch back to the actual task executor
-- to contuinue with correct executor type for check-full.
SELECT quote_literal(current_setting('citus.task_executor_type')) AS actual_task_executor
\gset
SET citus.task_executor_type TO 'real-time';
SELECT count(*) FROM orders_hash_partitioned;
count
-------
0
(1 row)
SELECT count(*) FROM orders_hash_partitioned WHERE o_orderkey = 1;
DEBUG: predicate pruning for shardId 630001
DEBUG: predicate pruning for shardId 630002
DEBUG: predicate pruning for shardId 630003
DEBUG: Creating router plan
DEBUG: Plan is router executable
count
-------
0
(1 row)
SELECT count(*) FROM orders_hash_partitioned WHERE o_orderkey = 2;
DEBUG: predicate pruning for shardId 630000
DEBUG: predicate pruning for shardId 630001
DEBUG: predicate pruning for shardId 630002
DEBUG: Creating router plan
DEBUG: Plan is router executable
count
-------
0
(1 row)
SELECT count(*) FROM orders_hash_partitioned WHERE o_orderkey = 3;
DEBUG: predicate pruning for shardId 630000
DEBUG: predicate pruning for shardId 630002
DEBUG: predicate pruning for shardId 630003
DEBUG: Creating router plan
DEBUG: Plan is router executable
count
-------
0
(1 row)
SELECT count(*) FROM orders_hash_partitioned WHERE o_orderkey = 4;
DEBUG: predicate pruning for shardId 630000
DEBUG: predicate pruning for shardId 630002
DEBUG: predicate pruning for shardId 630003
DEBUG: Creating router plan
DEBUG: Plan is router executable
count
-------
0
(1 row)
SELECT count(*) FROM orders_hash_partitioned
WHERE o_orderkey = 1 AND o_clerk = 'aaa';
DEBUG: predicate pruning for shardId 630001
DEBUG: predicate pruning for shardId 630002
DEBUG: predicate pruning for shardId 630003
DEBUG: Creating router plan
DEBUG: Plan is router executable
count
-------
0
(1 row)
SELECT count(*) FROM orders_hash_partitioned WHERE o_orderkey = abs(-1);
DEBUG: predicate pruning for shardId 630001
DEBUG: predicate pruning for shardId 630002
DEBUG: predicate pruning for shardId 630003
DEBUG: Creating router plan
DEBUG: Plan is router executable
count
-------
0
(1 row)
SET citus.task_executor_type TO 'task-tracker';
SELECT count(*) FROM orders_hash_partitioned;
count
-------
0
(1 row)
SELECT count(*) FROM orders_hash_partitioned WHERE o_orderkey = 1;
DEBUG: predicate pruning for shardId 630001
DEBUG: predicate pruning for shardId 630002
DEBUG: predicate pruning for shardId 630003
count
-------
0
(1 row)
SELECT count(*) FROM orders_hash_partitioned WHERE o_orderkey = 2;
DEBUG: predicate pruning for shardId 630000
DEBUG: predicate pruning for shardId 630001
DEBUG: predicate pruning for shardId 630002
count
-------
0
(1 row)
SELECT count(*) FROM orders_hash_partitioned WHERE o_orderkey = 3;
DEBUG: predicate pruning for shardId 630000
DEBUG: predicate pruning for shardId 630002
DEBUG: predicate pruning for shardId 630003
count
-------
0
(1 row)
SELECT count(*) FROM orders_hash_partitioned WHERE o_orderkey = 4;
DEBUG: predicate pruning for shardId 630000
DEBUG: predicate pruning for shardId 630002
DEBUG: predicate pruning for shardId 630003
count
-------
0
(1 row)
SELECT count(*) FROM orders_hash_partitioned
WHERE o_orderkey = 1 AND o_clerk = 'aaa';
DEBUG: predicate pruning for shardId 630001
DEBUG: predicate pruning for shardId 630002
DEBUG: predicate pruning for shardId 630003
count
-------
0
(1 row)
SELECT count(*) FROM orders_hash_partitioned WHERE o_orderkey = abs(-1);
DEBUG: predicate pruning for shardId 630001
DEBUG: predicate pruning for shardId 630002
DEBUG: predicate pruning for shardId 630003
count
-------
0
(1 row)
SET citus.task_executor_type TO :actual_task_executor;
SELECT count(*) FROM orders_hash_partitioned WHERE o_orderkey is NULL;
count
-------
0
(1 row)
SELECT count(*) FROM orders_hash_partitioned WHERE o_orderkey is not NULL;
count
-------
0
(1 row)
SELECT count(*) FROM orders_hash_partitioned WHERE o_orderkey > 2;
count
-------
0
(1 row)
SELECT count(*) FROM orders_hash_partitioned
WHERE o_orderkey = 1 OR o_orderkey = 2;
DEBUG: predicate pruning for shardId 630001
DEBUG: predicate pruning for shardId 630002
count
-------
0
(1 row)
SELECT count(*) FROM orders_hash_partitioned
WHERE o_orderkey = 1 OR o_clerk = 'aaa';
count
-------
0
(1 row)
SELECT count(*) FROM orders_hash_partitioned
WHERE o_orderkey = 1 OR (o_orderkey = 3 AND o_clerk = 'aaa');
DEBUG: predicate pruning for shardId 630002
DEBUG: predicate pruning for shardId 630003
count
-------
0
(1 row)
SELECT count(*) FROM orders_hash_partitioned
WHERE o_orderkey = 1 OR o_orderkey is NULL;
count
-------
0
(1 row)
SELECT count(*) FROM
(SELECT o_orderkey FROM orders_hash_partitioned WHERE o_orderkey = 1) AS orderkeys;
DEBUG: predicate pruning for shardId 630001
DEBUG: predicate pruning for shardId 630002
DEBUG: predicate pruning for shardId 630003
count
-------
0
(1 row)
-- Check that we don't support pruning for ANY (array expression) and give
-- a notice message when used with the partition column
SELECT count(*) FROM orders_hash_partitioned
WHERE o_orderkey = ANY ('{1,2,3}');
NOTICE: cannot use shard pruning with ANY/ALL (array expression)
HINT: Consider rewriting the expression with OR/AND clauses.
count
-------
0
(1 row)
-- Check that we don't show the message if the operator is not
-- equality operator
SELECT count(*) FROM orders_hash_partitioned
WHERE o_orderkey < ALL ('{1,2,3}');
count
-------
0
(1 row)
-- Check that we don't give a spurious hint message when non-partition
-- columns are used with ANY/IN/ALL
SELECT count(*) FROM orders_hash_partitioned
WHERE o_orderkey = 1 OR o_totalprice IN (2, 5);
count
-------
0
(1 row)
-- Check that we cannot prune for mutable functions.
SELECT count(*) FROM orders_hash_partitioned WHERE o_orderkey = random();
count
-------
0
(1 row)
SELECT count(*) FROM orders_hash_partitioned
WHERE o_orderkey = random() OR o_orderkey = 1;
count
-------
0
(1 row)
SELECT count(*) FROM orders_hash_partitioned
WHERE o_orderkey = random() AND o_orderkey = 1;
DEBUG: predicate pruning for shardId 630001
DEBUG: predicate pruning for shardId 630002
DEBUG: predicate pruning for shardId 630003
count
-------
0
(1 row)
-- Check that we can do join pruning.
SELECT count(*)
FROM orders_hash_partitioned orders1, orders_hash_partitioned orders2
WHERE orders1.o_orderkey = orders2.o_orderkey;
DEBUG: join prunable for intervals [-2147483648,-1073741825] and [-1073741824,-1]
DEBUG: join prunable for intervals [-2147483648,-1073741825] and [0,1073741823]
DEBUG: join prunable for intervals [-2147483648,-1073741825] and [1073741824,2147483647]
DEBUG: join prunable for intervals [-1073741824,-1] and [-2147483648,-1073741825]
DEBUG: join prunable for intervals [-1073741824,-1] and [0,1073741823]
DEBUG: join prunable for intervals [-1073741824,-1] and [1073741824,2147483647]
DEBUG: join prunable for intervals [0,1073741823] and [-2147483648,-1073741825]
DEBUG: join prunable for intervals [0,1073741823] and [-1073741824,-1]
DEBUG: join prunable for intervals [0,1073741823] and [1073741824,2147483647]
DEBUG: join prunable for intervals [1073741824,2147483647] and [-2147483648,-1073741825]
DEBUG: join prunable for intervals [1073741824,2147483647] and [-1073741824,-1]
DEBUG: join prunable for intervals [1073741824,2147483647] and [0,1073741823]
count
-------
0
(1 row)
SELECT count(*)
FROM orders_hash_partitioned orders1, orders_hash_partitioned orders2
WHERE orders1.o_orderkey = orders2.o_orderkey
AND orders1.o_orderkey = 1
AND orders2.o_orderkey is NULL;
DEBUG: predicate pruning for shardId 630001
DEBUG: predicate pruning for shardId 630002
DEBUG: predicate pruning for shardId 630003
DEBUG: join prunable for intervals [-2147483648,-1073741825] and [-1073741824,-1]
DEBUG: join prunable for intervals [-2147483648,-1073741825] and [0,1073741823]
DEBUG: join prunable for intervals [-2147483648,-1073741825] and [1073741824,2147483647]
count
-------
0
(1 row)

2
src/test/regress/expected/multi_upsert.out
View File

@ -1,6 +1,4 @@
-- this test file aims to test UPSERT feature on Citus
-- note that output of this file for postgresql 9.4 will
-- be full syntax errors, which is expected.
ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 980000;
ALTER SEQUENCE pg_catalog.pg_dist_jobid_seq RESTART 980000;
CREATE TABLE upsert_test

4
src/test/regress/pg_regress_multi.pl Normal file → Executable file
View File

@ -298,10 +298,6 @@ if ($majorversion eq '9.5')
{
push(@arguments, '--bindir', "tmp_check/tmp-bin");
}
elsif ($majorversion eq '9.4')
{
push(@arguments, '--psqldir', "tmp_check/tmp-bin");
}
else
{
die "Citus is not compatible with the detected PostgreSQL version $majorversion";

2
src/test/regress/sql/multi_upsert.sql
View File

@ -1,6 +1,4 @@
-- this test file aims to test UPSERT feature on Citus
-- note that output of this file for postgresql 9.4 will
-- be full syntax errors, which is expected.
ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 980000;