From bbda64aaa1e7ace91ad9e75bc8ff8aa857d77e0f Mon Sep 17 00:00:00 2001 From: Jason Petersen Date: Wed, 10 Feb 2016 14:58:46 -0700 Subject: [PATCH 01/19] Handle hash-partitioned aliased data types When two data types have the same binary representation, PostgreSQL may add an implicit coercion between them by wrapping a node in a relabel type. This wrapper signals that the wrapped value is completely binary compatible with the designated "final type" of the relabel node. As an example, the varchar type is often relabeled to text, since functions provided for use with text (comparisons, hashes, etc.) are completely compatible with varchar as well. The hash-partitioned codepath contains functions that verify queries actually contain an equality constraint on the partition column, but those functions expect such constraints to be comparison operations between a Var and Const. The RelabelType wrapper node causes these functions to always return false, which bypasses shard pruning. --- .../distributed/planner/multi_physical_planner.c | 8 ++++++++ src/test/regress/expected/multi_data_types.out | 12 ++++++------ src/test/regress/sql/multi_data_types.sql | 8 ++++---- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/backend/distributed/planner/multi_physical_planner.c b/src/backend/distributed/planner/multi_physical_planner.c index 1b4172d3a..5e497970e 100644 --- a/src/backend/distributed/planner/multi_physical_planner.c +++ b/src/backend/distributed/planner/multi_physical_planner.c @@ -2808,6 +2808,10 @@ SimpleOpExpression(Expr *clause) return false; /* not a binary opclause */ } + /* strip coercions before doing check */ + leftOperand = strip_implicit_coercions(leftOperand); + rightOperand = strip_implicit_coercions(rightOperand); + if (IsA(rightOperand, Const) && IsA(leftOperand, Var)) { constantClause = (Const *) rightOperand; @@ -2919,6 +2923,10 @@ OpExpressionContainsColumn(OpExpr *operatorExpression, Var *partitionColumn) Node *rightOperand = get_rightop((Expr *) operatorExpression); Var *column = NULL; + /* strip coercions before doing check */ + leftOperand = strip_implicit_coercions(leftOperand); + rightOperand = strip_implicit_coercions(rightOperand); + if (IsA(leftOperand, Var)) { column = (Var *) leftOperand; diff --git a/src/test/regress/expected/multi_data_types.out b/src/test/regress/expected/multi_data_types.out index 8dc78f841..18b7a54dc 100644 --- a/src/test/regress/expected/multi_data_types.out +++ b/src/test/regress/expected/multi_data_types.out @@ -121,7 +121,7 @@ CREATE TABLE varchar_hash_partitioned_table id int, name varchar ); -SELECT master_create_distributed_table('varchar_hash_partitioned_table', 'id', 'hash'); +SELECT master_create_distributed_table('varchar_hash_partitioned_table', 'name', 'hash'); master_create_distributed_table --------------------------------- @@ -139,16 +139,16 @@ INSERT INTO varchar_hash_partitioned_table VALUES (2, 'Ozgun'); INSERT INTO varchar_hash_partitioned_table VALUES (3, 'Onder'); INSERT INTO varchar_hash_partitioned_table VALUES (4, 'Sumedh'); INSERT INTO varchar_hash_partitioned_table VALUES (5, 'Marco'); -SELECT * FROM varchar_hash_partitioned_table WHERE name = 'Onder'; +SELECT * FROM varchar_hash_partitioned_table WHERE id = 1; id | name ----+------- - 3 | Onder + 1 | Jason (1 row) -UPDATE varchar_hash_partitioned_table SET name = 'Samay' WHERE id = 5; -SELECT * FROM varchar_hash_partitioned_table WHERE name = 'Samay'; +UPDATE varchar_hash_partitioned_table SET id = 6 WHERE name = 'Jason'; +SELECT * FROM varchar_hash_partitioned_table WHERE id = 6; id | name ----+------- - 5 | Samay + 6 | Jason (1 row) diff --git a/src/test/regress/sql/multi_data_types.sql b/src/test/regress/sql/multi_data_types.sql index 99c603e63..68b84044f 100644 --- a/src/test/regress/sql/multi_data_types.sql +++ b/src/test/regress/sql/multi_data_types.sql @@ -104,7 +104,7 @@ CREATE TABLE varchar_hash_partitioned_table name varchar ); -SELECT master_create_distributed_table('varchar_hash_partitioned_table', 'id', 'hash'); +SELECT master_create_distributed_table('varchar_hash_partitioned_table', 'name', 'hash'); SELECT master_create_worker_shards('varchar_hash_partitioned_table', 4, 1); -- execute INSERT, SELECT and UPDATE queries on composite_type_partitioned_table @@ -114,8 +114,8 @@ INSERT INTO varchar_hash_partitioned_table VALUES (3, 'Onder'); INSERT INTO varchar_hash_partitioned_table VALUES (4, 'Sumedh'); INSERT INTO varchar_hash_partitioned_table VALUES (5, 'Marco'); -SELECT * FROM varchar_hash_partitioned_table WHERE name = 'Onder'; +SELECT * FROM varchar_hash_partitioned_table WHERE id = 1; -UPDATE varchar_hash_partitioned_table SET name = 'Samay' WHERE id = 5; +UPDATE varchar_hash_partitioned_table SET id = 6 WHERE name = 'Jason'; -SELECT * FROM varchar_hash_partitioned_table WHERE name = 'Samay'; +SELECT * FROM varchar_hash_partitioned_table WHERE id = 6; From ef7206348d1d0cdf9f63c19a2dfeecbf666f18e2 Mon Sep 17 00:00:00 2001 From: Jason Petersen Date: Thu, 11 Feb 2016 16:09:35 -0700 Subject: [PATCH 02/19] Allow DML commands on append-partitioned tables This entirely removes any restriction on the type of partitioning during DML planning and execution. Though there aren't actually any technical limitations preventing DML commands against append- (or even range-) partitioned tables, we had initially forbidden this, as any future stage operation could cause shards to overlap, banning all subsequent DML operations to partition values contained within more than one shards. This ended up mostly restricting us, so we're now removing that restriction. --- .../distributed/planner/modify_planner.c | 10 ------- .../regress/expected/multi_modifications.out | 29 ++++++++++++++----- .../regress/expected/multi_simple_queries.out | 26 ++++++++--------- src/test/regress/sql/multi_modifications.sql | 21 ++++++++++---- 4 files changed, 49 insertions(+), 37 deletions(-) diff --git a/src/backend/distributed/planner/modify_planner.c b/src/backend/distributed/planner/modify_planner.c index 15a08ea16..faeb6b6b0 100644 --- a/src/backend/distributed/planner/modify_planner.c +++ b/src/backend/distributed/planner/modify_planner.c @@ -121,16 +121,6 @@ ErrorIfQueryNotSupported(Query *queryTree) Assert(commandType == CMD_INSERT || commandType == CMD_UPDATE || commandType == CMD_DELETE); - if (!(partitionMethod == DISTRIBUTE_BY_HASH || - partitionMethod == DISTRIBUTE_BY_RANGE)) - { - ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot perform distributed planning for the given" - " modification"), - errdetail("Only hash- or range-partitioned tables may be the " - "target of distributed modifications"))); - } - /* * Reject subqueries which are in SELECT or WHERE clause. * Queries which include subqueries in FROM clauses are rejected below. diff --git a/src/test/regress/expected/multi_modifications.out b/src/test/regress/expected/multi_modifications.out index 5da80026b..d48c47551 100644 --- a/src/test/regress/expected/multi_modifications.out +++ b/src/test/regress/expected/multi_modifications.out @@ -61,10 +61,14 @@ SELECT master_create_empty_shard('range_partitioned') AS new_shard_id \gset UPDATE pg_dist_shard SET shardminvalue = 50000, shardmaxvalue = 99999 WHERE shardid = :new_shard_id; --- create append-partitioned shard +-- create append-partitioned shards SELECT master_create_empty_shard('append_partitioned') AS new_shard_id \gset -UPDATE pg_dist_shard SET shardminvalue = 0, shardmaxvalue = 100000 +UPDATE pg_dist_shard SET shardminvalue = 0, shardmaxvalue = 500000 +WHERE shardid = :new_shard_id; +SELECT master_create_empty_shard('append_partitioned') AS new_shard_id +\gset +UPDATE pg_dist_shard SET shardminvalue = 500000, shardmaxvalue = 1000000 WHERE shardid = :new_shard_id; -- basic single-row INSERT INSERT INTO limit_orders VALUES (32743, 'AAPL', 9580, '2004-10-19 10:23:54', 'buy', @@ -82,7 +86,10 @@ ERROR: distributed modifications must target exactly one shard -- try an insert to a range-partitioned table INSERT INTO range_partitioned VALUES (32743, 'AAPL', 9580, '2004-10-19 10:23:54', 'buy', 20.69); --- ensure the value is where we put it and query to find it is properly pruned +-- also insert to an append-partitioned table +INSERT INTO append_partitioned VALUES (414123, 'AAPL', 9580, '2004-10-19 10:23:54', 'buy', + 20.69); +-- ensure the values are where we put them and query to ensure they are properly pruned SET client_min_messages TO 'DEBUG2'; SET citusdb.task_executor_type TO 'router'; SELECT * FROM range_partitioned WHERE id = 32743; @@ -92,17 +99,23 @@ DEBUG: predicate pruning for shardId 103070 32743 | AAPL | 9580 | Tue Oct 19 10:23:54 2004 | buy | 20.69 (1 row) +SELECT * FROM append_partitioned WHERE id = 414123; +DEBUG: predicate pruning for shardId 103072 + id | symbol | bidder_id | placed_at | kind | limit_price +--------+--------+-----------+--------------------------+------+------------- + 414123 | AAPL | 9580 | Tue Oct 19 10:23:54 2004 | buy | 20.69 +(1 row) + SET client_min_messages TO DEFAULT; SET citusdb.task_executor_type TO DEFAULT; --- also try inserting without a range-partitioned shard to receive the value +-- try inserting without a range-partitioned shard to receive the value INSERT INTO range_partitioned VALUES (999999, 'AAPL', 9580, '2004-10-19 10:23:54', 'buy', 20.69); ERROR: distributed modifications must target exactly one shard --- try an insert to an append-partitioned table -INSERT INTO append_partitioned VALUES (414123, 'AAPL', 9580, '2004-10-19 10:23:54', 'buy', +-- and insert into an append-partitioned table with a value that spans shards: +INSERT INTO append_partitioned VALUES (500000, 'AAPL', 9580, '2004-10-19 10:23:54', 'buy', 20.69); -ERROR: cannot perform distributed planning for the given modification -DETAIL: Only hash- or range-partitioned tables may be the target of distributed modifications +ERROR: distributed modifications must target exactly one shard -- INSERT with DEFAULT in the target list INSERT INTO limit_orders VALUES (12756, 'MSFT', 10959, '2013-05-08 07:29:23', 'sell', DEFAULT); diff --git a/src/test/regress/expected/multi_simple_queries.out b/src/test/regress/expected/multi_simple_queries.out index 4407ec697..734dfbde2 100644 --- a/src/test/regress/expected/multi_simple_queries.out +++ b/src/test/regress/expected/multi_simple_queries.out @@ -309,7 +309,7 @@ SET client_min_messages TO 'DEBUG2'; SELECT * FROM articles WHERE author_id = 1; -DEBUG: predicate pruning for shardId 103093 +DEBUG: predicate pruning for shardId 103094 id | author_id | title | word_count ----+-----------+--------------+------------ 1 | 1 | arsenous | 9572 @@ -323,7 +323,7 @@ DEBUG: predicate pruning for shardId 103093 SELECT * FROM articles WHERE author_id = 1 OR author_id = 17; -DEBUG: predicate pruning for shardId 103093 +DEBUG: predicate pruning for shardId 103094 id | author_id | title | word_count ----+-----------+--------------+------------ 1 | 1 | arsenous | 9572 @@ -343,7 +343,7 @@ HINT: Set citusdb.task_executor_type to "real-time" or "task-tracker". SELECT id as article_id, word_count * id as random_value FROM articles WHERE author_id = 1; -DEBUG: predicate pruning for shardId 103093 +DEBUG: predicate pruning for shardId 103094 article_id | random_value ------------+-------------- 1 | 9572 @@ -360,7 +360,7 @@ SELECT a.author_id as first_author, b.word_count as second_word_count WHERE a.author_id = 10 and a.author_id = b.author_id LIMIT 3; DEBUG: push down of limit count: 3 -DEBUG: predicate pruning for shardId 103093 +DEBUG: predicate pruning for shardId 103094 DEBUG: join prunable for intervals [-2147483648,-1] and [0,2147483647] first_author | second_word_count --------------+------------------- @@ -375,7 +375,7 @@ SELECT a.author_id as first_author, b.word_count as second_word_count WHERE a.author_id = 10 and a.author_id = b.author_id LIMIT 3; DEBUG: push down of limit count: 3 -DEBUG: predicate pruning for shardId 103093 +DEBUG: predicate pruning for shardId 103094 ERROR: cannot use router executor with JOINs HINT: Set citusdb.task_executor_type to "real-time" or "task-tracker". -- do not create the master query for LIMIT on a single shard SELECT @@ -384,7 +384,7 @@ SELECT * WHERE author_id = 1 LIMIT 2; DEBUG: push down of limit count: 2 -DEBUG: predicate pruning for shardId 103093 +DEBUG: predicate pruning for shardId 103094 id | author_id | title | word_count ----+-----------+----------+------------ 1 | 1 | arsenous | 9572 @@ -398,7 +398,7 @@ SELECT id FROM articles WHERE author_id = 1 GROUP BY id; -DEBUG: predicate pruning for shardId 103093 +DEBUG: predicate pruning for shardId 103094 id ---- 41 @@ -415,7 +415,7 @@ COPY articles_single_shard TO stdout; SELECT avg(word_count) FROM articles WHERE author_id = 2; -DEBUG: predicate pruning for shardId 103092 +DEBUG: predicate pruning for shardId 103093 ERROR: cannot use router executor with aggregates HINT: Set citusdb.task_executor_type to "real-time" or "task-tracker". -- max, min, sum, count is somehow implemented @@ -424,7 +424,7 @@ SELECT max(word_count) as max, min(word_count) as min, sum(word_count) as sum, count(word_count) as cnt FROM articles WHERE author_id = 2; -DEBUG: predicate pruning for shardId 103092 +DEBUG: predicate pruning for shardId 103093 ERROR: cannot use router executor with aggregates HINT: Set citusdb.task_executor_type to "real-time" or "task-tracker". -- error out for queries with ORDER BY @@ -432,7 +432,7 @@ SELECT * FROM articles WHERE author_id = 1 ORDER BY word_count; -DEBUG: predicate pruning for shardId 103093 +DEBUG: predicate pruning for shardId 103094 ERROR: cannot use router executor with ORDER BY clauses HINT: Set citusdb.task_executor_type to "real-time" or "task-tracker". -- error out for queries with ORDER BY and LIMIT @@ -442,7 +442,7 @@ SELECT * ORDER BY word_count LIMIT 2; DEBUG: push down of limit count: 2 -DEBUG: predicate pruning for shardId 103093 +DEBUG: predicate pruning for shardId 103094 ERROR: cannot use router executor with ORDER BY clauses HINT: Set citusdb.task_executor_type to "real-time" or "task-tracker". -- error out for queries with aggregates and GROUP BY @@ -450,14 +450,14 @@ SELECT max(word_count) FROM articles WHERE author_id = 1 GROUP BY author_id; -DEBUG: predicate pruning for shardId 103093 +DEBUG: predicate pruning for shardId 103094 ERROR: cannot use router executor with aggregates HINT: Set citusdb.task_executor_type to "real-time" or "task-tracker". -- error out for queries with repartition jobs SELECT * FROM articles a, articles b WHERE a.id = b.id AND a.author_id = 1; -DEBUG: predicate pruning for shardId 103093 +DEBUG: predicate pruning for shardId 103094 DEBUG: join prunable for task partitionId 0 and 1 DEBUG: join prunable for task partitionId 0 and 2 DEBUG: join prunable for task partitionId 0 and 3 diff --git a/src/test/regress/sql/multi_modifications.sql b/src/test/regress/sql/multi_modifications.sql index 040411079..390913dfb 100644 --- a/src/test/regress/sql/multi_modifications.sql +++ b/src/test/regress/sql/multi_modifications.sql @@ -40,10 +40,15 @@ SELECT master_create_empty_shard('range_partitioned') AS new_shard_id UPDATE pg_dist_shard SET shardminvalue = 50000, shardmaxvalue = 99999 WHERE shardid = :new_shard_id; --- create append-partitioned shard +-- create append-partitioned shards SELECT master_create_empty_shard('append_partitioned') AS new_shard_id \gset -UPDATE pg_dist_shard SET shardminvalue = 0, shardmaxvalue = 100000 +UPDATE pg_dist_shard SET shardminvalue = 0, shardmaxvalue = 500000 +WHERE shardid = :new_shard_id; + +SELECT master_create_empty_shard('append_partitioned') AS new_shard_id +\gset +UPDATE pg_dist_shard SET shardminvalue = 500000, shardmaxvalue = 1000000 WHERE shardid = :new_shard_id; -- basic single-row INSERT @@ -59,19 +64,23 @@ INSERT INTO insufficient_shards VALUES (32743, 'AAPL', 9580, '2004-10-19 10:23:5 INSERT INTO range_partitioned VALUES (32743, 'AAPL', 9580, '2004-10-19 10:23:54', 'buy', 20.69); --- ensure the value is where we put it and query to find it is properly pruned +-- also insert to an append-partitioned table +INSERT INTO append_partitioned VALUES (414123, 'AAPL', 9580, '2004-10-19 10:23:54', 'buy', + 20.69); +-- ensure the values are where we put them and query to ensure they are properly pruned SET client_min_messages TO 'DEBUG2'; SET citusdb.task_executor_type TO 'router'; SELECT * FROM range_partitioned WHERE id = 32743; +SELECT * FROM append_partitioned WHERE id = 414123; SET client_min_messages TO DEFAULT; SET citusdb.task_executor_type TO DEFAULT; --- also try inserting without a range-partitioned shard to receive the value +-- try inserting without a range-partitioned shard to receive the value INSERT INTO range_partitioned VALUES (999999, 'AAPL', 9580, '2004-10-19 10:23:54', 'buy', 20.69); --- try an insert to an append-partitioned table -INSERT INTO append_partitioned VALUES (414123, 'AAPL', 9580, '2004-10-19 10:23:54', 'buy', +-- and insert into an append-partitioned table with a value that spans shards: +INSERT INTO append_partitioned VALUES (500000, 'AAPL', 9580, '2004-10-19 10:23:54', 'buy', 20.69); -- INSERT with DEFAULT in the target list From 0a6839e544e2a99b1e81788a48072a914cbfcbaf Mon Sep 17 00:00:00 2001 From: Onder Kalaci Date: Thu, 11 Feb 2016 19:42:31 +0200 Subject: [PATCH 03/19] Perform distributed planning in the calling memory context Previously we used, for historical reasons, MessageContext. That is problematic if a single message from the client causes a lot of statements to be planned. E.g. for the copy_to_distributed_table script one insert statement is planned for each row inserted via COPY, and only freed when COPY has finished. --- src/backend/distributed/planner/multi_planner.c | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/src/backend/distributed/planner/multi_planner.c b/src/backend/distributed/planner/multi_planner.c index 93d05817e..e6cd82b50 100644 --- a/src/backend/distributed/planner/multi_planner.c +++ b/src/backend/distributed/planner/multi_planner.c @@ -52,19 +52,10 @@ multi_planner(Query *parse, int cursorOptions, ParamListInfo boundParams) if (NeedsDistributedPlanning(parse)) { - MemoryContext oldcontext = NULL; - MultiPlan *physicalPlan = NULL; - - /* Switch to top level message context */ - oldcontext = MemoryContextSwitchTo(MessageContext); - - physicalPlan = CreatePhysicalPlan(parse); + MultiPlan *physicalPlan = CreatePhysicalPlan(parse); /* store required data into the planned statement */ result = MultiQueryContainerNode(result, physicalPlan); - - /* Now switch back to original context */ - MemoryContextSwitchTo(oldcontext); } return result; From f2bf7fc802eea7915d893e8c33adf1dd4f1f7b97 Mon Sep 17 00:00:00 2001 From: Jason Petersen Date: Thu, 11 Feb 2016 15:21:40 -0700 Subject: [PATCH 04/19] Rename GetConnection to address name conflict The postgres_fdw extension has an extern function with an identical signature, which can cause problems when both extensions are loaded. A simple rename can fix this for now (this is the only function with) such a conflict. --- .../distributed/executor/multi_router_executor.c | 4 ++-- src/backend/distributed/test/connection_cache.c | 8 ++++---- src/backend/distributed/utils/connection_cache.c | 12 ++++++------ src/include/distributed/connection_cache.h | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/backend/distributed/executor/multi_router_executor.c b/src/backend/distributed/executor/multi_router_executor.c index 5a6cdd06e..5c2f04165 100644 --- a/src/backend/distributed/executor/multi_router_executor.c +++ b/src/backend/distributed/executor/multi_router_executor.c @@ -250,7 +250,7 @@ ExecuteDistributedModify(Task *task) Assert(taskPlacement->shardState == FILE_FINALIZED); - connection = GetConnection(nodeName, nodePort); + connection = GetOrEstablishConnection(nodeName, nodePort); if (connection == NULL) { failedPlacementList = lappend(failedPlacementList, taskPlacement); @@ -383,7 +383,7 @@ ExecuteTaskAndStoreResults(Task *task, TupleDesc tupleDescriptor, bool queryOK = false; bool storedOK = false; - PGconn *connection = GetConnection(nodeName, nodePort); + PGconn *connection = GetOrEstablishConnection(nodeName, nodePort); if (connection == NULL) { continue; diff --git a/src/backend/distributed/test/connection_cache.c b/src/backend/distributed/test/connection_cache.c index 25fcbf593..1b256695c 100644 --- a/src/backend/distributed/test/connection_cache.c +++ b/src/backend/distributed/test/connection_cache.c @@ -48,7 +48,7 @@ initialize_remote_temp_table(PG_FUNCTION_ARGS) int32 nodePort = PG_GETARG_INT32(1); PGresult *result = NULL; - PGconn *connection = GetConnection(nodeName, nodePort); + PGconn *connection = GetOrEstablishConnection(nodeName, nodePort); if (connection == NULL) { PG_RETURN_BOOL(false); @@ -79,7 +79,7 @@ count_remote_temp_table_rows(PG_FUNCTION_ARGS) Datum count = Int32GetDatum(-1); PGresult *result = NULL; - PGconn *connection = GetConnection(nodeName, nodePort); + PGconn *connection = GetOrEstablishConnection(nodeName, nodePort); if (connection == NULL) { PG_RETURN_DATUM(count); @@ -114,7 +114,7 @@ get_and_purge_connection(PG_FUNCTION_ARGS) char *nodeName = PG_GETARG_CSTRING(0); int32 nodePort = PG_GETARG_INT32(1); - PGconn *connection = GetConnection(nodeName, nodePort); + PGconn *connection = GetOrEstablishConnection(nodeName, nodePort); if (connection == NULL) { PG_RETURN_BOOL(false); @@ -136,7 +136,7 @@ set_connection_status_bad(PG_FUNCTION_ARGS) char *nodeName = PG_GETARG_CSTRING(0); int32 nodePort = PG_GETARG_INT32(1); - PGconn *connection = GetConnection(nodeName, nodePort); + PGconn *connection = GetOrEstablishConnection(nodeName, nodePort); if (connection == NULL) { PG_RETURN_BOOL(false); diff --git a/src/backend/distributed/utils/connection_cache.c b/src/backend/distributed/utils/connection_cache.c index 4281a4a59..a4aa14cd7 100644 --- a/src/backend/distributed/utils/connection_cache.c +++ b/src/backend/distributed/utils/connection_cache.c @@ -32,7 +32,7 @@ /* * NodeConnectionHash is the connection hash itself. It begins uninitialized. - * The first call to GetConnection triggers hash creation. + * The first call to GetOrEstablishConnection triggers hash creation. */ static HTAB *NodeConnectionHash = NULL; @@ -44,10 +44,10 @@ static char * ConnectionGetOptionValue(PGconn *connection, char *optionKeyword); /* - * GetConnection returns a PGconn which can be used to execute queries on a - * remote PostgreSQL server. If no suitable connection to the specified node on - * the specified port yet exists, the function establishes a new connection and - * returns that. + * GetOrEstablishConnection returns a PGconn which can be used to execute + * queries on a remote PostgreSQL server. If no suitable connection to the + * specified node on the specified port yet exists, the function establishes + * a new connection and adds it to the connection cache before returning it. * * Returned connections are guaranteed to be in the CONNECTION_OK state. If the * requested connection cannot be established, or if it was previously created @@ -56,7 +56,7 @@ static char * ConnectionGetOptionValue(PGconn *connection, char *optionKeyword); * This function throws an error if a hostname over 255 characters is provided. */ PGconn * -GetConnection(char *nodeName, int32 nodePort) +GetOrEstablishConnection(char *nodeName, int32 nodePort) { PGconn *connection = NULL; NodeConnectionKey nodeConnectionKey; diff --git a/src/include/distributed/connection_cache.h b/src/include/distributed/connection_cache.h index 2a1f997d6..45f61742b 100644 --- a/src/include/distributed/connection_cache.h +++ b/src/include/distributed/connection_cache.h @@ -51,7 +51,7 @@ typedef struct NodeConnectionEntry /* function declarations for obtaining and using a connection */ -extern PGconn * GetConnection(char *nodeName, int32 nodePort); +extern PGconn * GetOrEstablishConnection(char *nodeName, int32 nodePort); extern void PurgeConnection(PGconn *connection); extern void ReportRemoteError(PGconn *connection, PGresult *result); From c1d377b7d2d66bca84aef8b75c2d534a3d0b640b Mon Sep 17 00:00:00 2001 From: Murat Tuncer Date: Mon, 15 Feb 2016 16:04:31 +0200 Subject: [PATCH 05/19] Changed product name to citus All citusdb references in - extension, binary names - file headers - all configuration name prefixes - error/warning messages - some functions names - regression tests are changed to be citus. --- Makefile | 12 +-- Makefile.global.in | 22 +++--- autogen.sh | 2 +- configure | 28 +++---- configure.in | 12 +-- prep_buildtree | 4 +- src/backend/distributed/.gitignore | 2 +- src/backend/distributed/Makefile | 16 ++-- src/backend/distributed/citus.control | 6 ++ .../distributed/{citusdb.sql => citus.sql} | 74 +++++++++---------- src/backend/distributed/citusdb.control | 6 -- .../commands/create_distributed_table.c | 12 +-- .../executor/multi_server_executor.c | 18 ++--- .../distributed/executor/multi_utility.c | 32 ++++---- .../distributed/planner/multi_explain.c | 2 +- .../distributed/planner/multi_join_order.c | 2 +- .../distributed/planner/multi_planner.c | 10 +-- src/backend/distributed/shared_library_init.c | 74 +++++++++---------- .../distributed/test/connection_cache.c | 2 +- src/backend/distributed/test/create_shards.c | 2 +- .../distributed/test/distribution_metadata.c | 2 +- .../distributed/test/generate_ddl_commands.c | 2 +- .../distributed/test/prune_shard_list.c | 2 +- .../distributed/test/test_helper_functions.c | 2 +- .../distributed/utils/citus_nodefuncs.c | 10 +-- .../distributed/utils/citus_outfuncs.c | 6 +- .../distributed/utils/citus_readfuncs_94.c | 2 +- .../distributed/utils/citus_readfuncs_95.c | 2 +- .../distributed/utils/connection_cache.c | 6 +- .../distributed/utils/metadata_cache.c | 16 ++-- .../distributed/utils/multi_resowner.c | 2 +- src/backend/distributed/utils/resource_lock.c | 2 +- src/backend/distributed/worker/task_tracker.c | 4 +- src/bin/csql/Makefile | 8 +- src/bin/csql/copy_options.c | 2 +- src/bin/csql/copy_options.h | 2 +- src/bin/csql/help.c | 2 +- src/bin/csql/mainloop.c | 2 +- src/bin/csql/stage.c | 2 +- src/bin/csql/stage.h | 2 +- src/include/.gitignore | 4 +- ...{citusdb_config.h.in => citus_config.h.in} | 6 +- src/include/distributed/citus_nodefuncs.h | 2 +- src/include/distributed/citus_nodes.h | 6 +- src/include/distributed/citus_ruleutils.h | 4 +- src/include/distributed/listutils.h | 6 +- src/include/distributed/metadata_cache.h | 4 +- src/include/distributed/multi_executor.h | 2 +- src/include/distributed/multi_explain.h | 2 +- src/include/distributed/multi_planner.h | 2 +- src/include/distributed/multi_resowner.h | 2 +- src/include/distributed/multi_utility.h | 2 +- src/include/distributed/resource_lock.h | 6 +- .../distributed/test_helper_functions.h | 6 +- src/include/distributed/worker_manager.h | 2 +- src/test/regress/Makefile | 50 ++++++------- .../multi_agg_approximate_distinct.out | 8 +- .../multi_binary_master_copy_format.out | 6 +- .../expected/multi_connection_cache.out | 8 +- .../regress/expected/multi_create_fdw.out | 2 +- .../expected/multi_create_insert_proxy.out | 4 +- .../regress/expected/multi_create_shards.out | 6 +- .../regress/expected/multi_create_table.out | 4 +- .../expected/multi_distribution_metadata.out | 26 +++---- .../expected/multi_generate_ddl_commands.out | 6 +- .../multi_generate_ddl_commands_0.out | 6 +- .../expected/multi_join_order_additional.out | 6 +- .../expected/multi_join_order_tpch_large.out | 4 +- .../expected/multi_join_order_tpch_small.out | 4 +- .../regress/expected/multi_join_pruning.out | 2 +- .../multi_large_table_join_planning.out | 4 +- .../multi_large_table_join_planning_0.out | 4 +- .../expected/multi_large_table_pruning.out | 4 +- .../multi_large_table_task_assignment.out | 8 +- .../multi_limit_clause_approximate.out | 12 +-- .../regress/expected/multi_modifications.out | 4 +- .../multi_null_minmax_value_pruning.out | 2 +- .../regress/expected/multi_prepare_plsql.out | 4 +- .../regress/expected/multi_prepare_sql.out | 4 +- .../expected/multi_prune_shard_list.out | 10 +-- .../regress/expected/multi_simple_queries.out | 34 ++++----- .../multi_single_relation_subquery.out | 2 +- src/test/regress/expected/multi_table_ddl.out | 12 +-- .../expected/multi_task_assignment_policy.out | 8 +- .../regress/expected/multi_tpch_query1.out | 2 +- .../regress/expected/multi_tpch_query10.out | 2 +- .../regress/expected/multi_tpch_query12.out | 2 +- .../regress/expected/multi_tpch_query14.out | 2 +- .../regress/expected/multi_tpch_query19.out | 2 +- .../regress/expected/multi_tpch_query3.out | 2 +- .../regress/expected/multi_tpch_query6.out | 2 +- .../regress/expected/multi_tpch_query7.out | 2 +- .../expected/multi_tpch_query7_nested.out | 2 +- src/test/regress/expected/multi_upsert.out | 6 +- src/test/regress/expected/multi_upsert_0.out | 6 +- .../expected/multi_utility_statements.out | 2 +- .../expected/multi_utility_warnings.out | 12 +-- .../regress/input/multi_agg_distinct.source | 4 +- .../input/multi_append_table_to_shard.source | 6 +- .../regress/input/multi_fdw_stage_data.source | 2 +- .../regress/input/multi_outer_join.source | 4 +- .../regress/input/multi_stage_data.source | 6 +- .../input/multi_stage_large_records.source | 4 +- .../input/multi_stage_more_data.source | 2 +- src/test/regress/input/multi_subquery.source | 6 +- .../regress/output/multi_agg_distinct.source | 4 +- .../output/multi_append_table_to_shard.source | 6 +- .../regress/output/multi_create_schema.source | 12 +-- .../output/multi_fdw_stage_data.source | 2 +- .../regress/output/multi_outer_join.source | 4 +- .../regress/output/multi_stage_data.source | 6 +- .../output/multi_stage_large_records.source | 4 +- .../output/multi_stage_more_data.source | 2 +- src/test/regress/output/multi_subquery.source | 6 +- src/test/regress/pg_regress_multi.pl | 20 ++--- .../sql/multi_agg_approximate_distinct.sql | 8 +- .../sql/multi_binary_master_copy_format.sql | 6 +- .../regress/sql/multi_connection_cache.sql | 8 +- src/test/regress/sql/multi_create_fdw.sql | 2 +- src/test/regress/sql/multi_create_shards.sql | 4 +- src/test/regress/sql/multi_create_table.sql | 4 +- .../sql/multi_distribution_metadata.sql | 26 +++---- .../sql/multi_generate_ddl_commands.sql | 2 +- .../sql/multi_join_order_additional.sql | 6 +- .../sql/multi_join_order_tpch_large.sql | 4 +- .../sql/multi_join_order_tpch_small.sql | 4 +- src/test/regress/sql/multi_join_pruning.sql | 2 +- .../sql/multi_large_table_join_planning.sql | 4 +- .../regress/sql/multi_large_table_pruning.sql | 4 +- .../sql/multi_large_table_task_assignment.sql | 8 +- .../sql/multi_limit_clause_approximate.sql | 12 +-- src/test/regress/sql/multi_modifications.sql | 4 +- .../sql/multi_null_minmax_value_pruning.sql | 2 +- src/test/regress/sql/multi_prepare_plsql.sql | 4 +- src/test/regress/sql/multi_prepare_sql.sql | 4 +- .../regress/sql/multi_prune_shard_list.sql | 10 +-- src/test/regress/sql/multi_simple_queries.sql | 16 ++-- .../sql/multi_single_relation_subquery.sql | 2 +- src/test/regress/sql/multi_table_ddl.sql | 8 +- .../sql/multi_task_assignment_policy.sql | 8 +- src/test/regress/sql/multi_tpch_query1.sql | 2 +- src/test/regress/sql/multi_tpch_query10.sql | 2 +- src/test/regress/sql/multi_tpch_query12.sql | 2 +- src/test/regress/sql/multi_tpch_query14.sql | 2 +- src/test/regress/sql/multi_tpch_query19.sql | 2 +- src/test/regress/sql/multi_tpch_query3.sql | 2 +- src/test/regress/sql/multi_tpch_query6.sql | 2 +- src/test/regress/sql/multi_tpch_query7.sql | 2 +- .../regress/sql/multi_tpch_query7_nested.sql | 2 +- src/test/regress/sql/multi_upsert.sql | 6 +- .../regress/sql/multi_utility_statements.sql | 2 +- 151 files changed, 529 insertions(+), 529 deletions(-) create mode 100644 src/backend/distributed/citus.control rename src/backend/distributed/{citusdb.sql => citus.sql} (91%) delete mode 100644 src/backend/distributed/citusdb.control rename src/include/{citusdb_config.h.in => citus_config.h.in} (71%) diff --git a/Makefile b/Makefile index e024ef4ff..149cc8c99 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,11 @@ -# CitusDB toplevel Makefile +# Citus toplevel Makefile -citusdb_subdir = . -citusdb_top_builddir = . +citus_subdir = . +citus_top_builddir = . # Hint that configure should be run first ifeq (,$(wildcard Makefile.global)) - $(error ./configure needs to be run before compiling CitusDB) + $(error ./configure needs to be run before compiling Citus) endif include Makefile.global @@ -20,9 +20,9 @@ install-extension: install-headers: $(MKDIR_P) '$(includedir_server)/distributed/' # generated headers are located in the build directory - $(INSTALL_DATA) src/include/citusdb_config.h '$(includedir_server)/' + $(INSTALL_DATA) src/include/citus_config.h '$(includedir_server)/' # the rest in the source tree - $(INSTALL_DATA) $(citusdb_abs_srcdir)/src/include/distributed/*.h '$(includedir_server)/distributed/' + $(INSTALL_DATA) $(citus_abs_srcdir)/src/include/distributed/*.h '$(includedir_server)/distributed/' clean-extension: $(MAKE) -C src/backend/distributed/ clean .PHONY: extension install-extension clean-extension diff --git a/Makefile.global.in b/Makefile.global.in index 087072749..60b3d31f0 100644 --- a/Makefile.global.in +++ b/Makefile.global.in @@ -9,40 +9,40 @@ # makefiles, particulary central handling of compilation flags and # rules. -citusdb_abs_srcdir:=@abs_top_srcdir@/${citusdb_subdir} -citusdb_abs_top_srcdir:=@abs_top_srcdir@ +citus_abs_srcdir:=@abs_top_srcdir@/${citus_subdir} +citus_abs_top_srcdir:=@abs_top_srcdir@ PG_CONFIG:=@PG_CONFIG@ PGXS:=$(shell $(PG_CONFIG) --pgxs) # Support for VPATH builds (i.e. builds from outside the source tree) vpath_build=@vpath_build@ ifeq ($(vpath_build),yes) - VPATH:=$(citusdb_abs_srcdir) + VPATH:=$(citus_abs_srcdir) USE_VPATH:=$(VPATH) endif -# CitusDB is built using PostgreSQL's pgxs +# Citus is built using PostgreSQL's pgxs USE_PGXS=1 include $(PGXS) # Remake Makefile.global from Makefile.global.in if the latter # changed. In order to trigger this rule, the including file must -# write `include $(citusdb_top_builddir)/Makefile.global', not some +# write `include $(citus_top_builddir)/Makefile.global', not some # shortcut thereof. This makes it less likely to accidentally run # with some outdated Makefile.global. # Make internally restarts whenever included Makefiles are # regenerated. -$(citusdb_top_builddir)/Makefile.global: $(citusdb_top_builddir)/Makefile.global.in @top_srcdir@/configure $(citusdb_top_builddir)/config.status +$(citus_top_builddir)/Makefile.global: $(citus_top_builddir)/Makefile.global.in @top_srcdir@/configure $(citus_top_builddir)/config.status cd @abs_top_builddir@ && ./config.status Makefile.global # Ensure configuration is generated by the most recent configure, # useful for longer existing build directories. -$(citusdb_top_builddir)/config.status: @top_srcdir@/configure +$(citus_top_builddir)/config.status: @top_srcdir@/configure cd @abs_top_builddir@ && ./config.status --recheck # Regenerate configure if configure.in changed -@top_srcdir@/configure: $(citusdb_abs_srcdir)/configure.in - cd ${citusdb_abs_srcdir} && ./autogen.sh +@top_srcdir@/configure: $(citus_abs_srcdir)/configure.in + cd ${citus_abs_srcdir} && ./autogen.sh # If specified via configure, replace the default compiler. Normally # we'll build with the one postgres was built with. But it's useful to @@ -54,8 +54,8 @@ endif # Add options passed to configure or computed therein, to CFLAGS/CPPFLAGS/... override CFLAGS += @CFLAGS@ @CITUS_CFLAGS@ -override CPPFLAGS := @CPPFLAGS@ -I '${citusdb_abs_top_srcdir}/src/include' $(CPPFLAGS) +override CPPFLAGS := @CPPFLAGS@ -I '${citus_abs_top_srcdir}/src/include' $(CPPFLAGS) override LDFLAGS += @LDFLAGS@ # optional file with user defined, additional, rules --include ${citusdb_abs_srcdir}/src/Makefile.custom +-include ${citus_abs_srcdir}/src/Makefile.custom diff --git a/autogen.sh b/autogen.sh index 9db001b0f..145d333cf 100755 --- a/autogen.sh +++ b/autogen.sh @@ -1,7 +1,7 @@ #!/bin/bash # # autogen.sh converts configure.in to configure and creates -# citusdb_config.h.in. The resuting resulting files are checked into +# citus_config.h.in. The resuting resulting files are checked into # the SCM, to avoid everyone needing autoconf installed. autoreconf -f diff --git a/configure b/configure index ce37e3d49..aaf84be6a 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for CitusDB 5.0. +# Generated by GNU Autoconf 2.69 for Citus 5.0. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. @@ -577,10 +577,10 @@ MFLAGS= MAKEFLAGS= # Identity of this package. -PACKAGE_NAME='CitusDB' -PACKAGE_TARNAME='citusdb' +PACKAGE_NAME='Citus' +PACKAGE_TARNAME='citus' PACKAGE_VERSION='5.0' -PACKAGE_STRING='CitusDB 5.0' +PACKAGE_STRING='Citus 5.0' PACKAGE_BUGREPORT='' PACKAGE_URL='' @@ -1190,7 +1190,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures CitusDB 5.0 to adapt to many kinds of systems. +\`configure' configures Citus 5.0 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1238,7 +1238,7 @@ Fine tuning of the installation directories: --infodir=DIR info documentation [DATAROOTDIR/info] --localedir=DIR locale-dependent data [DATAROOTDIR/locale] --mandir=DIR man documentation [DATAROOTDIR/man] - --docdir=DIR documentation root [DATAROOTDIR/doc/citusdb] + --docdir=DIR documentation root [DATAROOTDIR/doc/citus] --htmldir=DIR html documentation [DOCDIR] --dvidir=DIR dvi documentation [DOCDIR] --pdfdir=DIR pdf documentation [DOCDIR] @@ -1251,7 +1251,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of CitusDB 5.0:";; + short | recursive ) echo "Configuration of Citus 5.0:";; esac cat <<\_ACEOF @@ -1333,7 +1333,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -CitusDB configure 5.0 +Citus configure 5.0 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1390,7 +1390,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by CitusDB $as_me 5.0, which was +It was created by Citus $as_me 5.0, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -1871,7 +1871,7 @@ if test -z "$version_num"; then fi if test "$version_num" != '9.4' -a "$version_num" != '9.5'; then - as_fn_error $? "CitusDB is not compatible with the detected PostgreSQL version ${version_num}." "$LINENO" 5 + as_fn_error $? "Citus is not compatible with the detected PostgreSQL version ${version_num}." "$LINENO" 5 else { $as_echo "$as_me:${as_lineno-$LINENO}: building against PostgreSQL $version_num" >&5 $as_echo "$as_me: building against PostgreSQL $version_num" >&6;} @@ -2893,7 +2893,7 @@ CITUS_CFLAGS="$CITUS_CFLAGS" ac_config_files="$ac_config_files Makefile.global" -ac_config_headers="$ac_config_headers src/include/citusdb_config.h" +ac_config_headers="$ac_config_headers src/include/citus_config.h" cat >confcache <<\_ACEOF @@ -3402,7 +3402,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by CitusDB $as_me 5.0, which was +This file was extended by Citus $as_me 5.0, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -3464,7 +3464,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -CitusDB config.status 5.0 +Citus config.status 5.0 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" @@ -3586,7 +3586,7 @@ for ac_config_target in $ac_config_targets do case $ac_config_target in "Makefile.global") CONFIG_FILES="$CONFIG_FILES Makefile.global" ;; - "src/include/citusdb_config.h") CONFIG_HEADERS="$CONFIG_HEADERS src/include/citusdb_config.h" ;; + "src/include/citus_config.h") CONFIG_HEADERS="$CONFIG_HEADERS src/include/citus_config.h" ;; *) as_fn_error $? "invalid argument: \`$ac_config_target'" "$LINENO" 5;; esac diff --git a/configure.in b/configure.in index b3bacd4cc..5248f36f5 100644 --- a/configure.in +++ b/configure.in @@ -1,11 +1,11 @@ -# CitusDB autoconf input script. +# Citus autoconf input script. # # Converted into an actual configure script by autogen.sh. This # conversion only has to be done when configure.in changes. To avoid # everyone needing autoconf installed, the resulting files are checked # into the SCM. -AC_INIT([CitusDB], [5.0], [], [citusdb], []) +AC_INIT([Citus], [5.0], [], [citus], []) AC_COPYRIGHT([Copyright (c) Copyright (c) 2012-2015, Citus Data, Inc.]) AC_PROG_SED @@ -32,7 +32,7 @@ if test -z "$version_num"; then fi if test "$version_num" != '9.4' -a "$version_num" != '9.5'; then - AC_MSG_ERROR([CitusDB is not compatible with the detected PostgreSQL version ${version_num}.]) + AC_MSG_ERROR([Citus is not compatible with the detected PostgreSQL version ${version_num}.]) else AC_MSG_NOTICE([building against PostgreSQL $version_num]) fi; @@ -96,11 +96,11 @@ CITUSAC_PROG_CC_CFLAGS_OPT([-Wmissing-prototypes]) AC_SUBST(CITUS_CFLAGS, "$CITUS_CFLAGS") AC_CONFIG_FILES([Makefile.global]) -AC_CONFIG_HEADERS([src/include/citusdb_config.h]) +AC_CONFIG_HEADERS([src/include/citus_config.h]) AH_TOP([ /* - * citusdb_config.h.in is generated by autoconf/autoheader and - * converted into citusdb_config.h by configure. Include when code needs to + * citus_config.h.in is generated by autoconf/autoheader and + * converted into citus_config.h by configure. Include when code needs to * depend on determinations made by configure. * * Do not manually edit! diff --git a/prep_buildtree b/prep_buildtree index dcd7869d7..f39f36c90 100644 --- a/prep_buildtree +++ b/prep_buildtree @@ -1,8 +1,8 @@ #! /bin/sh # -# CitusDB copy of PostgreSQL's config/prep_buildtree +# Citus copy of PostgreSQL's config/prep_buildtree # -# This script prepares a CitusDB build tree for an out-of-tree/VPATH +# This script prepares a Citus build tree for an out-of-tree/VPATH # build. It is intended to be run by the configure script. me=`basename $0` diff --git a/src/backend/distributed/.gitignore b/src/backend/distributed/.gitignore index b5abe3631..3c9a69e03 100644 --- a/src/backend/distributed/.gitignore +++ b/src/backend/distributed/.gitignore @@ -10,4 +10,4 @@ /tmp_check* # ignore latest install file -citusdb--5.0.sql +citus--5.0.sql diff --git a/src/backend/distributed/Makefile b/src/backend/distributed/Makefile index ab24267ef..8f4bd899a 100644 --- a/src/backend/distributed/Makefile +++ b/src/backend/distributed/Makefile @@ -1,13 +1,13 @@ -# Makefile for the CitusDB extension +# Makefile for the Citus extension -citusdb_subdir = src/backend/distributed -citusdb_top_builddir = ../../.. +citus_subdir = src/backend/distributed +citus_top_builddir = ../../.. -MODULE_big = citusdb -EXTENSION = citusdb +MODULE_big = citus +EXTENSION = citus EXTVERSION = 5.0 DATA_built = $(EXTENSION)--$(EXTVERSION).sql -SCRIPTS = $(wildcard $(citusdb_top_builddir)/src/bin/scripts/*) +SCRIPTS = $(wildcard $(citus_top_builddir)/src/bin/scripts/*) # directories with source files SUBDIRS = . commands executor master planner relay test utils worker @@ -15,7 +15,7 @@ SUBDIRS = . commands executor master planner relay test utils worker # That patsubst rule searches all directories listed in SUBDIRS for .c # files, and adds the corresponding .o files to OBJS OBJS += \ - $(patsubst $(citusdb_abs_srcdir)/%.c,%.o,$(foreach dir,$(SUBDIRS), $(wildcard $(citusdb_abs_srcdir)/$(dir)/*.c))) + $(patsubst $(citus_abs_srcdir)/%.c,%.o,$(foreach dir,$(SUBDIRS), $(wildcard $(citus_abs_srcdir)/$(dir)/*.c))) # define build process for latest install file $(EXTENSION)--$(EXTVERSION).sql: $(EXTENSION).sql @@ -28,6 +28,6 @@ NO_PGXS = 1 SHLIB_LINK = $(libpq) -include $(citusdb_top_builddir)/Makefile.global +include $(citus_top_builddir)/Makefile.global override CPPFLAGS += -I$(libpq_srcdir) diff --git a/src/backend/distributed/citus.control b/src/backend/distributed/citus.control new file mode 100644 index 000000000..0f25b1dd7 --- /dev/null +++ b/src/backend/distributed/citus.control @@ -0,0 +1,6 @@ +# Citus extension +comment = 'Citus distributed database' +default_version = '5.0' +module_pathname = '$libdir/citus' +relocatable = false +schema = pg_catalog diff --git a/src/backend/distributed/citusdb.sql b/src/backend/distributed/citus.sql similarity index 91% rename from src/backend/distributed/citusdb.sql rename to src/backend/distributed/citus.sql index 1090d5bd4..4e1178f04 100644 --- a/src/backend/distributed/citusdb.sql +++ b/src/backend/distributed/citus.sql @@ -1,24 +1,24 @@ -/* citusdb.sql */ +/* citus.sql */ -- complain if script is sourced in psql, rather than via CREATE EXTENSION -\echo Use "CREATE EXTENSION citusdb" to load this file. \quit +\echo Use "CREATE EXTENSION citus" to load this file. \quit -CREATE SCHEMA citusdb; +CREATE SCHEMA citus; --- Ensure CREATE EXTENSION is not run against an old citusdb data +-- Ensure CREATE EXTENSION is not run against an old citus data -- directory, we're not compatible (due to the builtin functions/tables) DO $$ BEGIN IF EXISTS(SELECT * FROM pg_proc WHERE proname = 'worker_apply_shard_ddl_command') THEN - RAISE 'cannot install citusdb extension in CitusDB 4 data directory'; + RAISE 'cannot install citus extension in Citus 4 data directory'; END IF; END; $$; /***************************************************************************** - * CitusDB data types + * Citus data types *****************************************************************************/ -CREATE TYPE citusdb.distribution_type AS ENUM ( +CREATE TYPE citus.distribution_type AS ENUM ( 'hash', 'range', 'append' @@ -26,18 +26,18 @@ CREATE TYPE citusdb.distribution_type AS ENUM ( /***************************************************************************** - * CitusDB tables & corresponding indexes + * Citus tables & corresponding indexes *****************************************************************************/ -CREATE TABLE citusdb.pg_dist_partition( +CREATE TABLE citus.pg_dist_partition( logicalrelid Oid NOT NULL, partmethod "char" NOT NULL, partkey text NOT NULL ); CREATE UNIQUE INDEX pg_dist_partition_logical_relid_index -ON citusdb.pg_dist_partition using btree(logicalrelid); -ALTER TABLE citusdb.pg_dist_partition SET SCHEMA pg_catalog; +ON citus.pg_dist_partition using btree(logicalrelid); +ALTER TABLE citus.pg_dist_partition SET SCHEMA pg_catalog; -CREATE TABLE citusdb.pg_dist_shard( +CREATE TABLE citus.pg_dist_shard( logicalrelid oid NOT NULL, shardid int8 NOT NULL, shardstorage "char" NOT NULL, @@ -46,12 +46,12 @@ CREATE TABLE citusdb.pg_dist_shard( shardmaxvalue text ); CREATE UNIQUE INDEX pg_dist_shard_shardid_index -ON citusdb.pg_dist_shard using btree(shardid); +ON citus.pg_dist_shard using btree(shardid); CREATE INDEX pg_dist_shard_logical_relid_index -ON citusdb.pg_dist_shard using btree(logicalrelid); -ALTER TABLE citusdb.pg_dist_shard SET SCHEMA pg_catalog; +ON citus.pg_dist_shard using btree(logicalrelid); +ALTER TABLE citus.pg_dist_shard SET SCHEMA pg_catalog; -CREATE TABLE citusdb.pg_dist_shard_placement( +CREATE TABLE citus.pg_dist_shard_placement( shardid int8 NOT NULL, shardstate int4 NOT NULL, shardlength int8 NOT NULL, @@ -59,40 +59,40 @@ CREATE TABLE citusdb.pg_dist_shard_placement( nodeport int8 NOT NULL ) WITH oids; CREATE UNIQUE INDEX pg_dist_shard_placement_oid_index -ON citusdb.pg_dist_shard_placement using btree(oid); +ON citus.pg_dist_shard_placement using btree(oid); CREATE INDEX pg_dist_shard_placement_shardid_index -ON citusdb.pg_dist_shard_placement using btree(shardid); +ON citus.pg_dist_shard_placement using btree(shardid); CREATE INDEX pg_dist_shard_placement_nodeid_index -ON citusdb.pg_dist_shard_placement using btree(nodename, nodeport); -ALTER TABLE citusdb.pg_dist_shard_placement SET SCHEMA pg_catalog; +ON citus.pg_dist_shard_placement using btree(nodename, nodeport); +ALTER TABLE citus.pg_dist_shard_placement SET SCHEMA pg_catalog; /***************************************************************************** - * CitusDB sequences + * Citus sequences *****************************************************************************/ /* * Unternal sequence to generate 64-bit shard ids. These identifiers are then * used to identify shards in the distributed database. */ -CREATE SEQUENCE citusdb.pg_dist_shardid_seq +CREATE SEQUENCE citus.pg_dist_shardid_seq MINVALUE 102008 NO CYCLE; -ALTER SEQUENCE citusdb.pg_dist_shardid_seq SET SCHEMA pg_catalog; +ALTER SEQUENCE citus.pg_dist_shardid_seq SET SCHEMA pg_catalog; /* * internal sequence to generate 32-bit jobIds. These identifiers are then * used to identify jobs in the distributed database; and they wrap at 32-bits * to allow for slave nodes to independently execute their distributed jobs. */ -CREATE SEQUENCE citusdb.pg_dist_jobid_seq +CREATE SEQUENCE citus.pg_dist_jobid_seq MINVALUE 2 /* first jobId reserved for clean up jobs */ MAXVALUE 4294967296; -ALTER SEQUENCE citusdb.pg_dist_jobid_seq SET SCHEMA pg_catalog; +ALTER SEQUENCE citus.pg_dist_jobid_seq SET SCHEMA pg_catalog; /***************************************************************************** - * CitusDB functions + * Citus functions *****************************************************************************/ /* For backward compatibility and ease of use create functions et al. in pg_catalog */ @@ -173,13 +173,13 @@ COMMENT ON FUNCTION master_get_round_robin_candidate_nodes(shard_id bigint) CREATE FUNCTION master_create_distributed_table(table_name regclass, distribution_column text, - distribution_method citusdb.distribution_type) + distribution_method citus.distribution_type) RETURNS void LANGUAGE C STRICT AS 'MODULE_PATHNAME', $$master_create_distributed_table$$; COMMENT ON FUNCTION master_create_distributed_table(table_name regclass, distribution_column text, - distribution_method citusdb.distribution_type) + distribution_method citus.distribution_type) IS 'define the table distribution functions'; -- define shard creation function for hash-partitioned tables @@ -314,7 +314,7 @@ COMMENT ON FUNCTION worker_append_table_to_shard(text, text, text, integer) /* trigger functions */ -CREATE OR REPLACE FUNCTION citusdb_drop_trigger() +CREATE OR REPLACE FUNCTION citus_drop_trigger() RETURNS event_trigger LANGUAGE plpgsql SET search_path = pg_catalog @@ -349,7 +349,7 @@ BEGIN END LOOP; END; $cdbdt$; -COMMENT ON FUNCTION citusdb_drop_trigger() +COMMENT ON FUNCTION citus_drop_trigger() IS 'perform checks and actions at the end of DROP actions'; CREATE FUNCTION master_dist_partition_cache_invalidate() @@ -369,21 +369,21 @@ COMMENT ON FUNCTION master_dist_shard_cache_invalidate() /* internal functions, not user accessible */ -CREATE FUNCTION citusdb_extradata_container(INTERNAL) +CREATE FUNCTION citus_extradata_container(INTERNAL) RETURNS void LANGUAGE C - AS 'MODULE_PATHNAME', $$citusdb_extradata_container$$; -COMMENT ON FUNCTION pg_catalog.citusdb_extradata_container(INTERNAL) + AS 'MODULE_PATHNAME', $$citus_extradata_container$$; +COMMENT ON FUNCTION pg_catalog.citus_extradata_container(INTERNAL) IS 'placeholder function to store additional data in postgres node trees'; /***************************************************************************** - * CitusDB triggers + * Citus triggers *****************************************************************************/ -CREATE EVENT TRIGGER citusdb_cascade_to_partition +CREATE EVENT TRIGGER citus_cascade_to_partition ON SQL_DROP - EXECUTE PROCEDURE citusdb_drop_trigger(); + EXECUTE PROCEDURE citus_drop_trigger(); CREATE TRIGGER dist_partition_cache_invalidate AFTER INSERT OR UPDATE OR DELETE @@ -397,7 +397,7 @@ CREATE TRIGGER dist_shard_cache_invalidate /***************************************************************************** - * CitusDB aggregates + * Citus aggregates *****************************************************************************/ CREATE AGGREGATE array_cat_agg(anyarray) (SFUNC = array_cat, STYPE = anyarray); COMMENT ON AGGREGATE array_cat_agg(anyarray) diff --git a/src/backend/distributed/citusdb.control b/src/backend/distributed/citusdb.control deleted file mode 100644 index ae36339b5..000000000 --- a/src/backend/distributed/citusdb.control +++ /dev/null @@ -1,6 +0,0 @@ -# CitusDB extension -comment = 'CitusDB distributed database' -default_version = '5.0' -module_pathname = '$libdir/citusdb' -relocatable = false -schema = pg_catalog diff --git a/src/backend/distributed/commands/create_distributed_table.c b/src/backend/distributed/commands/create_distributed_table.c index b4a4c802b..d960b281f 100644 --- a/src/backend/distributed/commands/create_distributed_table.c +++ b/src/backend/distributed/commands/create_distributed_table.c @@ -165,7 +165,7 @@ master_create_distributed_table(PG_FUNCTION_ARGS) * * Similarly, do not allow UNIQUE constraint and/or PRIMARY KEY if it does not * include partition column. This check is important for two reasons. First, - * currently CitusDB does not enforce uniqueness constraint on multiple shards. + * currently Citus does not enforce uniqueness constraint on multiple shards. * Second, INSERT INTO .. ON CONFLICT (i.e., UPSERT) queries can be executed with no * further check for constraints. */ @@ -191,7 +191,7 @@ master_create_distributed_table(PG_FUNCTION_ARGS) } /* - * CitusDB cannot enforce uniqueness constraints with overlapping shards. Thus, + * Citus cannot enforce uniqueness constraints with overlapping shards. Thus, * emit a warning for unique indexes on append partitioned tables. */ if (distributionMethod == DISTRIBUTE_BY_APPEND) @@ -262,7 +262,7 @@ master_create_distributed_table(PG_FUNCTION_ARGS) * necessary for a distributed relation in addition to the preexisting ones * for a normal relation. * - * We create one dependency from the (now distributed) relation to the citusdb + * We create one dependency from the (now distributed) relation to the citus * extension to prevent the extension from being dropped while distributed * tables exist. Furthermore a dependency from pg_dist_partition's * distribution clause to the underlying columns is created, but it's marked @@ -281,7 +281,7 @@ RecordDistributedRelationDependencies(Oid distributedRelationId, Node *distribut relationAddr.objectSubId = 0; citusExtensionAddr.classId = ExtensionRelationId; - citusExtensionAddr.objectId = get_extension_oid("citusdb", false); + citusExtensionAddr.objectId = get_extension_oid("citus", false); citusExtensionAddr.objectSubId = 0; /* dependency from table entry to extension */ @@ -294,10 +294,10 @@ RecordDistributedRelationDependencies(Oid distributedRelationId, Node *distribut /* - * LookupDistributionMethod maps the oids of citusdb.distribution_type enum + * LookupDistributionMethod maps the oids of citus.distribution_type enum * values to pg_dist_partition.partmethod values. * - * The passed in oid has to belong to a value of citusdb.distribution_type. + * The passed in oid has to belong to a value of citus.distribution_type. */ static char LookupDistributionMethod(Oid distributionMethodOid) diff --git a/src/backend/distributed/executor/multi_server_executor.c b/src/backend/distributed/executor/multi_server_executor.c index 1f143778d..1672e5332 100644 --- a/src/backend/distributed/executor/multi_server_executor.c +++ b/src/backend/distributed/executor/multi_server_executor.c @@ -73,7 +73,7 @@ JobExecutorType(MultiPlan *multiPlan) ereport(WARNING, (errmsg("this query uses more connections than the " "configured max_connections limit"), errhint("Consider increasing max_connections or setting " - "citusdb.task_executor_type to " + "citus.task_executor_type to " "\"task-tracker\"."))); } @@ -88,7 +88,7 @@ JobExecutorType(MultiPlan *multiPlan) ereport(WARNING, (errmsg("this query uses more file descriptors than the " "configured max_files_per_process limit"), errhint("Consider increasing max_files_per_process or " - "setting citusdb.task_executor_type to " + "setting citus.task_executor_type to " "\"task-tracker\"."))); } @@ -96,7 +96,7 @@ JobExecutorType(MultiPlan *multiPlan) if (dependedJobCount > 0) { ereport(ERROR, (errmsg("cannot use real time executor with repartition jobs"), - errhint("Set citusdb.task_executor_type to " + errhint("Set citus.task_executor_type to " "\"task-tracker\"."))); } } @@ -119,7 +119,7 @@ JobExecutorType(MultiPlan *multiPlan) if (dependedJobCount > 0) { ereport(ERROR, (errmsg("cannot use router executor with repartition jobs"), - errhint("Set citusdb.task_executor_type to " + errhint("Set citus.task_executor_type to " "\"task-tracker\"."))); } @@ -128,7 +128,7 @@ JobExecutorType(MultiPlan *multiPlan) { ereport(ERROR, (errmsg("cannot use router executor with queries that " "hit multiple shards"), - errhint("Set citusdb.task_executor_type to \"real-time\" or " + errhint("Set citus.task_executor_type to \"real-time\" or " "\"task-tracker\"."))); } @@ -138,7 +138,7 @@ JobExecutorType(MultiPlan *multiPlan) if (list_length(workerDependentTaskList) > 0) { ereport(ERROR, (errmsg("cannot use router executor with JOINs"), - errhint("Set citusdb.task_executor_type to \"real-time\" or " + errhint("Set citus.task_executor_type to \"real-time\" or " "\"task-tracker\"."))); } @@ -146,7 +146,7 @@ JobExecutorType(MultiPlan *multiPlan) if (masterQuery != NULL && list_length(masterQuery->sortClause) > 0) { ereport(ERROR, (errmsg("cannot use router executor with ORDER BY clauses"), - errhint("Set citusdb.task_executor_type to \"real-time\" or " + errhint("Set citus.task_executor_type to \"real-time\" or " "\"task-tracker\"."))); } @@ -158,7 +158,7 @@ JobExecutorType(MultiPlan *multiPlan) if (masterQueryHasAggregates) { ereport(ERROR, (errmsg("cannot use router executor with aggregates"), - errhint("Set citusdb.task_executor_type to \"real-time\" or " + errhint("Set citus.task_executor_type to \"real-time\" or " "\"task-tracker\"."))); } } @@ -173,7 +173,7 @@ JobExecutorType(MultiPlan *multiPlan) * Every task requires 2 FDs, one file and one connection. Some FDs are taken by * the VFD pool and there is currently no way to reclaim these before opening a * connection. We therefore assume some FDs to be reserved for VFDs, based on - * observing a typical size of the pool on a CitusDB master. + * observing a typical size of the pool on a Citus master. */ int MaxMasterConnectionCount(void) diff --git a/src/backend/distributed/executor/multi_utility.c b/src/backend/distributed/executor/multi_utility.c index ccd05c021..f54d8c674 100644 --- a/src/backend/distributed/executor/multi_utility.c +++ b/src/backend/distributed/executor/multi_utility.c @@ -1,6 +1,6 @@ /*------------------------------------------------------------------------- * multi_utility.c - * CitusDB utility hook and related functionality. + * Citus utility hook and related functionality. * * Copyright (c) 2012-2015, Citus Data, Inc. *------------------------------------------------------------------------- @@ -76,7 +76,7 @@ static void RangeVarCallbackForDropIndex(const RangeVar *rel, Oid relOid, Oid ol /* - * Utility for handling citusdb specific concerns around utility statements. + * Utility for handling citus specific concerns around utility statements. * * There's two basic types of concerns here: * 1) Intercept utility statements that run after distributed query @@ -168,29 +168,29 @@ multi_ProcessUtility(Node *parsetree, /* * Inform the user about potential caveats. * - * To prevent failures in aborted transactions, CitusDBHasBeenLoaded() needs + * To prevent failures in aborted transactions, CitusHasBeenLoaded() needs * to be the second condition. See RelationIdGetRelation() which is called - * by CitusDBHasBeenLoaded(). + * by CitusHasBeenLoaded(). */ - if (IsA(parsetree, CreatedbStmt) && CitusDBHasBeenLoaded()) + if (IsA(parsetree, CreatedbStmt) && CitusHasBeenLoaded()) { - ereport(NOTICE, (errmsg("CitusDB partially supports CREATE DATABASE for " + ereport(NOTICE, (errmsg("Citus partially supports CREATE DATABASE for " "distributed databases"), - errdetail("CitusDB does not propagate CREATE DATABASE " + errdetail("Citus does not propagate CREATE DATABASE " "command to workers"), errhint("You can manually create a database and its " "extensions on workers."))); } - else if (IsA(parsetree, CreateSchemaStmt) && CitusDBHasBeenLoaded()) + else if (IsA(parsetree, CreateSchemaStmt) && CitusHasBeenLoaded()) { - ereport(NOTICE, (errmsg("CitusDB partially supports CREATE SCHEMA " + ereport(NOTICE, (errmsg("Citus partially supports CREATE SCHEMA " "for distributed databases"), errdetail("schema usage in joins and in some UDFs " - "provided by CitusDB are not supported yet"))); + "provided by Citus are not supported yet"))); } - else if (IsA(parsetree, CreateRoleStmt) && CitusDBHasBeenLoaded()) + else if (IsA(parsetree, CreateRoleStmt) && CitusHasBeenLoaded()) { - ereport(NOTICE, (errmsg("CitusDB does not support CREATE ROLE/USER " + ereport(NOTICE, (errmsg("Citus does not support CREATE ROLE/USER " "for distributed databases"), errdetail("Multiple roles are currently supported " "only for local tables"))); @@ -204,7 +204,7 @@ multi_ProcessUtility(Node *parsetree, /* * WarnIfDropCitusExtension prints a WARNING if dropStatement includes dropping - * citusdb extension. + * citus extension. */ static void WarnIfDropCitusExtension(DropStmt *dropStatement) @@ -218,8 +218,8 @@ WarnIfDropCitusExtension(DropStmt *dropStatement) List *objectNameList = lfirst(dropStatementObject); char *objectName = NameListToString(objectNameList); - /* we're only concerned with the citusdb extension */ - if (strncmp("citusdb", objectName, NAMEDATALEN) == 0) + /* we're only concerned with the citus extension */ + if (strncmp("citus", objectName, NAMEDATALEN) == 0) { /* * Warn the user about the possibility of invalid cache. Also, see @@ -296,7 +296,7 @@ VerifyTransmitStmt(CopyStmt *copyStatement) /* - * ProcessCopyStmt handles CitusDB specific concerns for COPY like supporting + * ProcessCopyStmt handles Citus specific concerns for COPY like supporting * COPYing from distributed tables and preventing unsupported actions. */ static Node * diff --git a/src/backend/distributed/planner/multi_explain.c b/src/backend/distributed/planner/multi_explain.c index 35e0ca79b..45bc52b36 100644 --- a/src/backend/distributed/planner/multi_explain.c +++ b/src/backend/distributed/planner/multi_explain.c @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * multi_explain.c - * CitusDB explain support. + * Citus explain support. * * Copyright (c) 2012-2015, Citus Data, Inc. *------------------------------------------------------------------------- diff --git a/src/backend/distributed/planner/multi_join_order.c b/src/backend/distributed/planner/multi_join_order.c index e8cc68f07..ad47a518b 100644 --- a/src/backend/distributed/planner/multi_join_order.c +++ b/src/backend/distributed/planner/multi_join_order.c @@ -175,7 +175,7 @@ FixedJoinOrderList(FromExpr *fromExpr, List *tableEntryList) "query"), errdetail("Cannot perform outer joins with broadcast " "joins of more than 1 shard"), - errhint("Set citusdb.large_table_shard_count to 1"))); + errhint("Set citus.large_table_shard_count to 1"))); } } else if (nextJoinNode->joinRuleType == LOCAL_PARTITION_JOIN) diff --git a/src/backend/distributed/planner/multi_planner.c b/src/backend/distributed/planner/multi_planner.c index e6cd82b50..28780fbdf 100644 --- a/src/backend/distributed/planner/multi_planner.c +++ b/src/backend/distributed/planner/multi_planner.c @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * multi_planner.c - * General CitusDB planner code. + * General Citus planner code. * * Copyright (c) 2012-2015, Citus Data, Inc. *------------------------------------------------------------------------- @@ -45,7 +45,7 @@ multi_planner(Query *parse, int cursorOptions, ParamListInfo boundParams) PlannedStmt *result = NULL; /* - * First call into standard planner. This is required because the CitusDB + * First call into standard planner. This is required because the Citus * planner relies on parse tree transformations made by postgres' planner. */ result = standard_planner(parse, cursorOptions, boundParams); @@ -90,7 +90,7 @@ CreatePhysicalPlan(Query *parse) /* * This check is here to make it likely that all node types used in - * CitusDB are dumpable. Explain can dump logical and physical plans + * Citus are dumpable. Explain can dump logical and physical plans * using the extended outfuncs infrastructure, but it's infeasible to * test most plans. MultiQueryContainerNode always serializes the * physical plan, so there's no need to check that separately. @@ -132,7 +132,7 @@ HasCitusToplevelNode(PlannedStmt *result) * yet. Directly return false, part of the required infrastructure for * further checks might not be present. */ - if (!CitusDBHasBeenLoaded()) + if (!CitusHasBeenLoaded()) { return false; } @@ -259,7 +259,7 @@ GetMultiPlanString(PlannedStmt *result) if (list_length(fauxFuncExpr->args) != 1) { ereport(ERROR, (errmsg("unexpected number of function arguments to " - "citusdb_extradata_container"))); + "citus_extradata_container"))); } multiPlanData = (Const *) linitial(fauxFuncExpr->args); diff --git a/src/backend/distributed/shared_library_init.c b/src/backend/distributed/shared_library_init.c index 344665eb0..1d3099f80 100644 --- a/src/backend/distributed/shared_library_init.c +++ b/src/backend/distributed/shared_library_init.c @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * shared_library_init.c - * Initialize CitusDB extension + * Initialize Citus extension * * Copyright (c) 2012-2015, Citus Data, Inc. *------------------------------------------------------------------------- @@ -74,8 +74,8 @@ _PG_init(void) { if (!process_shared_preload_libraries_in_progress) { - ereport(ERROR, (errmsg("CitusDB can only be loaded via shared_preload_libraries"), - errhint("Add citusdb to shared_preload_libraries."))); + ereport(ERROR, (errmsg("Citus can only be loaded via shared_preload_libraries"), + errhint("Add citus to shared_preload_libraries."))); } /* @@ -95,8 +95,8 @@ _PG_init(void) ExecutorEnd_hook != NULL || ProcessUtility_hook != NULL) { - ereport(ERROR, (errmsg("CitusDB has to be loaded first"), - errhint("Place citusdb at the beginning of " + ereport(ERROR, (errmsg("Citus has to be loaded first"), + errhint("Place citus at the beginning of " "shared_preload_libraries."))); } @@ -107,7 +107,7 @@ _PG_init(void) CreateRequiredDirectories(); /* - * Register CitusDB configuration variables. Do so before intercepting + * Register Citus configuration variables. Do so before intercepting * hooks or calling initialization functions, in case we want to do the * latter in a configuration dependent manner. */ @@ -137,7 +137,7 @@ _PG_init(void) /* - * CreateRequiredDirectories - Create directories required for CitusDB to + * CreateRequiredDirectories - Create directories required for Citus to * function. * * These used to be created by initdb, but that's not possible anymore. @@ -166,12 +166,12 @@ CreateRequiredDirectories(void) } -/* Register CitusDB configuration variables. */ +/* Register Citus configuration variables. */ static void RegisterCitusConfigVariables(void) { DefineCustomStringVariable( - "citusdb.worker_list_file", + "citus.worker_list_file", gettext_noop("Sets the server's \"worker_list\" configuration file."), NULL, &WorkerListFileName, @@ -182,7 +182,7 @@ RegisterCitusConfigVariables(void) NormalizeWorkerListPath(); DefineCustomBoolVariable( - "citusdb.binary_master_copy_format", + "citus.binary_master_copy_format", gettext_noop("Use the binary master copy format."), gettext_noop("When enabled, data is copied from workers to the master " "in PostgreSQL's binary serialization format."), @@ -193,7 +193,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomBoolVariable( - "citusdb.binary_worker_copy_format", + "citus.binary_worker_copy_format", gettext_noop("Use the binary worker copy format."), gettext_noop("When enabled, data is copied from workers to workers " "in PostgreSQL's binary serialization format when " @@ -205,7 +205,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomBoolVariable( - "citusdb.expire_cached_shards", + "citus.expire_cached_shards", gettext_noop("Enables shard cache expiration if a shard's size on disk has changed. "), gettext_noop("When appending to an existing shard, old data may still be cached on " "other workers. This configuration entry activates automatic " @@ -217,7 +217,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomBoolVariable( - "citusdb.subquery_pushdown", + "citus.subquery_pushdown", gettext_noop("Enables supported subquery pushdown to workers."), NULL, &SubqueryPushdown, @@ -227,7 +227,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomBoolVariable( - "citusdb.log_multi_join_order", + "citus.log_multi_join_order", gettext_noop("Logs the distributed join order to the server log."), gettext_noop("We use this private configuration entry as a debugging aid. " "If enabled, we print the distributed join order."), @@ -238,7 +238,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomBoolVariable( - "citusdb.explain_multi_logical_plan", + "citus.explain_multi_logical_plan", gettext_noop("Enables Explain to print out distributed logical plans."), gettext_noop("We use this private configuration entry as a debugging aid. " "If enabled, the Explain command prints out the optimized " @@ -250,7 +250,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomBoolVariable( - "citusdb.explain_multi_physical_plan", + "citus.explain_multi_physical_plan", gettext_noop("Enables Explain to print out distributed physical plans."), gettext_noop("We use this private configuration entry as a debugging aid. " "If enabled, the Explain command prints out the physical " @@ -262,7 +262,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomBoolVariable( - "citusdb.all_modifications_commutative", + "citus.all_modifications_commutative", gettext_noop("Bypasses commutativity checks when enabled"), NULL, &AllModificationsCommutative, @@ -272,7 +272,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomIntVariable( - "citusdb.shard_replication_factor", + "citus.shard_replication_factor", gettext_noop("Sets the replication factor for shards."), gettext_noop("Shards are replicated across nodes according to this " "replication factor. Note that shards read this " @@ -285,7 +285,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomIntVariable( - "citusdb.shard_max_size", + "citus.shard_max_size", gettext_noop("Sets the maximum size a shard will grow before it gets split."), gettext_noop("Shards store table and file data. When the source " "file's size for one shard exceeds this configuration " @@ -300,7 +300,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomIntVariable( - "citusdb.max_worker_nodes_tracked", + "citus.max_worker_nodes_tracked", gettext_noop("Sets the maximum number of worker nodes that are tracked."), gettext_noop("Worker nodes' network locations, their membership and " "health status are tracked in a shared hash table on " @@ -314,7 +314,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomIntVariable( - "citusdb.remote_task_check_interval", + "citus.remote_task_check_interval", gettext_noop("Sets the frequency at which we check job statuses."), gettext_noop("The master node assigns tasks to workers nodes, and " "then regularly checks with them about each task's " @@ -327,7 +327,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomIntVariable( - "citusdb.task_tracker_delay", + "citus.task_tracker_delay", gettext_noop("Task tracker sleep time between task management rounds."), gettext_noop("The task tracker process wakes up regularly, walks over " "all tasks assigned to it, and schedules and executes these " @@ -341,7 +341,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomIntVariable( - "citusdb.max_assign_task_batch_size", + "citus.max_assign_task_batch_size", gettext_noop("Sets the maximum number of tasks to assign per round."), gettext_noop("The master node synchronously assigns tasks to workers in " "batches. Bigger batches allow for faster task assignment, " @@ -355,7 +355,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomIntVariable( - "citusdb.max_tracked_tasks_per_node", + "citus.max_tracked_tasks_per_node", gettext_noop("Sets the maximum number of tracked tasks per node."), gettext_noop("The task tracker processes keeps all assigned tasks in " "a shared hash table, and schedules and executes these " @@ -369,7 +369,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomIntVariable( - "citusdb.max_running_tasks_per_node", + "citus.max_running_tasks_per_node", gettext_noop("Sets the maximum number of tasks to run concurrently per node."), gettext_noop("The task tracker process schedules and executes the tasks " "assigned to it as appropriate. This configuration value " @@ -382,7 +382,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomIntVariable( - "citusdb.partition_buffer_size", + "citus.partition_buffer_size", gettext_noop("Sets the buffer size to use for partition operations."), gettext_noop("Worker nodes allow for table data to be repartitioned " "into multiple text files, much like Hadoop's Map " @@ -396,7 +396,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomIntVariable( - "citusdb.large_table_shard_count", + "citus.large_table_shard_count", gettext_noop("The shard count threshold over which a table is considered large."), gettext_noop("A distributed table is considered to be large if it has " "more shards than the value specified here. This largeness " @@ -409,7 +409,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomIntVariable( - "citusdb.limit_clause_row_fetch_count", + "citus.limit_clause_row_fetch_count", gettext_noop("Number of rows to fetch per task for limit clause optimization."), gettext_noop("Select queries get partitioned and executed as smaller " "tasks. In some cases, select queries with limit clauses " @@ -424,7 +424,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomRealVariable( - "citusdb.count_distinct_error_rate", + "citus.count_distinct_error_rate", gettext_noop("Desired error rate when calculating count(distinct) " "approximates using the postgresql-hll extension. " "0.0 disables approximations for count(distinct); 1.0 " @@ -437,7 +437,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomEnumVariable( - "citusdb.task_assignment_policy", + "citus.task_assignment_policy", gettext_noop("Sets the policy to use when assigning tasks to worker nodes."), gettext_noop("The master node assigns tasks to worker nodes based on shard " "locations. This configuration value specifies the policy to " @@ -454,7 +454,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomEnumVariable( - "citusdb.task_executor_type", + "citus.task_executor_type", gettext_noop("Sets the executor type to be used for distributed queries."), gettext_noop("The master node chooses between three different executor types " "when executing a distributed query. The router executor is " @@ -472,7 +472,7 @@ RegisterCitusConfigVariables(void) NULL, NULL, NULL); DefineCustomEnumVariable( - "citusdb.shard_placement_policy", + "citus.shard_placement_policy", gettext_noop("Sets the policy to use when choosing nodes for shard placement."), gettext_noop("The master node chooses which worker nodes to place new shards " "on. This configuration value specifies the policy to use when " @@ -486,8 +486,8 @@ RegisterCitusConfigVariables(void) 0, NULL, NULL, NULL); - /* warn about config items in the citusdb namespace that are not registered above */ - EmitWarningsOnPlaceholders("citusdb"); + /* warn about config items in the citus namespace that are not registered above */ + EmitWarningsOnPlaceholders("citus"); /* Also warn about citus namespace, as that's a very likely misspelling */ EmitWarningsOnPlaceholders("citus"); } @@ -495,7 +495,7 @@ RegisterCitusConfigVariables(void) /* * NormalizeWorkerListPath converts the path configured via - * citusdb.worker_list_file into an absolute path, falling back to the default + * citus.worker_list_file into an absolute path, falling back to the default * value if necessary. The previous value of the config variable is * overwritten with the normalized value. * @@ -525,11 +525,11 @@ NormalizeWorkerListPath(void) ereport(FATAL, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("%s does not know where to find the \"worker_list_file\" " "configuration file.\n" - "This can be specified as \"citusdb.worker_list_file\" in " + "This can be specified as \"citus.worker_list_file\" in " "\"%s\", or by the -D invocation option, or by the PGDATA " "environment variable.\n", progname, ConfigFileName))); } - SetConfigOption("citusdb.worker_list_file", absoluteFileName, PGC_POSTMASTER, PGC_S_OVERRIDE); + SetConfigOption("citus.worker_list_file", absoluteFileName, PGC_POSTMASTER, PGC_S_OVERRIDE); free(absoluteFileName); } diff --git a/src/backend/distributed/test/connection_cache.c b/src/backend/distributed/test/connection_cache.c index 1b256695c..691020a55 100644 --- a/src/backend/distributed/test/connection_cache.c +++ b/src/backend/distributed/test/connection_cache.c @@ -2,7 +2,7 @@ * * test/src/connection_cache.c * - * This file contains functions to exercise CitusDB's connection hash + * This file contains functions to exercise Citus's connection hash * functionality for purposes of unit testing. * * Copyright (c) 2014-2015, Citus Data, Inc. diff --git a/src/backend/distributed/test/create_shards.c b/src/backend/distributed/test/create_shards.c index e6b2b5762..eb8a4d8af 100644 --- a/src/backend/distributed/test/create_shards.c +++ b/src/backend/distributed/test/create_shards.c @@ -3,7 +3,7 @@ * test/src/create_shards.c * * This file contains functions to exercise shard creation functionality - * within CitusDB. + * within Citus. * * Copyright (c) 2014-2015, Citus Data, Inc. * diff --git a/src/backend/distributed/test/distribution_metadata.c b/src/backend/distributed/test/distribution_metadata.c index f54359a19..83bb02c7c 100644 --- a/src/backend/distributed/test/distribution_metadata.c +++ b/src/backend/distributed/test/distribution_metadata.c @@ -3,7 +3,7 @@ * test/src/distribution_metadata.c * * This file contains functions to exercise distributed table metadata - * functionality within CitusDB. + * functionality within Citus. * * Copyright (c) 2014-2015, Citus Data, Inc. * diff --git a/src/backend/distributed/test/generate_ddl_commands.c b/src/backend/distributed/test/generate_ddl_commands.c index 4cbc225cf..4b6ea04bc 100644 --- a/src/backend/distributed/test/generate_ddl_commands.c +++ b/src/backend/distributed/test/generate_ddl_commands.c @@ -3,7 +3,7 @@ * test/src/generate_ddl_commands.c * * This file contains functions to exercise DDL generation functionality - * within CitusDB. + * within Citus. * * Copyright (c) 2014-2015, Citus Data, Inc. * diff --git a/src/backend/distributed/test/prune_shard_list.c b/src/backend/distributed/test/prune_shard_list.c index f722a70ec..b7edf661e 100644 --- a/src/backend/distributed/test/prune_shard_list.c +++ b/src/backend/distributed/test/prune_shard_list.c @@ -3,7 +3,7 @@ * test/src/create_shards.c * * This file contains functions to exercise shard creation functionality - * within CitusDB. + * within Citus. * * Copyright (c) 2014-2015, Citus Data, Inc. * diff --git a/src/backend/distributed/test/test_helper_functions.c b/src/backend/distributed/test/test_helper_functions.c index bc463c4c6..28458d96d 100644 --- a/src/backend/distributed/test/test_helper_functions.c +++ b/src/backend/distributed/test/test_helper_functions.c @@ -2,7 +2,7 @@ * * test/src/test_helper_functions.c * - * This file contains helper functions used in many CitusDB tests. + * This file contains helper functions used in many Citus tests. * * Copyright (c) 2014-2015, Citus Data, Inc. * diff --git a/src/backend/distributed/utils/citus_nodefuncs.c b/src/backend/distributed/utils/citus_nodefuncs.c index b3858cbdf..4e5a40260 100644 --- a/src/backend/distributed/utils/citus_nodefuncs.c +++ b/src/backend/distributed/utils/citus_nodefuncs.c @@ -16,7 +16,7 @@ /* exports for SQL callable functions */ -PG_FUNCTION_INFO_V1(citusdb_extradata_container); +PG_FUNCTION_INFO_V1(citus_extradata_container); /* @@ -189,7 +189,7 @@ ExtractRangeTblExtraData(RangeTblEntry *rte, CitusRTEKind *rteKind, if (list_length(fauxFuncExpr->args) != 4) { ereport(ERROR, (errmsg("unexpected number of function arguments to " - "citusdb_extradata_container"))); + "citus_extradata_container"))); return; } @@ -290,13 +290,13 @@ GetRangeTblKind(RangeTblEntry *rte) /* - * citusdb_extradata_container is a placeholder function to store information - * needed by CitusDB in plain postgres node trees. Executor and other hooks + * citus_extradata_container is a placeholder function to store information + * needed by Citus in plain postgres node trees. Executor and other hooks * should always intercept statements containing calls to this function. It's * not actually SQL callable by the user because of an INTERNAL argument. */ Datum -citusdb_extradata_container(PG_FUNCTION_ARGS) +citus_extradata_container(PG_FUNCTION_ARGS) { ereport(ERROR, (errmsg("not supposed to get here, did you cheat?"))); diff --git a/src/backend/distributed/utils/citus_outfuncs.c b/src/backend/distributed/utils/citus_outfuncs.c index bb620987a..babd10620 100644 --- a/src/backend/distributed/utils/citus_outfuncs.c +++ b/src/backend/distributed/utils/citus_outfuncs.c @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * citus_outfuncs.c - * Output functions for CitusDB tree nodes. + * Output functions for Citus tree nodes. * * Portions Copyright (c) 1996-2014, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California @@ -9,7 +9,7 @@ * * NOTES * This is a wrapper around postgres' nodeToString() that additionally - * supports CitusDB node types. + * supports Citus node types. * * Keep as closely aligned with the upstream version as possible. * @@ -220,7 +220,7 @@ _outDatum(StringInfo str, Datum value, int typlen, bool typbyval) /***************************************************************************** - * Output routines for CitusDB node types + * Output routines for Citus node types *****************************************************************************/ static void diff --git a/src/backend/distributed/utils/citus_readfuncs_94.c b/src/backend/distributed/utils/citus_readfuncs_94.c index ace76bb3a..7cbf124c4 100644 --- a/src/backend/distributed/utils/citus_readfuncs_94.c +++ b/src/backend/distributed/utils/citus_readfuncs_94.c @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * citus_readfuncs.c - * CitusDB adapted reader functions for Citus & Postgres tree nodes + * Citus adapted reader functions for Citus & Postgres tree nodes * * Portions Copyright (c) 1996-2014, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California diff --git a/src/backend/distributed/utils/citus_readfuncs_95.c b/src/backend/distributed/utils/citus_readfuncs_95.c index 84866f632..87b1d827a 100644 --- a/src/backend/distributed/utils/citus_readfuncs_95.c +++ b/src/backend/distributed/utils/citus_readfuncs_95.c @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * citus_readfuncs.c - * CitusDB adapted reader functions for Citus & Postgres tree nodes + * Citus adapted reader functions for Citus & Postgres tree nodes * * Portions Copyright (c) 1996-2014, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California diff --git a/src/backend/distributed/utils/connection_cache.c b/src/backend/distributed/utils/connection_cache.c index a4aa14cd7..a7391aa5b 100644 --- a/src/backend/distributed/utils/connection_cache.c +++ b/src/backend/distributed/utils/connection_cache.c @@ -249,7 +249,7 @@ CreateNodeConnectionHash(void) info.hcxt = CacheMemoryContext; hashFlags = (HASH_ELEM | HASH_FUNCTION | HASH_CONTEXT); - nodeConnectionHash = hash_create("citusdb connection cache", 32, &info, hashFlags); + nodeConnectionHash = hash_create("citus connection cache", 32, &info, hashFlags); return nodeConnectionHash; } @@ -257,7 +257,7 @@ CreateNodeConnectionHash(void) /* * ConnectToNode opens a connection to a remote PostgreSQL server. The function - * configures the connection's fallback application name to 'citusdb' and sets + * configures the connection's fallback application name to 'citus' and sets * the remote encoding to match the local one. This function requires that the * port be specified as a string for easier use with libpq functions. * @@ -277,7 +277,7 @@ ConnectToNode(char *nodeName, char *nodePort) "client_encoding", "connect_timeout", "dbname", NULL }; const char *valueArray[] = { - nodeName, nodePort, "citusdb", clientEncoding, + nodeName, nodePort, "citus", clientEncoding, CLIENT_CONNECT_TIMEOUT_SECONDS, dbname, NULL }; diff --git a/src/backend/distributed/utils/metadata_cache.c b/src/backend/distributed/utils/metadata_cache.c index 868b11313..910b90f73 100644 --- a/src/backend/distributed/utils/metadata_cache.c +++ b/src/backend/distributed/utils/metadata_cache.c @@ -77,7 +77,7 @@ IsDistributedTable(Oid relationId) * yet. As we can't do lookups in nonexistent tables, directly return * false. */ - if (!CitusDBHasBeenLoaded()) + if (!CitusHasBeenLoaded()) { return false; } @@ -155,7 +155,7 @@ DistributedTableCacheEntry(Oid distributedRelationId) * yet. As we can't do lookups in nonexistent tables, directly return NULL * here. */ - if (!CitusDBHasBeenLoaded()) + if (!CitusHasBeenLoaded()) { return NULL; } @@ -292,7 +292,7 @@ LookupDistTableCacheEntry(Oid relationId) /* - * CitusDBHasBeenLoaded returns true if the citusdb extension has been created + * CitusHasBeenLoaded returns true if the citus extension has been created * in the current database and the extension script has been executed. Otherwise, * it returns false. The result is cached as this is called very frequently. * @@ -301,17 +301,17 @@ LookupDistTableCacheEntry(Oid relationId) * acceptable. */ bool -CitusDBHasBeenLoaded(void) +CitusHasBeenLoaded(void) { static bool extensionLoaded = false; - /* recheck presence until citusdb has been loaded */ + /* recheck presence until citus has been loaded */ if (!extensionLoaded) { bool extensionPresent = false; bool extensionScriptExecuted = true; - Oid extensionOid = get_extension_oid("citusdb", true); + Oid extensionOid = get_extension_oid("citus", true); if (extensionOid != InvalidOid) { extensionPresent = true; @@ -319,7 +319,7 @@ CitusDBHasBeenLoaded(void) if (extensionPresent) { - /* check if CitusDB extension objects are still being created */ + /* check if Citus extension objects are still being created */ if (creating_extension && CurrentExtensionObject == extensionOid) { extensionScriptExecuted = false; @@ -428,7 +428,7 @@ CitusExtraDataContainerFuncId(void) if (cachedOid == InvalidOid) { nameList = list_make2(makeString("pg_catalog"), - makeString("citusdb_extradata_container")); + makeString("citus_extradata_container")); cachedOid = LookupFuncName(nameList, 1, paramOids, false); } diff --git a/src/backend/distributed/utils/multi_resowner.c b/src/backend/distributed/utils/multi_resowner.c index 80aecf7e5..597360ffd 100644 --- a/src/backend/distributed/utils/multi_resowner.c +++ b/src/backend/distributed/utils/multi_resowner.c @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * multi_resowner.c - * CitusDB resource owner integration + * Citus resource owner integration * * An extension can't directly add members to ResourceOwnerData. Instead we * have to use the resource owner callback mechanism. Right now it's diff --git a/src/backend/distributed/utils/resource_lock.c b/src/backend/distributed/utils/resource_lock.c index a2552d46b..1147cc0e6 100644 --- a/src/backend/distributed/utils/resource_lock.c +++ b/src/backend/distributed/utils/resource_lock.c @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * resource_lock.c - * Locking Infrastructure for CitusDB. + * Locking Infrastructure for Citus. * * To avoid introducing a new type of locktag - that then could not be * displayed by core functionality - we reuse advisory locks. If we'd just diff --git a/src/backend/distributed/worker/task_tracker.c b/src/backend/distributed/worker/task_tracker.c index b8a6bde6a..6c69e8d7a 100644 --- a/src/backend/distributed/worker/task_tracker.c +++ b/src/backend/distributed/worker/task_tracker.c @@ -240,7 +240,7 @@ TaskTrackerMain(Datum main_arg) /* * Reload worker membership file. For now we do that in the task * tracker because that's currently the only background worker in - * CitusDB. And only background workers allow us to safely + * Citus. And only background workers allow us to safely * register a SIGHUP handler. */ LoadWorkerNodeList(WorkerListFileName); @@ -295,7 +295,7 @@ WorkerTasksHashEnter(uint64 jobId, uint32 taskId) { ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of shared memory"), - errhint("Try increasing citusdb.max_tracked_tasks_per_node."))); + errhint("Try increasing citus.max_tracked_tasks_per_node."))); } /* check that we do not have the same task assigned twice to this node */ diff --git a/src/bin/csql/Makefile b/src/bin/csql/Makefile index cbfe7c01f..2c1ae79af 100644 --- a/src/bin/csql/Makefile +++ b/src/bin/csql/Makefile @@ -9,12 +9,12 @@ # #------------------------------------------------------------------------- -citusdb_subdir = src/bin/csql -citusdb_top_builddir = ../../.. +citus_subdir = src/bin/csql +citus_top_builddir = ../../.. PROGRAM = csql -PGFILEDESC = "csql - the CitusDB interactive terminal" +PGFILEDESC = "csql - the Citus interactive terminal" PGAPPICON=win32 OBJS =command.o common.o help.o input.o stringutils.o mainloop.o copy.o \ @@ -26,7 +26,7 @@ OBJS =command.o common.o help.o input.o stringutils.o mainloop.o copy.o \ PG_LIBS = $(libpq) -include $(citusdb_top_builddir)/Makefile.global +include $(citus_top_builddir)/Makefile.global override CPPFLAGS += -I$(libpq_srcdir) -I$(top_srcdir)/src/bin/csql diff --git a/src/bin/csql/copy_options.c b/src/bin/csql/copy_options.c index 357ae726c..866344e4e 100644 --- a/src/bin/csql/copy_options.c +++ b/src/bin/csql/copy_options.c @@ -1,5 +1,5 @@ /* - * csql - the CitusDB interactive terminal + * csql - the Citus interactive terminal * copy_options.c * Routines for parsing copy and stage meta commands. * diff --git a/src/bin/csql/copy_options.h b/src/bin/csql/copy_options.h index 07a3aeb09..91ac98028 100644 --- a/src/bin/csql/copy_options.h +++ b/src/bin/csql/copy_options.h @@ -1,5 +1,5 @@ /* - * csql - the CitusDB interactive terminal + * csql - the Citus interactive terminal * copy_options.h * Shared declarations for parsing copy and stage meta-commands. The stage * meta-command borrows from copy's syntax, but does not yet support diff --git a/src/bin/csql/help.c b/src/bin/csql/help.c index 10304564e..b5c2e405e 100644 --- a/src/bin/csql/help.c +++ b/src/bin/csql/help.c @@ -67,7 +67,7 @@ usage(unsigned short int pager) output = PageOutput(59, pager ? &(pset.popt.topt) : NULL); - printf(_("csql is the CitusDB interactive terminal.\n\n")); + printf(_("csql is the Citus interactive terminal.\n\n")); fprintf(output, _("Usage:\n")); printf(_(" csql [OPTION]... [DBNAME [USERNAME]]\n\n")); diff --git a/src/bin/csql/mainloop.c b/src/bin/csql/mainloop.c index 507dd5986..65070e643 100644 --- a/src/bin/csql/mainloop.c +++ b/src/bin/csql/mainloop.c @@ -200,7 +200,7 @@ MainLoop(FILE *source) (line[4] == '\0' || line[4] == ';' || isspace((unsigned char) line[4]))) { free(line); - puts(_("You are using csql, the command-line interface to CitusDB.")); + puts(_("You are using csql, the command-line interface to Citus.")); printf(_("Type: \\copyright for distribution terms\n" " \\h for help with SQL commands\n" " \\? for help with csql commands\n" diff --git a/src/bin/csql/stage.c b/src/bin/csql/stage.c index 1863b8bcb..9d47184b9 100644 --- a/src/bin/csql/stage.c +++ b/src/bin/csql/stage.c @@ -1,5 +1,5 @@ /* - * csql - the CitusDB interactive terminal + * csql - the Citus interactive terminal * stage.c * Helper routines to execute the csql meta-command \stage. These routines * communicate with the master and worker nodes; and create new shards and diff --git a/src/bin/csql/stage.h b/src/bin/csql/stage.h index 01575f886..b7e75ac9c 100644 --- a/src/bin/csql/stage.h +++ b/src/bin/csql/stage.h @@ -1,5 +1,5 @@ /* - * csql - the CitusDB interactive terminal + * csql - the Citus interactive terminal * stage.h * Declarations for the csql meta-command \stage. These declarations define a * protocol for the client to communicate to the master and worker nodes. diff --git a/src/include/.gitignore b/src/include/.gitignore index 30f297ab7..5207873bf 100644 --- a/src/include/.gitignore +++ b/src/include/.gitignore @@ -1,4 +1,4 @@ /stamp-h /stamp-ext-h -/citusdb_config.h -/citusdb_config.h.in~ +/citus_config.h +/citus_config.h.in~ diff --git a/src/include/citusdb_config.h.in b/src/include/citus_config.h.in similarity index 71% rename from src/include/citusdb_config.h.in rename to src/include/citus_config.h.in index 61ced741c..33da446e7 100644 --- a/src/include/citusdb_config.h.in +++ b/src/include/citus_config.h.in @@ -1,9 +1,9 @@ -/* src/include/citusdb_config.h.in. Generated from configure.in by autoheader. */ +/* src/include/citus_config.h.in. Generated from configure.in by autoheader. */ /* - * citusdb_config.h.in is generated by autoconf/autoheader and - * converted into citusdb_config.h by configure. Include when code needs to + * citus_config.h.in is generated by autoconf/autoheader and + * converted into citus_config.h by configure. Include when code needs to * depend on determinations made by configure. * * Do not manually edit! diff --git a/src/include/distributed/citus_nodefuncs.h b/src/include/distributed/citus_nodefuncs.h index cd273d587..e0c8373e5 100644 --- a/src/include/distributed/citus_nodefuncs.h +++ b/src/include/distributed/citus_nodefuncs.h @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * citus_nodefuncs.h - * Node (de-)serialization support for CitusDB. + * Node (de-)serialization support for Citus. * * Copyright (c) 2012-2015, Citus Data, Inc. * diff --git a/src/include/distributed/citus_nodes.h b/src/include/distributed/citus_nodes.h index 05a536cf0..3bab073b1 100644 --- a/src/include/distributed/citus_nodes.h +++ b/src/include/distributed/citus_nodes.h @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * citus_nodes.h - * Additional node types, and related infrastructure, for CitusDB. + * Additional node types, and related infrastructure, for Citus. * * Copyright (c) 2012-2015, Citus Data, Inc. * @@ -52,7 +52,7 @@ typedef enum CitusNodeTag #define CitusIsA(nodeptr,_type_) (CitusNodeTag(nodeptr) == T_##_type_) -/* CitusDB variant of newNode(), don't use directly. */ +/* Citus variant of newNode(), don't use directly. */ #define CitusNewNode(size, tag) \ ({ Node *_result; \ AssertMacro((size) >= sizeof(Node)); /* need the tag, at least */ \ @@ -63,7 +63,7 @@ typedef enum CitusNodeTag /* - * CitusMakeNode is CitusDB variant of makeNode(). Use it to create nodes of + * CitusMakeNode is Citus variant of makeNode(). Use it to create nodes of * the types listed in the CitusNodeTag enum and plain NodeTag. Initializes * memory, besides the node tag, to 0. */ diff --git a/src/include/distributed/citus_ruleutils.h b/src/include/distributed/citus_ruleutils.h index 0f2402748..6cf837c7e 100644 --- a/src/include/distributed/citus_ruleutils.h +++ b/src/include/distributed/citus_ruleutils.h @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * citus_ruleutils.h - * CitusDB ruleutils wrapper functions and exported PostgreSQL ruleutils + * Citus ruleutils wrapper functions and exported PostgreSQL ruleutils * functions. * * Copyright (c) 2012-2015, Citus Data, Inc. @@ -15,7 +15,7 @@ #include "nodes/parsenodes.h" -/* Function declarations for version independent CitusDB ruleutils wrapper functions */ +/* Function declarations for version independent Citus ruleutils wrapper functions */ extern char *pg_get_extensiondef_string(Oid tableRelationId); extern char *pg_get_serverdef_string(Oid tableRelationId); extern char *pg_get_tableschemadef_string(Oid tableRelationId); diff --git a/src/include/distributed/listutils.h b/src/include/distributed/listutils.h index 3e9cd3917..62b4ec6f1 100644 --- a/src/include/distributed/listutils.h +++ b/src/include/distributed/listutils.h @@ -9,8 +9,8 @@ *------------------------------------------------------------------------- */ -#ifndef CITUSDB_LISTUTILS_H -#define CITUSDB_LISTUTILS_H +#ifndef CITUS_LISTUTILS_H +#define CITUS_LISTUTILS_H #include "postgres.h" #include "c.h" @@ -23,4 +23,4 @@ extern List * SortList(List *pointerList, int (*ComparisonFunction)(const void *, const void *)); -#endif /* CITUSDB_LISTUTILS_H */ +#endif /* CITUS_LISTUTILS_H */ diff --git a/src/include/distributed/metadata_cache.h b/src/include/distributed/metadata_cache.h index 0c002b8da..d439d0456 100644 --- a/src/include/distributed/metadata_cache.h +++ b/src/include/distributed/metadata_cache.h @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * metadata_cache.h - * Executor support for CitusDB. + * Executor support for Citus. * * Copyright (c) 2012-2015, Citus Data, Inc. * @@ -46,7 +46,7 @@ extern bool IsDistributedTable(Oid relationId); extern ShardInterval * LoadShardInterval(uint64 shardId); extern DistTableCacheEntry * DistributedTableCacheEntry(Oid distributedRelationId); -extern bool CitusDBHasBeenLoaded(void); +extern bool CitusHasBeenLoaded(void); /* relation oids */ extern Oid DistPartitionRelationId(void); diff --git a/src/include/distributed/multi_executor.h b/src/include/distributed/multi_executor.h index bcf22fe2e..5f82f10ac 100644 --- a/src/include/distributed/multi_executor.h +++ b/src/include/distributed/multi_executor.h @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * multi_executor.h - * Executor support for CitusDB. + * Executor support for Citus. * * Copyright (c) 2012-2015, Citus Data, Inc. *------------------------------------------------------------------------- diff --git a/src/include/distributed/multi_explain.h b/src/include/distributed/multi_explain.h index 71ed52948..95912a38c 100644 --- a/src/include/distributed/multi_explain.h +++ b/src/include/distributed/multi_explain.h @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * multi_explain.h - * Explain support for CitusDB. + * Explain support for Citus. * * Copyright (c) 2012-2015, Citus Data, Inc. *------------------------------------------------------------------------- diff --git a/src/include/distributed/multi_planner.h b/src/include/distributed/multi_planner.h index a14b2b65e..1a90e550a 100644 --- a/src/include/distributed/multi_planner.h +++ b/src/include/distributed/multi_planner.h @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * multi_planner.h - * General CitusDB planner code. + * General Citus planner code. * * Copyright (c) 2012-2015, Citus Data, Inc. *------------------------------------------------------------------------- diff --git a/src/include/distributed/multi_resowner.h b/src/include/distributed/multi_resowner.h index 195e2d9f0..3ffc04016 100644 --- a/src/include/distributed/multi_resowner.h +++ b/src/include/distributed/multi_resowner.h @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * multi_resowner.h - * CitusDB resource owner integration. + * Citus resource owner integration. * * Copyright (c) 2012-2015, Citus Data, Inc. *------------------------------------------------------------------------- diff --git a/src/include/distributed/multi_utility.h b/src/include/distributed/multi_utility.h index 0a50b5b08..d37c2f3c8 100644 --- a/src/include/distributed/multi_utility.h +++ b/src/include/distributed/multi_utility.h @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * multi_utility.h - * CitusDB utility hook and related functionality. + * Citus utility hook and related functionality. * * Copyright (c) 2012-2015, Citus Data, Inc. *------------------------------------------------------------------------- diff --git a/src/include/distributed/resource_lock.h b/src/include/distributed/resource_lock.h index 6c1c8ffcf..f1ca62add 100644 --- a/src/include/distributed/resource_lock.h +++ b/src/include/distributed/resource_lock.h @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * resource_lock.h - * Locking Infrastructure for CitusDB. + * Locking Infrastructure for Citus. * * Copyright (c) 2012-2015, Citus Data, Inc. *------------------------------------------------------------------------- @@ -21,7 +21,7 @@ * advisory locks. Only 1 and 2 are used allowing us to define non-conflicting * lock methods. * - * In case postgres starts to use additional values, CitusDB's values + * In case postgres starts to use additional values, Citus's values * will have to be changed. That just requires re-compiling and a restart. */ typedef enum AdvisoryLocktagClass @@ -29,7 +29,7 @@ typedef enum AdvisoryLocktagClass /* values defined in postgres' lockfuncs.c */ ADV_LOCKTAG_CLASS_INT64 = 1, ADV_LOCKTAG_CLASS_INT32 = 2, - /* CitusDB lock types */ + /* Citus lock types */ ADV_LOCKTAG_CLASS_CITUS_SHARD_METADATA = 4, ADV_LOCKTAG_CLASS_CITUS_SHARD = 5, ADV_LOCKTAG_CLASS_CITUS_JOB = 6 diff --git a/src/include/distributed/test_helper_functions.h b/src/include/distributed/test_helper_functions.h index 1d304bd4d..410239099 100644 --- a/src/include/distributed/test_helper_functions.h +++ b/src/include/distributed/test_helper_functions.h @@ -10,8 +10,8 @@ *------------------------------------------------------------------------- */ -#ifndef CITUSDB_TEST_HELPER_FUNCTIONS_H -#define CITUSDB_TEST_HELPER_FUNCTIONS_H +#ifndef CITUS_TEST_HELPER_FUNCTIONS_H +#define CITUS_TEST_HELPER_FUNCTIONS_H #include "postgres.h" #include "c.h" @@ -70,4 +70,4 @@ extern Datum prune_using_both_values(PG_FUNCTION_ARGS); extern Datum debug_equality_expression(PG_FUNCTION_ARGS); -#endif /* CITUSDB_TEST_HELPER_FUNCTIONS_H */ +#endif /* CITUS_TEST_HELPER_FUNCTIONS_H */ diff --git a/src/include/distributed/worker_manager.h b/src/include/distributed/worker_manager.h index 57a38194a..3d3df4078 100644 --- a/src/include/distributed/worker_manager.h +++ b/src/include/distributed/worker_manager.h @@ -23,7 +23,7 @@ /* Maximum length of worker port number (represented as string) */ #define MAX_PORT_LENGTH 10 -/* default filename for citusdb.worker_list_file */ +/* default filename for citus.worker_list_file */ #define WORKER_LIST_FILENAME "pg_worker_list.conf" /* Implementation specific definitions used in finding worker nodes */ diff --git a/src/test/regress/Makefile b/src/test/regress/Makefile index 2b9ef7078..398ed9cac 100644 --- a/src/test/regress/Makefile +++ b/src/test/regress/Makefile @@ -1,9 +1,9 @@ -# Makefile for tests of the CitusDB extension +# Makefile for tests of the Citus extension -citusdb_subdir = src/test/regress -citusdb_top_builddir = ../../.. +citus_subdir = src/test/regress +citus_top_builddir = ../../.. -include $(citusdb_top_builddir)/Makefile.global +include $(citus_top_builddir)/Makefile.global # ensure MAJORVERSION is defined (missing in older versions) ifndef MAJORVERSION @@ -11,11 +11,11 @@ MAJORVERSION := $(basename $(VERSION)) endif ## -## CitusDB regression support +## Citus regression support ## MULTI_INSTALLDIR=$(CURDIR)/tmp_check/install -pg_regress_multi_check = $(PERL) $(citusdb_abs_srcdir)/pg_regress_multi.pl --pgxsdir="$(pgxsdir)" --bindir="$(bindir)" --libdir="$(libdir)" --majorversion="$(MAJORVERSION)" -MULTI_REGRESS_OPTS = --inputdir=$(citusdb_abs_srcdir) $(pg_regress_locale_flags) +pg_regress_multi_check = $(PERL) $(citus_abs_srcdir)/pg_regress_multi.pl --pgxsdir="$(pgxsdir)" --bindir="$(bindir)" --libdir="$(libdir)" --majorversion="$(MAJORVERSION)" +MULTI_REGRESS_OPTS = --inputdir=$(citus_abs_srcdir) $(pg_regress_locale_flags) # XXX: Can't actually do useful testruns against install - $libdir # etc will point to the directory configured during postgres' @@ -26,12 +26,12 @@ cleandir-main: ### tempinstall-main: cleandir-main #### mkdir -p $(MULTI_INSTALLDIR) -### $(MAKE) DESTDIR=$(MULTI_INSTALLDIR) -C $(citusdb_top_builddir) install > tmp_check/install.log 2>&1 +### $(MAKE) DESTDIR=$(MULTI_INSTALLDIR) -C $(citus_top_builddir) install > tmp_check/install.log 2>&1 # Test input and expected files. These are created by pg_regress itself, so we # don't have a rule to create them. We do need rules to clean them however. -input_files := $(patsubst $(citusdb_abs_srcdir)/input/%.source,sql/%.sql, $(wildcard $(citusdb_abs_srcdir)/input/*.source)) -output_files := $(patsubst $(citusdb_abs_srcdir)/output/%.source,expected/%.out, $(wildcard $(citusdb_abs_srcdir)/output/*.source)) +input_files := $(patsubst $(citus_abs_srcdir)/input/%.source,sql/%.sql, $(wildcard $(citus_abs_srcdir)/input/*.source)) +output_files := $(patsubst $(citus_abs_srcdir)/output/%.source,expected/%.out, $(wildcard $(citus_abs_srcdir)/output/*.source)) # have make check actually run all tests, but keep check-full as an # intermediate, for muscle memory backward compatibility. @@ -42,32 +42,32 @@ check-full: check-multi check-multi-task-tracker check-multi-binary check-worker # using pg_regress_multi_check unnecessarily starts up multiple nodes, which isn't needed # for check-worker. But that's harmless besides a few cycles. check-worker: all - $(pg_regress_multi_check) --load-extension=citusdb \ - -- $(MULTI_REGRESS_OPTS) --schedule=$(citusdb_abs_srcdir)/worker_schedule $(EXTRA_TESTS) + $(pg_regress_multi_check) --load-extension=citus \ + -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/worker_schedule $(EXTRA_TESTS) check-multi: all tempinstall-main - $(pg_regress_multi_check) --load-extension=citusdb \ - -- $(MULTI_REGRESS_OPTS) --schedule=$(citusdb_abs_srcdir)/multi_schedule $(EXTRA_TESTS) + $(pg_regress_multi_check) --load-extension=citus \ + -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/multi_schedule $(EXTRA_TESTS) check-multi-fdw: all tempinstall-main - $(pg_regress_multi_check) --load-extension=citusdb --load-extension=file_fdw -- \ - $(MULTI_REGRESS_OPTS) --schedule=$(citusdb_abs_srcdir)/multi_fdw_schedule $(EXTRA_TESTS) + $(pg_regress_multi_check) --load-extension=citus --load-extension=file_fdw -- \ + $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/multi_fdw_schedule $(EXTRA_TESTS) check-multi-hll: all tempinstall-main - $(pg_regress_multi_check) --load-extension=citusdb --load-extension=hll -- \ + $(pg_regress_multi_check) --load-extension=citus --load-extension=hll -- \ $(MULTI_REGRESS_OPTS) $(EXTRA_TESTS) multi_create_table multi_master_protocol multi_stage_data multi_agg_approximate_distinct check-multi-task-tracker: all tempinstall-main - $(pg_regress_multi_check) --load-extension=citusdb \ - --server-option=citusdb.task_executor_type=task-tracker \ - --server-option=citusdb.task_tracker_delay=50ms \ - --server-option=citusdb.large_table_shard_count=1 \ - -- $(MULTI_REGRESS_OPTS) --schedule=$(citusdb_abs_srcdir)/multi_schedule $(EXTRA_TESTS) + $(pg_regress_multi_check) --load-extension=citus \ + --server-option=citus.task_executor_type=task-tracker \ + --server-option=citus.task_tracker_delay=50ms \ + --server-option=citus.large_table_shard_count=1 \ + -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/multi_schedule $(EXTRA_TESTS) check-multi-binary: all tempinstall-main - $(pg_regress_multi_check) --load-extension=citusdb \ - --server-option=citusdb.binary_worker_copy_format=on \ - -- $(MULTI_REGRESS_OPTS) --schedule=$(citusdb_abs_srcdir)/multi_schedule $(EXTRA_TESTS) + $(pg_regress_multi_check) --load-extension=citus \ + --server-option=citus.binary_worker_copy_format=on \ + -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/multi_schedule $(EXTRA_TESTS) clean distclean maintainer-clean: rm -f $(output_files) $(input_files) diff --git a/src/test/regress/expected/multi_agg_approximate_distinct.out b/src/test/regress/expected/multi_agg_approximate_distinct.out index 13f4fcdf5..852142d0f 100644 --- a/src/test/regress/expected/multi_agg_approximate_distinct.out +++ b/src/test/regress/expected/multi_agg_approximate_distinct.out @@ -7,14 +7,14 @@ ERROR: cannot compute aggregate (distinct) DETAIL: table partitioning is unsuitable for aggregate (distinct) HINT: You can load the hll extension from contrib packages and enable distinct approximations. -- Check approximate count(distinct) at different precisions / error rates -SET citusdb.count_distinct_error_rate = 0.1; +SET citus.count_distinct_error_rate = 0.1; SELECT count(distinct l_orderkey) FROM lineitem; count ------- 2612 (1 row) -SET citusdb.count_distinct_error_rate = 0.01; +SET citus.count_distinct_error_rate = 0.01; SELECT count(distinct l_orderkey) FROM lineitem; count ------- @@ -102,7 +102,7 @@ SELECT count(DISTINCT l_orderkey) as distinct_order_count, l_quantity FROM linei -- If we have an order by on count(distinct) that we intend to push down to -- worker nodes, we need to error out. Otherwise, we are fine. -SET citusdb.limit_clause_row_fetch_count = 1000; +SET citus.limit_clause_row_fetch_count = 1000; SELECT l_returnflag, count(DISTINCT l_shipdate) as count_distinct, count(*) as total FROM lineitem GROUP BY l_returnflag @@ -123,7 +123,7 @@ SELECT l_returnflag, count(DISTINCT l_shipdate) as count_distinct, count(*) as t (3 rows) -- Check that we can revert config and disable count(distinct) approximations -SET citusdb.count_distinct_error_rate = 0.0; +SET citus.count_distinct_error_rate = 0.0; SELECT count(distinct l_orderkey) FROM lineitem; ERROR: cannot compute aggregate (distinct) DETAIL: table partitioning is unsuitable for aggregate (distinct) diff --git a/src/test/regress/expected/multi_binary_master_copy_format.out b/src/test/regress/expected/multi_binary_master_copy_format.out index eef2f7f05..505d18a34 100644 --- a/src/test/regress/expected/multi_binary_master_copy_format.out +++ b/src/test/regress/expected/multi_binary_master_copy_format.out @@ -2,8 +2,8 @@ -- MULTI_BINARY_MASTER_COPY -- -- Try binary master copy for different executors -SET citusdb.binary_master_copy_format TO 'on'; -SET citusdb.task_executor_type TO 'task-tracker'; +SET citus.binary_master_copy_format TO 'on'; +SET citus.task_executor_type TO 'task-tracker'; SELECT count(*) FROM lineitem; count ------- @@ -17,7 +17,7 @@ SELECT l_shipmode FROM lineitem WHERE l_partkey = 67310 OR l_partkey = 155190; MAIL (2 rows) -SET citusdb.task_executor_type TO 'real-time'; +SET citus.task_executor_type TO 'real-time'; SELECT count(*) FROM lineitem; count ------- diff --git a/src/test/regress/expected/multi_connection_cache.out b/src/test/regress/expected/multi_connection_cache.out index acd921a1c..ac808a0af 100644 --- a/src/test/regress/expected/multi_connection_cache.out +++ b/src/test/regress/expected/multi_connection_cache.out @@ -3,19 +3,19 @@ -- =================================================================== CREATE FUNCTION initialize_remote_temp_table(cstring, integer) RETURNS bool - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION count_remote_temp_table_rows(cstring, integer) RETURNS integer - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION get_and_purge_connection(cstring, integer) RETURNS bool - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION set_connection_status_bad(cstring, integer) RETURNS bool - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; -- =================================================================== -- test connection hash functionality diff --git a/src/test/regress/expected/multi_create_fdw.out b/src/test/regress/expected/multi_create_fdw.out index abd6a5616..006ad6d1a 100644 --- a/src/test/regress/expected/multi_create_fdw.out +++ b/src/test/regress/expected/multi_create_fdw.out @@ -4,7 +4,7 @@ -- create fake fdw for use in tests CREATE FUNCTION fake_fdw_handler() RETURNS fdw_handler -AS 'citusdb' +AS 'citus' LANGUAGE C STRICT; CREATE FOREIGN DATA WRAPPER fake_fdw HANDLER fake_fdw_handler; CREATE SERVER fake_fdw_server FOREIGN DATA WRAPPER fake_fdw; diff --git a/src/test/regress/expected/multi_create_insert_proxy.out b/src/test/regress/expected/multi_create_insert_proxy.out index 5737338ef..d90262c4a 100644 --- a/src/test/regress/expected/multi_create_insert_proxy.out +++ b/src/test/regress/expected/multi_create_insert_proxy.out @@ -9,8 +9,8 @@ CREATE SCHEMA "A$AP Mob" id bigint PRIMARY KEY, data text NOT NULL DEFAULT 'lorem ipsum' ); -NOTICE: CitusDB partially supports CREATE SCHEMA for distributed databases -DETAIL: schema usage in joins and in some UDFs provided by CitusDB are not supported yet +NOTICE: Citus partially supports CREATE SCHEMA for distributed databases +DETAIL: schema usage in joins and in some UDFs provided by Citus are not supported yet \set insert_target '"A$AP Mob"."Dr. Bronner''s ""Magic"" Soaps"' -- create proxy and save proxy table name SELECT create_insert_proxy_for_table(:'insert_target') AS proxy_tablename diff --git a/src/test/regress/expected/multi_create_shards.out b/src/test/regress/expected/multi_create_shards.out index 01e44580d..39cb45eef 100644 --- a/src/test/regress/expected/multi_create_shards.out +++ b/src/test/regress/expected/multi_create_shards.out @@ -3,7 +3,7 @@ -- =================================================================== CREATE FUNCTION sort_names(cstring, cstring, cstring) RETURNS cstring - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; -- create a custom type... CREATE TYPE dummy_type AS ( @@ -47,7 +47,7 @@ SELECT master_create_distributed_table('table_to_distribute', 'bad_column', 'has ERROR: column "bad_column" of relation "table_to_distribute" does not exist -- use unrecognized partition type SELECT master_create_distributed_table('table_to_distribute', 'name', 'unrecognized'); -ERROR: invalid input value for enum citusdb.distribution_type: "unrecognized" +ERROR: invalid input value for enum citus.distribution_type: "unrecognized" LINE 1: ..._distributed_table('table_to_distribute', 'name', 'unrecogni... ^ -- use a partition column of a type lacking any default operator class @@ -209,7 +209,7 @@ SELECT master_create_worker_shards('weird_shard_count', 7, 1); (1 row) --- CitusDB ensures all shards are roughly the same size +-- Citus ensures all shards are roughly the same size SELECT shardmaxvalue::integer - shardminvalue::integer AS shard_size FROM pg_dist_shard WHERE logicalrelid = 'weird_shard_count'::regclass diff --git a/src/test/regress/expected/multi_create_table.out b/src/test/regress/expected/multi_create_table.out index b079d7873..dd3f1c167 100644 --- a/src/test/regress/expected/multi_create_table.out +++ b/src/test/regress/expected/multi_create_table.out @@ -109,7 +109,7 @@ SELECT master_create_distributed_table('supplier', 's_suppkey', 'append'); (1 row) --- now test that CitusDB cannot distribute unique constraints that do not include +-- now test that Citus cannot distribute unique constraints that do not include -- the partition column CREATE TABLE primary_key_on_non_part_col ( @@ -127,7 +127,7 @@ CREATE TABLE unique_const_on_non_part_col SELECT master_create_distributed_table('primary_key_on_non_part_col', 'partition_col', 'hash'); ERROR: cannot distribute relation: "primary_key_on_non_part_col" DETAIL: Distributed relations cannot have UNIQUE constraints or PRIMARY KEYs that do not include the partition column. --- now show that CitusDB can distribute unique constrints that include +-- now show that Citus can distribute unique constrints that include -- the partition column CREATE TABLE primary_key_on_part_col ( diff --git a/src/test/regress/expected/multi_distribution_metadata.out b/src/test/regress/expected/multi_distribution_metadata.out index 6c69be7b8..d626b179e 100644 --- a/src/test/regress/expected/multi_distribution_metadata.out +++ b/src/test/regress/expected/multi_distribution_metadata.out @@ -3,55 +3,55 @@ -- =================================================================== CREATE FUNCTION load_shard_id_array(regclass) RETURNS bigint[] - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION load_shard_interval_array(bigint, anyelement) RETURNS anyarray - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION load_shard_placement_array(bigint, bool) RETURNS text[] - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION partition_column_id(regclass) RETURNS smallint - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION partition_type(regclass) RETURNS "char" - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION is_distributed_table(regclass) RETURNS boolean - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION column_name_to_column_id(regclass, cstring) RETURNS smallint - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION create_monolithic_shard_row(regclass) RETURNS bigint - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION create_healthy_local_shard_placement_row(bigint) RETURNS void - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION delete_shard_placement_row(bigint, text, bigint) RETURNS bool - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION update_shard_placement_row_state(bigint, text, bigint, int) RETURNS bool - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION acquire_shared_shard_lock(bigint) RETURNS void - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION column_name_to_column(regclass, text) RETURNS text - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; -- =================================================================== -- test distribution metadata functionality diff --git a/src/test/regress/expected/multi_generate_ddl_commands.out b/src/test/regress/expected/multi_generate_ddl_commands.out index aa4d58822..ef63548ab 100644 --- a/src/test/regress/expected/multi_generate_ddl_commands.out +++ b/src/test/regress/expected/multi_generate_ddl_commands.out @@ -3,7 +3,7 @@ -- =================================================================== CREATE FUNCTION table_ddl_command_array(regclass) RETURNS text[] - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; -- =================================================================== -- test ddl command generation functionality @@ -33,8 +33,8 @@ SELECT table_ddl_command_array('not_null_table'); -- ensure tables not in search path are schema-prefixed CREATE SCHEMA not_in_path CREATE TABLE simple_table (id bigint); -NOTICE: CitusDB partially supports CREATE SCHEMA for distributed databases -DETAIL: schema usage in joins and in some UDFs provided by CitusDB are not supported yet +NOTICE: Citus partially supports CREATE SCHEMA for distributed databases +DETAIL: schema usage in joins and in some UDFs provided by Citus are not supported yet SELECT table_ddl_command_array('not_in_path.simple_table'); table_ddl_command_array ------------------------------------------------------------------------------------------------- diff --git a/src/test/regress/expected/multi_generate_ddl_commands_0.out b/src/test/regress/expected/multi_generate_ddl_commands_0.out index 1d042bc26..ff856b423 100644 --- a/src/test/regress/expected/multi_generate_ddl_commands_0.out +++ b/src/test/regress/expected/multi_generate_ddl_commands_0.out @@ -3,7 +3,7 @@ -- =================================================================== CREATE FUNCTION table_ddl_command_array(regclass) RETURNS text[] - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; -- =================================================================== -- test ddl command generation functionality @@ -33,8 +33,8 @@ SELECT table_ddl_command_array('not_null_table'); -- ensure tables not in search path are schema-prefixed CREATE SCHEMA not_in_path CREATE TABLE simple_table (id bigint); -NOTICE: CitusDB partially supports CREATE SCHEMA for distributed databases -DETAIL: schema usage in joins and in some UDFs provided by CitusDB are not supported yet +NOTICE: Citus partially supports CREATE SCHEMA for distributed databases +DETAIL: schema usage in joins and in some UDFs provided by Citus are not supported yet SELECT table_ddl_command_array('not_in_path.simple_table'); table_ddl_command_array ------------------------------------------------------------------------------------------------- diff --git a/src/test/regress/expected/multi_join_order_additional.out b/src/test/regress/expected/multi_join_order_additional.out index c641140a7..9fd9cac07 100644 --- a/src/test/regress/expected/multi_join_order_additional.out +++ b/src/test/regress/expected/multi_join_order_additional.out @@ -2,7 +2,7 @@ -- MULTI_JOIN_ORDER_ADDITIONAL -- -- Set configuration to print table join order and pruned shards -SET citusdb.log_multi_join_order TO TRUE; +SET citus.log_multi_join_order TO TRUE; SET client_min_messages TO DEBUG2; -- The following query checks that we can correctly handle self-joins EXPLAIN SELECT l1.l_quantity FROM lineitem l1, lineitem l2 @@ -42,7 +42,7 @@ DEBUG: join prunable for intervals [1,2496] and [2497,4964] (1 row) -- Update configuration to treat lineitem and orders tables as large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; SET client_min_messages TO LOG; -- The following queries check that we correctly handle joins and OR clauses. In -- particular, these queries check that we factorize out OR clauses if possible, @@ -106,7 +106,7 @@ LOG: join order: [ "customer" ][ broadcast join "nation" ] UPDATE pg_dist_partition SET partmethod = 'a' WHERE logicalrelid = (SELECT relfilenode FROM pg_class WHERE relname = 'customer'); -- Update the large table shard count for all the following tests. -SET citusdb.large_table_shard_count TO 1; +SET citus.large_table_shard_count TO 1; -- Validate that we don't use a single-partition join method for a hash -- re-partitioned table, thus preventing a partition of just the customer table. EXPLAIN SELECT count(*) FROM orders, lineitem, customer diff --git a/src/test/regress/expected/multi_join_order_tpch_large.out b/src/test/regress/expected/multi_join_order_tpch_large.out index 14bc84cff..deb1f20da 100644 --- a/src/test/regress/expected/multi_join_order_tpch_large.out +++ b/src/test/regress/expected/multi_join_order_tpch_large.out @@ -2,13 +2,13 @@ -- MULTI_JOIN_ORDER_TPCH_LARGE -- -- Enable configuration to print table join order -SET citusdb.log_multi_join_order TO TRUE; +SET citus.log_multi_join_order TO TRUE; SET client_min_messages TO LOG; -- Change configuration to treat lineitem, orders, customer, and part tables as -- large. The following queries are basically the same as the ones in tpch_small -- except that more data has been staged to customer and part tables. Therefore, -- we will apply different distributed join strategies for these queries. -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; -- Query #6 from the TPC-H decision support benchmark EXPLAIN SELECT sum(l_extendedprice * l_discount) as revenue diff --git a/src/test/regress/expected/multi_join_order_tpch_small.out b/src/test/regress/expected/multi_join_order_tpch_small.out index 696736a00..6601fb53c 100644 --- a/src/test/regress/expected/multi_join_order_tpch_small.out +++ b/src/test/regress/expected/multi_join_order_tpch_small.out @@ -2,10 +2,10 @@ -- MULTI_JOIN_ORDER_TPCH_SMALL -- -- Enable configuration to print table join order -SET citusdb.log_multi_join_order TO TRUE; +SET citus.log_multi_join_order TO TRUE; SET client_min_messages TO LOG; -- Change configuration to treat lineitem and orders tables as large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; -- Query #6 from the TPC-H decision support benchmark EXPLAIN SELECT sum(l_extendedprice * l_discount) as revenue diff --git a/src/test/regress/expected/multi_join_pruning.out b/src/test/regress/expected/multi_join_pruning.out index e690636ee..578964ac5 100644 --- a/src/test/regress/expected/multi_join_pruning.out +++ b/src/test/regress/expected/multi_join_pruning.out @@ -6,7 +6,7 @@ -- future we want to check for pruning between re-partitioned relations as well. SET client_min_messages TO DEBUG2; -- Change configuration to treat all tables as large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; SELECT sum(l_linenumber), avg(l_linenumber) FROM lineitem, orders WHERE l_orderkey = o_orderkey; DEBUG: join prunable for intervals [13921,14947] and [1,5986] diff --git a/src/test/regress/expected/multi_large_table_join_planning.out b/src/test/regress/expected/multi_large_table_join_planning.out index 499a1f5be..3fbdf0b42 100644 --- a/src/test/regress/expected/multi_large_table_join_planning.out +++ b/src/test/regress/expected/multi_large_table_join_planning.out @@ -8,11 +8,11 @@ BEGIN; SET client_min_messages TO DEBUG4; DEBUG: CommitTransactionCommand -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; DEBUG: StartTransactionCommand DEBUG: ProcessUtility DEBUG: CommitTransactionCommand -SET citusdb.task_executor_type TO 'task-tracker'; +SET citus.task_executor_type TO 'task-tracker'; DEBUG: StartTransactionCommand DEBUG: ProcessUtility DEBUG: CommitTransactionCommand diff --git a/src/test/regress/expected/multi_large_table_join_planning_0.out b/src/test/regress/expected/multi_large_table_join_planning_0.out index 48c7f8a77..4b08b7fe0 100644 --- a/src/test/regress/expected/multi_large_table_join_planning_0.out +++ b/src/test/regress/expected/multi_large_table_join_planning_0.out @@ -8,11 +8,11 @@ BEGIN; SET client_min_messages TO DEBUG4; DEBUG: CommitTransactionCommand -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; DEBUG: StartTransactionCommand DEBUG: ProcessUtility DEBUG: CommitTransactionCommand -SET citusdb.task_executor_type TO 'task-tracker'; +SET citus.task_executor_type TO 'task-tracker'; DEBUG: StartTransactionCommand DEBUG: ProcessUtility DEBUG: CommitTransactionCommand diff --git a/src/test/regress/expected/multi_large_table_pruning.out b/src/test/regress/expected/multi_large_table_pruning.out index d957e7669..20b62fa4d 100644 --- a/src/test/regress/expected/multi_large_table_pruning.out +++ b/src/test/regress/expected/multi_large_table_pruning.out @@ -4,9 +4,9 @@ -- Tests covering partition and join-pruning for large table joins. Note that we -- set executor type to task tracker executor here, as we cannot run repartition -- jobs with real time executor. -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; SET client_min_messages TO DEBUG2; -SET citusdb.task_executor_type TO 'task-tracker'; +SET citus.task_executor_type TO 'task-tracker'; -- Single range-repartition join to test join-pruning behaviour. SELECT count(*) diff --git a/src/test/regress/expected/multi_large_table_task_assignment.out b/src/test/regress/expected/multi_large_table_task_assignment.out index 9bf12ff00..33ec87b9b 100644 --- a/src/test/regress/expected/multi_large_table_task_assignment.out +++ b/src/test/regress/expected/multi_large_table_task_assignment.out @@ -8,11 +8,11 @@ BEGIN; SET client_min_messages TO DEBUG3; DEBUG: CommitTransactionCommand -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; DEBUG: StartTransactionCommand DEBUG: ProcessUtility DEBUG: CommitTransactionCommand -SET citusdb.task_executor_type TO 'task-tracker'; +SET citus.task_executor_type TO 'task-tracker'; DEBUG: StartTransactionCommand DEBUG: ProcessUtility DEBUG: CommitTransactionCommand @@ -52,7 +52,7 @@ DEBUG: CommitTransactionCommand -- more than one shard. This situation results in multiple sql tasks depending on -- the same merge task, and tests our constraint group creation and assignment -- propagation. Here 'orders' is considered the small table. -SET citusdb.large_table_shard_count TO 3; +SET citus.large_table_shard_count TO 3; DEBUG: StartTransactionCommand DEBUG: ProcessUtility DEBUG: CommitTransactionCommand @@ -128,7 +128,7 @@ DEBUG: CommitTransactionCommand 11998 (1 row) -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; DEBUG: StartTransactionCommand DEBUG: ProcessUtility DEBUG: CommitTransactionCommand diff --git a/src/test/regress/expected/multi_limit_clause_approximate.out b/src/test/regress/expected/multi_limit_clause_approximate.out index d65c6ae6f..d1bad570e 100644 --- a/src/test/regress/expected/multi_limit_clause_approximate.out +++ b/src/test/regress/expected/multi_limit_clause_approximate.out @@ -24,7 +24,7 @@ SELECT l_partkey, sum(l_partkey * (1 + l_suppkey)) AS aggregate FROM lineitem (10 rows) -- Enable limit optimization to fetch one third of each shard's data -SET citusdb.limit_clause_row_fetch_count TO 600; +SET citus.limit_clause_row_fetch_count TO 600; SELECT l_partkey, sum(l_partkey * (1 + l_suppkey)) AS aggregate FROM lineitem GROUP BY l_partkey ORDER BY aggregate DESC LIMIT 10; @@ -45,7 +45,7 @@ DEBUG: push down of limit count: 600 -- Disable limit optimization for our second test. This time, we have a query -- that joins several tables, and that groups and orders the results. -RESET citusdb.limit_clause_row_fetch_count; +RESET citus.limit_clause_row_fetch_count; SELECT c_custkey, c_name, count(*) as lineitem_count FROM customer, orders, lineitem WHERE c_custkey = o_custkey AND l_orderkey = o_orderkey @@ -68,8 +68,8 @@ SELECT c_custkey, c_name, count(*) as lineitem_count -- Now, enable limit optimization to fetch half of each task's results. For this -- test, we also change a config setting to ensure that we don't repartition any -- of the tables during the query. -SET citusdb.limit_clause_row_fetch_count TO 150; -SET citusdb.large_table_shard_count TO 2; +SET citus.limit_clause_row_fetch_count TO 150; +SET citus.large_table_shard_count TO 2; SELECT c_custkey, c_name, count(*) as lineitem_count FROM customer, orders, lineitem WHERE c_custkey = o_custkey AND l_orderkey = o_orderkey @@ -90,7 +90,7 @@ DEBUG: push down of limit count: 150 145 | Customer#000000145 | 30 (10 rows) -RESET citusdb.large_table_shard_count; +RESET citus.large_table_shard_count; -- We now test scenarios where applying the limit optimization wouldn't produce -- meaningful results. First, we check that we don't push down the limit clause -- for non-commutative aggregates. @@ -147,5 +147,5 @@ SELECT count(*) count_quantity, l_quantity FROM lineitem WHERE l_quantity < 10.0 258 | 9.00 (9 rows) -RESET citusdb.limit_clause_row_fetch_count; +RESET citus.limit_clause_row_fetch_count; RESET client_min_messages; diff --git a/src/test/regress/expected/multi_modifications.out b/src/test/regress/expected/multi_modifications.out index d48c47551..3b62f7c38 100644 --- a/src/test/regress/expected/multi_modifications.out +++ b/src/test/regress/expected/multi_modifications.out @@ -91,7 +91,7 @@ INSERT INTO append_partitioned VALUES (414123, 'AAPL', 9580, '2004-10-19 10:23:5 20.69); -- ensure the values are where we put them and query to ensure they are properly pruned SET client_min_messages TO 'DEBUG2'; -SET citusdb.task_executor_type TO 'router'; +SET citus.task_executor_type TO 'router'; SELECT * FROM range_partitioned WHERE id = 32743; DEBUG: predicate pruning for shardId 103070 id | symbol | bidder_id | placed_at | kind | limit_price @@ -107,7 +107,7 @@ DEBUG: predicate pruning for shardId 103072 (1 row) SET client_min_messages TO DEFAULT; -SET citusdb.task_executor_type TO DEFAULT; +SET citus.task_executor_type TO DEFAULT; -- try inserting without a range-partitioned shard to receive the value INSERT INTO range_partitioned VALUES (999999, 'AAPL', 9580, '2004-10-19 10:23:54', 'buy', 20.69); diff --git a/src/test/regress/expected/multi_null_minmax_value_pruning.out b/src/test/regress/expected/multi_null_minmax_value_pruning.out index 53e5a10a5..486db51fb 100644 --- a/src/test/regress/expected/multi_null_minmax_value_pruning.out +++ b/src/test/regress/expected/multi_null_minmax_value_pruning.out @@ -5,7 +5,7 @@ -- and that we don't partition or join prune shards that have null values. SET client_min_messages TO DEBUG2; -- Change configuration to treat lineitem and orders tables as large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; SELECT shardminvalue, shardmaxvalue from pg_dist_shard WHERE shardid = 102009; shardminvalue | shardmaxvalue ---------------+--------------- diff --git a/src/test/regress/expected/multi_prepare_plsql.out b/src/test/regress/expected/multi_prepare_plsql.out index c2848b7cb..e76482076 100644 --- a/src/test/regress/expected/multi_prepare_plsql.out +++ b/src/test/regress/expected/multi_prepare_plsql.out @@ -173,7 +173,7 @@ BEGIN l_year; END; $$ LANGUAGE plpgsql; -SET citusdb.task_executor_type TO 'task-tracker'; +SET citus.task_executor_type TO 'task-tracker'; SET client_min_messages TO INFO; -- now, run plain SQL functions SELECT sql_test_no_1(); @@ -313,7 +313,7 @@ SELECT plpgsql_test_2(); -- run the tests which do not require re-partition -- with real-time executor -SET citusdb.task_executor_type TO 'real-time'; +SET citus.task_executor_type TO 'real-time'; -- now, run plain SQL functions SELECT sql_test_no_1(); sql_test_no_1 diff --git a/src/test/regress/expected/multi_prepare_sql.out b/src/test/regress/expected/multi_prepare_sql.out index 1aff5bccf..b0d097a3f 100644 --- a/src/test/regress/expected/multi_prepare_sql.out +++ b/src/test/regress/expected/multi_prepare_sql.out @@ -95,7 +95,7 @@ ORDER BY supp_nation, cust_nation, l_year; -SET citusdb.task_executor_type TO 'task-tracker'; +SET citus.task_executor_type TO 'task-tracker'; SET client_min_messages TO INFO; -- execute prepared statements EXECUTE prepared_test_1; @@ -217,7 +217,7 @@ SELECT * from prepared_sql_test_7; (1 row) -- now, run some of the tests with real-time executor -SET citusdb.task_executor_type TO 'real-time'; +SET citus.task_executor_type TO 'real-time'; -- execute prepared statements EXECUTE prepared_test_1; count diff --git a/src/test/regress/expected/multi_prune_shard_list.out b/src/test/regress/expected/multi_prune_shard_list.out index 029228841..2f74b1991 100644 --- a/src/test/regress/expected/multi_prune_shard_list.out +++ b/src/test/regress/expected/multi_prune_shard_list.out @@ -3,23 +3,23 @@ -- =================================================================== CREATE FUNCTION prune_using_no_values(regclass) RETURNS text[] - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION prune_using_single_value(regclass, text) RETURNS text[] - AS 'citusdb' + AS 'citus' LANGUAGE C; CREATE FUNCTION prune_using_either_value(regclass, text, text) RETURNS text[] - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION prune_using_both_values(regclass, text, text) RETURNS text[] - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION debug_equality_expression(regclass) RETURNS cstring - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; -- =================================================================== -- test shard pruning functionality diff --git a/src/test/regress/expected/multi_simple_queries.out b/src/test/regress/expected/multi_simple_queries.out index 734dfbde2..764e3c705 100644 --- a/src/test/regress/expected/multi_simple_queries.out +++ b/src/test/regress/expected/multi_simple_queries.out @@ -169,7 +169,7 @@ SELECT title, author_id FROM articles (10 rows) -- add in some grouping expressions, still on same shard --- having queries unsupported in CitusDB +-- having queries unsupported in Citus SELECT author_id, sum(word_count) AS corpus_size FROM articles WHERE author_id = 1 OR author_id = 7 OR author_id = 8 OR author_id = 10 GROUP BY author_id @@ -190,7 +190,7 @@ ERROR: cannot plan queries that include both regular and partitioned relations SELECT * FROM articles, position('om' in 'Thomas'); ERROR: cannot perform distributed planning on this query DETAIL: Complex table expressions are currently unsupported --- subqueries are not supported in WHERE clause in CitusDB +-- subqueries are not supported in WHERE clause in Citus SELECT * FROM articles WHERE author_id IN (SELECT id FROM authors WHERE name LIKE '%a'); ERROR: cannot plan queries that include both regular and partitioned relations -- subqueries are supported in FROM clause @@ -285,7 +285,7 @@ SELECT COUNT(*) FROM articles; 50 (1 row) --- having queries unsupported in CitusDB +-- having queries unsupported in Citus SELECT author_id, sum(word_count) AS corpus_size FROM articles GROUP BY author_id HAVING sum(word_count) > 25000 @@ -293,17 +293,17 @@ SELECT author_id, sum(word_count) AS corpus_size FROM articles LIMIT 5; ERROR: cannot perform distributed planning on this query DETAIL: Having qual is currently unsupported --- more proof CitusDB doesn't support having clauses +-- more proof Citus doesn't support having clauses SELECT author_id FROM articles GROUP BY author_id HAVING sum(word_count) > 50000 ORDER BY author_id; ERROR: cannot perform distributed planning on this query DETAIL: Having qual is currently unsupported --- now, test the cases where CitusDB do or do not need to create +-- now, test the cases where Citus do or do not need to create -- the master queries -SET citusdb.task_executor_type TO 'router'; -SET citusdb.large_table_shard_count TO 2; +SET citus.task_executor_type TO 'router'; +SET citus.large_table_shard_count TO 2; SET client_min_messages TO 'DEBUG2'; -- start with the simple lookup query SELECT * @@ -338,7 +338,7 @@ SELECT * FROM articles WHERE author_id = 1 OR author_id = 18; ERROR: cannot use router executor with queries that hit multiple shards -HINT: Set citusdb.task_executor_type to "real-time" or "task-tracker". +HINT: Set citus.task_executor_type to "real-time" or "task-tracker". -- rename the output columns on a no master query case SELECT id as article_id, word_count * id as random_value FROM articles @@ -377,7 +377,7 @@ SELECT a.author_id as first_author, b.word_count as second_word_count DEBUG: push down of limit count: 3 DEBUG: predicate pruning for shardId 103094 ERROR: cannot use router executor with JOINs -HINT: Set citusdb.task_executor_type to "real-time" or "task-tracker". +HINT: Set citus.task_executor_type to "real-time" or "task-tracker". -- do not create the master query for LIMIT on a single shard SELECT SELECT * FROM articles @@ -417,7 +417,7 @@ SELECT avg(word_count) WHERE author_id = 2; DEBUG: predicate pruning for shardId 103093 ERROR: cannot use router executor with aggregates -HINT: Set citusdb.task_executor_type to "real-time" or "task-tracker". +HINT: Set citus.task_executor_type to "real-time" or "task-tracker". -- max, min, sum, count is somehow implemented -- differently in distributed planning but, still error out SELECT max(word_count) as max, min(word_count) as min, @@ -426,7 +426,7 @@ SELECT max(word_count) as max, min(word_count) as min, WHERE author_id = 2; DEBUG: predicate pruning for shardId 103093 ERROR: cannot use router executor with aggregates -HINT: Set citusdb.task_executor_type to "real-time" or "task-tracker". +HINT: Set citus.task_executor_type to "real-time" or "task-tracker". -- error out for queries with ORDER BY SELECT * FROM articles @@ -434,7 +434,7 @@ SELECT * ORDER BY word_count; DEBUG: predicate pruning for shardId 103094 ERROR: cannot use router executor with ORDER BY clauses -HINT: Set citusdb.task_executor_type to "real-time" or "task-tracker". +HINT: Set citus.task_executor_type to "real-time" or "task-tracker". -- error out for queries with ORDER BY and LIMIT SELECT * FROM articles @@ -444,7 +444,7 @@ SELECT * DEBUG: push down of limit count: 2 DEBUG: predicate pruning for shardId 103094 ERROR: cannot use router executor with ORDER BY clauses -HINT: Set citusdb.task_executor_type to "real-time" or "task-tracker". +HINT: Set citus.task_executor_type to "real-time" or "task-tracker". -- error out for queries with aggregates and GROUP BY SELECT max(word_count) FROM articles @@ -452,7 +452,7 @@ SELECT max(word_count) GROUP BY author_id; DEBUG: predicate pruning for shardId 103094 ERROR: cannot use router executor with aggregates -HINT: Set citusdb.task_executor_type to "real-time" or "task-tracker". +HINT: Set citus.task_executor_type to "real-time" or "task-tracker". -- error out for queries with repartition jobs SELECT * FROM articles a, articles b @@ -487,12 +487,12 @@ DETAIL: Creating dependency on merge taskId 9 DEBUG: pruning merge fetch taskId 11 DETAIL: Creating dependency on merge taskId 14 ERROR: cannot use router executor with repartition jobs -HINT: Set citusdb.task_executor_type to "task-tracker". +HINT: Set citus.task_executor_type to "task-tracker". -- error out for queries which hit more than 1 shards SELECT * FROM articles WHERE author_id >= 1 AND author_id <= 3; ERROR: cannot use router executor with queries that hit multiple shards -HINT: Set citusdb.task_executor_type to "real-time" or "task-tracker". +HINT: Set citus.task_executor_type to "real-time" or "task-tracker". SET client_min_messages to 'NOTICE'; -SET citusdb.task_executor_type TO 'real-time'; +SET citus.task_executor_type TO 'real-time'; diff --git a/src/test/regress/expected/multi_single_relation_subquery.out b/src/test/regress/expected/multi_single_relation_subquery.out index 1573e3a0a..267007770 100644 --- a/src/test/regress/expected/multi_single_relation_subquery.out +++ b/src/test/regress/expected/multi_single_relation_subquery.out @@ -2,7 +2,7 @@ -- MULTI_SINGLE_RELATION_SUBQUERY -- -- This test checks that we are able to run selected set of distributed SQL subqueries. -SET citusdb.task_executor_type TO 'task-tracker'; +SET citus.task_executor_type TO 'task-tracker'; select number_sum, count(*) as total, diff --git a/src/test/regress/expected/multi_table_ddl.out b/src/test/regress/expected/multi_table_ddl.out index 9c2488fe0..15fe60150 100644 --- a/src/test/regress/expected/multi_table_ddl.out +++ b/src/test/regress/expected/multi_table_ddl.out @@ -9,12 +9,12 @@ SELECT master_create_distributed_table('testtableddl', 'distributecol', 'append' (1 row) --- verify that the citusdb extension can't be dropped while distributed tables exist -DROP EXTENSION citusdb; +-- verify that the citus extension can't be dropped while distributed tables exist +DROP EXTENSION citus; WARNING: could not clean the metadata cache on DROP EXTENSION command HINT: Reconnect to the server again. -ERROR: cannot drop extension citusdb because other objects depend on it -DETAIL: table testtableddl depends on extension citusdb +ERROR: cannot drop extension citus because other objects depend on it +DETAIL: table testtableddl depends on extension citus HINT: Use DROP ... CASCADE to drop the dependent objects too. -- verify that the distribution column can't have its type changed ALTER TABLE testtableddl ALTER COLUMN distributecol TYPE text; @@ -63,8 +63,8 @@ SELECT * FROM pg_dist_shard_placement; -- check that the extension now can be dropped (and recreated). We reconnect -- before creating the extension to expire extension specific variables which -- are cached for performance. -DROP EXTENSION citusdb; +DROP EXTENSION citus; WARNING: could not clean the metadata cache on DROP EXTENSION command HINT: Reconnect to the server again. \c -CREATE EXTENSION citusdb; +CREATE EXTENSION citus; diff --git a/src/test/regress/expected/multi_task_assignment_policy.out b/src/test/regress/expected/multi_task_assignment_policy.out index 11f4b5115..381d57b50 100644 --- a/src/test/regress/expected/multi_task_assignment_policy.out +++ b/src/test/regress/expected/multi_task_assignment_policy.out @@ -46,7 +46,7 @@ BEGIN; SET client_min_messages TO DEBUG3; DEBUG: CommitTransactionCommand -- First test the default greedy task assignment policy -SET citusdb.task_assignment_policy TO 'greedy'; +SET citus.task_assignment_policy TO 'greedy'; DEBUG: StartTransactionCommand DEBUG: ProcessUtility DEBUG: CommitTransactionCommand @@ -75,7 +75,7 @@ DEBUG: CommitTransactionCommand (1 row) -- Next test the first-replica task assignment policy -SET citusdb.task_assignment_policy TO 'first-replica'; +SET citus.task_assignment_policy TO 'first-replica'; DEBUG: StartTransactionCommand DEBUG: ProcessUtility DEBUG: CommitTransactionCommand @@ -118,7 +118,7 @@ DEBUG: CommitTransactionCommand (1 row) -- Finally test the round-robin task assignment policy -SET citusdb.task_assignment_policy TO 'round-robin'; +SET citus.task_assignment_policy TO 'round-robin'; DEBUG: StartTransactionCommand DEBUG: ProcessUtility DEBUG: CommitTransactionCommand @@ -158,7 +158,7 @@ DEBUG: CommitTransactionCommand explain statements for distributed queries are currently unsupported (1 row) -RESET citusdb.task_assignment_policy; +RESET citus.task_assignment_policy; DEBUG: StartTransactionCommand DEBUG: ProcessUtility DEBUG: CommitTransactionCommand diff --git a/src/test/regress/expected/multi_tpch_query1.out b/src/test/regress/expected/multi_tpch_query1.out index 39e75276a..23a507e06 100644 --- a/src/test/regress/expected/multi_tpch_query1.out +++ b/src/test/regress/expected/multi_tpch_query1.out @@ -2,7 +2,7 @@ -- MULTI_TPCH_QUERY1 -- -- Change configuration to treat lineitem and orders tables as large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; -- Query #1 from the TPC-H decision support benchmark SELECT l_returnflag, diff --git a/src/test/regress/expected/multi_tpch_query10.out b/src/test/regress/expected/multi_tpch_query10.out index ea6f8406e..82eefa6df 100644 --- a/src/test/regress/expected/multi_tpch_query10.out +++ b/src/test/regress/expected/multi_tpch_query10.out @@ -2,7 +2,7 @@ -- MULTI_TPCH_QUERY10 -- -- Query #10 from the TPC-H decision support benchmark. Unlike other TPC-H tests, --- we don't set citusdb.large_table_shard_count here, and instead use the default value +-- we don't set citus.large_table_shard_count here, and instead use the default value -- coming from postgresql.conf or multi_task_tracker_executor.conf. SELECT c_custkey, diff --git a/src/test/regress/expected/multi_tpch_query12.out b/src/test/regress/expected/multi_tpch_query12.out index 8d692752d..e54d3e0a4 100644 --- a/src/test/regress/expected/multi_tpch_query12.out +++ b/src/test/regress/expected/multi_tpch_query12.out @@ -2,7 +2,7 @@ -- MULTI_TPCH_QUERY12 -- -- Change configuration to treat lineitem and orders tables as large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; -- Query #12 from the TPC-H decision support benchmark SELECT l_shipmode, diff --git a/src/test/regress/expected/multi_tpch_query14.out b/src/test/regress/expected/multi_tpch_query14.out index 1f96ff96c..2eded6dda 100644 --- a/src/test/regress/expected/multi_tpch_query14.out +++ b/src/test/regress/expected/multi_tpch_query14.out @@ -2,7 +2,7 @@ -- MULTI_TPCH_QUERY14 -- -- Change configuration to treat lineitem and orders tables as large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; -- Query #14 from the TPC-H decision support benchmark SELECT 100.00 * sum(case diff --git a/src/test/regress/expected/multi_tpch_query19.out b/src/test/regress/expected/multi_tpch_query19.out index 784c2f5ce..d6ee657c1 100644 --- a/src/test/regress/expected/multi_tpch_query19.out +++ b/src/test/regress/expected/multi_tpch_query19.out @@ -2,7 +2,7 @@ -- MULTI_TPCH_QUERY19 -- -- Change configuration to treat lineitem and orders tables as large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; -- Query #19 from the TPC-H decision support benchmark. Note that we modified -- the query from its original to make it work on smaller data sets. SELECT diff --git a/src/test/regress/expected/multi_tpch_query3.out b/src/test/regress/expected/multi_tpch_query3.out index c16171339..da87dc8f5 100644 --- a/src/test/regress/expected/multi_tpch_query3.out +++ b/src/test/regress/expected/multi_tpch_query3.out @@ -2,7 +2,7 @@ -- MULTI_TPCH_QUERY3 -- -- Query #3 from the TPC-H decision support benchmark. Unlike other TPC-H tests, --- we don't set citusdb.large_table_shard_count here, and instead use the default value +-- we don't set citus.large_table_shard_count here, and instead use the default value -- coming from postgresql.conf or multi_task_tracker_executor.conf. SELECT l_orderkey, diff --git a/src/test/regress/expected/multi_tpch_query6.out b/src/test/regress/expected/multi_tpch_query6.out index 977009bbb..9badebd9e 100644 --- a/src/test/regress/expected/multi_tpch_query6.out +++ b/src/test/regress/expected/multi_tpch_query6.out @@ -2,7 +2,7 @@ -- MULTI_TPCH_QUERY6 -- -- Change configuration to treat lineitem and orders tables as large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; -- Query #6 from the TPC-H decision support benchmark SELECT sum(l_extendedprice * l_discount) as revenue diff --git a/src/test/regress/expected/multi_tpch_query7.out b/src/test/regress/expected/multi_tpch_query7.out index 253da5c71..55ab9ae63 100644 --- a/src/test/regress/expected/multi_tpch_query7.out +++ b/src/test/regress/expected/multi_tpch_query7.out @@ -2,7 +2,7 @@ -- MULTI_TPCH_QUERY7 -- -- Change configuration to treat lineitem AND orders tables as large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; -- Query #7 from the TPC-H decision support benchmark SELECT supp_nation, diff --git a/src/test/regress/expected/multi_tpch_query7_nested.out b/src/test/regress/expected/multi_tpch_query7_nested.out index 493b3bf1e..d6d6acf71 100644 --- a/src/test/regress/expected/multi_tpch_query7_nested.out +++ b/src/test/regress/expected/multi_tpch_query7_nested.out @@ -2,7 +2,7 @@ -- MULTI_TPCH_QUERY7_NESTED -- -- Change configuration to treat lineitem AND orders tables AS large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; -- Query #7 from the TPC-H benchmark; modified to include sub-selects SELECT supp_nation, diff --git a/src/test/regress/expected/multi_upsert.out b/src/test/regress/expected/multi_upsert.out index 6222c87e8..cc16748d0 100644 --- a/src/test/regress/expected/multi_upsert.out +++ b/src/test/regress/expected/multi_upsert.out @@ -1,4 +1,4 @@ --- this test file aims to test UPSERT feature on CitusDB +-- this test file aims to test UPSERT feature on Citus -- note that output of this file for postgresql 9.4 will -- be full syntax errors, which is expected. CREATE TABLE upsert_test @@ -124,7 +124,7 @@ SELECT master_create_worker_shards('upsert_test_2', '4', '2'); (1 row) --- now show that CitusDB works with multiple columns as the PRIMARY KEY, including the partiton key +-- now show that Citus works with multiple columns as the PRIMARY KEY, including the partiton key INSERT INTO upsert_test_2 (part_key, other_col) VALUES (1, 1); INSERT INTO upsert_test_2 (part_key, other_col) VALUES (1, 1) ON CONFLICT (part_key, other_col) DO NOTHING; -- this errors out since there is no unique constraint on partition key @@ -214,7 +214,7 @@ INSERT INTO dropcol_distributed AS dropcol (key, keep1) VALUES (1, '5') ON CONFL ALTER TABLE dropcol_distributed DROP COLUMN drop1; INSERT INTO dropcol_distributed AS dropcol (key, keep1) VALUES (1, '5') ON CONFLICT(key) DO UPDATE SET keep1 = dropcol.keep1; --- below we test the cases that CitusDB does not support +-- below we test the cases that Citus does not support -- subquery in the SET clause INSERT INTO upsert_test (part_key, other_col) VALUES (1, 1) ON CONFLICT (part_key) DO UPDATE SET other_col = (SELECT count(*) from upsert_test); diff --git a/src/test/regress/expected/multi_upsert_0.out b/src/test/regress/expected/multi_upsert_0.out index 461eb804c..c8c1e705e 100644 --- a/src/test/regress/expected/multi_upsert_0.out +++ b/src/test/regress/expected/multi_upsert_0.out @@ -1,4 +1,4 @@ --- this test file aims to test UPSERT feature on CitusDB +-- this test file aims to test UPSERT feature on Citus -- note that output of this file for postgresql 9.4 will -- be full syntax errors, which is expected. CREATE TABLE upsert_test @@ -157,7 +157,7 @@ SELECT master_create_worker_shards('upsert_test_2', '4', '2'); (1 row) --- now show that CitusDB works with multiple columns as the PRIMARY KEY, including the partiton key +-- now show that Citus works with multiple columns as the PRIMARY KEY, including the partiton key INSERT INTO upsert_test_2 (part_key, other_col) VALUES (1, 1); INSERT INTO upsert_test_2 (part_key, other_col) VALUES (1, 1) ON CONFLICT (part_key, other_col) DO NOTHING; ERROR: syntax error at or near "ON" @@ -284,7 +284,7 @@ INSERT INTO dropcol_distributed AS dropcol (key, keep1) VALUES (1, '5') ON CONFL ERROR: syntax error at or near "AS" LINE 1: INSERT INTO dropcol_distributed AS dropcol (key, keep1) VALU... ^ --- below we test the cases that CitusDB does not support +-- below we test the cases that Citus does not support -- subquery in the SET clause INSERT INTO upsert_test (part_key, other_col) VALUES (1, 1) ON CONFLICT (part_key) DO UPDATE SET other_col = (SELECT count(*) from upsert_test); diff --git a/src/test/regress/expected/multi_utility_statements.out b/src/test/regress/expected/multi_utility_statements.out index 04317ce81..ca5d7b4ad 100644 --- a/src/test/regress/expected/multi_utility_statements.out +++ b/src/test/regress/expected/multi_utility_statements.out @@ -38,7 +38,7 @@ SELECT * FROM lineitem_pricing_summary ORDER BY l_returnflag, l_linestatus; (4 rows) -- Test we can handle joins -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; CREATE TABLE shipping_priority AS ( SELECT diff --git a/src/test/regress/expected/multi_utility_warnings.out b/src/test/regress/expected/multi_utility_warnings.out index 6b564d473..b10cc7598 100644 --- a/src/test/regress/expected/multi_utility_warnings.out +++ b/src/test/regress/expected/multi_utility_warnings.out @@ -4,15 +4,15 @@ -- Tests to check if we inform the user about potential caveats of creating new -- databases, schemas, and roles. CREATE DATABASE new_database; -NOTICE: CitusDB partially supports CREATE DATABASE for distributed databases -DETAIL: CitusDB does not propagate CREATE DATABASE command to workers +NOTICE: Citus partially supports CREATE DATABASE for distributed databases +DETAIL: Citus does not propagate CREATE DATABASE command to workers HINT: You can manually create a database and its extensions on workers. CREATE SCHEMA new_schema; -NOTICE: CitusDB partially supports CREATE SCHEMA for distributed databases -DETAIL: schema usage in joins and in some UDFs provided by CitusDB are not supported yet +NOTICE: Citus partially supports CREATE SCHEMA for distributed databases +DETAIL: schema usage in joins and in some UDFs provided by Citus are not supported yet CREATE ROLE new_role; -NOTICE: CitusDB does not support CREATE ROLE/USER for distributed databases +NOTICE: Citus does not support CREATE ROLE/USER for distributed databases DETAIL: Multiple roles are currently supported only for local tables CREATE USER new_user; -NOTICE: CitusDB does not support CREATE ROLE/USER for distributed databases +NOTICE: Citus does not support CREATE ROLE/USER for distributed databases DETAIL: Multiple roles are currently supported only for local tables diff --git a/src/test/regress/input/multi_agg_distinct.source b/src/test/regress/input/multi_agg_distinct.source index 5d7fa7e53..69e431f5a 100644 --- a/src/test/regress/input/multi_agg_distinct.source +++ b/src/test/regress/input/multi_agg_distinct.source @@ -35,14 +35,14 @@ SELECT avg(distinct l_orderkey) FROM lineitem_range; -- sharded table. For this test, we also change a config setting to ensure that -- we don't repartition any of the tables during the query. -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; SELECT p_partkey, count(distinct l_orderkey) FROM lineitem_range, part WHERE l_partkey = p_partkey GROUP BY p_partkey ORDER BY p_partkey LIMIT 10; -RESET citusdb.large_table_shard_count; +RESET citus.large_table_shard_count; -- Check that we don't support count(distinct) on non-partition column, and -- complex expressions. diff --git a/src/test/regress/input/multi_append_table_to_shard.source b/src/test/regress/input/multi_append_table_to_shard.source index 3c33fe34c..04ff276d4 100644 --- a/src/test/regress/input/multi_append_table_to_shard.source +++ b/src/test/regress/input/multi_append_table_to_shard.source @@ -17,16 +17,16 @@ CREATE TABLE multi_append_table_to_shard_left SELECT master_create_distributed_table('multi_append_table_to_shard_left', 'left_number', 'append'); -- Replicate 'left' table on both workers -SELECT set_config('citusdb.shard_replication_factor', '2', false); +SELECT set_config('citus.shard_replication_factor', '2', false); \STAGE multi_append_table_to_shard_left FROM '@abs_srcdir@/data/agg.data' \STAGE multi_append_table_to_shard_left FROM '@abs_srcdir@/data/agg.data' -- Place 'right' table only on the primary worker -SELECT set_config('citusdb.shard_replication_factor', '1', false); +SELECT set_config('citus.shard_replication_factor', '1', false); \STAGE multi_append_table_to_shard_right FROM '@abs_srcdir@/data/agg.data' -- Reset shard replication factor to ensure tasks will be assigned to both workers -SELECT set_config('citusdb.shard_replication_factor', '2', false); +SELECT set_config('citus.shard_replication_factor', '2', false); -- All 8 rows in left table match a row in right table SELECT COUNT(*) diff --git a/src/test/regress/input/multi_fdw_stage_data.source b/src/test/regress/input/multi_fdw_stage_data.source index 67e45d351..8597407ca 100644 --- a/src/test/regress/input/multi_fdw_stage_data.source +++ b/src/test/regress/input/multi_fdw_stage_data.source @@ -10,7 +10,7 @@ \STAGE orders FROM '@abs_srcdir@/data/orders.1.data' \STAGE orders FROM '@abs_srcdir@/data/orders.2.data' -SELECT set_config('citusdb.shard_replication_factor', '1', false); +SELECT set_config('citus.shard_replication_factor', '1', false); \STAGE customer FROM '@abs_srcdir@/data/customer.1.data' \STAGE nation FROM '@abs_srcdir@/data/nation.data' diff --git a/src/test/regress/input/multi_outer_join.source b/src/test/regress/input/multi_outer_join.source index 7b46790ce..b0d0fce73 100644 --- a/src/test/regress/input/multi_outer_join.source +++ b/src/test/regress/input/multi_outer_join.source @@ -1,5 +1,5 @@ -SET citusdb.large_table_shard_count TO 2; -SET citusdb.log_multi_join_order to true; +SET citus.large_table_shard_count TO 2; +SET citus.log_multi_join_order to true; SET client_min_messages TO LOG; CREATE TABLE multi_outer_join_left diff --git a/src/test/regress/input/multi_stage_data.source b/src/test/regress/input/multi_stage_data.source index 3e093f9ed..a65b05c69 100644 --- a/src/test/regress/input/multi_stage_data.source +++ b/src/test/regress/input/multi_stage_data.source @@ -3,12 +3,12 @@ -- -- Tests for staging data in a distributed cluster. Please note that the number --- of shards uploaded depends on two config values: citusdb.shard_replication_factor and --- citusdb.shard_max_size. These values are manually set in pg_regress.c. We also set +-- of shards uploaded depends on two config values: citus.shard_replication_factor and +-- citus.shard_max_size. These values are manually set in pg_regress.c. We also set -- the shard placement policy to the local-node-first policy as other regression -- tests expect the placements to be in that order. -SET citusdb.shard_placement_policy TO 'local-node-first'; +SET citus.shard_placement_policy TO 'local-node-first'; \STAGE lineitem FROM '@abs_srcdir@/data/lineitem.1.data' with delimiter '|' \STAGE lineitem FROM '@abs_srcdir@/data/lineitem.2.data' with delimiter '|' diff --git a/src/test/regress/input/multi_stage_large_records.source b/src/test/regress/input/multi_stage_large_records.source index 5e92bb960..8e015d537 100644 --- a/src/test/regress/input/multi_stage_large_records.source +++ b/src/test/regress/input/multi_stage_large_records.source @@ -6,7 +6,7 @@ -- size, which is 32kB) in a distributed cluster. These tests make sure that we -- are creating shards of correct size even when records are large. -SET citusdb.shard_max_size TO "256kB"; +SET citus.shard_max_size TO "256kB"; CREATE TABLE large_records_table (data_id integer, data text); SELECT master_create_distributed_table('large_records_table', 'data_id', 'append'); @@ -17,4 +17,4 @@ SELECT shardminvalue, shardmaxvalue FROM pg_dist_shard, pg_class WHERE pg_class.oid=logicalrelid AND relname='large_records_table' ORDER BY shardid; -RESET citusdb.shard_max_size; +RESET citus.shard_max_size; diff --git a/src/test/regress/input/multi_stage_more_data.source b/src/test/regress/input/multi_stage_more_data.source index 4b685988a..63a24d5c1 100644 --- a/src/test/regress/input/multi_stage_more_data.source +++ b/src/test/regress/input/multi_stage_more_data.source @@ -8,7 +8,7 @@ -- We also set the shard placement policy to the local-node-first policy as other -- regression tests expect the placements to be in that order. -SET citusdb.shard_placement_policy TO 'local-node-first'; +SET citus.shard_placement_policy TO 'local-node-first'; \STAGE customer FROM '@abs_srcdir@/data/customer.2.data' with delimiter '|' \STAGE customer FROM '@abs_srcdir@/data/customer.3.data' with delimiter '|' diff --git a/src/test/regress/input/multi_subquery.source b/src/test/regress/input/multi_subquery.source index c4d8bb86f..0accf9135 100644 --- a/src/test/regress/input/multi_subquery.source +++ b/src/test/regress/input/multi_subquery.source @@ -37,7 +37,7 @@ CREATE TABLE orders_subquery ( PRIMARY KEY(o_orderkey) ); SELECT master_create_distributed_table('orders_subquery', 'o_orderkey', 'range'); -SET citusdb.task_executor_type TO 'task-tracker'; +SET citus.task_executor_type TO 'task-tracker'; -- Check that we don't allow subquery pushdown in default settings. @@ -55,7 +55,7 @@ FROM GROUP BY l_orderkey) AS unit_prices; -SET citusdb.subquery_pushdown to TRUE; +SET citus.subquery_pushdown to TRUE; -- Check that we don't crash if there are not any shards. @@ -75,7 +75,7 @@ FROM -- Stage data to tables. -SET citusdb.shard_max_size TO "1MB"; +SET citus.shard_max_size TO "1MB"; \STAGE lineitem_subquery FROM '@abs_srcdir@/data/lineitem.1.data' with delimiter '|' \STAGE lineitem_subquery FROM '@abs_srcdir@/data/lineitem.2.data' with delimiter '|' diff --git a/src/test/regress/output/multi_agg_distinct.source b/src/test/regress/output/multi_agg_distinct.source index 19510dbd0..060bb39d2 100644 --- a/src/test/regress/output/multi_agg_distinct.source +++ b/src/test/regress/output/multi_agg_distinct.source @@ -43,7 +43,7 @@ SELECT avg(distinct l_orderkey) FROM lineitem_range; -- Run count(distinct) on join between a range partitioned table and a single -- sharded table. For this test, we also change a config setting to ensure that -- we don't repartition any of the tables during the query. -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; SELECT p_partkey, count(distinct l_orderkey) FROM lineitem_range, part WHERE l_partkey = p_partkey GROUP BY p_partkey @@ -62,7 +62,7 @@ SELECT p_partkey, count(distinct l_orderkey) FROM lineitem_range, part 222 | 1 (10 rows) -RESET citusdb.large_table_shard_count; +RESET citus.large_table_shard_count; -- Check that we don't support count(distinct) on non-partition column, and -- complex expressions. SELECT count(distinct l_partkey) FROM lineitem_range; diff --git a/src/test/regress/output/multi_append_table_to_shard.source b/src/test/regress/output/multi_append_table_to_shard.source index 62580647c..682671c74 100644 --- a/src/test/regress/output/multi_append_table_to_shard.source +++ b/src/test/regress/output/multi_append_table_to_shard.source @@ -25,7 +25,7 @@ SELECT master_create_distributed_table('multi_append_table_to_shard_left', 'left (1 row) -- Replicate 'left' table on both workers -SELECT set_config('citusdb.shard_replication_factor', '2', false); +SELECT set_config('citus.shard_replication_factor', '2', false); set_config ------------ 2 @@ -34,7 +34,7 @@ SELECT set_config('citusdb.shard_replication_factor', '2', false); \STAGE multi_append_table_to_shard_left FROM '@abs_srcdir@/data/agg.data' \STAGE multi_append_table_to_shard_left FROM '@abs_srcdir@/data/agg.data' -- Place 'right' table only on the primary worker -SELECT set_config('citusdb.shard_replication_factor', '1', false); +SELECT set_config('citus.shard_replication_factor', '1', false); set_config ------------ 1 @@ -42,7 +42,7 @@ SELECT set_config('citusdb.shard_replication_factor', '1', false); \STAGE multi_append_table_to_shard_right FROM '@abs_srcdir@/data/agg.data' -- Reset shard replication factor to ensure tasks will be assigned to both workers -SELECT set_config('citusdb.shard_replication_factor', '2', false); +SELECT set_config('citus.shard_replication_factor', '2', false); set_config ------------ 2 diff --git a/src/test/regress/output/multi_create_schema.source b/src/test/regress/output/multi_create_schema.source index 26512abd3..690a309d4 100644 --- a/src/test/regress/output/multi_create_schema.source +++ b/src/test/regress/output/multi_create_schema.source @@ -4,8 +4,8 @@ CREATE TABLE nation ( n_name char(25) not null, n_regionkey integer not null, n_comment varchar(152)); -NOTICE: CitusDB partially supports CREATE SCHEMA for distributed databases -DETAIL: schema usage in joins and in some UDFs provided by CitusDB are not supported yet +NOTICE: Citus partially supports CREATE SCHEMA for distributed databases +DETAIL: schema usage in joins and in some UDFs provided by Citus are not supported yet SELECT master_create_distributed_table('tpch.nation', 'n_nationkey', 'append'); master_create_distributed_table --------------------------------- @@ -13,10 +13,10 @@ SELECT master_create_distributed_table('tpch.nation', 'n_nationkey', 'append'); (1 row) \STAGE tpch.nation FROM '@abs_srcdir@/data/nation.data' with delimiter '|' -NOTICE: CitusDB partially supports CREATE SCHEMA for distributed databases -DETAIL: schema usage in joins and in some UDFs provided by CitusDB are not supported yet -NOTICE: CitusDB partially supports CREATE SCHEMA for distributed databases -DETAIL: schema usage in joins and in some UDFs provided by CitusDB are not supported yet +NOTICE: Citus partially supports CREATE SCHEMA for distributed databases +DETAIL: schema usage in joins and in some UDFs provided by Citus are not supported yet +NOTICE: Citus partially supports CREATE SCHEMA for distributed databases +DETAIL: schema usage in joins and in some UDFs provided by Citus are not supported yet SELECT count(*) from tpch.nation; count ------- diff --git a/src/test/regress/output/multi_fdw_stage_data.source b/src/test/regress/output/multi_fdw_stage_data.source index b32c1321b..3cb781624 100644 --- a/src/test/regress/output/multi_fdw_stage_data.source +++ b/src/test/regress/output/multi_fdw_stage_data.source @@ -14,7 +14,7 @@ NOTICE: extension "file_fdw" already exists, skipping \STAGE orders FROM '@abs_srcdir@/data/orders.2.data' NOTICE: extension "file_fdw" already exists, skipping NOTICE: extension "file_fdw" already exists, skipping -SELECT set_config('citusdb.shard_replication_factor', '1', false); +SELECT set_config('citus.shard_replication_factor', '1', false); set_config ------------ 1 diff --git a/src/test/regress/output/multi_outer_join.source b/src/test/regress/output/multi_outer_join.source index 02e0ce21f..025f258fe 100644 --- a/src/test/regress/output/multi_outer_join.source +++ b/src/test/regress/output/multi_outer_join.source @@ -1,5 +1,5 @@ -SET citusdb.large_table_shard_count TO 2; -SET citusdb.log_multi_join_order to true; +SET citus.large_table_shard_count TO 2; +SET citus.log_multi_join_order to true; SET client_min_messages TO LOG; CREATE TABLE multi_outer_join_left ( diff --git a/src/test/regress/output/multi_stage_data.source b/src/test/regress/output/multi_stage_data.source index 58a8b15a0..74a4d84e2 100644 --- a/src/test/regress/output/multi_stage_data.source +++ b/src/test/regress/output/multi_stage_data.source @@ -2,11 +2,11 @@ -- MULTI_STAGE_DATA -- -- Tests for staging data in a distributed cluster. Please note that the number --- of shards uploaded depends on two config values: citusdb.shard_replication_factor and --- citusdb.shard_max_size. These values are manually set in pg_regress.c. We also set +-- of shards uploaded depends on two config values: citus.shard_replication_factor and +-- citus.shard_max_size. These values are manually set in pg_regress.c. We also set -- the shard placement policy to the local-node-first policy as other regression -- tests expect the placements to be in that order. -SET citusdb.shard_placement_policy TO 'local-node-first'; +SET citus.shard_placement_policy TO 'local-node-first'; \STAGE lineitem FROM '@abs_srcdir@/data/lineitem.1.data' with delimiter '|' \STAGE lineitem FROM '@abs_srcdir@/data/lineitem.2.data' with delimiter '|' \STAGE orders FROM '@abs_srcdir@/data/orders.1.data' with delimiter '|' diff --git a/src/test/regress/output/multi_stage_large_records.source b/src/test/regress/output/multi_stage_large_records.source index a8e7549f3..24eaa14e6 100644 --- a/src/test/regress/output/multi_stage_large_records.source +++ b/src/test/regress/output/multi_stage_large_records.source @@ -4,7 +4,7 @@ -- Tests for staging data with large records (i.e. greater than the read buffer -- size, which is 32kB) in a distributed cluster. These tests make sure that we -- are creating shards of correct size even when records are large. -SET citusdb.shard_max_size TO "256kB"; +SET citus.shard_max_size TO "256kB"; CREATE TABLE large_records_table (data_id integer, data text); SELECT master_create_distributed_table('large_records_table', 'data_id', 'append'); master_create_distributed_table @@ -22,4 +22,4 @@ SELECT shardminvalue, shardmaxvalue FROM pg_dist_shard, pg_class 2 | 2 (2 rows) -RESET citusdb.shard_max_size; +RESET citus.shard_max_size; diff --git a/src/test/regress/output/multi_stage_more_data.source b/src/test/regress/output/multi_stage_more_data.source index d4b95c01d..0a57bc64e 100644 --- a/src/test/regress/output/multi_stage_more_data.source +++ b/src/test/regress/output/multi_stage_more_data.source @@ -6,7 +6,7 @@ -- evaluate plans where some of the underlying tables need to be repartitioned. -- We also set the shard placement policy to the local-node-first policy as other -- regression tests expect the placements to be in that order. -SET citusdb.shard_placement_policy TO 'local-node-first'; +SET citus.shard_placement_policy TO 'local-node-first'; \STAGE customer FROM '@abs_srcdir@/data/customer.2.data' with delimiter '|' \STAGE customer FROM '@abs_srcdir@/data/customer.3.data' with delimiter '|' \STAGE part FROM '@abs_srcdir@/data/part.more.data' with delimiter '|' diff --git a/src/test/regress/output/multi_subquery.source b/src/test/regress/output/multi_subquery.source index 84e38aa5c..f0e6f0250 100644 --- a/src/test/regress/output/multi_subquery.source +++ b/src/test/regress/output/multi_subquery.source @@ -43,7 +43,7 @@ SELECT master_create_distributed_table('orders_subquery', 'o_orderkey', 'range') (1 row) -SET citusdb.task_executor_type TO 'task-tracker'; +SET citus.task_executor_type TO 'task-tracker'; -- Check that we don't allow subquery pushdown in default settings. SELECT avg(unit_price) @@ -60,7 +60,7 @@ FROM l_orderkey) AS unit_prices; ERROR: cannot perform distributed planning on this query DETAIL: Join in subqueries is not supported yet -SET citusdb.subquery_pushdown to TRUE; +SET citus.subquery_pushdown to TRUE; -- Check that we don't crash if there are not any shards. SELECT avg(unit_price) @@ -81,7 +81,7 @@ FROM (1 row) -- Stage data to tables. -SET citusdb.shard_max_size TO "1MB"; +SET citus.shard_max_size TO "1MB"; \STAGE lineitem_subquery FROM '@abs_srcdir@/data/lineitem.1.data' with delimiter '|' \STAGE lineitem_subquery FROM '@abs_srcdir@/data/lineitem.2.data' with delimiter '|' \STAGE orders_subquery FROM '@abs_srcdir@/data/orders.1.data' with delimiter '|' diff --git a/src/test/regress/pg_regress_multi.pl b/src/test/regress/pg_regress_multi.pl index e1f106f90..aa863021e 100644 --- a/src/test/regress/pg_regress_multi.pl +++ b/src/test/regress/pg_regress_multi.pl @@ -1,7 +1,7 @@ #!/usr/bin/perl -w #---------------------------------------------------------------------- # -# pg_regress_multi.pl - Test runner for CitusDB +# pg_regress_multi.pl - Test runner for Citus # # Portions Copyright (c) 2012-2015, Citus Data, Inc. # Portions Copyright (c) 1996-2015, PostgreSQL Global Development Group @@ -19,7 +19,7 @@ use Getopt::Long; sub Usage() { - print "pg_regress_multi - CitusDB test runner\n"; + print "pg_regress_multi - Citus test runner\n"; print "\n"; print "Usage:\n"; print " pg_regress_multi [MULTI OPTIONS] -- [PG REGRESS OPTS]\n"; @@ -56,7 +56,7 @@ GetOptions( # Update environment to include [DY]LD_LIBRARY_PATH/LIBDIR/etc - # pointing to the libdir - that's required so the right version of -# libpq, citusdb et al is being picked up. +# libpq, citus et al is being picked up. # # XXX: There's some issues with el capitan's SIP here, causing # DYLD_LIBRARY_PATH not being inherited if SIP is enabled. That's a @@ -87,12 +87,12 @@ push(@pgOptions, '-c', "listen_addresses='${host}'"); # not required, and we don't necessarily have access to the default directory push(@pgOptions, '-c', "unix_socket_directories="); push(@pgOptions, '-c', "fsync=off"); -push(@pgOptions, '-c', "shared_preload_libraries=citusdb"); +push(@pgOptions, '-c', "shared_preload_libraries=citus"); -# CitusDB options set for the tests -push(@pgOptions, '-c', "citusdb.shard_max_size=300kB"); -push(@pgOptions, '-c', "citusdb.max_running_tasks_per_node=4"); -push(@pgOptions, '-c', "citusdb.expire_cached_shards=on"); +# Citus options set for the tests +push(@pgOptions, '-c', "citus.shard_max_size=300kB"); +push(@pgOptions, '-c', "citus.max_running_tasks_per_node=4"); +push(@pgOptions, '-c', "citus.expire_cached_shards=on"); # Add externally added options last, so they overwrite the default ones above for my $option (@userPgOptions) @@ -107,7 +107,7 @@ for my $option (@userPgOptions) 'bug_status', ' ENUM (\'new\', \'open\', \'closed\')'); # define functions as signature->definition -%functions = ('fake_fdw_handler()', 'fdw_handler AS \'citusdb\' LANGUAGE C STRICT;'); +%functions = ('fake_fdw_handler()', 'fdw_handler AS \'citus\' LANGUAGE C STRICT;'); #define fdws as name->handler name %fdws = ('fake_fdw', 'fake_fdw_handler'); @@ -274,7 +274,7 @@ elsif ($majorversion eq '9.4') } else { - die "CitusDB is not compatible with the detected PostgreSQL version $majorversion"; + die "Citus is not compatible with the detected PostgreSQL version $majorversion"; } # Add load extension parameters to the argument list diff --git a/src/test/regress/sql/multi_agg_approximate_distinct.sql b/src/test/regress/sql/multi_agg_approximate_distinct.sql index 386ca3abb..5ed4d527c 100644 --- a/src/test/regress/sql/multi_agg_approximate_distinct.sql +++ b/src/test/regress/sql/multi_agg_approximate_distinct.sql @@ -8,10 +8,10 @@ SELECT count(distinct l_orderkey) FROM lineitem; -- Check approximate count(distinct) at different precisions / error rates -SET citusdb.count_distinct_error_rate = 0.1; +SET citus.count_distinct_error_rate = 0.1; SELECT count(distinct l_orderkey) FROM lineitem; -SET citusdb.count_distinct_error_rate = 0.01; +SET citus.count_distinct_error_rate = 0.01; SELECT count(distinct l_orderkey) FROM lineitem; -- Check approximate count(distinct) for different data types @@ -50,7 +50,7 @@ SELECT count(DISTINCT l_orderkey) as distinct_order_count, l_quantity FROM linei -- If we have an order by on count(distinct) that we intend to push down to -- worker nodes, we need to error out. Otherwise, we are fine. -SET citusdb.limit_clause_row_fetch_count = 1000; +SET citus.limit_clause_row_fetch_count = 1000; SELECT l_returnflag, count(DISTINCT l_shipdate) as count_distinct, count(*) as total FROM lineitem GROUP BY l_returnflag @@ -65,5 +65,5 @@ SELECT l_returnflag, count(DISTINCT l_shipdate) as count_distinct, count(*) as t -- Check that we can revert config and disable count(distinct) approximations -SET citusdb.count_distinct_error_rate = 0.0; +SET citus.count_distinct_error_rate = 0.0; SELECT count(distinct l_orderkey) FROM lineitem; diff --git a/src/test/regress/sql/multi_binary_master_copy_format.sql b/src/test/regress/sql/multi_binary_master_copy_format.sql index 37623ce36..4c67eb2d2 100644 --- a/src/test/regress/sql/multi_binary_master_copy_format.sql +++ b/src/test/regress/sql/multi_binary_master_copy_format.sql @@ -4,13 +4,13 @@ -- Try binary master copy for different executors -SET citusdb.binary_master_copy_format TO 'on'; -SET citusdb.task_executor_type TO 'task-tracker'; +SET citus.binary_master_copy_format TO 'on'; +SET citus.task_executor_type TO 'task-tracker'; SELECT count(*) FROM lineitem; SELECT l_shipmode FROM lineitem WHERE l_partkey = 67310 OR l_partkey = 155190; -SET citusdb.task_executor_type TO 'real-time'; +SET citus.task_executor_type TO 'real-time'; SELECT count(*) FROM lineitem; SELECT l_shipmode FROM lineitem WHERE l_partkey = 67310 OR l_partkey = 155190; diff --git a/src/test/regress/sql/multi_connection_cache.sql b/src/test/regress/sql/multi_connection_cache.sql index 33bad147e..ea1bdff7d 100644 --- a/src/test/regress/sql/multi_connection_cache.sql +++ b/src/test/regress/sql/multi_connection_cache.sql @@ -4,22 +4,22 @@ CREATE FUNCTION initialize_remote_temp_table(cstring, integer) RETURNS bool - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION count_remote_temp_table_rows(cstring, integer) RETURNS integer - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION get_and_purge_connection(cstring, integer) RETURNS bool - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION set_connection_status_bad(cstring, integer) RETURNS bool - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; -- =================================================================== diff --git a/src/test/regress/sql/multi_create_fdw.sql b/src/test/regress/sql/multi_create_fdw.sql index 0f237be11..dd828a66b 100644 --- a/src/test/regress/sql/multi_create_fdw.sql +++ b/src/test/regress/sql/multi_create_fdw.sql @@ -5,7 +5,7 @@ -- create fake fdw for use in tests CREATE FUNCTION fake_fdw_handler() RETURNS fdw_handler -AS 'citusdb' +AS 'citus' LANGUAGE C STRICT; CREATE FOREIGN DATA WRAPPER fake_fdw HANDLER fake_fdw_handler; diff --git a/src/test/regress/sql/multi_create_shards.sql b/src/test/regress/sql/multi_create_shards.sql index 5b51dccd2..1180338ac 100644 --- a/src/test/regress/sql/multi_create_shards.sql +++ b/src/test/regress/sql/multi_create_shards.sql @@ -4,7 +4,7 @@ CREATE FUNCTION sort_names(cstring, cstring, cstring) RETURNS cstring - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; -- create a custom type... @@ -125,7 +125,7 @@ CREATE TABLE weird_shard_count SELECT master_create_distributed_table('weird_shard_count', 'id', 'hash'); SELECT master_create_worker_shards('weird_shard_count', 7, 1); --- CitusDB ensures all shards are roughly the same size +-- Citus ensures all shards are roughly the same size SELECT shardmaxvalue::integer - shardminvalue::integer AS shard_size FROM pg_dist_shard WHERE logicalrelid = 'weird_shard_count'::regclass diff --git a/src/test/regress/sql/multi_create_table.sql b/src/test/regress/sql/multi_create_table.sql index 987988771..3509436f6 100644 --- a/src/test/regress/sql/multi_create_table.sql +++ b/src/test/regress/sql/multi_create_table.sql @@ -83,7 +83,7 @@ CREATE TABLE supplier SELECT master_create_distributed_table('supplier', 's_suppkey', 'append'); --- now test that CitusDB cannot distribute unique constraints that do not include +-- now test that Citus cannot distribute unique constraints that do not include -- the partition column CREATE TABLE primary_key_on_non_part_col ( @@ -99,7 +99,7 @@ CREATE TABLE unique_const_on_non_part_col ); SELECT master_create_distributed_table('primary_key_on_non_part_col', 'partition_col', 'hash'); --- now show that CitusDB can distribute unique constrints that include +-- now show that Citus can distribute unique constrints that include -- the partition column CREATE TABLE primary_key_on_part_col ( diff --git a/src/test/regress/sql/multi_distribution_metadata.sql b/src/test/regress/sql/multi_distribution_metadata.sql index 51b3fe2ee..d53d88c14 100644 --- a/src/test/regress/sql/multi_distribution_metadata.sql +++ b/src/test/regress/sql/multi_distribution_metadata.sql @@ -4,67 +4,67 @@ CREATE FUNCTION load_shard_id_array(regclass) RETURNS bigint[] - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION load_shard_interval_array(bigint, anyelement) RETURNS anyarray - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION load_shard_placement_array(bigint, bool) RETURNS text[] - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION partition_column_id(regclass) RETURNS smallint - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION partition_type(regclass) RETURNS "char" - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION is_distributed_table(regclass) RETURNS boolean - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION column_name_to_column_id(regclass, cstring) RETURNS smallint - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION create_monolithic_shard_row(regclass) RETURNS bigint - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION create_healthy_local_shard_placement_row(bigint) RETURNS void - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION delete_shard_placement_row(bigint, text, bigint) RETURNS bool - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION update_shard_placement_row_state(bigint, text, bigint, int) RETURNS bool - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION acquire_shared_shard_lock(bigint) RETURNS void - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION column_name_to_column(regclass, text) RETURNS text - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; -- =================================================================== diff --git a/src/test/regress/sql/multi_generate_ddl_commands.sql b/src/test/regress/sql/multi_generate_ddl_commands.sql index 5e7c4be99..0bd2c28fa 100644 --- a/src/test/regress/sql/multi_generate_ddl_commands.sql +++ b/src/test/regress/sql/multi_generate_ddl_commands.sql @@ -4,7 +4,7 @@ CREATE FUNCTION table_ddl_command_array(regclass) RETURNS text[] - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; -- =================================================================== diff --git a/src/test/regress/sql/multi_join_order_additional.sql b/src/test/regress/sql/multi_join_order_additional.sql index 678d9077c..c1668f061 100644 --- a/src/test/regress/sql/multi_join_order_additional.sql +++ b/src/test/regress/sql/multi_join_order_additional.sql @@ -4,7 +4,7 @@ -- Set configuration to print table join order and pruned shards -SET citusdb.log_multi_join_order TO TRUE; +SET citus.log_multi_join_order TO TRUE; SET client_min_messages TO DEBUG2; -- The following query checks that we can correctly handle self-joins @@ -14,7 +14,7 @@ EXPLAIN SELECT l1.l_quantity FROM lineitem l1, lineitem l2 -- Update configuration to treat lineitem and orders tables as large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; SET client_min_messages TO LOG; -- The following queries check that we correctly handle joins and OR clauses. In @@ -66,7 +66,7 @@ UPDATE pg_dist_partition SET partmethod = 'a' WHERE logicalrelid = (SELECT relfilenode FROM pg_class WHERE relname = 'customer'); -- Update the large table shard count for all the following tests. -SET citusdb.large_table_shard_count TO 1; +SET citus.large_table_shard_count TO 1; -- Validate that we don't use a single-partition join method for a hash -- re-partitioned table, thus preventing a partition of just the customer table. diff --git a/src/test/regress/sql/multi_join_order_tpch_large.sql b/src/test/regress/sql/multi_join_order_tpch_large.sql index 7b0431fd8..a0d1e5a9f 100644 --- a/src/test/regress/sql/multi_join_order_tpch_large.sql +++ b/src/test/regress/sql/multi_join_order_tpch_large.sql @@ -4,7 +4,7 @@ -- Enable configuration to print table join order -SET citusdb.log_multi_join_order TO TRUE; +SET citus.log_multi_join_order TO TRUE; SET client_min_messages TO LOG; -- Change configuration to treat lineitem, orders, customer, and part tables as @@ -12,7 +12,7 @@ SET client_min_messages TO LOG; -- except that more data has been staged to customer and part tables. Therefore, -- we will apply different distributed join strategies for these queries. -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; -- Query #6 from the TPC-H decision support benchmark diff --git a/src/test/regress/sql/multi_join_order_tpch_small.sql b/src/test/regress/sql/multi_join_order_tpch_small.sql index 545bab32c..b4663cbe9 100644 --- a/src/test/regress/sql/multi_join_order_tpch_small.sql +++ b/src/test/regress/sql/multi_join_order_tpch_small.sql @@ -4,12 +4,12 @@ -- Enable configuration to print table join order -SET citusdb.log_multi_join_order TO TRUE; +SET citus.log_multi_join_order TO TRUE; SET client_min_messages TO LOG; -- Change configuration to treat lineitem and orders tables as large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; -- Query #6 from the TPC-H decision support benchmark diff --git a/src/test/regress/sql/multi_join_pruning.sql b/src/test/regress/sql/multi_join_pruning.sql index 29df48095..293b05f4c 100644 --- a/src/test/regress/sql/multi_join_pruning.sql +++ b/src/test/regress/sql/multi_join_pruning.sql @@ -10,7 +10,7 @@ SET client_min_messages TO DEBUG2; -- Change configuration to treat all tables as large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; SELECT sum(l_linenumber), avg(l_linenumber) FROM lineitem, orders WHERE l_orderkey = o_orderkey; diff --git a/src/test/regress/sql/multi_large_table_join_planning.sql b/src/test/regress/sql/multi_large_table_join_planning.sql index 06a914361..8ad2a3fe6 100644 --- a/src/test/regress/sql/multi_large_table_join_planning.sql +++ b/src/test/regress/sql/multi_large_table_join_planning.sql @@ -9,8 +9,8 @@ BEGIN; SET client_min_messages TO DEBUG4; -SET citusdb.large_table_shard_count TO 2; -SET citusdb.task_executor_type TO 'task-tracker'; +SET citus.large_table_shard_count TO 2; +SET citus.task_executor_type TO 'task-tracker'; -- Debug4 log messages display jobIds within them. We explicitly set the jobId -- sequence here so that the regression output becomes independent of the number diff --git a/src/test/regress/sql/multi_large_table_pruning.sql b/src/test/regress/sql/multi_large_table_pruning.sql index 897aa73a7..ec46eff0c 100644 --- a/src/test/regress/sql/multi_large_table_pruning.sql +++ b/src/test/regress/sql/multi_large_table_pruning.sql @@ -6,9 +6,9 @@ -- set executor type to task tracker executor here, as we cannot run repartition -- jobs with real time executor. -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; SET client_min_messages TO DEBUG2; -SET citusdb.task_executor_type TO 'task-tracker'; +SET citus.task_executor_type TO 'task-tracker'; -- Single range-repartition join to test join-pruning behaviour. diff --git a/src/test/regress/sql/multi_large_table_task_assignment.sql b/src/test/regress/sql/multi_large_table_task_assignment.sql index 8fd902950..0793e4492 100644 --- a/src/test/regress/sql/multi_large_table_task_assignment.sql +++ b/src/test/regress/sql/multi_large_table_task_assignment.sql @@ -9,8 +9,8 @@ BEGIN; SET client_min_messages TO DEBUG3; -SET citusdb.large_table_shard_count TO 2; -SET citusdb.task_executor_type TO 'task-tracker'; +SET citus.large_table_shard_count TO 2; +SET citus.task_executor_type TO 'task-tracker'; -- Single range repartition join to test anchor-shard based task assignment and -- assignment propagation to merge and data-fetch tasks. @@ -27,7 +27,7 @@ WHERE -- the same merge task, and tests our constraint group creation and assignment -- propagation. Here 'orders' is considered the small table. -SET citusdb.large_table_shard_count TO 3; +SET citus.large_table_shard_count TO 3; SELECT count(*) @@ -37,7 +37,7 @@ WHERE o_custkey = c_custkey AND o_orderkey = l_orderkey; -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; -- The next test, dual hash repartition join, uses the current jobId to assign -- tasks in a round-robin fashion. We therefore need to ensure that jobIds start diff --git a/src/test/regress/sql/multi_limit_clause_approximate.sql b/src/test/regress/sql/multi_limit_clause_approximate.sql index 8daca0930..659e91ab6 100644 --- a/src/test/regress/sql/multi_limit_clause_approximate.sql +++ b/src/test/regress/sql/multi_limit_clause_approximate.sql @@ -16,7 +16,7 @@ SELECT l_partkey, sum(l_partkey * (1 + l_suppkey)) AS aggregate FROM lineitem -- Enable limit optimization to fetch one third of each shard's data -SET citusdb.limit_clause_row_fetch_count TO 600; +SET citus.limit_clause_row_fetch_count TO 600; SELECT l_partkey, sum(l_partkey * (1 + l_suppkey)) AS aggregate FROM lineitem GROUP BY l_partkey @@ -25,7 +25,7 @@ SELECT l_partkey, sum(l_partkey * (1 + l_suppkey)) AS aggregate FROM lineitem -- Disable limit optimization for our second test. This time, we have a query -- that joins several tables, and that groups and orders the results. -RESET citusdb.limit_clause_row_fetch_count; +RESET citus.limit_clause_row_fetch_count; SELECT c_custkey, c_name, count(*) as lineitem_count FROM customer, orders, lineitem @@ -37,8 +37,8 @@ SELECT c_custkey, c_name, count(*) as lineitem_count -- test, we also change a config setting to ensure that we don't repartition any -- of the tables during the query. -SET citusdb.limit_clause_row_fetch_count TO 150; -SET citusdb.large_table_shard_count TO 2; +SET citus.limit_clause_row_fetch_count TO 150; +SET citus.large_table_shard_count TO 2; SELECT c_custkey, c_name, count(*) as lineitem_count FROM customer, orders, lineitem @@ -46,7 +46,7 @@ SELECT c_custkey, c_name, count(*) as lineitem_count GROUP BY c_custkey, c_name ORDER BY lineitem_count DESC, c_custkey LIMIT 10; -RESET citusdb.large_table_shard_count; +RESET citus.large_table_shard_count; -- We now test scenarios where applying the limit optimization wouldn't produce -- meaningful results. First, we check that we don't push down the limit clause @@ -69,5 +69,5 @@ SELECT count(*) count_quantity, l_quantity FROM lineitem WHERE l_quantity < 10.0 GROUP BY l_quantity ORDER BY count_quantity ASC, l_quantity ASC; -RESET citusdb.limit_clause_row_fetch_count; +RESET citus.limit_clause_row_fetch_count; RESET client_min_messages; diff --git a/src/test/regress/sql/multi_modifications.sql b/src/test/regress/sql/multi_modifications.sql index 390913dfb..605f8905f 100644 --- a/src/test/regress/sql/multi_modifications.sql +++ b/src/test/regress/sql/multi_modifications.sql @@ -69,11 +69,11 @@ INSERT INTO append_partitioned VALUES (414123, 'AAPL', 9580, '2004-10-19 10:23:5 20.69); -- ensure the values are where we put them and query to ensure they are properly pruned SET client_min_messages TO 'DEBUG2'; -SET citusdb.task_executor_type TO 'router'; +SET citus.task_executor_type TO 'router'; SELECT * FROM range_partitioned WHERE id = 32743; SELECT * FROM append_partitioned WHERE id = 414123; SET client_min_messages TO DEFAULT; -SET citusdb.task_executor_type TO DEFAULT; +SET citus.task_executor_type TO DEFAULT; -- try inserting without a range-partitioned shard to receive the value INSERT INTO range_partitioned VALUES (999999, 'AAPL', 9580, '2004-10-19 10:23:54', 'buy', diff --git a/src/test/regress/sql/multi_null_minmax_value_pruning.sql b/src/test/regress/sql/multi_null_minmax_value_pruning.sql index 91828359d..9604f9ab4 100644 --- a/src/test/regress/sql/multi_null_minmax_value_pruning.sql +++ b/src/test/regress/sql/multi_null_minmax_value_pruning.sql @@ -9,7 +9,7 @@ SET client_min_messages TO DEBUG2; -- Change configuration to treat lineitem and orders tables as large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; SELECT shardminvalue, shardmaxvalue from pg_dist_shard WHERE shardid = 102009; SELECT shardminvalue, shardmaxvalue from pg_dist_shard WHERE shardid = 102010; diff --git a/src/test/regress/sql/multi_prepare_plsql.sql b/src/test/regress/sql/multi_prepare_plsql.sql index 2a91d5483..f62e256df 100644 --- a/src/test/regress/sql/multi_prepare_plsql.sql +++ b/src/test/regress/sql/multi_prepare_plsql.sql @@ -187,7 +187,7 @@ BEGIN END; $$ LANGUAGE plpgsql; -SET citusdb.task_executor_type TO 'task-tracker'; +SET citus.task_executor_type TO 'task-tracker'; SET client_min_messages TO INFO; -- now, run plain SQL functions @@ -222,7 +222,7 @@ SELECT plpgsql_test_2(); -- run the tests which do not require re-partition -- with real-time executor -SET citusdb.task_executor_type TO 'real-time'; +SET citus.task_executor_type TO 'real-time'; -- now, run plain SQL functions SELECT sql_test_no_1(); diff --git a/src/test/regress/sql/multi_prepare_sql.sql b/src/test/regress/sql/multi_prepare_sql.sql index 64a1c0b10..346b8de6a 100644 --- a/src/test/regress/sql/multi_prepare_sql.sql +++ b/src/test/regress/sql/multi_prepare_sql.sql @@ -104,7 +104,7 @@ ORDER BY cust_nation, l_year; -SET citusdb.task_executor_type TO 'task-tracker'; +SET citus.task_executor_type TO 'task-tracker'; SET client_min_messages TO INFO; -- execute prepared statements @@ -136,7 +136,7 @@ CREATE TEMP TABLE prepared_sql_test_7 AS EXECUTE prepared_test_7('UNITED KINGDOM SELECT * from prepared_sql_test_7; -- now, run some of the tests with real-time executor -SET citusdb.task_executor_type TO 'real-time'; +SET citus.task_executor_type TO 'real-time'; -- execute prepared statements EXECUTE prepared_test_1; diff --git a/src/test/regress/sql/multi_prune_shard_list.sql b/src/test/regress/sql/multi_prune_shard_list.sql index f684bfa83..2a215c5a5 100644 --- a/src/test/regress/sql/multi_prune_shard_list.sql +++ b/src/test/regress/sql/multi_prune_shard_list.sql @@ -4,27 +4,27 @@ CREATE FUNCTION prune_using_no_values(regclass) RETURNS text[] - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION prune_using_single_value(regclass, text) RETURNS text[] - AS 'citusdb' + AS 'citus' LANGUAGE C; CREATE FUNCTION prune_using_either_value(regclass, text, text) RETURNS text[] - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION prune_using_both_values(regclass, text, text) RETURNS text[] - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; CREATE FUNCTION debug_equality_expression(regclass) RETURNS cstring - AS 'citusdb' + AS 'citus' LANGUAGE C STRICT; -- =================================================================== diff --git a/src/test/regress/sql/multi_simple_queries.sql b/src/test/regress/sql/multi_simple_queries.sql index 1e774ebb6..45fc1ebf2 100644 --- a/src/test/regress/sql/multi_simple_queries.sql +++ b/src/test/regress/sql/multi_simple_queries.sql @@ -112,7 +112,7 @@ SELECT title, author_id FROM articles ORDER BY author_id ASC, id; -- add in some grouping expressions, still on same shard --- having queries unsupported in CitusDB +-- having queries unsupported in Citus SELECT author_id, sum(word_count) AS corpus_size FROM articles WHERE author_id = 1 OR author_id = 7 OR author_id = 8 OR author_id = 10 GROUP BY author_id @@ -130,7 +130,7 @@ SELECT title FROM articles; -- queries which involve functions in FROM clause are unsupported. SELECT * FROM articles, position('om' in 'Thomas'); --- subqueries are not supported in WHERE clause in CitusDB +-- subqueries are not supported in WHERE clause in Citus SELECT * FROM articles WHERE author_id IN (SELECT id FROM authors WHERE name LIKE '%a'); -- subqueries are supported in FROM clause @@ -166,23 +166,23 @@ $sharded_sql$; -- test cross-shard queries SELECT COUNT(*) FROM articles; --- having queries unsupported in CitusDB +-- having queries unsupported in Citus SELECT author_id, sum(word_count) AS corpus_size FROM articles GROUP BY author_id HAVING sum(word_count) > 25000 ORDER BY sum(word_count) DESC LIMIT 5; --- more proof CitusDB doesn't support having clauses +-- more proof Citus doesn't support having clauses SELECT author_id FROM articles GROUP BY author_id HAVING sum(word_count) > 50000 ORDER BY author_id; --- now, test the cases where CitusDB do or do not need to create +-- now, test the cases where Citus do or do not need to create -- the master queries -SET citusdb.task_executor_type TO 'router'; -SET citusdb.large_table_shard_count TO 2; +SET citus.task_executor_type TO 'router'; +SET citus.large_table_shard_count TO 2; SET client_min_messages TO 'DEBUG2'; -- start with the simple lookup query @@ -277,4 +277,4 @@ SELECT * WHERE author_id >= 1 AND author_id <= 3; SET client_min_messages to 'NOTICE'; -SET citusdb.task_executor_type TO 'real-time'; +SET citus.task_executor_type TO 'real-time'; diff --git a/src/test/regress/sql/multi_single_relation_subquery.sql b/src/test/regress/sql/multi_single_relation_subquery.sql index 19e56f60e..691e51c7a 100644 --- a/src/test/regress/sql/multi_single_relation_subquery.sql +++ b/src/test/regress/sql/multi_single_relation_subquery.sql @@ -4,7 +4,7 @@ -- This test checks that we are able to run selected set of distributed SQL subqueries. -SET citusdb.task_executor_type TO 'task-tracker'; +SET citus.task_executor_type TO 'task-tracker'; select number_sum, diff --git a/src/test/regress/sql/multi_table_ddl.sql b/src/test/regress/sql/multi_table_ddl.sql index f2f8a6290..2fe55b758 100644 --- a/src/test/regress/sql/multi_table_ddl.sql +++ b/src/test/regress/sql/multi_table_ddl.sql @@ -6,8 +6,8 @@ CREATE TABLE testtableddl(somecol int, distributecol text NOT NULL); SELECT master_create_distributed_table('testtableddl', 'distributecol', 'append'); --- verify that the citusdb extension can't be dropped while distributed tables exist -DROP EXTENSION citusdb; +-- verify that the citus extension can't be dropped while distributed tables exist +DROP EXTENSION citus; -- verify that the distribution column can't have its type changed ALTER TABLE testtableddl ALTER COLUMN distributecol TYPE text; @@ -34,6 +34,6 @@ SELECT * FROM pg_dist_shard_placement; -- check that the extension now can be dropped (and recreated). We reconnect -- before creating the extension to expire extension specific variables which -- are cached for performance. -DROP EXTENSION citusdb; +DROP EXTENSION citus; \c -CREATE EXTENSION citusdb; +CREATE EXTENSION citus; diff --git a/src/test/regress/sql/multi_task_assignment_policy.sql b/src/test/regress/sql/multi_task_assignment_policy.sql index 935c13558..5ee4cddfc 100644 --- a/src/test/regress/sql/multi_task_assignment_policy.sql +++ b/src/test/regress/sql/multi_task_assignment_policy.sql @@ -55,7 +55,7 @@ SET client_min_messages TO DEBUG3; -- First test the default greedy task assignment policy -SET citusdb.task_assignment_policy TO 'greedy'; +SET citus.task_assignment_policy TO 'greedy'; EXPLAIN SELECT count(*) FROM task_assignment_test_table; @@ -63,7 +63,7 @@ EXPLAIN SELECT count(*) FROM task_assignment_test_table; -- Next test the first-replica task assignment policy -SET citusdb.task_assignment_policy TO 'first-replica'; +SET citus.task_assignment_policy TO 'first-replica'; EXPLAIN SELECT count(*) FROM task_assignment_test_table; @@ -80,7 +80,7 @@ SELECT case when (currval('pg_dist_jobid_seq') % 2) = 0 -- Finally test the round-robin task assignment policy -SET citusdb.task_assignment_policy TO 'round-robin'; +SET citus.task_assignment_policy TO 'round-robin'; EXPLAIN SELECT count(*) FROM task_assignment_test_table; @@ -88,7 +88,7 @@ EXPLAIN SELECT count(*) FROM task_assignment_test_table; EXPLAIN SELECT count(*) FROM task_assignment_test_table; -RESET citusdb.task_assignment_policy; +RESET citus.task_assignment_policy; RESET client_min_messages; COMMIT; diff --git a/src/test/regress/sql/multi_tpch_query1.sql b/src/test/regress/sql/multi_tpch_query1.sql index f79aa6586..d6727d15a 100644 --- a/src/test/regress/sql/multi_tpch_query1.sql +++ b/src/test/regress/sql/multi_tpch_query1.sql @@ -4,7 +4,7 @@ -- Change configuration to treat lineitem and orders tables as large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; -- Query #1 from the TPC-H decision support benchmark diff --git a/src/test/regress/sql/multi_tpch_query10.sql b/src/test/regress/sql/multi_tpch_query10.sql index 581647403..75bb475a5 100644 --- a/src/test/regress/sql/multi_tpch_query10.sql +++ b/src/test/regress/sql/multi_tpch_query10.sql @@ -3,7 +3,7 @@ -- -- Query #10 from the TPC-H decision support benchmark. Unlike other TPC-H tests, --- we don't set citusdb.large_table_shard_count here, and instead use the default value +-- we don't set citus.large_table_shard_count here, and instead use the default value -- coming from postgresql.conf or multi_task_tracker_executor.conf. SELECT diff --git a/src/test/regress/sql/multi_tpch_query12.sql b/src/test/regress/sql/multi_tpch_query12.sql index fd85c82c0..380165bb2 100644 --- a/src/test/regress/sql/multi_tpch_query12.sql +++ b/src/test/regress/sql/multi_tpch_query12.sql @@ -4,7 +4,7 @@ -- Change configuration to treat lineitem and orders tables as large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; -- Query #12 from the TPC-H decision support benchmark diff --git a/src/test/regress/sql/multi_tpch_query14.sql b/src/test/regress/sql/multi_tpch_query14.sql index 83cf9e954..82b696fec 100644 --- a/src/test/regress/sql/multi_tpch_query14.sql +++ b/src/test/regress/sql/multi_tpch_query14.sql @@ -4,7 +4,7 @@ -- Change configuration to treat lineitem and orders tables as large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; -- Query #14 from the TPC-H decision support benchmark diff --git a/src/test/regress/sql/multi_tpch_query19.sql b/src/test/regress/sql/multi_tpch_query19.sql index 32255e74c..dfff2ea5d 100644 --- a/src/test/regress/sql/multi_tpch_query19.sql +++ b/src/test/regress/sql/multi_tpch_query19.sql @@ -4,7 +4,7 @@ -- Change configuration to treat lineitem and orders tables as large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; -- Query #19 from the TPC-H decision support benchmark. Note that we modified -- the query from its original to make it work on smaller data sets. diff --git a/src/test/regress/sql/multi_tpch_query3.sql b/src/test/regress/sql/multi_tpch_query3.sql index e23ba3329..62232e5fa 100644 --- a/src/test/regress/sql/multi_tpch_query3.sql +++ b/src/test/regress/sql/multi_tpch_query3.sql @@ -3,7 +3,7 @@ -- -- Query #3 from the TPC-H decision support benchmark. Unlike other TPC-H tests, --- we don't set citusdb.large_table_shard_count here, and instead use the default value +-- we don't set citus.large_table_shard_count here, and instead use the default value -- coming from postgresql.conf or multi_task_tracker_executor.conf. SELECT diff --git a/src/test/regress/sql/multi_tpch_query6.sql b/src/test/regress/sql/multi_tpch_query6.sql index 7a31c47fe..994b9467b 100644 --- a/src/test/regress/sql/multi_tpch_query6.sql +++ b/src/test/regress/sql/multi_tpch_query6.sql @@ -4,7 +4,7 @@ -- Change configuration to treat lineitem and orders tables as large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; -- Query #6 from the TPC-H decision support benchmark diff --git a/src/test/regress/sql/multi_tpch_query7.sql b/src/test/regress/sql/multi_tpch_query7.sql index 875c1e86a..b9b3c5cd3 100644 --- a/src/test/regress/sql/multi_tpch_query7.sql +++ b/src/test/regress/sql/multi_tpch_query7.sql @@ -4,7 +4,7 @@ -- Change configuration to treat lineitem AND orders tables as large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; -- Query #7 from the TPC-H decision support benchmark diff --git a/src/test/regress/sql/multi_tpch_query7_nested.sql b/src/test/regress/sql/multi_tpch_query7_nested.sql index fd93a17c9..ad34dd7b5 100644 --- a/src/test/regress/sql/multi_tpch_query7_nested.sql +++ b/src/test/regress/sql/multi_tpch_query7_nested.sql @@ -4,7 +4,7 @@ -- Change configuration to treat lineitem AND orders tables AS large -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; -- Query #7 from the TPC-H benchmark; modified to include sub-selects diff --git a/src/test/regress/sql/multi_upsert.sql b/src/test/regress/sql/multi_upsert.sql index 3e47862c6..b61833eaf 100644 --- a/src/test/regress/sql/multi_upsert.sql +++ b/src/test/regress/sql/multi_upsert.sql @@ -1,4 +1,4 @@ --- this test file aims to test UPSERT feature on CitusDB +-- this test file aims to test UPSERT feature on Citus -- note that output of this file for postgresql 9.4 will -- be full syntax errors, which is expected. @@ -93,7 +93,7 @@ CREATE TABLE upsert_test_2 SELECT master_create_distributed_table('upsert_test_2', 'part_key', 'hash'); SELECT master_create_worker_shards('upsert_test_2', '4', '2'); --- now show that CitusDB works with multiple columns as the PRIMARY KEY, including the partiton key +-- now show that Citus works with multiple columns as the PRIMARY KEY, including the partiton key INSERT INTO upsert_test_2 (part_key, other_col) VALUES (1, 1); INSERT INTO upsert_test_2 (part_key, other_col) VALUES (1, 1) ON CONFLICT (part_key, other_col) DO NOTHING; @@ -165,7 +165,7 @@ ALTER TABLE dropcol_distributed DROP COLUMN drop1; INSERT INTO dropcol_distributed AS dropcol (key, keep1) VALUES (1, '5') ON CONFLICT(key) DO UPDATE SET keep1 = dropcol.keep1; --- below we test the cases that CitusDB does not support +-- below we test the cases that Citus does not support -- subquery in the SET clause INSERT INTO upsert_test (part_key, other_col) VALUES (1, 1) ON CONFLICT (part_key) DO UPDATE SET other_col = (SELECT count(*) from upsert_test); diff --git a/src/test/regress/sql/multi_utility_statements.sql b/src/test/regress/sql/multi_utility_statements.sql index ddf0b37a5..ad67f2b36 100644 --- a/src/test/regress/sql/multi_utility_statements.sql +++ b/src/test/regress/sql/multi_utility_statements.sql @@ -35,7 +35,7 @@ SELECT * FROM lineitem_pricing_summary ORDER BY l_returnflag, l_linestatus; -- Test we can handle joins -SET citusdb.large_table_shard_count TO 2; +SET citus.large_table_shard_count TO 2; CREATE TABLE shipping_priority AS ( From 166f96bb836a2c6e30f636bd7e811cc6fb33aede Mon Sep 17 00:00:00 2001 From: Jason Petersen Date: Fri, 5 Feb 2016 13:18:54 -0700 Subject: [PATCH 06/19] First formatting attempt Skipped csql, ruleutils, readfuncs, and functions obviously copied from PostgreSQL. Seeing how this looks, then continuing. --- .../commands/create_distributed_table.c | 10 +- src/backend/distributed/commands/transmit.c | 28 +- .../executor/multi_client_executor.c | 8 +- .../distributed/executor/multi_executor.c | 3 +- .../executor/multi_real_time_executor.c | 619 ++++++------ .../executor/multi_router_executor.c | 18 +- .../executor/multi_server_executor.c | 6 +- .../executor/multi_task_tracker_executor.c | 883 +++++++++--------- .../distributed/executor/multi_utility.c | 21 +- .../distributed/master/master_create_shards.c | 2 +- .../master/master_delete_protocol.c | 45 +- .../master/master_metadata_utility.c | 2 +- .../distributed/master/master_node_protocol.c | 35 +- .../master/master_stage_protocol.c | 10 +- .../distributed/planner/modify_planner.c | 8 +- .../distributed/planner/multi_explain.c | 2 +- .../distributed/planner/multi_join_order.c | 73 +- .../planner/multi_logical_optimizer.c | 83 +- .../planner/multi_logical_planner.c | 52 +- .../planner/multi_master_planner.c | 18 +- .../planner/multi_physical_planner.c | 86 +- .../distributed/relay/relay_event_utility.c | 51 +- src/backend/distributed/shared_library_init.c | 45 +- src/backend/distributed/test/fake_fdw.c | 8 +- .../distributed/utils/citus_nodefuncs.c | 7 +- .../distributed/utils/citus_ruleutils.c | 18 +- .../distributed/utils/metadata_cache.c | 48 +- .../distributed/utils/multi_resowner.c | 26 +- src/backend/distributed/utils/resource_lock.c | 10 +- src/backend/distributed/worker/task_tracker.c | 44 +- .../worker/task_tracker_protocol.c | 8 +- .../worker/worker_data_fetch_protocol.c | 25 +- .../worker/worker_file_access_protocol.c | 24 +- .../worker/worker_merge_protocol.c | 12 +- .../worker/worker_partition_protocol.c | 32 +- src/include/distributed/citus_ruleutils.h | 19 +- .../distributed/master_metadata_utility.h | 14 +- src/include/distributed/master_protocol.h | 13 +- src/include/distributed/modify_planner.h | 1 + .../distributed/multi_client_executor.h | 33 +- src/include/distributed/multi_executor.h | 6 +- src/include/distributed/multi_join_order.h | 15 +- .../distributed/multi_logical_optimizer.h | 11 +- .../distributed/multi_logical_planner.h | 13 +- .../distributed/multi_physical_planner.h | 40 +- src/include/distributed/multi_planner.h | 4 +- .../distributed/multi_server_executor.h | 25 +- src/include/distributed/pg_dist_partition.h | 22 +- src/include/distributed/pg_dist_shard.h | 36 +- .../distributed/pg_dist_shard_placement.h | 24 +- src/include/distributed/relay_utility.h | 3 +- src/include/distributed/resource_lock.h | 1 + src/include/distributed/task_tracker.h | 27 +- src/include/distributed/worker_manager.h | 5 +- src/include/distributed/worker_protocol.h | 23 +- 55 files changed, 1391 insertions(+), 1314 deletions(-) diff --git a/src/backend/distributed/commands/create_distributed_table.c b/src/backend/distributed/commands/create_distributed_table.c index b4a4c802b..691981485 100644 --- a/src/backend/distributed/commands/create_distributed_table.c +++ b/src/backend/distributed/commands/create_distributed_table.c @@ -197,11 +197,11 @@ master_create_distributed_table(PG_FUNCTION_ARGS) if (distributionMethod == DISTRIBUTE_BY_APPEND) { ereport(WARNING, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("table \"%s\" has a unique constraint", - distributedRelationName), - errdetail("Unique constraints and primary keys on " - "append-partitioned tables cannot be enforced."), - errhint("Consider using hash partitioning."))); + errmsg("table \"%s\" has a unique constraint", + distributedRelationName), + errdetail("Unique constraints and primary keys on " + "append-partitioned tables cannot be enforced."), + errhint("Consider using hash partitioning."))); } attributeCount = indexInfo->ii_NumIndexAttrs; diff --git a/src/backend/distributed/commands/transmit.c b/src/backend/distributed/commands/transmit.c index 0ab90d0ef..6e0d78136 100644 --- a/src/backend/distributed/commands/transmit.c +++ b/src/backend/distributed/commands/transmit.c @@ -136,7 +136,7 @@ static File FileOpenForTransmit(const char *filename, int fileFlags, int fileMode) { File fileDesc = -1; - int fileStated = -1; + int fileStated = -1; struct stat fileStat; fileStated = stat(filename, &fileStat); @@ -145,7 +145,7 @@ FileOpenForTransmit(const char *filename, int fileFlags, int fileMode) if (S_ISDIR(fileStat.st_mode)) { ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("\"%s\" is a directory", filename))); + errmsg("\"%s\" is a directory", filename))); } } @@ -270,18 +270,28 @@ ReceiveCopyData(StringInfo copyData) switch (messageType) { - case 'd': /* CopyData */ + case 'd': /* CopyData */ + { copyDone = false; break; - case 'c': /* CopyDone */ + } + + case 'c': /* CopyDone */ + { copyDone = true; break; - case 'f': /* CopyFail */ + } + + case 'f': /* CopyFail */ + { ereport(ERROR, (errcode(ERRCODE_QUERY_CANCELED), errmsg("COPY data failed: %s", pq_getmsgstring(copyData)))); break; - case 'H': /* Flush */ - case 'S': /* Sync */ + } + + case 'H': /* Flush */ + case 'S': /* Sync */ + { /* * Ignore Flush/Sync for the convenience of client libraries (such * as libpq) that may send those without noticing that the command @@ -289,11 +299,15 @@ ReceiveCopyData(StringInfo copyData) */ copyDone = false; break; + } + default: + { ereport(ERROR, (errcode(ERRCODE_PROTOCOL_VIOLATION), errmsg("unexpected message type 0x%02X during COPY data", messageType))); break; + } } return copyDone; diff --git a/src/backend/distributed/executor/multi_client_executor.c b/src/backend/distributed/executor/multi_client_executor.c index e6b4ceef7..57ab6c1b9 100644 --- a/src/backend/distributed/executor/multi_client_executor.c +++ b/src/backend/distributed/executor/multi_client_executor.c @@ -301,7 +301,7 @@ MultiClientCancel(int32 connectionId) if (cancelSent == 0) { ereport(WARNING, (errmsg("could not issue cancel request"), - errdetail("Client error: %s", errorBuffer))); + errdetail("Client error: %s", errorBuffer))); canceled = false; } @@ -348,7 +348,7 @@ MultiClientResultStatus(int32 connectionId) } else { - ereport(WARNING, (errmsg("could not consume data from worker node"))); + ereport(WARNING, (errmsg("could not consume data from worker node"))); resultStatus = CLIENT_RESULT_UNAVAILABLE; } @@ -589,7 +589,7 @@ MultiClientCopyData(int32 connectionId, int32 fileDescriptor) while (receiveLength > 0) { /* received copy data; append these data to file */ - int appended = -1; + int appended = -1; errno = 0; appended = write(fileDescriptor, receiveBuffer, receiveLength); @@ -706,7 +706,7 @@ ClientConnectionReady(PGconn *connection, PostgresPollingStatusType pollingStatu fd_set readFileDescriptorSet; fd_set writeFileDescriptorSet; fd_set exceptionFileDescriptorSet; - struct timeval immediateTimeout = {0, 0}; + struct timeval immediateTimeout = { 0, 0 }; int connectionFileDescriptor = PQsocket(connection); FD_ZERO(&readFileDescriptorSet); diff --git a/src/backend/distributed/executor/multi_executor.c b/src/backend/distributed/executor/multi_executor.c index 25fd12640..145abb4d5 100644 --- a/src/backend/distributed/executor/multi_executor.c +++ b/src/backend/distributed/executor/multi_executor.c @@ -157,7 +157,6 @@ multi_ExecutorStart(QueryDesc *queryDesc, int eflags) queryDesc->plannedstmt = masterSelectPlan; eflags |= EXEC_FLAG_CITUS_MASTER_SELECT; } - } /* if the execution is not done for router executor, drop into standard executor */ @@ -253,7 +252,7 @@ multi_ExecutorEnd(QueryDesc *queryDesc) RangeTblEntry *rangeTableEntry = linitial(planStatement->rtable); Oid masterTableRelid = rangeTableEntry->relid; - ObjectAddress masterTableObject = {InvalidOid, InvalidOid, 0}; + ObjectAddress masterTableObject = { InvalidOid, InvalidOid, 0 }; masterTableObject.classId = RelationRelationId; masterTableObject.objectId = masterTableRelid; diff --git a/src/backend/distributed/executor/multi_real_time_executor.c b/src/backend/distributed/executor/multi_real_time_executor.c index 77436612c..e3050f64b 100644 --- a/src/backend/distributed/executor/multi_real_time_executor.c +++ b/src/backend/distributed/executor/multi_real_time_executor.c @@ -89,7 +89,7 @@ MultiRealTimeExecute(Job *job) } /* loop around until all tasks complete, one task fails, or user cancels */ - while ( !(allTasksCompleted || taskFailed || QueryCancelPending) ) + while (!(allTasksCompleted || taskFailed || QueryCancelPending)) { uint32 taskCount = list_length(taskList); uint32 completedTaskCount = 0; @@ -230,333 +230,338 @@ ManageTaskExecution(Task *task, TaskExecution *taskExecution) switch (currentStatus) { - case EXEC_TASK_CONNECT_START: - { - int32 connectionId = INVALID_CONNECTION_ID; - char *nodeDatabase = NULL; - - /* we use the same database name on the master and worker nodes */ - nodeDatabase = get_database_name(MyDatabaseId); - - connectionId = MultiClientConnectStart(nodeName, nodePort, nodeDatabase); - connectionIdArray[currentIndex] = connectionId; - - /* if valid, poll the connection until the connection is initiated */ - if (connectionId != INVALID_CONNECTION_ID) + case EXEC_TASK_CONNECT_START: { - taskStatusArray[currentIndex] = EXEC_TASK_CONNECT_POLL; - taskExecution->connectPollCount = 0; - connectAction = CONNECT_ACTION_OPENED; - } - else - { - AdjustStateForFailure(taskExecution); - } + int32 connectionId = INVALID_CONNECTION_ID; + char *nodeDatabase = NULL; - break; - } + /* we use the same database name on the master and worker nodes */ + nodeDatabase = get_database_name(MyDatabaseId); - case EXEC_TASK_CONNECT_POLL: - { - int32 connectionId = connectionIdArray[currentIndex]; - ConnectStatus pollStatus = MultiClientConnectPoll(connectionId); + connectionId = MultiClientConnectStart(nodeName, nodePort, nodeDatabase); + connectionIdArray[currentIndex] = connectionId; - /* - * If the connection is established, we reset the data fetch counter and - * change our status to data fetching. - */ - if (pollStatus == CLIENT_CONNECTION_READY) - { - taskExecution->dataFetchTaskIndex = -1; - taskStatusArray[currentIndex] = EXEC_FETCH_TASK_LOOP; - } - else if (pollStatus == CLIENT_CONNECTION_BUSY) - { - taskStatusArray[currentIndex] = EXEC_TASK_CONNECT_POLL; - } - else if (pollStatus == CLIENT_CONNECTION_BAD) - { - taskStatusArray[currentIndex] = EXEC_TASK_FAILED; - } - - /* now check if we have been trying to connect for too long */ - taskExecution->connectPollCount++; - if (pollStatus == CLIENT_CONNECTION_BUSY) - { - uint32 maxCount = REMOTE_NODE_CONNECT_TIMEOUT / RemoteTaskCheckInterval; - uint32 currentCount = taskExecution->connectPollCount; - if (currentCount >= maxCount) + /* if valid, poll the connection until the connection is initiated */ + if (connectionId != INVALID_CONNECTION_ID) { - ereport(WARNING, (errmsg("could not establish asynchronous connection " - "after %u ms", REMOTE_NODE_CONNECT_TIMEOUT))); + taskStatusArray[currentIndex] = EXEC_TASK_CONNECT_POLL; + taskExecution->connectPollCount = 0; + connectAction = CONNECT_ACTION_OPENED; + } + else + { + AdjustStateForFailure(taskExecution); + } + break; + } + + case EXEC_TASK_CONNECT_POLL: + { + int32 connectionId = connectionIdArray[currentIndex]; + ConnectStatus pollStatus = MultiClientConnectPoll(connectionId); + + /* + * If the connection is established, we reset the data fetch counter and + * change our status to data fetching. + */ + if (pollStatus == CLIENT_CONNECTION_READY) + { + taskExecution->dataFetchTaskIndex = -1; + taskStatusArray[currentIndex] = EXEC_FETCH_TASK_LOOP; + } + else if (pollStatus == CLIENT_CONNECTION_BUSY) + { + taskStatusArray[currentIndex] = EXEC_TASK_CONNECT_POLL; + } + else if (pollStatus == CLIENT_CONNECTION_BAD) + { taskStatusArray[currentIndex] = EXEC_TASK_FAILED; } - } - break; - } - - case EXEC_TASK_FAILED: - { - /* - * On task failure, we close the connection. We also reset our execution - * status assuming that we might fail on all other worker nodes and come - * back to this failed node. In that case, we will retry the same fetch - * and compute task(s) on this node again. - */ - int32 connectionId = connectionIdArray[currentIndex]; - MultiClientDisconnect(connectionId); - connectionIdArray[currentIndex] = INVALID_CONNECTION_ID; - connectAction = CONNECT_ACTION_CLOSED; - - taskStatusArray[currentIndex] = EXEC_TASK_CONNECT_START; - - /* try next worker node */ - AdjustStateForFailure(taskExecution); - - break; - } - - case EXEC_FETCH_TASK_LOOP: - { - List *dataFetchTaskList = task->dependedTaskList; - int32 dataFetchTaskCount = list_length(dataFetchTaskList); - - /* move to the next data fetch task */ - taskExecution->dataFetchTaskIndex++; - - if (taskExecution->dataFetchTaskIndex < dataFetchTaskCount) - { - taskStatusArray[currentIndex] = EXEC_FETCH_TASK_START; - } - else - { - taskStatusArray[currentIndex] = EXEC_COMPUTE_TASK_START; - } - - break; - } - - case EXEC_FETCH_TASK_START: - { - List *dataFetchTaskList = task->dependedTaskList; - int32 dataFetchTaskIndex = taskExecution->dataFetchTaskIndex; - Task *dataFetchTask = (Task *) list_nth(dataFetchTaskList, dataFetchTaskIndex); - - char *dataFetchQuery = dataFetchTask->queryString; - int32 connectionId = connectionIdArray[currentIndex]; - - bool querySent = MultiClientSendQuery(connectionId, dataFetchQuery); - if (querySent) - { - taskStatusArray[currentIndex] = EXEC_FETCH_TASK_RUNNING; - } - else - { - taskStatusArray[currentIndex] = EXEC_TASK_FAILED; - } - - break; - } - - case EXEC_FETCH_TASK_RUNNING: - { - int32 connectionId = connectionIdArray[currentIndex]; - ResultStatus resultStatus = MultiClientResultStatus(connectionId); - QueryStatus queryStatus = CLIENT_INVALID_QUERY; - - /* check if query results are in progress or unavailable */ - if (resultStatus == CLIENT_RESULT_BUSY) - { - taskStatusArray[currentIndex] = EXEC_FETCH_TASK_RUNNING; - break; - } - else if (resultStatus == CLIENT_RESULT_UNAVAILABLE) - { - taskStatusArray[currentIndex] = EXEC_TASK_FAILED; - break; - } - - Assert(resultStatus == CLIENT_RESULT_READY); - - /* - * If the query executed successfully, loop onto the next data fetch - * task. Else if the query failed, try data fetching on another node. - */ - queryStatus = MultiClientQueryStatus(connectionId); - if (queryStatus == CLIENT_QUERY_DONE) - { - taskStatusArray[currentIndex] = EXEC_FETCH_TASK_LOOP; - } - else if (queryStatus == CLIENT_QUERY_FAILED) - { - taskStatusArray[currentIndex] = EXEC_TASK_FAILED; - } - else - { - ereport(FATAL, (errmsg("invalid query status: %d", queryStatus))); - } - - break; - } - - case EXEC_COMPUTE_TASK_START: - { - int32 connectionId = connectionIdArray[currentIndex]; - bool querySent = false; - - /* construct new query to copy query results to stdout */ - char *queryString = task->queryString; - StringInfo computeTaskQuery = makeStringInfo(); - if (BinaryMasterCopyFormat) - { - appendStringInfo(computeTaskQuery, COPY_QUERY_TO_STDOUT_BINARY, queryString); - } - else - { - appendStringInfo(computeTaskQuery, COPY_QUERY_TO_STDOUT_TEXT, queryString); - } - - querySent = MultiClientSendQuery(connectionId, computeTaskQuery->data); - if (querySent) - { - taskStatusArray[currentIndex] = EXEC_COMPUTE_TASK_RUNNING; - } - else - { - taskStatusArray[currentIndex] = EXEC_TASK_FAILED; - } - - break; - } - - case EXEC_COMPUTE_TASK_RUNNING: - { - int32 connectionId = connectionIdArray[currentIndex]; - ResultStatus resultStatus = MultiClientResultStatus(connectionId); - QueryStatus queryStatus = CLIENT_INVALID_QUERY; - - /* check if query results are in progress or unavailable */ - if (resultStatus == CLIENT_RESULT_BUSY) - { - taskStatusArray[currentIndex] = EXEC_COMPUTE_TASK_RUNNING; - break; - } - else if (resultStatus == CLIENT_RESULT_UNAVAILABLE) - { - taskStatusArray[currentIndex] = EXEC_TASK_FAILED; - break; - } - - Assert(resultStatus == CLIENT_RESULT_READY); - - /* check if our request to copy query results has been acknowledged */ - queryStatus = MultiClientQueryStatus(connectionId); - if (queryStatus == CLIENT_QUERY_COPY) - { - StringInfo jobDirectoryName = JobDirectoryName(task->jobId); - StringInfo taskFilename = TaskFilename(jobDirectoryName, task->taskId); - - char *filename = taskFilename->data; - int fileFlags = (O_APPEND | O_CREAT | O_RDWR | O_TRUNC | PG_BINARY); - int fileMode = (S_IRUSR | S_IWUSR); - - int32 fileDescriptor = BasicOpenFile(filename, fileFlags, fileMode); - if (fileDescriptor >= 0) + /* now check if we have been trying to connect for too long */ + taskExecution->connectPollCount++; + if (pollStatus == CLIENT_CONNECTION_BUSY) + { + uint32 maxCount = REMOTE_NODE_CONNECT_TIMEOUT / RemoteTaskCheckInterval; + uint32 currentCount = taskExecution->connectPollCount; + if (currentCount >= maxCount) + { + ereport(WARNING, (errmsg("could not establish asynchronous " + "connection after %u ms", + REMOTE_NODE_CONNECT_TIMEOUT))); + + taskStatusArray[currentIndex] = EXEC_TASK_FAILED; + } + } + + break; + } + + case EXEC_TASK_FAILED: + { + /* + * On task failure, we close the connection. We also reset our execution + * status assuming that we might fail on all other worker nodes and come + * back to this failed node. In that case, we will retry the same fetch + * and compute task(s) on this node again. + */ + int32 connectionId = connectionIdArray[currentIndex]; + MultiClientDisconnect(connectionId); + connectionIdArray[currentIndex] = INVALID_CONNECTION_ID; + connectAction = CONNECT_ACTION_CLOSED; + + taskStatusArray[currentIndex] = EXEC_TASK_CONNECT_START; + + /* try next worker node */ + AdjustStateForFailure(taskExecution); + + break; + } + + case EXEC_FETCH_TASK_LOOP: + { + List *dataFetchTaskList = task->dependedTaskList; + int32 dataFetchTaskCount = list_length(dataFetchTaskList); + + /* move to the next data fetch task */ + taskExecution->dataFetchTaskIndex++; + + if (taskExecution->dataFetchTaskIndex < dataFetchTaskCount) + { + taskStatusArray[currentIndex] = EXEC_FETCH_TASK_START; + } + else + { + taskStatusArray[currentIndex] = EXEC_COMPUTE_TASK_START; + } + + break; + } + + case EXEC_FETCH_TASK_START: + { + List *dataFetchTaskList = task->dependedTaskList; + int32 dataFetchTaskIndex = taskExecution->dataFetchTaskIndex; + Task *dataFetchTask = (Task *) list_nth(dataFetchTaskList, + dataFetchTaskIndex); + + char *dataFetchQuery = dataFetchTask->queryString; + int32 connectionId = connectionIdArray[currentIndex]; + + bool querySent = MultiClientSendQuery(connectionId, dataFetchQuery); + if (querySent) + { + taskStatusArray[currentIndex] = EXEC_FETCH_TASK_RUNNING; + } + else + { + taskStatusArray[currentIndex] = EXEC_TASK_FAILED; + } + + break; + } + + case EXEC_FETCH_TASK_RUNNING: + { + int32 connectionId = connectionIdArray[currentIndex]; + ResultStatus resultStatus = MultiClientResultStatus(connectionId); + QueryStatus queryStatus = CLIENT_INVALID_QUERY; + + /* check if query results are in progress or unavailable */ + if (resultStatus == CLIENT_RESULT_BUSY) + { + taskStatusArray[currentIndex] = EXEC_FETCH_TASK_RUNNING; + break; + } + else if (resultStatus == CLIENT_RESULT_UNAVAILABLE) + { + taskStatusArray[currentIndex] = EXEC_TASK_FAILED; + break; + } + + Assert(resultStatus == CLIENT_RESULT_READY); + + /* + * If the query executed successfully, loop onto the next data fetch + * task. Else if the query failed, try data fetching on another node. + */ + queryStatus = MultiClientQueryStatus(connectionId); + if (queryStatus == CLIENT_QUERY_DONE) + { + taskStatusArray[currentIndex] = EXEC_FETCH_TASK_LOOP; + } + else if (queryStatus == CLIENT_QUERY_FAILED) + { + taskStatusArray[currentIndex] = EXEC_TASK_FAILED; + } + else + { + ereport(FATAL, (errmsg("invalid query status: %d", queryStatus))); + } + + break; + } + + case EXEC_COMPUTE_TASK_START: + { + int32 connectionId = connectionIdArray[currentIndex]; + bool querySent = false; + + /* construct new query to copy query results to stdout */ + char *queryString = task->queryString; + StringInfo computeTaskQuery = makeStringInfo(); + if (BinaryMasterCopyFormat) + { + appendStringInfo(computeTaskQuery, COPY_QUERY_TO_STDOUT_BINARY, + queryString); + } + else + { + appendStringInfo(computeTaskQuery, COPY_QUERY_TO_STDOUT_TEXT, + queryString); + } + + querySent = MultiClientSendQuery(connectionId, computeTaskQuery->data); + if (querySent) + { + taskStatusArray[currentIndex] = EXEC_COMPUTE_TASK_RUNNING; + } + else + { + taskStatusArray[currentIndex] = EXEC_TASK_FAILED; + } + + break; + } + + case EXEC_COMPUTE_TASK_RUNNING: + { + int32 connectionId = connectionIdArray[currentIndex]; + ResultStatus resultStatus = MultiClientResultStatus(connectionId); + QueryStatus queryStatus = CLIENT_INVALID_QUERY; + + /* check if query results are in progress or unavailable */ + if (resultStatus == CLIENT_RESULT_BUSY) + { + taskStatusArray[currentIndex] = EXEC_COMPUTE_TASK_RUNNING; + break; + } + else if (resultStatus == CLIENT_RESULT_UNAVAILABLE) + { + taskStatusArray[currentIndex] = EXEC_TASK_FAILED; + break; + } + + Assert(resultStatus == CLIENT_RESULT_READY); + + /* check if our request to copy query results has been acknowledged */ + queryStatus = MultiClientQueryStatus(connectionId); + if (queryStatus == CLIENT_QUERY_COPY) + { + StringInfo jobDirectoryName = JobDirectoryName(task->jobId); + StringInfo taskFilename = TaskFilename(jobDirectoryName, task->taskId); + + char *filename = taskFilename->data; + int fileFlags = (O_APPEND | O_CREAT | O_RDWR | O_TRUNC | PG_BINARY); + int fileMode = (S_IRUSR | S_IWUSR); + + int32 fileDescriptor = BasicOpenFile(filename, fileFlags, fileMode); + if (fileDescriptor >= 0) + { + /* + * All files inside the job directory get automatically cleaned + * up on transaction commit or abort. + */ + fileDescriptorArray[currentIndex] = fileDescriptor; + taskStatusArray[currentIndex] = EXEC_COMPUTE_TASK_COPYING; + } + else + { + ereport(WARNING, (errcode_for_file_access(), + errmsg("could not open file \"%s\": %m", + filename))); + + taskStatusArray[currentIndex] = EXEC_TASK_FAILED; + } + } + else if (queryStatus == CLIENT_QUERY_FAILED) + { + taskStatusArray[currentIndex] = EXEC_TASK_FAILED; + } + else + { + ereport(FATAL, (errmsg("invalid query status: %d", queryStatus))); + } + + break; + } + + case EXEC_COMPUTE_TASK_COPYING: + { + int32 connectionId = connectionIdArray[currentIndex]; + int32 fileDesc = fileDescriptorArray[currentIndex]; + int closed = -1; + + /* copy data from worker node, and write to local file */ + CopyStatus copyStatus = MultiClientCopyData(connectionId, fileDesc); + + /* if worker node will continue to send more data, keep reading */ + if (copyStatus == CLIENT_COPY_MORE) { - /* - * All files inside the job directory get automatically cleaned - * up on transaction commit or abort. - */ - fileDescriptorArray[currentIndex] = fileDescriptor; taskStatusArray[currentIndex] = EXEC_COMPUTE_TASK_COPYING; } - else + else if (copyStatus == CLIENT_COPY_DONE) { - ereport(WARNING, (errcode_for_file_access(), - errmsg("could not open file \"%s\": %m", filename))); + closed = close(fileDesc); + fileDescriptorArray[currentIndex] = -1; + if (closed >= 0) + { + taskStatusArray[currentIndex] = EXEC_TASK_DONE; + + /* we are done executing; we no longer need the connection */ + MultiClientDisconnect(connectionId); + connectionIdArray[currentIndex] = INVALID_CONNECTION_ID; + connectAction = CONNECT_ACTION_CLOSED; + } + else + { + ereport(WARNING, (errcode_for_file_access(), + errmsg("could not close copied file: %m"))); + + taskStatusArray[currentIndex] = EXEC_TASK_FAILED; + } + } + else if (copyStatus == CLIENT_COPY_FAILED) + { taskStatusArray[currentIndex] = EXEC_TASK_FAILED; + + closed = close(fileDesc); + fileDescriptorArray[currentIndex] = -1; + + if (closed < 0) + { + ereport(WARNING, (errcode_for_file_access(), + errmsg("could not close copy file: %m"))); + } } + + break; } - else if (queryStatus == CLIENT_QUERY_FAILED) + + case EXEC_TASK_DONE: { - taskStatusArray[currentIndex] = EXEC_TASK_FAILED; + /* we are done with this task's execution */ + break; } - else + + default: { - ereport(FATAL, (errmsg("invalid query status: %d", queryStatus))); + /* we fatal here to avoid leaking client-side resources */ + ereport(FATAL, (errmsg("invalid execution status: %d", currentStatus))); + break; } - - break; - } - - case EXEC_COMPUTE_TASK_COPYING: - { - int32 connectionId = connectionIdArray[currentIndex]; - int32 fileDesc = fileDescriptorArray[currentIndex]; - int closed = -1; - - /* copy data from worker node, and write to local file */ - CopyStatus copyStatus = MultiClientCopyData(connectionId, fileDesc); - - /* if worker node will continue to send more data, keep reading */ - if (copyStatus == CLIENT_COPY_MORE) - { - taskStatusArray[currentIndex] = EXEC_COMPUTE_TASK_COPYING; - } - else if (copyStatus == CLIENT_COPY_DONE) - { - closed = close(fileDesc); - fileDescriptorArray[currentIndex] = -1; - - if (closed >= 0) - { - taskStatusArray[currentIndex] = EXEC_TASK_DONE; - - /* we are done executing; we no longer need the connection */ - MultiClientDisconnect(connectionId); - connectionIdArray[currentIndex] = INVALID_CONNECTION_ID; - connectAction = CONNECT_ACTION_CLOSED; - } - else - { - ereport(WARNING, (errcode_for_file_access(), - errmsg("could not close copied file: %m"))); - - taskStatusArray[currentIndex] = EXEC_TASK_FAILED; - } - } - else if (copyStatus == CLIENT_COPY_FAILED) - { - taskStatusArray[currentIndex] = EXEC_TASK_FAILED; - - closed = close(fileDesc); - fileDescriptorArray[currentIndex] = -1; - - if (closed < 0) - { - ereport(WARNING, (errcode_for_file_access(), - errmsg("could not close copy file: %m"))); - } - } - - break; - } - - case EXEC_TASK_DONE: - { - /* we are done with this task's execution */ - break; - } - - default: - { - /* we fatal here to avoid leaking client-side resources */ - ereport(FATAL, (errmsg("invalid execution status: %d", currentStatus))); - break; - } } return connectAction; diff --git a/src/backend/distributed/executor/multi_router_executor.c b/src/backend/distributed/executor/multi_router_executor.c index 5c2f04165..ae6eea97d 100644 --- a/src/backend/distributed/executor/multi_router_executor.c +++ b/src/backend/distributed/executor/multi_router_executor.c @@ -80,6 +80,7 @@ RouterExecutorStart(QueryDesc *queryDesc, int eflags, Task *task) queryDesc->estate = executorState; #if (PG_VERSION_NUM < 90500) + /* make sure that upsertQuery is false for versions that UPSERT is not available */ Assert(task->upsertQuery == false); #endif @@ -177,14 +178,14 @@ RouterExecutorRun(QueryDesc *queryDesc, ScanDirection direction, long count, Tas if (!ScanDirectionIsForward(direction)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("scan directions other than forward scans " - "are unsupported"))); + errmsg("scan directions other than forward scans " + "are unsupported"))); } if (count != 0) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("fetching rows from a query using a cursor " - "is unsupported"))); + errmsg("fetching rows from a query using a cursor " + "is unsupported"))); } oldcontext = MemoryContextSwitchTo(estate->es_query_cxt); @@ -210,7 +211,7 @@ RouterExecutorRun(QueryDesc *queryDesc, ScanDirection direction, long count, Tas else { ereport(ERROR, (errmsg("unrecognized operation code: %d", - (int) operation))); + (int) operation))); } if (queryDesc->totaltime != NULL) @@ -219,9 +220,9 @@ RouterExecutorRun(QueryDesc *queryDesc, ScanDirection direction, long count, Tas } MemoryContextSwitchTo(oldcontext); - } + /* * ExecuteDistributedModify is the main entry point for modifying distributed * tables. A distributed modification is successful if any placement of the @@ -532,9 +533,10 @@ StoreQueryResult(PGconn *connection, TupleDesc tupleDescriptor, return true; } + /* -* RouterExecutorFinish cleans up after a distributed execution. -*/ + * RouterExecutorFinish cleans up after a distributed execution. + */ void RouterExecutorFinish(QueryDesc *queryDesc) { diff --git a/src/backend/distributed/executor/multi_server_executor.c b/src/backend/distributed/executor/multi_server_executor.c index 1f143778d..1abc6f007 100644 --- a/src/backend/distributed/executor/multi_server_executor.c +++ b/src/backend/distributed/executor/multi_server_executor.c @@ -303,13 +303,13 @@ AdjustStateForFailure(TaskExecution *taskExecution) if (taskExecution->currentNodeIndex < maxNodeIndex) { - taskExecution->currentNodeIndex++; /* try next worker node */ + taskExecution->currentNodeIndex++; /* try next worker node */ } else { taskExecution->currentNodeIndex = 0; /* go back to the first worker node */ } - taskExecution->dataFetchTaskIndex = -1; /* reset data fetch counter */ - taskExecution->failureCount++; /* record failure */ + taskExecution->dataFetchTaskIndex = -1; /* reset data fetch counter */ + taskExecution->failureCount++; /* record failure */ } diff --git a/src/backend/distributed/executor/multi_task_tracker_executor.c b/src/backend/distributed/executor/multi_task_tracker_executor.c index f67c82271..14c26aef9 100644 --- a/src/backend/distributed/executor/multi_task_tracker_executor.c +++ b/src/backend/distributed/executor/multi_task_tracker_executor.c @@ -38,10 +38,9 @@ int MaxAssignTaskBatchSize = 64; /* maximum number of tasks to assign per round /* TaskMapKey is used as a key in task hash */ typedef struct TaskMapKey { - TaskType taskType; - uint64 jobId; - uint32 taskId; - + TaskType taskType; + uint64 jobId; + uint32 taskId; } TaskMapKey; @@ -51,9 +50,8 @@ typedef struct TaskMapKey */ typedef struct TaskMapEntry { - TaskMapKey key; - Task *task; - + TaskMapKey key; + Task *task; } TaskMapEntry; @@ -83,7 +81,8 @@ static TaskTracker * TrackerHashLookup(HTAB *trackerHash, const char *nodeName, static TaskExecStatus ManageTaskExecution(TaskTracker *taskTracker, TaskTracker *sourceTaskTracker, Task *task, TaskExecution *taskExecution); -static TransmitExecStatus ManageTransmitExecution(TaskTracker *transmitTracker, Task *task, +static TransmitExecStatus ManageTransmitExecution(TaskTracker *transmitTracker, + Task *task, TaskExecution *taskExecution); static bool TaskExecutionsCompleted(List *taskList); static StringInfo MapFetchTaskQueryString(Task *mapFetchTask, Task *mapTask); @@ -194,8 +193,8 @@ MultiTaskTrackerExecute(Job *job) TrackerHashConnect(transmitTrackerHash); /* loop around until all tasks complete, one task fails, or user cancels */ - while ( !(allTasksCompleted || taskFailed || taskTransmitFailed || - clusterFailed || QueryCancelPending) ) + while (!(allTasksCompleted || taskFailed || taskTransmitFailed || + clusterFailed || QueryCancelPending)) { TaskTracker *taskTracker = NULL; TaskTracker *transmitTracker = NULL; @@ -493,8 +492,8 @@ TaskAndExecutionList(List *jobTaskList) */ if (!dependendTaskInHash) { - dependendTaskInHash = TaskHashEnter(taskHash, dependendTask); - taskQueue = lappend(taskQueue, dependendTaskInHash); + dependendTaskInHash = TaskHashEnter(taskHash, dependendTask); + taskQueue = lappend(taskQueue, dependendTaskInHash); } /* update dependedTaskList element to the one which is in the hash */ @@ -557,7 +556,7 @@ TaskHashEnter(HTAB *taskHash, Task *task) if (handleFound) { ereport(ERROR, (errmsg("multiple entries for task: \"%d:%ld:%d\"", - task->taskType, task->jobId, task->taskId))); + task->taskType, task->jobId, task->taskId))); } /* save the pointer to the original task in the hash */ @@ -820,82 +819,84 @@ TrackerConnectPoll(TaskTracker *taskTracker) { switch (taskTracker->trackerStatus) { - case TRACKER_CONNECT_START: - { - char *nodeName = taskTracker->workerName; - uint32 nodePort = taskTracker->workerPort; - char *nodeDatabase = get_database_name(MyDatabaseId); - - int32 connectionId = MultiClientConnectStart(nodeName, nodePort, nodeDatabase); - if (connectionId != INVALID_CONNECTION_ID) + case TRACKER_CONNECT_START: { - taskTracker->connectionId = connectionId; - taskTracker->trackerStatus = TRACKER_CONNECT_POLL; - } - else - { - taskTracker->trackerStatus = TRACKER_CONNECTION_FAILED; - } + char *nodeName = taskTracker->workerName; + uint32 nodePort = taskTracker->workerPort; + char *nodeDatabase = get_database_name(MyDatabaseId); - break; - } - - case TRACKER_CONNECT_POLL: - { - int32 connectionId = taskTracker->connectionId; - - ConnectStatus pollStatus = MultiClientConnectPoll(connectionId); - if (pollStatus == CLIENT_CONNECTION_READY) - { - taskTracker->trackerStatus = TRACKER_CONNECTED; - } - else if (pollStatus == CLIENT_CONNECTION_BUSY) - { - taskTracker->trackerStatus = TRACKER_CONNECT_POLL; - } - else if (pollStatus == CLIENT_CONNECTION_BAD) - { - taskTracker->trackerStatus = TRACKER_CONNECTION_FAILED; - - MultiClientDisconnect(connectionId); - taskTracker->connectionId = INVALID_CONNECTION_ID; - } - - /* now check if we have been trying to connect for too long */ - taskTracker->connectPollCount++; - if (pollStatus == CLIENT_CONNECTION_BUSY) - { - uint32 maxCount = REMOTE_NODE_CONNECT_TIMEOUT / RemoteTaskCheckInterval; - uint32 currentCount = taskTracker->connectPollCount; - if (currentCount >= maxCount) + int32 connectionId = MultiClientConnectStart(nodeName, nodePort, + nodeDatabase); + if (connectionId != INVALID_CONNECTION_ID) { - ereport(WARNING, (errmsg("could not establish asynchronous connection " - "after %u ms", REMOTE_NODE_CONNECT_TIMEOUT))); + taskTracker->connectionId = connectionId; + taskTracker->trackerStatus = TRACKER_CONNECT_POLL; + } + else + { + taskTracker->trackerStatus = TRACKER_CONNECTION_FAILED; + } + break; + } + + case TRACKER_CONNECT_POLL: + { + int32 connectionId = taskTracker->connectionId; + + ConnectStatus pollStatus = MultiClientConnectPoll(connectionId); + if (pollStatus == CLIENT_CONNECTION_READY) + { + taskTracker->trackerStatus = TRACKER_CONNECTED; + } + else if (pollStatus == CLIENT_CONNECTION_BUSY) + { + taskTracker->trackerStatus = TRACKER_CONNECT_POLL; + } + else if (pollStatus == CLIENT_CONNECTION_BAD) + { taskTracker->trackerStatus = TRACKER_CONNECTION_FAILED; MultiClientDisconnect(connectionId); taskTracker->connectionId = INVALID_CONNECTION_ID; } + + /* now check if we have been trying to connect for too long */ + taskTracker->connectPollCount++; + if (pollStatus == CLIENT_CONNECTION_BUSY) + { + uint32 maxCount = REMOTE_NODE_CONNECT_TIMEOUT / RemoteTaskCheckInterval; + uint32 currentCount = taskTracker->connectPollCount; + if (currentCount >= maxCount) + { + ereport(WARNING, (errmsg("could not establish asynchronous " + "connection after %u ms", + REMOTE_NODE_CONNECT_TIMEOUT))); + + taskTracker->trackerStatus = TRACKER_CONNECTION_FAILED; + + MultiClientDisconnect(connectionId); + taskTracker->connectionId = INVALID_CONNECTION_ID; + } + } + + break; } - break; - } + case TRACKER_CONNECTED: + case TRACKER_CONNECTION_FAILED: + { + /* if connected or failed to connect in previous pass, reset poll count */ + taskTracker->connectPollCount = 0; + break; + } - case TRACKER_CONNECTED: - case TRACKER_CONNECTION_FAILED: - { - /* if connected or failed to connect in previous pass, reset poll count */ - taskTracker->connectPollCount = 0; - break; - } - - default: - { - int trackerStatus = (int) taskTracker->trackerStatus; - ereport(FATAL, (errmsg("invalid task tracker status: %d", trackerStatus))); - break; - } + default: + { + int trackerStatus = (int) taskTracker->trackerStatus; + ereport(FATAL, (errmsg("invalid task tracker status: %d", trackerStatus))); + break; + } } return taskTracker->trackerStatus; @@ -1008,213 +1009,214 @@ ManageTaskExecution(TaskTracker *taskTracker, TaskTracker *sourceTaskTracker, switch (currentExecutionStatus) { - case EXEC_TASK_UNASSIGNED: - { - bool taskExecutionsCompleted = true; - TaskType taskType = TASK_TYPE_INVALID_FIRST; - - bool trackerHealthy = TrackerHealthy(taskTracker); - if (!trackerHealthy) + case EXEC_TASK_UNASSIGNED: { - nextExecutionStatus = EXEC_TASK_TRACKER_FAILED; - break; - } + bool taskExecutionsCompleted = true; + TaskType taskType = TASK_TYPE_INVALID_FIRST; - /* - * We first retrieve this task's downstream dependencies, and then check - * if these dependencies' executions have completed. - */ - taskExecutionsCompleted = TaskExecutionsCompleted(task->dependedTaskList); - if (!taskExecutionsCompleted) - { - nextExecutionStatus = EXEC_TASK_UNASSIGNED; - break; - } - - /* if map fetch task, create query string from completed map task */ - taskType = task->taskType; - if (taskType == MAP_OUTPUT_FETCH_TASK) - { - StringInfo mapFetchTaskQueryString = NULL; - Task *mapTask = (Task *) linitial(task->dependedTaskList); - TaskExecution *mapTaskExecution = mapTask->taskExecution; - - mapFetchTaskQueryString = MapFetchTaskQueryString(task, mapTask); - task->queryString = mapFetchTaskQueryString->data; - taskExecution->querySourceNodeIndex = mapTaskExecution->currentNodeIndex; - } - - /* - * We finally queue this task for execution. Note that we queue sql and - * other tasks slightly differently. - */ - if (taskType == SQL_TASK) - { - TrackerQueueSqlTask(taskTracker, task); - } - else - { - TrackerQueueTask(taskTracker, task); - } - - nextExecutionStatus = EXEC_TASK_QUEUED; - break; - } - - case EXEC_TASK_QUEUED: - { - TaskStatus remoteTaskStatus = TASK_STATUS_INVALID_FIRST; - - bool trackerHealthy = TrackerHealthy(taskTracker); - if (!trackerHealthy) - { - nextExecutionStatus = EXEC_TASK_TRACKER_FAILED; - break; - } - - remoteTaskStatus = TrackerTaskStatus(taskTracker, task); - if (remoteTaskStatus == TASK_SUCCEEDED) - { - nextExecutionStatus = EXEC_TASK_DONE; - } - else if (remoteTaskStatus == TASK_CLIENT_SIDE_ASSIGN_FAILED || - remoteTaskStatus == TASK_CLIENT_SIDE_STATUS_FAILED) - { - nextExecutionStatus = EXEC_TASK_TRACKER_RETRY; - } - else if (remoteTaskStatus == TASK_PERMANENTLY_FAILED) - { - /* - * If a map output fetch task failed, we assume the problem lies with - * the map task (and the source task tracker it runs on). Otherwise, - * we assume the task tracker crashed, and fail over to the next task - * tracker. - */ - if (task->taskType == MAP_OUTPUT_FETCH_TASK) + bool trackerHealthy = TrackerHealthy(taskTracker); + if (!trackerHealthy) { - nextExecutionStatus = EXEC_SOURCE_TASK_TRACKER_RETRY; + nextExecutionStatus = EXEC_TASK_TRACKER_FAILED; + break; + } + + /* + * We first retrieve this task's downstream dependencies, and then check + * if these dependencies' executions have completed. + */ + taskExecutionsCompleted = TaskExecutionsCompleted(task->dependedTaskList); + if (!taskExecutionsCompleted) + { + nextExecutionStatus = EXEC_TASK_UNASSIGNED; + break; + } + + /* if map fetch task, create query string from completed map task */ + taskType = task->taskType; + if (taskType == MAP_OUTPUT_FETCH_TASK) + { + StringInfo mapFetchTaskQueryString = NULL; + Task *mapTask = (Task *) linitial(task->dependedTaskList); + TaskExecution *mapTaskExecution = mapTask->taskExecution; + + mapFetchTaskQueryString = MapFetchTaskQueryString(task, mapTask); + task->queryString = mapFetchTaskQueryString->data; + taskExecution->querySourceNodeIndex = mapTaskExecution->currentNodeIndex; + } + + /* + * We finally queue this task for execution. Note that we queue sql and + * other tasks slightly differently. + */ + if (taskType == SQL_TASK) + { + TrackerQueueSqlTask(taskTracker, task); + } + else + { + TrackerQueueTask(taskTracker, task); + } + + nextExecutionStatus = EXEC_TASK_QUEUED; + break; + } + + case EXEC_TASK_QUEUED: + { + TaskStatus remoteTaskStatus = TASK_STATUS_INVALID_FIRST; + + bool trackerHealthy = TrackerHealthy(taskTracker); + if (!trackerHealthy) + { + nextExecutionStatus = EXEC_TASK_TRACKER_FAILED; + break; + } + + remoteTaskStatus = TrackerTaskStatus(taskTracker, task); + if (remoteTaskStatus == TASK_SUCCEEDED) + { + nextExecutionStatus = EXEC_TASK_DONE; + } + else if (remoteTaskStatus == TASK_CLIENT_SIDE_ASSIGN_FAILED || + remoteTaskStatus == TASK_CLIENT_SIDE_STATUS_FAILED) + { + nextExecutionStatus = EXEC_TASK_TRACKER_RETRY; + } + else if (remoteTaskStatus == TASK_PERMANENTLY_FAILED) + { + /* + * If a map output fetch task failed, we assume the problem lies with + * the map task (and the source task tracker it runs on). Otherwise, + * we assume the task tracker crashed, and fail over to the next task + * tracker. + */ + if (task->taskType == MAP_OUTPUT_FETCH_TASK) + { + nextExecutionStatus = EXEC_SOURCE_TASK_TRACKER_RETRY; + } + else + { + nextExecutionStatus = EXEC_TASK_TRACKER_FAILED; + } + } + else + { + /* assume task is still in progress */ + nextExecutionStatus = EXEC_TASK_QUEUED; + } + + break; + } + + case EXEC_TASK_TRACKER_RETRY: + { + bool trackerHealthy = false; + bool trackerConnectionUp = false; + + /* + * This case statement usually handles connection related issues. Some + * edge cases however, like a user sending a SIGTERM to the worker node, + * keep the connection open but disallow task assignments. We therefore + * need to track those as intermittent tracker failures here. + */ + trackerConnectionUp = TrackerConnectionUp(taskTracker); + if (trackerConnectionUp) + { + taskTracker->trackerFailureCount++; + } + + trackerHealthy = TrackerHealthy(taskTracker); + if (trackerHealthy) + { + TaskStatus remoteTaskStatus = TrackerTaskStatus(taskTracker, task); + if (remoteTaskStatus == TASK_CLIENT_SIDE_ASSIGN_FAILED) + { + nextExecutionStatus = EXEC_TASK_UNASSIGNED; + } + else if (remoteTaskStatus == TASK_CLIENT_SIDE_STATUS_FAILED) + { + nextExecutionStatus = EXEC_TASK_QUEUED; + } } else { nextExecutionStatus = EXEC_TASK_TRACKER_FAILED; } - } - else - { - /* assume task is still in progress */ - nextExecutionStatus = EXEC_TASK_QUEUED; + + break; } - break; - } - - case EXEC_TASK_TRACKER_RETRY: - { - bool trackerHealthy = false; - bool trackerConnectionUp = false; - - /* - * This case statement usually handles connection related issues. Some - * edge cases however, like a user sending a SIGTERM to the worker node, - * keep the connection open but disallow task assignments. We therefore - * need to track those as intermittent tracker failures here. - */ - trackerConnectionUp = TrackerConnectionUp(taskTracker); - if (trackerConnectionUp) + case EXEC_SOURCE_TASK_TRACKER_RETRY: { - taskTracker->trackerFailureCount++; - } + Task *mapTask = (Task *) linitial(task->dependedTaskList); + TaskExecution *mapTaskExecution = mapTask->taskExecution; + uint32 sourceNodeIndex = mapTaskExecution->currentNodeIndex; - trackerHealthy = TrackerHealthy(taskTracker); - if (trackerHealthy) - { - TaskStatus remoteTaskStatus = TrackerTaskStatus(taskTracker, task); - if (remoteTaskStatus == TASK_CLIENT_SIDE_ASSIGN_FAILED) + bool sourceTrackerHealthy = false; + Assert(sourceTaskTracker != NULL); + Assert(task->taskType == MAP_OUTPUT_FETCH_TASK); + + /* + * As this map fetch task was running, another map fetch that depends on + * another map task might have failed. We would have then reassigned the + * map task and potentially other map tasks in its constraint group. So + * this map fetch's source node might have changed underneath us. If it + * did, we don't want to record a failure for the new source tracker. + */ + if (taskExecution->querySourceNodeIndex == sourceNodeIndex) { + bool sourceTrackerConnectionUp = TrackerConnectionUp(sourceTaskTracker); + if (sourceTrackerConnectionUp) + { + sourceTaskTracker->trackerFailureCount++; + } + } + + sourceTrackerHealthy = TrackerHealthy(sourceTaskTracker); + if (sourceTrackerHealthy) + { + /* + * We change our status to unassigned. In that status, we queue an + * "update map fetch task" on the task tracker, and retry fetching + * the map task's output from the same source node. + */ nextExecutionStatus = EXEC_TASK_UNASSIGNED; } - else if (remoteTaskStatus == TASK_CLIENT_SIDE_STATUS_FAILED) + else { - nextExecutionStatus = EXEC_TASK_QUEUED; + nextExecutionStatus = EXEC_SOURCE_TASK_TRACKER_FAILED; } - } - else - { - nextExecutionStatus = EXEC_TASK_TRACKER_FAILED; + + break; } - break; - } - - case EXEC_SOURCE_TASK_TRACKER_RETRY: - { - Task *mapTask = (Task *) linitial(task->dependedTaskList); - TaskExecution *mapTaskExecution = mapTask->taskExecution; - uint32 sourceNodeIndex = mapTaskExecution->currentNodeIndex; - - bool sourceTrackerHealthy = false; - Assert(sourceTaskTracker != NULL); - Assert(task->taskType == MAP_OUTPUT_FETCH_TASK); - - /* - * As this map fetch task was running, another map fetch that depends on - * another map task might have failed. We would have then reassigned the - * map task and potentially other map tasks in its constraint group. So - * this map fetch's source node might have changed underneath us. If it - * did, we don't want to record a failure for the new source tracker. - */ - if (taskExecution->querySourceNodeIndex == sourceNodeIndex) - { - bool sourceTrackerConnectionUp = TrackerConnectionUp(sourceTaskTracker); - if (sourceTrackerConnectionUp) - { - sourceTaskTracker->trackerFailureCount++; - } - } - - sourceTrackerHealthy = TrackerHealthy(sourceTaskTracker); - if (sourceTrackerHealthy) + case EXEC_TASK_TRACKER_FAILED: + case EXEC_SOURCE_TASK_TRACKER_FAILED: { /* - * We change our status to unassigned. In that status, we queue an - * "update map fetch task" on the task tracker, and retry fetching - * the map task's output from the same source node. + * These two cases exist to signal to the caller that we failed. In both + * cases, the caller is responsible for reassigning task(s) and running + * the appropriate recovery logic. */ nextExecutionStatus = EXEC_TASK_UNASSIGNED; + break; } - else + + case EXEC_TASK_DONE: { - nextExecutionStatus = EXEC_SOURCE_TASK_TRACKER_FAILED; + /* we are done with this task's execution */ + nextExecutionStatus = EXEC_TASK_DONE; + break; } - break; - } - - case EXEC_TASK_TRACKER_FAILED: - case EXEC_SOURCE_TASK_TRACKER_FAILED: - { - /* - * These two cases exist to signal to the caller that we failed. In both - * cases, the caller is responsible for reassigning task(s) and running - * the appropriate recovery logic. - */ - nextExecutionStatus = EXEC_TASK_UNASSIGNED; - break; - } - - case EXEC_TASK_DONE: - { - /* we are done with this task's execution */ - nextExecutionStatus = EXEC_TASK_DONE; - break; - } - - default: - { - /* we fatal here to avoid leaking client-side resources */ - ereport(FATAL, (errmsg("invalid execution status: %d", currentExecutionStatus))); - break; - } + default: + { + /* we fatal here to avoid leaking client-side resources */ + ereport(FATAL, (errmsg("invalid execution status: %d", + currentExecutionStatus))); + break; + } } /* update task execution's status for most recent task tracker */ @@ -1247,225 +1249,227 @@ ManageTransmitExecution(TaskTracker *transmitTracker, switch (currentTransmitStatus) { - case EXEC_TRANSMIT_UNASSIGNED: - { - TaskExecStatus *taskStatusArray = taskExecution->taskStatusArray; - TaskExecStatus currentExecutionStatus = taskStatusArray[currentNodeIndex]; - bool trackerHealthy = false; - - /* if top level task's in progress, nothing to do */ - if (currentExecutionStatus != EXEC_TASK_DONE) + case EXEC_TRANSMIT_UNASSIGNED: { - nextTransmitStatus = EXEC_TRANSMIT_UNASSIGNED; - break; - } + TaskExecStatus *taskStatusArray = taskExecution->taskStatusArray; + TaskExecStatus currentExecutionStatus = taskStatusArray[currentNodeIndex]; + bool trackerHealthy = false; - trackerHealthy = TrackerHealthy(transmitTracker); - if (!trackerHealthy) - { - nextTransmitStatus = EXEC_TRANSMIT_TRACKER_FAILED; - break; - } + /* if top level task's in progress, nothing to do */ + if (currentExecutionStatus != EXEC_TASK_DONE) + { + nextTransmitStatus = EXEC_TRANSMIT_UNASSIGNED; + break; + } - TrackerQueueFileTransmit(transmitTracker, task); - nextTransmitStatus = EXEC_TRANSMIT_QUEUED; - break; - } + trackerHealthy = TrackerHealthy(transmitTracker); + if (!trackerHealthy) + { + nextTransmitStatus = EXEC_TRANSMIT_TRACKER_FAILED; + break; + } - case EXEC_TRANSMIT_QUEUED: - { - QueryStatus queryStatus = CLIENT_INVALID_QUERY; - int32 connectionId = INVALID_CONNECTION_ID; - TaskStatus taskStatus = TASK_STATUS_INVALID_FIRST; - - bool trackerHealthy = TrackerHealthy(transmitTracker); - if (!trackerHealthy) - { - nextTransmitStatus = EXEC_TRANSMIT_TRACKER_FAILED; - break; - } - - taskStatus = TrackerTaskStatus(transmitTracker, task); - if (taskStatus == TASK_FILE_TRANSMIT_QUEUED) - { - /* remain in queued status until tracker assigns this task */ + TrackerQueueFileTransmit(transmitTracker, task); nextTransmitStatus = EXEC_TRANSMIT_QUEUED; break; } - else if (taskStatus == TASK_CLIENT_SIDE_TRANSMIT_FAILED) + + case EXEC_TRANSMIT_QUEUED: { - nextTransmitStatus = EXEC_TRANSMIT_TRACKER_RETRY; - break; - } + QueryStatus queryStatus = CLIENT_INVALID_QUERY; + int32 connectionId = INVALID_CONNECTION_ID; + TaskStatus taskStatus = TASK_STATUS_INVALID_FIRST; - /* the open connection belongs to this task */ - connectionId = TransmitTrackerConnectionId(transmitTracker, task); - Assert(connectionId != INVALID_CONNECTION_ID); - Assert(taskStatus == TASK_ASSIGNED); - - /* start copy protocol */ - queryStatus = MultiClientQueryStatus(connectionId); - if (queryStatus == CLIENT_QUERY_COPY) - { - StringInfo jobDirectoryName = JobDirectoryName(task->jobId); - StringInfo taskFilename = TaskFilename(jobDirectoryName, task->taskId); - - char *filename = taskFilename->data; - int fileFlags = (O_APPEND | O_CREAT | O_RDWR | O_TRUNC | PG_BINARY); - int fileMode = (S_IRUSR | S_IWUSR); - - int32 fileDescriptor = BasicOpenFile(filename, fileFlags, fileMode); - if (fileDescriptor >= 0) + bool trackerHealthy = TrackerHealthy(transmitTracker); + if (!trackerHealthy) { - /* - * All files inside the job directory get automatically cleaned - * up on transaction commit or abort. - */ - fileDescriptorArray[currentNodeIndex] = fileDescriptor; - nextTransmitStatus = EXEC_TRANSMIT_COPYING; + nextTransmitStatus = EXEC_TRANSMIT_TRACKER_FAILED; + break; + } + + taskStatus = TrackerTaskStatus(transmitTracker, task); + if (taskStatus == TASK_FILE_TRANSMIT_QUEUED) + { + /* remain in queued status until tracker assigns this task */ + nextTransmitStatus = EXEC_TRANSMIT_QUEUED; + break; + } + else if (taskStatus == TASK_CLIENT_SIDE_TRANSMIT_FAILED) + { + nextTransmitStatus = EXEC_TRANSMIT_TRACKER_RETRY; + break; + } + + /* the open connection belongs to this task */ + connectionId = TransmitTrackerConnectionId(transmitTracker, task); + Assert(connectionId != INVALID_CONNECTION_ID); + Assert(taskStatus == TASK_ASSIGNED); + + /* start copy protocol */ + queryStatus = MultiClientQueryStatus(connectionId); + if (queryStatus == CLIENT_QUERY_COPY) + { + StringInfo jobDirectoryName = JobDirectoryName(task->jobId); + StringInfo taskFilename = TaskFilename(jobDirectoryName, task->taskId); + + char *filename = taskFilename->data; + int fileFlags = (O_APPEND | O_CREAT | O_RDWR | O_TRUNC | PG_BINARY); + int fileMode = (S_IRUSR | S_IWUSR); + + int32 fileDescriptor = BasicOpenFile(filename, fileFlags, fileMode); + if (fileDescriptor >= 0) + { + /* + * All files inside the job directory get automatically cleaned + * up on transaction commit or abort. + */ + fileDescriptorArray[currentNodeIndex] = fileDescriptor; + nextTransmitStatus = EXEC_TRANSMIT_COPYING; + } + else + { + ereport(WARNING, (errcode_for_file_access(), + errmsg("could not open file \"%s\": %m", + filename))); + + nextTransmitStatus = EXEC_TRANSMIT_TRACKER_RETRY; + } } else { - ereport(WARNING, (errcode_for_file_access(), - errmsg("could not open file \"%s\": %m", filename))); - nextTransmitStatus = EXEC_TRANSMIT_TRACKER_RETRY; } - } - else - { - nextTransmitStatus = EXEC_TRANSMIT_TRACKER_RETRY; + + /* + * We use task tracker logic to manage file transmits as well, but that + * abstraction starts to leak after we drop into the copy protocol. To + * make our task tracker logic work, we need to "void" the tracker's + * connection if the transmit task failed in here. + */ + if (nextTransmitStatus == EXEC_TRANSMIT_TRACKER_RETRY) + { + transmitTracker->connectionBusy = false; + transmitTracker->connectionBusyOnTask = NULL; + } + + break; } - /* - * We use task tracker logic to manage file transmits as well, but that - * abstraction starts to leak after we drop into the copy protocol. To - * make our task tracker logic work, we need to "void" the tracker's - * connection if the transmit task failed in here. - */ - if (nextTransmitStatus == EXEC_TRANSMIT_TRACKER_RETRY) + case EXEC_TRANSMIT_COPYING: { + int32 fileDescriptor = fileDescriptorArray[currentNodeIndex]; + CopyStatus copyStatus = CLIENT_INVALID_COPY; + int closed = -1; + + /* the open connection belongs to this task */ + int32 connectionId = TransmitTrackerConnectionId(transmitTracker, task); + Assert(connectionId != INVALID_CONNECTION_ID); + + copyStatus = MultiClientCopyData(connectionId, fileDescriptor); + if (copyStatus == CLIENT_COPY_MORE) + { + /* worker node continues to send more data, keep reading */ + nextTransmitStatus = EXEC_TRANSMIT_COPYING; + break; + } + + /* we are done copying data */ + if (copyStatus == CLIENT_COPY_DONE) + { + closed = close(fileDescriptor); + fileDescriptorArray[currentNodeIndex] = -1; + + if (closed >= 0) + { + nextTransmitStatus = EXEC_TRANSMIT_DONE; + } + else + { + ereport(WARNING, (errcode_for_file_access(), + errmsg("could not close copied file: %m"))); + + nextTransmitStatus = EXEC_TRANSMIT_TRACKER_RETRY; + } + } + else if (copyStatus == CLIENT_COPY_FAILED) + { + nextTransmitStatus = EXEC_TRANSMIT_TRACKER_RETRY; + + closed = close(fileDescriptor); + fileDescriptorArray[currentNodeIndex] = -1; + + if (closed < 0) + { + ereport(WARNING, (errcode_for_file_access(), + errmsg("could not close copy file: %m"))); + } + } + + /* + * We use task tracker logic to manage file transmits as well, but that + * abstraction leaks after we drop into the copy protocol. To make it + * work, we reset transmit tracker's connection for next file transmit. + */ transmitTracker->connectionBusy = false; transmitTracker->connectionBusyOnTask = NULL; - } - break; - } - - case EXEC_TRANSMIT_COPYING: - { - int32 fileDescriptor = fileDescriptorArray[currentNodeIndex]; - CopyStatus copyStatus = CLIENT_INVALID_COPY; - int closed = -1; - - /* the open connection belongs to this task */ - int32 connectionId = TransmitTrackerConnectionId(transmitTracker, task); - Assert(connectionId != INVALID_CONNECTION_ID); - - copyStatus = MultiClientCopyData(connectionId, fileDescriptor); - if (copyStatus == CLIENT_COPY_MORE) - { - /* worker node continues to send more data, keep reading */ - nextTransmitStatus = EXEC_TRANSMIT_COPYING; break; } - /* we are done copying data */ - if (copyStatus == CLIENT_COPY_DONE) + case EXEC_TRANSMIT_TRACKER_RETRY: { - closed = close(fileDescriptor); - fileDescriptorArray[currentNodeIndex] = -1; + bool trackerHealthy = false; + bool trackerConnectionUp = false; - if (closed >= 0) + /* + * The task tracker proxy handles connection errors. On the off chance + * that our connection is still up and the transmit tracker misbehaved, + * we capture this as an intermittent tracker failure. + */ + trackerConnectionUp = TrackerConnectionUp(transmitTracker); + if (trackerConnectionUp) { - nextTransmitStatus = EXEC_TRANSMIT_DONE; + transmitTracker->trackerFailureCount++; + } + + trackerHealthy = TrackerHealthy(transmitTracker); + if (trackerHealthy) + { + nextTransmitStatus = EXEC_TRANSMIT_UNASSIGNED; } else { - ereport(WARNING, (errcode_for_file_access(), - errmsg("could not close copied file: %m"))); - - nextTransmitStatus = EXEC_TRANSMIT_TRACKER_RETRY; + nextTransmitStatus = EXEC_TRANSMIT_TRACKER_FAILED; } - } - else if (copyStatus == CLIENT_COPY_FAILED) - { - nextTransmitStatus = EXEC_TRANSMIT_TRACKER_RETRY; - closed = close(fileDescriptor); - fileDescriptorArray[currentNodeIndex] = -1; - - if (closed < 0) - { - ereport(WARNING, (errcode_for_file_access(), - errmsg("could not close copy file: %m"))); - } + break; } - /* - * We use task tracker logic to manage file transmits as well, but that - * abstraction leaks after we drop into the copy protocol. To make it - * work, we reset transmit tracker's connection for next file transmit. - */ - transmitTracker->connectionBusy = false; - transmitTracker->connectionBusyOnTask = NULL; - - break; - } - - case EXEC_TRANSMIT_TRACKER_RETRY: - { - bool trackerHealthy = false; - bool trackerConnectionUp = false; - - /* - * The task tracker proxy handles connection errors. On the off chance - * that our connection is still up and the transmit tracker misbehaved, - * we capture this as an intermittent tracker failure. - */ - trackerConnectionUp = TrackerConnectionUp(transmitTracker); - if (trackerConnectionUp) - { - transmitTracker->trackerFailureCount++; - } - - trackerHealthy = TrackerHealthy(transmitTracker); - if (trackerHealthy) + case EXEC_TRANSMIT_TRACKER_FAILED: { + /* + * This case exists to signal to the caller that we failed. The caller + * is now responsible for reassigning the transmit task (and downstream + * SQL task dependencies) and running the appropriate recovery logic. + */ nextTransmitStatus = EXEC_TRANSMIT_UNASSIGNED; + break; } - else + + case EXEC_TRANSMIT_DONE: { - nextTransmitStatus = EXEC_TRANSMIT_TRACKER_FAILED; + /* we are done with fetching task results to the master node */ + nextTransmitStatus = EXEC_TRANSMIT_DONE; + break; } - break; - } - - case EXEC_TRANSMIT_TRACKER_FAILED: - { - /* - * This case exists to signal to the caller that we failed. The caller - * is now responsible for reassigning the transmit task (and downstream - * SQL task dependencies) and running the appropriate recovery logic. - */ - nextTransmitStatus = EXEC_TRANSMIT_UNASSIGNED; - break; - } - - case EXEC_TRANSMIT_DONE: - { - /* we are done with fetching task results to the master node */ - nextTransmitStatus = EXEC_TRANSMIT_DONE; - break; - } - - default: - { - /* we fatal here to avoid leaking client-side resources */ - ereport(FATAL, (errmsg("invalid transmit status: %d", currentTransmitStatus))); - break; - } + default: + { + /* we fatal here to avoid leaking client-side resources */ + ereport(FATAL, (errmsg("invalid transmit status: %d", + currentTransmitStatus))); + break; + } } /* update file transmit status for most recent transmit tracker */ @@ -2317,7 +2321,7 @@ AssignQueuedTasks(TaskTracker *taskTracker) { StringInfo taskAssignmentQuery = taskState->taskAssignmentQuery; - if(taskAssignmentCount > 0) + if (taskAssignmentCount > 0) { appendStringInfo(multiAssignQuery, ";"); } @@ -2336,7 +2340,7 @@ AssignQueuedTasks(TaskTracker *taskTracker) taskState = (TrackerTaskState *) hash_seq_search(&status); } - if(taskAssignmentCount > 0) + if (taskAssignmentCount > 0) { void *queryResult = NULL; int rowCount = 0; @@ -2833,7 +2837,8 @@ TrackerHashCleanupJob(HTAB *taskTrackerHash, Task *jobCleanupTask) if (queryStatus == CLIENT_QUERY_DONE) { ereport(DEBUG4, (errmsg("completed cleanup query for job " UINT64_FORMAT - " on node \"%s:%u\"", jobId, nodeName, nodePort))); + " on node \"%s:%u\"", jobId, nodeName, + nodePort))); /* clear connection for future cleanup queries */ taskTracker->connectionBusy = false; diff --git a/src/backend/distributed/executor/multi_utility.c b/src/backend/distributed/executor/multi_utility.c index ccd05c021..be4148f75 100644 --- a/src/backend/distributed/executor/multi_utility.c +++ b/src/backend/distributed/executor/multi_utility.c @@ -39,9 +39,9 @@ */ struct DropRelationCallbackState { - char relkind; - Oid heapOid; - bool concurrent; + char relkind; + Oid heapOid; + bool concurrent; }; @@ -190,10 +190,10 @@ multi_ProcessUtility(Node *parsetree, } else if (IsA(parsetree, CreateRoleStmt) && CitusDBHasBeenLoaded()) { - ereport(NOTICE, (errmsg("CitusDB does not support CREATE ROLE/USER " - "for distributed databases"), - errdetail("Multiple roles are currently supported " - "only for local tables"))); + ereport(NOTICE, (errmsg("CitusDB does not support CREATE ROLE/USER " + "for distributed databases"), + errdetail("Multiple roles are currently supported " + "only for local tables"))); } /* now drop into standard process utility */ @@ -757,7 +757,7 @@ IsAlterTableRenameStmt(RenameStmt *renameStmt) isAlterTableRenameStmt = true; } -#if (PG_VERSION_NUM >=90500) +#if (PG_VERSION_NUM >= 90500) else if (renameStmt->renameType == OBJECT_TABCONSTRAINT) { isAlterTableRenameStmt = true; @@ -905,8 +905,9 @@ ExecuteCommandOnWorkerShards(Oid relationId, const char *commandString, } else { - ereport(DEBUG2, (errmsg("applied command on shard " UINT64_FORMAT " on " - "node %s:%d", shardId, workerName, workerPort))); + ereport(DEBUG2, (errmsg("applied command on shard " UINT64_FORMAT + " on node %s:%d", shardId, workerName, + workerPort))); } isFirstPlacement = false; diff --git a/src/backend/distributed/master/master_create_shards.c b/src/backend/distributed/master/master_create_shards.c index 0617d8d59..9adc0e21c 100644 --- a/src/backend/distributed/master/master_create_shards.c +++ b/src/backend/distributed/master/master_create_shards.c @@ -185,7 +185,7 @@ master_create_worker_shards(PG_FUNCTION_ARGS) LockShardDistributionMetadata(shardId, ExclusiveLock); CreateShardPlacements(shardId, ddlCommandList, workerNodeList, - roundRobinNodeIndex, replicationFactor); + roundRobinNodeIndex, replicationFactor); InsertShardRow(distributedTableId, shardId, shardStorageType, minHashTokenText, maxHashTokenText); diff --git a/src/backend/distributed/master/master_delete_protocol.c b/src/backend/distributed/master/master_delete_protocol.c index 35835c7d0..efeeb78bc 100644 --- a/src/backend/distributed/master/master_delete_protocol.c +++ b/src/backend/distributed/master/master_delete_protocol.c @@ -115,9 +115,9 @@ master_apply_delete_command(PG_FUNCTION_ARGS) if ((partitionMethod == DISTRIBUTE_BY_HASH) && (deleteCriteria != NULL)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot delete from distributed table"), - errdetail("Delete statements on hash-partitioned tables " - "with where clause is not supported"))); + errmsg("cannot delete from distributed table"), + errdetail("Delete statements on hash-partitioned tables " + "with where clause is not supported"))); } CheckDeleteCriteria(deleteCriteria); @@ -138,15 +138,15 @@ master_apply_delete_command(PG_FUNCTION_ARGS) else { deletableShardIntervalList = ShardsMatchingDeleteCriteria(relationId, - shardIntervalList, - deleteCriteria); + shardIntervalList, + deleteCriteria); } foreach(shardIntervalCell, deletableShardIntervalList) { List *shardPlacementList = NIL; List *droppedPlacementList = NIL; - List *lingeringPlacementList= NIL; + List *lingeringPlacementList = NIL; ListCell *shardPlacementCell = NULL; ListCell *droppedPlacementCell = NULL; ListCell *lingeringPlacementCell = NULL; @@ -167,7 +167,8 @@ master_apply_delete_command(PG_FUNCTION_ARGS) shardPlacementList = ShardPlacementList(shardId); foreach(shardPlacementCell, shardPlacementList) { - ShardPlacement *shardPlacement = (ShardPlacement *) lfirst(shardPlacementCell); + ShardPlacement *shardPlacement = + (ShardPlacement *) lfirst(shardPlacementCell); char *workerName = shardPlacement->nodeName; uint32 workerPort = shardPlacement->nodePort; bool dropSuccessful = false; @@ -176,14 +177,17 @@ master_apply_delete_command(PG_FUNCTION_ARGS) char tableType = get_rel_relkind(relationId); if (tableType == RELKIND_RELATION) { - appendStringInfo(workerDropQuery, DROP_REGULAR_TABLE_COMMAND, quotedShardName); + appendStringInfo(workerDropQuery, DROP_REGULAR_TABLE_COMMAND, + quotedShardName); } else if (tableType == RELKIND_FOREIGN_TABLE) { - appendStringInfo(workerDropQuery, DROP_FOREIGN_TABLE_COMMAND, quotedShardName); + appendStringInfo(workerDropQuery, DROP_FOREIGN_TABLE_COMMAND, + quotedShardName); } - dropSuccessful = ExecuteRemoteCommand(workerName, workerPort, workerDropQuery); + dropSuccessful = ExecuteRemoteCommand(workerName, workerPort, + workerDropQuery); if (dropSuccessful) { droppedPlacementList = lappend(droppedPlacementList, shardPlacement); @@ -227,12 +231,13 @@ master_apply_delete_command(PG_FUNCTION_ARGS) if (QueryCancelPending) { - ereport(WARNING, (errmsg("cancel requests are ignored during shard deletion"))); + ereport(WARNING, (errmsg("cancel requests are ignored during shard " + "deletion"))); QueryCancelPending = false; } RESUME_INTERRUPTS(); - } + } deleteCriteriaShardCount = list_length(deletableShardIntervalList); PG_RETURN_INT32(deleteCriteriaShardCount); @@ -257,7 +262,7 @@ CheckTableCount(Query *deleteQuery) static void CheckDeleteCriteria(Node *deleteCriteria) { - bool simpleOpExpression = true; + bool simpleOpExpression = true; if (deleteCriteria == NULL) { @@ -286,7 +291,7 @@ CheckDeleteCriteria(Node *deleteCriteria) } else { - simpleOpExpression = false; + simpleOpExpression = false; } if (!simpleOpExpression) @@ -298,15 +303,15 @@ CheckDeleteCriteria(Node *deleteCriteria) } - /* - * CheckPartitionColumn checks that the given where clause is based only on the - * partition key of the given relation id. - */ +/* + * CheckPartitionColumn checks that the given where clause is based only on the + * partition key of the given relation id. + */ static void CheckPartitionColumn(Oid relationId, Node *whereClause) { Var *partitionColumn = PartitionKey(relationId); - ListCell *columnCell = NULL; + ListCell *columnCell = NULL; List *columnList = pull_var_clause_default(whereClause); foreach(columnCell, columnList) @@ -332,7 +337,7 @@ CheckPartitionColumn(Oid relationId, Node *whereClause) */ static List * ShardsMatchingDeleteCriteria(Oid relationId, List *shardIntervalList, - Node *deleteCriteria) + Node *deleteCriteria) { List *dropShardIntervalList = NIL; List *deleteCriteriaList = NIL; diff --git a/src/backend/distributed/master/master_metadata_utility.c b/src/backend/distributed/master/master_metadata_utility.c index d8ac90997..34db8da9a 100644 --- a/src/backend/distributed/master/master_metadata_utility.c +++ b/src/backend/distributed/master/master_metadata_utility.c @@ -219,7 +219,7 @@ ShardLength(uint64 shardId) if (shardPlacementList == NIL) { ereport(ERROR, (errmsg("could not find length of shard " UINT64_FORMAT, shardId), - errdetail("Could not find any shard placements for the shard."))); + errdetail("Could not find any shard placements for the shard."))); } else { diff --git a/src/backend/distributed/master/master_node_protocol.c b/src/backend/distributed/master/master_node_protocol.c index fc9413666..064756d77 100644 --- a/src/backend/distributed/master/master_node_protocol.c +++ b/src/backend/distributed/master/master_node_protocol.c @@ -49,7 +49,7 @@ /* Shard related configuration */ int ShardReplicationFactor = 2; /* desired replication factor for shards */ -int ShardMaxSize = 1048576; /* maximum size in KB one shard can grow to */ +int ShardMaxSize = 1048576; /* maximum size in KB one shard can grow to */ int ShardPlacementPolicy = SHARD_PLACEMENT_ROUND_ROBIN; @@ -210,7 +210,7 @@ master_get_table_ddl_events(PG_FUNCTION_ARGS) tableDDLEventCell = list_head(tableDDLEventList); functionContext->user_fctx = tableDDLEventCell; - + MemoryContextSwitchTo(oldContext); } @@ -226,8 +226,8 @@ master_get_table_ddl_events(PG_FUNCTION_ARGS) if (tableDDLEventCell != NULL) { char *ddlStatement = (char *) lfirst(tableDDLEventCell); - text *ddlStatementText = cstring_to_text(ddlStatement); - + text *ddlStatementText = cstring_to_text(ddlStatement); + functionContext->user_fctx = lnext(tableDDLEventCell); SRF_RETURN_NEXT(functionContext, PointerGetDatum(ddlStatementText)); @@ -252,7 +252,7 @@ Datum master_get_new_shardid(PG_FUNCTION_ARGS) { text *sequenceName = cstring_to_text(SHARDID_SEQUENCE_NAME); - Oid sequenceId = ResolveRelationId(sequenceName); + Oid sequenceId = ResolveRelationId(sequenceName); Datum sequenceIdDatum = ObjectIdGetDatum(sequenceId); /* generate new and unique shardId from sequence */ @@ -281,7 +281,7 @@ master_get_local_first_candidate_nodes(PG_FUNCTION_ARGS) if (SRF_IS_FIRSTCALL()) { - MemoryContext oldContext = NULL; + MemoryContext oldContext = NULL; TupleDesc tupleDescriptor = NULL; uint32 liveNodeCount = 0; bool hasOid = false; @@ -396,7 +396,7 @@ master_get_round_robin_candidate_nodes(PG_FUNCTION_ARGS) if (SRF_IS_FIRSTCALL()) { - MemoryContext oldContext = NULL; + MemoryContext oldContext = NULL; TupleDesc tupleDescriptor = NULL; List *workerNodeList = NIL; TypeFuncClass resultTypeClass = 0; @@ -477,7 +477,7 @@ master_get_active_worker_nodes(PG_FUNCTION_ARGS) if (SRF_IS_FIRSTCALL()) { - MemoryContext oldContext = NULL; + MemoryContext oldContext = NULL; List *workerNodeList = NIL; uint32 workerNodeCount = 0; TupleDesc tupleDescriptor = NULL; @@ -567,7 +567,7 @@ GetTableDDLEvents(Oid relationId) Relation pgIndex = NULL; SysScanDesc scanDescriptor = NULL; - ScanKeyData scanKey[1]; + ScanKeyData scanKey[1]; int scanKeyCount = 1; HeapTuple heapTuple = NULL; @@ -599,13 +599,13 @@ GetTableDDLEvents(Oid relationId) /* fetch table schema and column option definitions */ tableSchemaDef = pg_get_tableschemadef_string(relationId); tableColumnOptionsDef = pg_get_tablecolumnoptionsdef_string(relationId); - + tableDDLEventList = lappend(tableDDLEventList, tableSchemaDef); if (tableColumnOptionsDef != NULL) { tableDDLEventList = lappend(tableDDLEventList, tableColumnOptionsDef); } - + /* open system catalog and scan all indexes that belong to this table */ pgIndex = heap_open(IndexRelationId, AccessShareLock); @@ -660,7 +660,7 @@ GetTableDDLEvents(Oid relationId) { statementDef = pg_get_indexdef_string(indexId); } - + /* append found constraint or index definition to the list */ tableDDLEventList = lappend(tableDDLEventList, statementDef); @@ -695,8 +695,8 @@ hostname_client_addr(void) Port *port = MyProcPort; char *remoteHost = NULL; int remoteHostLen = NI_MAXHOST; - int flags = NI_NAMEREQD; /* require fully qualified hostname */ - int nameFound = 0; + int flags = NI_NAMEREQD; /* require fully qualified hostname */ + int nameFound = 0; if (port == NULL) { @@ -709,10 +709,15 @@ hostname_client_addr(void) #ifdef HAVE_IPV6 case AF_INET6: #endif - break; + { + break; + } + default: + { ereport(ERROR, (errmsg("invalid address family in connection"))); break; + } } remoteHost = palloc0(remoteHostLen); diff --git a/src/backend/distributed/master/master_stage_protocol.c b/src/backend/distributed/master/master_stage_protocol.c index 1c13237cc..e4c4f7fca 100644 --- a/src/backend/distributed/master/master_stage_protocol.c +++ b/src/backend/distributed/master/master_stage_protocol.c @@ -93,7 +93,7 @@ master_create_empty_shard(PG_FUNCTION_ARGS) if (partitionMethod == DISTRIBUTE_BY_HASH) { ereport(ERROR, (errmsg("relation \"%s\" is a hash partitioned table", - relationName), + relationName), errdetail("We currently don't support creating shards " "on hash-partitioned tables"))); } @@ -128,7 +128,7 @@ master_create_empty_shard(PG_FUNCTION_ARGS) } CreateShardPlacements(shardId, ddlEventList, candidateNodeList, 0, - ShardReplicationFactor); + ShardReplicationFactor); InsertShardRow(relationId, shardId, SHARD_STORAGE_TABLE, nullMinValue, nullMaxValue); @@ -361,7 +361,7 @@ CheckDistributedTable(Oid relationId) */ void CreateShardPlacements(int64 shardId, List *ddlEventList, List *workerNodeList, - int workerStartIndex, int replicationFactor) + int workerStartIndex, int replicationFactor) { int attemptCount = replicationFactor; int workerNodeCount = list_length(workerNodeList); @@ -393,7 +393,7 @@ CreateShardPlacements(int64 shardId, List *ddlEventList, List *workerNodeList, else { ereport(WARNING, (errmsg("could not create shard on \"%s:%u\"", - nodeName, nodePort))); + nodeName, nodePort))); } if (placementsCreated >= replicationFactor) @@ -406,7 +406,7 @@ CreateShardPlacements(int64 shardId, List *ddlEventList, List *workerNodeList, if (placementsCreated < replicationFactor) { ereport(ERROR, (errmsg("could only create %u of %u of required shard replicas", - placementsCreated, replicationFactor))); + placementsCreated, replicationFactor))); } } diff --git a/src/backend/distributed/planner/modify_planner.c b/src/backend/distributed/planner/modify_planner.c index faeb6b6b0..4484e53bb 100644 --- a/src/backend/distributed/planner/modify_planner.c +++ b/src/backend/distributed/planner/modify_planner.c @@ -393,6 +393,7 @@ DistributedModifyTask(Query *query) query->onConflict = RebuildOnConflict(relationId, query->onConflict); } #else + /* always set to false for PG_VERSION_NUM < 90500 */ upsertQuery = false; #endif @@ -414,6 +415,7 @@ DistributedModifyTask(Query *query) #if (PG_VERSION_NUM >= 90500) + /* * RebuildOnConflict rebuilds OnConflictExpr for correct deparsing. The function * makes WHERE clause elements explicit and filters dropped columns @@ -433,7 +435,7 @@ RebuildOnConflict(Oid relationId, OnConflictExpr *originalOnConflict) /* Convert onConflictWhere qualifiers to an explicitly and'd clause */ updatedOnConflict->onConflictWhere = - (Node *) make_ands_explicit((List *) onConflictWhere); + (Node *) make_ands_explicit((List *) onConflictWhere); /* * Here we handle dropped columns on the distributed table. onConflictSet @@ -448,7 +450,7 @@ RebuildOnConflict(Oid relationId, OnConflictExpr *originalOnConflict) foreach(targetEntryCell, onConflictSet) { TargetEntry *targetEntry = (TargetEntry *) lfirst(targetEntryCell); - FormData_pg_attribute *tableAttribute = tableAttributes[targetEntry->resno -1]; + FormData_pg_attribute *tableAttribute = tableAttributes[targetEntry->resno - 1]; /* skip dropped columns */ if (tableAttribute->attisdropped) @@ -468,6 +470,8 @@ RebuildOnConflict(Oid relationId, OnConflictExpr *originalOnConflict) return updatedOnConflict; } + + #endif diff --git a/src/backend/distributed/planner/multi_explain.c b/src/backend/distributed/planner/multi_explain.c index 35e0ca79b..c7fbdefed 100644 --- a/src/backend/distributed/planner/multi_explain.c +++ b/src/backend/distributed/planner/multi_explain.c @@ -45,7 +45,7 @@ MultiExplainOneQuery(Query *query, IntoClause *into, ExplainState *es, if (localQuery) { PlannedStmt *plan = NULL; - instr_time planstart; + instr_time planstart; instr_time planduration; INSTR_TIME_SET_CURRENT(planstart); diff --git a/src/backend/distributed/planner/multi_join_order.c b/src/backend/distributed/planner/multi_join_order.c index e8cc68f07..04a91d311 100644 --- a/src/backend/distributed/planner/multi_join_order.c +++ b/src/backend/distributed/planner/multi_join_order.c @@ -33,18 +33,18 @@ /* Config variables managed via guc.c */ -int LargeTableShardCount = 4; /* shard counts for a large table */ +int LargeTableShardCount = 4; /* shard counts for a large table */ bool LogMultiJoinOrder = false; /* print join order as a debugging aid */ /* Function pointer type definition for join rule evaluation functions */ -typedef JoinOrderNode * (*RuleEvalFunction) (JoinOrderNode *currentJoinNode, - TableEntry *candidateTable, - List *candidateShardList, - List *applicableJoinClauses, - JoinType joinType); +typedef JoinOrderNode *(*RuleEvalFunction) (JoinOrderNode *currentJoinNode, + TableEntry *candidateTable, + List *candidateShardList, + List *applicableJoinClauses, + JoinType joinType); -static char * RuleNameArray[JOIN_RULE_LAST] = {0}; /* ordered join rule names */ -static RuleEvalFunction RuleEvalFunctionArray[JOIN_RULE_LAST] = {0}; /* join rules */ +static char *RuleNameArray[JOIN_RULE_LAST] = { 0 }; /* ordered join rule names */ +static RuleEvalFunction RuleEvalFunctionArray[JOIN_RULE_LAST] = { 0 }; /* join rules */ /* Local functions forward declarations */ @@ -54,7 +54,8 @@ static bool JoinExprListWalker(Node *node, List **joinList); static bool ExtractLeftMostRangeTableIndex(Node *node, int *rangeTableIndex); static List * MergeShardIntervals(List *leftShardIntervalList, List *rightShardIntervalList, JoinType joinType); -static bool ShardIntervalsMatch(List *leftShardIntervalList, List *rightShardIntervalList); +static bool ShardIntervalsMatch(List *leftShardIntervalList, + List *rightShardIntervalList); static List * LoadSortedShardIntervalList(Oid relationId); static List * JoinOrderForTable(TableEntry *firstTable, List *tableEntryList, List *joinClauseList); @@ -68,31 +69,41 @@ static List * TableEntryListDifference(List *lhsTableList, List *rhsTableList); static TableEntry * FindTableEntry(List *tableEntryList, uint32 tableId); /* Local functions forward declarations for join evaluations */ -static JoinOrderNode * EvaluateJoinRules(List *joinedTableList, JoinOrderNode *currentJoinNode, - TableEntry *candidateTable, List *candidateShardList, +static JoinOrderNode * EvaluateJoinRules(List *joinedTableList, + JoinOrderNode *currentJoinNode, + TableEntry *candidateTable, + List *candidateShardList, List *joinClauseList, JoinType joinType); static List * RangeTableIdList(List *tableList); static RuleEvalFunction JoinRuleEvalFunction(JoinRuleType ruleType); static char * JoinRuleName(JoinRuleType ruleType); static JoinOrderNode * BroadcastJoin(JoinOrderNode *joinNode, TableEntry *candidateTable, - List *candidateShardList, List *applicableJoinClauses, + List *candidateShardList, + List *applicableJoinClauses, JoinType joinType); static JoinOrderNode * LocalJoin(JoinOrderNode *joinNode, TableEntry *candidateTable, List *candidateShardList, List *applicableJoinClauses, JoinType joinType); static bool JoinOnColumns(Var *currentPartitioncolumn, Var *candidatePartitionColumn, List *joinClauseList); -static JoinOrderNode * SinglePartitionJoin(JoinOrderNode *joinNode, TableEntry *candidateTable, - List *candidateShardList, List *applicableJoinClauses, +static JoinOrderNode * SinglePartitionJoin(JoinOrderNode *joinNode, + TableEntry *candidateTable, + List *candidateShardList, + List *applicableJoinClauses, JoinType joinType); -static JoinOrderNode * DualPartitionJoin(JoinOrderNode *joinNode, TableEntry *candidateTable, - List *candidateShardList, List *applicableJoinClauses, +static JoinOrderNode * DualPartitionJoin(JoinOrderNode *joinNode, + TableEntry *candidateTable, + List *candidateShardList, + List *applicableJoinClauses, JoinType joinType); -static JoinOrderNode * CartesianProduct(JoinOrderNode *joinNode, TableEntry *candidateTable, - List *candidateShardList, List *applicableJoinClauses, +static JoinOrderNode * CartesianProduct(JoinOrderNode *joinNode, + TableEntry *candidateTable, + List *candidateShardList, + List *applicableJoinClauses, JoinType joinType); -static JoinOrderNode * MakeJoinOrderNode(TableEntry *tableEntry, JoinRuleType joinRuleType, - Var *partitionColumn, char partitionMethod); +static JoinOrderNode * MakeJoinOrderNode(TableEntry *tableEntry, JoinRuleType + joinRuleType, Var *partitionColumn, + char partitionMethod); /* @@ -106,7 +117,7 @@ List * FixedJoinOrderList(FromExpr *fromExpr, List *tableEntryList) { List *joinList = NIL; - ListCell * joinCell = NULL; + ListCell *joinCell = NULL; List *joinWhereClauseList = NIL; List *joinOrderList = NIL; List *joinedTableList = NIL; @@ -199,7 +210,6 @@ FixedJoinOrderList(FromExpr *fromExpr, List *tableEntryList) "query"), errdetail("Shards of relations in outer join queries " "must have 1-to-1 shard partitioning"))); - } } else @@ -439,7 +449,7 @@ MergeShardIntervals(List *leftShardIntervalList, List *rightShardIntervalList, bool nextMaxSmaller = comparisonResult > 0; if ((shardUnion && nextMaxLarger) || - (!shardUnion && nextMaxSmaller) ) + (!shardUnion && nextMaxSmaller)) { newShardInterval->maxValue = datumCopy(nextMax, typeByValue, typeLen); } @@ -586,7 +596,8 @@ ShardIntervalsMatch(List *leftShardIntervalList, List *rightShardIntervalList) nextRightIntervalCell = lnext(rightShardIntervalCell); if (nextRightIntervalCell != NULL) { - ShardInterval *nextRightInterval = (ShardInterval *) lfirst(nextRightIntervalCell); + ShardInterval *nextRightInterval = + (ShardInterval *) lfirst(nextRightIntervalCell); shardIntervalsIntersect = ShardIntervalsOverlap(leftInterval, nextRightInterval); if (shardIntervalsIntersect) @@ -730,7 +741,7 @@ JoinOrderForTable(TableEntry *firstTable, List *tableEntryList, List *joinClause * BestJoinOrder takes in a list of candidate join orders, and determines the * best join order among these candidates. The function uses two heuristics for * this. First, the function chooses join orders that have the fewest number of - * join operators that cause large data transfers. Second, the function chooses + * join operators that cause large data transfers. Second, the function chooses * join orders where large data transfers occur later in the execution. */ static List * @@ -1009,7 +1020,7 @@ EvaluateJoinRules(List *joinedTableList, JoinOrderNode *currentJoinNode, uint32 candidateTableId = 0; List *joinedTableIdList = NIL; List *applicableJoinClauses = NIL; - uint32 lowestValidIndex = JOIN_RULE_INVALID_FIRST + 1; + uint32 lowestValidIndex = JOIN_RULE_INVALID_FIRST + 1; uint32 highestValidIndex = JOIN_RULE_LAST - 1; uint32 ruleIndex = 0; @@ -1028,11 +1039,11 @@ EvaluateJoinRules(List *joinedTableList, JoinOrderNode *currentJoinNode, JoinRuleType ruleType = (JoinRuleType) ruleIndex; RuleEvalFunction ruleEvalFunction = JoinRuleEvalFunction(ruleType); - nextJoinNode = (*ruleEvalFunction) (currentJoinNode, - candidateTable, - candidateShardList, - applicableJoinClauses, - joinType); + nextJoinNode = (*ruleEvalFunction)(currentJoinNode, + candidateTable, + candidateShardList, + applicableJoinClauses, + joinType); /* break after finding the first join rule that applies */ if (nextJoinNode != NULL) diff --git a/src/backend/distributed/planner/multi_logical_optimizer.c b/src/backend/distributed/planner/multi_logical_optimizer.c index 3aebfcb37..7acf85d7e 100644 --- a/src/backend/distributed/planner/multi_logical_optimizer.c +++ b/src/backend/distributed/planner/multi_logical_optimizer.c @@ -91,7 +91,8 @@ static void ParentSetNewChild(MultiNode *parentNode, MultiNode *oldChildNode, /* Local functions forward declarations for aggregate expressions */ static void ApplyExtendedOpNodes(MultiExtendedOp *originalNode, - MultiExtendedOp *masterNode, MultiExtendedOp *workerNode); + MultiExtendedOp *masterNode, + MultiExtendedOp *workerNode); static void TransformSubqueryNode(MultiTable *subqueryNode); static MultiExtendedOp * MasterExtendedOpNode(MultiExtendedOp *originalOpNode); static Node * MasterAggregateMutator(Node *originalNode, AttrNumber *columnId); @@ -117,7 +118,8 @@ static void ErrorIfUnsupportedArrayAggregate(Aggref *arrayAggregateExpression); static void ErrorIfUnsupportedAggregateDistinct(Aggref *aggregateExpression, MultiNode *logicalPlanNode); static Var * AggregateDistinctColumn(Aggref *aggregateExpression); -static bool TablePartitioningSupportsDistinct(List *tableNodeList, MultiExtendedOp *opNode, +static bool TablePartitioningSupportsDistinct(List *tableNodeList, + MultiExtendedOp *opNode, Var *distinctColumn); static bool GroupedByColumn(List *groupClauseList, List *targetList, Var *column); @@ -488,7 +490,7 @@ AddressProjectSpecialConditions(MultiProject *projectNode) /* * We check if we need to include any child columns in the project node to - * address the following special conditions. + * address the following special conditions. * * SNC1: project node must include child node's projected columns, or * SNC2: project node must include child node's partition column, or @@ -637,7 +639,7 @@ Commutative(MultiUnaryNode *parentNode, MultiUnaryNode *childNode) { PushDownStatus pushDownStatus = PUSH_DOWN_NOT_VALID; CitusNodeTag parentNodeTag = CitusNodeTag(parentNode); - CitusNodeTag childNodeTag = CitusNodeTag(childNode); + CitusNodeTag childNodeTag = CitusNodeTag(childNode); /* we cannot be commutative with non-query operators */ if (childNodeTag == T_MultiTreeRoot || childNodeTag == T_MultiTable) @@ -692,7 +694,7 @@ Distributive(MultiUnaryNode *parentNode, MultiBinaryNode *childNode) { PushDownStatus pushDownStatus = PUSH_DOWN_NOT_VALID; CitusNodeTag parentNodeTag = CitusNodeTag(parentNode); - CitusNodeTag childNodeTag = CitusNodeTag(childNode); + CitusNodeTag childNodeTag = CitusNodeTag(childNode); /* special condition checks for partition operator are not implemented */ Assert(parentNodeTag != T_MultiPartition); @@ -751,7 +753,7 @@ Factorizable(MultiBinaryNode *parentNode, MultiUnaryNode *childNode) { PullUpStatus pullUpStatus = PULL_UP_NOT_VALID; CitusNodeTag parentNodeTag = CitusNodeTag(parentNode); - CitusNodeTag childNodeTag = CitusNodeTag(childNode); + CitusNodeTag childNodeTag = CitusNodeTag(childNode); /* * The following nodes are factorizable with their parents, but we don't @@ -1220,7 +1222,7 @@ MasterExtendedOpNode(MultiExtendedOp *originalOpNode) bool hasAggregates = contain_agg_clause((Node *) originalExpression); if (hasAggregates) { - Node *newNode = MasterAggregateMutator((Node*) originalExpression, + Node *newNode = MasterAggregateMutator((Node *) originalExpression, &columnId); newExpression = (Expr *) newNode; } @@ -1826,7 +1828,7 @@ WorkerAggregateExpressionList(Aggref *originalAggregate) static AggregateType GetAggregateType(Oid aggFunctionId) { - char *aggregateProcName = NULL; + char *aggregateProcName = NULL; uint32 aggregateCount = 0; uint32 aggregateIndex = 0; bool found = false; @@ -1980,22 +1982,30 @@ CountDistinctHashFunctionName(Oid argumentType) switch (argumentType) { case INT4OID: + { hashFunctionName = pstrdup(HLL_HASH_INTEGER_FUNC_NAME); break; + } case INT8OID: + { hashFunctionName = pstrdup(HLL_HASH_BIGINT_FUNC_NAME); break; + } case TEXTOID: case BPCHAROID: case VARCHAROID: + { hashFunctionName = pstrdup(HLL_HASH_TEXT_FUNC_NAME); break; + } default: + { hashFunctionName = pstrdup(HLL_HASH_ANY_FUNC_NAME); break; + } } return hashFunctionName; @@ -2479,7 +2489,7 @@ ErrorIfCannotPushdownSubquery(Query *subqueryTree, bool outerQueryHasLimit) if (subqueryTree->setOperations) { SetOperationStmt *setOperationStatement = - (SetOperationStmt *) subqueryTree->setOperations; + (SetOperationStmt *) subqueryTree->setOperations; if (setOperationStatement->op == SETOP_UNION) { @@ -2563,7 +2573,7 @@ ErrorIfCannotPushdownSubquery(Query *subqueryTree, bool outerQueryHasLimit) List *joinTreeTableIndexList = NIL; uint32 joiningTableCount = 0; - ExtractRangeTableIndexWalker((Node*) subqueryTree->jointree, + ExtractRangeTableIndexWalker((Node *) subqueryTree->jointree, &joinTreeTableIndexList); joiningTableCount = list_length(joinTreeTableIndexList); @@ -2587,7 +2597,7 @@ ErrorIfCannotPushdownSubquery(Query *subqueryTree, bool outerQueryHasLimit) List *distinctTargetEntryList = GroupTargetEntryList(distinctClauseList, targetEntryList); bool distinctOnPartitionColumn = - TargetListOnPartitionColumn(subqueryTree, distinctTargetEntryList); + TargetListOnPartitionColumn(subqueryTree, distinctTargetEntryList); if (!distinctOnPartitionColumn) { preconditionsSatisfied = false; @@ -2609,7 +2619,7 @@ ErrorIfCannotPushdownSubquery(Query *subqueryTree, bool outerQueryHasLimit) foreach(rangeTableEntryCell, subqueryEntryList) { RangeTblEntry *rangeTableEntry = - (RangeTblEntry *) lfirst(rangeTableEntryCell); + (RangeTblEntry *) lfirst(rangeTableEntryCell); Query *innerSubquery = rangeTableEntry->subquery; ErrorIfCannotPushdownSubquery(innerSubquery, outerQueryHasLimit); @@ -2639,7 +2649,7 @@ ErrorIfUnsupportedTableCombination(Query *queryTree) * Extract all range table indexes from the join tree. Note that sub-queries * that get pulled up by PostgreSQL don't appear in this join tree. */ - ExtractRangeTableIndexWalker((Node*) queryTree->jointree, &joinTreeTableIndexList); + ExtractRangeTableIndexWalker((Node *) queryTree->jointree, &joinTreeTableIndexList); foreach(joinTreeTableIndexCell, joinTreeTableIndexList) { /* @@ -2768,7 +2778,7 @@ ErrorIfUnsupportedUnionQuery(Query *unionQuery) leftQueryOnPartitionColumn = TargetListOnPartitionColumn(leftQuery, leftQuery->targetList); rightQueryOnPartitionColumn = TargetListOnPartitionColumn(rightQuery, - rightQuery->targetList); + rightQuery->targetList); if (!(leftQueryOnPartitionColumn && rightQueryOnPartitionColumn)) { @@ -2807,7 +2817,7 @@ GroupTargetEntryList(List *groupClauseList, List *targetEntryList) { SortGroupClause *groupClause = (SortGroupClause *) lfirst(groupClauseCell); TargetEntry *groupTargetEntry = - get_sortgroupclause_tle(groupClause, targetEntryList); + get_sortgroupclause_tle(groupClause, targetEntryList); groupTargetEntryList = lappend(groupTargetEntryList, groupTargetEntry); } @@ -2890,7 +2900,7 @@ IsPartitionColumnRecursive(Expr *columnExpression, Query *query) else if (IsA(columnExpression, FieldSelect)) { FieldSelect *compositeField = (FieldSelect *) columnExpression; - Expr *fieldExpression = compositeField->arg; + Expr *fieldExpression = compositeField->arg; if (IsA(fieldExpression, Var)) { @@ -2909,7 +2919,7 @@ IsPartitionColumnRecursive(Expr *columnExpression, Query *query) return false; } - rangeTableEntryIndex = candidateColumn->varno - 1; + rangeTableEntryIndex = candidateColumn->varno - 1; rangeTableEntry = list_nth(rangetableList, rangeTableEntryIndex); if (rangeTableEntry->rtekind == RTE_RELATION) @@ -2980,7 +2990,7 @@ CompositeFieldRecursive(Expr *expression, Query *query) return NULL; } - rangeTableEntryIndex = candidateColumn->varno - 1; + rangeTableEntryIndex = candidateColumn->varno - 1; rangeTableEntry = list_nth(rangetableList, rangeTableEntryIndex); if (rangeTableEntry->rtekind == RTE_SUBQUERY) @@ -3019,7 +3029,7 @@ FullCompositeFieldList(List *compositeFieldList) uint32 fieldIndex = 0; ListCell *fieldSelectCell = NULL; - foreach (fieldSelectCell, compositeFieldList) + foreach(fieldSelectCell, compositeFieldList) { FieldSelect *fieldSelect = (FieldSelect *) lfirst(fieldSelectCell); uint32 compositeFieldIndex = 0; @@ -3226,9 +3236,10 @@ SupportedLateralQuery(Query *parentQuery, Query *lateralQuery) if (outerColumnIsPartitionColumn && localColumnIsPartitionColumn) { FieldSelect *outerCompositeField = - CompositeFieldRecursive(outerQueryExpression, parentQuery); + CompositeFieldRecursive(outerQueryExpression, parentQuery); FieldSelect *localCompositeField = - CompositeFieldRecursive(localQueryExpression, lateralQuery); + CompositeFieldRecursive(localQueryExpression, lateralQuery); + /* * If partition colums are composite fields, add them to list to * check later if all composite fields are used. @@ -3251,12 +3262,12 @@ SupportedLateralQuery(Query *parentQuery, Query *lateralQuery) } /* check composite fields */ - if(!supportedLateralQuery) + if (!supportedLateralQuery) { bool outerFullCompositeFieldList = - FullCompositeFieldList(outerCompositeFieldList); + FullCompositeFieldList(outerCompositeFieldList); bool localFullCompositeFieldList = - FullCompositeFieldList(localCompositeFieldList); + FullCompositeFieldList(localCompositeFieldList); if (outerFullCompositeFieldList && localFullCompositeFieldList) { @@ -3301,15 +3312,15 @@ JoinOnPartitionColumn(Query *query) if (isLeftColumnPartitionColumn && isRightColumnPartitionColumn) { FieldSelect *leftCompositeField = - CompositeFieldRecursive(leftArgument, query); + CompositeFieldRecursive(leftArgument, query); FieldSelect *rightCompositeField = - CompositeFieldRecursive(rightArgument, query); + CompositeFieldRecursive(rightArgument, query); /* * If partition colums are composite fields, add them to list to * check later if all composite fields are used. */ - if(leftCompositeField && rightCompositeField) + if (leftCompositeField && rightCompositeField) { leftCompositeFieldList = lappend(leftCompositeFieldList, leftCompositeField); @@ -3318,7 +3329,7 @@ JoinOnPartitionColumn(Query *query) } /* if both sides are not composite fields, they are normal columns */ - if(!(leftCompositeField && rightCompositeField)) + if (!(leftCompositeField && rightCompositeField)) { joinOnPartitionColumn = true; break; @@ -3327,12 +3338,12 @@ JoinOnPartitionColumn(Query *query) } /* check composite fields */ - if(!joinOnPartitionColumn) + if (!joinOnPartitionColumn) { bool leftFullCompositeFieldList = - FullCompositeFieldList(leftCompositeFieldList); + FullCompositeFieldList(leftCompositeFieldList); bool rightFullCompositeFieldList = - FullCompositeFieldList(rightCompositeFieldList); + FullCompositeFieldList(rightCompositeFieldList); if (leftFullCompositeFieldList && rightFullCompositeFieldList) { @@ -3409,7 +3420,7 @@ ErrorIfUnsupportedShardDistribution(Query *query) /* check if this table has 1-1 shard partitioning with first table */ coPartitionedTables = CoPartitionedTables(firstShardIntervalList, - currentShardIntervalList); + currentShardIntervalList); if (!coPartitionedTables) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -3437,7 +3448,7 @@ RelationIdList(Query *query) foreach(tableEntryCell, tableEntryList) { - TableEntry *tableEntry = (TableEntry *) lfirst(tableEntryCell); + TableEntry *tableEntry = (TableEntry *) lfirst(tableEntryCell); Oid relationId = tableEntry->relationId; relationIdList = list_append_unique_oid(relationIdList, relationId); @@ -3617,7 +3628,7 @@ ExtractQueryWalker(Node *node, List **queryList) Query *query = (Query *) node; (*queryList) = lappend(*queryList, query); - walkerResult = query_tree_walker(query, ExtractQueryWalker, queryList, + walkerResult = query_tree_walker(query, ExtractQueryWalker, queryList, QTW_EXAMINE_RTES); } @@ -3641,7 +3652,7 @@ LeafQuery(Query *queryTree) * Extract all range table indexes from the join tree. Note that sub-queries * that get pulled up by PostgreSQL don't appear in this join tree. */ - ExtractRangeTableIndexWalker((Node*) queryTree->jointree, &joinTreeTableIndexList); + ExtractRangeTableIndexWalker((Node *) queryTree->jointree, &joinTreeTableIndexList); foreach(joinTreeTableIndexCell, joinTreeTableIndexList) { /* @@ -3725,7 +3736,7 @@ PartitionColumnOpExpressionList(Query *query) } else if (IsA(leftArgument, Const) && IsA(leftArgument, Var)) { - candidatePartitionColumn = (Var *) rightArgument; + candidatePartitionColumn = (Var *) rightArgument; } else { diff --git a/src/backend/distributed/planner/multi_logical_planner.c b/src/backend/distributed/planner/multi_logical_planner.c index 8f866c074..a9edcfc39 100644 --- a/src/backend/distributed/planner/multi_logical_planner.c +++ b/src/backend/distributed/planner/multi_logical_planner.c @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * multi_logical_planner.c - * + * * Routines for constructing a logical plan tree from the given Query tree * structure. This new logical plan is based on multi-relational algebra rules. * @@ -39,11 +39,11 @@ bool SubqueryPushdown = false; /* is subquery pushdown enabled */ /* Function pointer type definition for apply join rule functions */ -typedef MultiNode * (*RuleApplyFunction) (MultiNode *leftNode, MultiNode *rightNode, - Var *partitionColumn, JoinType joinType, - List *joinClauses); +typedef MultiNode *(*RuleApplyFunction) (MultiNode *leftNode, MultiNode *rightNode, + Var *partitionColumn, JoinType joinType, + List *joinClauses); -static RuleApplyFunction RuleApplyFunctionArray[JOIN_RULE_LAST] = {0}; /* join rules */ +static RuleApplyFunction RuleApplyFunctionArray[JOIN_RULE_LAST] = { 0 }; /* join rules */ /* Local functions forward declarations */ static MultiNode * MultiPlanTree(Query *queryTree); @@ -157,7 +157,7 @@ SubqueryEntryList(Query *queryTree) * only walk over range table entries at this level and do not recurse into * subqueries. */ - ExtractRangeTableIndexWalker((Node*) queryTree->jointree, &joinTreeTableIndexList); + ExtractRangeTableIndexWalker((Node *) queryTree->jointree, &joinTreeTableIndexList); foreach(joinTreeTableIndexCell, joinTreeTableIndexList) { /* @@ -285,6 +285,7 @@ MultiPlanTree(Query *queryTree) else { bool hasOuterJoin = false; + /* * We calculate the join order using the list of tables in the query and * the join clauses between them. Note that this function owns the table @@ -465,6 +466,7 @@ ErrorIfQueryNotSupported(Query *queryTree) #if (PG_VERSION_NUM >= 90500) + /* HasTablesample returns tree if the query contains tablesample */ static bool HasTablesample(Query *queryTree) @@ -485,6 +487,8 @@ HasTablesample(Query *queryTree) return hasTablesample; } + + #endif @@ -529,7 +533,8 @@ HasUnsupportedJoinWalker(Node *node, void *context) * ErrorIfSubqueryNotSupported checks that we can perform distributed planning for * the given subquery. */ -static void ErrorIfSubqueryNotSupported(Query *subqueryTree) +static void +ErrorIfSubqueryNotSupported(Query *subqueryTree) { char *errorDetail = NULL; bool preconditionsSatisfied = true; @@ -587,7 +592,6 @@ HasOuterJoin(Query *queryTree) static bool HasOuterJoinWalker(Node *node, void *context) { - bool hasOuterJoin = false; if (node == NULL) { @@ -657,7 +661,7 @@ HasComplexRangeTableType(Query *queryTree) * Extract all range table indexes from the join tree. Note that sub-queries * that get pulled up by PostgreSQL don't appear in this join tree. */ - ExtractRangeTableIndexWalker((Node*) queryTree->jointree, &joinTreeTableIndexList); + ExtractRangeTableIndexWalker((Node *) queryTree->jointree, &joinTreeTableIndexList); foreach(joinTreeTableIndexCell, joinTreeTableIndexList) { /* @@ -675,7 +679,7 @@ HasComplexRangeTableType(Query *queryTree) * subquery. */ if (rangeTableEntry->rtekind != RTE_RELATION && - rangeTableEntry->rtekind != RTE_SUBQUERY) + rangeTableEntry->rtekind != RTE_SUBQUERY) { hasComplexRangeTableType = true; } @@ -966,7 +970,7 @@ TableEntryList(List *rangeTableList) foreach(rangeTableCell, rangeTableList) { - RangeTblEntry *rangeTableEntry = (RangeTblEntry *) lfirst(rangeTableCell); + RangeTblEntry *rangeTableEntry = (RangeTblEntry *) lfirst(rangeTableCell); if (rangeTableEntry->rtekind == RTE_RELATION) { @@ -1178,8 +1182,8 @@ IsSelectClause(Node *clause) /* we currently consider the following nodes as select clauses */ NodeTag nodeTag = nodeTag(clause); - if ( !(nodeTag == T_OpExpr || nodeTag == T_ScalarArrayOpExpr || - nodeTag == T_NullTest || nodeTag == T_BooleanTest) ) + if (!(nodeTag == T_OpExpr || nodeTag == T_ScalarArrayOpExpr || + nodeTag == T_NullTest || nodeTag == T_BooleanTest)) { return false; } @@ -1317,9 +1321,9 @@ UnaryOperator(MultiNode *node) { bool unaryOperator = false; - if (CitusIsA(node, MultiTreeRoot) || CitusIsA(node, MultiTable) || - CitusIsA(node, MultiCollect) || CitusIsA(node, MultiSelect) || - CitusIsA(node, MultiProject) || CitusIsA(node, MultiPartition) || + if (CitusIsA(node, MultiTreeRoot) || CitusIsA(node, MultiTable) || + CitusIsA(node, MultiCollect) || CitusIsA(node, MultiSelect) || + CitusIsA(node, MultiProject) || CitusIsA(node, MultiPartition) || CitusIsA(node, MultiExtendedOp)) { unaryOperator = true; @@ -1403,7 +1407,7 @@ FindNodesOfType(MultiNode *node, int type) } else if (BinaryOperator(node)) { - MultiNode *leftChildNode = ((MultiBinaryNode *) node)->leftChildNode; + MultiNode *leftChildNode = ((MultiBinaryNode *) node)->leftChildNode; MultiNode *rightChildNode = ((MultiBinaryNode *) node)->rightChildNode; List *leftChildNodeList = FindNodesOfType(leftChildNode, type); @@ -1533,9 +1537,9 @@ ExtractRangeTableEntryWalker(Node *node, List **rangeTableList) List * pull_var_clause_default(Node *node) { - List *columnList = pull_var_clause(node, PVC_RECURSE_AGGREGATES, - PVC_REJECT_PLACEHOLDERS); - return columnList; + List *columnList = pull_var_clause(node, PVC_RECURSE_AGGREGATES, + PVC_REJECT_PLACEHOLDERS); + return columnList; } @@ -1552,7 +1556,7 @@ ApplyJoinRule(MultiNode *leftNode, MultiNode *rightNode, JoinRuleType ruleType, MultiNode *multiNode = NULL; List *applicableJoinClauses = NIL; - List *leftTableIdList = OutputTableIdList(leftNode); + List *leftTableIdList = OutputTableIdList(leftNode); List *rightTableIdList = OutputTableIdList(rightNode); int rightTableIdCount = 0; uint32 rightTableId = 0; @@ -1567,8 +1571,8 @@ ApplyJoinRule(MultiNode *leftNode, MultiNode *rightNode, JoinRuleType ruleType, /* call the join rule application function to create the new join node */ ruleApplyFunction = JoinRuleApplyFunction(ruleType); - multiNode = (*ruleApplyFunction) (leftNode, rightNode, partitionColumn, - joinType, applicableJoinClauses); + multiNode = (*ruleApplyFunction)(leftNode, rightNode, partitionColumn, + joinType, applicableJoinClauses); if (joinType != JOIN_INNER && CitusIsA(multiNode, MultiJoin)) { @@ -1918,7 +1922,7 @@ ErrorIfSubqueryJoin(Query *queryTree) * Extract all range table indexes from the join tree. Note that sub-queries * that get pulled up by PostgreSQL don't appear in this join tree. */ - ExtractRangeTableIndexWalker((Node*) queryTree->jointree, &joinTreeTableIndexList); + ExtractRangeTableIndexWalker((Node *) queryTree->jointree, &joinTreeTableIndexList); joiningRangeTableCount = list_length(joinTreeTableIndexList); if (joiningRangeTableCount > 1) diff --git a/src/backend/distributed/planner/multi_master_planner.c b/src/backend/distributed/planner/multi_master_planner.c index 88fda53a3..b328e2e7c 100644 --- a/src/backend/distributed/planner/multi_master_planner.c +++ b/src/backend/distributed/planner/multi_master_planner.c @@ -122,7 +122,7 @@ BuildAggregatePlan(Query *masterQuery, Plan *subPlan) AggStrategy aggregateStrategy = AGG_PLAIN; AggClauseCosts aggregateCosts; AttrNumber *groupColumnIdArray = NULL; - List *aggregateTargetList = NIL; + List *aggregateTargetList = NIL; List *groupColumnList = NIL; List *columnList = NIL; ListCell *columnCell = NULL; @@ -168,13 +168,13 @@ BuildAggregatePlan(Query *masterQuery, Plan *subPlan) /* finally create the plan */ #if (PG_VERSION_NUM >= 90500) - aggregatePlan = make_agg(NULL, aggregateTargetList, NIL, aggregateStrategy, - &aggregateCosts, groupColumnCount, groupColumnIdArray, - groupColumnOpArray, NIL, rowEstimate, subPlan); + aggregatePlan = make_agg(NULL, aggregateTargetList, NIL, aggregateStrategy, + &aggregateCosts, groupColumnCount, groupColumnIdArray, + groupColumnOpArray, NIL, rowEstimate, subPlan); #else - aggregatePlan = make_agg(NULL, aggregateTargetList, NIL, aggregateStrategy, - &aggregateCosts, groupColumnCount, groupColumnIdArray, - groupColumnOpArray, rowEstimate, subPlan); + aggregatePlan = make_agg(NULL, aggregateTargetList, NIL, aggregateStrategy, + &aggregateCosts, groupColumnCount, groupColumnIdArray, + groupColumnOpArray, rowEstimate, subPlan); #endif return aggregatePlan; @@ -211,7 +211,7 @@ BuildSelectStatement(Query *masterQuery, char *masterTableName, rangeTableEntry = copyObject(queryRangeTableEntry); rangeTableEntry->rtekind = RTE_RELATION; rangeTableEntry->eref = makeAlias(masterTableName, NIL); - rangeTableEntry->relid = 0; /* to be filled in exec_Start */ + rangeTableEntry->relid = 0; /* to be filled in exec_Start */ rangeTableEntry->inh = false; rangeTableEntry->inFromCl = true; @@ -220,7 +220,7 @@ BuildSelectStatement(Query *masterQuery, char *masterTableName, /* (2) build and initialize sequential scan node */ sequentialScan = makeNode(SeqScan); - sequentialScan->scanrelid = 1; /* always one */ + sequentialScan->scanrelid = 1; /* always one */ /* (3) add an aggregation plan if needed */ if (masterQuery->hasAggs || masterQuery->groupClause) diff --git a/src/backend/distributed/planner/multi_physical_planner.c b/src/backend/distributed/planner/multi_physical_planner.c index 5e497970e..b34c36b4c 100644 --- a/src/backend/distributed/planner/multi_physical_planner.c +++ b/src/backend/distributed/planner/multi_physical_planner.c @@ -138,7 +138,7 @@ static OpExpr * MakeOpExpressionWithZeroConst(void); static List * BuildRestrictInfoList(List *qualList); static List * FragmentCombinationList(List *rangeTableFragmentsList, Query *jobQuery, List *dependedJobList); -static JoinSequenceNode * JoinSequenceArray(List * rangeTableFragmentsList, +static JoinSequenceNode * JoinSequenceArray(List *rangeTableFragmentsList, Query *jobQuery, List *dependedJobList); static bool PartitionedOnColumn(Var *column, List *rangeTableList, List *dependedJobList); static void CheckJoinBetweenColumns(OpExpr *joinClause); @@ -155,7 +155,8 @@ static StringInfo DatumArrayString(Datum *datumArray, uint32 datumCount, Oid dat static Task * CreateBasicTask(uint64 jobId, uint32 taskId, TaskType taskType, char *queryString); static void UpdateRangeTableAlias(List *rangeTableList, List *fragmentList); -static Alias * FragmentAlias(RangeTblEntry *rangeTableEntry, RangeTableFragment *fragment); +static Alias * FragmentAlias(RangeTblEntry *rangeTableEntry, + RangeTableFragment *fragment); static uint64 AnchorShardId(List *fragmentList, uint32 anchorRangeTableId); static List * PruneSqlTaskDependencies(List *sqlTaskList); static List * AssignTaskList(List *sqlTaskList); @@ -167,7 +168,7 @@ static Task * GreedyAssignTask(WorkerNode *workerNode, List *taskList, static List * RoundRobinAssignTaskList(List *taskList); static List * RoundRobinReorder(Task *task, List *placementList); static List * ReorderAndAssignTaskList(List *taskList, - List * (*reorderFunction) (Task *, List *)); + List * (*reorderFunction)(Task *, List *)); static int CompareTasksByShardId(const void *leftElement, const void *rightElement); static List * ActiveShardPlacementLists(List *taskList); static List * ActivePlacementList(List *placementList); @@ -309,6 +310,7 @@ BuildJobTree(MultiTreeRoot *multiTree) partitionKey, partitionType, baseRelationId, JOIN_MAP_MERGE_JOB); + /* reset depended job list */ loopDependedJobList = NIL; loopDependedJobList = list_make1(mapMergeJob); @@ -538,7 +540,7 @@ BuildJobQuery(MultiNode *multiNode, List *dependedJobList) * If we are building this query on a repartitioned subquery job then we * don't need to update column attributes. */ - if(dependedJobList != NIL) + if (dependedJobList != NIL) { Job *job = (Job *) linitial(dependedJobList); if (CitusIsA(job, MapMergeJob)) @@ -628,10 +630,10 @@ BuildJobQuery(MultiNode *multiNode, List *dependedJobList) jobQuery->rtable = rangeTableList; jobQuery->targetList = targetList; jobQuery->jointree = joinTree; - jobQuery->sortClause = sortClauseList; + jobQuery->sortClause = sortClauseList; jobQuery->groupClause = groupClauseList; jobQuery->limitOffset = limitOffset; - jobQuery->limitCount = limitCount; + jobQuery->limitCount = limitCount; jobQuery->hasAggs = contain_agg_clause((Node *) targetList); return jobQuery; @@ -718,10 +720,10 @@ BuildReduceQuery(MultiExtendedOp *extendedOpNode, List *dependedJobList) reduceQuery->rtable = derivedRangeTableList; reduceQuery->targetList = targetList; reduceQuery->jointree = joinTree; - reduceQuery->sortClause = extendedOpNode->sortClauseList; + reduceQuery->sortClause = extendedOpNode->sortClauseList; reduceQuery->groupClause = extendedOpNode->groupClauseList; reduceQuery->limitOffset = extendedOpNode->limitOffset; - reduceQuery->limitCount = extendedOpNode->limitCount; + reduceQuery->limitCount = extendedOpNode->limitCount; reduceQuery->hasAggs = contain_agg_clause((Node *) targetList); return reduceQuery; @@ -754,7 +756,7 @@ BaseRangeTableList(MultiNode *multiNode) */ MultiTable *multiTable = (MultiTable *) multiNode; if (multiTable->relationId != SUBQUERY_RELATION_ID && - multiTable->relationId != HEAP_ANALYTICS_SUBQUERY_RELATION_ID) + multiTable->relationId != HEAP_ANALYTICS_SUBQUERY_RELATION_ID) { RangeTblEntry *rangeTableEntry = makeNode(RangeTblEntry); rangeTableEntry->inFromCl = true; @@ -870,7 +872,7 @@ TargetEntryList(List *expressionList) Expr *expression = (Expr *) lfirst(expressionCell); TargetEntry *targetEntry = makeTargetEntry(expression, - list_length(targetEntryList)+1, + list_length(targetEntryList) + 1, NULL, false); targetEntryList = lappend(targetEntryList, targetEntry); } @@ -1044,7 +1046,7 @@ QueryJoinTree(MultiNode *multiNode, List *dependedJobList, List **rangeTableList /* fix the column attributes in ON (...) clauses */ columnList = pull_var_clause_default((Node *) joinNode->joinClauseList); - foreach (columnCell, columnList) + foreach(columnCell, columnList) { Var *column = (Var *) lfirst(columnCell); UpdateColumnAttributes(column, *rangeTableList, dependedJobList); @@ -1093,7 +1095,8 @@ QueryJoinTree(MultiNode *multiNode, List *dependedJobList, List **rangeTableList uint32 columnCount = (uint32) list_length(dependedTargetList); List *columnNameList = DerivedColumnNameList(columnCount, dependedJob->jobId); - RangeTblEntry *rangeTableEntry = DerivedRangeTableEntry(multiNode, columnNameList, + RangeTblEntry *rangeTableEntry = DerivedRangeTableEntry(multiNode, + columnNameList, tableIdList); RangeTblRef *rangeTableRef = makeNode(RangeTblRef); @@ -1246,7 +1249,7 @@ ExtractColumns(RangeTblEntry *rangeTableEntry, int rangeTableId, List *dependedJ else if (rangeTableKind == CITUS_RTE_RELATION) { /* - * For distributed tables, we construct a regular table RTE to call + * For distributed tables, we construct a regular table RTE to call * expandRTE, which will extract columns from the distributed table * schema. */ @@ -1405,10 +1408,10 @@ BuildSubqueryJobQuery(MultiNode *multiNode) jobQuery->rtable = rangeTableList; jobQuery->targetList = targetList; jobQuery->jointree = joinTree; - jobQuery->sortClause = sortClauseList; + jobQuery->sortClause = sortClauseList; jobQuery->groupClause = groupClauseList; jobQuery->limitOffset = limitOffset; - jobQuery->limitCount = limitCount; + jobQuery->limitCount = limitCount; jobQuery->hasAggs = contain_agg_clause((Node *) targetList); return jobQuery; @@ -1646,7 +1649,7 @@ static uint64 UniqueJobId(void) { text *sequenceName = cstring_to_text(JOBID_SEQUENCE_NAME); - Oid sequenceId = ResolveRelationId(sequenceName); + Oid sequenceId = ResolveRelationId(sequenceName); Datum sequenceIdDatum = ObjectIdGetDatum(sequenceId); /* generate new and unique jobId from sequence */ @@ -1747,7 +1750,7 @@ HashPartitionCount(void) uint32 nodeCount = WorkerGetLiveNodeCount(); double maxReduceTasksPerNode = MaxRunningTasksPerNode / 2.0; - uint32 partitionCount = (uint32) rint(nodeCount * maxReduceTasksPerNode); + uint32 partitionCount = (uint32) rint(nodeCount * maxReduceTasksPerNode); return partitionCount; } @@ -1864,8 +1867,9 @@ SplitPointObject(ShardInterval **shardIntervalArray, uint32 shardIntervalCount) return splitPointObject; } + /* ------------------------------------------------------------ - * Functions that relate to building and assigning tasks follow + * Functions that relate to building and assigning tasks follow * ------------------------------------------------------------ */ @@ -1986,7 +1990,7 @@ SubquerySqlTaskList(Job *job) ListCell *rangeTableCell = NULL; ListCell *queryCell = NULL; Node *whereClauseTree = NULL; - uint32 taskIdIndex = 1; /* 0 is reserved for invalid taskId */ + uint32 taskIdIndex = 1; /* 0 is reserved for invalid taskId */ uint32 anchorRangeTableId = 0; uint32 rangeTableIndex = 0; const uint32 fragmentSize = sizeof(RangeTableFragment); @@ -2036,10 +2040,10 @@ SubquerySqlTaskList(Job *job) if (opExpressionList != NIL) { Var *partitionColumn = PartitionColumn(relationId, tableId); - List *whereClauseList = ReplaceColumnsInOpExpressionList(opExpressionList, + List *whereClauseList = ReplaceColumnsInOpExpressionList(opExpressionList, partitionColumn); finalShardIntervalList = PruneShardList(relationId, tableId, whereClauseList, - shardIntervalList); + shardIntervalList); } else { @@ -2146,7 +2150,7 @@ static List * SqlTaskList(Job *job) { List *sqlTaskList = NIL; - uint32 taskIdIndex = 1; /* 0 is reserved for invalid taskId */ + uint32 taskIdIndex = 1; /* 0 is reserved for invalid taskId */ uint64 jobId = job->jobId; bool anchorRangeTableBasedAssignment = false; uint32 anchorRangeTableId = 0; @@ -2472,8 +2476,8 @@ RangeTableFragmentsList(List *rangeTableList, List *whereClauseList, List *shardIntervalList = LoadShardIntervalList(relationId); List *prunedShardIntervalList = PruneShardList(relationId, tableId, - whereClauseList, - shardIntervalList); + whereClauseList, + shardIntervalList); /* * If we prune all shards for one table, query results will be empty. @@ -2548,7 +2552,7 @@ RangeTableFragmentsList(List *rangeTableList, List *whereClauseList, */ List * PruneShardList(Oid relationId, Index tableId, List *whereClauseList, - List *shardIntervalList) + List *shardIntervalList) { List *remainingShardList = NIL; ListCell *shardIntervalCell = NULL; @@ -2653,7 +2657,7 @@ MakeOpExpression(Var *variable, int16 strategyNumber) Oid accessMethodId = BTREE_AM_OID; Oid operatorId = InvalidOid; Oid operatorClassInputType = InvalidOid; - Const *constantValue = NULL; + Const *constantValue = NULL; OpExpr *expression = NULL; char typeType = 0; @@ -2679,7 +2683,7 @@ MakeOpExpression(Var *variable, int16 strategyNumber) /* Now make the expression with the given variable and a null constant */ expression = (OpExpr *) make_opclause(operatorId, InvalidOid, /* no result type yet */ - false, /* no return set */ + false, /* no return set */ (Expr *) variable, (Expr *) constantValue, InvalidOid, collationId); @@ -2900,7 +2904,7 @@ HashableClauseMutator(Node *originalNode, Var *partitionColumn) * If this node is not hashable, continue walking down the expression tree * to find and hash clauses which are eligible. */ - if(newNode == NULL) + if (newNode == NULL) { newNode = expression_tree_mutator(originalNode, HashableClauseMutator, (void *) partitionColumn); @@ -3045,7 +3049,7 @@ MakeInt4Constant(Datum constantValue) bool constantIsNull = false; bool constantByValue = true; - Const *int4Constant = makeConst(constantType, constantTypeMode, constantCollationId, + Const *int4Constant = makeConst(constantType, constantTypeMode, constantCollationId, constantLength, constantValue, constantIsNull, constantByValue); return int4Constant; @@ -3102,7 +3106,7 @@ UpdateConstraint(Node *baseConstraint, ShardInterval *shardInterval) Node *greaterThanExpr = (Node *) lsecond(andExpr->args); Node *minNode = get_rightop((Expr *) greaterThanExpr); /* right op */ - Node *maxNode = get_rightop((Expr *) lessThanExpr); /* right op */ + Node *maxNode = get_rightop((Expr *) lessThanExpr); /* right op */ Const *minConstant = NULL; Const *maxConstant = NULL; @@ -3273,7 +3277,7 @@ JoinSequenceArray(List *rangeTableFragmentsList, Query *jobQuery, List *depended joinSequenceArray[joinedTableCount].joiningRangeTableId = NON_PRUNABLE_JOIN; joinedTableCount++; - foreach (joinExprCell, joinExprList) + foreach(joinExprCell, joinExprList) { JoinExpr *joinExpr = (JoinExpr *) lfirst(joinExprCell); JoinType joinType = joinExpr->jointype; @@ -3347,7 +3351,7 @@ JoinSequenceArray(List *rangeTableFragmentsList, Query *jobQuery, List *depended if (IS_OUTER_JOIN(joinType)) { int innerRangeTableId = 0; - List * tableFragments = NIL; + List *tableFragments = NIL; int fragmentCount = 0; if (joinType == JOIN_RIGHT) @@ -3500,7 +3504,7 @@ FindRangeTableFragmentsList(List *rangeTableFragmentsList, int tableId) if (tableFragments != NIL) { RangeTableFragment *tableFragment = - (RangeTableFragment*) linitial(tableFragments); + (RangeTableFragment *) linitial(tableFragments); if (tableFragment->rangeTableId == tableId) { foundTableFragments = tableFragments; @@ -3706,7 +3710,7 @@ UniqueFragmentList(List *fragmentList) foreach(uniqueFragmentCell, uniqueFragmentList) { RangeTableFragment *uniqueFragment = - (RangeTableFragment *) lfirst(uniqueFragmentCell); + (RangeTableFragment *) lfirst(uniqueFragmentCell); uint64 *uniqueShardId = uniqueFragment->fragmentReference; if (*shardId == *uniqueShardId) @@ -4046,6 +4050,7 @@ FragmentAlias(RangeTblEntry *rangeTableEntry, RangeTableFragment *fragment) return alias; } + /* * AnchorShardId walks over each fragment in the given fragment list, finds the * fragment that corresponds to the given anchor range tableId, and returns this @@ -4360,7 +4365,7 @@ MergeTaskList(MapMergeJob *mapMergeJob, List *mapTaskList, uint32 taskIdIndex) StringInfo intermediateTableQueryString = IntermediateTableQueryString(jobId, taskIdIndex, reduceQuery); - StringInfo mergeAndRunQueryString= makeStringInfo(); + StringInfo mergeAndRunQueryString = makeStringInfo(); appendStringInfo(mergeAndRunQueryString, MERGE_FILES_AND_RUN_QUERY_COMMAND, jobId, taskIdIndex, mergeTableQueryString->data, intermediateTableQueryString->data); @@ -4686,7 +4691,7 @@ TaskListAppendUnique(List *list, Task *task) List * TaskListConcatUnique(List *list1, List *list2) { - ListCell *taskCell = NULL; + ListCell *taskCell = NULL; foreach(taskCell, list2) { @@ -4960,7 +4965,7 @@ List * FirstReplicaAssignTaskList(List *taskList) { /* No additional reordering need take place for this algorithm */ - List * (*reorderFunction)(Task *, List *) = NULL; + List *(*reorderFunction)(Task *, List *) = NULL; taskList = ReorderAndAssignTaskList(taskList, reorderFunction); @@ -4984,6 +4989,7 @@ RoundRobinAssignTaskList(List *taskList) return taskList; } + /* * RoundRobinReorder implements the core of the round-robin assignment policy. * It takes a task and placement list and rotates a copy of the placement list @@ -5116,7 +5122,8 @@ ActiveShardPlacementLists(List *taskList) List *activeShardPlacementList = ActivePlacementList(shardPlacementList); /* sort shard placements by their insertion time */ - activeShardPlacementList = SortList(activeShardPlacementList, CompareShardPlacements); + activeShardPlacementList = SortList(activeShardPlacementList, + CompareShardPlacements); shardPlacementLists = lappend(shardPlacementLists, activeShardPlacementList); } @@ -5257,7 +5264,8 @@ AssignDualHashTaskList(List *taskList) uint32 replicaIndex = 0; for (replicaIndex = 0; replicaIndex < ShardReplicationFactor; replicaIndex++) { - uint32 assignmentOffset = beginningNodeIndex + assignedTaskIndex + replicaIndex; + uint32 assignmentOffset = beginningNodeIndex + assignedTaskIndex + + replicaIndex; uint32 assignmentIndex = assignmentOffset % workerNodeCount; WorkerNode *workerNode = list_nth(workerNodeList, assignmentIndex); diff --git a/src/backend/distributed/relay/relay_event_utility.c b/src/backend/distributed/relay/relay_event_utility.c index 54f7a09a5..ce02ef9a0 100644 --- a/src/backend/distributed/relay/relay_event_utility.c +++ b/src/backend/distributed/relay/relay_event_utility.c @@ -35,7 +35,7 @@ /* Local functions forward declarations */ static bool TypeAddIndexConstraint(const AlterTableCmd *command); -static bool TypeDropIndexConstraint(const AlterTableCmd *command, +static bool TypeDropIndexConstraint(const AlterTableCmd *command, const RangeVar *relation, uint64 shardId); static void AppendShardIdToConstraintName(AlterTableCmd *command, uint64 shardId); @@ -67,7 +67,7 @@ RelayEventExtendNames(Node *parseTree, uint64 shardId) AppendShardIdToName(sequenceName, shardId); break; } - + case T_AlterTableStmt: { /* @@ -79,7 +79,7 @@ RelayEventExtendNames(Node *parseTree, uint64 shardId) AlterTableStmt *alterTableStmt = (AlterTableStmt *) parseTree; char **relationName = &(alterTableStmt->relation->relname); - RangeVar *relation = alterTableStmt->relation; /* for constraints */ + RangeVar *relation = alterTableStmt->relation; /* for constraints */ List *commandList = alterTableStmt->cmds; ListCell *commandCell = NULL; @@ -179,15 +179,15 @@ RelayEventExtendNames(Node *parseTree, uint64 shardId) objectType == OBJECT_INDEX || objectType == OBJECT_FOREIGN_TABLE || objectType == OBJECT_FOREIGN_SERVER) { - List *relationNameList = NULL; - int relationNameListLength = 0; + List *relationNameList = NULL; + int relationNameListLength = 0; Value *relationNameValue = NULL; - char **relationName = NULL; + char **relationName = NULL; uint32 dropCount = list_length(dropStmt->objects); if (dropCount > 1) { - ereport(ERROR, + ereport(ERROR, (errmsg("cannot extend name for multiple drop objects"))); } @@ -205,19 +205,30 @@ RelayEventExtendNames(Node *parseTree, uint64 shardId) switch (relationNameListLength) { case 1: + { relationNameValue = linitial(relationNameList); break; + } + case 2: + { relationNameValue = lsecond(relationNameList); break; + } + case 3: + { relationNameValue = lthird(relationNameList); break; + } + default: + { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("improper relation name: \"%s\"", NameListToString(relationNameList)))); break; + } } relationName = &(relationNameValue->val.str); @@ -304,7 +315,7 @@ RelayEventExtendNames(Node *parseTree, uint64 shardId) { RenameStmt *renameStmt = (RenameStmt *) parseTree; ObjectType objectType = renameStmt->renameType; - + if (objectType == OBJECT_TABLE || objectType == OBJECT_SEQUENCE || objectType == OBJECT_INDEX) { @@ -335,7 +346,7 @@ RelayEventExtendNames(Node *parseTree, uint64 shardId) * We currently do not support truncate statements. This is * primarily because truncates allow implicit modifications to * sequences through table column dependencies. As we have not - * determined our dependency model for sequences, we error here. + * determined our dependency model for sequences, we error here. */ ereport(ERROR, (errmsg("cannot extend name for truncate statement"))); break; @@ -384,18 +395,18 @@ TypeAddIndexConstraint(const AlterTableCmd *command) * associated with an index. */ static bool -TypeDropIndexConstraint(const AlterTableCmd *command, +TypeDropIndexConstraint(const AlterTableCmd *command, const RangeVar *relation, uint64 shardId) { Relation pgConstraint = NULL; SysScanDesc scanDescriptor = NULL; - ScanKeyData scanKey[1]; + ScanKeyData scanKey[1]; int scanKeyCount = 1; HeapTuple heapTuple = NULL; char *searchedConstraintName = NULL; - bool indexConstraint = false; - Oid relationId = InvalidOid; + bool indexConstraint = false; + Oid relationId = InvalidOid; bool failOK = true; if (command->subtype != AT_DropConstraint) @@ -423,8 +434,8 @@ TypeDropIndexConstraint(const AlterTableCmd *command, ScanKeyInit(&scanKey[0], Anum_pg_constraint_conrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relationId)); - - scanDescriptor = systable_beginscan(pgConstraint, + + scanDescriptor = systable_beginscan(pgConstraint, ConstraintRelidIndexId, true, /* indexOK */ NULL, scanKeyCount, scanKey); @@ -433,7 +444,7 @@ TypeDropIndexConstraint(const AlterTableCmd *command, { Form_pg_constraint constraintForm = (Form_pg_constraint) GETSTRUCT(heapTuple); char *constraintName = NameStr(constraintForm->conname); - + if (strncmp(constraintName, searchedConstraintName, NAMEDATALEN) == 0) { /* we found the constraint, now check if it is for an index */ @@ -442,7 +453,7 @@ TypeDropIndexConstraint(const AlterTableCmd *command, { indexConstraint = true; } - + break; } @@ -451,7 +462,7 @@ TypeDropIndexConstraint(const AlterTableCmd *command, systable_endscan(scanDescriptor); heap_close(pgConstraint, AccessShareLock); - + pfree(searchedConstraintName); return indexConstraint; @@ -489,10 +500,10 @@ AppendShardIdToConstraintName(AlterTableCmd *command, uint64 shardId) void AppendShardIdToName(char **name, uint64 shardId) { - char extendedName[NAMEDATALEN]; + char extendedName[NAMEDATALEN]; uint32 extendedNameLength = 0; - snprintf(extendedName, NAMEDATALEN, "%s%c" UINT64_FORMAT, + snprintf(extendedName, NAMEDATALEN, "%s%c" UINT64_FORMAT, (*name), SHARD_NAME_SEPARATOR, shardId); /* diff --git a/src/backend/distributed/shared_library_init.c b/src/backend/distributed/shared_library_init.c index 344665eb0..8402ab0cf 100644 --- a/src/backend/distributed/shared_library_init.c +++ b/src/backend/distributed/shared_library_init.c @@ -48,23 +48,23 @@ static void NormalizeWorkerListPath(void); /* GUC enum definitions */ static const struct config_enum_entry task_assignment_policy_options[] = { - {"greedy", TASK_ASSIGNMENT_GREEDY, false}, - {"first-replica", TASK_ASSIGNMENT_FIRST_REPLICA, false}, - {"round-robin", TASK_ASSIGNMENT_ROUND_ROBIN, false}, - {NULL, 0, false} + { "greedy", TASK_ASSIGNMENT_GREEDY, false }, + { "first-replica", TASK_ASSIGNMENT_FIRST_REPLICA, false }, + { "round-robin", TASK_ASSIGNMENT_ROUND_ROBIN, false }, + { NULL, 0, false } }; static const struct config_enum_entry task_executor_type_options[] = { - {"real-time", MULTI_EXECUTOR_REAL_TIME, false}, - {"task-tracker", MULTI_EXECUTOR_TASK_TRACKER, false}, - {"router", MULTI_EXECUTOR_ROUTER, false}, - {NULL, 0, false} + { "real-time", MULTI_EXECUTOR_REAL_TIME, false }, + { "task-tracker", MULTI_EXECUTOR_TASK_TRACKER, false }, + { "router", MULTI_EXECUTOR_ROUTER, false }, + { NULL, 0, false } }; static const struct config_enum_entry shard_placement_policy_options[] = { - {"local-node-first", SHARD_PLACEMENT_LOCAL_NODE_FIRST, false}, - {"round-robin", SHARD_PLACEMENT_ROUND_ROBIN, false}, - {NULL, 0, false} + { "local-node-first", SHARD_PLACEMENT_LOCAL_NODE_FIRST, false }, + { "round-robin", SHARD_PLACEMENT_ROUND_ROBIN, false }, + { NULL, 0, false } }; @@ -206,9 +206,10 @@ RegisterCitusConfigVariables(void) DefineCustomBoolVariable( "citusdb.expire_cached_shards", - gettext_noop("Enables shard cache expiration if a shard's size on disk has changed. "), - gettext_noop("When appending to an existing shard, old data may still be cached on " - "other workers. This configuration entry activates automatic " + gettext_noop("Enables shard cache expiration if a shard's size on disk has " + "changed."), + gettext_noop("When appending to an existing shard, old data may still be cached " + "on other workers. This configuration entry activates automatic " "expiration, but should not be used with manual updates to shards."), &ExpireCachedShards, false, @@ -440,11 +441,11 @@ RegisterCitusConfigVariables(void) "citusdb.task_assignment_policy", gettext_noop("Sets the policy to use when assigning tasks to worker nodes."), gettext_noop("The master node assigns tasks to worker nodes based on shard " - "locations. This configuration value specifies the policy to " - "use when making these assignments. The greedy policy aims to " - "evenly distribute tasks across worker nodes, first-replica just " - "assigns tasks in the order shard placements were created, " - "and the round-robin policy assigns tasks to worker nodes in " + "locations. This configuration value specifies the policy to " + "use when making these assignments. The greedy policy aims to " + "evenly distribute tasks across worker nodes, first-replica just " + "assigns tasks in the order shard placements were created, " + "and the round-robin policy assigns tasks to worker nodes in " "a round-robin fashion."), &TaskAssignmentPolicy, TASK_ASSIGNMENT_GREEDY, @@ -488,6 +489,7 @@ RegisterCitusConfigVariables(void) /* warn about config items in the citusdb namespace that are not registered above */ EmitWarningsOnPlaceholders("citusdb"); + /* Also warn about citus namespace, as that's a very likely misspelling */ EmitWarningsOnPlaceholders("citus"); } @@ -515,8 +517,10 @@ NormalizeWorkerListPath(void) { absoluteFileName = malloc(strlen(DataDir) + strlen(WORKER_LIST_FILENAME) + 2); if (absoluteFileName == NULL) + { ereport(FATAL, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); + } sprintf(absoluteFileName, "%s/%s", DataDir, WORKER_LIST_FILENAME); } @@ -530,6 +534,7 @@ NormalizeWorkerListPath(void) "environment variable.\n", progname, ConfigFileName))); } - SetConfigOption("citusdb.worker_list_file", absoluteFileName, PGC_POSTMASTER, PGC_S_OVERRIDE); + SetConfigOption("citusdb.worker_list_file", absoluteFileName, PGC_POSTMASTER, + PGC_S_OVERRIDE); free(absoluteFileName); } diff --git a/src/backend/distributed/test/fake_fdw.c b/src/backend/distributed/test/fake_fdw.c index 883c77755..9409dde3b 100644 --- a/src/backend/distributed/test/fake_fdw.c +++ b/src/backend/distributed/test/fake_fdw.c @@ -116,9 +116,9 @@ FakeGetForeignPlan(PlannerInfo *root, RelOptInfo *baserel, Oid foreigntableid, ForeignPath *best_path, List *tlist, List *scan_clauses) #else static ForeignScan * -FakeGetForeignPlan(PlannerInfo *root, RelOptInfo *baserel, Oid foreigntableid, - ForeignPath *best_path, List *tlist, List *scan_clauses, - Plan *outer_plan) +FakeGetForeignPlan(PlannerInfo * root, RelOptInfo * baserel, Oid foreigntableid, + ForeignPath * best_path, List * tlist, List * scan_clauses, + Plan * outer_plan) #endif { Index scan_relid = baserel->relid; @@ -129,7 +129,7 @@ FakeGetForeignPlan(PlannerInfo *root, RelOptInfo *baserel, Oid foreigntableid, return make_foreignscan(tlist, scan_clauses, scan_relid, NIL, NIL); #else return make_foreignscan(tlist, scan_clauses, scan_relid, NIL, NIL, NIL, NIL, - outer_plan); + outer_plan); #endif } diff --git a/src/backend/distributed/utils/citus_nodefuncs.c b/src/backend/distributed/utils/citus_nodefuncs.c index b3858cbdf..e5abb1d60 100644 --- a/src/backend/distributed/utils/citus_nodefuncs.c +++ b/src/backend/distributed/utils/citus_nodefuncs.c @@ -265,7 +265,7 @@ GetRangeTblKind(RangeTblEntry *rte) { CitusRTEKind rteKind = CITUS_RTE_RELATION /* invalid */; - switch(rte->rtekind) + switch (rte->rtekind) { /* directly rtekind if it's not possibly an extended RTE */ case RTE_RELATION: @@ -273,9 +273,13 @@ GetRangeTblKind(RangeTblEntry *rte) case RTE_JOIN: case RTE_VALUES: case RTE_CTE: + { rteKind = (CitusRTEKind) rte->rtekind; break; + } + case RTE_FUNCTION: + { /* * Extract extra data - correct even if a plain RTE_FUNCTION, not * an extended one, ExtractRangeTblExtraData handles that case @@ -283,6 +287,7 @@ GetRangeTblKind(RangeTblEntry *rte) */ ExtractRangeTblExtraData(rte, &rteKind, NULL, NULL, NULL); break; + } } return rteKind; diff --git a/src/backend/distributed/utils/citus_ruleutils.c b/src/backend/distributed/utils/citus_ruleutils.c index ad1ac7332..5f2c4a5ef 100644 --- a/src/backend/distributed/utils/citus_ruleutils.c +++ b/src/backend/distributed/utils/citus_ruleutils.c @@ -186,7 +186,7 @@ AppendOptionListToString(StringInfo stringBuffer, List *optionList) foreach(optionCell, optionList) { - DefElem *option = (DefElem*) lfirst(optionCell); + DefElem *option = (DefElem *) lfirst(optionCell); char *optionName = option->defname; char *optionValue = defGetString(option); @@ -219,7 +219,7 @@ pg_get_tableschemadef_string(Oid tableRelationId) char relationKind = 0; TupleDesc tupleDescriptor = NULL; TupleConstr *tupleConstraints = NULL; - int attributeIndex = 0; + int attributeIndex = 0; bool firstAttributePrinted = false; AttrNumber defaultValueIndex = 0; AttrNumber constraintIndex = 0; @@ -447,21 +447,35 @@ pg_get_tablecolumnoptionsdef_string(Oid tableRelationId) switch (attributeForm->attstorage) { case 'p': + { storageName = "PLAIN"; break; + } + case 'e': + { storageName = "EXTERNAL"; break; + } + case 'm': + { storageName = "MAIN"; break; + } + case 'x': + { storageName = "EXTENDED"; break; + } + default: + { ereport(ERROR, (errmsg("unrecognized storage type: %c", attributeForm->attstorage))); break; + } } appendStringInfo(&statement, "ALTER COLUMN %s ", diff --git a/src/backend/distributed/utils/metadata_cache.c b/src/backend/distributed/utils/metadata_cache.c index 868b11313..61127371f 100644 --- a/src/backend/distributed/utils/metadata_cache.c +++ b/src/backend/distributed/utils/metadata_cache.c @@ -51,10 +51,10 @@ static void InvalidateDistRelationCacheCallback(Datum argument, Oid relationId); static HeapTuple LookupDistPartitionTuple(Oid relationId); static List * LookupDistShardTuples(Oid relationId); static void GetPartitionTypeInputInfo(char *partitionKeyString, char partitionMethod, - Oid *intervalTypeId, int32 *intervalTypeMod); + Oid *intervalTypeId, int32 *intervalTypeMod); static ShardInterval * TupleToShardInterval(HeapTuple heapTuple, - TupleDesc tupleDescriptor, Oid intervalTypeId, - int32 intervalTypeMod); + TupleDesc tupleDescriptor, Oid intervalTypeId, + int32 intervalTypeMod); static void CachedRelationLookup(const char *relationName, Oid *cachedOid); @@ -87,6 +87,7 @@ IsDistributedTable(Oid relationId) return cacheEntry->isDistributedTable; } + /* * LoadShardInterval reads shard metadata for given shardId from pg_dist_shard, * and converts min/max values in these metadata to their properly typed datum @@ -98,7 +99,7 @@ LoadShardInterval(uint64 shardId) { ShardInterval *shardInterval; SysScanDesc scanDescriptor = NULL; - ScanKeyData scanKey[1]; + ScanKeyData scanKey[1]; int scanKeyCount = 1; HeapTuple heapTuple = NULL; Form_pg_dist_shard shardForm = NULL; @@ -127,11 +128,11 @@ LoadShardInterval(uint64 shardId) partitionEntry = DistributedTableCacheEntry(shardForm->logicalrelid); GetPartitionTypeInputInfo(partitionEntry->partitionKeyString, - partitionEntry->partitionMethod, &intervalTypeId, - &intervalTypeMod); + partitionEntry->partitionMethod, &intervalTypeId, + &intervalTypeMod); shardInterval = TupleToShardInterval(heapTuple, tupleDescriptor, intervalTypeId, - intervalTypeMod); + intervalTypeMod); systable_endscan(scanDescriptor); heap_close(pgDistShard, AccessShareLock); @@ -139,6 +140,7 @@ LoadShardInterval(uint64 shardId) return shardInterval; } + /* * DistributedTableCacheEntry looks up a pg_dist_partition entry for a * relation. @@ -239,19 +241,19 @@ LookupDistTableCacheEntry(Oid relationId) int32 intervalTypeMod = -1; GetPartitionTypeInputInfo(partitionKeyString, partitionMethod, &intervalTypeId, - &intervalTypeMod); + &intervalTypeMod); shardIntervalArray = MemoryContextAllocZero(CacheMemoryContext, - shardIntervalArrayLength * - sizeof(ShardInterval)); + shardIntervalArrayLength * + sizeof(ShardInterval)); foreach(distShardTupleCell, distShardTupleList) { HeapTuple shardTuple = lfirst(distShardTupleCell); ShardInterval *shardInterval = TupleToShardInterval(shardTuple, - distShardTupleDesc, - intervalTypeId, - intervalTypeMod); + distShardTupleDesc, + intervalTypeId, + intervalTypeMod); MemoryContext oldContext = MemoryContextSwitchTo(CacheMemoryContext); CopyShardInterval(shardInterval, &shardIntervalArray[arrayIndex]); @@ -773,7 +775,7 @@ LookupDistShardTuples(Oid relationId) scanKey[0].sk_argument = ObjectIdGetDatum(relationId); scanDescriptor = systable_beginscan(pgDistShard, DistShardLogicalRelidIndexId(), true, - NULL, 1, scanKey); + NULL, 1, scanKey); currentShardTuple = systable_getnext(scanDescriptor); while (HeapTupleIsValid(currentShardTuple)) @@ -797,7 +799,7 @@ LookupDistShardTuples(Oid relationId) */ static void GetPartitionTypeInputInfo(char *partitionKeyString, char partitionMethod, - Oid *intervalTypeId, int32 *intervalTypeMod) + Oid *intervalTypeId, int32 *intervalTypeMod) { *intervalTypeId = InvalidOid; *intervalTypeMod = -1; @@ -826,7 +828,7 @@ GetPartitionTypeInputInfo(char *partitionKeyString, char partitionMethod, { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("unsupported table partition type: %c", - partitionMethod))); + partitionMethod))); } } } @@ -838,7 +840,7 @@ GetPartitionTypeInputInfo(char *partitionKeyString, char partitionMethod, */ static ShardInterval * TupleToShardInterval(HeapTuple heapTuple, TupleDesc tupleDescriptor, Oid intervalTypeId, - int32 intervalTypeMod) + int32 intervalTypeMod) { ShardInterval *shardInterval = NULL; bool isNull = false; @@ -847,16 +849,16 @@ TupleToShardInterval(HeapTuple heapTuple, TupleDesc tupleDescriptor, Oid interva Oid inputFunctionId = InvalidOid; Oid typeIoParam = InvalidOid; Datum relationIdDatum = heap_getattr(heapTuple, Anum_pg_dist_shard_logicalrelid, - tupleDescriptor, &isNull); + tupleDescriptor, &isNull); Datum shardIdDatum = heap_getattr(heapTuple, Anum_pg_dist_shard_shardid, - tupleDescriptor, &isNull); + tupleDescriptor, &isNull); Datum storageTypeDatum = heap_getattr(heapTuple, Anum_pg_dist_shard_shardstorage, - tupleDescriptor, &isNull); + tupleDescriptor, &isNull); Datum minValueTextDatum = heap_getattr(heapTuple, Anum_pg_dist_shard_shardminvalue, - tupleDescriptor, &minValueNull); + tupleDescriptor, &minValueNull); Datum maxValueTextDatum = heap_getattr(heapTuple, Anum_pg_dist_shard_shardmaxvalue, - tupleDescriptor, &maxValueNull); + tupleDescriptor, &maxValueNull); Oid relationId = DatumGetObjectId(relationIdDatum); int64 shardId = DatumGetInt64(shardIdDatum); @@ -877,7 +879,7 @@ TupleToShardInterval(HeapTuple heapTuple, TupleDesc tupleDescriptor, Oid interva /* TODO: move this up the call stack to avoid per-tuple invocation? */ get_type_io_data(intervalTypeId, IOFunc_input, &intervalTypeLen, &intervalByVal, - &intervalAlign, &intervalDelim, &typeIoParam, &inputFunctionId); + &intervalAlign, &intervalDelim, &typeIoParam, &inputFunctionId); /* finally convert min/max values to their actual types */ minValue = OidInputFunctionCall(inputFunctionId, minValueString, diff --git a/src/backend/distributed/utils/multi_resowner.c b/src/backend/distributed/utils/multi_resowner.c index 80aecf7e5..21d78d1aa 100644 --- a/src/backend/distributed/utils/multi_resowner.c +++ b/src/backend/distributed/utils/multi_resowner.c @@ -22,7 +22,8 @@ #include "distributed/multi_resowner.h" -typedef struct JobDirectoryEntry { +typedef struct JobDirectoryEntry +{ ResourceOwner owner; uint64 jobId; } JobDirectoryEntry; @@ -44,8 +45,8 @@ MultiResourceOwnerReleaseCallback(ResourceReleasePhase phase, bool isTopLevel, void *arg) { - int lastJobIndex = NumRegisteredJobDirectories - 1; - int jobIndex = 0; + int lastJobIndex = NumRegisteredJobDirectories - 1; + int jobIndex = 0; if (phase == RESOURCE_RELEASE_AFTER_LOCKS) { @@ -79,7 +80,7 @@ MultiResourceOwnerReleaseCallback(ResourceReleasePhase phase, void ResourceOwnerEnlargeJobDirectories(ResourceOwner owner) { - int newMax = 0; + int newMax = 0; /* ensure callback is registered */ if (!RegisteredResownerCallback) @@ -91,15 +92,17 @@ ResourceOwnerEnlargeJobDirectories(ResourceOwner owner) if (RegisteredJobDirectories == NULL) { newMax = 16; - RegisteredJobDirectories = (JobDirectoryEntry *) - MemoryContextAlloc(TopMemoryContext, newMax * sizeof(JobDirectoryEntry)); + RegisteredJobDirectories = + (JobDirectoryEntry *) MemoryContextAlloc(TopMemoryContext, + newMax * sizeof(JobDirectoryEntry)); NumAllocatedJobDirectories = newMax; } else if (NumRegisteredJobDirectories + 1 > NumAllocatedJobDirectories) { newMax = NumAllocatedJobDirectories * 2; - RegisteredJobDirectories = (JobDirectoryEntry *) - repalloc(RegisteredJobDirectories, newMax * sizeof(JobDirectoryEntry)); + RegisteredJobDirectories = + (JobDirectoryEntry *) repalloc(RegisteredJobDirectories, + newMax * sizeof(JobDirectoryEntry)); NumAllocatedJobDirectories = newMax; } } @@ -123,8 +126,8 @@ ResourceOwnerRememberJobDirectory(ResourceOwner owner, uint64 jobId) void ResourceOwnerForgetJobDirectory(ResourceOwner owner, uint64 jobId) { - int lastJobIndex = NumRegisteredJobDirectories - 1; - int jobIndex = 0; + int lastJobIndex = NumRegisteredJobDirectories - 1; + int jobIndex = 0; for (jobIndex = lastJobIndex; jobIndex >= 0; jobIndex--) { @@ -135,7 +138,8 @@ ResourceOwnerForgetJobDirectory(ResourceOwner owner, uint64 jobId) /* move all later entries one up */ while (jobIndex < lastJobIndex) { - RegisteredJobDirectories[jobIndex] = RegisteredJobDirectories[jobIndex + 1]; + RegisteredJobDirectories[jobIndex] = + RegisteredJobDirectories[jobIndex + 1]; jobIndex++; } NumRegisteredJobDirectories = lastJobIndex; diff --git a/src/backend/distributed/utils/resource_lock.c b/src/backend/distributed/utils/resource_lock.c index a2552d46b..3f1b739af 100644 --- a/src/backend/distributed/utils/resource_lock.c +++ b/src/backend/distributed/utils/resource_lock.c @@ -30,7 +30,7 @@ void LockShardDistributionMetadata(int64 shardId, LOCKMODE lockMode) { - LOCKTAG tag; + LOCKTAG tag; const bool sessionLock = false; const bool dontWait = false; @@ -64,7 +64,7 @@ LockRelationDistributionMetadata(Oid relationId, LOCKMODE lockMode) void LockShardResource(uint64 shardId, LOCKMODE lockmode) { - LOCKTAG tag; + LOCKTAG tag; const bool sessionLock = false; const bool dontWait = false; @@ -78,7 +78,7 @@ LockShardResource(uint64 shardId, LOCKMODE lockmode) void UnlockShardResource(uint64 shardId, LOCKMODE lockmode) { - LOCKTAG tag; + LOCKTAG tag; const bool sessionLock = false; SET_LOCKTAG_SHARD_RESOURCE(tag, MyDatabaseId, shardId); @@ -95,7 +95,7 @@ UnlockShardResource(uint64 shardId, LOCKMODE lockmode) void LockJobResource(uint64 jobId, LOCKMODE lockmode) { - LOCKTAG tag; + LOCKTAG tag; const bool sessionLock = false; const bool dontWait = false; @@ -109,7 +109,7 @@ LockJobResource(uint64 jobId, LOCKMODE lockmode) void UnlockJobResource(uint64 jobId, LOCKMODE lockmode) { - LOCKTAG tag; + LOCKTAG tag; const bool sessionLock = false; SET_LOCKTAG_JOB_RESOURCE(tag, MyDatabaseId, jobId); diff --git a/src/backend/distributed/worker/task_tracker.c b/src/backend/distributed/worker/task_tracker.c index b8a6bde6a..e3e7320db 100644 --- a/src/backend/distributed/worker/task_tracker.c +++ b/src/backend/distributed/worker/task_tracker.c @@ -50,7 +50,7 @@ #include "utils/memutils.h" -int TaskTrackerDelay = 200; /* process sleep interval in millisecs */ +int TaskTrackerDelay = 200; /* process sleep interval in millisecs */ int MaxRunningTasksPerNode = 16; /* max number of running tasks */ int MaxTrackedTasksPerNode = 1024; /* max number of tracked tasks */ WorkerTasksSharedStateData *WorkerTasksSharedState; /* shared memory state */ @@ -76,10 +76,10 @@ static void TrackerCleanupJobSchemas(void); static void TrackerCleanupConnections(HTAB *WorkerTasksHash); static void TrackerRegisterShutDown(HTAB *WorkerTasksHash); static void TrackerDelayLoop(void); -static List *SchedulableTaskList(HTAB *WorkerTasksHash); +static List * SchedulableTaskList(HTAB *WorkerTasksHash); static WorkerTask * SchedulableTaskPriorityQueue(HTAB *WorkerTasksHash); static uint32 CountTasksMatchingCriteria(HTAB *WorkerTasksHash, - bool (*CriteriaFunction) (WorkerTask *)); + bool (*CriteriaFunction)(WorkerTask *)); static bool RunningTask(WorkerTask *workerTask); static bool SchedulableTask(WorkerTask *workerTask); static int CompareTasksByTime(const void *first, const void *second); @@ -393,7 +393,7 @@ TrackerCleanupJobSchemas(void) /* * We create cleanup tasks since we can't remove schemas within the task * tracker process. We also assign high priorities to these tasks so - * that they get scheduled before everyone else. + * that they get scheduled before everyone else. */ cleanupTask = WorkerTasksHashEnter(jobId, taskIndex); cleanupTask->assignedAt = HIGH_PRIORITY_TASK_TIME; @@ -440,7 +440,7 @@ TrackerCleanupConnections(HTAB *WorkerTasksHash) currentTask->connectionId = INVALID_CONNECTION_ID; } - currentTask = (WorkerTask *) hash_seq_search(&status); + currentTask = (WorkerTask *) hash_seq_search(&status); } } @@ -494,8 +494,9 @@ TrackerDelayLoop(void) } } + /* ------------------------------------------------------------ - * Signal handling and shared hash initialization functions follow + * Signal handling and shared hash initialization functions follow * ------------------------------------------------------------ */ @@ -503,7 +504,7 @@ TrackerDelayLoop(void) static void TrackerSigHupHandler(SIGNAL_ARGS) { - int save_errno = errno; + int save_errno = errno; got_SIGHUP = true; if (MyProc != NULL) @@ -519,7 +520,7 @@ TrackerSigHupHandler(SIGNAL_ARGS) static void TrackerShutdownHandler(SIGNAL_ARGS) { - int save_errno = errno; + int save_errno = errno; got_SIGTERM = true; if (MyProc != NULL) @@ -579,10 +580,10 @@ TaskTrackerShmemInit(void) LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE); /* allocate struct containing task tracker related shared state */ - WorkerTasksSharedState = (WorkerTasksSharedStateData *) - ShmemInitStruct("Worker Task Control", - sizeof(WorkerTasksSharedStateData), - &alreadyInitialized); + WorkerTasksSharedState = + (WorkerTasksSharedStateData *) ShmemInitStruct("Worker Task Control", + sizeof(WorkerTasksSharedStateData), + &alreadyInitialized); if (!alreadyInitialized) { @@ -607,6 +608,7 @@ TaskTrackerShmemInit(void) } } + /* ------------------------------------------------------------ * Task scheduling and management functions follow * ------------------------------------------------------------ @@ -638,7 +640,7 @@ SchedulableTaskList(HTAB *WorkerTasksHash) schedulableTaskCount = CountTasksMatchingCriteria(WorkerTasksHash, &SchedulableTask); if (schedulableTaskCount == 0) { - return NIL; /* we do not have any new tasks to schedule */ + return NIL; /* we do not have any new tasks to schedule */ } tasksToScheduleCount = MaxRunningTasksPerNode - runningTaskCount; @@ -653,7 +655,7 @@ SchedulableTaskList(HTAB *WorkerTasksHash) for (queueIndex = 0; queueIndex < tasksToScheduleCount; queueIndex++) { WorkerTask *schedulableTask = (WorkerTask *) palloc0(sizeof(WorkerTask)); - schedulableTask->jobId = schedulableTaskQueue[queueIndex].jobId; + schedulableTask->jobId = schedulableTaskQueue[queueIndex].jobId; schedulableTask->taskId = schedulableTaskQueue[queueIndex].taskId; schedulableTaskList = lappend(schedulableTaskList, schedulableTask); @@ -681,13 +683,13 @@ SchedulableTaskPriorityQueue(HTAB *WorkerTasksHash) uint32 queueIndex = 0; /* our priority queue size equals to the number of schedulable tasks */ - queueSize = CountTasksMatchingCriteria(WorkerTasksHash, &SchedulableTask); + queueSize = CountTasksMatchingCriteria(WorkerTasksHash, &SchedulableTask); if (queueSize == 0) { return NULL; } - /* allocate an array of tasks for our priority queue */ + /* allocate an array of tasks for our priority queue */ priorityQueue = (WorkerTask *) palloc0(sizeof(WorkerTask) * queueSize); /* copy tasks in the shared hash to the priority queue */ @@ -719,7 +721,7 @@ SchedulableTaskPriorityQueue(HTAB *WorkerTasksHash) /* Counts the number of tasks that match the given criteria function. */ static uint32 CountTasksMatchingCriteria(HTAB *WorkerTasksHash, - bool (*CriteriaFunction) (WorkerTask *)) + bool (*CriteriaFunction)(WorkerTask *)) { HASH_SEQ_STATUS status; WorkerTask *currentTask = NULL; @@ -730,13 +732,13 @@ CountTasksMatchingCriteria(HTAB *WorkerTasksHash, currentTask = (WorkerTask *) hash_seq_search(&status); while (currentTask != NULL) { - bool matchesCriteria = (*CriteriaFunction) (currentTask); + bool matchesCriteria = (*CriteriaFunction)(currentTask); if (matchesCriteria) { taskCount++; } - currentTask = (WorkerTask *) hash_seq_search(&status); + currentTask = (WorkerTask *) hash_seq_search(&status); } return taskCount; @@ -775,7 +777,7 @@ SchedulableTask(WorkerTask *workerTask) static int CompareTasksByTime(const void *first, const void *second) { - WorkerTask *firstTask = (WorkerTask *) first; + WorkerTask *firstTask = (WorkerTask *) first; WorkerTask *secondTask = (WorkerTask *) second; /* tasks that are assigned earlier have higher priority */ @@ -893,7 +895,7 @@ ManageWorkerTask(WorkerTask *workerTask, HTAB *WorkerTasksHash) { case TASK_ASSIGNED: { - break; /* nothing to do until the task gets scheduled */ + break; /* nothing to do until the task gets scheduled */ } case TASK_SCHEDULED: diff --git a/src/backend/distributed/worker/task_tracker_protocol.c b/src/backend/distributed/worker/task_tracker_protocol.c index 84115f441..1b52be99a 100644 --- a/src/backend/distributed/worker/task_tracker_protocol.c +++ b/src/backend/distributed/worker/task_tracker_protocol.c @@ -57,7 +57,7 @@ task_tracker_assign_task(PG_FUNCTION_ARGS) { uint64 jobId = PG_GETARG_INT64(0); uint32 taskId = PG_GETARG_UINT32(1); - text *taskCallStringText = PG_GETARG_TEXT_P(2); + text *taskCallStringText = PG_GETARG_TEXT_P(2); StringInfo jobSchemaName = JobSchemaName(jobId); bool schemaExists = false; @@ -185,7 +185,7 @@ task_tracker_cleanup_job(PG_FUNCTION_ARGS) CleanupTask(currentTask); } - currentTask = (WorkerTask *) hash_seq_search(&status); + currentTask = (WorkerTask *) hash_seq_search(&status); } LWLockRelease(WorkerTasksSharedState->taskHashLock); @@ -308,7 +308,7 @@ CreateTask(uint64 jobId, uint32 taskId, char *taskCallString) } -/* +/* * UpdateTask updates the call string text for an already existing task. Note * that this function expects the caller to hold an exclusive lock over the * shared hash. @@ -331,7 +331,7 @@ UpdateTask(WorkerTask *workerTask, char *taskCallString) if (taskStatus == TASK_SUCCEEDED || taskStatus == TASK_CANCEL_REQUESTED || taskStatus == TASK_CANCELED) { - ; /* nothing to do */ + /* nothing to do */ } else if (taskStatus == TASK_PERMANENTLY_FAILED) { diff --git a/src/backend/distributed/worker/worker_data_fetch_protocol.c b/src/backend/distributed/worker/worker_data_fetch_protocol.c index 0e5b68a1d..d0c309c18 100644 --- a/src/backend/distributed/worker/worker_data_fetch_protocol.c +++ b/src/backend/distributed/worker/worker_data_fetch_protocol.c @@ -53,11 +53,14 @@ static void ReceiveResourceCleanup(int32 connectionId, const char *filename, static void DeleteFile(const char *filename); static void FetchTableCommon(text *tableName, uint64 remoteTableSize, ArrayType *nodeNameObject, ArrayType *nodePortObject, - bool (*FetchTableFunction) (const char *, uint32, StringInfo)); + bool (*FetchTableFunction)(const char *, uint32, + StringInfo)); static uint64 LocalTableSize(Oid relationId); static uint64 ExtractShardId(StringInfo tableName); -static bool FetchRegularTable(const char *nodeName, uint32 nodePort, StringInfo tableName); -static bool FetchForeignTable(const char *nodeName, uint32 nodePort, StringInfo tableName); +static bool FetchRegularTable(const char *nodeName, uint32 nodePort, + StringInfo tableName); +static bool FetchForeignTable(const char *nodeName, uint32 nodePort, + StringInfo tableName); static List * TableDDLCommandList(const char *nodeName, uint32 nodePort, StringInfo tableName); static StringInfo ForeignFilePath(const char *nodeName, uint32 nodePort, @@ -85,7 +88,7 @@ worker_fetch_partition_file(PG_FUNCTION_ARGS) uint64 jobId = PG_GETARG_INT64(0); uint32 partitionTaskId = PG_GETARG_UINT32(1); uint32 partitionFileId = PG_GETARG_UINT32(2); - uint32 upstreamTaskId = PG_GETARG_UINT32(3); + uint32 upstreamTaskId = PG_GETARG_UINT32(3); text *nodeNameText = PG_GETARG_TEXT_P(4); uint32 nodePort = PG_GETARG_UINT32(5); char *nodeName = NULL; @@ -226,7 +229,7 @@ ReceiveRegularFile(const char *nodeName, uint32 nodePort, char filename[MAXPGPATH]; int closed = -1; const int fileFlags = (O_APPEND | O_CREAT | O_RDWR | O_TRUNC | PG_BINARY); - const int fileMode = (S_IRUSR | S_IWUSR); + const int fileMode = (S_IRUSR | S_IWUSR); QueryStatus queryStatus = CLIENT_INVALID_QUERY; int32 connectionId = INVALID_CONNECTION_ID; @@ -309,7 +312,7 @@ ReceiveRegularFile(const char *nodeName, uint32 nodePort, } else if (copyStatus == CLIENT_COPY_MORE) { - ; /* remote node will continue to send more data */ + /* remote node will continue to send more data */ } else { @@ -468,7 +471,7 @@ worker_fetch_foreign_file(PG_FUNCTION_ARGS) static void FetchTableCommon(text *tableNameText, uint64 remoteTableSize, ArrayType *nodeNameObject, ArrayType *nodePortObject, - bool (*FetchTableFunction) (const char *, uint32, StringInfo)) + bool (*FetchTableFunction)(const char *, uint32, StringInfo)) { StringInfo tableName = NULL; char *tableNameCString = NULL; @@ -531,7 +534,7 @@ FetchTableCommon(text *tableNameText, uint64 remoteTableSize, if (remoteTableSize > localTableSize) { /* table is not up to date, drop the table */ - ObjectAddress tableObject = {InvalidOid, InvalidOid, 0}; + ObjectAddress tableObject = { InvalidOid, InvalidOid, 0 }; tableObject.classId = RelationRelationId; tableObject.objectId = relationId; @@ -554,7 +557,7 @@ FetchTableCommon(text *tableNameText, uint64 remoteTableSize, char *nodeName = TextDatumGetCString(nodeNameDatum); uint32 nodePort = DatumGetUInt32(nodePortDatum); - tableFetched = (*FetchTableFunction) (nodeName, nodePort, tableName); + tableFetched = (*FetchTableFunction)(nodeName, nodePort, tableName); nodeIndex++; } @@ -1010,7 +1013,7 @@ worker_append_table_to_shard(PG_FUNCTION_ARGS) * the transaction for this function commits, this lock will automatically * be released. This ensures appends to a shard happen in a serial manner. */ - shardId = ExtractShardId(shardNameString); + shardId = ExtractShardId(shardNameString); LockShardResource(shardId, AccessExclusiveLock); localFilePath = makeStringInfo(); @@ -1049,7 +1052,7 @@ worker_append_table_to_shard(PG_FUNCTION_ARGS) static bool check_log_statement(List *statementList) { - ListCell *statementCell; + ListCell *statementCell; if (log_statement == LOGSTMT_NONE) { diff --git a/src/backend/distributed/worker/worker_file_access_protocol.c b/src/backend/distributed/worker/worker_file_access_protocol.c index 04deb2881..6c9eb18bd 100644 --- a/src/backend/distributed/worker/worker_file_access_protocol.c +++ b/src/backend/distributed/worker/worker_file_access_protocol.c @@ -40,22 +40,22 @@ worker_foreign_file_path(PG_FUNCTION_ARGS) ForeignTable *foreignTable = GetForeignTable(relationId); ListCell *optionCell = NULL; - foreach(optionCell, foreignTable->options) - { - DefElem *option = (DefElem *) lfirst(optionCell); + foreach(optionCell, foreignTable->options) + { + DefElem *option = (DefElem *) lfirst(optionCell); char *optionName = option->defname; - int compareResult = strncmp(optionName, FOREIGN_FILENAME_OPTION, MAXPGPATH); - if (compareResult == 0) - { - char *optionValue = defGetString(option); - foreignFilePath = cstring_to_text(optionValue); - break; - } - } + int compareResult = strncmp(optionName, FOREIGN_FILENAME_OPTION, MAXPGPATH); + if (compareResult == 0) + { + char *optionValue = defGetString(option); + foreignFilePath = cstring_to_text(optionValue); + break; + } + } /* check that we found the filename option */ - if (foreignFilePath == NULL) + if (foreignFilePath == NULL) { char *relationName = get_rel_name(relationId); ereport(ERROR, (errmsg("could not find filename for foreign table: \"%s\"", diff --git a/src/backend/distributed/worker/worker_merge_protocol.c b/src/backend/distributed/worker/worker_merge_protocol.c index ee829e342..92afb1fb1 100644 --- a/src/backend/distributed/worker/worker_merge_protocol.c +++ b/src/backend/distributed/worker/worker_merge_protocol.c @@ -133,7 +133,7 @@ worker_merge_files_and_run_query(PG_FUNCTION_ARGS) const char *createMergeTableQuery = text_to_cstring(createMergeTableQueryText); const char *createIntermediateTableQuery = - text_to_cstring(createIntermediateTableQueryText); + text_to_cstring(createIntermediateTableQueryText); StringInfo taskDirectoryName = TaskDirectoryName(jobId, taskId); StringInfo jobSchemaName = JobSchemaName(jobId); @@ -170,14 +170,14 @@ worker_merge_files_and_run_query(PG_FUNCTION_ARGS) if (setSearchPathResult < 0) { ereport(ERROR, (errmsg("execution was not successful \"%s\"", - setSearchPathString->data))); + setSearchPathString->data))); } createMergeTableResult = SPI_exec(createMergeTableQuery, 0); if (createMergeTableResult < 0) { ereport(ERROR, (errmsg("execution was not successful \"%s\"", - createMergeTableQuery))); + createMergeTableQuery))); } appendStringInfo(mergeTableName, "%s%s", intermediateTableName->data, @@ -188,7 +188,7 @@ worker_merge_files_and_run_query(PG_FUNCTION_ARGS) if (createIntermediateTableResult < 0) { ereport(ERROR, (errmsg("execution was not successful \"%s\"", - createIntermediateTableQuery))); + createIntermediateTableQuery))); } finished = SPI_finish(); @@ -256,8 +256,8 @@ JobSchemaName(uint64 jobId) */ #ifdef HAVE_INTTYPES_H StringInfo jobSchemaName = makeStringInfo(); - appendStringInfo(jobSchemaName, "%s%0*"PRIu64, - JOB_SCHEMA_PREFIX, MIN_JOB_DIRNAME_WIDTH, jobId); + appendStringInfo(jobSchemaName, "%s%0*" PRIu64, JOB_SCHEMA_PREFIX, + MIN_JOB_DIRNAME_WIDTH, jobId); #else StringInfo jobSchemaName = makeStringInfo(); appendStringInfo(jobSchemaName, "%s%0*llu", diff --git a/src/backend/distributed/worker/worker_partition_protocol.c b/src/backend/distributed/worker/worker_partition_protocol.c index c6578295b..9ab21e588 100644 --- a/src/backend/distributed/worker/worker_partition_protocol.c +++ b/src/backend/distributed/worker/worker_partition_protocol.c @@ -59,7 +59,7 @@ static void FileOutputStreamWrite(FileOutputStream file, StringInfo dataToWrite) static void FileOutputStreamFlush(FileOutputStream file); static void FilterAndPartitionTable(const char *filterQuery, const char *columnName, Oid columnType, - uint32 (*PartitionIdFunction) (Datum, const void *), + uint32 (*PartitionIdFunction)(Datum, const void *), const void *partitionIdContext, FileOutputStream *partitionFileArray, uint32 fileCount); @@ -105,7 +105,7 @@ worker_range_partition_table(PG_FUNCTION_ARGS) uint32 taskId = PG_GETARG_UINT32(1); text *filterQueryText = PG_GETARG_TEXT_P(2); text *partitionColumnText = PG_GETARG_TEXT_P(3); - Oid partitionColumnType = PG_GETARG_OID(4); + Oid partitionColumnType = PG_GETARG_OID(4); ArrayType *splitPointObject = PG_GETARG_ARRAYTYPE_P(5); const char *filterQuery = text_to_cstring(filterQueryText); @@ -125,7 +125,7 @@ worker_range_partition_table(PG_FUNCTION_ARGS) if (splitPointType != partitionColumnType) { ereport(ERROR, (errmsg("partition column type %u and split point type %u " - "do not match", partitionColumnType, splitPointType))); + "do not match", partitionColumnType, splitPointType))); } /* use column's type information to get the comparison function */ @@ -181,7 +181,7 @@ worker_hash_partition_table(PG_FUNCTION_ARGS) uint32 taskId = PG_GETARG_UINT32(1); text *filterQueryText = PG_GETARG_TEXT_P(2); text *partitionColumnText = PG_GETARG_TEXT_P(3); - Oid partitionColumnType = PG_GETARG_OID(4); + Oid partitionColumnType = PG_GETARG_OID(4); uint32 partitionCount = PG_GETARG_UINT32(5); const char *filterQuery = text_to_cstring(filterQueryText); @@ -463,7 +463,7 @@ JobDirectoryName(uint64 jobId) */ #ifdef HAVE_INTTYPES_H StringInfo jobDirectoryName = makeStringInfo(); - appendStringInfo(jobDirectoryName, "base/%s/%s%0*"PRIu64, + appendStringInfo(jobDirectoryName, "base/%s/%s%0*" PRIu64, PG_JOB_CACHE_DIR, JOB_DIRECTORY_PREFIX, MIN_JOB_DIRNAME_WIDTH, jobId); #else @@ -726,7 +726,7 @@ FileOutputStreamFlush(FileOutputStream file) static void FilterAndPartitionTable(const char *filterQuery, const char *partitionColumnName, Oid partitionColumnType, - uint32 (*PartitionIdFunction) (Datum, const void *), + uint32 (*PartitionIdFunction)(Datum, const void *), const void *partitionIdContext, FileOutputStream *partitionFileArray, uint32 fileCount) @@ -794,7 +794,7 @@ FilterAndPartitionTable(const char *filterQuery, FileOutputStream partitionFile = { 0, 0, 0 }; StringInfo rowText = NULL; Datum partitionKey = 0; - bool partitionKeyNull = false; + bool partitionKeyNull = false; uint32 partitionId = 0; partitionKey = SPI_getbinval(row, rowDescriptor, @@ -808,7 +808,7 @@ FilterAndPartitionTable(const char *filterQuery, */ if (!partitionKeyNull) { - partitionId = (*PartitionIdFunction) (partitionKey, partitionIdContext); + partitionId = (*PartitionIdFunction)(partitionKey, partitionIdContext); } else { @@ -926,7 +926,7 @@ InitRowOutputState(void) /* initialize defaults for printing null values */ char *nullPrint = pstrdup("\\N"); - int nullPrintLen = strlen(nullPrint); + int nullPrintLen = strlen(nullPrint); char *nullPrintClient = pg_server_to_any(nullPrint, nullPrintLen, fileEncoding); /* set default text output characters */ @@ -946,7 +946,7 @@ InitRowOutputState(void) } /* set up transcoding information and default text output characters */ - if ( (fileEncoding != databaseEncoding) || (databaseEncodingMaxLength > 1) ) + if ((fileEncoding != databaseEncoding) || (databaseEncodingMaxLength > 1)) { rowOutputState->need_transcoding = true; } @@ -1057,7 +1057,7 @@ OutputRow(HeapTuple row, TupleDesc rowDescriptor, CopySendString(rowOutputState, rowOutputState->null_print_client); } - lastColumn = ((columnIndex+1) == columnCount); + lastColumn = ((columnIndex + 1) == columnCount); if (!lastColumn) { CopySendChar(rowOutputState, rowOutputState->delim[0]); @@ -1094,9 +1094,9 @@ OutputBinaryHeaders(FileOutputStream *partitionFileArray, uint32 fileCount) { /* Generate header for a binary copy */ const int32 zero = 0; - FileOutputStream partitionFile = {0, 0, 0}; + FileOutputStream partitionFile = { 0, 0, 0 }; PartialCopyStateData headerOutputStateData; - PartialCopyState headerOutputState = (PartialCopyState) &headerOutputStateData; + PartialCopyState headerOutputState = (PartialCopyState) & headerOutputStateData; memset(headerOutputState, 0, sizeof(PartialCopyStateData)); headerOutputState->fe_msgbuf = makeStringInfo(); @@ -1128,9 +1128,9 @@ OutputBinaryFooters(FileOutputStream *partitionFileArray, uint32 fileCount) { /* Generate footer for a binary copy */ int16 negative = -1; - FileOutputStream partitionFile = {0, 0, 0}; + FileOutputStream partitionFile = { 0, 0, 0 }; PartialCopyStateData footerOutputStateData; - PartialCopyState footerOutputState = (PartialCopyState) &footerOutputStateData; + PartialCopyState footerOutputState = (PartialCopyState) & footerOutputStateData; memset(footerOutputState, 0, sizeof(PartialCopyStateData)); footerOutputState->fe_msgbuf = makeStringInfo(); @@ -1359,7 +1359,7 @@ RangePartitionId(Datum partitionValue, const void *context) currentLength = currentLength - halfLength - 1; } } - + return firstIndex; } diff --git a/src/include/distributed/citus_ruleutils.h b/src/include/distributed/citus_ruleutils.h index 0f2402748..92e1777e7 100644 --- a/src/include/distributed/citus_ruleutils.h +++ b/src/include/distributed/citus_ruleutils.h @@ -2,7 +2,7 @@ * * citus_ruleutils.h * CitusDB ruleutils wrapper functions and exported PostgreSQL ruleutils - * functions. + * functions. * * Copyright (c) 2012-2015, Citus Data, Inc. *------------------------------------------------------------------------- @@ -16,16 +16,17 @@ /* Function declarations for version independent CitusDB ruleutils wrapper functions */ -extern char *pg_get_extensiondef_string(Oid tableRelationId); -extern char *pg_get_serverdef_string(Oid tableRelationId); -extern char *pg_get_tableschemadef_string(Oid tableRelationId); -extern char *pg_get_tablecolumnoptionsdef_string(Oid tableRelationId); -extern char *pg_get_indexclusterdef_string(Oid indexRelationId); +extern char * pg_get_extensiondef_string(Oid tableRelationId); +extern char * pg_get_serverdef_string(Oid tableRelationId); +extern char * pg_get_tableschemadef_string(Oid tableRelationId); +extern char * pg_get_tablecolumnoptionsdef_string(Oid tableRelationId); +extern char * pg_get_indexclusterdef_string(Oid indexRelationId); /* Function declarations for version dependent PostgreSQL ruleutils functions */ -extern void pg_get_query_def(Query *query, StringInfo buffer); -extern void deparse_shard_query(Query *query, Oid distrelid, int64 shardid, StringInfo buffer); -extern char *generate_relation_name(Oid relid, List *namespaces); +extern void pg_get_query_def(Query *query, StringInfo buffer); +extern void deparse_shard_query(Query *query, Oid distrelid, int64 shardid, StringInfo + buffer); +extern char * generate_relation_name(Oid relid, List *namespaces); #endif /* CITUS_RULEUTILS_H */ diff --git a/src/include/distributed/master_metadata_utility.h b/src/include/distributed/master_metadata_utility.h index b3cfb6548..4fb377571 100644 --- a/src/include/distributed/master_metadata_utility.h +++ b/src/include/distributed/master_metadata_utility.h @@ -30,15 +30,14 @@ typedef struct ShardInterval CitusNodeTag type; Oid relationId; char storageType; - Oid valueTypeId; /* min/max value datum's typeId */ - int valueTypeLen; /* min/max value datum's typelen */ - bool valueByVal; /* min/max value datum's byval */ + Oid valueTypeId; /* min/max value datum's typeId */ + int valueTypeLen; /* min/max value datum's typelen */ + bool valueByVal; /* min/max value datum's byval */ bool minValueExists; bool maxValueExists; - Datum minValue; /* a shard's typed min value datum */ - Datum maxValue; /* a shard's typed max value datum */ + Datum minValue; /* a shard's typed min value datum */ + Datum maxValue; /* a shard's typed max value datum */ uint64 shardId; - } ShardInterval; @@ -46,13 +45,12 @@ typedef struct ShardInterval typedef struct ShardPlacement { CitusNodeTag type; - Oid tupleOid; /* unique oid that implies this row's insertion order */ + Oid tupleOid; /* unique oid that implies this row's insertion order */ uint64 shardId; uint64 shardLength; RelayFileState shardState; char *nodeName; uint32 nodePort; - } ShardPlacement; diff --git a/src/include/distributed/master_protocol.h b/src/include/distributed/master_protocol.h index f39ce865b..94344de0c 100644 --- a/src/include/distributed/master_protocol.h +++ b/src/include/distributed/master_protocol.h @@ -49,10 +49,10 @@ #define SHARDID_SEQUENCE_NAME "pg_dist_shardid_seq" /* Remote call definitions to help with data staging and deletion */ -#define WORKER_APPLY_SHARD_DDL_COMMAND "SELECT worker_apply_shard_ddl_command \ - ("UINT64_FORMAT", %s)" -#define WORKER_APPEND_TABLE_TO_SHARD "SELECT worker_append_table_to_shard \ - (%s, %s, %s, %u)" +#define WORKER_APPLY_SHARD_DDL_COMMAND \ + "SELECT worker_apply_shard_ddl_command (" UINT64_FORMAT ", %s)" +#define WORKER_APPEND_TABLE_TO_SHARD \ + "SELECT worker_append_table_to_shard (%s, %s, %s, %u)" #define SHARD_MIN_VALUE_QUERY "SELECT min(%s) FROM %s" #define SHARD_MAX_VALUE_QUERY "SELECT max(%s) FROM %s" #define SHARD_TABLE_SIZE_QUERY "SELECT pg_table_size('%s')" @@ -67,7 +67,6 @@ typedef enum SHARD_PLACEMENT_INVALID_FIRST = 0, SHARD_PLACEMENT_LOCAL_NODE_FIRST = 1, SHARD_PLACEMENT_ROUND_ROBIN = 2 - } ShardPlacementPolicyType; @@ -83,8 +82,8 @@ extern Oid ResolveRelationId(text *relationName); extern List * GetTableDDLEvents(Oid relationId); extern void CheckDistributedTable(Oid relationId); extern void CreateShardPlacements(int64 shardId, List *ddlEventList, - List *workerNodeList, int workerStartIndex, - int replicationFactor); + List *workerNodeList, int workerStartIndex, + int replicationFactor); /* Function declarations for generating metadata for shard creation */ extern Datum master_get_table_metadata(PG_FUNCTION_ARGS); diff --git a/src/include/distributed/modify_planner.h b/src/include/distributed/modify_planner.h index 3e52cfa5a..bd6df4755 100644 --- a/src/include/distributed/modify_planner.h +++ b/src/include/distributed/modify_planner.h @@ -24,6 +24,7 @@ #define INVALID_TASK_ID 0 #if (PG_VERSION_NUM >= 90500) + /* reserved alias name for UPSERTs */ #define UPSERT_ALIAS "citus_table_alias" #endif diff --git a/src/include/distributed/multi_client_executor.h b/src/include/distributed/multi_client_executor.h index 470b53673..0f1203031 100644 --- a/src/include/distributed/multi_client_executor.h +++ b/src/include/distributed/multi_client_executor.h @@ -15,21 +15,20 @@ #define MULTI_CLIENT_EXECUTOR_H -#define INVALID_CONNECTION_ID -1 /* identifies an invalid connection */ -#define CLIENT_CONNECT_TIMEOUT 5 /* connection timeout in seconds */ +#define INVALID_CONNECTION_ID -1 /* identifies an invalid connection */ +#define CLIENT_CONNECT_TIMEOUT 5 /* connection timeout in seconds */ #define MAX_CONNECTION_COUNT 2048 /* simultaneous client connection count */ -#define STRING_BUFFER_SIZE 1024 /* buffer size for character arrays */ +#define STRING_BUFFER_SIZE 1024 /* buffer size for character arrays */ #define CONN_INFO_TEMPLATE "host=%s port=%u dbname=%s connect_timeout=%u" /* Enumeration to track one client connection's status */ typedef enum { - CLIENT_INVALID_CONNECT = 0, - CLIENT_CONNECTION_BAD = 1, - CLIENT_CONNECTION_BUSY = 2, + CLIENT_INVALID_CONNECT = 0, + CLIENT_CONNECTION_BAD = 1, + CLIENT_CONNECTION_BUSY = 2, CLIENT_CONNECTION_READY = 3 - } ConnectStatus; @@ -38,9 +37,8 @@ typedef enum { CLIENT_INVALID_RESULT_STATUS = 0, CLIENT_RESULT_UNAVAILABLE = 1, - CLIENT_RESULT_BUSY = 2, + CLIENT_RESULT_BUSY = 2, CLIENT_RESULT_READY = 3 - } ResultStatus; @@ -48,10 +46,9 @@ typedef enum typedef enum { CLIENT_INVALID_QUERY = 0, - CLIENT_QUERY_FAILED = 1, + CLIENT_QUERY_FAILED = 1, CLIENT_QUERY_DONE = 2, CLIENT_QUERY_COPY = 3 - } QueryStatus; @@ -59,21 +56,19 @@ typedef enum typedef enum { CLIENT_INVALID_COPY = 0, - CLIENT_COPY_MORE = 1, - CLIENT_COPY_FAILED = 2, - CLIENT_COPY_DONE = 3 - + CLIENT_COPY_MORE = 1, + CLIENT_COPY_FAILED = 2, + CLIENT_COPY_DONE = 3 } CopyStatus; /* Enumeration to track the status of a query in a batch on the client */ typedef enum { - CLIENT_INVALID_BATCH_QUERY = 0, - CLIENT_BATCH_QUERY_FAILED = 1, + CLIENT_INVALID_BATCH_QUERY = 0, + CLIENT_BATCH_QUERY_FAILED = 1, CLIENT_BATCH_QUERY_CONTINUE = 2, - CLIENT_BATCH_QUERY_DONE = 3 - + CLIENT_BATCH_QUERY_DONE = 3 } BatchQueryStatus; diff --git a/src/include/distributed/multi_executor.h b/src/include/distributed/multi_executor.h index bcf22fe2e..b777858d8 100644 --- a/src/include/distributed/multi_executor.h +++ b/src/include/distributed/multi_executor.h @@ -14,12 +14,12 @@ #include "nodes/parsenodes.h" /* signal currently executed statement is a master select statement or router execution */ -#define EXEC_FLAG_CITUS_MASTER_SELECT 0x100 -#define EXEC_FLAG_CITUS_ROUTER_EXECUTOR 0x200 +#define EXEC_FLAG_CITUS_MASTER_SELECT 0x100 +#define EXEC_FLAG_CITUS_ROUTER_EXECUTOR 0x200 extern void multi_ExecutorStart(QueryDesc *queryDesc, int eflags); extern void multi_ExecutorRun(QueryDesc *queryDesc, - ScanDirection direction, long count); + ScanDirection direction, long count); extern void multi_ExecutorFinish(QueryDesc *queryDesc); extern void multi_ExecutorEnd(QueryDesc *queryDesc); diff --git a/src/include/distributed/multi_join_order.h b/src/include/distributed/multi_join_order.h index cccde6917..56ede010b 100644 --- a/src/include/distributed/multi_join_order.h +++ b/src/include/distributed/multi_join_order.h @@ -29,7 +29,7 @@ typedef enum JoinRuleType { JOIN_RULE_INVALID_FIRST = 0, BROADCAST_JOIN = 1, - LOCAL_PARTITION_JOIN = 2, + LOCAL_PARTITION_JOIN = 2, SINGLE_PARTITION_JOIN = 3, DUAL_PARTITION_JOIN = 4, CARTESIAN_PRODUCT = 5, @@ -40,7 +40,6 @@ typedef enum JoinRuleType * RuleNameArray. */ JOIN_RULE_LAST - } JoinRuleType; @@ -53,7 +52,6 @@ typedef struct TableEntry { Oid relationId; uint32 rangeTableId; - } TableEntry; @@ -65,14 +63,13 @@ typedef struct TableEntry */ typedef struct JoinOrderNode { - TableEntry *tableEntry; /* this node's relation and range table id */ - JoinRuleType joinRuleType; /* not relevant for the first table */ - JoinType joinType; /* not relevant for the first table */ - Var *partitionColumn; /* not relevant for the first table */ + TableEntry *tableEntry; /* this node's relation and range table id */ + JoinRuleType joinRuleType; /* not relevant for the first table */ + JoinType joinType; /* not relevant for the first table */ + Var *partitionColumn; /* not relevant for the first table */ char partitionMethod; - List *joinClauseList; /* not relevant for the first table */ + List *joinClauseList; /* not relevant for the first table */ List *shardIntervalList; - } JoinOrderNode; diff --git a/src/include/distributed/multi_logical_optimizer.h b/src/include/distributed/multi_logical_optimizer.h index f8c145751..4efc1bfed 100644 --- a/src/include/distributed/multi_logical_optimizer.h +++ b/src/include/distributed/multi_logical_optimizer.h @@ -44,7 +44,7 @@ * * Please note that the order of values in this enumeration is tied to the order * of elements in the following AggregateNames array. This order needs to be - * preserved. + * preserved. */ typedef enum { @@ -55,7 +55,6 @@ typedef enum AGGREGATE_SUM = 4, AGGREGATE_COUNT = 5, AGGREGATE_ARRAY_AGG = 6 - } AggregateType; @@ -69,7 +68,6 @@ typedef enum PUSH_DOWN_VALID = 1, PUSH_DOWN_NOT_VALID = 2, PUSH_DOWN_SPECIAL_CONDITIONS = 3 - } PushDownStatus; @@ -82,7 +80,6 @@ typedef enum PULL_UP_INVALID_FIRST = 0, PULL_UP_VALID = 1, PULL_UP_NOT_VALID = 2 - } PullUpStatus; @@ -97,8 +94,10 @@ typedef enum * Please note that the order of elements in this array is tied to the order of * values in the preceding AggregateType enum. This order needs to be preserved. */ -static const char * const AggregateNames[] = { "invalid", "avg", "min", "max", - "sum", "count", "array_agg" }; +static const char *const AggregateNames[] = { + "invalid", "avg", "min", "max", "sum", + "count", "array_agg" +}; /* Config variable managed via guc.c */ diff --git a/src/include/distributed/multi_logical_planner.h b/src/include/distributed/multi_logical_planner.h index 15b0cba07..40ee39c70 100644 --- a/src/include/distributed/multi_logical_planner.h +++ b/src/include/distributed/multi_logical_planner.h @@ -40,8 +40,8 @@ typedef struct MultiNode CitusNodeTag type; struct MultiNode *parentNode; - /* child node(s) are defined in unary and binary nodes */ + /* child node(s) are defined in unary and binary nodes */ } MultiNode; @@ -51,7 +51,6 @@ typedef struct MultiUnaryNode MultiNode node; struct MultiNode *childNode; - } MultiUnaryNode; @@ -62,7 +61,6 @@ typedef struct MultiBinaryNode struct MultiNode *leftChildNode; struct MultiNode *rightChildNode; - } MultiBinaryNode; @@ -73,7 +71,6 @@ typedef struct MultiBinaryNode typedef struct MultiTreeRoot { MultiUnaryNode unaryNode; - } MultiTreeRoot; @@ -91,7 +88,6 @@ typedef struct MultiTable Alias *alias; Alias *referenceNames; Query *subquery; /* this field is only valid for non-relation subquery types */ - } MultiTable; @@ -100,7 +96,6 @@ typedef struct MultiProject { MultiUnaryNode unaryNode; List *columnList; - } MultiProject; @@ -112,7 +107,6 @@ typedef struct MultiProject typedef struct MultiCollect { MultiUnaryNode unaryNode; - } MultiCollect; @@ -125,7 +119,6 @@ typedef struct MultiSelect { MultiUnaryNode unaryNode; List *selectClauseList; - } MultiSelect; @@ -140,7 +133,6 @@ typedef struct MultiJoin List *joinClauseList; JoinRuleType joinRuleType; JoinType joinType; - } MultiJoin; @@ -150,7 +142,6 @@ typedef struct MultiPartition MultiUnaryNode unaryNode; Var *partitionColumn; uint32 splitPointTableId; - } MultiPartition; @@ -158,7 +149,6 @@ typedef struct MultiPartition typedef struct MultiCartesianProduct { MultiBinaryNode binaryNode; - } MultiCartesianProduct; @@ -183,7 +173,6 @@ typedef struct MultiExtendedOp List *sortClauseList; Node *limitCount; Node *limitOffset; - } MultiExtendedOp; diff --git a/src/include/distributed/multi_physical_planner.h b/src/include/distributed/multi_physical_planner.h index 09d28e952..3ad053b22 100644 --- a/src/include/distributed/multi_physical_planner.h +++ b/src/include/distributed/multi_physical_planner.h @@ -2,7 +2,7 @@ * * multi_physical_planner.h * Type and function declarations used in creating the distributed execution - * plan. + * plan. * * Copyright (c) 2012, Citus Data, Inc. * @@ -40,17 +40,18 @@ (" UINT64_FORMAT ", %d, %s, '%s', %d, %d)" #define MERGE_FILES_INTO_TABLE_COMMAND "SELECT worker_merge_files_into_table \ (" UINT64_FORMAT ", %d, '%s', '%s')" -#define MERGE_FILES_AND_RUN_QUERY_COMMAND "SELECT worker_merge_files_and_run_query(" UINT64_FORMAT ", %d, '%s', '%s')" +#define MERGE_FILES_AND_RUN_QUERY_COMMAND \ + "SELECT worker_merge_files_and_run_query(" UINT64_FORMAT ", %d, '%s', '%s')" typedef enum CitusRTEKind { - CITUS_RTE_RELATION = RTE_RELATION, /* ordinary relation reference */ - CITUS_RTE_SUBQUERY = RTE_SUBQUERY, /* subquery in FROM */ - CITUS_RTE_JOIN = RTE_JOIN, /* join */ - CITUS_RTE_FUNCTION = RTE_FUNCTION, /* function in FROM */ - CITUS_RTE_VALUES = RTE_VALUES, /* VALUES (), (), ... */ - CITUS_RTE_CTE = RTE_CTE, /* common table expr (WITH list element) */ + CITUS_RTE_RELATION = RTE_RELATION, /* ordinary relation reference */ + CITUS_RTE_SUBQUERY = RTE_SUBQUERY, /* subquery in FROM */ + CITUS_RTE_JOIN = RTE_JOIN, /* join */ + CITUS_RTE_FUNCTION = RTE_FUNCTION, /* function in FROM */ + CITUS_RTE_VALUES = RTE_VALUES, /* VALUES (), (), ... */ + CITUS_RTE_CTE = RTE_CTE, /* common table expr (WITH list element) */ CITUS_RTE_SHARD, CITUS_RTE_REMOTE_QUERY } CitusRTEKind; @@ -61,8 +62,7 @@ typedef enum { PARTITION_INVALID_FIRST = 0, RANGE_PARTITION_TYPE = 1, - HASH_PARTITION_TYPE = 2 - + HASH_PARTITION_TYPE = 2 } PartitionType; @@ -77,7 +77,6 @@ typedef enum MAP_OUTPUT_FETCH_TASK = 5, MERGE_FETCH_TASK = 6, MODIFY_TASK = 7 - } TaskType; @@ -88,7 +87,6 @@ typedef enum TASK_ASSIGNMENT_GREEDY = 1, TASK_ASSIGNMENT_ROUND_ROBIN = 2, TASK_ASSIGNMENT_FIRST_REPLICA = 3 - } TaskAssignmentPolicyType; @@ -99,7 +97,6 @@ typedef enum JOIN_MAP_MERGE_JOB = 1, SUBQUERY_MAP_MERGE_JOB = 2, TOP_LEVEL_WORKER_JOB = 3 - } BoundaryNodeJobType; @@ -133,7 +130,6 @@ typedef struct MapMergeJob ShardInterval **sortedShardIntervalArray; /* only applies to range partitioning */ List *mapTaskList; List *mergeTaskList; - } MapMergeJob; @@ -153,18 +149,17 @@ typedef struct Task uint64 jobId; uint32 taskId; char *queryString; - uint64 anchorShardId; /* only applies to compute tasks */ - List *taskPlacementList; /* only applies to compute tasks */ - List *dependedTaskList; /* only applies to compute tasks */ + uint64 anchorShardId; /* only applies to compute tasks */ + List *taskPlacementList; /* only applies to compute tasks */ + List *dependedTaskList; /* only applies to compute tasks */ uint32 partitionId; - uint32 upstreamTaskId; /* only applies to data fetch tasks */ + uint32 upstreamTaskId; /* only applies to data fetch tasks */ ShardInterval *shardInterval; /* only applies to merge tasks */ bool assignmentConstrained; /* only applies to merge tasks */ - uint64 shardId; /* only applies to shard fetch tasks */ + uint64 shardId; /* only applies to shard fetch tasks */ TaskExecution *taskExecution; /* used by task tracker executor */ - bool upsertQuery; /* only applies to modify tasks */ - + bool upsertQuery; /* only applies to modify tasks */ } Task; @@ -177,7 +172,6 @@ typedef struct RangeTableFragment CitusRTEKind fragmentType; void *fragmentReference; uint32 rangeTableId; - } RangeTableFragment; @@ -190,7 +184,6 @@ typedef struct JoinSequenceNode { uint32 rangeTableId; int32 joiningRangeTableId; - } JoinSequenceNode; @@ -203,7 +196,6 @@ typedef struct MultiPlan Job *workerJob; Query *masterQuery; char *masterTableName; - } MultiPlan; diff --git a/src/include/distributed/multi_planner.h b/src/include/distributed/multi_planner.h index a14b2b65e..c3e2511e8 100644 --- a/src/include/distributed/multi_planner.h +++ b/src/include/distributed/multi_planner.h @@ -13,8 +13,8 @@ #include "nodes/plannodes.h" #include "nodes/relation.h" -extern PlannedStmt *multi_planner(Query *parse, int cursorOptions, - ParamListInfo boundParams); +extern PlannedStmt * multi_planner(Query *parse, int cursorOptions, + ParamListInfo boundParams); extern bool HasCitusToplevelNode(PlannedStmt *planStatement); struct MultiPlan; diff --git a/src/include/distributed/multi_server_executor.h b/src/include/distributed/multi_server_executor.h index 83105cc54..e6e17f566 100644 --- a/src/include/distributed/multi_server_executor.h +++ b/src/include/distributed/multi_server_executor.h @@ -20,9 +20,9 @@ #define MAX_TASK_EXECUTION_FAILURES 3 /* allowed failure count for one task */ -#define MAX_TRACKER_FAILURE_COUNT 3 /* allowed failure count for one tracker */ +#define MAX_TRACKER_FAILURE_COUNT 3 /* allowed failure count for one tracker */ #define REMOTE_NODE_CONNECT_TIMEOUT 4000 /* async connect timeout in ms */ -#define RESERVED_FD_COUNT 64 /* file descriptors unavailable to executor */ +#define RESERVED_FD_COUNT 64 /* file descriptors unavailable to executor */ /* copy out query results */ #define COPY_QUERY_TO_STDOUT_TEXT "COPY (%s) TO STDOUT" @@ -32,9 +32,9 @@ /* Task tracker executor related defines */ #define TASK_ASSIGNMENT_QUERY "SELECT task_tracker_assign_task \ - ("UINT64_FORMAT", %u, %s)" -#define TASK_STATUS_QUERY "SELECT task_tracker_task_status("UINT64_FORMAT", %u)" -#define JOB_CLEANUP_QUERY "SELECT task_tracker_cleanup_job("UINT64_FORMAT")" + ("UINT64_FORMAT ", %u, %s)" +#define TASK_STATUS_QUERY "SELECT task_tracker_task_status("UINT64_FORMAT ", %u)" +#define JOB_CLEANUP_QUERY "SELECT task_tracker_cleanup_job("UINT64_FORMAT ")" #define JOB_CLEANUP_TASK_ID INT_MAX @@ -43,9 +43,9 @@ typedef enum { EXEC_TASK_INVALID_FIRST = 0, EXEC_TASK_CONNECT_START = 1, - EXEC_TASK_CONNECT_POLL = 2, + EXEC_TASK_CONNECT_POLL = 2, EXEC_TASK_FAILED = 3, - EXEC_FETCH_TASK_LOOP = 4, + EXEC_FETCH_TASK_LOOP = 4, EXEC_FETCH_TASK_START = 5, EXEC_FETCH_TASK_RUNNING = 6, EXEC_COMPUTE_TASK_START = 7, @@ -60,7 +60,6 @@ typedef enum EXEC_TASK_TRACKER_FAILED = 14, EXEC_SOURCE_TASK_TRACKER_RETRY = 15, EXEC_SOURCE_TASK_TRACKER_FAILED = 16 - } TaskExecStatus; @@ -74,7 +73,6 @@ typedef enum EXEC_TRANSMIT_TRACKER_RETRY = 4, EXEC_TRANSMIT_TRACKER_FAILED = 5, EXEC_TRANSMIT_DONE = 6 - } TransmitExecStatus; @@ -86,7 +84,6 @@ typedef enum TRACKER_CONNECT_POLL = 2, TRACKER_CONNECTED = 3, TRACKER_CONNECTION_FAILED = 4 - } TrackerStatus; @@ -97,7 +94,6 @@ typedef enum MULTI_EXECUTOR_REAL_TIME = 1, MULTI_EXECUTOR_TASK_TRACKER = 2, MULTI_EXECUTOR_ROUTER = 3 - } MultiExecutorType; @@ -107,7 +103,6 @@ typedef enum CONNECT_ACTION_NONE = 0, CONNECT_ACTION_OPENED = 1, CONNECT_ACTION_CLOSED = 2 - } ConnectAction; @@ -132,7 +127,6 @@ struct TaskExecution uint32 querySourceNodeIndex; /* only applies to map fetch tasks */ int32 dataFetchTaskIndex; uint32 failureCount; - }; @@ -147,7 +141,6 @@ typedef struct TrackerTaskState uint32 taskId; TaskStatus status; StringInfo taskAssignmentQuery; - } TrackerTaskState; @@ -158,7 +151,7 @@ typedef struct TrackerTaskState */ typedef struct TaskTracker { - uint32 workerPort; /* node's port; part of hash table key */ + uint32 workerPort; /* node's port; part of hash table key */ char workerName[WORKER_LENGTH]; /* node's name; part of hash table key */ TrackerStatus trackerStatus; int32 connectionId; @@ -171,7 +164,6 @@ typedef struct TaskTracker int32 currentTaskIndex; bool connectionBusy; TrackerTaskState *connectionBusyOnTask; - } TaskTracker; @@ -184,7 +176,6 @@ typedef struct WorkerNodeState uint32 workerPort; char workerName[WORKER_LENGTH]; uint32 openConnectionCount; - } WorkerNodeState; diff --git a/src/include/distributed/pg_dist_partition.h b/src/include/distributed/pg_dist_partition.h index d3db82638..d277bc8ce 100644 --- a/src/include/distributed/pg_dist_partition.h +++ b/src/include/distributed/pg_dist_partition.h @@ -21,9 +21,9 @@ */ typedef struct FormData_pg_dist_partition { - Oid logicalrelid; /* logical relation id; references pg_class oid */ - char partmethod; /* partition method; see codes below */ - text partkey; /* partition key expression */ + Oid logicalrelid; /* logical relation id; references pg_class oid */ + char partmethod; /* partition method; see codes below */ + text partkey; /* partition key expression */ } FormData_pg_dist_partition; /* ---------------- @@ -37,16 +37,16 @@ typedef FormData_pg_dist_partition *Form_pg_dist_partition; * compiler constants for pg_dist_partitions * ---------------- */ -#define Natts_pg_dist_partition 3 -#define Anum_pg_dist_partition_logicalrelid 1 -#define Anum_pg_dist_partition_partmethod 2 -#define Anum_pg_dist_partition_partkey 3 +#define Natts_pg_dist_partition 3 +#define Anum_pg_dist_partition_logicalrelid 1 +#define Anum_pg_dist_partition_partmethod 2 +#define Anum_pg_dist_partition_partkey 3 /* valid values for partmethod include append, hash, and range */ -#define DISTRIBUTE_BY_APPEND 'a' -#define DISTRIBUTE_BY_HASH 'h' -#define DISTRIBUTE_BY_RANGE 'r' -#define REDISTRIBUTE_BY_HASH 'x' +#define DISTRIBUTE_BY_APPEND 'a' +#define DISTRIBUTE_BY_HASH 'h' +#define DISTRIBUTE_BY_RANGE 'r' +#define REDISTRIBUTE_BY_HASH 'x' #endif /* PG_DIST_PARTITION_H */ diff --git a/src/include/distributed/pg_dist_shard.h b/src/include/distributed/pg_dist_shard.h index dfe1c86c7..b093bb59d 100644 --- a/src/include/distributed/pg_dist_shard.h +++ b/src/include/distributed/pg_dist_shard.h @@ -22,13 +22,13 @@ */ typedef struct FormData_pg_dist_shard { - Oid logicalrelid; /* logical relation id; references pg_class oid */ - int64 shardid; /* global shardId representing remote partition */ - char shardstorage; /* shard storage type; see codes below */ -#ifdef CATALOG_VARLEN /* variable-length fields start here */ - text shardalias; /* user specified table name for shard, if any */ - text shardminvalue; /* partition key's minimum value in shard */ - text shardmaxvalue; /* partition key's maximum value in shard */ + Oid logicalrelid; /* logical relation id; references pg_class oid */ + int64 shardid; /* global shardId representing remote partition */ + char shardstorage; /* shard storage type; see codes below */ +#ifdef CATALOG_VARLEN /* variable-length fields start here */ + text shardalias; /* user specified table name for shard, if any */ + text shardminvalue; /* partition key's minimum value in shard */ + text shardmaxvalue; /* partition key's maximum value in shard */ #endif } FormData_pg_dist_shard; @@ -43,22 +43,22 @@ typedef FormData_pg_dist_shard *Form_pg_dist_shard; * compiler constants for pg_dist_shards * ---------------- */ -#define Natts_pg_dist_shard 6 -#define Anum_pg_dist_shard_logicalrelid 1 -#define Anum_pg_dist_shard_shardid 2 -#define Anum_pg_dist_shard_shardstorage 3 -#define Anum_pg_dist_shard_shardalias 4 -#define Anum_pg_dist_shard_shardminvalue 5 -#define Anum_pg_dist_shard_shardmaxvalue 6 +#define Natts_pg_dist_shard 6 +#define Anum_pg_dist_shard_logicalrelid 1 +#define Anum_pg_dist_shard_shardid 2 +#define Anum_pg_dist_shard_shardstorage 3 +#define Anum_pg_dist_shard_shardalias 4 +#define Anum_pg_dist_shard_shardminvalue 5 +#define Anum_pg_dist_shard_shardmaxvalue 6 /* * Valid values for shard storage types include relay file, foreign table, * (standard) table and columnar table. Relay file types are currently unused. */ -#define SHARD_STORAGE_RELAY 'r' -#define SHARD_STORAGE_FOREIGN 'f' -#define SHARD_STORAGE_TABLE 't' -#define SHARD_STORAGE_COLUMNAR 'c' +#define SHARD_STORAGE_RELAY 'r' +#define SHARD_STORAGE_FOREIGN 'f' +#define SHARD_STORAGE_TABLE 't' +#define SHARD_STORAGE_COLUMNAR 'c' #endif /* PG_DIST_SHARD_H */ diff --git a/src/include/distributed/pg_dist_shard_placement.h b/src/include/distributed/pg_dist_shard_placement.h index 505daffa2..955e4efa6 100644 --- a/src/include/distributed/pg_dist_shard_placement.h +++ b/src/include/distributed/pg_dist_shard_placement.h @@ -23,12 +23,12 @@ */ typedef struct FormData_pg_dist_shard_placement { - int64 shardid; /* global shardId on remote node */ - int32 shardstate; /* shard state on remote node; see RelayFileState */ - int64 shardlength; /* shard length on remote node; stored as bigint */ -#ifdef CATALOG_VARLEN /* variable-length fields start here */ - text nodename; /* remote node's host name */ - int32 nodeport; /* remote node's port number */ + int64 shardid; /* global shardId on remote node */ + int32 shardstate; /* shard state on remote node; see RelayFileState */ + int64 shardlength; /* shard length on remote node; stored as bigint */ +#ifdef CATALOG_VARLEN /* variable-length fields start here */ + text nodename; /* remote node's host name */ + int32 nodeport; /* remote node's port number */ #endif } FormData_pg_dist_shard_placement; @@ -43,12 +43,12 @@ typedef FormData_pg_dist_shard_placement *Form_pg_dist_shard_placement; * compiler constants for pg_dist_shard_placement * ---------------- */ -#define Natts_pg_dist_shard_placement 5 -#define Anum_pg_dist_shard_placement_shardid 1 -#define Anum_pg_dist_shard_placement_shardstate 2 -#define Anum_pg_dist_shard_placement_shardlength 3 -#define Anum_pg_dist_shard_placement_nodename 4 -#define Anum_pg_dist_shard_placement_nodeport 5 +#define Natts_pg_dist_shard_placement 5 +#define Anum_pg_dist_shard_placement_shardid 1 +#define Anum_pg_dist_shard_placement_shardstate 2 +#define Anum_pg_dist_shard_placement_shardlength 3 +#define Anum_pg_dist_shard_placement_nodename 4 +#define Anum_pg_dist_shard_placement_nodeport 5 #endif /* PG_DIST_SHARD_PLACEMENT_H */ diff --git a/src/include/distributed/relay_utility.h b/src/include/distributed/relay_utility.h index 592f61632..bd4657a01 100644 --- a/src/include/distributed/relay_utility.h +++ b/src/include/distributed/relay_utility.h @@ -3,7 +3,7 @@ * relay_utility.h * * Header and type declarations that extend relation, index and constraint names - * with the appropriate shard identifiers. + * with the appropriate shard identifiers. * * Copyright (c) 2012, Citus Data, Inc. * @@ -35,7 +35,6 @@ typedef enum FILE_CACHED = 2, FILE_INACTIVE = 3, FILE_TO_DELETE = 4 - } RelayFileState; diff --git a/src/include/distributed/resource_lock.h b/src/include/distributed/resource_lock.h index 6c1c8ffcf..1406da9c5 100644 --- a/src/include/distributed/resource_lock.h +++ b/src/include/distributed/resource_lock.h @@ -29,6 +29,7 @@ typedef enum AdvisoryLocktagClass /* values defined in postgres' lockfuncs.c */ ADV_LOCKTAG_CLASS_INT64 = 1, ADV_LOCKTAG_CLASS_INT32 = 2, + /* CitusDB lock types */ ADV_LOCKTAG_CLASS_CITUS_SHARD_METADATA = 4, ADV_LOCKTAG_CLASS_CITUS_SHARD = 5, diff --git a/src/include/distributed/task_tracker.h b/src/include/distributed/task_tracker.h index ad41b8589..2fc657d49 100644 --- a/src/include/distributed/task_tracker.h +++ b/src/include/distributed/task_tracker.h @@ -19,10 +19,10 @@ #include "utils/hsearch.h" -#define HIGH_PRIORITY_TASK_TIME 1 /* assignment time for high priority tasks */ -#define RESERVED_JOB_ID 1 /* reserved for cleanup and shutdown tasks */ +#define HIGH_PRIORITY_TASK_TIME 1 /* assignment time for high priority tasks */ +#define RESERVED_JOB_ID 1 /* reserved for cleanup and shutdown tasks */ #define SHUTDOWN_MARKER_TASK_ID UINT_MAX /* used to identify task tracker shutdown */ -#define MAX_TASK_FAILURE_COUNT 2 /* allowed failure count for one task */ +#define MAX_TASK_FAILURE_COUNT 2 /* allowed failure count for one task */ #define LOCAL_HOST_NAME "localhost" /* connect to local backends using this name */ #define TASK_CALL_STRING_SIZE 12288 /* max length of task call string */ #define TEMPLATE0_NAME "template0" /* skip job schema cleanup for template0 */ @@ -37,13 +37,13 @@ typedef enum { TASK_STATUS_INVALID_FIRST = 0, - TASK_ASSIGNED = 1, /* master node and task tracker */ + TASK_ASSIGNED = 1, /* master node and task tracker */ TASK_SCHEDULED = 2, TASK_RUNNING = 3, - TASK_FAILED = 4, + TASK_FAILED = 4, TASK_PERMANENTLY_FAILED = 5, TASK_SUCCEEDED = 6, - TASK_CANCEL_REQUESTED = 7, /* master node only */ + TASK_CANCEL_REQUESTED = 7, /* master node only */ TASK_CANCELED = 8, TASK_TO_REMOVE = 9, @@ -63,7 +63,6 @@ typedef enum * TASK_STATUS_LAST, should never have their numbers changed. */ TASK_STATUS_LAST - } TaskStatus; @@ -76,16 +75,15 @@ typedef enum */ typedef struct WorkerTask { - uint64 jobId; /* job id (upper 32-bits reserved); part of hash table key */ - uint32 taskId; /* task id; part of hash table key */ + uint64 jobId; /* job id (upper 32-bits reserved); part of hash table key */ + uint32 taskId; /* task id; part of hash table key */ uint32 assignedAt; /* task assignment time in epoch seconds */ char taskCallString[TASK_CALL_STRING_SIZE]; /* query or function call string */ - TaskStatus taskStatus; /* task's current execution status */ - char databaseName[NAMEDATALEN]; /* name to use for local backend connection */ - int32 connectionId; /* connection id to local backend */ - uint32 failureCount; /* number of task failures */ - + TaskStatus taskStatus; /* task's current execution status */ + char databaseName[NAMEDATALEN]; /* name to use for local backend connection */ + int32 connectionId; /* connection id to local backend */ + uint32 failureCount; /* number of task failures */ } WorkerTask; @@ -97,6 +95,7 @@ typedef struct WorkerTasksSharedStateData { /* Hash table shared by the task tracker and task tracker protocol functions */ HTAB *taskHash; + /* Lock protecting workerNodesHash */ LWLock *taskHashLock; } WorkerTasksSharedStateData; diff --git a/src/include/distributed/worker_manager.h b/src/include/distributed/worker_manager.h index 57a38194a..f23a659d0 100644 --- a/src/include/distributed/worker_manager.h +++ b/src/include/distributed/worker_manager.h @@ -43,12 +43,11 @@ */ typedef struct WorkerNode { - uint32 workerPort; /* node's port; part of hash table key */ + uint32 workerPort; /* node's port; part of hash table key */ char workerName[WORKER_LENGTH]; /* node's name; part of hash table key */ char workerRack[WORKER_LENGTH]; /* node's network location */ - bool inWorkerFile; /* is node in current membership file? */ - + bool inWorkerFile; /* is node in current membership file? */ } WorkerNode; diff --git a/src/include/distributed/worker_protocol.h b/src/include/distributed/worker_protocol.h index e797b7396..ac2985f6a 100644 --- a/src/include/distributed/worker_protocol.h +++ b/src/include/distributed/worker_protocol.h @@ -64,8 +64,7 @@ typedef struct RangePartitionContext { FmgrInfo *comparisonFunction; Datum *splitPointArray; - int32 splitPointCount; - + int32 splitPointCount; } RangePartitionContext; @@ -77,7 +76,6 @@ typedef struct HashPartitionContext { FmgrInfo *hashFunction; uint32 partitionCount; - } HashPartitionContext; @@ -88,16 +86,16 @@ typedef struct HashPartitionContext */ typedef struct PartialCopyStateData { - StringInfo fe_msgbuf; /* used for all dests during COPY TO, only for - * dest == COPY_NEW_FE in COPY FROM */ - int file_encoding; /* file or remote side's character encoding */ - bool need_transcoding; /* file encoding diff from server? */ - bool binary; /* binary format? */ - char *null_print; /* NULL marker string (server encoding!) */ - char *null_print_client; /* same converted to file encoding */ - char *delim; /* column delimiter (must be 1 byte) */ + StringInfo fe_msgbuf; /* used for all dests during COPY TO, only for + * dest == COPY_NEW_FE in COPY FROM */ + int file_encoding; /* file or remote side's character encoding */ + bool need_transcoding; /* file encoding diff from server? */ + bool binary; /* binary format? */ + char *null_print; /* NULL marker string (server encoding!) */ + char *null_print_client; /* same converted to file encoding */ + char *delim; /* column delimiter (must be 1 byte) */ - MemoryContext rowcontext; /* per-row evaluation context */ + MemoryContext rowcontext; /* per-row evaluation context */ } PartialCopyStateData; typedef struct PartialCopyStateData *PartialCopyState; @@ -114,7 +112,6 @@ typedef struct FileOutputStream File fileDescriptor; StringInfo fileBuffer; StringInfo filePath; - } FileOutputStream; From 1ea3f46194501b844bd6b16892cfdf34d23d3d62 Mon Sep 17 00:00:00 2001 From: Jason Petersen Date: Fri, 5 Feb 2016 16:53:04 -0700 Subject: [PATCH 07/19] Omit RangeVarCallbackForDropIndex from formatting I removed two braces to have this function remain more similar to the original PostgreSQL function and added uncrustify commands to disable formatting of its contents. --- src/backend/distributed/executor/multi_utility.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/distributed/executor/multi_utility.c b/src/backend/distributed/executor/multi_utility.c index be4148f75..84d74edf2 100644 --- a/src/backend/distributed/executor/multi_utility.c +++ b/src/backend/distributed/executor/multi_utility.c @@ -989,6 +989,7 @@ AllFinalizedPlacementsAccessible(Oid relationId) static void RangeVarCallbackForDropIndex(const RangeVar *rel, Oid relOid, Oid oldRelOid, void *arg) { + /* *INDENT-OFF* */ HeapTuple tuple; struct DropRelationCallbackState *state; char relkind; @@ -1023,10 +1024,8 @@ RangeVarCallbackForDropIndex(const RangeVar *rel, Oid relOid, Oid oldRelOid, voi classform = (Form_pg_class) GETSTRUCT(tuple); if (classform->relkind != relkind) - { ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("\"%s\" is not an index", rel->relname))); - } /* Allow DROP to either table owner or schema owner */ if (!pg_class_ownercheck(relOid, GetUserId()) && @@ -1055,4 +1054,5 @@ RangeVarCallbackForDropIndex(const RangeVar *rel, Oid relOid, Oid oldRelOid, voi if (OidIsValid(state->heapOid)) LockRelationOid(state->heapOid, heap_lockmode); } + /* *INDENT-ON* */ } From 628ed846a29aeb40324413dbe8cecbc53b4b5de1 Mon Sep 17 00:00:00 2001 From: Jason Petersen Date: Fri, 5 Feb 2016 17:19:43 -0700 Subject: [PATCH 08/19] Omit get_extension_schema from formatting It exactly matches the implementation in extension.c. --- src/backend/distributed/utils/citus_ruleutils.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/backend/distributed/utils/citus_ruleutils.c b/src/backend/distributed/utils/citus_ruleutils.c index 5f2c4a5ef..dda9a86fc 100644 --- a/src/backend/distributed/utils/citus_ruleutils.c +++ b/src/backend/distributed/utils/citus_ruleutils.c @@ -102,6 +102,7 @@ pg_get_extensiondef_string(Oid tableRelationId) static Oid get_extension_schema(Oid ext_oid) { + /* *INDENT-OFF* */ Oid result; Relation rel; SysScanDesc scandesc; @@ -131,6 +132,7 @@ get_extension_schema(Oid ext_oid) heap_close(rel, AccessShareLock); return result; + /* *INDENT-ON* */ } From 20ba6c659e3bf88537ca052c3d9ec17f221400b0 Mon Sep 17 00:00:00 2001 From: Jason Petersen Date: Sat, 6 Feb 2016 14:56:12 -0700 Subject: [PATCH 09/19] Omit backend/copy.c-inspired parts from formatting I think we need to assess whether this function is still as in-sync with upstream as we believe, but for now I'm omitting it from formatting. --- src/backend/distributed/worker/worker_partition_protocol.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/backend/distributed/worker/worker_partition_protocol.c b/src/backend/distributed/worker/worker_partition_protocol.c index 9ab21e588..9ad956ed6 100644 --- a/src/backend/distributed/worker/worker_partition_protocol.c +++ b/src/backend/distributed/worker/worker_partition_protocol.c @@ -1143,6 +1143,7 @@ OutputBinaryFooters(FileOutputStream *partitionFileArray, uint32 fileCount) } +/* *INDENT-OFF* */ /* Append data to the copy buffer in outputState */ static void CopySendData(PartialCopyState outputState, const void *databuf, int datasize) @@ -1282,6 +1283,7 @@ CopyAttributeOutText(PartialCopyState cstate, char *string) } +/* *INDENT-ON* */ /* Helper function to send pending copy output */ static inline void CopyFlushOutput(PartialCopyState cstate, char *start, char *pointer) From b48182574e1191c51ca500cf957fc9f6d3a893cd Mon Sep 17 00:00:00 2001 From: Jason Petersen Date: Mon, 8 Feb 2016 12:44:05 -0700 Subject: [PATCH 10/19] Make copy_options.ch similar to PostgreSQL copy.c We reorganized these functions in our copy; not sure why (makes diffing harder). I'm moving it back. --- src/bin/csql/copy_options.c | 30 +++++++++++++++--------------- src/bin/csql/copy_options.h | 2 +- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/bin/csql/copy_options.c b/src/bin/csql/copy_options.c index 357ae726c..da5dd5826 100644 --- a/src/bin/csql/copy_options.c +++ b/src/bin/csql/copy_options.c @@ -16,7 +16,21 @@ #include "stringutils.h" -/* Concatenates "more" onto "var", and frees the original value of *var. */ +void +free_copy_options(copy_options * ptr) +{ + if (!ptr) + return; + free(ptr->before_tofrom); + free(ptr->after_tofrom); + free(ptr->file); + free(ptr->tableName); + free(ptr->columnList); + free(ptr); +} + + +/* concatenate "more" onto "var", freeing the original value of *var */ static void xstrcat(char **var, const char *more) { @@ -212,20 +226,6 @@ error: /* Frees copy options. */ -void -free_copy_options(copy_options * ptr) -{ - if (!ptr) - return; - free(ptr->before_tofrom); - free(ptr->after_tofrom); - free(ptr->file); - free(ptr->tableName); - free(ptr->columnList); - free(ptr); -} - - /* * ParseStageOptions takes the given copy options, parses the additional options * needed for the \stage command, and sets them in the copy options structure. diff --git a/src/bin/csql/copy_options.h b/src/bin/csql/copy_options.h index 07a3aeb09..4a2e15222 100644 --- a/src/bin/csql/copy_options.h +++ b/src/bin/csql/copy_options.h @@ -46,7 +46,7 @@ typedef struct copy_options bool psql_inout; /* true = use psql stdin/stdout */ bool from; /* true = FROM, false = TO */ - char *tableName; /* table name to stage data to */ + char *tableName; /* table name to stage data to */ char *columnList; /* optional column list used in staging */ } copy_options; From 63f1497ecd7dbfa0fea2d716c79f00decc68a160 Mon Sep 17 00:00:00 2001 From: Jason Petersen Date: Mon, 8 Feb 2016 12:49:25 -0700 Subject: [PATCH 11/19] Omit most of copy_options from formatting Only a small portion is Citus style. --- src/bin/csql/copy_options.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/bin/csql/copy_options.c b/src/bin/csql/copy_options.c index da5dd5826..7c6542a98 100644 --- a/src/bin/csql/copy_options.c +++ b/src/bin/csql/copy_options.c @@ -16,6 +16,7 @@ #include "stringutils.h" +/* *INDENT-OFF* */ void free_copy_options(copy_options * ptr) { @@ -224,8 +225,10 @@ error: return NULL; } +/* *INDENT-ON* */ /* Frees copy options. */ + /* * ParseStageOptions takes the given copy options, parses the additional options * needed for the \stage command, and sets them in the copy options structure. From 3722d66c41611027f58e5c0b8579a6d7b11c9a51 Mon Sep 17 00:00:00 2001 From: Jason Petersen Date: Mon, 8 Feb 2016 13:19:52 -0700 Subject: [PATCH 12/19] Format csql's stage files These are entirely Citus-produced, so need full formatting. --- src/bin/csql/stage.c | 72 ++++++++++++++++++++------------------- src/bin/csql/stage.h | 80 ++++++++++++++++++++++---------------------- 2 files changed, 77 insertions(+), 75 deletions(-) diff --git a/src/bin/csql/stage.c b/src/bin/csql/stage.c index 1863b8bcb..de8bd5e9b 100644 --- a/src/bin/csql/stage.c +++ b/src/bin/csql/stage.c @@ -26,7 +26,8 @@ static bool FileSize(char *filename, uint64 *fileSize); static PGconn * ConnectToWorkerNode(const char *nodeName, uint32 nodePort, const char *nodeDatabase); -static PGresult * ExecuteRemoteCommand(PGconn *remoteConnection, const char *remoteCommand, +static PGresult * ExecuteRemoteCommand(PGconn *remoteConnection, + const char *remoteCommand, const char **parameterValues, int parameterCount); static TableMetadata * InitTableMetadata(const char *tableName); static ShardMetadata * InitShardMetadata(int shardPlacementPolicy); @@ -41,7 +42,8 @@ static uint64 GetValueUint64(const PGresult *result, int rowNumber, int columnNu static bool MasterGetTableMetadata(const char *tableName, TableMetadata *tableMetadata); static bool MasterGetTableDDLEvents(const char *tableName, TableMetadata *tableMetadata); static bool MasterGetNewShardId(ShardMetadata *shardMetadata); -static bool MasterGetCandidateNodes(ShardMetadata *shardMetadata, int shardPlacementPolicy); +static bool MasterGetCandidateNodes(ShardMetadata *shardMetadata, + int shardPlacementPolicy); static bool MasterInsertShardRow(uint32 logicalRelid, char storageType, const ShardMetadata *shardMetadata); static bool MasterInsertPlacementRows(const ShardMetadata *shardMetadata); @@ -62,7 +64,8 @@ static bool ApplyShardDDLCommand(PGconn *workerNode, uint64 shardId, const char static bool TransmitTableData(PGconn *workerNode, uint64 shardId, uint64 shardMaxSize, copy_options *stageOptions, uint64 currentFileOffset, uint64 *nextFileOffset); -static bool TransmitFile(PGconn *workerNode, const char *localPath, const char *remotePath); +static bool TransmitFile(PGconn *workerNode, const char *localPath, + const char *remotePath); static bool FileStreamOK(const copy_options *stageOptions); static PQExpBuffer CreateCopyQueryString(const char *tableName, const char *columnList, const char *afterToFrom); @@ -166,7 +169,7 @@ DoStageData(const char *stageCommand) if (partitionMethod == DISTRIBUTE_BY_HASH) { psql_error("\\stage: staging data into hash partitioned tables is not " - "supported\n"); + "supported\n"); free_copy_options(stageOptions); FreeTableMetadata(tableMetadata); @@ -179,7 +182,7 @@ DoStageData(const char *stageCommand) bool tableOptionsOK = ColumnarTableOptionsOK(tableMetadata->logicalRelid); if (!tableOptionsOK) { - return false; /* error message already displayed */ + return false; /* error message already displayed */ } } @@ -225,7 +228,7 @@ DoStageData(const char *stageCommand) */ FreeCommonStageData(stageOptions, tableMetadata, shardMetadataList); - return false; /* abort immediately */ + return false; /* abort immediately */ } /* save allocated shard metadata */ @@ -245,7 +248,7 @@ DoStageData(const char *stageCommand) */ for (nodeIndex = 0; nodeIndex < shardMetadata->nodeCount; nodeIndex++) { - char *remoteNodeName = shardMetadata->nodeNameList[nodeIndex]; + char *remoteNodeName = shardMetadata->nodeNameList[nodeIndex]; uint32 remoteNodePort = shardMetadata->nodePortList[nodeIndex]; PGconn *remoteNode = NULL; @@ -341,7 +344,6 @@ DoStageData(const char *stageCommand) /* update current file offset */ currentFileOffset = nextFileOffset; - } /* while more file data left for sharding */ /* @@ -390,9 +392,9 @@ ConnectToWorkerNode(const char *nodeName, uint32 nodePort, const char *nodeDatab char nodePortString[MAXPGPATH]; char connInfoString[MAXPGPATH]; - /* transcribe port number and connection info to their string values */ + /* transcribe port number and connection info to their string values */ snprintf(nodePortString, MAXPGPATH, "%u", nodePort); - snprintf(connInfoString, MAXPGPATH, CONN_INFO_TEMPLATE, + snprintf(connInfoString, MAXPGPATH, CONN_INFO_TEMPLATE, nodeDatabase, CLIENT_CONNECT_TIMEOUT); workerNode = PQsetdb(nodeName, nodePortString, nodeOptions, nodeTty, connInfoString); @@ -421,16 +423,16 @@ ExecuteRemoteCommand(PGconn *remoteConnection, const char *remoteCommand, { PGresult *result = NULL; - const Oid *parameterType = NULL; /* let the backend deduce type */ + const Oid *parameterType = NULL; /* let the backend deduce type */ const int *parameterLength = NULL; /* text params do not need length */ const int *parameterFormat = NULL; /* text params have Null by default */ - const int resultFormat = 0; /* ask for results in text format */ + const int resultFormat = 0; /* ask for results in text format */ result = PQexecParams(remoteConnection, remoteCommand, parameterCount, parameterType, parameterValues, parameterLength, parameterFormat, resultFormat); - if (PQresultStatus(result) != PGRES_COMMAND_OK && + if (PQresultStatus(result) != PGRES_COMMAND_OK && PQresultStatus(result) != PGRES_TUPLES_OK) { psql_error("remote command \"%s\" failed with %s", @@ -488,7 +490,7 @@ FreeTableMetadata(TableMetadata *tableMetadata) for (eventIndex = 0; eventIndex < eventCount; eventIndex++) { char *ddlEvent = tableMetadata->ddlEventList[eventIndex]; - + free(ddlEvent); ddlEvent = NULL; } @@ -552,7 +554,7 @@ FreeShardMetadata(ShardMetadata *shardMetadata) for (nodeIndex = 0; nodeIndex < nodeCount; nodeIndex++) { char *nodeName = shardMetadata->nodeNameList[nodeIndex]; - + free(nodeName); nodeName = NULL; } @@ -655,7 +657,7 @@ ExtendTablename(const char *baseTablename, uint64 shardId) { char *extendedTablename = (char *) pg_malloc0(NAMEDATALEN); - snprintf(extendedTablename, NAMEDATALEN, "%s%c" UINT64_FORMAT, + snprintf(extendedTablename, NAMEDATALEN, "%s%c" UINT64_FORMAT, baseTablename, SHARD_NAME_SEPARATOR, shardId); return extendedTablename; @@ -678,7 +680,7 @@ GetValueUint64(const PGresult *result, int rowNumber, int columnNumber) errno = 0; value = strtoull(valueString, &valueStringEnd, 0); - + if (errno != 0 || (*valueStringEnd) != '\0') { return INVALID_UINT64; @@ -716,7 +718,7 @@ MasterGetTableMetadata(const char *tableName, TableMetadata *tableMetadata) char *tableStorageType = NULL; char *partitionMethod = NULL; char *partitionKey = NULL; - int partitionKeyLength = 0; + int partitionKeyLength = 0; uint64 logicalRelid = 0; uint64 shardReplicaCount = 0; uint64 shardMaxSize = 0; @@ -727,7 +729,7 @@ MasterGetTableMetadata(const char *tableName, TableMetadata *tableMetadata) parameterValue, parameterCount); if (result == NULL) { - return false; /* error message already displayed */ + return false; /* error message already displayed */ } /* find column numbers associated with column names */ @@ -798,13 +800,13 @@ MasterGetTableDDLEvents(const char *tableName, TableMetadata *tableMetadata) int ddlEventIndex = 0; /* fetch DDL events needed for table creation */ - result = ExecuteRemoteCommand(masterNode, remoteCommand, + result = ExecuteRemoteCommand(masterNode, remoteCommand, parameterValue, parameterCount); if (result == NULL) { return false; } - + /* check that we have at least one DDL event */ ddlEventCount = PQntuples(result); if (ddlEventCount <= 0) @@ -825,7 +827,7 @@ MasterGetTableDDLEvents(const char *tableName, TableMetadata *tableMetadata) { char *ddlEvent = NULL; char *ddlEventValue = PQgetvalue(result, ddlEventIndex, 0); - int ddlEventLength = PQgetlength(result, ddlEventIndex, 0); + int ddlEventLength = PQgetlength(result, ddlEventIndex, 0); if (ddlEventLength <= 0) { @@ -866,7 +868,7 @@ MasterGetNewShardId(ShardMetadata *shardMetadata) uint64 shardId = 0; /* fetch unique shardId for shard to be created */ - result = ExecuteRemoteCommand(masterNode, remoteCommand, + result = ExecuteRemoteCommand(masterNode, remoteCommand, parameterValue, parameterCount); if (result == NULL) { @@ -877,7 +879,7 @@ MasterGetNewShardId(ShardMetadata *shardMetadata) shardId = GetValueUint64(result, 0, 0); if (shardId == INVALID_UINT64) { - psql_error("remote command \"%s\" failed with invalid shardId\n", + psql_error("remote command \"%s\" failed with invalid shardId\n", remoteCommand); PQclear(result); @@ -996,11 +998,11 @@ MasterGetCandidateNodes(ShardMetadata *shardMetadata, int shardPlacementPolicy) /* walk over fetched node name/port list, and assign them to metadata */ for (nodeIndex = 0; nodeIndex < nodeCount; nodeIndex++) { - char *nodeName = NULL; + char *nodeName = NULL; uint64 nodePort = 0; - char *nodeNameValue = PQgetvalue(result, nodeIndex, nodeNameIndex); - int nodeNameLength = PQgetlength(result, nodeIndex, nodeNameIndex); + char *nodeNameValue = PQgetvalue(result, nodeIndex, nodeNameIndex); + int nodeNameLength = PQgetlength(result, nodeIndex, nodeNameIndex); if (nodeNameLength <= 0) { @@ -1014,7 +1016,7 @@ MasterGetCandidateNodes(ShardMetadata *shardMetadata, int shardPlacementPolicy) /* deep copy node name and assign to metadata */ nodeName = (char *) pg_malloc0(nodeNameLength + 1); strncpy(nodeName, nodeNameValue, nodeNameLength + 1); - + shardMetadata->nodeNameList[nodeIndex] = nodeName; /* convert port value string to 64-bit integer, and assign to metadata */ @@ -1024,7 +1026,7 @@ MasterGetCandidateNodes(ShardMetadata *shardMetadata, int shardPlacementPolicy) psql_error("remote command \"%s\" failed to fetch valid port number\n", remoteCommand); PQclear(result); - + return false; } @@ -1107,12 +1109,12 @@ MasterInsertPlacementRows(const ShardMetadata *shardMetadata) bool staged = shardMetadata->nodeStageList[nodeIndex]; if (staged) { - char *nodeName = shardMetadata->nodeNameList[nodeIndex]; + char *nodeName = shardMetadata->nodeNameList[nodeIndex]; uint32 nodePort = shardMetadata->nodePortList[nodeIndex]; /* convert parameter to its string representation */ snprintf(nodePortString, NAMEDATALEN, "%u", nodePort); - + parameterValue[3] = nodeName; parameterValue[4] = nodePortString; @@ -1136,7 +1138,7 @@ MasterInsertPlacementRows(const ShardMetadata *shardMetadata) * staged to worker nodes. The function executes shard metadata insert commands * within a single transaction so that either all or none of the metadata are * finalized. On success, the function commits the transaction and returns true. - * On failure, the function rolls back the transaction and returns false. + * On failure, the function rolls back the transaction and returns false. */ static bool MasterInsertShardMetadata(uint32 logicalRelid, char storageType, @@ -1199,7 +1201,7 @@ IssueTransactionCommand(PGconn *connection, const char *command) return false; } - + PQclear(result); return true; } @@ -1729,7 +1731,7 @@ ShardColumnarTableSize(PGconn *workerNode, const char *tablename, uint64 shardId * failure, the function returns false. */ static bool -ShardMinMaxValues(PGconn *workerNode, const char *tablename, +ShardMinMaxValues(PGconn *workerNode, const char *tablename, const char *partitionKey, ShardMetadata *shardMetadata) { const int MinValueIndex = 0; @@ -1744,7 +1746,7 @@ ShardMinMaxValues(PGconn *workerNode, const char *tablename, int maxValueLength = 0; extendedTablename = ExtendTablename(tablename, shardMetadata->shardId); - snprintf(remoteCommand, MAXPGPATH, SHARD_MIN_MAX_COMMAND, + snprintf(remoteCommand, MAXPGPATH, SHARD_MIN_MAX_COMMAND, partitionKey, partitionKey, extendedTablename); result = PQexec(workerNode, remoteCommand); diff --git a/src/bin/csql/stage.h b/src/bin/csql/stage.h index 01575f886..0863adff5 100644 --- a/src/bin/csql/stage.h +++ b/src/bin/csql/stage.h @@ -30,42 +30,44 @@ #define ROLLBACK_COMMAND "ROLLBACK" /* Names of remote function calls to execute on the master. */ -#define MASTER_GET_TABLE_METADATA "SELECT * FROM master_get_table_metadata($1::text)" +#define MASTER_GET_TABLE_METADATA "SELECT * FROM master_get_table_metadata($1::text)" #define MASTER_GET_TABLE_DDL_EVENTS "SELECT * FROM master_get_table_ddl_events($1::text)" -#define MASTER_GET_NEW_SHARDID "SELECT * FROM master_get_new_shardid()" -#define MASTER_GET_LOCAL_FIRST_CANDIDATE_NODES "SELECT * FROM \ - master_get_local_first_candidate_nodes()" -#define MASTER_GET_ROUND_ROBIN_CANDIDATE_NODES "SELECT * FROM \ - master_get_round_robin_candidate_nodes($1::int8)" +#define MASTER_GET_NEW_SHARDID "SELECT * FROM master_get_new_shardid()" +#define MASTER_GET_LOCAL_FIRST_CANDIDATE_NODES \ + "SELECT * FROM master_get_local_first_candidate_nodes()" +#define MASTER_GET_ROUND_ROBIN_CANDIDATE_NODES \ + "SELECT * FROM master_get_round_robin_candidate_nodes($1::int8)" -#define MASTER_INSERT_SHARD_ROW "INSERT INTO pg_dist_shard \ - (logicalrelid, shardid, shardstorage, shardminvalue, shardmaxvalue) VALUES \ - ($1::oid, $2::int8, $3::char, $4::text, $5::text)" -#define MASTER_INSERT_PLACEMENT_ROW "INSERT INTO pg_dist_shard_placement \ - (shardid, shardstate, shardlength, nodename, nodeport) VALUES \ - ($1::int8, $2::int4, $3::int8, $4::text, $5::int4)" +#define MASTER_INSERT_SHARD_ROW \ + "INSERT INTO pg_dist_shard " \ + "(logicalrelid, shardid, shardstorage, shardminvalue, shardmaxvalue) VALUES " \ + "($1::oid, $2::int8, $3::char, $4::text, $5::text)" +#define MASTER_INSERT_PLACEMENT_ROW \ + "INSERT INTO pg_dist_shard_placement " \ + "(shardid, shardstate, shardlength, nodename, nodeport) VALUES " \ + "($1::int8, $2::int4, $3::int8, $4::text, $5::int4)" /* Column names used to identify response fields as returned from the master. */ -#define LOGICAL_RELID_FIELD "logical_relid" -#define PART_STORAGE_TYPE_FIELD "part_storage_type" -#define PART_METHOD_FIELD "part_method" -#define PART_KEY_FIELD "part_key" -#define PART_REPLICA_COUNT_FIELD "part_replica_count" -#define PART_MAX_SIZE_FIELD "part_max_size" -#define PART_PLACEMENT_POLICY_FIELD "part_placement_policy" -#define NODE_NAME_FIELD "node_name" -#define NODE_PORT_FIELD "node_port" +#define LOGICAL_RELID_FIELD "logical_relid" +#define PART_STORAGE_TYPE_FIELD "part_storage_type" +#define PART_METHOD_FIELD "part_method" +#define PART_KEY_FIELD "part_key" +#define PART_REPLICA_COUNT_FIELD "part_replica_count" +#define PART_MAX_SIZE_FIELD "part_max_size" +#define PART_PLACEMENT_POLICY_FIELD "part_placement_policy" +#define NODE_NAME_FIELD "node_name" +#define NODE_PORT_FIELD "node_port" /* the tablename in the overloaded COPY statement is the to-be-transferred file */ #define TRANSMIT_REGULAR_COMMAND "COPY \"%s\" FROM STDIN WITH (format 'transmit')" -#define SHARD_MIN_MAX_COMMAND "SELECT min(%s), max(%s) FROM %s" +#define SHARD_MIN_MAX_COMMAND "SELECT min(%s), max(%s) FROM %s" #define SHARD_TABLE_SIZE_COMMAND "SELECT pg_table_size('%s')" #define SET_FOREIGN_TABLE_FILENAME "ALTER FOREIGN TABLE %s OPTIONS (SET filename '%s')" -#define GET_COLUMNAR_TABLE_FILENAME_OPTION "SELECT * FROM \ - (SELECT (pg_options_to_table(ftoptions)).* FROM pg_foreign_table \ - WHERE ftrelid = %u) AS Q WHERE option_name = 'filename';" -#define APPLY_SHARD_DDL_COMMAND "SELECT * FROM worker_apply_shard_ddl_command \ - ($1::int8, $2::text)" +#define GET_COLUMNAR_TABLE_FILENAME_OPTION \ + "SELECT * FROM (SELECT (pg_options_to_table(ftoptions)).* FROM pg_foreign_table " \ + "WHERE ftrelid = %u) AS Q WHERE option_name = 'filename';" +#define APPLY_SHARD_DDL_COMMAND \ + "SELECT * FROM worker_apply_shard_ddl_command ($1::int8, $2::text)" #define REMOTE_FILE_SIZE_COMMAND "SELECT size FROM pg_stat_file('%s')" #define SHARD_COLUMNAR_TABLE_SIZE_COMMAND "SELECT cstore_table_size('%s')" @@ -90,17 +92,16 @@ */ typedef struct TableMetadata { - uint32 logicalRelid; /* table's relationId on the master */ - char tableStorageType; /* relay file, foreign table, or table */ - char partitionMethod; /* table's partition method */ - char *partitionKey; /* partition key expression */ - uint32 shardReplicaCount; /* shard replication factor */ - uint64 shardMaxSize; /* create new shard when shard reaches max size */ + uint32 logicalRelid; /* table's relationId on the master */ + char tableStorageType; /* relay file, foreign table, or table */ + char partitionMethod; /* table's partition method */ + char *partitionKey; /* partition key expression */ + uint32 shardReplicaCount; /* shard replication factor */ + uint64 shardMaxSize; /* create new shard when shard reaches max size */ uint32 shardPlacementPolicy; /* policy to use when choosing nodes to place shards */ char **ddlEventList; /* DDL statements used for creating new shard */ uint32 ddlEventCount; /* DDL statement count; statement list size */ - } TableMetadata; @@ -112,17 +113,16 @@ typedef struct TableMetadata */ typedef struct ShardMetadata { - uint64 shardId; /* global shardId; created on the master node */ + uint64 shardId; /* global shardId; created on the master node */ - char **nodeNameList; /* candidate node name list for shard uploading */ + char **nodeNameList; /* candidate node name list for shard uploading */ uint32 *nodePortList; /* candidate node port list for shard uploading */ - uint32 nodeCount; /* candidate node count; node list size */ - bool *nodeStageList; /* shard uploaded to corresponding candidate node? */ + uint32 nodeCount; /* candidate node count; node list size */ + bool *nodeStageList; /* shard uploaded to corresponding candidate node? */ char *shardMinValue; /* partition key's minimum value in shard */ char *shardMaxValue; /* partition key's maximum value in shard */ - uint64 shardSize; /* shard size; updated during staging */ - + uint64 shardSize; /* shard size; updated during staging */ } ShardMetadata; From f249d9f3a49d7abadc20c635ad178c21a9146fe0 Mon Sep 17 00:00:00 2001 From: Jason Petersen Date: Wed, 10 Feb 2016 12:34:32 -0700 Subject: [PATCH 13/19] Switch to using git attributes to ignore files Ties into the script introduced [here][1]. [1]: https://github.com/citusdata/tools/pull/2 --- .gitattributes | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.gitattributes b/.gitattributes index e3ee9cf5b..5eb7bda7b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -20,3 +20,23 @@ src/test/regress/output/*.source -whitespace # These files are maintained or generated elsewhere. We take them as is. configure -whitespace + +# all C files (implementation and header) use our style... +*.[ch] citus-style + +# except these exceptions... +src/backend/distributed/utils/citus_outfuncs.c -citus-style +src/backend/distributed/utils/citus_read.c -citus-style +src/backend/distributed/utils/citus_readfuncs_94.c -citus-style +src/backend/distributed/utils/citus_readfuncs_95.c -citus-style +src/backend/distributed/utils/ruleutils_94.c -citus-style +src/backend/distributed/utils/ruleutils_95.c -citus-style +src/include/distributed/citus_nodes.h -citus-style +src/include/dumputils.h -citus-style + +# all csql files use PostgreSQL style... +src/bin/csql/*.[ch] -citus-style + +# except these exceptions +src/bin/csql/copy_options.c citus-style +src/bin/csql/stage.[ch] citus-style From 44d7721b4cbff6be547f1a0d404c0e3b8e5f9c03 Mon Sep 17 00:00:00 2001 From: Murat Tuncer Date: Fri, 12 Feb 2016 14:41:32 +0200 Subject: [PATCH 14/19] Add support for appending to cstore table shards - Flexed the check which prevented append operation cstore tables since its storage type is not SHARD_STORAGE_TABLE. - Used process utility function to perform copy operation in worker_append_table_to shard() instead of directly calling postgresql DoCopy(). - Removed the additional check in master_create_empty_shard() function. This check was redundant and erroneous since it was called after CheckDistributedTable() call. - Modified WorkerTableSize() function to retrieve cstore table shard size correctly. --- .../master/master_stage_protocol.c | 34 ++++++++++++------- .../worker/worker_data_fetch_protocol.c | 12 ++++--- src/include/distributed/master_protocol.h | 1 + 3 files changed, 31 insertions(+), 16 deletions(-) diff --git a/src/backend/distributed/master/master_stage_protocol.c b/src/backend/distributed/master/master_stage_protocol.c index 1c13237cc..388394856 100644 --- a/src/backend/distributed/master/master_stage_protocol.c +++ b/src/backend/distributed/master/master_stage_protocol.c @@ -45,7 +45,8 @@ static bool WorkerCreateShard(char *nodeName, uint32 nodePort, static bool WorkerShardStats(char *nodeName, uint32 nodePort, Oid relationId, char *shardName, uint64 *shardLength, text **shardMinValue, text **shardMaxValue); -static uint64 WorkerTableSize(char *nodeName, uint32 nodePort, char *tableName); +static uint64 WorkerTableSize(char *nodeName, uint32 nodePort, Oid relationId, + char *tableName); static StringInfo WorkerPartitionValue(char *nodeName, uint32 nodePort, Oid relationId, char *shardName, char *selectQuery); @@ -77,16 +78,15 @@ master_create_empty_shard(PG_FUNCTION_ARGS) List *candidateNodeList = NIL; text *nullMinValue = NULL; text *nullMaxValue = NULL; - char tableType = 0; char partitionMethod = 0; + char storageType = SHARD_STORAGE_TABLE; Oid relationId = ResolveRelationId(relationNameText); CheckDistributedTable(relationId); - tableType = get_rel_relkind(relationId); - if (tableType != RELKIND_RELATION) + if (CStoreTable(relationId)) { - ereport(ERROR, (errmsg("relation \"%s\" is not a regular table", relationName))); + storageType = SHARD_STORAGE_COLUMNAR; } partitionMethod = PartitionMethod(relationId); @@ -130,7 +130,7 @@ master_create_empty_shard(PG_FUNCTION_ARGS) CreateShardPlacements(shardId, ddlEventList, candidateNodeList, 0, ShardReplicationFactor); - InsertShardRow(relationId, shardId, SHARD_STORAGE_TABLE, nullMinValue, nullMaxValue); + InsertShardRow(relationId, shardId, storageType, nullMinValue, nullMaxValue); PG_RETURN_INT64(shardId); } @@ -171,9 +171,10 @@ master_append_table_to_shard(PG_FUNCTION_ARGS) ShardInterval *shardInterval = LoadShardInterval(shardId); Oid relationId = shardInterval->relationId; + bool cstoreTable = CStoreTable(relationId); char storageType = shardInterval->storageType; - if (storageType != SHARD_STORAGE_TABLE) + if (storageType != SHARD_STORAGE_TABLE && !cstoreTable) { ereport(ERROR, (errmsg("cannot append to shardId " UINT64_FORMAT, shardId), errdetail("The underlying shard is not a regular table"))); @@ -457,7 +458,7 @@ WorkerShardStats(char *nodeName, uint32 nodePort, Oid relationId, char *shardNam PG_TRY(); { - uint64 tableSize = WorkerTableSize(nodeName, nodePort, shardName); + uint64 tableSize = WorkerTableSize(nodeName, nodePort, relationId, shardName); StringInfo minValue = WorkerPartitionValue(nodeName, nodePort, relationId, shardName, SHARD_MIN_VALUE_QUERY); StringInfo maxValue = WorkerPartitionValue(nodeName, nodePort, relationId, @@ -479,18 +480,27 @@ WorkerShardStats(char *nodeName, uint32 nodePort, Oid relationId, char *shardNam /* * WorkerTableSize queries the worker node to extract the disk space used by the - * given relation. The function assumes the relation represents a regular table. + * given relation. The function assumes the relation represents a regular table or + * a cstore_fdw table. */ static uint64 -WorkerTableSize(char *nodeName, uint32 nodePort, char *tableName) +WorkerTableSize(char *nodeName, uint32 nodePort, Oid relationId, char *tableName) { uint64 tableSize = 0; List *queryResultList = NIL; StringInfo tableSizeString = NULL; char *tableSizeStringEnd = NULL; - + bool cstoreTable = CStoreTable(relationId); StringInfo tableSizeQuery = makeStringInfo(); - appendStringInfo(tableSizeQuery, SHARD_TABLE_SIZE_QUERY, tableName); + + if (cstoreTable) + { + appendStringInfo(tableSizeQuery, SHARD_CSTORE_TABLE_SIZE_QUERY, tableName); + } + else + { + appendStringInfo(tableSizeQuery, SHARD_TABLE_SIZE_QUERY, tableName); + } queryResultList = ExecuteRemoteQuery(nodeName, nodePort, tableSizeQuery); if (queryResultList == NIL) diff --git a/src/backend/distributed/worker/worker_data_fetch_protocol.c b/src/backend/distributed/worker/worker_data_fetch_protocol.c index 0e5b68a1d..10cd02324 100644 --- a/src/backend/distributed/worker/worker_data_fetch_protocol.c +++ b/src/backend/distributed/worker/worker_data_fetch_protocol.c @@ -994,11 +994,10 @@ worker_append_table_to_shard(PG_FUNCTION_ARGS) StringInfo remoteCopyCommand = NULL; CopyStmt *localCopyCommand = NULL; RangeVar *localTable = NULL; - uint64 copiedRowCount = 0; uint64 shardId = INVALID_SHARD_ID; bool received = false; char *quotedTableName = NULL; - const char *queryString = NULL; + StringInfo queryString = NULL; const char *schemaName = NULL; /* copy remote table's data to this node */ @@ -1032,8 +1031,13 @@ worker_append_table_to_shard(PG_FUNCTION_ARGS) localTable = makeRangeVar((char *) schemaName, shardNameString->data, -1); localCopyCommand = CopyStatement(localTable, localFilePath->data); - DoCopy(localCopyCommand, queryString, &copiedRowCount); - (void) copiedRowCount; + quotedTableName = quote_qualified_identifier(schemaName, shardNameString->data); + + queryString = makeStringInfo(); + appendStringInfo(queryString, COPY_IN_COMMAND, quotedTableName, localFilePath->data); + + ProcessUtility((Node *) localCopyCommand, queryString->data, + PROCESS_UTILITY_TOPLEVEL, NULL, None_Receiver, NULL); /* finally delete the temporary file we created */ DeleteFile(localFilePath->data); diff --git a/src/include/distributed/master_protocol.h b/src/include/distributed/master_protocol.h index f39ce865b..daa3b2414 100644 --- a/src/include/distributed/master_protocol.h +++ b/src/include/distributed/master_protocol.h @@ -56,6 +56,7 @@ #define SHARD_MIN_VALUE_QUERY "SELECT min(%s) FROM %s" #define SHARD_MAX_VALUE_QUERY "SELECT max(%s) FROM %s" #define SHARD_TABLE_SIZE_QUERY "SELECT pg_table_size('%s')" +#define SHARD_CSTORE_TABLE_SIZE_QUERY "SELECT cstore_table_size('%s')" #define DROP_REGULAR_TABLE_COMMAND "DROP TABLE IF EXISTS %s" #define DROP_FOREIGN_TABLE_COMMAND "DROP FOREIGN TABLE IF EXISTS %s" #define CREATE_SCHEMA_COMMAND "CREATE SCHEMA IF NOT EXISTS %s" From 130e65f5be38fb1595e2da28ed2e2c2de56864d6 Mon Sep 17 00:00:00 2001 From: Jason Petersen Date: Tue, 16 Feb 2016 11:20:18 -0700 Subject: [PATCH 15/19] Ensure router executor acquires proper shard lock Though Citus' Task struct has a shardId field, it doesn't have the same semantics as the one previously used in pg_shard code. The analogous field in the Citus Task is anchorShardId. I've also added an argument check to the relevant locking function to catch future locking attempts which pass an invalid argument. --- src/backend/distributed/executor/multi_router_executor.c | 2 +- src/backend/distributed/utils/resource_lock.c | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/backend/distributed/executor/multi_router_executor.c b/src/backend/distributed/executor/multi_router_executor.c index 5c2f04165..d9900798f 100644 --- a/src/backend/distributed/executor/multi_router_executor.c +++ b/src/backend/distributed/executor/multi_router_executor.c @@ -153,7 +153,7 @@ CommutativityRuleToLockMode(CmdType commandType, bool upsertQuery) static void AcquireExecutorShardLock(Task *task, LOCKMODE lockMode) { - int64 shardId = task->shardId; + int64 shardId = task->anchorShardId; LockShardResource(shardId, lockMode); } diff --git a/src/backend/distributed/utils/resource_lock.c b/src/backend/distributed/utils/resource_lock.c index a2552d46b..fecd703d1 100644 --- a/src/backend/distributed/utils/resource_lock.c +++ b/src/backend/distributed/utils/resource_lock.c @@ -14,9 +14,10 @@ */ #include "postgres.h" - +#include "c.h" #include "miscadmin.h" +#include "distributed/relay_utility.h" #include "distributed/resource_lock.h" #include "storage/lmgr.h" @@ -68,6 +69,8 @@ LockShardResource(uint64 shardId, LOCKMODE lockmode) const bool sessionLock = false; const bool dontWait = false; + AssertArg(shardId != INVALID_SHARD_ID); + SET_LOCKTAG_SHARD_RESOURCE(tag, MyDatabaseId, shardId); (void) LockAcquire(&tag, lockmode, sessionLock, dontWait); From ce37149dcc03ed29b9260b95470351065eec577a Mon Sep 17 00:00:00 2001 From: Jason Petersen Date: Tue, 16 Feb 2016 11:34:01 -0700 Subject: [PATCH 16/19] Add make targets for applying and checking style Need to change to the project's top srcdir, as citus_indent expects to be able to find styled files using git ls-files, and VPATH builds would otherwise not return any results. --- Makefile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Makefile b/Makefile index e024ef4ff..7131e2ead 100644 --- a/Makefile +++ b/Makefile @@ -42,6 +42,13 @@ clean-csql: install: install-csql clean: clean-csql +# apply or check style +reindent: + cd ${citusdb_abs_top_srcdir} && citus_indent --quiet +check-style: + cd ${citusdb_abs_top_srcdir} && citus_indent --quiet --check +.PHONY: reindent check-style + # depend on install for now check: all install $(MAKE) -C src/test/regress check-full From 87ff558c1c7d1ec23db6b1814f4c7b824dc4c572 Mon Sep 17 00:00:00 2001 From: Metin Doslu Date: Fri, 12 Feb 2016 19:12:42 +0200 Subject: [PATCH 17/19] Add check for count distinct on single table subqueries Fixes #314 --- .../planner/multi_logical_optimizer.c | 19 +++++++++++++++++-- .../multi_single_relation_subquery.out | 12 ++++++++++++ .../sql/multi_single_relation_subquery.sql | 12 ++++++++++++ 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/src/backend/distributed/planner/multi_logical_optimizer.c b/src/backend/distributed/planner/multi_logical_optimizer.c index 7acf85d7e..787937079 100644 --- a/src/backend/distributed/planner/multi_logical_optimizer.c +++ b/src/backend/distributed/planner/multi_logical_optimizer.c @@ -259,6 +259,7 @@ MultiLogicalPlanOptimize(MultiTreeRoot *multiLogicalPlan) MultiTable *tableNode = (MultiTable *) lfirst(tableNodeCell); if (tableNode->relationId == SUBQUERY_RELATION_ID) { + ErrorIfContainsUnsupportedAggregate((MultiNode *) tableNode); TransformSubqueryNode(tableNode); } } @@ -2145,8 +2146,9 @@ ErrorIfUnsupportedAggregateDistinct(Aggref *aggregateExpression, bool distinctSupported = true; List *repartitionNodeList = NIL; Var *distinctColumn = NULL; - - AggregateType aggregateType = GetAggregateType(aggregateExpression->aggfnoid); + List *multiTableNodeList = NIL; + ListCell *multiTableNodeCell = NULL; + AggregateType aggregateType = AGGREGATE_INVALID_FIRST; /* check if logical plan includes a subquery */ List *subqueryMultiTableList = SubqueryMultiTableList(logicalPlanNode); @@ -2157,7 +2159,20 @@ ErrorIfUnsupportedAggregateDistinct(Aggref *aggregateExpression, errdetail("distinct in the outermost query is unsupported"))); } + multiTableNodeList = FindNodesOfType(logicalPlanNode, T_MultiTable); + foreach(multiTableNodeCell, multiTableNodeList) + { + MultiTable *multiTable = (MultiTable *) lfirst(multiTableNodeCell); + if (multiTable->relationId == SUBQUERY_RELATION_ID) + { + ereport(ERROR, (errmsg("cannot compute count (distinct)"), + errdetail("Subqueries with aggregate (distinct) are " + "not supported yet"))); + } + } + /* if we have a count(distinct), and distinct approximation is enabled */ + aggregateType = GetAggregateType(aggregateExpression->aggfnoid); if (aggregateType == AGGREGATE_COUNT && CountDistinctErrorRate != DISABLE_DISTINCT_APPROXIMATION) { diff --git a/src/test/regress/expected/multi_single_relation_subquery.out b/src/test/regress/expected/multi_single_relation_subquery.out index 1573e3a0a..3a01f8a3a 100644 --- a/src/test/regress/expected/multi_single_relation_subquery.out +++ b/src/test/regress/expected/multi_single_relation_subquery.out @@ -171,6 +171,18 @@ from l_tax) as distributed_table; ERROR: cannot perform distributed planning on this query DETAIL: Subqueries without aggregates are not supported yet +-- Check that we don't support subqueries with count(distinct). +select + different_shipment_days +from + (select + count(distinct l_shipdate) as different_shipment_days + from + lineitem + group by + l_partkey) as distributed_table; +ERROR: cannot compute count (distinct) +DETAIL: Subqueries with aggregate (distinct) are not supported yet -- Check that if subquery is pulled, we don't error and run query properly. SELECT max(l_suppkey) FROM ( diff --git a/src/test/regress/sql/multi_single_relation_subquery.sql b/src/test/regress/sql/multi_single_relation_subquery.sql index 19e56f60e..08853b639 100644 --- a/src/test/regress/sql/multi_single_relation_subquery.sql +++ b/src/test/regress/sql/multi_single_relation_subquery.sql @@ -125,6 +125,18 @@ from group by l_tax) as distributed_table; +-- Check that we don't support subqueries with count(distinct). + +select + different_shipment_days +from + (select + count(distinct l_shipdate) as different_shipment_days + from + lineitem + group by + l_partkey) as distributed_table; + -- Check that if subquery is pulled, we don't error and run query properly. SELECT max(l_suppkey) FROM From b535910f3e2c04934af3430017ced4fa7ef11c13 Mon Sep 17 00:00:00 2001 From: Murat Tuncer Date: Wed, 17 Feb 2016 15:44:24 +0200 Subject: [PATCH 18/19] Fixed merge leftovers --- Makefile | 4 ++-- src/include/distributed/citus_ruleutils.h | 7 +------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index f1b7b9c56..191a6e6f7 100644 --- a/Makefile +++ b/Makefile @@ -44,9 +44,9 @@ clean: clean-csql # apply or check style reindent: - cd ${citusdb_abs_top_srcdir} && citus_indent --quiet + cd ${citus_abs_top_srcdir} && citus_indent --quiet check-style: - cd ${citusdb_abs_top_srcdir} && citus_indent --quiet --check + cd ${citus_abs_top_srcdir} && citus_indent --quiet --check .PHONY: reindent check-style # depend on install for now diff --git a/src/include/distributed/citus_ruleutils.h b/src/include/distributed/citus_ruleutils.h index 70b879291..e53b1c34d 100644 --- a/src/include/distributed/citus_ruleutils.h +++ b/src/include/distributed/citus_ruleutils.h @@ -1,13 +1,8 @@ /*------------------------------------------------------------------------- * * citus_ruleutils.h -<<<<<<< HEAD * Citus ruleutils wrapper functions and exported PostgreSQL ruleutils - * functions. -======= - * CitusDB ruleutils wrapper functions and exported PostgreSQL ruleutils - * functions. ->>>>>>> origin + * functions. * * Copyright (c) 2012-2015, Citus Data, Inc. *------------------------------------------------------------------------- From 0516555f97dc311b3c46c1f384f4d4629dd2a0f1 Mon Sep 17 00:00:00 2001 From: Jason Petersen Date: Wed, 17 Feb 2016 11:03:17 -0700 Subject: [PATCH 19/19] Change tests to use default staging policy The default staging policy is now round-robin, though tests were still configured to use local-first. Testing with the shipping default seems like the best option, correctness-wise, and since local-first has some issues with OSes where connecting from localhost doesn't always resolve to 'localhost', just going with the default is a win-win. --- src/test/regress/expected/multi_array_agg.out | 22 ++++---- .../expected/multi_index_statements.out | 54 +++++++++---------- .../expected/multi_index_statements_0.out | 54 +++++++++---------- .../multi_large_table_join_planning.out | 32 +++++------ .../multi_large_table_join_planning_0.out | 32 +++++------ .../multi_large_table_task_assignment.out | 38 ++++++------- .../regress/input/multi_stage_data.source | 7 +-- .../input/multi_stage_more_data.source | 4 -- .../regress/output/multi_stage_data.source | 6 +-- .../output/multi_stage_more_data.source | 3 -- 10 files changed, 120 insertions(+), 132 deletions(-) diff --git a/src/test/regress/expected/multi_array_agg.out b/src/test/regress/expected/multi_array_agg.out index 9a1d0b00c..b3fe1b307 100644 --- a/src/test/regress/expected/multi_array_agg.out +++ b/src/test/regress/expected/multi_array_agg.out @@ -96,10 +96,10 @@ SELECT l_quantity, count(*), avg(l_extendedprice), array_agg(l_orderkey) FROM li GROUP BY l_quantity ORDER BY l_quantity; l_quantity | count | avg | array_agg ------------+-------+-----------------------+-------------------------------------------------------------------------------------------------- - 1.00 | 17 | 1477.1258823529411765 | {5543,5633,5634,5698,5766,5856,5857,5986,8997,9026,9158,9184,9220,9222,9348,9383,9476} - 2.00 | 19 | 3078.4242105263157895 | {5506,5540,5573,5669,5703,5730,5798,5831,5893,5920,5923,9030,9058,9123,9124,9188,9344,9441,9476} - 3.00 | 14 | 4714.0392857142857143 | {5509,5543,5605,5606,5827,9124,9157,9184,9223,9254,9349,9414,9475,9477} - 4.00 | 19 | 5929.7136842105263158 | {5504,5507,5508,5511,5538,5764,5766,5826,5829,5862,5959,5985,9091,9120,9281,9347,9382,9440,9473} + 1.00 | 17 | 1477.1258823529411765 | {8997,9026,9158,9184,9220,9222,9348,9383,9476,5543,5633,5634,5698,5766,5856,5857,5986} + 2.00 | 19 | 3078.4242105263157895 | {9030,9058,9123,9124,9188,9344,9441,9476,5506,5540,5573,5669,5703,5730,5798,5831,5893,5920,5923} + 3.00 | 14 | 4714.0392857142857143 | {9124,9157,9184,9223,9254,9349,9414,9475,9477,5509,5543,5605,5606,5827} + 4.00 | 19 | 5929.7136842105263158 | {9091,9120,9281,9347,9382,9440,9473,5504,5507,5508,5511,5538,5764,5766,5826,5829,5862,5959,5985} (4 rows) SELECT l_quantity, array_agg(extract (month FROM o_orderdate)) AS my_month @@ -107,10 +107,10 @@ SELECT l_quantity, array_agg(extract (month FROM o_orderdate)) AS my_month AND l_orderkey > 5500 AND l_orderkey < 9500 GROUP BY l_quantity ORDER BY l_quantity; l_quantity | my_month ------------+------------------------------------------------ - 1.00 | {9,5,7,5,9,11,11,4,7,7,4,7,4,2,6,3,5} - 2.00 | {11,10,8,5,5,12,3,11,7,11,5,7,6,6,10,1,12,6,5} - 3.00 | {4,9,8,11,7,10,6,7,8,5,8,9,11,3} - 4.00 | {1,5,6,11,12,10,9,6,1,2,5,1,11,6,2,8,2,6,10} + 1.00 | {7,7,4,7,4,2,6,3,5,9,5,7,5,9,11,11,4} + 2.00 | {7,6,6,10,1,12,6,5,11,10,8,5,5,12,3,11,7,11,5} + 3.00 | {10,6,7,8,5,8,9,11,3,4,9,8,11,7} + 4.00 | {11,6,2,8,2,6,10,1,5,6,11,12,10,9,6,1,2,5,1} (4 rows) SELECT l_quantity, array_agg(l_orderkey * 2 + 1) FROM lineitem WHERE l_quantity < 5 @@ -118,10 +118,10 @@ SELECT l_quantity, array_agg(l_orderkey * 2 + 1) FROM lineitem WHERE l_quantity AND l_orderkey > 5500 AND l_orderkey < 9500 GROUP BY l_quantity ORDER BY l_quantity; l_quantity | array_agg ------------+--------------------------------------------- - 1.00 | {11269,11397,11713,11715,11973,18317,18445} - 2.00 | {11847,18061,18247,18953} + 1.00 | {18317,18445,11269,11397,11713,11715,11973} + 2.00 | {18061,18247,18953,11847} 3.00 | {18249,18315,18699,18951,18955} - 4.00 | {11653,11659,18241,18765} + 4.00 | {18241,18765,11653,11659} (4 rows) -- Check that we can execute array_agg() with an expression containing NULL values diff --git a/src/test/regress/expected/multi_index_statements.out b/src/test/regress/expected/multi_index_statements.out index 245e2870f..7461e6025 100644 --- a/src/test/regress/expected/multi_index_statements.out +++ b/src/test/regress/expected/multi_index_statements.out @@ -10,59 +10,59 @@ SET client_min_messages TO DEBUG2; -- -- Verify that we can create different types of indexes CREATE INDEX lineitem_orderkey_index ON lineitem (l_orderkey); -DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102014 on node localhost:57638 +DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57638 -DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102012 on node localhost:57638 +DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57638 -DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102010 on node localhost:57638 +DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57638 DEBUG: building index "lineitem_orderkey_index" on table "lineitem" CREATE INDEX lineitem_partkey_desc_index ON lineitem (l_partkey DESC); -DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102014 on node localhost:57638 +DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57638 -DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102012 on node localhost:57638 +DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57638 -DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102010 on node localhost:57638 +DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57638 DEBUG: building index "lineitem_partkey_desc_index" on table "lineitem" CREATE INDEX lineitem_partial_index ON lineitem (l_shipdate) WHERE l_shipdate < '1995-01-01'; -DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102014 on node localhost:57638 +DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57638 -DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102012 on node localhost:57638 +DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57638 -DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102010 on node localhost:57638 +DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57638 DEBUG: building index "lineitem_partial_index" on table "lineitem" CREATE INDEX lineitem_orderkey_hash_index ON lineitem USING hash (l_partkey); -DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102014 on node localhost:57638 +DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57638 -DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102012 on node localhost:57638 +DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57638 -DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102010 on node localhost:57638 +DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57638 DEBUG: building index "lineitem_orderkey_hash_index" on table "lineitem" @@ -88,15 +88,15 @@ ERROR: specifying tablespaces with CREATE INDEX statements is currently unsuppo -- Verify that we error out in case of postgres errors on supported statement -- types. CREATE INDEX lineitem_orderkey_index ON lineitem (l_orderkey); -WARNING: could not receive query results from localhost:57637 +WARNING: could not receive query results from localhost:57638 DETAIL: Client error: relation "lineitem_orderkey_index_102014" already exists ERROR: could not execute DDL command on worker node shards CREATE INDEX try_index ON lineitem USING gist (l_orderkey); -WARNING: could not receive query results from localhost:57637 +WARNING: could not receive query results from localhost:57638 DETAIL: Client error: data type bigint has no default operator class for access method "gist" ERROR: could not execute DDL command on worker node shards CREATE INDEX try_index ON lineitem (non_existent_column); -WARNING: could not receive query results from localhost:57637 +WARNING: could not receive query results from localhost:57638 DETAIL: Client error: column "non_existent_column" does not exist ERROR: could not execute DDL command on worker node shards -- Verify that none of failed indexes got created on the master node @@ -123,42 +123,42 @@ DROP INDEX CONCURRENTLY lineitem_orderkey_index; ERROR: dropping indexes concurrently on distributed tables is currently unsupported -- Verify that we can succesfully drop indexes DROP INDEX lineitem_orderkey_index; -DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102014 on node localhost:57638 +DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57638 -DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102012 on node localhost:57638 +DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57638 -DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102010 on node localhost:57638 +DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57638 DROP INDEX lineitem_partkey_desc_index; -DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102014 on node localhost:57638 +DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57638 -DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102012 on node localhost:57638 +DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57638 -DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102010 on node localhost:57638 +DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57638 DROP INDEX lineitem_partial_index; -DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102014 on node localhost:57638 +DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57638 -DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102012 on node localhost:57638 +DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57638 -DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102010 on node localhost:57638 +DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57638 -- Verify that we handle if exists statements correctly @@ -167,16 +167,16 @@ ERROR: index "non_existent_index" does not exist DROP INDEX IF EXISTS non_existent_index; NOTICE: index "non_existent_index" does not exist, skipping DROP INDEX IF EXISTS lineitem_orderkey_hash_index; -DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102014 on node localhost:57638 +DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57638 -DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102012 on node localhost:57638 +DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57638 -DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102010 on node localhost:57638 +DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57638 DROP INDEX lineitem_orderkey_hash_index; diff --git a/src/test/regress/expected/multi_index_statements_0.out b/src/test/regress/expected/multi_index_statements_0.out index e7cc610e4..0df6d6470 100644 --- a/src/test/regress/expected/multi_index_statements_0.out +++ b/src/test/regress/expected/multi_index_statements_0.out @@ -10,59 +10,59 @@ SET client_min_messages TO DEBUG2; -- -- Verify that we can create different types of indexes CREATE INDEX lineitem_orderkey_index ON lineitem (l_orderkey); -DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102014 on node localhost:57638 +DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57638 -DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102012 on node localhost:57638 +DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57638 -DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102010 on node localhost:57638 +DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57638 DEBUG: building index "lineitem_orderkey_index" on table "lineitem" CREATE INDEX lineitem_partkey_desc_index ON lineitem (l_partkey DESC); -DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102014 on node localhost:57638 +DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57638 -DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102012 on node localhost:57638 +DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57638 -DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102010 on node localhost:57638 +DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57638 DEBUG: building index "lineitem_partkey_desc_index" on table "lineitem" CREATE INDEX lineitem_partial_index ON lineitem (l_shipdate) WHERE l_shipdate < '1995-01-01'; -DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102014 on node localhost:57638 +DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57638 -DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102012 on node localhost:57638 +DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57638 -DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102010 on node localhost:57638 +DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57638 DEBUG: building index "lineitem_partial_index" on table "lineitem" CREATE INDEX lineitem_orderkey_hash_index ON lineitem USING hash (l_partkey); -DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102014 on node localhost:57638 +DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57638 -DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102012 on node localhost:57638 +DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57638 -DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102010 on node localhost:57638 +DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57638 WARNING: hash indexes are not WAL-logged and their use is discouraged @@ -89,15 +89,15 @@ ERROR: specifying tablespaces with CREATE INDEX statements is currently unsuppo -- Verify that we error out in case of postgres errors on supported statement -- types. CREATE INDEX lineitem_orderkey_index ON lineitem (l_orderkey); -WARNING: could not receive query results from localhost:57637 +WARNING: could not receive query results from localhost:57638 DETAIL: Client error: relation "lineitem_orderkey_index_102014" already exists ERROR: could not execute DDL command on worker node shards CREATE INDEX try_index ON lineitem USING gist (l_orderkey); -WARNING: could not receive query results from localhost:57637 +WARNING: could not receive query results from localhost:57638 DETAIL: Client error: data type bigint has no default operator class for access method "gist" ERROR: could not execute DDL command on worker node shards CREATE INDEX try_index ON lineitem (non_existent_column); -WARNING: could not receive query results from localhost:57637 +WARNING: could not receive query results from localhost:57638 DETAIL: Client error: column "non_existent_column" does not exist ERROR: could not execute DDL command on worker node shards -- Verify that none of failed indexes got created on the master node @@ -124,44 +124,44 @@ DROP INDEX CONCURRENTLY lineitem_orderkey_index; ERROR: dropping indexes concurrently on distributed tables is currently unsupported -- Verify that we can succesfully drop indexes DROP INDEX lineitem_orderkey_index; -DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102014 on node localhost:57638 +DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57638 -DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102012 on node localhost:57638 +DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57638 -DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102010 on node localhost:57638 +DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57638 DEBUG: EventTriggerInvoke 16532 DROP INDEX lineitem_partkey_desc_index; -DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102014 on node localhost:57638 +DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57638 -DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102012 on node localhost:57638 +DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57638 -DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102010 on node localhost:57638 +DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57638 DEBUG: EventTriggerInvoke 16532 DROP INDEX lineitem_partial_index; -DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102014 on node localhost:57638 +DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57638 -DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102012 on node localhost:57638 +DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57638 -DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102010 on node localhost:57638 +DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57638 DEBUG: EventTriggerInvoke 16532 @@ -171,16 +171,16 @@ ERROR: index "non_existent_index" does not exist DROP INDEX IF EXISTS non_existent_index; NOTICE: index "non_existent_index" does not exist, skipping DROP INDEX IF EXISTS lineitem_orderkey_hash_index; -DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102014 on node localhost:57638 +DEBUG: applied command on shard 102014 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57637 DEBUG: applied command on shard 102013 on node localhost:57638 -DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102012 on node localhost:57638 +DEBUG: applied command on shard 102012 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57637 DEBUG: applied command on shard 102011 on node localhost:57638 -DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102010 on node localhost:57638 +DEBUG: applied command on shard 102010 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57637 DEBUG: applied command on shard 102009 on node localhost:57638 DEBUG: EventTriggerInvoke 16532 diff --git a/src/test/regress/expected/multi_large_table_join_planning.out b/src/test/regress/expected/multi_large_table_join_planning.out index 3fbdf0b42..842f164cc 100644 --- a/src/test/regress/expected/multi_large_table_join_planning.out +++ b/src/test/regress/expected/multi_large_table_join_planning.out @@ -63,12 +63,12 @@ DEBUG: generated sql query for job 1250 and task 15 DETAIL: query string: "SELECT lineitem.l_partkey, orders.o_orderkey, lineitem.l_quantity, lineitem.l_extendedprice, orders.o_custkey FROM (lineitem_102010 lineitem JOIN orders_102015 orders ON ((lineitem.l_orderkey = orders.o_orderkey))) WHERE (orders.o_totalprice > '10'::numeric)" DEBUG: generated sql query for job 1250 and task 18 DETAIL: query string: "SELECT lineitem.l_partkey, orders.o_orderkey, lineitem.l_quantity, lineitem.l_extendedprice, orders.o_custkey FROM (lineitem_102009 lineitem JOIN orders_102015 orders ON ((lineitem.l_orderkey = orders.o_orderkey))) WHERE (orders.o_totalprice > '10'::numeric)" -DEBUG: assigned task 18 to node localhost:57637 -DEBUG: assigned task 15 to node localhost:57638 -DEBUG: assigned task 12 to node localhost:57637 -DEBUG: assigned task 9 to node localhost:57638 -DEBUG: assigned task 6 to node localhost:57637 -DEBUG: assigned task 3 to node localhost:57638 +DEBUG: assigned task 15 to node localhost:57637 +DEBUG: assigned task 18 to node localhost:57638 +DEBUG: assigned task 9 to node localhost:57637 +DEBUG: assigned task 12 to node localhost:57638 +DEBUG: assigned task 3 to node localhost:57637 +DEBUG: assigned task 6 to node localhost:57638 DEBUG: join prunable for intervals [1,1000] and [6001,7000] DEBUG: join prunable for intervals [6001,7000] and [1,1000] DEBUG: generated sql query for job 1251 and task 3 @@ -99,9 +99,9 @@ DEBUG: pruning merge fetch taskId 4 DETAIL: Creating dependency on merge taskId 10 DEBUG: pruning merge fetch taskId 7 DETAIL: Creating dependency on merge taskId 13 -DEBUG: assigned task 3 to node localhost:57637 -DEBUG: assigned task 9 to node localhost:57638 DEBUG: assigned task 6 to node localhost:57637 +DEBUG: assigned task 3 to node localhost:57638 +DEBUG: assigned task 9 to node localhost:57637 DEBUG: completed cleanup query for job 1252 on node "localhost:57638" DEBUG: completed cleanup query for job 1252 on node "localhost:57637" DEBUG: completed cleanup query for job 1251 on node "localhost:57638" @@ -167,18 +167,18 @@ DEBUG: generated sql query for job 1253 and task 10 DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_102010 lineitem WHERE (l_quantity < 5.0)" DEBUG: generated sql query for job 1253 and task 12 DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_102009 lineitem WHERE (l_quantity < 5.0)" -DEBUG: assigned task 12 to node localhost:57637 -DEBUG: assigned task 10 to node localhost:57638 -DEBUG: assigned task 8 to node localhost:57637 -DEBUG: assigned task 6 to node localhost:57638 -DEBUG: assigned task 4 to node localhost:57637 -DEBUG: assigned task 2 to node localhost:57638 +DEBUG: assigned task 10 to node localhost:57637 +DEBUG: assigned task 12 to node localhost:57638 +DEBUG: assigned task 6 to node localhost:57637 +DEBUG: assigned task 8 to node localhost:57638 +DEBUG: assigned task 2 to node localhost:57637 +DEBUG: assigned task 4 to node localhost:57638 DEBUG: generated sql query for job 1254 and task 2 DETAIL: query string: "SELECT o_orderkey, o_shippriority FROM orders_102016 orders WHERE (o_totalprice <> 4.0)" DEBUG: generated sql query for job 1254 and task 4 DETAIL: query string: "SELECT o_orderkey, o_shippriority FROM orders_102015 orders WHERE (o_totalprice <> 4.0)" -DEBUG: assigned task 4 to node localhost:57637 -DEBUG: assigned task 2 to node localhost:57638 +DEBUG: assigned task 2 to node localhost:57637 +DEBUG: assigned task 4 to node localhost:57638 DEBUG: join prunable for task partitionId 0 and 1 DEBUG: join prunable for task partitionId 0 and 2 DEBUG: join prunable for task partitionId 0 and 3 diff --git a/src/test/regress/expected/multi_large_table_join_planning_0.out b/src/test/regress/expected/multi_large_table_join_planning_0.out index 4b08b7fe0..33700d111 100644 --- a/src/test/regress/expected/multi_large_table_join_planning_0.out +++ b/src/test/regress/expected/multi_large_table_join_planning_0.out @@ -63,12 +63,12 @@ DEBUG: generated sql query for job 1250 and task 15 DETAIL: query string: "SELECT lineitem.l_partkey, orders.o_orderkey, lineitem.l_quantity, lineitem.l_extendedprice, orders.o_custkey FROM (lineitem_102010 lineitem JOIN orders_102015 orders ON ((lineitem.l_orderkey = orders.o_orderkey))) WHERE (orders.o_totalprice > 10::numeric)" DEBUG: generated sql query for job 1250 and task 18 DETAIL: query string: "SELECT lineitem.l_partkey, orders.o_orderkey, lineitem.l_quantity, lineitem.l_extendedprice, orders.o_custkey FROM (lineitem_102009 lineitem JOIN orders_102015 orders ON ((lineitem.l_orderkey = orders.o_orderkey))) WHERE (orders.o_totalprice > 10::numeric)" -DEBUG: assigned task 18 to node localhost:57637 -DEBUG: assigned task 15 to node localhost:57638 -DEBUG: assigned task 12 to node localhost:57637 -DEBUG: assigned task 9 to node localhost:57638 -DEBUG: assigned task 6 to node localhost:57637 -DEBUG: assigned task 3 to node localhost:57638 +DEBUG: assigned task 15 to node localhost:57637 +DEBUG: assigned task 18 to node localhost:57638 +DEBUG: assigned task 9 to node localhost:57637 +DEBUG: assigned task 12 to node localhost:57638 +DEBUG: assigned task 3 to node localhost:57637 +DEBUG: assigned task 6 to node localhost:57638 DEBUG: join prunable for intervals [1,1000] and [6001,7000] DEBUG: join prunable for intervals [6001,7000] and [1,1000] DEBUG: generated sql query for job 1251 and task 3 @@ -99,9 +99,9 @@ DEBUG: pruning merge fetch taskId 4 DETAIL: Creating dependency on merge taskId 10 DEBUG: pruning merge fetch taskId 7 DETAIL: Creating dependency on merge taskId 13 -DEBUG: assigned task 3 to node localhost:57637 -DEBUG: assigned task 9 to node localhost:57638 DEBUG: assigned task 6 to node localhost:57637 +DEBUG: assigned task 3 to node localhost:57638 +DEBUG: assigned task 9 to node localhost:57637 DEBUG: completed cleanup query for job 1252 on node "localhost:57638" DEBUG: completed cleanup query for job 1252 on node "localhost:57637" DEBUG: completed cleanup query for job 1251 on node "localhost:57638" @@ -167,18 +167,18 @@ DEBUG: generated sql query for job 1253 and task 10 DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_102010 lineitem WHERE (l_quantity < 5.0)" DEBUG: generated sql query for job 1253 and task 12 DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_102009 lineitem WHERE (l_quantity < 5.0)" -DEBUG: assigned task 12 to node localhost:57637 -DEBUG: assigned task 10 to node localhost:57638 -DEBUG: assigned task 8 to node localhost:57637 -DEBUG: assigned task 6 to node localhost:57638 -DEBUG: assigned task 4 to node localhost:57637 -DEBUG: assigned task 2 to node localhost:57638 +DEBUG: assigned task 10 to node localhost:57637 +DEBUG: assigned task 12 to node localhost:57638 +DEBUG: assigned task 6 to node localhost:57637 +DEBUG: assigned task 8 to node localhost:57638 +DEBUG: assigned task 2 to node localhost:57637 +DEBUG: assigned task 4 to node localhost:57638 DEBUG: generated sql query for job 1254 and task 2 DETAIL: query string: "SELECT o_orderkey, o_shippriority FROM orders_102016 orders WHERE (o_totalprice <> 4.0)" DEBUG: generated sql query for job 1254 and task 4 DETAIL: query string: "SELECT o_orderkey, o_shippriority FROM orders_102015 orders WHERE (o_totalprice <> 4.0)" -DEBUG: assigned task 4 to node localhost:57637 -DEBUG: assigned task 2 to node localhost:57638 +DEBUG: assigned task 2 to node localhost:57637 +DEBUG: assigned task 4 to node localhost:57638 DEBUG: join prunable for task partitionId 0 and 1 DEBUG: join prunable for task partitionId 0 and 2 DEBUG: join prunable for task partitionId 0 and 3 diff --git a/src/test/regress/expected/multi_large_table_task_assignment.out b/src/test/regress/expected/multi_large_table_task_assignment.out index 33ec87b9b..7ba87fa94 100644 --- a/src/test/regress/expected/multi_large_table_task_assignment.out +++ b/src/test/regress/expected/multi_large_table_task_assignment.out @@ -25,8 +25,8 @@ FROM WHERE o_custkey = c_custkey; DEBUG: StartTransactionCommand -DEBUG: assigned task 4 to node localhost:57637 -DEBUG: assigned task 2 to node localhost:57638 +DEBUG: assigned task 2 to node localhost:57637 +DEBUG: assigned task 4 to node localhost:57638 DEBUG: join prunable for intervals [1,1000] and [1001,2000] DEBUG: join prunable for intervals [1,1000] and [6001,7000] DEBUG: join prunable for intervals [1001,2000] and [6001,7000] @@ -39,9 +39,9 @@ DEBUG: pruning merge fetch taskId 4 DETAIL: Creating dependency on merge taskId 8 DEBUG: pruning merge fetch taskId 7 DETAIL: Creating dependency on merge taskId 11 -DEBUG: assigned task 3 to node localhost:57637 -DEBUG: assigned task 9 to node localhost:57638 DEBUG: assigned task 6 to node localhost:57637 +DEBUG: assigned task 3 to node localhost:57638 +DEBUG: assigned task 9 to node localhost:57637 DEBUG: CommitTransactionCommand count ------- @@ -64,12 +64,12 @@ WHERE o_custkey = c_custkey AND o_orderkey = l_orderkey; DEBUG: StartTransactionCommand -DEBUG: assigned task 15 to node localhost:57637 +DEBUG: assigned task 3 to node localhost:57637 +DEBUG: assigned task 15 to node localhost:57638 +DEBUG: assigned task 6 to node localhost:57637 DEBUG: assigned task 18 to node localhost:57638 DEBUG: assigned task 9 to node localhost:57637 DEBUG: assigned task 12 to node localhost:57638 -DEBUG: assigned task 3 to node localhost:57637 -DEBUG: assigned task 6 to node localhost:57638 DEBUG: join prunable for intervals [1,2496] and [13921,14947] DEBUG: join prunable for intervals [1,2496] and [11554,13920] DEBUG: join prunable for intervals [1,2496] and [8997,11554] @@ -114,12 +114,12 @@ DEBUG: pruning merge fetch taskId 19 DETAIL: Creating dependency on merge taskId 47 DEBUG: pruning merge fetch taskId 22 DETAIL: Creating dependency on merge taskId 54 -DEBUG: assigned task 3 to node localhost:57637 -DEBUG: assigned task 6 to node localhost:57638 -DEBUG: assigned task 9 to node localhost:57637 -DEBUG: assigned task 12 to node localhost:57638 -DEBUG: assigned task 18 to node localhost:57637 -DEBUG: assigned task 24 to node localhost:57638 +DEBUG: assigned task 6 to node localhost:57637 +DEBUG: assigned task 3 to node localhost:57638 +DEBUG: assigned task 24 to node localhost:57637 +DEBUG: assigned task 9 to node localhost:57638 +DEBUG: assigned task 12 to node localhost:57637 +DEBUG: assigned task 18 to node localhost:57638 DEBUG: propagating assignment from merge task 40 to constrained sql task 15 DEBUG: propagating assignment from merge task 47 to constrained sql task 21 DEBUG: CommitTransactionCommand @@ -154,15 +154,15 @@ FROM WHERE l_partkey = c_nationkey; DEBUG: StartTransactionCommand -DEBUG: assigned task 12 to node localhost:57637 -DEBUG: assigned task 10 to node localhost:57638 -DEBUG: assigned task 8 to node localhost:57637 -DEBUG: assigned task 6 to node localhost:57638 -DEBUG: assigned task 4 to node localhost:57637 -DEBUG: assigned task 2 to node localhost:57638 +DEBUG: assigned task 10 to node localhost:57637 +DEBUG: assigned task 12 to node localhost:57638 DEBUG: assigned task 6 to node localhost:57637 +DEBUG: assigned task 8 to node localhost:57638 +DEBUG: assigned task 2 to node localhost:57637 DEBUG: assigned task 4 to node localhost:57638 DEBUG: assigned task 2 to node localhost:57637 +DEBUG: assigned task 6 to node localhost:57638 +DEBUG: assigned task 4 to node localhost:57637 DEBUG: join prunable for task partitionId 0 and 1 DEBUG: join prunable for task partitionId 0 and 2 DEBUG: join prunable for task partitionId 0 and 3 diff --git a/src/test/regress/input/multi_stage_data.source b/src/test/regress/input/multi_stage_data.source index a65b05c69..9effe58d5 100644 --- a/src/test/regress/input/multi_stage_data.source +++ b/src/test/regress/input/multi_stage_data.source @@ -4,11 +4,8 @@ -- Tests for staging data in a distributed cluster. Please note that the number -- of shards uploaded depends on two config values: citus.shard_replication_factor and --- citus.shard_max_size. These values are manually set in pg_regress.c. We also set --- the shard placement policy to the local-node-first policy as other regression --- tests expect the placements to be in that order. - -SET citus.shard_placement_policy TO 'local-node-first'; +-- citus.shard_max_size. These values are set in pg_regress_multi.pl. Shard placement +-- policy is left to the default value (round-robin) to test the common install case. \STAGE lineitem FROM '@abs_srcdir@/data/lineitem.1.data' with delimiter '|' \STAGE lineitem FROM '@abs_srcdir@/data/lineitem.2.data' with delimiter '|' diff --git a/src/test/regress/input/multi_stage_more_data.source b/src/test/regress/input/multi_stage_more_data.source index 63a24d5c1..3ec97813f 100644 --- a/src/test/regress/input/multi_stage_more_data.source +++ b/src/test/regress/input/multi_stage_more_data.source @@ -5,10 +5,6 @@ -- We stage more data to customer and part tables to test distributed joins. The -- staging causes the planner to consider customer and part tables as large, and -- evaluate plans where some of the underlying tables need to be repartitioned. --- We also set the shard placement policy to the local-node-first policy as other --- regression tests expect the placements to be in that order. - -SET citus.shard_placement_policy TO 'local-node-first'; \STAGE customer FROM '@abs_srcdir@/data/customer.2.data' with delimiter '|' \STAGE customer FROM '@abs_srcdir@/data/customer.3.data' with delimiter '|' diff --git a/src/test/regress/output/multi_stage_data.source b/src/test/regress/output/multi_stage_data.source index 74a4d84e2..0eed8965f 100644 --- a/src/test/regress/output/multi_stage_data.source +++ b/src/test/regress/output/multi_stage_data.source @@ -3,10 +3,8 @@ -- -- Tests for staging data in a distributed cluster. Please note that the number -- of shards uploaded depends on two config values: citus.shard_replication_factor and --- citus.shard_max_size. These values are manually set in pg_regress.c. We also set --- the shard placement policy to the local-node-first policy as other regression --- tests expect the placements to be in that order. -SET citus.shard_placement_policy TO 'local-node-first'; +-- citus.shard_max_size. These values are set in pg_regress_multi.pl. Shard placement +-- policy is left to the default value (round-robin) to test the common install case. \STAGE lineitem FROM '@abs_srcdir@/data/lineitem.1.data' with delimiter '|' \STAGE lineitem FROM '@abs_srcdir@/data/lineitem.2.data' with delimiter '|' \STAGE orders FROM '@abs_srcdir@/data/orders.1.data' with delimiter '|' diff --git a/src/test/regress/output/multi_stage_more_data.source b/src/test/regress/output/multi_stage_more_data.source index 0a57bc64e..2e0f38f1d 100644 --- a/src/test/regress/output/multi_stage_more_data.source +++ b/src/test/regress/output/multi_stage_more_data.source @@ -4,9 +4,6 @@ -- We stage more data to customer and part tables to test distributed joins. The -- staging causes the planner to consider customer and part tables as large, and -- evaluate plans where some of the underlying tables need to be repartitioned. --- We also set the shard placement policy to the local-node-first policy as other --- regression tests expect the placements to be in that order. -SET citus.shard_placement_policy TO 'local-node-first'; \STAGE customer FROM '@abs_srcdir@/data/customer.2.data' with delimiter '|' \STAGE customer FROM '@abs_srcdir@/data/customer.3.data' with delimiter '|' \STAGE part FROM '@abs_srcdir@/data/part.more.data' with delimiter '|'