external
/
citus
mirror of https://github.com/citusdata/citus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into run-test-isolation_update_node

pull/7425/head
Jelte Fennema-Nio 2024-01-17 16:20:14 +01:00 committed by GitHub
parent c744ac2b78 6cf6cf37fd
commit 5b9ed03956
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 81 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/backend/distributed/executor/adaptive_executor.c
View File

@ -401,7 +401,7 @@ typedef struct WorkerPool
/*
* Placement executions destined for worker node, but not assigned to any
* connection and not ready to start.
* connection and ready to start.
*/
dlist_head readyTaskQueue;
int readyTaskCount;
@ -492,8 +492,6 @@ typedef struct WorkerSession
} WorkerSession;
struct TaskPlacementExecution;
/* GUC, determining whether Citus opens 1 connection per task */
bool ForceMaxQueryParallelization = false;
int MaxAdaptiveExecutorPoolSize = 16;
@ -585,7 +583,7 @@ typedef enum TaskPlacementExecutionState
} TaskPlacementExecutionState;
/*
* TaskPlacementExecution represents the an execution of a command
* TaskPlacementExecution represents the execution of a command
* on a shard placement.
*/
typedef struct TaskPlacementExecution
@ -1908,7 +1906,7 @@ RunDistributedExecution(DistributedExecution *execution)
/*
* Iterate until all the tasks are finished. Once all the tasks
* are finished, ensure that that all the connection initializations
* are finished, ensure that all the connection initializations
* are also finished. Otherwise, those connections are terminated
* abruptly before they are established (or failed). Instead, we let
* the ConnectionStateMachine() to properly handle them.
@ -3118,7 +3116,7 @@ ConnectionStateMachine(WorkerSession *session)
*
* We can only retry connection when the remote transaction has
* not started over the connection. Otherwise, we'd have to deal
* with restoring the transaction state, which iis beyond our
* with restoring the transaction state, which is beyond our
* purpose at this time.
*/
RemoteTransaction *transaction = &connection->remoteTransaction;

2
src/backend/distributed/executor/multi_executor.c
View File

@ -168,7 +168,7 @@ CitusExecutorRun(QueryDesc *queryDesc,
executorBoundParams = queryDesc->params;
/*
* We do some potentially time consuming operations our self now before we hand of
* We do some potentially time consuming operations ourself now before we hand off
* control to postgres' executor. To make sure that time spent is accurately measured
* we remove the totaltime instrumentation from the queryDesc. Instead we will start
* and stop the instrumentation of the total time and put it back on the queryDesc

2
src/backend/distributed/planner/fast_path_router_planner.c
View File

@ -252,7 +252,7 @@ FastPathRouterQuery(Query *query, Node **distributionKeyValue)
/*
* Distribution column must be used in a simple equality match check and it must be
* place at top level conjustion operator. In simple words, we should have
* place at top level conjunction operator. In simple words, we should have
* WHERE dist_key = VALUE [AND ....];
*
* We're also not allowing any other appearances of the distribution key in the quals.

4
src/backend/distributed/planner/multi_explain.c
View File

@ -197,9 +197,7 @@ CitusExplainScan(CustomScanState *node, List *ancestors, struct ExplainState *es
if (!ExplainDistributedQueries)
{
appendStringInfoSpaces(es->str, es->indent * 2);
appendStringInfo(es->str, "explain statements for distributed queries ");
appendStringInfo(es->str, "are not enabled\n");
ExplainPropertyBool("citus.explain_distributed_queries", false, es);
return;
}

6
src/backend/distributed/planner/multi_logical_planner.c
View File

@ -715,8 +715,8 @@ MultiNodeTree(Query *queryTree)
/*
* ContainsReadIntermediateResultFunction determines whether an expresion tree contains
* a call to the read_intermediate_result function.
* ContainsReadIntermediateResultFunction determines whether an expression tree
* contains a call to the read_intermediate_result function.
*/
bool
ContainsReadIntermediateResultFunction(Node *node)
@ -726,7 +726,7 @@ ContainsReadIntermediateResultFunction(Node *node)
/*
* ContainsReadIntermediateResultArrayFunction determines whether an expresion
* ContainsReadIntermediateResultArrayFunction determines whether an expression
* tree contains a call to the read_intermediate_results(result_ids, format)
* function.
*/

20
src/backend/distributed/planner/multi_router_planner.c
View File

@ -434,7 +434,7 @@ ExtractSelectRangeTableEntry(Query *query)
* for the given modification query.
*
* The function errors out if the input query is not a
* modify query (e.g., INSERT, UPDATE or DELETE). So, this
* modify query (e.g., INSERT, UPDATE, DELETE or MERGE). So, this
* function is not expected to be called on SELECT queries.
*/
Oid
@ -2271,13 +2271,13 @@ SelectsFromDistributedTable(List *rangeTableList, Query *query)
/*
* RouterQuery runs router pruning logic for SELECT, UPDATE, DELETE, and MERGE queries.
* If there are shards present and query is routable, all RTEs have been updated
* to point to the relevant shards in the originalQuery. Also, placementList is
* filled with the list of worker nodes that has all the required shard placements
* for the query execution. anchorShardId is set to the first pruned shardId of
* the given query. Finally, relationShardList is filled with the list of
* relation-to-shard mappings for the query.
* PlanRouterQuery runs router pruning logic for SELECT, UPDATE, DELETE, and
* MERGE queries. If there are shards present and query is routable, all RTEs
* have been updated to point to the relevant shards in the originalQuery. Also,
* placementList is filled with the list of worker nodes that has all the
* required shard placements for the query execution. anchorShardId is set to
* the first pruned shardId of the given query. Finally, relationShardList is
* filled with the list of relation-to-shard mappings for the query.
*
* If the given query is not routable, it fills planningError with the related
* DeferredErrorMessage. The caller can check this error message to see if query
@ -2510,7 +2510,7 @@ AllShardsColocated(List *relationShardList)
if (currentTableType == RANGE_DISTRIBUTED ||
currentTableType == APPEND_DISTRIBUTED)
{
/* we do not have further strict colocation chceks */
/* we do not have further strict colocation checks */
continue;
}
}
@ -2932,7 +2932,7 @@ TargetShardIntervalsForRestrictInfo(RelationRestrictionContext *restrictionConte
}
/*
* Different resrictions might have different partition columns.
* Different restrictions might have different partition columns.
* We report partition column value if there is only one.
*/
if (multiplePartitionValuesExist)

2
src/include/distributed/distributed_planner.h
View File

@ -104,7 +104,7 @@ typedef struct FastPathRestrictionContext
* Set to true when distKey = Param; in the queryTree
*/
bool distributionKeyHasParam;
}FastPathRestrictionContext;
} FastPathRestrictionContext;
typedef struct PlannerRestrictionContext
{

6
src/include/distributed/multi_physical_planner.h
View File

@ -238,8 +238,8 @@ typedef struct Task
TaskQuery taskQuery;
/*
* A task can have multiple queries, in which case queryCount will be > 1. If
* a task has more one query, then taskQuery->queryType == TASK_QUERY_TEXT_LIST.
* A task can have multiple queries, in which case queryCount will be > 1, and
* taskQuery->queryType == TASK_QUERY_TEXT_LIST.
*/
int queryCount;
@ -290,7 +290,7 @@ typedef struct Task
/*
* When we evaluate functions and parameters in the query string then
* we should no longer send the list of parameters long with the
* we should no longer send the list of parameters along with the
* query.
*/
bool parametersInQueryStringResolved;

36
src/test/regress/expected/multi_join_order_additional.out
View File

@ -77,10 +77,10 @@ DEBUG: Router planner cannot handle multi-shard select queries
LOG: join order: [ "lineitem" ][ local partition join "lineitem" ]
DEBUG: join prunable for intervals [-2147483648,-1] and [0,2147483647]
DEBUG: join prunable for intervals [0,2147483647] and [-2147483648,-1]
QUERY PLAN
QUERY PLAN
---------------------------------------------------------------------
Custom Scan (Citus Adaptive)
explain statements for distributed queries are not enabled
citus.explain_distributed_queries: false
(2 rows)
SET client_min_messages TO LOG;
@ -92,11 +92,11 @@ SELECT count(*) FROM lineitem, orders
WHERE (l_orderkey = o_orderkey AND l_quantity > 5)
OR (l_orderkey = o_orderkey AND l_quantity < 10);
LOG: join order: [ "lineitem" ][ local partition join "orders" ]
QUERY PLAN
QUERY PLAN
---------------------------------------------------------------------
Aggregate
-> Custom Scan (Citus Adaptive)
explain statements for distributed queries are not enabled
citus.explain_distributed_queries: false
(3 rows)
EXPLAIN (COSTS OFF)
@ -107,11 +107,11 @@ EXPLAIN (COSTS OFF)
SELECT count(*) FROM orders, lineitem_hash
WHERE o_orderkey = l_orderkey;
LOG: join order: [ "orders" ][ dual partition join "lineitem_hash" ]
QUERY PLAN
QUERY PLAN
---------------------------------------------------------------------
Aggregate
-> Custom Scan (Citus Adaptive)
explain statements for distributed queries are not enabled
citus.explain_distributed_queries: false
(3 rows)
-- Verify we handle local joins between two hash-partitioned tables.
@ -119,11 +119,11 @@ EXPLAIN (COSTS OFF)
SELECT count(*) FROM orders_hash, lineitem_hash
WHERE o_orderkey = l_orderkey;
LOG: join order: [ "orders_hash" ][ local partition join "lineitem_hash" ]
QUERY PLAN
QUERY PLAN
---------------------------------------------------------------------
Aggregate
-> Custom Scan (Citus Adaptive)
explain statements for distributed queries are not enabled
citus.explain_distributed_queries: false
(3 rows)
-- Validate that we can handle broadcast joins with hash-partitioned tables.
@ -131,11 +131,11 @@ EXPLAIN (COSTS OFF)
SELECT count(*) FROM customer_hash, nation
WHERE c_nationkey = n_nationkey;
LOG: join order: [ "customer_hash" ][ reference join "nation" ]
QUERY PLAN
QUERY PLAN
---------------------------------------------------------------------
Aggregate
-> Custom Scan (Citus Adaptive)
explain statements for distributed queries are not enabled
citus.explain_distributed_queries: false
(3 rows)
-- Validate that we don't use a single-partition join method for a hash
@ -144,11 +144,11 @@ EXPLAIN (COSTS OFF)
SELECT count(*) FROM orders, lineitem, customer_append
WHERE o_custkey = l_partkey AND o_custkey = c_nationkey;
LOG: join order: [ "orders" ][ dual partition join "lineitem" ][ dual partition join "customer_append" ]
QUERY PLAN
QUERY PLAN
---------------------------------------------------------------------
Aggregate
-> Custom Scan (Citus Adaptive)
explain statements for distributed queries are not enabled
citus.explain_distributed_queries: false
(3 rows)
-- Validate that we don't chose a single-partition join method with a
@ -157,11 +157,11 @@ EXPLAIN (COSTS OFF)
SELECT count(*) FROM orders, customer_hash
WHERE c_custkey = o_custkey;
LOG: join order: [ "orders" ][ dual partition join "customer_hash" ]
QUERY PLAN
QUERY PLAN
---------------------------------------------------------------------
Aggregate
-> Custom Scan (Citus Adaptive)
explain statements for distributed queries are not enabled
citus.explain_distributed_queries: false
(3 rows)
-- Validate that we can re-partition a hash partitioned table to join with a
@ -170,11 +170,11 @@ EXPLAIN (COSTS OFF)
SELECT count(*) FROM orders_hash, customer_append
WHERE c_custkey = o_custkey;
LOG: join order: [ "orders_hash" ][ dual partition join "customer_append" ]
QUERY PLAN
QUERY PLAN
---------------------------------------------------------------------
Aggregate
-> Custom Scan (Citus Adaptive)
explain statements for distributed queries are not enabled
citus.explain_distributed_queries: false
(3 rows)
-- Validate a 4 way join that could be done locally is planned as such by the logical
@ -195,11 +195,11 @@ JOIN (
WHERE event_type = 5
) AS some_users ON (some_users.user_id = bar.user_id);
LOG: join order: [ "users_table" ][ local partition join "events_table" ][ local partition join "users_table" ][ local partition join "events_table" ]
QUERY PLAN
QUERY PLAN
---------------------------------------------------------------------
Aggregate
-> Custom Scan (Citus Adaptive)
explain statements for distributed queries are not enabled
citus.explain_distributed_queries: false
(3 rows)
-- Reset client logging level to its previous value

18
src/test/regress/expected/multi_join_order_tpch_repartition.out
View File

@ -22,11 +22,11 @@ WHERE
and l_discount between 0.06 - 0.01 and 0.06 + 0.01
and l_quantity < 24;
LOG: join order: [ "lineitem" ]
QUERY PLAN
QUERY PLAN
---------------------------------------------------------------------
Aggregate
-> Custom Scan (Citus Adaptive)
explain statements for distributed queries are not enabled
citus.explain_distributed_queries: false
(3 rows)
-- Query #3 from the TPC-H decision support benchmark
@ -61,7 +61,7 @@ LOG: join order: [ "orders" ][ local partition join "lineitem" ][ dual partitio
-> HashAggregate
Group Key: remote_scan.l_orderkey, remote_scan.o_orderdate, remote_scan.o_shippriority
-> Custom Scan (Citus Adaptive)
explain statements for distributed queries are not enabled
citus.explain_distributed_queries: false
(6 rows)
-- Query #10 from the TPC-H decision support benchmark
@ -98,12 +98,12 @@ GROUP BY
ORDER BY
revenue DESC;
LOG: join order: [ "orders" ][ local partition join "lineitem" ][ dual partition join "customer_append" ][ reference join "nation" ]
QUERY PLAN
QUERY PLAN
---------------------------------------------------------------------
Sort
Sort Key: remote_scan.revenue DESC
-> Custom Scan (Citus Adaptive)
explain statements for distributed queries are not enabled
citus.explain_distributed_queries: false
(4 rows)
-- Query #19 from the TPC-H decision support benchmark (modified)
@ -138,11 +138,11 @@ WHERE
AND l_shipinstruct = 'DELIVER IN PERSON'
);
LOG: join order: [ "lineitem" ][ dual partition join "part_append" ]
QUERY PLAN
QUERY PLAN
---------------------------------------------------------------------
Aggregate
-> Custom Scan (Citus Adaptive)
explain statements for distributed queries are not enabled
citus.explain_distributed_queries: false
(3 rows)
-- Query to test multiple re-partition jobs in a single query
@ -158,12 +158,12 @@ WHERE
GROUP BY
l_partkey;
LOG: join order: [ "lineitem" ][ local partition join "orders" ][ dual partition join "part_append" ][ dual partition join "customer_append" ]
QUERY PLAN
QUERY PLAN
---------------------------------------------------------------------
HashAggregate
Group Key: remote_scan.l_partkey
-> Custom Scan (Citus Adaptive)
explain statements for distributed queries are not enabled
citus.explain_distributed_queries: false
(4 rows)
-- Reset client logging level to its previous value

14
src/test/regress/expected/multi_join_order_tpch_small.out
View File

@ -17,11 +17,11 @@ WHERE
and l_discount between 0.06 - 0.01 and 0.06 + 0.01
and l_quantity < 24;
LOG: join order: [ "lineitem" ]
QUERY PLAN
QUERY PLAN
---------------------------------------------------------------------
Aggregate
-> Custom Scan (Citus Adaptive)
explain statements for distributed queries are not enabled
citus.explain_distributed_queries: false
(3 rows)
-- Query #3 from the TPC-H decision support benchmark
@ -49,12 +49,12 @@ ORDER BY
revenue DESC,
o_orderdate;
LOG: join order: [ "orders" ][ reference join "customer" ][ local partition join "lineitem" ]
QUERY PLAN
QUERY PLAN
---------------------------------------------------------------------
Sort
Sort Key: remote_scan.revenue DESC, remote_scan.o_orderdate
-> Custom Scan (Citus Adaptive)
explain statements for distributed queries are not enabled
citus.explain_distributed_queries: false
(4 rows)
-- Query #10 from the TPC-H decision support benchmark
@ -98,7 +98,7 @@ LOG: join order: [ "orders" ][ reference join "customer" ][ reference join "nat
-> HashAggregate
Group Key: remote_scan.c_custkey, remote_scan.c_name, remote_scan.c_acctbal, remote_scan.c_phone, remote_scan.n_name, remote_scan.c_address, remote_scan.c_comment
-> Custom Scan (Citus Adaptive)
explain statements for distributed queries are not enabled
citus.explain_distributed_queries: false
(6 rows)
-- Query #19 from the TPC-H decision support benchmark (modified)
@ -133,11 +133,11 @@ WHERE
AND l_shipinstruct = 'DELIVER IN PERSON'
);
LOG: join order: [ "lineitem" ][ reference join "part" ]
QUERY PLAN
QUERY PLAN
---------------------------------------------------------------------
Aggregate
-> Custom Scan (Citus Adaptive)
explain statements for distributed queries are not enabled
citus.explain_distributed_queries: false
(3 rows)
-- Reset client logging level to its previous value

14
src/test/regress/expected/multi_join_pruning.out
View File

@ -104,11 +104,11 @@ SELECT count(*)
DEBUG: Router planner cannot handle multi-shard select queries
DEBUG: join prunable for intervals [{},{AZZXSP27F21T6,AZZXSP27F21T6}] and [{BA1000U2AMO4ZGX,BZZXSP27F21T6},{CA1000U2AMO4ZGX,CZZXSP27F21T6}]
DEBUG: join prunable for intervals [{BA1000U2AMO4ZGX,BZZXSP27F21T6},{CA1000U2AMO4ZGX,CZZXSP27F21T6}] and [{},{AZZXSP27F21T6,AZZXSP27F21T6}]
QUERY PLAN
QUERY PLAN
---------------------------------------------------------------------
Aggregate
-> Custom Scan (Citus Adaptive)
explain statements for distributed queries are not enabled
citus.explain_distributed_queries: false
(3 rows)
EXPLAIN (COSTS OFF)
@ -118,11 +118,11 @@ SELECT count(*)
DEBUG: Router planner cannot handle multi-shard select queries
DEBUG: join prunable for intervals [(a,3,b),(b,4,c)] and [(c,5,d),(d,6,e)]
DEBUG: join prunable for intervals [(c,5,d),(d,6,e)] and [(a,3,b),(b,4,c)]
QUERY PLAN
QUERY PLAN
---------------------------------------------------------------------
Aggregate
-> Custom Scan (Citus Adaptive)
explain statements for distributed queries are not enabled
citus.explain_distributed_queries: false
(3 rows)
-- Test that large table joins on partition varchar columns work
@ -133,14 +133,14 @@ SELECT count(*)
DEBUG: Router planner cannot handle multi-shard select queries
DEBUG: join prunable for intervals [AA1000U2AMO4ZGX,AZZXSP27F21T6] and [BA1000U2AMO4ZGX,BZZXSP27F21T6]
DEBUG: join prunable for intervals [BA1000U2AMO4ZGX,BZZXSP27F21T6] and [AA1000U2AMO4ZGX,AZZXSP27F21T6]
QUERY PLAN
QUERY PLAN
---------------------------------------------------------------------
Aggregate
-> Custom Scan (Citus Adaptive)
explain statements for distributed queries are not enabled
citus.explain_distributed_queries: false
(3 rows)
SET client_min_messages TO WARNING;
SET client_min_messages TO WARNING;
DROP TABLE varchar_partitioned_table;
DROP TABLE array_partitioned_table;
DROP TABLE composite_partitioned_table;

32
src/test/regress/expected/multi_task_assignment_policy.out
View File

@ -75,11 +75,11 @@ DEBUG: shard count after pruning for task_assignment_test_table: 3
DEBUG: assigned task to node localhost:xxxxx
DEBUG: assigned task to node localhost:xxxxx
DEBUG: assigned task to node localhost:xxxxx
QUERY PLAN
QUERY PLAN
---------------------------------------------------------------------
Aggregate
-> Custom Scan (Citus Adaptive)
explain statements for distributed queries are not enabled
citus.explain_distributed_queries: false
(3 rows)
EXPLAIN (COSTS OFF) SELECT count(*) FROM task_assignment_test_table;
@ -89,11 +89,11 @@ DEBUG: shard count after pruning for task_assignment_test_table: 3
DEBUG: assigned task to node localhost:xxxxx
DEBUG: assigned task to node localhost:xxxxx
DEBUG: assigned task to node localhost:xxxxx
QUERY PLAN
QUERY PLAN
---------------------------------------------------------------------
Aggregate
-> Custom Scan (Citus Adaptive)
explain statements for distributed queries are not enabled
citus.explain_distributed_queries: false
(3 rows)
-- Next test the first-replica task assignment policy
@ -105,11 +105,11 @@ DEBUG: shard count after pruning for task_assignment_test_table: 3
DEBUG: assigned task to node localhost:xxxxx
DEBUG: assigned task to node localhost:xxxxx
DEBUG: assigned task to node localhost:xxxxx
QUERY PLAN
QUERY PLAN
---------------------------------------------------------------------
Aggregate
-> Custom Scan (Citus Adaptive)
explain statements for distributed queries are not enabled
citus.explain_distributed_queries: false
(3 rows)
EXPLAIN (COSTS OFF) SELECT count(*) FROM task_assignment_test_table;
@ -119,11 +119,11 @@ DEBUG: shard count after pruning for task_assignment_test_table: 3
DEBUG: assigned task to node localhost:xxxxx
DEBUG: assigned task to node localhost:xxxxx
DEBUG: assigned task to node localhost:xxxxx
QUERY PLAN
QUERY PLAN
---------------------------------------------------------------------
Aggregate
-> Custom Scan (Citus Adaptive)
explain statements for distributed queries are not enabled
citus.explain_distributed_queries: false
(3 rows)
COMMIT;
@ -142,38 +142,38 @@ SET LOCAL citus.task_assignment_policy TO 'greedy';
EXPLAIN (COSTS FALSE) SELECT * FROM task_assignment_reference_table;
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
QUERY PLAN
QUERY PLAN
---------------------------------------------------------------------
Custom Scan (Citus Adaptive)
explain statements for distributed queries are not enabled
citus.explain_distributed_queries: false
(2 rows)
EXPLAIN (COSTS FALSE) SELECT * FROM task_assignment_reference_table;
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
QUERY PLAN
QUERY PLAN
---------------------------------------------------------------------
Custom Scan (Citus Adaptive)
explain statements for distributed queries are not enabled
citus.explain_distributed_queries: false
(2 rows)
SET LOCAL citus.task_assignment_policy TO 'first-replica';
EXPLAIN (COSTS FALSE) SELECT * FROM task_assignment_reference_table;
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
QUERY PLAN
QUERY PLAN
---------------------------------------------------------------------
Custom Scan (Citus Adaptive)
explain statements for distributed queries are not enabled
citus.explain_distributed_queries: false
(2 rows)
EXPLAIN (COSTS FALSE) SELECT * FROM task_assignment_reference_table;
DEBUG: Distributed planning for a fast-path router query
DEBUG: Creating router plan
QUERY PLAN
QUERY PLAN
---------------------------------------------------------------------
Custom Scan (Citus Adaptive)
explain statements for distributed queries are not enabled
citus.explain_distributed_queries: false
(2 rows)
ROLLBACK;