external
/
citus
mirror of https://github.com/citusdata/citus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'improve-shard-pruning' of https://github.com/MarkusSintonen/citus into MarkusSintonen-improve-shard-pruning

pull/3543/head
Hadi Moshayedi 2020-02-26 07:13:33 -08:00
parent 82d22b34fe 099e266a6c
commit 1b3e58f0c3
7 changed files with 1586 additions and 271 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

859
src/backend/distributed/planner/shard_pruning.c
View File

File diff suppressed because it is too large Load Diff

1
src/test/regress/expected/fast_path_router_modify.out
View File

@ -87,7 +87,6 @@ DEBUG: Plan is router executable
DELETE FROM modify_fast_path WHERE value_1 = 15 AND (key = 1 OR value_2 = 'citus');
DEBUG: Creating router plan
DEBUG: Plan is router executable
DETAIL: distribution column value: 1
-- goes through fast-path planning even if the key is updated to the same value
UPDATE modify_fast_path SET key = 1 WHERE key = 1;
DEBUG: Distributed planning for a fast-path router query

793
src/test/regress/expected/multi_hash_pruning.out
View File

File diff suppressed because it is too large Load Diff

32
src/test/regress/expected/multi_repartition_join_planning.out
View File

@ -65,6 +65,10 @@ GROUP BY
ORDER BY
l_partkey, o_orderkey;
DEBUG: Router planner does not support append-partitioned tables.
DEBUG: no valid constraints found
DEBUG: shard count: 2
DEBUG: no valid constraints found
DEBUG: shard count: 2
DEBUG: join prunable for intervals [1,5986] and [8997,14947]
DEBUG: join prunable for intervals [8997,14947] and [1,5986]
DEBUG: generated sql query for task 1
@ -73,6 +77,8 @@ DEBUG: generated sql query for task 2
DETAIL: query string: "SELECT lineitem.l_partkey, orders.o_orderkey, lineitem.l_quantity, lineitem.l_extendedprice, orders.o_custkey FROM (lineitem_290001 lineitem JOIN orders_290003 orders ON ((lineitem.l_orderkey OPERATOR(pg_catalog.=) orders.o_orderkey))) WHERE ((lineitem.l_partkey OPERATOR(pg_catalog.<) 1000) AND (orders.o_totalprice OPERATOR(pg_catalog.>) 10.0))"
DEBUG: assigned task to node localhost:xxxxx
DEBUG: assigned task to node localhost:xxxxx
DEBUG: no valid constraints found
DEBUG: shard count: 2
DEBUG: join prunable for intervals [1,1000] and [6001,7000]
DEBUG: join prunable for intervals [6001,7000] and [1,1000]
DEBUG: generated sql query for task 2
@ -85,6 +91,8 @@ DEBUG: pruning merge fetch taskId 3
DETAIL: Creating dependency on merge taskId 6
DEBUG: assigned task to node localhost:xxxxx
DEBUG: assigned task to node localhost:xxxxx
DEBUG: no valid constraints found
DEBUG: shard count: 3
DEBUG: join prunable for intervals [1,1000] and [1001,2000]
DEBUG: join prunable for intervals [1,1000] and [6001,7000]
DEBUG: join prunable for intervals [1001,2000] and [1,1000]
@ -158,12 +166,16 @@ GROUP BY
ORDER BY
l_partkey, o_orderkey;
DEBUG: Router planner does not support append-partitioned tables.
DEBUG: no valid constraints found
DEBUG: shard count: 2
DEBUG: generated sql query for task 1
DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290000 lineitem WHERE (l_quantity OPERATOR(pg_catalog.<) 5.0)"
DEBUG: generated sql query for task 2
DETAIL: query string: "SELECT l_partkey, l_suppkey FROM lineitem_290001 lineitem WHERE (l_quantity OPERATOR(pg_catalog.<) 5.0)"
DEBUG: assigned task to node localhost:xxxxx
DEBUG: assigned task to node localhost:xxxxx
DEBUG: no valid constraints found
DEBUG: shard count: 2
DEBUG: generated sql query for task 1
DETAIL: query string: "SELECT o_orderkey, o_shippriority FROM orders_290002 orders WHERE (o_totalprice OPERATOR(pg_catalog.<>) 4.0)"
DEBUG: generated sql query for task 2
@ -233,12 +245,16 @@ GROUP BY
ORDER BY
o_orderkey;
DEBUG: Router planner does not support append-partitioned tables.
DEBUG: no valid constraints found
DEBUG: shard count: 2
DEBUG: generated sql query for task 1
DETAIL: query string: "SELECT l_suppkey FROM lineitem_290000 lineitem WHERE true"
DEBUG: generated sql query for task 2
DETAIL: query string: "SELECT l_suppkey FROM lineitem_290001 lineitem WHERE true"
DEBUG: assigned task to node localhost:xxxxx
DEBUG: assigned task to node localhost:xxxxx
DEBUG: no valid constraints found
DEBUG: shard count: 2
DEBUG: generated sql query for task 1
DETAIL: query string: "SELECT o_orderkey, o_shippriority FROM orders_290002 orders WHERE true"
DEBUG: generated sql query for task 2
@ -310,12 +326,16 @@ GROUP BY
ORDER BY
o_orderkey;
DEBUG: Router planner does not support append-partitioned tables.
DEBUG: no valid constraints found
DEBUG: shard count: 2
DEBUG: generated sql query for task 1
DETAIL: query string: "SELECT l_suppkey FROM lineitem_290000 lineitem WHERE true"
DEBUG: generated sql query for task 2
DETAIL: query string: "SELECT l_suppkey FROM lineitem_290001 lineitem WHERE true"
DEBUG: assigned task to node localhost:xxxxx
DEBUG: assigned task to node localhost:xxxxx
DEBUG: no valid constraints found
DEBUG: shard count: 2
DEBUG: generated sql query for task 1
DETAIL: query string: "SELECT o_orderkey, o_shippriority FROM orders_290002 orders WHERE true"
DEBUG: generated sql query for task 2
@ -385,12 +405,16 @@ GROUP BY
ORDER BY
o_orderkey;
DEBUG: Router planner does not support append-partitioned tables.
DEBUG: no valid constraints found
DEBUG: shard count: 2
DEBUG: generated sql query for task 1
DETAIL: query string: "SELECT l_suppkey FROM lineitem_290000 lineitem WHERE true"
DEBUG: generated sql query for task 2
DETAIL: query string: "SELECT l_suppkey FROM lineitem_290001 lineitem WHERE true"
DEBUG: assigned task to node localhost:xxxxx
DEBUG: assigned task to node localhost:xxxxx
DEBUG: no valid constraints found
DEBUG: shard count: 2
DEBUG: generated sql query for task 1
DETAIL: query string: "SELECT o_orderkey, o_shippriority FROM orders_290002 orders WHERE true"
DEBUG: generated sql query for task 2
@ -457,7 +481,13 @@ select s_i_id
group by s_i_id, s_w_id, s_quantity
having s_quantity > random()
;
DEBUG: no valid constraints found
DEBUG: shard count: 4
DEBUG: no valid constraints found
DEBUG: shard count: 4
DEBUG: Router planner cannot handle multi-shard select queries
DEBUG: no valid constraints found
DEBUG: shard count: 4
DEBUG: generated sql query for task 1
DETAIL: query string: "SELECT s_i_id, s_w_id, s_quantity FROM stock_690004 stock WHERE true"
DEBUG: generated sql query for task 2
@ -470,6 +500,8 @@ DEBUG: assigned task to node localhost:xxxxx
DEBUG: assigned task to node localhost:xxxxx
DEBUG: assigned task to node localhost:xxxxx
DEBUG: assigned task to node localhost:xxxxx
DEBUG: no valid constraints found
DEBUG: shard count: 4
DEBUG: generated sql query for task 1
DETAIL: query string: "SELECT ol_i_id FROM order_line_690000 order_line WHERE true"
DEBUG: generated sql query for task 2

12
src/test/regress/expected/multi_repartition_join_task_assignment.out
View File

@ -18,8 +18,12 @@ FROM
WHERE
o_custkey = c_custkey;
DEBUG: Router planner does not support append-partitioned tables.
DEBUG: no valid constraints found
DEBUG: shard count: 2
DEBUG: assigned task to node localhost:xxxxx
DEBUG: assigned task to node localhost:xxxxx
DEBUG: no valid constraints found
DEBUG: shard count: 3
DEBUG: join prunable for intervals [1,1000] and [1001,2000]
DEBUG: join prunable for intervals [1,1000] and [6001,7000]
DEBUG: join prunable for intervals [1001,2000] and [1,1000]
@ -52,9 +56,13 @@ WHERE
o_custkey = c_custkey AND
o_orderkey = l_orderkey;
DEBUG: Router planner does not support append-partitioned tables.
DEBUG: no valid constraints found
DEBUG: shard count: 3
DEBUG: assigned task to node localhost:xxxxx
DEBUG: assigned task to node localhost:xxxxx
DEBUG: assigned task to node localhost:xxxxx
DEBUG: no valid constraints found
DEBUG: shard count: 2
DEBUG: join prunable for intervals [1,5986] and [8997,14947]
DEBUG: join prunable for intervals [8997,14947] and [1,5986]
DEBUG: pruning merge fetch taskId 1
@ -77,8 +85,12 @@ FROM
WHERE
l_partkey = c_nationkey;
DEBUG: Router planner does not support append-partitioned tables.
DEBUG: no valid constraints found
DEBUG: shard count: 2
DEBUG: assigned task to node localhost:xxxxx
DEBUG: assigned task to node localhost:xxxxx
DEBUG: no valid constraints found
DEBUG: shard count: 3
DEBUG: assigned task to node localhost:xxxxx
DEBUG: assigned task to node localhost:xxxxx
DEBUG: assigned task to node localhost:xxxxx

8
src/test/regress/expected/multi_task_assignment_policy.out
View File

@ -70,6 +70,8 @@ SET client_min_messages TO DEBUG3;
SET citus.task_assignment_policy TO 'greedy';
EXPLAIN (COSTS OFF) SELECT count(*) FROM task_assignment_test_table;
DEBUG: Router planner does not support append-partitioned tables.
DEBUG: no valid constraints found
DEBUG: shard count: 3
DEBUG: assigned task to node localhost:xxxxx
DEBUG: assigned task to node localhost:xxxxx
DEBUG: assigned task to node localhost:xxxxx
@ -82,6 +84,8 @@ DEBUG: assigned task to node localhost:xxxxx
EXPLAIN (COSTS OFF) SELECT count(*) FROM task_assignment_test_table;
DEBUG: Router planner does not support append-partitioned tables.
DEBUG: no valid constraints found
DEBUG: shard count: 3
DEBUG: assigned task to node localhost:xxxxx
DEBUG: assigned task to node localhost:xxxxx
DEBUG: assigned task to node localhost:xxxxx
@ -96,6 +100,8 @@ DEBUG: assigned task to node localhost:xxxxx
SET citus.task_assignment_policy TO 'first-replica';
EXPLAIN (COSTS OFF) SELECT count(*) FROM task_assignment_test_table;
DEBUG: Router planner does not support append-partitioned tables.
DEBUG: no valid constraints found
DEBUG: shard count: 3
DEBUG: assigned task to node localhost:xxxxx
DEBUG: assigned task to node localhost:xxxxx
DEBUG: assigned task to node localhost:xxxxx
@ -108,6 +114,8 @@ DEBUG: assigned task to node localhost:xxxxx
EXPLAIN (COSTS OFF) SELECT count(*) FROM task_assignment_test_table;
DEBUG: Router planner does not support append-partitioned tables.
DEBUG: no valid constraints found
DEBUG: shard count: 3
DEBUG: assigned task to node localhost:xxxxx
DEBUG: assigned task to node localhost:xxxxx
DEBUG: assigned task to node localhost:xxxxx

152
src/test/regress/sql/multi_hash_pruning.sql
View File

@ -32,6 +32,12 @@ CREATE TABLE orders_hash_partitioned (
o_comment varchar(79) );
SELECT create_distributed_table('orders_hash_partitioned', 'o_orderkey');
INSERT INTO orders_hash_partitioned (o_orderkey, o_custkey, o_totalprice, o_shippriority, o_clerk) VALUES
(1, 11, 10, 111, 'aaa'),
(2, 22, 20, 222, 'bbb'),
(3, 33, 30, 333, 'ccc'),
(4, 44, 40, 444, 'ddd');
SET client_min_messages TO DEBUG2;
-- Check that we can prune shards for simple cases, boolean expressions and
@ -68,13 +74,13 @@ SELECT count(*) FROM orders_hash_partitioned
SELECT count(*) FROM orders_hash_partitioned
WHERE o_orderkey = 1 OR o_clerk = 'aaa';
SELECT count(*) FROM orders_hash_partitioned
WHERE o_orderkey = 1 OR (o_orderkey = 3 AND o_clerk = 'aaa');
WHERE o_orderkey = 1 OR (o_orderkey = 3 AND o_clerk = 'ccc');
SELECT count(*) FROM orders_hash_partitioned
WHERE o_orderkey = 1 OR o_orderkey is NULL;
SELECT count(*) FROM
(SELECT o_orderkey FROM orders_hash_partitioned WHERE o_orderkey = 1) AS orderkeys;
SET client_min_messages TO DEFAULT;
SET client_min_messages TO DEBUG3;
-- Check that we support runing for ANY/IN with literal.
SELECT count(*) FROM lineitem_hash_part
@ -100,6 +106,13 @@ SELECT count(*) FROM lineitem_hash_part
SELECT count(*) FROM lineitem_hash_part WHERE l_orderkey IN (SELECT l_orderkey FROM lineitem_hash_part);
SELECT count(*) FROM lineitem_hash_part WHERE l_orderkey = ANY (SELECT l_orderkey FROM lineitem_hash_part);
-- Check whether we support range queries with append distributed table
SELECT count(*) FROM lineitem
WHERE l_orderkey >= 1 AND l_orderkey <= 3;
SELECT count(*) FROM lineitem
WHERE (l_orderkey >= 1 AND l_orderkey <= 3) AND (l_quantity > 11 AND l_quantity < 22);
-- Check whether we support IN/ANY in subquery with append and range distributed table
SELECT count(*) FROM lineitem
WHERE l_orderkey = ANY ('{1,2,3}');
@ -119,8 +132,6 @@ SELECT count(*) FROM lineitem_range
SELECT count(*) FROM lineitem_range
WHERE l_orderkey = ANY(NULL) OR TRUE;
SET client_min_messages TO DEBUG2;
-- Check that we don't show the message if the operator is not
-- equality operator
SELECT count(*) FROM orders_hash_partitioned
@ -129,15 +140,15 @@ SELECT count(*) FROM orders_hash_partitioned
-- Check that we don't give a spurious hint message when non-partition
-- columns are used with ANY/IN/ALL
SELECT count(*) FROM orders_hash_partitioned
WHERE o_orderkey = 1 OR o_totalprice IN (2, 5);
WHERE o_orderkey = 1 OR o_totalprice IN (20, 30);
-- Check that we cannot prune for mutable functions.
SELECT count(*) FROM orders_hash_partitioned WHERE o_orderkey = random();
SELECT count(*) FROM orders_hash_partitioned WHERE o_orderkey = (random() + 100);
SELECT count(*) FROM orders_hash_partitioned
WHERE o_orderkey = random() OR o_orderkey = 1;
WHERE o_orderkey = (random() + 100) OR o_orderkey = 1;
SELECT count(*) FROM orders_hash_partitioned
WHERE o_orderkey = random() AND o_orderkey = 1;
WHERE o_orderkey = (random() + 100) AND o_orderkey = 1;
-- Check that we can do join pruning.
@ -150,3 +161,128 @@ SELECT count(*)
WHERE orders1.o_orderkey = orders2.o_orderkey
AND orders1.o_orderkey = 1
AND orders2.o_orderkey is NULL;
-- All shards used without constraints
SELECT count(*) FROM orders_hash_partitioned;
-- Shards restricted correctly with prunable constraint
SELECT count(*) FROM orders_hash_partitioned
WHERE o_orderkey = 1;
-- Shards restricted correctly with prunable constraint ANDed with unprunable expression using OR
SELECT count(*) FROM orders_hash_partitioned
WHERE o_orderkey = 1 AND (o_custkey = 11 OR o_custkey = 22);
-- Shards restricted correctly with prunable constraints ORed
SELECT count(*) FROM orders_hash_partitioned
WHERE (o_orderkey = 1 OR o_orderkey = 2);
-- Shards restricted correctly with prunable constraints ANDed with unprunable expression using OR
SELECT count(*) FROM orders_hash_partitioned
WHERE (o_orderkey = 1 OR o_orderkey = 2) AND (o_custkey = 11 OR o_custkey = 22);
-- Shards restricted correctly with many different prunable constraints ORed
SELECT count(*) FROM orders_hash_partitioned
WHERE (o_orderkey = 1 AND o_custkey = 11) OR (o_orderkey = 1 AND o_custkey = 33) OR (o_orderkey = 2 AND o_custkey = 22) OR (o_orderkey = 2 AND o_custkey = 44);
-- Shards restricted correctly with prunable SAO constraint ANDed with unprunable expression using OR
SELECT count(*) FROM orders_hash_partitioned
WHERE (o_orderkey IN (1,2)) AND (o_custkey = 11 OR o_custkey = 22 OR o_custkey = 33);
-- Shards restricted correctly with prunable SAO constraint ANDed with multiple unprunable expressions
SELECT count(*) FROM orders_hash_partitioned
WHERE (o_orderkey IN (1,2)) AND (o_totalprice < 11 OR o_totalprice > 19) AND o_shippriority > 100 AND (o_custkey = 11 OR o_custkey = 22);
-- Shards restricted correctly with prunable SAO constraints ORed
SELECT count(*) FROM orders_hash_partitioned
WHERE (o_orderkey IN (1,2) AND o_custkey = 11) OR (o_orderkey IN (2,3) AND o_custkey = 22);
-- All shards used with prunable expression ORed with unprunable expression
SELECT count(*) FROM orders_hash_partitioned
WHERE o_orderkey IN (1,2) OR o_custkey = 33;
-- Shards restricted correctly with prunable constraint ORed
SELECT count(*) FROM orders_hash_partitioned
WHERE o_orderkey = 1 OR ((o_orderkey = 2 AND o_custkey = 22) OR (o_orderkey = 3 AND o_custkey = 33));
-- Shards restricted correctly with prunable constraint ORed with falsy expression
SELECT count(*) FROM orders_hash_partitioned
WHERE o_orderkey = 1 OR (o_orderkey = 2 AND (o_custkey = 11 OR (o_orderkey = 3 AND o_custkey = 44)));
-- Shards restricted correctly with prunable SAO constraint ORed with prunable nested EQ constraint
SELECT count(*) FROM orders_hash_partitioned
WHERE (o_orderkey IN (1,2)) AND (o_custkey = 11 OR o_custkey = 22 OR o_custkey = 33) AND o_totalprice <= 20;
-- Shards restricted correctly with prunable SAO constraint ANDed with unprunable expressions
SELECT count(*) FROM orders_hash_partitioned
WHERE (o_orderkey IN (1,2)) AND (o_custkey = 11 OR o_custkey = 33) AND o_custkey = 22;
-- All shards used with prunable SAO constraint ORed with unprunable nested expression
SELECT count(*) FROM orders_hash_partitioned
WHERE ((o_orderkey IN (1,2)) AND (o_custkey = 11 OR o_custkey = 22)) OR o_custkey = 33;
-- Shards restricted correctly with prunable SAO constraint ORed with prunable nested EQ constraint
SELECT count(*) FROM orders_hash_partitioned
WHERE ((o_orderkey IN (1,2)) AND (o_custkey = 11 OR o_custkey = 22)) OR (o_orderkey = 3 AND o_custkey = 33);
-- All shards used with ORed top level unprunable expression
SELECT count(*) FROM orders_hash_partitioned
WHERE o_custkey = 11 OR (o_orderkey = 2 AND o_custkey = 22);
-- Single shard used when deeply nested prunable expression is restrictive with nested ANDs
SELECT count(*) FROM orders_hash_partitioned
WHERE o_orderkey = 1 OR (o_orderkey = 2 AND (o_orderkey = 3 OR (o_orderkey = 1 AND o_custkey = 11)));
-- Single shard used when top prunable expression is restrictive with nested ANDs
SELECT count(*) FROM orders_hash_partitioned
WHERE o_orderkey = 1 AND ((o_orderkey = 2 OR o_orderkey = 3) AND (o_custkey = 11 OR o_custkey = 22));
-- Deeply nested prunable expression affects used shards
SELECT count(*) FROM orders_hash_partitioned
WHERE o_orderkey = 1 OR ((o_orderkey = 2 OR o_orderkey = 3) AND (o_custkey = 22 OR o_custkey = 33));
-- Deeply nested non prunable expression uses all shards
SELECT count(*) FROM orders_hash_partitioned
WHERE o_orderkey = 1 OR ((o_orderkey = 2 OR o_custkey = 11) AND (o_custkey = 22 OR o_custkey = 33));
-- a OR partkey != x Uses all shards
SELECT count(*) FROM orders_hash_partitioned
WHERE o_orderkey = 1 OR o_orderkey != 2;
-- a OR partkey IS NULL Uses all shards
SELECT count(*) FROM orders_hash_partitioned
WHERE o_orderkey = 1 OR o_orderkey IS NULL;
-- a OR partkey IS NOT NULL Uses all shards
SELECT count(*) FROM orders_hash_partitioned
WHERE o_orderkey = 1 OR o_orderkey IS NOT NULL;
-- Check that NOT is handled with NEQs ORed
SELECT count(*) FROM orders_hash_partitioned
WHERE NOT (o_orderkey != 2 OR o_orderkey != 3);
-- Check that NOT is handled with EQs ORed
SELECT count(*) FROM orders_hash_partitioned
WHERE NOT (o_orderkey = 2 OR o_orderkey = 3);
-- Check that NOT is handled with NEQs ANDed
SELECT count(*) FROM orders_hash_partitioned
WHERE NOT (o_orderkey != 2 AND o_orderkey != 3);
-- Check that NOT is handled with EQs ANDed
SELECT count(*) FROM orders_hash_partitioned
WHERE NOT (o_orderkey = 2 AND o_orderkey = 3);
SET citus.task_executor_type TO 'adaptive';
-- Check that subquery NOT is pruned when ANDed to a valid constraint
SELECT count(*) FROM orders_hash_partitioned
WHERE o_orderkey IN (1,2) AND o_custkey NOT IN (SELECT o_custkey FROM orders_hash_partitioned WHERE o_orderkey = 1);
-- Check that subquery NOT is unpruned when ORed to a valid constraint
SELECT count(*) FROM orders_hash_partitioned
WHERE o_orderkey IN (1,2) OR o_custkey NOT IN (SELECT o_custkey FROM orders_hash_partitioned WHERE o_orderkey = 3);
SET citus.task_executor_type TO DEFAULT;
SET client_min_messages TO DEFAULT;