From 13e0fbd6696593cdeda12a0ffa4e66d605c26e55 Mon Sep 17 00:00:00 2001 From: naisila Date: Mon, 8 Aug 2022 12:41:15 +0300 Subject: [PATCH] Handles EXPLAIN output diffs in PG15 To handle extra "Result" lines in explain outputs, we add explain methods to multi_test_helpers.sql file - coordinator_plan() is updated to remove "Result" lines - plan_without_result_lines() is added for cases where we want the whole explain output with only "Result" lines removed To handle differences in usage of GroupAggregate vs HashAggregate in cases where this detail doesn't seem to matter, we use coordinator_plan(). There are some cases where we have subplans so we add a new function that prints all Task Count lines as well - coordinator_plan_with_subplans() --- .../expected/columnar_chunk_filtering.out | 12 +++-- .../expected/columnar_citus_integration.out | 6 ++- src/test/regress/expected/cte_inline.out | 39 ++++------------ .../expected/insert_select_repartition.out | 23 ++++------ .../expected/multi_select_distinct.out | 28 ++++++++--- .../expected/multi_select_distinct_1.out | 28 ++++++++--- .../regress/expected/multi_test_helpers.out | 43 +++++++++++++++++ src/test/regress/expected/multi_view.out | 25 ++-------- src/test/regress/expected/subquery_view.out | 46 ++++--------------- .../regress/expected/window_functions.out | 4 +- .../regress/expected/window_functions_0.out | 4 +- .../regress/expected/window_functions_1.out | 4 +- .../regress/sql/columnar_chunk_filtering.sql | 4 ++ .../sql/columnar_citus_integration.sql | 4 ++ src/test/regress/sql/cte_inline.sql | 9 +++- .../regress/sql/insert_select_repartition.sql | 6 +++ .../regress/sql/multi_select_distinct.sql | 14 ++++++ src/test/regress/sql/multi_test_helpers.sql | 45 ++++++++++++++++++ src/test/regress/sql/multi_view.sql | 2 + src/test/regress/sql/subquery_view.sql | 4 ++ src/test/regress/sql/window_functions.sql | 2 + 21 files changed, 228 insertions(+), 124 deletions(-) diff --git a/src/test/regress/expected/columnar_chunk_filtering.out b/src/test/regress/expected/columnar_chunk_filtering.out index 09688d7aa..a35bd3e71 100644 --- a/src/test/regress/expected/columnar_chunk_filtering.out +++ b/src/test/regress/expected/columnar_chunk_filtering.out @@ -264,19 +264,23 @@ EXPLAIN (analyze on, costs off, timing off, summary off) Columnar Projected Columns: a (9 rows) +SELECT public.plan_without_result_lines($Q$ EXPLAIN (costs off, timing off, summary off) SELECT y, * FROM another_columnar_table; - QUERY PLAN +$Q$); + plan_without_result_lines --------------------------------------------------------------------- - Custom Scan (ColumnarScan) on another_columnar_table + -> Custom Scan (ColumnarScan) on another_columnar_table Columnar Projected Columns: x, y (2 rows) +SELECT public.plan_without_result_lines($Q$ EXPLAIN (costs off, timing off, summary off) SELECT *, x FROM another_columnar_table; - QUERY PLAN +$Q$); + plan_without_result_lines --------------------------------------------------------------------- - Custom Scan (ColumnarScan) on another_columnar_table + -> Custom Scan (ColumnarScan) on another_columnar_table Columnar Projected Columns: x, y (2 rows) diff --git a/src/test/regress/expected/columnar_citus_integration.out b/src/test/regress/expected/columnar_citus_integration.out index d1baced0c..8beb09edf 100644 --- a/src/test/regress/expected/columnar_citus_integration.out +++ b/src/test/regress/expected/columnar_citus_integration.out @@ -957,13 +957,16 @@ SELECT * FROM weird_col_explain; Columnar Projected Columns: "bbbbbbbbbbbbbbbbbbbbbbbbb\!bbbb'bbbbbbbbbbbbbbbbbbbbb''bbbbbbbb", "aaaaaaaaaaaa$aaaaaa$$aaaaaaaaaaaaaaaaaaaaaaaaaaaaa'aaaaaaaa'$a'" (7 rows) +\set VERBOSITY terse +SELECT public.plan_without_result_lines($Q$ EXPLAIN (COSTS OFF, SUMMARY OFF) SELECT *, "bbbbbbbbbbbbbbbbbbbbbbbbb\!bbbb'bbbbbbbbbbbbbbbbbbbbb''bbbbbbbb" FROM weird_col_explain WHERE "bbbbbbbbbbbbbbbbbbbbbbbbb\!bbbb'bbbbbbbbbbbbbbbbbbbbb''bbbbbbbb" * 2 > "aaaaaaaaaaaa$aaaaaa$$aaaaaaaaaaaaaaaaaaaaaaaaaaaaa'aaaaaaaa'$a'!"; +$Q$); NOTICE: identifier "aaaaaaaaaaaa$aaaaaa$$aaaaaaaaaaaaaaaaaaaaaaaaaaaaa'aaaaaaaa'$a'!" will be truncated to "aaaaaaaaaaaa$aaaaaa$$aaaaaaaaaaaaaaaaaaaaaaaaaaaaa'aaaaaaaa'$a'" - QUERY PLAN + plan_without_result_lines --------------------------------------------------------------------- Custom Scan (Citus Adaptive) Task Count: 4 @@ -975,6 +978,7 @@ NOTICE: identifier "aaaaaaaaaaaa$aaaaaa$$aaaaaaaaaaaaaaaaaaaaaaaaaaaaa'aaaaaaaa Columnar Projected Columns: "bbbbbbbbbbbbbbbbbbbbbbbbb\!bbbb'bbbbbbbbbbbbbbbbbbbbb''bbbbbbbb", "aaaaaaaaaaaa$aaaaaa$$aaaaaaaaaaaaaaaaaaaaaaaaaaaaa'aaaaaaaa'$a'" (8 rows) +\set VERBOSITY default -- should not project any columns EXPLAIN (COSTS OFF, SUMMARY OFF) SELECT COUNT(*) FROM weird_col_explain; diff --git a/src/test/regress/expected/cte_inline.out b/src/test/regress/expected/cte_inline.out index 072f076d6..82d6a8e49 100644 --- a/src/test/regress/expected/cte_inline.out +++ b/src/test/regress/expected/cte_inline.out @@ -423,6 +423,8 @@ DEBUG: join prunable for intervals [1073741824,2147483647] and [0,1073741823] (1 row) -- EXPLAIN should show the differences between MATERIALIZED and NOT MATERIALIZED +\set VERBOSITY terse +SELECT public.coordinator_plan_with_subplans($Q$ EXPLAIN (COSTS OFF) WITH cte_1 AS (SELECT * FROM test_table) SELECT count(*) @@ -431,36 +433,22 @@ FROM JOIN cte_1 as second_entry USING (key); +$Q$); DEBUG: Router planner cannot handle multi-shard select queries DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table DEBUG: Router planner cannot handle multi-shard select queries DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) first_entry JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) second_entry USING (key)) DEBUG: Creating router plan - QUERY PLAN + coordinator_plan_with_subplans --------------------------------------------------------------------- Custom Scan (Citus Adaptive) -> Distributed Subplan XXX_1 -> Custom Scan (Citus Adaptive) Task Count: 4 - Tasks Shown: One of 4 - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Seq Scan on test_table_1960000 test_table Task Count: 1 - Tasks Shown: All - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Aggregate - -> Merge Join - Merge Cond: (intermediate_result.key = intermediate_result_1.key) - -> Sort - Sort Key: intermediate_result.key - -> Function Scan on read_intermediate_result intermediate_result - -> Sort - Sort Key: intermediate_result_1.key - -> Function Scan on read_intermediate_result intermediate_result_1 -(21 rows) +(5 rows) +SELECT public.coordinator_plan_with_subplans($Q$ EXPLAIN (COSTS OFF) WITH cte_1 AS NOT MATERIALIZED (SELECT * FROM test_table) SELECT count(*) @@ -469,6 +457,7 @@ FROM JOIN cte_1 as second_entry USING (key); +$Q$); DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: Router planner cannot handle multi-shard select queries DEBUG: join prunable for intervals [-2147483648,-1073741825] and [-1073741824,-1] @@ -483,22 +472,14 @@ DEBUG: join prunable for intervals [0,1073741823] and [1073741824,2147483647] DEBUG: join prunable for intervals [1073741824,2147483647] and [-2147483648,-1073741825] DEBUG: join prunable for intervals [1073741824,2147483647] and [-1073741824,-1] DEBUG: join prunable for intervals [1073741824,2147483647] and [0,1073741823] - QUERY PLAN + coordinator_plan_with_subplans --------------------------------------------------------------------- Aggregate -> Custom Scan (Citus Adaptive) Task Count: 4 - Tasks Shown: One of 4 - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Aggregate - -> Hash Join - Hash Cond: (test_table.key = test_table_1.key) - -> Seq Scan on test_table_1960000 test_table - -> Hash - -> Seq Scan on test_table_1960000 test_table_1 -(12 rows) +(3 rows) +\set VERBOSITY default -- ctes with volatile functions are not -- inlined WITH cte_1 AS (SELECT *, random() FROM test_table) diff --git a/src/test/regress/expected/insert_select_repartition.out b/src/test/regress/expected/insert_select_repartition.out index f6e4f17a5..f6cc3db97 100644 --- a/src/test/regress/expected/insert_select_repartition.out +++ b/src/test/regress/expected/insert_select_repartition.out @@ -702,23 +702,16 @@ PREPARE insert_plan AS INSERT INTO target_table SELECT a, max(b) FROM source_table WHERE a BETWEEN 1 AND 2 GROUP BY a; +SELECT public.coordinator_plan($Q$ EXPLAIN EXECUTE insert_plan; - QUERY PLAN +$Q$); + coordinator_plan --------------------------------------------------------------------- Custom Scan (Citus INSERT ... SELECT) (cost=0.00..0.00 rows=0 width=0) INSERT/SELECT method: repartition -> Custom Scan (Citus Adaptive) (cost=0.00..0.00 rows=100000 width=8) Task Count: 4 - Tasks Shown: One of 4 - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> GroupAggregate (cost=44.09..44.28 rows=11 width=8) - Group Key: a - -> Sort (cost=44.09..44.12 rows=11 width=8) - Sort Key: a - -> Seq Scan on source_table_4213606 source_table (cost=0.00..43.90 rows=11 width=8) - Filter: ((a >= 1) AND (a <= 2)) -(13 rows) +(4 rows) SET client_min_messages TO DEBUG1; EXECUTE insert_plan; @@ -1254,8 +1247,10 @@ NOTICE: copying the data has completed (1 row) +SELECT public.plan_without_result_lines($Q$ explain (costs off) insert into table_with_sequences select y, x from table_with_sequences; - QUERY PLAN +$Q$); + plan_without_result_lines --------------------------------------------------------------------- Custom Scan (Citus INSERT ... SELECT) INSERT/SELECT method: pull to coordinator @@ -1280,8 +1275,10 @@ NOTICE: copying the data has completed (1 row) +SELECT public.plan_without_result_lines($Q$ explain (costs off) insert into table_with_user_sequences select y, x from table_with_user_sequences; - QUERY PLAN +$Q$); + plan_without_result_lines --------------------------------------------------------------------- Custom Scan (Citus INSERT ... SELECT) INSERT/SELECT method: pull to coordinator diff --git a/src/test/regress/expected/multi_select_distinct.out b/src/test/regress/expected/multi_select_distinct.out index bfb189095..599d85e1d 100644 --- a/src/test/regress/expected/multi_select_distinct.out +++ b/src/test/regress/expected/multi_select_distinct.out @@ -306,12 +306,14 @@ EXPLAIN (COSTS FALSE) -- check the plan if the hash aggreate is disabled. We expect to see sort+unique -- instead of aggregate plan node to handle distinct. SET enable_hashagg TO off; +SELECT public.plan_without_result_lines($Q$ EXPLAIN (COSTS FALSE) SELECT DISTINCT count(*) FROM lineitem_hash_part GROUP BY l_suppkey, l_linenumber ORDER BY 1; - QUERY PLAN +$Q$); + plan_without_result_lines --------------------------------------------------------------------- Unique -> Sort @@ -380,13 +382,15 @@ EXPLAIN (COSTS FALSE) -- check the plan if the hash aggreate is disabled. Similar to the explain of -- the query above. SET enable_hashagg TO off; +SELECT public.plan_without_result_lines($Q$ EXPLAIN (COSTS FALSE) SELECT DISTINCT l_suppkey, count(*) FROM lineitem_hash_part GROUP BY l_suppkey, l_linenumber ORDER BY 1 LIMIT 10; - QUERY PLAN +$Q$); + plan_without_result_lines --------------------------------------------------------------------- Limit -> Unique @@ -457,13 +461,15 @@ EXPLAIN (COSTS FALSE) -- check the plan if the hash aggreate is disabled. This explain errors out due -- to a bug right now, expectation must be corrected after fixing it. SET enable_hashagg TO off; +SELECT public.plan_without_result_lines($Q$ EXPLAIN (COSTS FALSE) SELECT DISTINCT l_suppkey, avg(l_partkey) FROM lineitem_hash_part GROUP BY l_suppkey, l_linenumber ORDER BY 1,2 LIMIT 10; - QUERY PLAN +$Q$); + plan_without_result_lines --------------------------------------------------------------------- Limit -> Unique @@ -533,13 +539,15 @@ EXPLAIN (COSTS FALSE) -- check the plan if the hash aggreate is disabled. We expect to see sort+unique to -- handle distinct on. SET enable_hashagg TO off; +SELECT public.plan_without_result_lines($Q$ EXPLAIN (COSTS FALSE) SELECT DISTINCT ON (l_suppkey) avg(l_partkey) FROM lineitem_hash_part GROUP BY l_suppkey, l_linenumber ORDER BY l_suppkey,1 LIMIT 10; - QUERY PLAN +$Q$); + plan_without_result_lines --------------------------------------------------------------------- Limit -> Unique @@ -608,13 +616,15 @@ EXPLAIN (COSTS FALSE) -- check the plan if the hash aggreate is disabled. This explain errors out due -- to a bug right now, expectation must be corrected after fixing it. SET enable_hashagg TO off; +SELECT public.plan_without_result_lines($Q$ EXPLAIN (COSTS FALSE) SELECT DISTINCT avg(ceil(l_partkey / 2)) FROM lineitem_hash_part GROUP BY l_suppkey, l_linenumber ORDER BY 1 LIMIT 10; - QUERY PLAN +$Q$); + plan_without_result_lines --------------------------------------------------------------------- Limit -> Unique @@ -683,13 +693,15 @@ EXPLAIN (COSTS FALSE) -- check the plan if the hash aggreate is disabled. This explain errors out due -- to a bug right now, expectation must be corrected after fixing it. SET enable_hashagg TO off; +SELECT public.plan_without_result_lines($Q$ EXPLAIN (COSTS FALSE) SELECT DISTINCT sum(l_suppkey) + count(l_partkey) AS dis FROM lineitem_hash_part GROUP BY l_suppkey, l_linenumber ORDER BY 1 LIMIT 10; - QUERY PLAN +$Q$); + plan_without_result_lines --------------------------------------------------------------------- Limit -> Unique @@ -914,12 +926,14 @@ EXPLAIN (COSTS FALSE) -- check the plan if the hash aggreate is disabled SET enable_hashagg TO off; +SELECT public.plan_without_result_lines($Q$ EXPLAIN (COSTS FALSE) SELECT DISTINCT ceil(count(case when l_partkey > 100000 THEN 1 ELSE 0 END) / 2) AS count FROM lineitem_hash_part GROUP BY l_suppkey ORDER BY 1; - QUERY PLAN +$Q$); + plan_without_result_lines --------------------------------------------------------------------- Unique -> Sort diff --git a/src/test/regress/expected/multi_select_distinct_1.out b/src/test/regress/expected/multi_select_distinct_1.out index 506ce215f..cfd948e78 100644 --- a/src/test/regress/expected/multi_select_distinct_1.out +++ b/src/test/regress/expected/multi_select_distinct_1.out @@ -306,12 +306,14 @@ EXPLAIN (COSTS FALSE) -- check the plan if the hash aggreate is disabled. We expect to see sort+unique -- instead of aggregate plan node to handle distinct. SET enable_hashagg TO off; +SELECT public.plan_without_result_lines($Q$ EXPLAIN (COSTS FALSE) SELECT DISTINCT count(*) FROM lineitem_hash_part GROUP BY l_suppkey, l_linenumber ORDER BY 1; - QUERY PLAN +$Q$); + plan_without_result_lines --------------------------------------------------------------------- Unique -> Sort @@ -380,13 +382,15 @@ EXPLAIN (COSTS FALSE) -- check the plan if the hash aggreate is disabled. Similar to the explain of -- the query above. SET enable_hashagg TO off; +SELECT public.plan_without_result_lines($Q$ EXPLAIN (COSTS FALSE) SELECT DISTINCT l_suppkey, count(*) FROM lineitem_hash_part GROUP BY l_suppkey, l_linenumber ORDER BY 1 LIMIT 10; - QUERY PLAN +$Q$); + plan_without_result_lines --------------------------------------------------------------------- Limit -> Unique @@ -457,13 +461,15 @@ EXPLAIN (COSTS FALSE) -- check the plan if the hash aggreate is disabled. This explain errors out due -- to a bug right now, expectation must be corrected after fixing it. SET enable_hashagg TO off; +SELECT public.plan_without_result_lines($Q$ EXPLAIN (COSTS FALSE) SELECT DISTINCT l_suppkey, avg(l_partkey) FROM lineitem_hash_part GROUP BY l_suppkey, l_linenumber ORDER BY 1,2 LIMIT 10; - QUERY PLAN +$Q$); + plan_without_result_lines --------------------------------------------------------------------- Limit -> Unique @@ -533,13 +539,15 @@ EXPLAIN (COSTS FALSE) -- check the plan if the hash aggreate is disabled. We expect to see sort+unique to -- handle distinct on. SET enable_hashagg TO off; +SELECT public.plan_without_result_lines($Q$ EXPLAIN (COSTS FALSE) SELECT DISTINCT ON (l_suppkey) avg(l_partkey) FROM lineitem_hash_part GROUP BY l_suppkey, l_linenumber ORDER BY l_suppkey,1 LIMIT 10; - QUERY PLAN +$Q$); + plan_without_result_lines --------------------------------------------------------------------- Limit -> Unique @@ -608,13 +616,15 @@ EXPLAIN (COSTS FALSE) -- check the plan if the hash aggreate is disabled. This explain errors out due -- to a bug right now, expectation must be corrected after fixing it. SET enable_hashagg TO off; +SELECT public.plan_without_result_lines($Q$ EXPLAIN (COSTS FALSE) SELECT DISTINCT avg(ceil(l_partkey / 2)) FROM lineitem_hash_part GROUP BY l_suppkey, l_linenumber ORDER BY 1 LIMIT 10; - QUERY PLAN +$Q$); + plan_without_result_lines --------------------------------------------------------------------- Limit -> Unique @@ -683,13 +693,15 @@ EXPLAIN (COSTS FALSE) -- check the plan if the hash aggreate is disabled. This explain errors out due -- to a bug right now, expectation must be corrected after fixing it. SET enable_hashagg TO off; +SELECT public.plan_without_result_lines($Q$ EXPLAIN (COSTS FALSE) SELECT DISTINCT sum(l_suppkey) + count(l_partkey) AS dis FROM lineitem_hash_part GROUP BY l_suppkey, l_linenumber ORDER BY 1 LIMIT 10; - QUERY PLAN +$Q$); + plan_without_result_lines --------------------------------------------------------------------- Limit -> Unique @@ -914,12 +926,14 @@ EXPLAIN (COSTS FALSE) -- check the plan if the hash aggreate is disabled SET enable_hashagg TO off; +SELECT public.plan_without_result_lines($Q$ EXPLAIN (COSTS FALSE) SELECT DISTINCT ceil(count(case when l_partkey > 100000 THEN 1 ELSE 0 END) / 2) AS count FROM lineitem_hash_part GROUP BY l_suppkey ORDER BY 1; - QUERY PLAN +$Q$); + plan_without_result_lines --------------------------------------------------------------------- Unique -> Sort diff --git a/src/test/regress/expected/multi_test_helpers.out b/src/test/regress/expected/multi_test_helpers.out index 5ae18f26b..118a052c2 100644 --- a/src/test/regress/expected/multi_test_helpers.out +++ b/src/test/regress/expected/multi_test_helpers.out @@ -17,10 +17,15 @@ BEGIN END; $$LANGUAGE plpgsql; -- Create a function to ignore worker plans in explain output +-- Also remove extra "-> Result" lines for PG15 support CREATE OR REPLACE FUNCTION coordinator_plan(explain_command text, out query_plan text) RETURNS SETOF TEXT AS $$ BEGIN FOR query_plan IN execute explain_command LOOP + IF (query_plan LIKE '%-> Result%' OR query_plan = 'Result') + THEN + CONTINUE; + END IF; RETURN next; IF query_plan LIKE '%Task Count:%' THEN @@ -29,6 +34,44 @@ BEGIN END LOOP; RETURN; END; $$ language plpgsql; +-- Create a function to ignore worker plans in explain output +-- It also shows task count for plan and subplans +-- Also remove extra "-> Result" lines for PG15 support +CREATE OR REPLACE FUNCTION coordinator_plan_with_subplans(explain_command text, out query_plan text) +RETURNS SETOF TEXT AS $$ +DECLARE + task_count_line_reached boolean := false; +BEGIN + FOR query_plan IN execute explain_command LOOP + IF (query_plan LIKE '%-> Result%' OR query_plan = 'Result') THEN + CONTINUE; + END IF; + IF NOT task_count_line_reached THEN + RETURN next; + END IF; + IF query_plan LIKE '%Task Count:%' THEN + IF NOT task_count_line_reached THEN + SELECT true INTO task_count_line_reached; + ELSE + RETURN next; + END IF; + END IF; + END LOOP; + RETURN; +END; $$ language plpgsql; +-- Create a function to ignore "-> Result" lines for PG15 support +-- In PG15 there are some extra "-> Result" lines +CREATE OR REPLACE FUNCTION plan_without_result_lines(explain_command text, out query_plan text) +RETURNS SETOF TEXT AS $$ +BEGIN + FOR query_plan IN execute explain_command LOOP + IF (query_plan LIKE '%-> Result%' OR query_plan = 'Result') THEN + CONTINUE; + END IF; + RETURN next; + END LOOP; + RETURN; +END; $$ language plpgsql; -- helper function that returns true if output of given explain has "is not null" (case in-sensitive) CREATE OR REPLACE FUNCTION explain_has_is_not_null(explain_command text) RETURNS BOOLEAN AS $$ diff --git a/src/test/regress/expected/multi_view.out b/src/test/regress/expected/multi_view.out index c7a6b44e0..70fa10874 100644 --- a/src/test/regress/expected/multi_view.out +++ b/src/test/regress/expected/multi_view.out @@ -785,6 +785,7 @@ EXPLAIN (COSTS FALSE) SELECT user_id FROM recent_selected_users GROUP BY 1 ORDER Filter: ((value_1 >= 1) AND (value_1 < 3)) (19 rows) +SELECT public.coordinator_plan($Q$ EXPLAIN (COSTS FALSE) SELECT * FROM ( (SELECT user_id FROM recent_users) @@ -792,32 +793,14 @@ EXPLAIN (COSTS FALSE) SELECT * (SELECT user_id FROM selected_users) ) u WHERE user_id < 4 AND user_id > 1 ORDER BY user_id; - QUERY PLAN +$Q$); + coordinator_plan --------------------------------------------------------------------- Sort Sort Key: remote_scan.user_id -> Custom Scan (Citus Adaptive) Task Count: 4 - Tasks Shown: One of 4 - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Unique - -> Sort - Sort Key: recent_users.user_id - -> Append - -> Subquery Scan on recent_users - -> Sort - Sort Key: (max(users_table."time")) DESC - -> GroupAggregate - Group Key: users_table.user_id - Filter: (max(users_table."time") > '2017-11-23 16:20:33.264457'::timestamp without time zone) - -> Sort - Sort Key: users_table.user_id - -> Seq Scan on users_table_1400256 users_table - Filter: ((user_id < 4) AND (user_id > 1)) - -> Seq Scan on users_table_1400256 users_table_1 - Filter: ((value_1 >= 1) AND (value_1 < 3) AND (user_id < 4) AND (user_id > 1)) -(23 rows) +(4 rows) EXPLAIN (COSTS FALSE) SELECT et.* FROM recent_10_users JOIN events_table et USING(user_id) ORDER BY et.time DESC LIMIT 10; QUERY PLAN diff --git a/src/test/regress/expected/subquery_view.out b/src/test/regress/expected/subquery_view.out index 535e356d5..32354e329 100644 --- a/src/test/regress/expected/subquery_view.out +++ b/src/test/regress/expected/subquery_view.out @@ -578,11 +578,13 @@ SELECT create_reference_table('reference_table'); (1 row) +SELECT public.coordinator_plan_with_subplans($Q$ EXPLAIN (COSTS OFF) WITH cte AS ( SELECT application_name AS text_col FROM pg_stat_activity ) SELECT * FROM reference_table JOIN cte USING (text_col); - QUERY PLAN +$Q$); + coordinator_plan_with_subplans --------------------------------------------------------------------- Custom Scan (Citus Adaptive) -> Distributed Subplan XXX_1 @@ -590,38 +592,17 @@ EXPLAIN (COSTS OFF) WITH cte AS ( -> Distributed Subplan XXX_2 -> Custom Scan (Citus Adaptive) Task Count: 1 - Tasks Shown: All - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Hash Left Join - Hash Cond: (intermediate_result.usesysid = u.oid) - -> Hash Left Join - Hash Cond: (intermediate_result.datid = d.oid) - -> Function Scan on read_intermediate_result intermediate_result - -> Hash - -> Seq Scan on pg_database d - -> Hash - -> Seq Scan on pg_authid u Task Count: 1 - Tasks Shown: All - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Merge Join - Merge Cond: (intermediate_result.application_name = reference_table.text_col) - -> Sort - Sort Key: intermediate_result.application_name - -> Function Scan on read_intermediate_result intermediate_result - -> Sort - Sort Key: reference_table.text_col - -> Seq Scan on reference_table_1512000 reference_table -(30 rows) +(7 rows) CREATE OR REPLACE VIEW view_on_views AS SELECT pg_stat_activity.application_name, pg_locks.pid FROM pg_stat_activity, pg_locks; +SELECT public.coordinator_plan_with_subplans($Q$ EXPLAIN (COSTS OFF) WITH cte AS ( SELECT application_name AS text_col FROM view_on_views ) SELECT * FROM reference_table JOIN cte USING (text_col); - QUERY PLAN +$Q$); + coordinator_plan_with_subplans --------------------------------------------------------------------- Custom Scan (Citus Adaptive) -> Distributed Subplan XXX_1 @@ -629,18 +610,7 @@ EXPLAIN (COSTS OFF) WITH cte AS ( -> Function Scan on pg_stat_get_activity s -> Function Scan on pg_lock_status l Task Count: 1 - Tasks Shown: All - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Merge Join - Merge Cond: (intermediate_result.text_col = reference_table.text_col) - -> Sort - Sort Key: intermediate_result.text_col - -> Function Scan on read_intermediate_result intermediate_result - -> Sort - Sort Key: reference_table.text_col - -> Seq Scan on reference_table_1512000 reference_table -(17 rows) +(6 rows) DROP SCHEMA subquery_view CASCADE; NOTICE: drop cascades to 19 other objects diff --git a/src/test/regress/expected/window_functions.out b/src/test/regress/expected/window_functions.out index 6657c3670..1b3e4016e 100644 --- a/src/test/regress/expected/window_functions.out +++ b/src/test/regress/expected/window_functions.out @@ -1506,10 +1506,12 @@ LIMIT 5; (22 rows) -- Grouping can be pushed down with aggregates even when window function can't +SELECT public.plan_without_result_lines($Q$ EXPLAIN (COSTS FALSE) SELECT user_id, count(value_1), stddev(value_1), count(user_id) OVER (PARTITION BY random()) FROM users_table GROUP BY user_id HAVING avg(value_1) > 2 LIMIT 1; - QUERY PLAN +$Q$); + plan_without_result_lines --------------------------------------------------------------------- Limit -> WindowAgg diff --git a/src/test/regress/expected/window_functions_0.out b/src/test/regress/expected/window_functions_0.out index 0a41bc0cc..a330b4cf4 100644 --- a/src/test/regress/expected/window_functions_0.out +++ b/src/test/regress/expected/window_functions_0.out @@ -1490,10 +1490,12 @@ LIMIT 5; (18 rows) -- Grouping can be pushed down with aggregates even when window function can't +SELECT public.plan_without_result_lines($Q$ EXPLAIN (COSTS FALSE) SELECT user_id, count(value_1), stddev(value_1), count(user_id) OVER (PARTITION BY random()) FROM users_table GROUP BY user_id HAVING avg(value_1) > 2 LIMIT 1; - QUERY PLAN +$Q$); + plan_without_result_lines --------------------------------------------------------------------- Limit -> WindowAgg diff --git a/src/test/regress/expected/window_functions_1.out b/src/test/regress/expected/window_functions_1.out index aea319c0b..d6203d41b 100644 --- a/src/test/regress/expected/window_functions_1.out +++ b/src/test/regress/expected/window_functions_1.out @@ -1486,10 +1486,12 @@ LIMIT 5; (17 rows) -- Grouping can be pushed down with aggregates even when window function can't +SELECT public.plan_without_result_lines($Q$ EXPLAIN (COSTS FALSE) SELECT user_id, count(value_1), stddev(value_1), count(user_id) OVER (PARTITION BY random()) FROM users_table GROUP BY user_id HAVING avg(value_1) > 2 LIMIT 1; - QUERY PLAN +$Q$); + plan_without_result_lines --------------------------------------------------------------------- Limit -> WindowAgg diff --git a/src/test/regress/sql/columnar_chunk_filtering.sql b/src/test/regress/sql/columnar_chunk_filtering.sql index 9e7d43363..5ed090832 100644 --- a/src/test/regress/sql/columnar_chunk_filtering.sql +++ b/src/test/regress/sql/columnar_chunk_filtering.sql @@ -130,11 +130,15 @@ INSERT INTO another_columnar_table SELECT generate_series(0,5); EXPLAIN (analyze on, costs off, timing off, summary off) SELECT a, y FROM multi_column_chunk_filtering, another_columnar_table WHERE x > 1; +SELECT public.plan_without_result_lines($Q$ EXPLAIN (costs off, timing off, summary off) SELECT y, * FROM another_columnar_table; +$Q$); +SELECT public.plan_without_result_lines($Q$ EXPLAIN (costs off, timing off, summary off) SELECT *, x FROM another_columnar_table; +$Q$); EXPLAIN (costs off, timing off, summary off) SELECT y, another_columnar_table FROM another_columnar_table; diff --git a/src/test/regress/sql/columnar_citus_integration.sql b/src/test/regress/sql/columnar_citus_integration.sql index a64a37108..566c3a9f6 100644 --- a/src/test/regress/sql/columnar_citus_integration.sql +++ b/src/test/regress/sql/columnar_citus_integration.sql @@ -428,11 +428,15 @@ SELECT create_distributed_table('weird_col_explain', 'bbbbbbbbbbbbbbbbbbbbbbbbb\ EXPLAIN (COSTS OFF, SUMMARY OFF) SELECT * FROM weird_col_explain; +\set VERBOSITY terse +SELECT public.plan_without_result_lines($Q$ EXPLAIN (COSTS OFF, SUMMARY OFF) SELECT *, "bbbbbbbbbbbbbbbbbbbbbbbbb\!bbbb'bbbbbbbbbbbbbbbbbbbbb''bbbbbbbb" FROM weird_col_explain WHERE "bbbbbbbbbbbbbbbbbbbbbbbbb\!bbbb'bbbbbbbbbbbbbbbbbbbbb''bbbbbbbb" * 2 > "aaaaaaaaaaaa$aaaaaa$$aaaaaaaaaaaaaaaaaaaaaaaaaaaaa'aaaaaaaa'$a'!"; +$Q$); +\set VERBOSITY default -- should not project any columns EXPLAIN (COSTS OFF, SUMMARY OFF) diff --git a/src/test/regress/sql/cte_inline.sql b/src/test/regress/sql/cte_inline.sql index 28691e35a..123ce8215 100644 --- a/src/test/regress/sql/cte_inline.sql +++ b/src/test/regress/sql/cte_inline.sql @@ -220,6 +220,10 @@ FROM USING (key); -- EXPLAIN should show the differences between MATERIALIZED and NOT MATERIALIZED + +\set VERBOSITY terse + +SELECT public.coordinator_plan_with_subplans($Q$ EXPLAIN (COSTS OFF) WITH cte_1 AS (SELECT * FROM test_table) SELECT count(*) @@ -228,7 +232,9 @@ FROM JOIN cte_1 as second_entry USING (key); +$Q$); +SELECT public.coordinator_plan_with_subplans($Q$ EXPLAIN (COSTS OFF) WITH cte_1 AS NOT MATERIALIZED (SELECT * FROM test_table) SELECT count(*) @@ -237,8 +243,9 @@ FROM JOIN cte_1 as second_entry USING (key); +$Q$); - +\set VERBOSITY default -- ctes with volatile functions are not -- inlined diff --git a/src/test/regress/sql/insert_select_repartition.sql b/src/test/regress/sql/insert_select_repartition.sql index 234d3374d..f88c67f50 100644 --- a/src/test/regress/sql/insert_select_repartition.sql +++ b/src/test/regress/sql/insert_select_repartition.sql @@ -354,7 +354,9 @@ INSERT INTO target_table SELECT a, max(b) FROM source_table WHERE a BETWEEN 1 AND 2 GROUP BY a; +SELECT public.coordinator_plan($Q$ EXPLAIN EXECUTE insert_plan; +$Q$); SET client_min_messages TO DEBUG1; EXECUTE insert_plan; @@ -623,7 +625,9 @@ DO UPDATE SET create table table_with_sequences (x int, y int, z bigserial); insert into table_with_sequences values (1,1); select create_distributed_table('table_with_sequences','x'); +SELECT public.plan_without_result_lines($Q$ explain (costs off) insert into table_with_sequences select y, x from table_with_sequences; +$Q$); -- verify that we don't report repartitioned insert/select for tables -- with user-defined sequences. @@ -631,7 +635,9 @@ CREATE SEQUENCE user_defined_sequence; create table table_with_user_sequences (x int, y int, z bigint default nextval('user_defined_sequence')); insert into table_with_user_sequences values (1,1); select create_distributed_table('table_with_user_sequences','x'); +SELECT public.plan_without_result_lines($Q$ explain (costs off) insert into table_with_user_sequences select y, x from table_with_user_sequences; +$Q$); -- clean-up SET client_min_messages TO WARNING; diff --git a/src/test/regress/sql/multi_select_distinct.sql b/src/test/regress/sql/multi_select_distinct.sql index 8254da5ba..39f49b746 100644 --- a/src/test/regress/sql/multi_select_distinct.sql +++ b/src/test/regress/sql/multi_select_distinct.sql @@ -113,11 +113,13 @@ EXPLAIN (COSTS FALSE) -- check the plan if the hash aggreate is disabled. We expect to see sort+unique -- instead of aggregate plan node to handle distinct. SET enable_hashagg TO off; +SELECT public.plan_without_result_lines($Q$ EXPLAIN (COSTS FALSE) SELECT DISTINCT count(*) FROM lineitem_hash_part GROUP BY l_suppkey, l_linenumber ORDER BY 1; +$Q$); SET enable_hashagg TO on; @@ -140,12 +142,14 @@ EXPLAIN (COSTS FALSE) -- check the plan if the hash aggreate is disabled. Similar to the explain of -- the query above. SET enable_hashagg TO off; +SELECT public.plan_without_result_lines($Q$ EXPLAIN (COSTS FALSE) SELECT DISTINCT l_suppkey, count(*) FROM lineitem_hash_part GROUP BY l_suppkey, l_linenumber ORDER BY 1 LIMIT 10; +$Q$); SET enable_hashagg TO on; @@ -169,12 +173,14 @@ EXPLAIN (COSTS FALSE) -- check the plan if the hash aggreate is disabled. This explain errors out due -- to a bug right now, expectation must be corrected after fixing it. SET enable_hashagg TO off; +SELECT public.plan_without_result_lines($Q$ EXPLAIN (COSTS FALSE) SELECT DISTINCT l_suppkey, avg(l_partkey) FROM lineitem_hash_part GROUP BY l_suppkey, l_linenumber ORDER BY 1,2 LIMIT 10; +$Q$); SET enable_hashagg TO on; @@ -197,12 +203,14 @@ EXPLAIN (COSTS FALSE) -- check the plan if the hash aggreate is disabled. We expect to see sort+unique to -- handle distinct on. SET enable_hashagg TO off; +SELECT public.plan_without_result_lines($Q$ EXPLAIN (COSTS FALSE) SELECT DISTINCT ON (l_suppkey) avg(l_partkey) FROM lineitem_hash_part GROUP BY l_suppkey, l_linenumber ORDER BY l_suppkey,1 LIMIT 10; +$Q$); SET enable_hashagg TO on; @@ -224,12 +232,14 @@ EXPLAIN (COSTS FALSE) -- check the plan if the hash aggreate is disabled. This explain errors out due -- to a bug right now, expectation must be corrected after fixing it. SET enable_hashagg TO off; +SELECT public.plan_without_result_lines($Q$ EXPLAIN (COSTS FALSE) SELECT DISTINCT avg(ceil(l_partkey / 2)) FROM lineitem_hash_part GROUP BY l_suppkey, l_linenumber ORDER BY 1 LIMIT 10; +$Q$); SET enable_hashagg TO on; @@ -251,12 +261,14 @@ EXPLAIN (COSTS FALSE) -- check the plan if the hash aggreate is disabled. This explain errors out due -- to a bug right now, expectation must be corrected after fixing it. SET enable_hashagg TO off; +SELECT public.plan_without_result_lines($Q$ EXPLAIN (COSTS FALSE) SELECT DISTINCT sum(l_suppkey) + count(l_partkey) AS dis FROM lineitem_hash_part GROUP BY l_suppkey, l_linenumber ORDER BY 1 LIMIT 10; +$Q$); SET enable_hashagg TO on; @@ -329,11 +341,13 @@ EXPLAIN (COSTS FALSE) -- check the plan if the hash aggreate is disabled SET enable_hashagg TO off; +SELECT public.plan_without_result_lines($Q$ EXPLAIN (COSTS FALSE) SELECT DISTINCT ceil(count(case when l_partkey > 100000 THEN 1 ELSE 0 END) / 2) AS count FROM lineitem_hash_part GROUP BY l_suppkey ORDER BY 1; +$Q$); SET enable_hashagg TO on; diff --git a/src/test/regress/sql/multi_test_helpers.sql b/src/test/regress/sql/multi_test_helpers.sql index 9e70cb971..c1bff60c9 100644 --- a/src/test/regress/sql/multi_test_helpers.sql +++ b/src/test/regress/sql/multi_test_helpers.sql @@ -20,10 +20,15 @@ END; $$LANGUAGE plpgsql; -- Create a function to ignore worker plans in explain output +-- Also remove extra "-> Result" lines for PG15 support CREATE OR REPLACE FUNCTION coordinator_plan(explain_command text, out query_plan text) RETURNS SETOF TEXT AS $$ BEGIN FOR query_plan IN execute explain_command LOOP + IF (query_plan LIKE '%-> Result%' OR query_plan = 'Result') + THEN + CONTINUE; + END IF; RETURN next; IF query_plan LIKE '%Task Count:%' THEN @@ -33,6 +38,46 @@ BEGIN RETURN; END; $$ language plpgsql; +-- Create a function to ignore worker plans in explain output +-- It also shows task count for plan and subplans +-- Also remove extra "-> Result" lines for PG15 support +CREATE OR REPLACE FUNCTION coordinator_plan_with_subplans(explain_command text, out query_plan text) +RETURNS SETOF TEXT AS $$ +DECLARE + task_count_line_reached boolean := false; +BEGIN + FOR query_plan IN execute explain_command LOOP + IF (query_plan LIKE '%-> Result%' OR query_plan = 'Result') THEN + CONTINUE; + END IF; + IF NOT task_count_line_reached THEN + RETURN next; + END IF; + IF query_plan LIKE '%Task Count:%' THEN + IF NOT task_count_line_reached THEN + SELECT true INTO task_count_line_reached; + ELSE + RETURN next; + END IF; + END IF; + END LOOP; + RETURN; +END; $$ language plpgsql; + +-- Create a function to ignore "-> Result" lines for PG15 support +-- In PG15 there are some extra "-> Result" lines +CREATE OR REPLACE FUNCTION plan_without_result_lines(explain_command text, out query_plan text) +RETURNS SETOF TEXT AS $$ +BEGIN + FOR query_plan IN execute explain_command LOOP + IF (query_plan LIKE '%-> Result%' OR query_plan = 'Result') THEN + CONTINUE; + END IF; + RETURN next; + END LOOP; + RETURN; +END; $$ language plpgsql; + -- helper function that returns true if output of given explain has "is not null" (case in-sensitive) CREATE OR REPLACE FUNCTION explain_has_is_not_null(explain_command text) RETURNS BOOLEAN AS $$ diff --git a/src/test/regress/sql/multi_view.sql b/src/test/regress/sql/multi_view.sql index 181fb4e70..193d669f6 100644 --- a/src/test/regress/sql/multi_view.sql +++ b/src/test/regress/sql/multi_view.sql @@ -374,6 +374,7 @@ VACUUM ANALYZE users_table; -- explain tests EXPLAIN (COSTS FALSE) SELECT user_id FROM recent_selected_users GROUP BY 1 ORDER BY 1; +SELECT public.coordinator_plan($Q$ EXPLAIN (COSTS FALSE) SELECT * FROM ( (SELECT user_id FROM recent_users) @@ -381,6 +382,7 @@ EXPLAIN (COSTS FALSE) SELECT * (SELECT user_id FROM selected_users) ) u WHERE user_id < 4 AND user_id > 1 ORDER BY user_id; +$Q$); EXPLAIN (COSTS FALSE) SELECT et.* FROM recent_10_users JOIN events_table et USING(user_id) ORDER BY et.time DESC LIMIT 10; SET citus.subquery_pushdown to ON; diff --git a/src/test/regress/sql/subquery_view.sql b/src/test/regress/sql/subquery_view.sql index 40798fccd..8f57ef5a3 100644 --- a/src/test/regress/sql/subquery_view.sql +++ b/src/test/regress/sql/subquery_view.sql @@ -427,17 +427,21 @@ SET client_min_messages TO DEFAULT; CREATE TABLE reference_table (text_col text, int_col int); SELECT create_reference_table('reference_table'); +SELECT public.coordinator_plan_with_subplans($Q$ EXPLAIN (COSTS OFF) WITH cte AS ( SELECT application_name AS text_col FROM pg_stat_activity ) SELECT * FROM reference_table JOIN cte USING (text_col); +$Q$); CREATE OR REPLACE VIEW view_on_views AS SELECT pg_stat_activity.application_name, pg_locks.pid FROM pg_stat_activity, pg_locks; +SELECT public.coordinator_plan_with_subplans($Q$ EXPLAIN (COSTS OFF) WITH cte AS ( SELECT application_name AS text_col FROM view_on_views ) SELECT * FROM reference_table JOIN cte USING (text_col); +$Q$); DROP SCHEMA subquery_view CASCADE; SET search_path TO public; diff --git a/src/test/regress/sql/window_functions.sql b/src/test/regress/sql/window_functions.sql index 5c94515a9..1488435ea 100644 --- a/src/test/regress/sql/window_functions.sql +++ b/src/test/regress/sql/window_functions.sql @@ -571,9 +571,11 @@ ORDER BY user_id, avg(value_1) DESC LIMIT 5; -- Grouping can be pushed down with aggregates even when window function can't +SELECT public.plan_without_result_lines($Q$ EXPLAIN (COSTS FALSE) SELECT user_id, count(value_1), stddev(value_1), count(user_id) OVER (PARTITION BY random()) FROM users_table GROUP BY user_id HAVING avg(value_1) > 2 LIMIT 1; +$Q$); -- Window function with inlined CTE WITH cte as (