diff --git a/src/test/regress/expected/cte_inline.out b/src/test/regress/expected/cte_inline.out index 4daaaa45c..98cd7343d 100644 --- a/src/test/regress/expected/cte_inline.out +++ b/src/test/regress/expected/cte_inline.out @@ -15,7 +15,7 @@ SHOW server_version \gset SELECT substring(:'server_version', '\d+')::int; substring --------------------------------------------------------------------- - 12 + 13 (1 row) SET client_min_messages TO DEBUG; @@ -747,8 +747,8 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: (SELECT intermediate DEBUG: Creating router plan key | value | other_value | ?column? --------------------------------------------------------------------- - 9 | test29 | {"f1": 29, "f2": 522, "f3": "test29"} | 1 9 | test9 | {"f1": 9, "f2": 162, "f3": "test9"} | 1 + 9 | test29 | {"f1": 29, "f2": 522, "f3": "test29"} | 1 9 | test19 | {"f1": 19, "f2": 342, "f3": "test19"} | 1 (3 rows) diff --git a/src/test/regress/expected/cte_inline_0.out b/src/test/regress/expected/cte_inline_0.out index 66efffa01..cd57658a7 100644 --- a/src/test/regress/expected/cte_inline_0.out +++ b/src/test/regress/expected/cte_inline_0.out @@ -15,7 +15,7 @@ SHOW server_version \gset SELECT substring(:'server_version', '\d+')::int; substring --------------------------------------------------------------------- - 11 + 12 (1 row) SET client_min_messages TO DEBUG; @@ -48,7 +48,19 @@ SELECT FROM cte_1 ORDER BY 2 DESC LIMIT 1; -ERROR: syntax error at or near "NOT" +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, other_value, (SELECT 1) FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 ORDER BY value DESC LIMIT 1 +DEBUG: Creating router plan +DEBUG: Plan is router executable + key | value | other_value | ?column? +--------------------------------------------------------------------- + 9 | test99 | {"f1": 99, "f2": 1782, "f3": "test99"} | 1 +(1 row) + -- the cte can be inlined because the unsupported -- part of the query (subquery in WHERE clause) -- doesn't access the cte @@ -292,7 +304,9 @@ FROM WHERE key = 1; DEBUG: CTE a is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Creating router plan +DEBUG: Plan is router executable +DETAIL: distribution column value: 1 count --------------------------------------------------------------------- 10 @@ -306,7 +320,15 @@ FROM a WHERE key = 1; -ERROR: syntax error at or near "NOT" +DEBUG: CTE a is going to be inlined via distributed planning +DEBUG: Creating router plan +DEBUG: Plan is router executable +DETAIL: distribution column value: 1 + count +--------------------------------------------------------------------- + 10 +(1 row) + -- using MATERIALIZED should cause inlining not to happen WITH a AS MATERIALIZED (SELECT * FROM test_table) SELECT @@ -315,7 +337,17 @@ FROM a WHERE key = 1; -ERROR: syntax error at or near "MATERIALIZED" +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE a: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) a WHERE (key OPERATOR(pg_catalog.=) 1) +DEBUG: Creating router plan +DEBUG: Plan is router executable + count +--------------------------------------------------------------------- + 10 +(1 row) + -- EXPLAIN should show the difference between materialized an not materialized EXPLAIN (COSTS OFF) WITH a AS (SELECT * FROM test_table) SELECT @@ -325,19 +357,20 @@ FROM WHERE key = 1; DEBUG: CTE a is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries - QUERY PLAN +DEBUG: Creating router plan +DEBUG: Plan is router executable +DETAIL: distribution column value: 1 + QUERY PLAN --------------------------------------------------------------------- - Aggregate - -> Custom Scan (Citus Adaptive) - Task Count: 4 - Tasks Shown: One of 4 - -> Task - Node: host=localhost port=xxxxx dbname=regression - -> Aggregate - -> Seq Scan on test_table_1960000 test_table - Filter: (key = 1) -(9 rows) + Custom Scan (Citus Adaptive) + Task Count: 1 + Tasks Shown: All + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Aggregate + -> Seq Scan on test_table_1960000 test_table + Filter: (key = 1) +(8 rows) EXPLAIN (COSTS OFF) WITH a AS MATERIALIZED (SELECT * FROM test_table) SELECT @@ -346,7 +379,31 @@ FROM a WHERE key = 1; -ERROR: syntax error at or near "MATERIALIZED" +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE a: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) a WHERE (key OPERATOR(pg_catalog.=) 1) +DEBUG: Creating router plan +DEBUG: Plan is router executable + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (Citus Adaptive) + -> Distributed Subplan XXX_1 + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on test_table_1960000 test_table + Task Count: 1 + Tasks Shown: All + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Aggregate + -> Function Scan on read_intermediate_result intermediate_result + Filter: (key = 1) +(15 rows) + -- citus should not inline the CTE because it is used multiple times WITH cte_1 AS (SELECT * FROM test_table) SELECT @@ -375,7 +432,25 @@ FROM JOIN cte_1 as second_entry USING (key); -ERROR: syntax error at or near "NOT" +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: join prunable for intervals [-2147483648,-1073741825] and [-1073741824,-1] +DEBUG: join prunable for intervals [-2147483648,-1073741825] and [0,1073741823] +DEBUG: join prunable for intervals [-2147483648,-1073741825] and [1073741824,2147483647] +DEBUG: join prunable for intervals [-1073741824,-1] and [-2147483648,-1073741825] +DEBUG: join prunable for intervals [-1073741824,-1] and [0,1073741823] +DEBUG: join prunable for intervals [-1073741824,-1] and [1073741824,2147483647] +DEBUG: join prunable for intervals [0,1073741823] and [-2147483648,-1073741825] +DEBUG: join prunable for intervals [0,1073741823] and [-1073741824,-1] +DEBUG: join prunable for intervals [0,1073741823] and [1073741824,2147483647] +DEBUG: join prunable for intervals [1073741824,2147483647] and [-2147483648,-1073741825] +DEBUG: join prunable for intervals [1073741824,2147483647] and [-1073741824,-1] +DEBUG: join prunable for intervals [1073741824,2147483647] and [0,1073741823] + count +--------------------------------------------------------------------- + 1021 +(1 row) + -- EXPLAIN should show the differences between MATERIALIZED and NOT MATERIALIZED EXPLAIN (COSTS OFF) WITH cte_1 AS (SELECT * FROM test_table) SELECT @@ -423,7 +498,36 @@ FROM JOIN cte_1 as second_entry USING (key); -ERROR: syntax error at or near "NOT" +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: join prunable for intervals [-2147483648,-1073741825] and [-1073741824,-1] +DEBUG: join prunable for intervals [-2147483648,-1073741825] and [0,1073741823] +DEBUG: join prunable for intervals [-2147483648,-1073741825] and [1073741824,2147483647] +DEBUG: join prunable for intervals [-1073741824,-1] and [-2147483648,-1073741825] +DEBUG: join prunable for intervals [-1073741824,-1] and [0,1073741823] +DEBUG: join prunable for intervals [-1073741824,-1] and [1073741824,2147483647] +DEBUG: join prunable for intervals [0,1073741823] and [-2147483648,-1073741825] +DEBUG: join prunable for intervals [0,1073741823] and [-1073741824,-1] +DEBUG: join prunable for intervals [0,1073741823] and [1073741824,2147483647] +DEBUG: join prunable for intervals [1073741824,2147483647] and [-2147483648,-1073741825] +DEBUG: join prunable for intervals [1073741824,2147483647] and [-1073741824,-1] +DEBUG: join prunable for intervals [1073741824,2147483647] and [0,1073741823] + QUERY PLAN +--------------------------------------------------------------------- + Aggregate + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Aggregate + -> Hash Join + Hash Cond: (test_table.key = test_table_1.key) + -> Seq Scan on test_table_1960000 test_table + -> Hash + -> Seq Scan on test_table_1960000 test_table_1 +(12 rows) + -- ctes with volatile functions are not -- inlined WITH cte_1 AS (SELECT *, random() FROM test_table) @@ -448,7 +552,17 @@ SELECT count(*) FROM cte_1; -ERROR: syntax error at or near "NOT" +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value, random() AS random FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value, intermediate_result.random FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb, random double precision)) cte_1 +DEBUG: Creating router plan +DEBUG: Plan is router executable + count +--------------------------------------------------------------------- + 101 +(1 row) + -- cte_1 should be able to inlined even if -- it is used one level below WITH cte_1 AS (SELECT * FROM test_table) @@ -504,11 +618,11 @@ SELECT count(*) LEFT JOIN test_table u2 ON u2.key = bar.key) AS foo ON TRUE; DEBUG: CTE bar is going to be inlined via distributed planning DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: skipping recursive planning for the subquery since it contains references to outer queries -DEBUG: skipping recursive planning for the subquery since it contains references to outer queries -DEBUG: skipping recursive planning for the subquery since it contains references to outer queries -DEBUG: Router planner cannot handle multi-shard select queries -ERROR: CTEs that refer to other subqueries are not supported in multi-shard queries + count +--------------------------------------------------------------------- + 10331 +(1 row) + -- inlined CTE contains a reference to outer query -- should be fine (even if the recursive planning fails -- to recursively plan the query) @@ -581,7 +695,9 @@ DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: CTE cte_1 is going to be inlined via distributed planning -DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Creating router plan +DEBUG: Plan is router executable +DETAIL: distribution column value: 1 count | key --------------------------------------------------------------------- (0 rows) @@ -695,17 +811,11 @@ FROM cte LEFT JOIN test_table USING (key); DEBUG: CTE cte is going to be inlined via distributed planning DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte LEFT JOIN cte_inline.test_table USING (key)) -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE cte: SELECT key, value, other_value FROM cte_inline.test_table -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte LEFT JOIN cte_inline.test_table USING (key)) -DEBUG: Router planner cannot handle multi-shard select queries -ERROR: cannot pushdown the subquery -DETAIL: Complex subqueries and CTEs cannot be in the outer part of the outer join + count +--------------------------------------------------------------------- + 1021 +(1 row) + -- the CTEs are very simple, so postgres -- can pull-up the subqueries after inlining -- the CTEs, and the query that we send to workers @@ -719,10 +829,18 @@ FROM DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: CTE cte_2 is going to be inlined via distributed planning DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for subquery SELECT key FROM cte_inline.test_table -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT test_table.key FROM cte_inline.test_table) cte_1 JOIN (SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) cte_2 USING (key)) -DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: join prunable for intervals [-2147483648,-1073741825] and [-1073741824,-1] +DEBUG: join prunable for intervals [-2147483648,-1073741825] and [0,1073741823] +DEBUG: join prunable for intervals [-2147483648,-1073741825] and [1073741824,2147483647] +DEBUG: join prunable for intervals [-1073741824,-1] and [-2147483648,-1073741825] +DEBUG: join prunable for intervals [-1073741824,-1] and [0,1073741823] +DEBUG: join prunable for intervals [-1073741824,-1] and [1073741824,2147483647] +DEBUG: join prunable for intervals [0,1073741823] and [-2147483648,-1073741825] +DEBUG: join prunable for intervals [0,1073741823] and [-1073741824,-1] +DEBUG: join prunable for intervals [0,1073741823] and [1073741824,2147483647] +DEBUG: join prunable for intervals [1073741824,2147483647] and [-2147483648,-1073741825] +DEBUG: join prunable for intervals [1073741824,2147483647] and [-1073741824,-1] +DEBUG: join prunable for intervals [1073741824,2147483647] and [0,1073741823] count --------------------------------------------------------------------- 1021 @@ -778,7 +896,12 @@ DEBUG: Creating router plan -- we de still don't support it WITH cte_1 AS NOT MATERIALIZED (SELECT * FROM test_table) DELETE FROM test_table WHERE key NOT IN (SELECT key FROM cte_1); -ERROR: syntax error at or near "NOT" +DEBUG: complex joins are only supported when all distributed tables are co-located and joined on their distribution columns +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: DELETE FROM cte_inline.test_table WHERE (NOT (key OPERATOR(pg_catalog.=) ANY (SELECT cte_1.key FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1))) +DEBUG: Creating router plan +DEBUG: Plan is router executable -- we don't inline CTEs if they are modifying CTEs WITH cte_1 AS (DELETE FROM test_table WHERE key % 3 = 1 RETURNING key) SELECT * FROM cte_1 ORDER BY 1 DESC LIMIT 3; @@ -797,7 +920,18 @@ DEBUG: Creating router plan -- NOT MATERIALIZED should not affect modifying CTEs WITH cte_1 AS NOT MATERIALIZED (DELETE FROM test_table WHERE key % 3 = 0 RETURNING key) SELECT count(*) FROM cte_1; -ERROR: syntax error at or near "NOT" +DEBUG: data-modifying statements are not supported in the WITH clauses of distributed queries +DEBUG: generating subplan XXX_1 for CTE cte_1: DELETE FROM cte_inline.test_table WHERE ((key OPERATOR(pg_catalog.%) 3) OPERATOR(pg_catalog.=) 0) RETURNING key +DEBUG: Creating router plan +DEBUG: Plan is router executable +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) cte_1 +DEBUG: Creating router plan +DEBUG: Plan is router executable + count +--------------------------------------------------------------------- + 164 +(1 row) + -- cte with column aliases SELECT * FROM test_table, (WITH cte_1 (x,y) AS (SELECT * FROM test_table), @@ -813,13 +947,13 @@ DEBUG: Router planner cannot handle multi-shard select queries DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT test_table.key, test_table.value, test_table.other_value, bar.z, bar.y, bar.key, bar.t, bar.m, bar.cte_2_key FROM cte_inline.test_table, (SELECT cte_2.z, cte_2.y, cte_2.key, cte_3.t, cte_3.m, cte_3.cte_2_key FROM (SELECT intermediate_result.value AS z, intermediate_result.other_value AS y, intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text, other_value jsonb, key integer)) cte_2, (SELECT cte_2_1.z AS t, cte_2_1.y AS m, cte_2_1.key AS cte_2_key FROM (SELECT intermediate_result.value AS z, intermediate_result.other_value AS y, intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text, other_value jsonb, key integer)) cte_2_1) cte_3) bar ORDER BY test_table.value, test_table.other_value, bar.z, bar.y, bar.t, bar.m, bar.cte_2_key LIMIT 5 DEBUG: Router planner cannot handle multi-shard select queries DEBUG: push down of limit count: 5 - key | value | other_value | z | y | key | t | m | cte_2_key + key | value | other_value | z | y | key | t | m | cte_2_key --------------------------------------------------------------------- - 0 | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | 0 | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | 0 - 0 | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | 0 | test0 | | 0 - 0 | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | 0 | test0 | | 0 - 0 | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | 0 | test0 | | 0 - 0 | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | 0 | test10 | {"f1": 10, "f2": 180, "f3": "test10"} | 0 + 2 | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | 2 | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | 2 + 2 | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | 2 | test12 | | 2 + 2 | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | 2 | test12 | | 2 + 2 | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | 2 | test12 | | 2 + 2 | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | test12 | {"f1": 12, "f2": 216, "f3": "test12"} | 2 | test15 | {"f1": 15, "f2": 270, "f3": "test15"} | 5 (5 rows) -- cte used in HAVING subquery just works fine @@ -843,12 +977,10 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, count(*) DEBUG: Router planner cannot handle multi-shard select queries key | count --------------------------------------------------------------------- - 0 | 44 - 9 | 40 8 | 40 - 6 | 40 5 | 40 -(5 rows) + 2 | 40 +(3 rows) -- cte used in ORDER BY just works fine -- even if it is inlined @@ -869,9 +1001,9 @@ DEBUG: Router planner cannot handle multi-shard select queries DEBUG: push down of limit count: 3 key --------------------------------------------------------------------- - 9 - 9 - 9 + 8 + 8 + 8 (3 rows) PREPARE inlined_cte_without_params AS @@ -904,7 +1036,7 @@ DEBUG: Router planner cannot handle multi-shard select queries DEBUG: push down of limit count: 3 count --------------------------------------------------------------------- - 44 + 40 40 40 (3 rows) @@ -912,7 +1044,7 @@ DEBUG: push down of limit count: 3 EXECUTE inlined_cte_without_params; count --------------------------------------------------------------------- - 44 + 40 40 40 (3 rows) @@ -920,7 +1052,7 @@ EXECUTE inlined_cte_without_params; EXECUTE inlined_cte_without_params; count --------------------------------------------------------------------- - 44 + 40 40 40 (3 rows) @@ -928,7 +1060,7 @@ EXECUTE inlined_cte_without_params; EXECUTE inlined_cte_without_params; count --------------------------------------------------------------------- - 44 + 40 40 40 (3 rows) @@ -936,7 +1068,7 @@ EXECUTE inlined_cte_without_params; EXECUTE inlined_cte_without_params; count --------------------------------------------------------------------- - 44 + 40 40 40 (3 rows) @@ -944,7 +1076,7 @@ EXECUTE inlined_cte_without_params; EXECUTE inlined_cte_without_params; count --------------------------------------------------------------------- - 44 + 40 40 40 (3 rows) @@ -959,49 +1091,49 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, o DEBUG: Creating router plan key | value | other_value | ?column? --------------------------------------------------------------------- - 9 | test99 | | 1 - 9 | test99 | | 1 - 9 | test99 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 (3 rows) EXECUTE non_inlined_cte_without_params; key | value | other_value | ?column? --------------------------------------------------------------------- - 9 | test99 | | 1 - 9 | test99 | | 1 - 9 | test99 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 (3 rows) EXECUTE non_inlined_cte_without_params; key | value | other_value | ?column? --------------------------------------------------------------------- - 9 | test99 | | 1 - 9 | test99 | | 1 - 9 | test99 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 (3 rows) EXECUTE non_inlined_cte_without_params; key | value | other_value | ?column? --------------------------------------------------------------------- - 9 | test99 | | 1 - 9 | test99 | | 1 - 9 | test99 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 (3 rows) EXECUTE non_inlined_cte_without_params; key | value | other_value | ?column? --------------------------------------------------------------------- - 9 | test99 | | 1 - 9 | test99 | | 1 - 9 | test99 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 (3 rows) EXECUTE non_inlined_cte_without_params; key | value | other_value | ?column? --------------------------------------------------------------------- - 9 | test99 | | 1 - 9 | test99 | | 1 - 9 | test99 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 (3 rows) EXECUTE inlined_cte_has_parameter_on_non_dist_key('test1'); @@ -1027,8 +1159,7 @@ DEBUG: Router planner cannot handle multi-shard select queries DEBUG: push down of limit count: 3 count --------------------------------------------------------------------- - 4 -(1 row) +(0 rows) EXECUTE inlined_cte_has_parameter_on_non_dist_key('test4'); DEBUG: CTE cte_1 is going to be inlined via distributed planning @@ -1055,8 +1186,7 @@ DEBUG: Router planner cannot handle multi-shard select queries DEBUG: push down of limit count: 3 count --------------------------------------------------------------------- - 4 -(1 row) +(0 rows) EXECUTE inlined_cte_has_parameter_on_dist_key(1); DEBUG: CTE cte_1 is going to be inlined via distributed planning @@ -1077,8 +1207,7 @@ DEBUG: push down of limit count: 3 --------------------------------------------------------------------- 40 40 - 40 -(3 rows) +(2 rows) EXECUTE inlined_cte_has_parameter_on_dist_key(3); DEBUG: CTE cte_1 is going to be inlined via distributed planning @@ -1088,8 +1217,7 @@ DEBUG: push down of limit count: 3 --------------------------------------------------------------------- 40 40 - 40 -(3 rows) +(2 rows) EXECUTE inlined_cte_has_parameter_on_dist_key(4); DEBUG: CTE cte_1 is going to be inlined via distributed planning @@ -1099,8 +1227,7 @@ DEBUG: push down of limit count: 3 --------------------------------------------------------------------- 40 40 - 40 -(3 rows) +(2 rows) EXECUTE inlined_cte_has_parameter_on_dist_key(5); DEBUG: CTE cte_1 is going to be inlined via distributed planning @@ -1109,9 +1236,7 @@ DEBUG: push down of limit count: 3 count --------------------------------------------------------------------- 40 - 40 - 40 -(3 rows) +(1 row) EXECUTE inlined_cte_has_parameter_on_dist_key(6); DEBUG: CTE cte_1 is going to be inlined via distributed planning @@ -1122,8 +1247,7 @@ DEBUG: push down of limit count: 3 count --------------------------------------------------------------------- 40 - 40 -(2 rows) +(1 row) EXECUTE non_inlined_cte_has_parameter_on_dist_key(1); DEBUG: CTE cte_1 is going to be inlined via distributed planning @@ -1135,9 +1259,9 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, o DEBUG: Creating router plan key | value | other_value | ?column? --------------------------------------------------------------------- - 9 | test99 | | 1 - 9 | test99 | | 1 - 9 | test99 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 (3 rows) EXECUTE non_inlined_cte_has_parameter_on_dist_key(2); @@ -1150,9 +1274,9 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, o DEBUG: Creating router plan key | value | other_value | ?column? --------------------------------------------------------------------- - 9 | test99 | | 1 - 9 | test99 | | 1 - 9 | test99 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 (3 rows) EXECUTE non_inlined_cte_has_parameter_on_dist_key(3); @@ -1165,9 +1289,9 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, o DEBUG: Creating router plan key | value | other_value | ?column? --------------------------------------------------------------------- - 9 | test99 | | 1 - 9 | test99 | | 1 - 9 | test99 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 (3 rows) EXECUTE non_inlined_cte_has_parameter_on_dist_key(4); @@ -1180,9 +1304,9 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, o DEBUG: Creating router plan key | value | other_value | ?column? --------------------------------------------------------------------- - 9 | test99 | | 1 - 9 | test99 | | 1 - 9 | test99 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 (3 rows) EXECUTE non_inlined_cte_has_parameter_on_dist_key(5); @@ -1195,9 +1319,9 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, o DEBUG: Creating router plan key | value | other_value | ?column? --------------------------------------------------------------------- - 9 | test99 | | 1 - 9 | test99 | | 1 - 9 | test99 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 (3 rows) EXECUTE non_inlined_cte_has_parameter_on_dist_key(6); @@ -1212,49 +1336,49 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, o DEBUG: Creating router plan key | value | other_value | ?column? --------------------------------------------------------------------- - 9 | test99 | | 1 - 9 | test99 | | 1 - 9 | test99 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 + 8 | test98 | | 1 (3 rows) EXECUTE retry_planning(1); DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: Router planner cannot handle multi-shard select queries - json_object_agg + json_object_agg --------------------------------------------------------------------- - { "2" : "test12", "2" : "test2", "2" : "test22", "2" : "test32", "2" : "test42", "2" : "test52", "2" : "test62", "2" : "test72", "2" : "test82", "2" : "test92", "3" : "test13", "3" : "test23", "3" : "test3", "3" : "test33", "3" : "test43", "3" : "test53", "3" : "test63", "3" : "test73", "3" : "test83", "3" : "test93", "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "6" : "test16", "6" : "test26", "6" : "test36", "6" : "test46", "6" : "test56", "6" : "test6", "6" : "test66", "6" : "test76", "6" : "test86", "6" : "test96", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } + { "2" : "test12", "2" : "test2", "2" : "test22", "2" : "test32", "2" : "test42", "2" : "test52", "2" : "test62", "2" : "test72", "2" : "test82", "2" : "test92", "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98" } (1 row) EXECUTE retry_planning(2); DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: Router planner cannot handle multi-shard select queries - json_object_agg + json_object_agg --------------------------------------------------------------------- - { "3" : "test13", "3" : "test23", "3" : "test3", "3" : "test33", "3" : "test43", "3" : "test53", "3" : "test63", "3" : "test73", "3" : "test83", "3" : "test93", "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "6" : "test16", "6" : "test26", "6" : "test36", "6" : "test46", "6" : "test56", "6" : "test6", "6" : "test66", "6" : "test76", "6" : "test86", "6" : "test96", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } + { "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98" } (1 row) EXECUTE retry_planning(3); DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: Router planner cannot handle multi-shard select queries - json_object_agg + json_object_agg --------------------------------------------------------------------- - { "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "6" : "test16", "6" : "test26", "6" : "test36", "6" : "test46", "6" : "test56", "6" : "test6", "6" : "test66", "6" : "test76", "6" : "test86", "6" : "test96", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } + { "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98" } (1 row) EXECUTE retry_planning(4); DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: Router planner cannot handle multi-shard select queries - json_object_agg + json_object_agg --------------------------------------------------------------------- - { "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "6" : "test16", "6" : "test26", "6" : "test36", "6" : "test46", "6" : "test56", "6" : "test6", "6" : "test66", "6" : "test76", "6" : "test86", "6" : "test96", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } + { "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98" } (1 row) EXECUTE retry_planning(5); DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: Router planner cannot handle multi-shard select queries - json_object_agg + json_object_agg --------------------------------------------------------------------- - { "6" : "test16", "6" : "test26", "6" : "test36", "6" : "test46", "6" : "test56", "6" : "test6", "6" : "test66", "6" : "test76", "6" : "test86", "6" : "test96", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } + { "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98" } (1 row) EXECUTE retry_planning(6); @@ -1262,9 +1386,9 @@ DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: Router planner cannot handle multi-shard select queries DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: Router planner cannot handle multi-shard select queries - json_object_agg + json_object_agg --------------------------------------------------------------------- - { "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } + { "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98" } (1 row) -- this test can only work if the CTE is recursively @@ -1286,7 +1410,7 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS c DEBUG: Creating router plan count --------------------------------------------------------------------- - 11536 + 4800 (1 row) -- this becomes a non-colocated subquery join @@ -1304,7 +1428,7 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS c DEBUG: Router planner cannot handle multi-shard select queries count --------------------------------------------------------------------- - 1136 + 480 (1 row) -- cte a has to be recursively planned because of OFFSET 0 @@ -1328,7 +1452,7 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT min(a.key) AS DEBUG: Creating router plan min --------------------------------------------------------------------- - 0 + 2 (1 row) -- after both CTEs are inlined, this becomes non-colocated subquery join @@ -1343,11 +1467,11 @@ DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FRO DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT cte_1.key, cte_1.value, cte_1.other_value, cte_2.key, cte_2.value, cte_2.other_value FROM ((SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) cte_1 JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_2 ON ((cte_1.value OPERATOR(pg_catalog.>) cte_2.value))) ORDER BY cte_1.key, cte_1.value, cte_1.other_value, cte_2.key, cte_2.value, cte_2.other_value DESC LIMIT 3 DEBUG: Router planner cannot handle multi-shard select queries DEBUG: push down of limit count: 3 - key | value | other_value | key | value | other_value + key | value | other_value | key | value | other_value --------------------------------------------------------------------- - 0 | test10 | {"f1": 10, "f2": 180, "f3": "test10"} | 0 | test0 | - 0 | test10 | {"f1": 10, "f2": 180, "f3": "test10"} | 0 | test0 | - 0 | test10 | {"f1": 10, "f2": 180, "f3": "test10"} | 0 | test0 | + 2 | test2 | {"f1": 2, "f2": 36, "f3": "test2"} | 2 | test12 | + 2 | test2 | {"f1": 2, "f2": 36, "f3": "test2"} | 2 | test12 | + 2 | test2 | {"f1": 2, "f2": 36, "f3": "test2"} | 2 | test12 | (3 rows) -- full join is only supported when both sides are @@ -1371,9 +1495,9 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT value FROM (( DEBUG: Creating router plan value --------------------------------------------------------------------- - test99 - test99 - test99 + test98 + test98 + test98 (3 rows) -- an unsupported agg. for multi-shard queries @@ -1382,9 +1506,9 @@ WITH cte_1 AS (SELECT * FROM test_table WHERE key > 1) SELECT json_object_agg(DISTINCT key, value) FROM cte_1; DEBUG: CTE cte_1 is going to be inlined via distributed planning DEBUG: Router planner cannot handle multi-shard select queries - json_object_agg + json_object_agg --------------------------------------------------------------------- - { "2" : "test12", "2" : "test2", "2" : "test22", "2" : "test32", "2" : "test42", "2" : "test52", "2" : "test62", "2" : "test72", "2" : "test82", "2" : "test92", "3" : "test13", "3" : "test23", "3" : "test3", "3" : "test33", "3" : "test43", "3" : "test53", "3" : "test63", "3" : "test73", "3" : "test83", "3" : "test93", "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "6" : "test16", "6" : "test26", "6" : "test36", "6" : "test46", "6" : "test56", "6" : "test6", "6" : "test66", "6" : "test76", "6" : "test86", "6" : "test96", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } + { "2" : "test12", "2" : "test2", "2" : "test22", "2" : "test32", "2" : "test42", "2" : "test52", "2" : "test62", "2" : "test72", "2" : "test82", "2" : "test92", "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98" } (1 row) -- both cte_1 and cte_2 are going to be inlined. diff --git a/src/test/regress/expected/cte_inline_1.out b/src/test/regress/expected/cte_inline_1.out new file mode 100644 index 000000000..f5d011bd0 --- /dev/null +++ b/src/test/regress/expected/cte_inline_1.out @@ -0,0 +1,1443 @@ +CREATE SCHEMA cte_inline; +SET search_path TO cte_inline; +SET citus.next_shard_id TO 1960000; +CREATE TABLE test_table (key int, value text, other_value jsonb); +SELECT create_distributed_table ('test_table', 'key'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO test_table SELECT i % 10, 'test' || i, row_to_json(row(i, i*18, 'test' || i)) FROM generate_series (0, 100) i; +-- server version because CTE inlining might produce +-- different debug messages in PG 11 vs PG 12 +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int; + substring +--------------------------------------------------------------------- + 11 +(1 row) + +SET client_min_messages TO DEBUG; +-- Citus should not inline this CTE because otherwise it cannot +-- plan the query +WITH cte_1 AS (SELECT * FROM test_table) +SELECT + *, (SELECT 1) +FROM + cte_1 +ORDER BY 1 DESC LIMIT 3; +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, other_value, (SELECT 1) FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 ORDER BY key DESC LIMIT 3 +DEBUG: Creating router plan +DEBUG: Plan is router executable + key | value | other_value | ?column? +--------------------------------------------------------------------- + 9 | test9 | {"f1": 9, "f2": 162, "f3": "test9"} | 1 + 9 | test19 | {"f1": 19, "f2": 342, "f3": "test19"} | 1 + 9 | test29 | {"f1": 29, "f2": 522, "f3": "test29"} | 1 +(3 rows) + +-- Should still not be inlined even if NOT MATERIALIZED is passed +WITH cte_1 AS NOT MATERIALIZED (SELECT * FROM test_table) +SELECT + *, (SELECT 1) +FROM + cte_1 +ORDER BY 2 DESC LIMIT 1; +ERROR: syntax error at or near "NOT" +-- the cte can be inlined because the unsupported +-- part of the query (subquery in WHERE clause) +-- doesn't access the cte +WITH cte_1 AS (SELECT * FROM test_table) +SELECT + count(*) +FROM + cte_1 +WHERE + key IN ( + SELECT + (SELECT 1) + FROM + test_table WHERE key = 1 + ); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Creating router plan +DEBUG: Plan is router executable +DETAIL: distribution column value: 1 +DEBUG: generating subplan XXX_1 for subquery SELECT (SELECT 1) FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.=) 1) +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) cte_1 WHERE (key OPERATOR(pg_catalog.=) ANY (SELECT intermediate_result."?column?" FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result("?column?" integer))) +DEBUG: Router planner cannot handle multi-shard select queries + count +--------------------------------------------------------------------- + 10 +(1 row) + +-- a similar query as the above, and this time the planning +-- fails, but it fails because the subquery in WHERE clause +-- cannot be planned by Citus +WITH cte_1 AS (SELECT * FROM test_table) +SELECT + count(*) +FROM + cte_1 +WHERE + key IN ( + SELECT + key + FROM + test_table + FOR UPDATE + ); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: SELECT FOR UPDATE with table replication factor > 1 not supported for non-reference tables. +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: SELECT FOR UPDATE with table replication factor > 1 not supported for non-reference tables. +ERROR: could not run distributed query with FOR UPDATE/SHARE commands +HINT: Consider using an equality filter on the distributed table's partition column. +-- Citus does the inlining, the planning fails +-- and retries without inlining, which works +-- fine later via recursive planning +WITH cte_1 AS + (SELECT * + FROM test_table) +SELECT *, (SELECT 1) +FROM + (SELECT * + FROM cte_1) AS foo +ORDER BY 2 DESC LIMIT 1; +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, other_value, (SELECT 1) FROM (SELECT cte_1.key, cte_1.value, cte_1.other_value FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1) foo ORDER BY value DESC LIMIT 1 +DEBUG: Creating router plan +DEBUG: Plan is router executable + key | value | other_value | ?column? +--------------------------------------------------------------------- + 9 | test99 | {"f1": 99, "f2": 1782, "f3": "test99"} | 1 +(1 row) + +-- a little more complicated query tree +-- Citus does the inlining, the planning fails +-- and retries without inlining, which works +WITH top_cte AS + (SELECT * + FROM test_table) +SELECT count(*) +FROM top_cte, + (WITH cte_1 AS + (SELECT * + FROM test_table) SELECT *, (SELECT 1) + FROM + (SELECT * + FROM cte_1) AS foo) AS bar; +DEBUG: CTE top_cte is going to be inlined via distributed planning +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE top_cte: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_2 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Creating router plan +DEBUG: Plan is router executable +DEBUG: generating subplan XXX_3 for subquery SELECT key, value, other_value, (SELECT 1) FROM (SELECT cte_1.key, cte_1.value, cte_1.other_value FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1) foo +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) top_cte, (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value, intermediate_result."?column?" FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb, "?column?" integer)) bar +DEBUG: Creating router plan +DEBUG: Plan is router executable + count +--------------------------------------------------------------------- + 10201 +(1 row) + +-- CTE is used inside a subquery in WHERE clause +-- the query wouldn't work by inlining, so Citus +-- retries again via recursive planning, which +-- works fine +WITH cte_1 AS + (SELECT * + FROM test_table) +SELECT count(*) +FROM test_table +WHERE KEY IN + (SELECT (SELECT 1) + FROM + (SELECT *, + random() + FROM + (SELECT * + FROM cte_1) AS foo) AS bar); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Creating router plan +DEBUG: Plan is router executable +DEBUG: generating subplan XXX_2 for subquery SELECT (SELECT 1) FROM (SELECT foo.key, foo.value, foo.other_value, random() AS random FROM (SELECT cte_1.key, cte_1.value, cte_1.other_value FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1) foo) bar +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.=) ANY (SELECT intermediate_result."?column?" FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result("?column?" integer))) +DEBUG: Router planner cannot handle multi-shard select queries + count +--------------------------------------------------------------------- + 10 +(1 row) + +-- cte_1 is used inside another CTE, but still +-- doesn't work when inlined because it is finally +-- used in an unsupported query +-- but still works fine because recursive planning +-- kicks in +WITH cte_1 AS + (SELECT * + FROM test_table) +SELECT (SELECT 1) AS KEY FROM ( + WITH cte_2 AS (SELECT *, random() + FROM (SELECT *,random() FROM cte_1) as foo) +SELECT *, random() FROM cte_2) as bar ORDER BY 1 DESC LIMIT 3; +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_2: SELECT key, value, other_value, random, random() AS random FROM (SELECT cte_1.key, cte_1.value, cte_1.other_value, random() AS random FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) cte_1) foo +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT (SELECT 1) AS key FROM (SELECT cte_2.key, cte_2.value, cte_2.other_value, cte_2.random, cte_2.random_1 AS random, random() AS random FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value, intermediate_result.random, intermediate_result.random_1 AS random FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb, random double precision, random_1 double precision)) cte_2(key, value, other_value, random, random_1)) bar(key, value, other_value, random, random_1, random_2) ORDER BY (SELECT 1) DESC LIMIT 3 +DEBUG: Creating router plan +DEBUG: Plan is router executable + key +--------------------------------------------------------------------- + 1 + 1 + 1 +(3 rows) + +-- in this example, cte_2 can be inlined, because it is not used +-- on any query that Citus cannot plan. However, cte_1 should not be +-- inlined, because it is used with a subquery in target list +WITH cte_1 AS (SELECT * FROM test_table), + cte_2 AS (select * from test_table) +SELECT + count(*) +FROM + (SELECT *, (SELECT 1) FROM cte_1) as foo + JOIN + cte_2 + ON (true); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: CTE cte_2 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT cte_1.key, cte_1.value, cte_1.other_value, (SELECT 1) FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) cte_1) foo JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_2 ON (true)) +DEBUG: Router planner cannot handle multi-shard select queries + count +--------------------------------------------------------------------- + 10201 +(1 row) + +-- unreferenced CTEs are just ignored +-- by Citus/Postgres +WITH a AS (SELECT * FROM test_table) +SELECT + *, row_number() OVER () +FROM + test_table +WHERE + key = 1 +ORDER BY 3 DESC +LIMIT 5; +DEBUG: Creating router plan +DEBUG: Plan is router executable +DETAIL: distribution column value: 1 + key | value | other_value | row_number +--------------------------------------------------------------------- + 1 | test91 | {"f1": 91, "f2": 1638, "f3": "test91"} | 10 + 1 | test81 | {"f1": 81, "f2": 1458, "f3": "test81"} | 9 + 1 | test71 | {"f1": 71, "f2": 1278, "f3": "test71"} | 8 + 1 | test61 | {"f1": 61, "f2": 1098, "f3": "test61"} | 7 + 1 | test51 | {"f1": 51, "f2": 918, "f3": "test51"} | 6 +(5 rows) + +-- router queries are affected by the distributed +-- cte inlining +WITH a AS (SELECT * FROM test_table WHERE key = 1) +SELECT + *, (SELECT 1) +FROM + a +WHERE + key = 1 +ORDER BY 1 DESC +LIMIT 5; +DEBUG: CTE a is going to be inlined via distributed planning +DEBUG: Creating router plan +DEBUG: Plan is router executable +DETAIL: distribution column value: 1 + key | value | other_value | ?column? +--------------------------------------------------------------------- + 1 | test1 | {"f1": 1, "f2": 18, "f3": "test1"} | 1 + 1 | test11 | {"f1": 11, "f2": 198, "f3": "test11"} | 1 + 1 | test21 | {"f1": 21, "f2": 378, "f3": "test21"} | 1 + 1 | test31 | {"f1": 31, "f2": 558, "f3": "test31"} | 1 + 1 | test41 | {"f1": 41, "f2": 738, "f3": "test41"} | 1 +(5 rows) + +-- non router queries are affected by the distributed +-- cte inlining as well +WITH a AS (SELECT * FROM test_table) +SELECT + count(*) +FROM + a +WHERE + key = 1; +DEBUG: CTE a is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries + count +--------------------------------------------------------------------- + 10 +(1 row) + +-- explicitely using NOT MATERIALIZED should result in the same +WITH a AS NOT MATERIALIZED (SELECT * FROM test_table) +SELECT + count(*) +FROM + a +WHERE + key = 1; +ERROR: syntax error at or near "NOT" +-- using MATERIALIZED should cause inlining not to happen +WITH a AS MATERIALIZED (SELECT * FROM test_table) +SELECT + count(*) +FROM + a +WHERE + key = 1; +ERROR: syntax error at or near "MATERIALIZED" +-- EXPLAIN should show the difference between materialized an not materialized +EXPLAIN (COSTS OFF) WITH a AS (SELECT * FROM test_table) +SELECT + count(*) +FROM + a +WHERE + key = 1; +DEBUG: CTE a is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries + QUERY PLAN +--------------------------------------------------------------------- + Aggregate + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Aggregate + -> Seq Scan on test_table_1960000 test_table + Filter: (key = 1) +(9 rows) + +EXPLAIN (COSTS OFF) WITH a AS MATERIALIZED (SELECT * FROM test_table) +SELECT + count(*) +FROM + a +WHERE + key = 1; +ERROR: syntax error at or near "MATERIALIZED" +-- citus should not inline the CTE because it is used multiple times +WITH cte_1 AS (SELECT * FROM test_table) +SELECT + count(*) +FROM + cte_1 as first_entry + JOIN + cte_1 as second_entry + USING (key); +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) first_entry JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) second_entry USING (key)) +DEBUG: Creating router plan +DEBUG: Plan is router executable + count +--------------------------------------------------------------------- + 1021 +(1 row) + +-- NOT MATERIALIZED should cause the query to be inlined twice +WITH cte_1 AS NOT MATERIALIZED (SELECT * FROM test_table) +SELECT + count(*) +FROM + cte_1 as first_entry + JOIN + cte_1 as second_entry + USING (key); +ERROR: syntax error at or near "NOT" +-- EXPLAIN should show the differences between MATERIALIZED and NOT MATERIALIZED +EXPLAIN (COSTS OFF) WITH cte_1 AS (SELECT * FROM test_table) +SELECT + count(*) +FROM + cte_1 as first_entry + JOIN + cte_1 as second_entry + USING (key); +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) first_entry JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) second_entry USING (key)) +DEBUG: Creating router plan +DEBUG: Plan is router executable + QUERY PLAN +--------------------------------------------------------------------- + Custom Scan (Citus Adaptive) + -> Distributed Subplan XXX_1 + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Seq Scan on test_table_1960000 test_table + Task Count: 1 + Tasks Shown: All + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Aggregate + -> Merge Join + Merge Cond: (intermediate_result.key = intermediate_result_1.key) + -> Sort + Sort Key: intermediate_result.key + -> Function Scan on read_intermediate_result intermediate_result + -> Sort + Sort Key: intermediate_result_1.key + -> Function Scan on read_intermediate_result intermediate_result_1 +(21 rows) + +EXPLAIN (COSTS OFF) WITH cte_1 AS NOT MATERIALIZED (SELECT * FROM test_table) +SELECT + count(*) +FROM + cte_1 as first_entry + JOIN + cte_1 as second_entry + USING (key); +ERROR: syntax error at or near "NOT" +-- ctes with volatile functions are not +-- inlined +WITH cte_1 AS (SELECT *, random() FROM test_table) +SELECT + key, value +FROM + cte_1 +ORDER BY 2 DESC LIMIT 1; +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value, random() AS random FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value, intermediate_result.random FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb, random double precision)) cte_1 ORDER BY value DESC LIMIT 1 +DEBUG: Creating router plan +DEBUG: Plan is router executable + key | value +--------------------------------------------------------------------- + 9 | test99 +(1 row) + +-- even with NOT MATERIALIZED volatile functions should not be inlined +WITH cte_1 AS NOT MATERIALIZED (SELECT *, random() FROM test_table) +SELECT + count(*) +FROM + cte_1; +ERROR: syntax error at or near "NOT" +-- cte_1 should be able to inlined even if +-- it is used one level below +WITH cte_1 AS (SELECT * FROM test_table) +SELECT + count(*) +FROM +( + WITH ct2 AS (SELECT * FROM cte_1) + SELECT * FROM ct2 +) as foo; +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: CTE ct2 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries + count +--------------------------------------------------------------------- + 101 +(1 row) + +-- a similar query, but there is also +-- one more cte, which relies on the previous +-- CTE +WITH cte_1 AS (SELECT * FROM test_table) +SELECT + count(DISTINCT key) +FROM +( + WITH cte_2 AS (SELECT * FROM cte_1), + cte_3 AS (SELECT * FROM cte_2) + SELECT * FROM cte_3 +) as foo; +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: CTE cte_2 is going to be inlined via distributed planning +DEBUG: CTE cte_3 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries + count +--------------------------------------------------------------------- + 10 +(1 row) + +-- inlined CTE contains a reference to outer query +-- should be fine (because we pushdown the whole query) +SELECT count(*) + FROM + (SELECT * + FROM test_table) AS test_table_cte + JOIN LATERAL + (WITH bar AS (SELECT * + FROM test_table + WHERE key = test_table_cte.key) + SELECT * + FROM + bar + LEFT JOIN test_table u2 ON u2.key = bar.key) AS foo ON TRUE; +DEBUG: CTE bar is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: skipping recursive planning for the subquery since it contains references to outer queries +DEBUG: skipping recursive planning for the subquery since it contains references to outer queries +DEBUG: skipping recursive planning for the subquery since it contains references to outer queries +DEBUG: Router planner cannot handle multi-shard select queries +ERROR: CTEs that refer to other subqueries are not supported in multi-shard queries +-- inlined CTE contains a reference to outer query +-- should be fine (even if the recursive planning fails +-- to recursively plan the query) +SELECT count(*) + FROM + (SELECT * + FROM test_table) AS test_table_cte + JOIN LATERAL + (WITH bar AS (SELECT * + FROM test_table + WHERE key = test_table_cte.key) + SELECT * + FROM + bar + LEFT JOIN test_table u2 ON u2.key = bar.value::int) AS foo ON TRUE; +DEBUG: CTE bar is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: skipping recursive planning for the subquery since it contains references to outer queries +DEBUG: skipping recursive planning for the subquery since it contains references to outer queries +DEBUG: skipping recursive planning for the subquery since it contains references to outer queries +DEBUG: Router planner cannot handle multi-shard select queries +ERROR: CTEs that refer to other subqueries are not supported in multi-shard queries +-- inlined CTE can recursively planned later, that's the decision +-- recursive planning makes +-- LIMIT 5 in cte2 triggers recusrive planning, after cte inlining +WITH cte_1 AS (SELECT * FROM test_table) +SELECT + * +FROM +( + WITH ct2 AS (SELECT * FROM cte_1 ORDER BY 1, 2, 3 LIMIT 5) + SELECT * FROM ct2 +) as foo ORDER BY 1 DESC, 2 DESC, 3 DESC LIMIT 5; +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: CTE ct2 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: push down of limit count: 5 +DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) cte_1 ORDER BY key, value, other_value LIMIT 5 +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, other_value FROM (SELECT ct2.key, ct2.value, ct2.other_value FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) ct2) foo ORDER BY key DESC, value DESC, other_value DESC LIMIT 5 +DEBUG: Creating router plan +DEBUG: Plan is router executable + key | value | other_value +--------------------------------------------------------------------- + 0 | test30 | {"f1": 30, "f2": 540, "f3": "test30"} + 0 | test20 | {"f1": 20, "f2": 360, "f3": "test20"} + 0 | test100 | {"f1": 100, "f2": 1800, "f3": "test100"} + 0 | test10 | {"f1": 10, "f2": 180, "f3": "test10"} + 0 | test0 | {"f1": 0, "f2": 0, "f3": "test0"} +(5 rows) + +-- all nested CTEs can be inlinied +WITH cte_1 AS ( + WITH cte_1 AS ( + WITH cte_1 AS ( + WITH cte_1 AS ( + WITH cte_1 AS ( + WITH cte_1 AS ( + WITH cte_1 AS (SELECT count(*), key FROM test_table GROUP BY key) + SELECT * FROM cte_1) + SELECT * FROM cte_1 WHERE key = 1) + SELECT * FROM cte_1 WHERE key = 2) + SELECT * FROM cte_1 WHERE key = 3) + SELECT * FROM cte_1 WHERE key = 4) + SELECT * FROM cte_1 WHERE key = 5) +SELECT * FROM cte_1 WHERE key = 6; +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries + count | key +--------------------------------------------------------------------- +(0 rows) + +-- ctes can be inlined even if they are used +-- in set operations +WITH cte_1 AS (SELECT * FROM test_table), + cte_2 AS (SELECT * FROM test_table) +SELECT count(*) FROM ( +(SELECT * FROM cte_1 EXCEPT SELECT * FROM test_table) +UNION +(SELECT * FROM cte_2)) as foo; +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: CTE cte_2 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) cte_1 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_2 for subquery SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_3 for subquery SELECT key, value, other_value FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) cte_2 +DEBUG: Creating router plan +DEBUG: Plan is router executable +DEBUG: generating subplan XXX_4 for subquery (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb) EXCEPT SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) UNION SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb) +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_4'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) foo +DEBUG: Creating router plan +DEBUG: Plan is router executable + count +--------------------------------------------------------------------- + 101 +(1 row) + +-- cte_1 is going to be inlined even inside another set operation +WITH cte_1 AS (SELECT * FROM test_table), + cte_2 AS (SELECT * FROM test_table ORDER BY 1 DESC LIMIT 3) +(SELECT *, (SELECT 1) FROM cte_1 EXCEPT SELECT *, 1 FROM test_table) +UNION +(SELECT *, 1 FROM cte_2); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: CTE cte_2 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: push down of limit count: 3 +DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM cte_inline.test_table ORDER BY key DESC LIMIT 3 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_2 for CTE cte_2: SELECT key, value, other_value FROM cte_inline.test_table ORDER BY key DESC LIMIT 3 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: push down of limit count: 3 +DEBUG: Creating router plan +DEBUG: Plan is router executable +DEBUG: generating subplan XXX_3 for subquery SELECT key, value, other_value, (SELECT 1) FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_4 for subquery SELECT key, value, other_value, 1 FROM cte_inline.test_table +DEBUG: Plan XXX query after replacing subqueries and CTEs: (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value, intermediate_result."?column?" FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb, "?column?" integer) EXCEPT SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value, intermediate_result."?column?" FROM read_intermediate_result('XXX_4'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb, "?column?" integer)) UNION SELECT cte_2.key, cte_2.value, cte_2.other_value, 1 FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_2 +DEBUG: Creating router plan +DEBUG: Plan is router executable + key | value | other_value | ?column? +--------------------------------------------------------------------- + 9 | test29 | {"f1": 29, "f2": 522, "f3": "test29"} | 1 + 9 | test9 | {"f1": 9, "f2": 162, "f3": "test9"} | 1 + 9 | test19 | {"f1": 19, "f2": 342, "f3": "test19"} | 1 +(3 rows) + +-- cte_1 is safe to inline, even if because after inlining +-- it'd be in a query tree where there is a query that is +-- not supported by Citus unless recursively planned +-- cte_2 is on another queryTree, should be fine +WITH cte_1 AS (SELECT * FROM test_table), + cte_2 AS (SELECT * FROM test_table) +(SELECT *, (SELECT key FROM cte_1) FROM test_table) +UNION +(SELECT *, 1 FROM cte_2); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: CTE cte_2 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_2 for CTE cte_2: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +ERROR: could not run distributed query with subquery outside the FROM, WHERE and HAVING clauses +HINT: Consider using an equality filter on the distributed table's partition column. +-- after inlining CTEs, the query becomes +-- subquery pushdown with set operations +WITH cte_1 AS (SELECT * FROM test_table), + cte_2 AS (SELECT * FROM test_table) +SELECT max(key) FROM +( + SELECT * FROM cte_1 + UNION + SELECT * FROM cte_2 +) as bar; +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: CTE cte_2 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries + max +--------------------------------------------------------------------- + 9 +(1 row) + +-- cte LEFT JOIN subquery should only work +-- when CTE is inlined, as Citus currently +-- doesn't know how to handle intermediate +-- results in the outer parts of outer +-- queries +WITH cte AS (SELECT * FROM test_table) +SELECT + count(*) +FROM + cte LEFT JOIN test_table USING (key); +DEBUG: CTE cte is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte LEFT JOIN cte_inline.test_table USING (key)) +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte LEFT JOIN cte_inline.test_table USING (key)) +DEBUG: Router planner cannot handle multi-shard select queries +ERROR: cannot pushdown the subquery +DETAIL: Complex subqueries and CTEs cannot be in the outer part of the outer join +-- the CTEs are very simple, so postgres +-- can pull-up the subqueries after inlining +-- the CTEs, and the query that we send to workers +-- becomes a join between two tables +WITH cte_1 AS (SELECT key FROM test_table), + cte_2 AS (SELECT key FROM test_table) +SELECT + count(*) +FROM + cte_1 JOIN cte_2 USING (key); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: CTE cte_2 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for subquery SELECT key FROM cte_inline.test_table +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT test_table.key FROM cte_inline.test_table) cte_1 JOIN (SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) cte_2 USING (key)) +DEBUG: Router planner cannot handle multi-shard select queries + count +--------------------------------------------------------------------- + 1021 +(1 row) + +-- the following query is kind of interesting +-- During INSERT .. SELECT via coordinator, +-- Citus moves the CTEs into SELECT part, and plans/execute +-- the SELECT separately. Thus, fist_table_cte can be inlined +-- by Citus -- but not by Postgres +WITH fist_table_cte AS + (SELECT * FROM test_table) +INSERT INTO test_table + (key, value) + SELECT + key, value + FROM + fist_table_cte; +DEBUG: distributed INSERT ... SELECT can only select from distributed tables +DEBUG: CTE fist_table_cte is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: performing repartitioned INSERT ... SELECT +DEBUG: partitioning SELECT query by column index 0 with name 'key' +DEBUG: distributed statement: INSERT INTO cte_inline.test_table_1960000 AS citus_table_alias (key, value) SELECT key, value FROM read_intermediate_results('{repartitioned_results_xxxxx_from_1960000_to_0}'::text[], 'binary'::citus_copy_format) intermediate_result(key integer, value text) +DEBUG: distributed statement: INSERT INTO cte_inline.test_table_1960001 AS citus_table_alias (key, value) SELECT key, value FROM read_intermediate_results('{repartitioned_results_xxxxx_from_1960001_to_1}'::text[], 'binary'::citus_copy_format) intermediate_result(key integer, value text) +DEBUG: distributed statement: INSERT INTO cte_inline.test_table_1960002 AS citus_table_alias (key, value) SELECT key, value FROM read_intermediate_results('{repartitioned_results_xxxxx_from_1960002_to_2}'::text[], 'binary'::citus_copy_format) intermediate_result(key integer, value text) +DEBUG: distributed statement: INSERT INTO cte_inline.test_table_1960003 AS citus_table_alias (key, value) SELECT key, value FROM read_intermediate_results('{repartitioned_results_xxxxx_from_1960003_to_3}'::text[], 'binary'::citus_copy_format) intermediate_result(key integer, value text) +-- the following INSERT..SELECT is even more interesting +-- the CTE becomes pushdownable +INSERT INTO test_table +WITH fist_table_cte AS + (SELECT * FROM test_table) + SELECT + key, value + FROM + fist_table_cte; +DEBUG: CTE fist_table_cte is going to be inlined via distributed planning +DEBUG: distributed statement: INSERT INTO cte_inline.test_table_1960000 AS citus_table_alias (key, value) SELECT key, value FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table_1960000 test_table) fist_table_cte WHERE ((worker_hash(key) OPERATOR(pg_catalog.>=) '-2147483648'::integer) AND (worker_hash(key) OPERATOR(pg_catalog.<=) '-1073741825'::integer)) +DEBUG: distributed statement: INSERT INTO cte_inline.test_table_1960001 AS citus_table_alias (key, value) SELECT key, value FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table_1960001 test_table) fist_table_cte WHERE ((worker_hash(key) OPERATOR(pg_catalog.>=) '-1073741824'::integer) AND (worker_hash(key) OPERATOR(pg_catalog.<=) '-1'::integer)) +DEBUG: distributed statement: INSERT INTO cte_inline.test_table_1960002 AS citus_table_alias (key, value) SELECT key, value FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table_1960002 test_table) fist_table_cte WHERE ((worker_hash(key) OPERATOR(pg_catalog.>=) 0) AND (worker_hash(key) OPERATOR(pg_catalog.<=) 1073741823)) +DEBUG: distributed statement: INSERT INTO cte_inline.test_table_1960003 AS citus_table_alias (key, value) SELECT key, value FROM (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table_1960003 test_table) fist_table_cte WHERE ((worker_hash(key) OPERATOR(pg_catalog.>=) 1073741824) AND (worker_hash(key) OPERATOR(pg_catalog.<=) 2147483647)) +DEBUG: Plan is router executable +-- update/delete/modifying ctes +-- we don't support any cte inlining in modifications +-- queries and modifying CTEs +WITH cte_1 AS (SELECT * FROM test_table) + DELETE FROM test_table WHERE key NOT IN (SELECT key FROM cte_1); +DEBUG: complex joins are only supported when all distributed tables are co-located and joined on their distribution columns +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: DELETE FROM cte_inline.test_table WHERE (NOT (key OPERATOR(pg_catalog.=) ANY (SELECT cte_1.key FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1))) +DEBUG: Creating router plan +DEBUG: Plan is router executable +-- NOT MATERIALIZED should not CTEs that are used in a modifying query, because +-- we de still don't support it +WITH cte_1 AS NOT MATERIALIZED (SELECT * FROM test_table) + DELETE FROM test_table WHERE key NOT IN (SELECT key FROM cte_1); +ERROR: syntax error at or near "NOT" +-- we don't inline CTEs if they are modifying CTEs +WITH cte_1 AS (DELETE FROM test_table WHERE key % 3 = 1 RETURNING key) +SELECT * FROM cte_1 ORDER BY 1 DESC LIMIT 3; +DEBUG: data-modifying statements are not supported in the WITH clauses of distributed queries +DEBUG: generating subplan XXX_1 for CTE cte_1: DELETE FROM cte_inline.test_table WHERE ((key OPERATOR(pg_catalog.%) 3) OPERATOR(pg_catalog.=) 1) RETURNING key +DEBUG: Creating router plan +DEBUG: Plan is router executable +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key FROM (SELECT intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer)) cte_1 ORDER BY key DESC LIMIT 3 +DEBUG: Creating router plan +DEBUG: Plan is router executable + key +--------------------------------------------------------------------- + 7 + 7 + 7 +(3 rows) + +-- NOT MATERIALIZED should not affect modifying CTEs +WITH cte_1 AS NOT MATERIALIZED (DELETE FROM test_table WHERE key % 3 = 0 RETURNING key) +SELECT count(*) FROM cte_1; +ERROR: syntax error at or near "NOT" +-- cte with column aliases +SELECT * FROM test_table, +(WITH cte_1 (x,y) AS (SELECT * FROM test_table), + cte_2 (z,y) AS (SELECT value, other_value, key FROM test_table), + cte_3 (t,m) AS (SELECT z, y, key as cte_2_key FROM cte_2) + SELECT * FROM cte_2, cte_3) as bar +ORDER BY value, other_value, z, y, t, m, cte_2_key +LIMIT 5; +DEBUG: CTE cte_3 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_2: SELECT value, other_value, key FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT test_table.key, test_table.value, test_table.other_value, bar.z, bar.y, bar.key, bar.t, bar.m, bar.cte_2_key FROM cte_inline.test_table, (SELECT cte_2.z, cte_2.y, cte_2.key, cte_3.t, cte_3.m, cte_3.cte_2_key FROM (SELECT intermediate_result.value AS z, intermediate_result.other_value AS y, intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text, other_value jsonb, key integer)) cte_2, (SELECT cte_2_1.z AS t, cte_2_1.y AS m, cte_2_1.key AS cte_2_key FROM (SELECT intermediate_result.value AS z, intermediate_result.other_value AS y, intermediate_result.key FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text, other_value jsonb, key integer)) cte_2_1) cte_3) bar ORDER BY test_table.value, test_table.other_value, bar.z, bar.y, bar.t, bar.m, bar.cte_2_key LIMIT 5 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: push down of limit count: 5 + key | value | other_value | z | y | key | t | m | cte_2_key +--------------------------------------------------------------------- + 0 | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | 0 | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | 0 + 0 | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | 0 | test0 | | 0 + 0 | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | 0 | test0 | | 0 + 0 | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | 0 | test0 | | 0 + 0 | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | test0 | {"f1": 0, "f2": 0, "f3": "test0"} | 0 | test10 | {"f1": 10, "f2": 180, "f3": "test10"} | 0 +(5 rows) + +-- cte used in HAVING subquery just works fine +-- even if it is inlined +WITH cte_1 AS (SELECT max(key) as max FROM test_table) +SELECT + key, count(*) +FROM + test_table +GROUP BY + key +HAVING + (count(*) > (SELECT max FROM cte_1)) +ORDER BY 2 DESC, 1 DESC +LIMIT 5; +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for subquery SELECT max(key) AS max FROM cte_inline.test_table +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, count(*) AS count FROM cte_inline.test_table GROUP BY key HAVING (count(*) OPERATOR(pg_catalog.>) (SELECT cte_1.max FROM (SELECT intermediate_result.max FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(max integer)) cte_1)) ORDER BY (count(*)) DESC, key DESC LIMIT 5 +DEBUG: Router planner cannot handle multi-shard select queries + key | count +--------------------------------------------------------------------- + 0 | 44 + 9 | 40 + 8 | 40 + 6 | 40 + 5 | 40 +(5 rows) + +-- cte used in ORDER BY just works fine +-- even if it is inlined +WITH cte_1 AS (SELECT max(key) as max FROM test_table) +SELECT + key +FROM + test_table JOIN cte_1 ON (key = max) +ORDER BY + cte_1.max +LIMIT 3; +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for subquery SELECT max(key) AS max FROM cte_inline.test_table +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT test_table.key FROM (cte_inline.test_table JOIN (SELECT intermediate_result.max FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(max integer)) cte_1 ON ((test_table.key OPERATOR(pg_catalog.=) cte_1.max))) ORDER BY cte_1.max LIMIT 3 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: push down of limit count: 3 + key +--------------------------------------------------------------------- + 9 + 9 + 9 +(3 rows) + +PREPARE inlined_cte_without_params AS + WITH cte_1 AS (SELECT count(*) FROM test_table GROUP BY key) + SELECT * FROM cte_1 ORDER BY 1 DESC LIMIT 3; +PREPARE non_inlined_cte_without_params AS + WITH cte_1 AS (SELECT * FROM test_table) + SELECT + *, (SELECT 1) + FROM + cte_1 ORDER BY 1 DESC, 2 DESC, 3 DESC LIMIT 3; +PREPARE inlined_cte_has_parameter_on_non_dist_key(text) AS + WITH cte_1 AS (SELECT count(*) FROM test_table WHERE value = $1 GROUP BY key) + SELECT * FROM cte_1 ORDER BY 1 DESC LIMIT 3; +PREPARE inlined_cte_has_parameter_on_dist_key(int) AS + WITH cte_1 AS (SELECT count(*) FROM test_table WHERE key > $1 GROUP BY key) + SELECT * FROM cte_1 ORDER BY 1 DESC LIMIT 3; +PREPARE non_inlined_cte_has_parameter_on_dist_key(int) AS + WITH cte_1 AS (SELECT * FROM test_table where key > $1) + SELECT + *, (SELECT 1) + FROM + cte_1 ORDER BY 1 DESC, 2 DESC, 3 DESC LIMIT 3; +PREPARE retry_planning(int) AS + WITH cte_1 AS (SELECT * FROM test_table WHERE key > $1) + SELECT json_object_agg(DISTINCT key, value) FROM cte_1 ORDER BY max(key), min(value) DESC LIMIT 3; +EXECUTE inlined_cte_without_params; +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: push down of limit count: 3 + count +--------------------------------------------------------------------- + 44 + 40 + 40 +(3 rows) + +EXECUTE inlined_cte_without_params; + count +--------------------------------------------------------------------- + 44 + 40 + 40 +(3 rows) + +EXECUTE inlined_cte_without_params; + count +--------------------------------------------------------------------- + 44 + 40 + 40 +(3 rows) + +EXECUTE inlined_cte_without_params; + count +--------------------------------------------------------------------- + 44 + 40 + 40 +(3 rows) + +EXECUTE inlined_cte_without_params; + count +--------------------------------------------------------------------- + 44 + 40 + 40 +(3 rows) + +EXECUTE inlined_cte_without_params; + count +--------------------------------------------------------------------- + 44 + 40 + 40 +(3 rows) + +EXECUTE non_inlined_cte_without_params; +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, other_value, (SELECT 1) FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 ORDER BY key DESC, value DESC, other_value DESC LIMIT 3 +DEBUG: Creating router plan +DEBUG: Plan is router executable + key | value | other_value | ?column? +--------------------------------------------------------------------- + 9 | test99 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 +(3 rows) + +EXECUTE non_inlined_cte_without_params; + key | value | other_value | ?column? +--------------------------------------------------------------------- + 9 | test99 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 +(3 rows) + +EXECUTE non_inlined_cte_without_params; + key | value | other_value | ?column? +--------------------------------------------------------------------- + 9 | test99 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 +(3 rows) + +EXECUTE non_inlined_cte_without_params; + key | value | other_value | ?column? +--------------------------------------------------------------------- + 9 | test99 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 +(3 rows) + +EXECUTE non_inlined_cte_without_params; + key | value | other_value | ?column? +--------------------------------------------------------------------- + 9 | test99 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 +(3 rows) + +EXECUTE non_inlined_cte_without_params; + key | value | other_value | ?column? +--------------------------------------------------------------------- + 9 | test99 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 +(3 rows) + +EXECUTE inlined_cte_has_parameter_on_non_dist_key('test1'); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: push down of limit count: 3 + count +--------------------------------------------------------------------- +(0 rows) + +EXECUTE inlined_cte_has_parameter_on_non_dist_key('test2'); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: push down of limit count: 3 + count +--------------------------------------------------------------------- + 4 +(1 row) + +EXECUTE inlined_cte_has_parameter_on_non_dist_key('test3'); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: push down of limit count: 3 + count +--------------------------------------------------------------------- + 4 +(1 row) + +EXECUTE inlined_cte_has_parameter_on_non_dist_key('test4'); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: push down of limit count: 3 + count +--------------------------------------------------------------------- +(0 rows) + +EXECUTE inlined_cte_has_parameter_on_non_dist_key('test5'); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: push down of limit count: 3 + count +--------------------------------------------------------------------- + 4 +(1 row) + +EXECUTE inlined_cte_has_parameter_on_non_dist_key('test6'); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: push down of limit count: 3 + count +--------------------------------------------------------------------- + 4 +(1 row) + +EXECUTE inlined_cte_has_parameter_on_dist_key(1); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: push down of limit count: 3 + count +--------------------------------------------------------------------- + 40 + 40 + 40 +(3 rows) + +EXECUTE inlined_cte_has_parameter_on_dist_key(2); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: push down of limit count: 3 + count +--------------------------------------------------------------------- + 40 + 40 + 40 +(3 rows) + +EXECUTE inlined_cte_has_parameter_on_dist_key(3); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: push down of limit count: 3 + count +--------------------------------------------------------------------- + 40 + 40 + 40 +(3 rows) + +EXECUTE inlined_cte_has_parameter_on_dist_key(4); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: push down of limit count: 3 + count +--------------------------------------------------------------------- + 40 + 40 + 40 +(3 rows) + +EXECUTE inlined_cte_has_parameter_on_dist_key(5); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: push down of limit count: 3 + count +--------------------------------------------------------------------- + 40 + 40 + 40 +(3 rows) + +EXECUTE inlined_cte_has_parameter_on_dist_key(6); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: push down of limit count: 3 + count +--------------------------------------------------------------------- + 40 + 40 +(2 rows) + +EXECUTE non_inlined_cte_has_parameter_on_dist_key(1); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.>) 1) +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, other_value, (SELECT 1) FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 ORDER BY key DESC, value DESC, other_value DESC LIMIT 3 +DEBUG: Creating router plan +DEBUG: Plan is router executable + key | value | other_value | ?column? +--------------------------------------------------------------------- + 9 | test99 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 +(3 rows) + +EXECUTE non_inlined_cte_has_parameter_on_dist_key(2); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.>) 2) +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, other_value, (SELECT 1) FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 ORDER BY key DESC, value DESC, other_value DESC LIMIT 3 +DEBUG: Creating router plan +DEBUG: Plan is router executable + key | value | other_value | ?column? +--------------------------------------------------------------------- + 9 | test99 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 +(3 rows) + +EXECUTE non_inlined_cte_has_parameter_on_dist_key(3); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.>) 3) +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, other_value, (SELECT 1) FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 ORDER BY key DESC, value DESC, other_value DESC LIMIT 3 +DEBUG: Creating router plan +DEBUG: Plan is router executable + key | value | other_value | ?column? +--------------------------------------------------------------------- + 9 | test99 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 +(3 rows) + +EXECUTE non_inlined_cte_has_parameter_on_dist_key(4); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.>) 4) +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, other_value, (SELECT 1) FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 ORDER BY key DESC, value DESC, other_value DESC LIMIT 3 +DEBUG: Creating router plan +DEBUG: Plan is router executable + key | value | other_value | ?column? +--------------------------------------------------------------------- + 9 | test99 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 +(3 rows) + +EXECUTE non_inlined_cte_has_parameter_on_dist_key(5); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.>) 5) +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, other_value, (SELECT 1) FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 ORDER BY key DESC, value DESC, other_value DESC LIMIT 3 +DEBUG: Creating router plan +DEBUG: Plan is router executable + key | value | other_value | ?column? +--------------------------------------------------------------------- + 9 | test99 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 +(3 rows) + +EXECUTE non_inlined_cte_has_parameter_on_dist_key(6); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT key, value, other_value FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.>) 6) +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT key, value, other_value, (SELECT 1) FROM (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_1 ORDER BY key DESC, value DESC, other_value DESC LIMIT 3 +DEBUG: Creating router plan +DEBUG: Plan is router executable + key | value | other_value | ?column? +--------------------------------------------------------------------- + 9 | test99 | | 1 + 9 | test99 | | 1 + 9 | test99 | | 1 +(3 rows) + +EXECUTE retry_planning(1); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries + json_object_agg +--------------------------------------------------------------------- + { "2" : "test12", "2" : "test2", "2" : "test22", "2" : "test32", "2" : "test42", "2" : "test52", "2" : "test62", "2" : "test72", "2" : "test82", "2" : "test92", "3" : "test13", "3" : "test23", "3" : "test3", "3" : "test33", "3" : "test43", "3" : "test53", "3" : "test63", "3" : "test73", "3" : "test83", "3" : "test93", "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "6" : "test16", "6" : "test26", "6" : "test36", "6" : "test46", "6" : "test56", "6" : "test6", "6" : "test66", "6" : "test76", "6" : "test86", "6" : "test96", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } +(1 row) + +EXECUTE retry_planning(2); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries + json_object_agg +--------------------------------------------------------------------- + { "3" : "test13", "3" : "test23", "3" : "test3", "3" : "test33", "3" : "test43", "3" : "test53", "3" : "test63", "3" : "test73", "3" : "test83", "3" : "test93", "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "6" : "test16", "6" : "test26", "6" : "test36", "6" : "test46", "6" : "test56", "6" : "test6", "6" : "test66", "6" : "test76", "6" : "test86", "6" : "test96", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } +(1 row) + +EXECUTE retry_planning(3); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries + json_object_agg +--------------------------------------------------------------------- + { "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "6" : "test16", "6" : "test26", "6" : "test36", "6" : "test46", "6" : "test56", "6" : "test6", "6" : "test66", "6" : "test76", "6" : "test86", "6" : "test96", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } +(1 row) + +EXECUTE retry_planning(4); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries + json_object_agg +--------------------------------------------------------------------- + { "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "6" : "test16", "6" : "test26", "6" : "test36", "6" : "test46", "6" : "test56", "6" : "test6", "6" : "test66", "6" : "test76", "6" : "test86", "6" : "test96", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } +(1 row) + +EXECUTE retry_planning(5); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries + json_object_agg +--------------------------------------------------------------------- + { "6" : "test16", "6" : "test26", "6" : "test36", "6" : "test46", "6" : "test56", "6" : "test6", "6" : "test66", "6" : "test76", "6" : "test86", "6" : "test96", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } +(1 row) + +EXECUTE retry_planning(6); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries + json_object_agg +--------------------------------------------------------------------- + { "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } +(1 row) + +-- this test can only work if the CTE is recursively +-- planned +WITH b AS (SELECT * FROM test_table) +SELECT count(*) FROM (SELECT key as x FROM test_table OFFSET 0) as ref LEFT JOIN b ON (ref.x = b.key); +DEBUG: CTE b is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for subquery SELECT key AS x FROM cte_inline.test_table OFFSET 0 +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT intermediate_result.x FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(x integer)) ref LEFT JOIN (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) b ON ((ref.x OPERATOR(pg_catalog.=) b.key))) +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE b: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_2 for subquery SELECT key AS x FROM cte_inline.test_table OFFSET 0 +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT intermediate_result.x FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(x integer)) ref LEFT JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) b ON ((ref.x OPERATOR(pg_catalog.=) b.key))) +DEBUG: Creating router plan +DEBUG: Plan is router executable + count +--------------------------------------------------------------------- + 11536 +(1 row) + +-- this becomes a non-colocated subquery join +-- because after the CTEs are inlined the joins +-- become a non-colocated subquery join +WITH a AS (SELECT * FROM test_table), +b AS (SELECT * FROM test_table) +SELECT count(*) FROM a LEFT JOIN b ON (a.value = b.value); +DEBUG: CTE a is going to be inlined via distributed planning +DEBUG: CTE b is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) a LEFT JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) b ON ((a.value OPERATOR(pg_catalog.=) b.value))) +DEBUG: Router planner cannot handle multi-shard select queries + count +--------------------------------------------------------------------- + 1136 +(1 row) + +-- cte a has to be recursively planned because of OFFSET 0 +-- after that, cte b also requires recursive planning +WITH a AS (SELECT * FROM test_table OFFSET 0), +b AS (SELECT * FROM test_table) +SELECT min(a.key) FROM a LEFT JOIN b ON (a.value = b.value); +DEBUG: CTE a is going to be inlined via distributed planning +DEBUG: CTE b is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM cte_inline.test_table OFFSET 0 +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT min(a.key) AS min FROM ((SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) a LEFT JOIN (SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) b ON ((a.value OPERATOR(pg_catalog.=) b.value))) +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE a: SELECT key, value, other_value FROM cte_inline.test_table OFFSET 0 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_2 for CTE b: SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT min(a.key) AS min FROM ((SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) a LEFT JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) b ON ((a.value OPERATOR(pg_catalog.=) b.value))) +DEBUG: Creating router plan +DEBUG: Plan is router executable + min +--------------------------------------------------------------------- + 0 +(1 row) + +-- after both CTEs are inlined, this becomes non-colocated subquery join +WITH cte_1 AS (SELECT * FROM test_table), +cte_2 AS (SELECT * FROM test_table) +SELECT * FROM cte_1 JOIN cte_2 ON (cte_1.value > cte_2.value) ORDER BY 1,2,3,4,5,6 DESC LIMIT 3;; +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: CTE cte_2 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for subquery SELECT key, value, other_value FROM cte_inline.test_table +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT cte_1.key, cte_1.value, cte_1.other_value, cte_2.key, cte_2.value, cte_2.other_value FROM ((SELECT test_table.key, test_table.value, test_table.other_value FROM cte_inline.test_table) cte_1 JOIN (SELECT intermediate_result.key, intermediate_result.value, intermediate_result.other_value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(key integer, value text, other_value jsonb)) cte_2 ON ((cte_1.value OPERATOR(pg_catalog.>) cte_2.value))) ORDER BY cte_1.key, cte_1.value, cte_1.other_value, cte_2.key, cte_2.value, cte_2.other_value DESC LIMIT 3 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: push down of limit count: 3 + key | value | other_value | key | value | other_value +--------------------------------------------------------------------- + 0 | test10 | {"f1": 10, "f2": 180, "f3": "test10"} | 0 | test0 | + 0 | test10 | {"f1": 10, "f2": 180, "f3": "test10"} | 0 | test0 | + 0 | test10 | {"f1": 10, "f2": 180, "f3": "test10"} | 0 | test0 | +(3 rows) + +-- full join is only supported when both sides are +-- recursively planned +WITH cte_1 AS (SELECT value FROM test_table WHERE key > 1), + cte_2 AS (SELECT value FROM test_table WHERE key > 3) +SELECT * FROM cte_1 FULL JOIN cte_2 USING (value) ORDER BY 1 DESC LIMIT 3;; +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: CTE cte_2 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for subquery SELECT value FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.>) 3) +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT value FROM ((SELECT test_table.value FROM cte_inline.test_table WHERE (test_table.key OPERATOR(pg_catalog.>) 1)) cte_1 FULL JOIN (SELECT intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text)) cte_2 USING (value)) ORDER BY value DESC LIMIT 3 +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for CTE cte_1: SELECT value FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.>) 1) +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_2 for CTE cte_2: SELECT value FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.>) 3) +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT value FROM ((SELECT intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text)) cte_1 FULL JOIN (SELECT intermediate_result.value FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(value text)) cte_2 USING (value)) ORDER BY value DESC LIMIT 3 +DEBUG: Creating router plan +DEBUG: Plan is router executable + value +--------------------------------------------------------------------- + test99 + test99 + test99 +(3 rows) + +-- an unsupported agg. for multi-shard queries +-- so CTE has to be recursively planned +WITH cte_1 AS (SELECT * FROM test_table WHERE key > 1) +SELECT json_object_agg(DISTINCT key, value) FROM cte_1; +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries + json_object_agg +--------------------------------------------------------------------- + { "2" : "test12", "2" : "test2", "2" : "test22", "2" : "test32", "2" : "test42", "2" : "test52", "2" : "test62", "2" : "test72", "2" : "test82", "2" : "test92", "3" : "test13", "3" : "test23", "3" : "test3", "3" : "test33", "3" : "test43", "3" : "test53", "3" : "test63", "3" : "test73", "3" : "test83", "3" : "test93", "5" : "test15", "5" : "test25", "5" : "test35", "5" : "test45", "5" : "test5", "5" : "test55", "5" : "test65", "5" : "test75", "5" : "test85", "5" : "test95", "6" : "test16", "6" : "test26", "6" : "test36", "6" : "test46", "6" : "test56", "6" : "test6", "6" : "test66", "6" : "test76", "6" : "test86", "6" : "test96", "8" : "test18", "8" : "test28", "8" : "test38", "8" : "test48", "8" : "test58", "8" : "test68", "8" : "test78", "8" : "test8", "8" : "test88", "8" : "test98", "9" : "test19", "9" : "test29", "9" : "test39", "9" : "test49", "9" : "test59", "9" : "test69", "9" : "test79", "9" : "test89", "9" : "test9", "9" : "test99" } +(1 row) + +-- both cte_1 and cte_2 are going to be inlined. +-- later, cte_2 is recursively planned since it doesn't have +-- GROUP BY but aggragate in a subquery. +-- this is an important example of being able to recursively plan +-- "some" of the CTEs +WITH cte_1 AS (SELECT value FROM test_table WHERE key > 1), + cte_2 AS (SELECT max(value) as value FROM test_table WHERE key > 3) +SELECT count(*) FROM cte_1 JOIN cte_2 USING (value); +DEBUG: CTE cte_1 is going to be inlined via distributed planning +DEBUG: CTE cte_2 is going to be inlined via distributed planning +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: Router planner cannot handle multi-shard select queries +DEBUG: generating subplan XXX_1 for subquery SELECT max(value) AS value FROM cte_inline.test_table WHERE (key OPERATOR(pg_catalog.>) 3) +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM ((SELECT test_table.value FROM cte_inline.test_table WHERE (test_table.key OPERATOR(pg_catalog.>) 1)) cte_1 JOIN (SELECT intermediate_result.value FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value text)) cte_2 USING (value)) +DEBUG: Router planner cannot handle multi-shard select queries + count +--------------------------------------------------------------------- + 4 +(1 row) + +-- prevent DROP CASCADE to give notices +SET client_min_messages TO ERROR; +DROP SCHEMA cte_inline CASCADE; diff --git a/src/test/regress/expected/multi_complex_count_distinct_1.out b/src/test/regress/expected/multi_complex_count_distinct_1.out new file mode 100644 index 000000000..b14e9c7e8 --- /dev/null +++ b/src/test/regress/expected/multi_complex_count_distinct_1.out @@ -0,0 +1,1128 @@ +-- +-- COMPLEX_COUNT_DISTINCT +-- +SET citus.next_shard_id TO 240000; +SET citus.shard_count TO 8; +SET citus.shard_replication_factor TO 1; +SET citus.coordinator_aggregation_strategy TO 'disabled'; +CREATE TABLE lineitem_hash ( + l_orderkey bigint not null, + l_partkey integer not null, + l_suppkey integer not null, + l_linenumber integer not null, + l_quantity decimal(15, 2) not null, + l_extendedprice decimal(15, 2) not null, + l_discount decimal(15, 2) not null, + l_tax decimal(15, 2) not null, + l_returnflag char(1) not null, + l_linestatus char(1) not null, + l_shipdate date not null, + l_commitdate date not null, + l_receiptdate date not null, + l_shipinstruct char(25) not null, + l_shipmode char(10) not null, + l_comment varchar(44) not null, + PRIMARY KEY(l_orderkey, l_linenumber) ); +SELECT create_distributed_table('lineitem_hash', 'l_orderkey', 'hash'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +\copy lineitem_hash FROM '/home/talha/citus/src/test/regress/data/lineitem.1.data' with delimiter '|' +\copy lineitem_hash FROM '/home/talha/citus/src/test/regress/data/lineitem.2.data' with delimiter '|' +ANALYZE lineitem_hash; +SET citus.task_executor_type to "task-tracker"; +-- count(distinct) is supported on top level query if there +-- is a grouping on the partition key +SELECT + l_orderkey, count(DISTINCT l_partkey) + FROM lineitem_hash + GROUP BY l_orderkey + ORDER BY 2 DESC, 1 DESC + LIMIT 10; + l_orderkey | count +--------------------------------------------------------------------- + 14885 | 7 + 14884 | 7 + 14821 | 7 + 14790 | 7 + 14785 | 7 + 14755 | 7 + 14725 | 7 + 14694 | 7 + 14627 | 7 + 14624 | 7 +(10 rows) + +EXPLAIN (COSTS false, VERBOSE true) +SELECT + l_orderkey, count(DISTINCT l_partkey) + FROM lineitem_hash + GROUP BY l_orderkey + ORDER BY 2 DESC, 1 DESC + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + Output: remote_scan.l_orderkey, remote_scan.count + -> Sort + Output: remote_scan.l_orderkey, remote_scan.count + Sort Key: remote_scan.count DESC, remote_scan.l_orderkey DESC + -> Custom Scan (Citus Task-Tracker) + Output: remote_scan.l_orderkey, remote_scan.count + Task Count: 8 + Tasks Shown: One of 8 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Limit + Output: l_orderkey, (count(DISTINCT l_partkey)) + -> Sort + Output: l_orderkey, (count(DISTINCT l_partkey)) + Sort Key: (count(DISTINCT lineitem_hash.l_partkey)) DESC, lineitem_hash.l_orderkey DESC + -> GroupAggregate + Output: l_orderkey, count(DISTINCT l_partkey) + Group Key: lineitem_hash.l_orderkey + -> Index Scan Backward using lineitem_hash_pkey_240000 on public.lineitem_hash_240000 lineitem_hash + Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment +(21 rows) + +-- it is also supported if there is no grouping or grouping is on non-partition field +SELECT + count(DISTINCT l_partkey) + FROM lineitem_hash + ORDER BY 1 DESC + LIMIT 10; + count +--------------------------------------------------------------------- + 11661 +(1 row) + +EXPLAIN (COSTS false, VERBOSE true) +SELECT + count(DISTINCT l_partkey) + FROM lineitem_hash + ORDER BY 1 DESC + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + Output: (count(DISTINCT remote_scan.count)) + -> Sort + Output: (count(DISTINCT remote_scan.count)) + Sort Key: (count(DISTINCT remote_scan.count)) DESC + -> Aggregate + Output: count(DISTINCT remote_scan.count) + -> Custom Scan (Citus Task-Tracker) + Output: remote_scan.count + Task Count: 8 + Tasks Shown: One of 8 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: l_partkey + Group Key: lineitem_hash.l_partkey + -> Seq Scan on public.lineitem_hash_240000 lineitem_hash + Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment +(18 rows) + +SELECT + l_shipmode, count(DISTINCT l_partkey) + FROM lineitem_hash + GROUP BY l_shipmode + ORDER BY 2 DESC, 1 DESC + LIMIT 10; + l_shipmode | count +--------------------------------------------------------------------- + TRUCK | 1757 + MAIL | 1730 + AIR | 1702 + FOB | 1700 + RAIL | 1696 + SHIP | 1684 + REG AIR | 1676 +(7 rows) + +EXPLAIN (COSTS false, VERBOSE true) +SELECT + l_shipmode, count(DISTINCT l_partkey) + FROM lineitem_hash + GROUP BY l_shipmode + ORDER BY 2 DESC, 1 DESC + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + Output: remote_scan.l_shipmode, (count(DISTINCT remote_scan.count)) + -> Sort + Output: remote_scan.l_shipmode, (count(DISTINCT remote_scan.count)) + Sort Key: (count(DISTINCT remote_scan.count)) DESC, remote_scan.l_shipmode DESC + -> GroupAggregate + Output: remote_scan.l_shipmode, count(DISTINCT remote_scan.count) + Group Key: remote_scan.l_shipmode + -> Sort + Output: remote_scan.l_shipmode, remote_scan.count + Sort Key: remote_scan.l_shipmode DESC + -> Custom Scan (Citus Task-Tracker) + Output: remote_scan.l_shipmode, remote_scan.count + Task Count: 8 + Tasks Shown: One of 8 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: l_shipmode, l_partkey + Group Key: lineitem_hash.l_shipmode, lineitem_hash.l_partkey + -> Seq Scan on public.lineitem_hash_240000 lineitem_hash + Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment +(22 rows) + +-- mixed mode count distinct, grouped by partition column +SELECT + l_orderkey, count(distinct l_partkey), count(distinct l_shipmode) + FROM lineitem_hash + GROUP BY l_orderkey + ORDER BY 3 DESC, 2 DESC, 1 + LIMIT 10; + l_orderkey | count | count +--------------------------------------------------------------------- + 226 | 7 | 7 + 1316 | 7 | 7 + 1477 | 7 | 7 + 3555 | 7 | 7 + 12258 | 7 | 7 + 12835 | 7 | 7 + 768 | 7 | 6 + 1121 | 7 | 6 + 1153 | 7 | 6 + 1281 | 7 | 6 +(10 rows) + +EXPLAIN (COSTS false, VERBOSE true) +SELECT + l_orderkey, count(distinct l_partkey), count(distinct l_shipmode) + FROM lineitem_hash + GROUP BY l_orderkey + ORDER BY 3 DESC, 2 DESC, 1 + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + Output: remote_scan.l_orderkey, remote_scan.count, remote_scan.count_1 + -> Sort + Output: remote_scan.l_orderkey, remote_scan.count, remote_scan.count_1 + Sort Key: remote_scan.count_1 DESC, remote_scan.count DESC, remote_scan.l_orderkey + -> Custom Scan (Citus Task-Tracker) + Output: remote_scan.l_orderkey, remote_scan.count, remote_scan.count_1 + Task Count: 8 + Tasks Shown: One of 8 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Limit + Output: l_orderkey, (count(DISTINCT l_partkey)), (count(DISTINCT l_shipmode)) + -> Sort + Output: l_orderkey, (count(DISTINCT l_partkey)), (count(DISTINCT l_shipmode)) + Sort Key: (count(DISTINCT lineitem_hash.l_shipmode)) DESC, (count(DISTINCT lineitem_hash.l_partkey)) DESC, lineitem_hash.l_orderkey + -> GroupAggregate + Output: l_orderkey, count(DISTINCT l_partkey), count(DISTINCT l_shipmode) + Group Key: lineitem_hash.l_orderkey + -> Index Scan using lineitem_hash_pkey_240000 on public.lineitem_hash_240000 lineitem_hash + Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment +(21 rows) + +-- partition/non-partition column count distinct no grouping +SELECT + count(distinct l_orderkey), count(distinct l_partkey), count(distinct l_shipmode) + FROM lineitem_hash; + count | count | count +--------------------------------------------------------------------- + 2985 | 11661 | 7 +(1 row) + +EXPLAIN (COSTS false, VERBOSE true) +SELECT + count(distinct l_orderkey), count(distinct l_partkey), count(distinct l_shipmode) + FROM lineitem_hash; + QUERY PLAN +--------------------------------------------------------------------- + Aggregate + Output: count(DISTINCT remote_scan.count), count(DISTINCT remote_scan.count_1), count(DISTINCT remote_scan.count_2) + -> Custom Scan (Citus Task-Tracker) + Output: remote_scan.count, remote_scan.count_1, remote_scan.count_2 + Task Count: 8 + Tasks Shown: One of 8 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: l_orderkey, l_partkey, l_shipmode + Group Key: lineitem_hash.l_orderkey, lineitem_hash.l_partkey, lineitem_hash.l_shipmode + -> Seq Scan on public.lineitem_hash_240000 lineitem_hash + Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment +(13 rows) + +-- distinct/non-distinct on partition and non-partition columns +SELECT + count(distinct l_orderkey), count(l_orderkey), + count(distinct l_partkey), count(l_partkey), + count(distinct l_shipmode), count(l_shipmode) + FROM lineitem_hash; + count | count | count | count | count | count +--------------------------------------------------------------------- + 2985 | 12000 | 11661 | 12000 | 7 | 12000 +(1 row) + +-- mixed mode count distinct, grouped by non-partition column +SELECT + l_shipmode, count(distinct l_partkey), count(distinct l_orderkey) + FROM lineitem_hash + GROUP BY l_shipmode + ORDER BY 1, 2 DESC, 3 DESC; + l_shipmode | count | count +--------------------------------------------------------------------- + AIR | 1702 | 1327 + FOB | 1700 | 1276 + MAIL | 1730 | 1299 + RAIL | 1696 | 1265 + REG AIR | 1676 | 1275 + SHIP | 1684 | 1289 + TRUCK | 1757 | 1333 +(7 rows) + +-- mixed mode count distinct, grouped by non-partition column +-- having on partition column +SELECT + l_shipmode, count(distinct l_partkey), count(distinct l_orderkey) + FROM lineitem_hash + GROUP BY l_shipmode + HAVING count(distinct l_orderkey) > 1300 + ORDER BY 1, 2 DESC; + l_shipmode | count | count +--------------------------------------------------------------------- + AIR | 1702 | 1327 + TRUCK | 1757 | 1333 +(2 rows) + +-- same but having clause is not on target list +SELECT + l_shipmode, count(distinct l_partkey) + FROM lineitem_hash + GROUP BY l_shipmode + HAVING count(distinct l_orderkey) > 1300 + ORDER BY 1, 2 DESC; + l_shipmode | count +--------------------------------------------------------------------- + AIR | 1702 + TRUCK | 1757 +(2 rows) + +-- mixed mode count distinct, grouped by non-partition column +-- having on non-partition column +SELECT + l_shipmode, count(distinct l_partkey), count(distinct l_suppkey) + FROM lineitem_hash + GROUP BY l_shipmode + HAVING count(distinct l_suppkey) > 1550 + ORDER BY 1, 2 DESC; + l_shipmode | count | count +--------------------------------------------------------------------- + AIR | 1702 | 1564 + FOB | 1700 | 1571 + MAIL | 1730 | 1573 + RAIL | 1696 | 1581 + REG AIR | 1676 | 1557 + SHIP | 1684 | 1554 + TRUCK | 1757 | 1602 +(7 rows) + +-- same but having clause is not on target list +SELECT + l_shipmode, count(distinct l_partkey) + FROM lineitem_hash + GROUP BY l_shipmode + HAVING count(distinct l_suppkey) > 1550 + ORDER BY 1, 2 DESC; + l_shipmode | count +--------------------------------------------------------------------- + AIR | 1702 + FOB | 1700 + MAIL | 1730 + RAIL | 1696 + REG AIR | 1676 + SHIP | 1684 + TRUCK | 1757 +(7 rows) + +EXPLAIN (COSTS false, VERBOSE true) +SELECT + l_shipmode, count(distinct l_partkey) + FROM lineitem_hash + GROUP BY l_shipmode + HAVING count(distinct l_suppkey) > 1550 + ORDER BY 1, 2 DESC; + QUERY PLAN +--------------------------------------------------------------------- + Sort + Output: remote_scan.l_shipmode, (count(DISTINCT remote_scan.count)) + Sort Key: remote_scan.l_shipmode, (count(DISTINCT remote_scan.count)) DESC + -> GroupAggregate + Output: remote_scan.l_shipmode, count(DISTINCT remote_scan.count) + Group Key: remote_scan.l_shipmode + Filter: (count(DISTINCT remote_scan.worker_column_3) > 1550) + -> Sort + Output: remote_scan.l_shipmode, remote_scan.count, remote_scan.worker_column_3 + Sort Key: remote_scan.l_shipmode + -> Custom Scan (Citus Task-Tracker) + Output: remote_scan.l_shipmode, remote_scan.count, remote_scan.worker_column_3 + Task Count: 8 + Tasks Shown: One of 8 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: l_shipmode, l_partkey, l_suppkey + Group Key: lineitem_hash.l_shipmode, lineitem_hash.l_partkey, lineitem_hash.l_suppkey + -> Seq Scan on public.lineitem_hash_240000 lineitem_hash + Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment +(21 rows) + +-- count distinct is supported on single table subqueries +SELECT * + FROM ( + SELECT + l_orderkey, count(DISTINCT l_partkey) + FROM lineitem_hash + GROUP BY l_orderkey) sub + ORDER BY 2 DESC, 1 DESC + LIMIT 10; + l_orderkey | count +--------------------------------------------------------------------- + 14885 | 7 + 14884 | 7 + 14821 | 7 + 14790 | 7 + 14785 | 7 + 14755 | 7 + 14725 | 7 + 14694 | 7 + 14627 | 7 + 14624 | 7 +(10 rows) + +SELECT * + FROM ( + SELECT + l_partkey, count(DISTINCT l_orderkey) + FROM lineitem_hash + GROUP BY l_partkey) sub + ORDER BY 2 DESC, 1 DESC + LIMIT 10; + l_partkey | count +--------------------------------------------------------------------- + 199146 | 3 + 188804 | 3 + 177771 | 3 + 160895 | 3 + 149926 | 3 + 136884 | 3 + 87761 | 3 + 15283 | 3 + 6983 | 3 + 1927 | 3 +(10 rows) + +EXPLAIN (COSTS false, VERBOSE true) +SELECT * + FROM ( + SELECT + l_partkey, count(DISTINCT l_orderkey) + FROM lineitem_hash + GROUP BY l_partkey) sub + ORDER BY 2 DESC, 1 DESC + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + Output: remote_scan.l_partkey, remote_scan.count + -> Sort + Output: remote_scan.l_partkey, remote_scan.count + Sort Key: remote_scan.count DESC, remote_scan.l_partkey DESC + -> Custom Scan (Citus Task-Tracker) + Output: remote_scan.l_partkey, remote_scan.count + Task Count: 4 + Tasks Shown: None, not supported for re-partition queries + -> MapMergeJob + Map Task Count: 8 + Merge Task Count: 4 +(12 rows) + +-- count distinct with filters +SELECT + l_orderkey, + count(DISTINCT l_suppkey) FILTER (WHERE l_shipmode = 'AIR'), + count(DISTINCT l_suppkey) + FROM lineitem_hash + GROUP BY l_orderkey + ORDER BY 2 DESC, 3 DESC, 1 + LIMIT 10; + l_orderkey | count | count +--------------------------------------------------------------------- + 4964 | 4 | 7 + 12005 | 4 | 7 + 5409 | 4 | 6 + 164 | 3 | 7 + 322 | 3 | 7 + 871 | 3 | 7 + 1156 | 3 | 7 + 1574 | 3 | 7 + 2054 | 3 | 7 + 2309 | 3 | 7 +(10 rows) + +EXPLAIN (COSTS false, VERBOSE true) +SELECT + l_orderkey, + count(DISTINCT l_suppkey) FILTER (WHERE l_shipmode = 'AIR'), + count(DISTINCT l_suppkey) + FROM lineitem_hash + GROUP BY l_orderkey + ORDER BY 2 DESC, 3 DESC, 1 + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + Output: remote_scan.l_orderkey, remote_scan.count, remote_scan.count_1 + -> Sort + Output: remote_scan.l_orderkey, remote_scan.count, remote_scan.count_1 + Sort Key: remote_scan.count DESC, remote_scan.count_1 DESC, remote_scan.l_orderkey + -> Custom Scan (Citus Task-Tracker) + Output: remote_scan.l_orderkey, remote_scan.count, remote_scan.count_1 + Task Count: 8 + Tasks Shown: One of 8 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Limit + Output: l_orderkey, (count(DISTINCT l_suppkey) FILTER (WHERE (l_shipmode = 'AIR'::bpchar))), (count(DISTINCT l_suppkey)) + -> Sort + Output: l_orderkey, (count(DISTINCT l_suppkey) FILTER (WHERE (l_shipmode = 'AIR'::bpchar))), (count(DISTINCT l_suppkey)) + Sort Key: (count(DISTINCT lineitem_hash.l_suppkey) FILTER (WHERE (lineitem_hash.l_shipmode = 'AIR'::bpchar))) DESC, (count(DISTINCT lineitem_hash.l_suppkey)) DESC, lineitem_hash.l_orderkey + -> GroupAggregate + Output: l_orderkey, count(DISTINCT l_suppkey) FILTER (WHERE (l_shipmode = 'AIR'::bpchar)), count(DISTINCT l_suppkey) + Group Key: lineitem_hash.l_orderkey + -> Index Scan using lineitem_hash_pkey_240000 on public.lineitem_hash_240000 lineitem_hash + Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment +(21 rows) + +-- group by on non-partition column +SELECT + l_suppkey, count(DISTINCT l_partkey) FILTER (WHERE l_shipmode = 'AIR') + FROM lineitem_hash + GROUP BY l_suppkey + ORDER BY 2 DESC, 1 DESC + LIMIT 10; + l_suppkey | count +--------------------------------------------------------------------- + 7680 | 4 + 7703 | 3 + 7542 | 3 + 7072 | 3 + 6335 | 3 + 5873 | 3 + 1318 | 3 + 1042 | 3 + 160 | 3 + 9872 | 2 +(10 rows) + +-- explaining the same query fails +EXPLAIN (COSTS false, VERBOSE true) +SELECT + l_suppkey, count(DISTINCT l_partkey) FILTER (WHERE l_shipmode = 'AIR') + FROM lineitem_hash + GROUP BY l_suppkey + ORDER BY 2 DESC, 1 DESC + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + Output: remote_scan.l_suppkey, (count(DISTINCT remote_scan.count) FILTER (WHERE (remote_scan.count_1 = 'AIR'::bpchar))) + -> Sort + Output: remote_scan.l_suppkey, (count(DISTINCT remote_scan.count) FILTER (WHERE (remote_scan.count_1 = 'AIR'::bpchar))) + Sort Key: (count(DISTINCT remote_scan.count) FILTER (WHERE (remote_scan.count_1 = 'AIR'::bpchar))) DESC, remote_scan.l_suppkey DESC + -> GroupAggregate + Output: remote_scan.l_suppkey, count(DISTINCT remote_scan.count) FILTER (WHERE (remote_scan.count_1 = 'AIR'::bpchar)) + Group Key: remote_scan.l_suppkey + -> Sort + Output: remote_scan.l_suppkey, remote_scan.count, remote_scan.count_1 + Sort Key: remote_scan.l_suppkey DESC + -> Custom Scan (Citus Task-Tracker) + Output: remote_scan.l_suppkey, remote_scan.count, remote_scan.count_1 + Task Count: 8 + Tasks Shown: One of 8 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Output: l_suppkey, l_partkey, l_shipmode + Group Key: lineitem_hash.l_suppkey, lineitem_hash.l_partkey, lineitem_hash.l_shipmode + -> Seq Scan on public.lineitem_hash_240000 lineitem_hash + Output: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment +(22 rows) + +-- without group by, on partition column +SELECT + count(DISTINCT l_orderkey) FILTER (WHERE l_shipmode = 'AIR') + FROM lineitem_hash; + count +--------------------------------------------------------------------- + 1327 +(1 row) + +-- without group by, on non-partition column +SELECT + count(DISTINCT l_partkey) FILTER (WHERE l_shipmode = 'AIR') + FROM lineitem_hash; + count +--------------------------------------------------------------------- + 1702 +(1 row) + +SELECT + count(DISTINCT l_partkey) FILTER (WHERE l_shipmode = 'AIR'), + count(DISTINCT l_partkey), + count(DISTINCT l_shipdate) + FROM lineitem_hash; + count | count | count +--------------------------------------------------------------------- + 1702 | 11661 | 2470 +(1 row) + +-- filter column already exists in target list +SELECT * + FROM ( + SELECT + l_orderkey, count(DISTINCT l_partkey) FILTER (WHERE l_orderkey > 100) + FROM lineitem_hash + GROUP BY l_orderkey) sub + ORDER BY 2 DESC, 1 DESC + LIMIT 10; + l_orderkey | count +--------------------------------------------------------------------- + 14885 | 7 + 14884 | 7 + 14821 | 7 + 14790 | 7 + 14785 | 7 + 14755 | 7 + 14725 | 7 + 14694 | 7 + 14627 | 7 + 14624 | 7 +(10 rows) + +-- filter column does not exist in target list +SELECT * + FROM ( + SELECT + l_orderkey, count(DISTINCT l_partkey) FILTER (WHERE l_shipmode = 'AIR') + FROM lineitem_hash + GROUP BY l_orderkey) sub + ORDER BY 2 DESC, 1 DESC + LIMIT 10; + l_orderkey | count +--------------------------------------------------------------------- + 12005 | 4 + 5409 | 4 + 4964 | 4 + 14848 | 3 + 14496 | 3 + 13473 | 3 + 13122 | 3 + 12929 | 3 + 12645 | 3 + 12417 | 3 +(10 rows) + +-- case expr in count distinct is supported. +-- count orders partkeys if l_shipmode is air +SELECT * + FROM ( + SELECT + l_orderkey, count(DISTINCT CASE WHEN l_shipmode = 'AIR' THEN l_partkey ELSE NULL END) as count + FROM lineitem_hash + GROUP BY l_orderkey) sub + WHERE count > 0 + ORDER BY 2 DESC, 1 DESC + LIMIT 10; + l_orderkey | count +--------------------------------------------------------------------- + 12005 | 4 + 5409 | 4 + 4964 | 4 + 14848 | 3 + 14496 | 3 + 13473 | 3 + 13122 | 3 + 12929 | 3 + 12645 | 3 + 12417 | 3 +(10 rows) + +-- text like operator is also supported +SELECT * + FROM ( + SELECT + l_orderkey, count(DISTINCT CASE WHEN l_shipmode like '%A%' THEN l_partkey ELSE NULL END) as count + FROM lineitem_hash + GROUP BY l_orderkey) sub + WHERE count > 0 + ORDER BY 2 DESC, 1 DESC + LIMIT 10; + l_orderkey | count +--------------------------------------------------------------------- + 14275 | 7 + 14181 | 7 + 13605 | 7 + 12707 | 7 + 12384 | 7 + 11746 | 7 + 10727 | 7 + 10467 | 7 + 5636 | 7 + 4614 | 7 +(10 rows) + +-- count distinct is rejected if it does not reference any columns +SELECT * + FROM ( + SELECT + l_linenumber, count(DISTINCT 1) + FROM lineitem_hash + GROUP BY l_linenumber) sub + ORDER BY 2 DESC, 1 DESC + LIMIT 10; +ERROR: cannot compute aggregate (distinct) +DETAIL: aggregate (distinct) with no columns is unsupported +HINT: You can load the hll extension from contrib packages and enable distinct approximations. +-- count distinct is rejected if it does not reference any columns +SELECT * + FROM ( + SELECT + l_linenumber, count(DISTINCT (random() * 5)::int) + FROM lineitem_hash + GROUP BY l_linenumber) sub + ORDER BY 2 DESC, 1 DESC + LIMIT 10; +ERROR: cannot compute aggregate (distinct) +DETAIL: aggregate (distinct) with no columns is unsupported +HINT: You can load the hll extension from contrib packages and enable distinct approximations. +-- even non-const function calls are supported within count distinct +SELECT * + FROM ( + SELECT + l_orderkey, count(DISTINCT (random() * 5)::int = l_linenumber) + FROM lineitem_hash + GROUP BY l_orderkey) sub + ORDER BY 2 DESC, 1 DESC + LIMIT 0; + l_orderkey | count +--------------------------------------------------------------------- +(0 rows) + +-- multiple nested subquery +SELECT + total, + avg(avg_count) as total_avg_count + FROM ( + SELECT + number_sum, + count(DISTINCT l_suppkey) as total, + avg(total_count) avg_count + FROM ( + SELECT + l_suppkey, + sum(l_linenumber) as number_sum, + count(DISTINCT l_shipmode) as total_count + FROM + lineitem_hash + WHERE + l_partkey > 100 and + l_quantity > 2 and + l_orderkey < 10000 + GROUP BY + l_suppkey) as distributed_table + WHERE + number_sum >= 10 + GROUP BY + number_sum) as distributed_table_2 + GROUP BY + total + ORDER BY + total_avg_count DESC; + total | total_avg_count +--------------------------------------------------------------------- + 1 | 3.6000000000000000 + 6 | 2.8333333333333333 + 10 | 2.6000000000000000 + 27 | 2.5555555555555556 + 32 | 2.4687500000000000 + 77 | 2.1948051948051948 + 57 | 2.1754385964912281 +(7 rows) + +-- multiple cases query +SELECT * + FROM ( + SELECT + count(DISTINCT + CASE + WHEN l_shipmode = 'TRUCK' THEN l_partkey + WHEN l_shipmode = 'AIR' THEN l_quantity + WHEN l_shipmode = 'SHIP' THEN l_discount + ELSE l_suppkey + END) as count, + l_shipdate + FROM + lineitem_hash + GROUP BY + l_shipdate) sub + WHERE + count > 0 + ORDER BY + 1 DESC, 2 DESC + LIMIT 10; + count | l_shipdate +--------------------------------------------------------------------- + 14 | 07-30-1997 + 13 | 05-26-1998 + 13 | 08-08-1997 + 13 | 11-17-1995 + 13 | 01-09-1993 + 12 | 01-15-1998 + 12 | 10-15-1997 + 12 | 09-07-1997 + 12 | 06-02-1997 + 12 | 03-14-1997 +(10 rows) + +-- count DISTINCT expression +SELECT * + FROM ( + SELECT + l_quantity, count(DISTINCT ((l_orderkey / 1000) * 1000 )) as count + FROM + lineitem_hash + GROUP BY + l_quantity) sub + WHERE + count > 0 + ORDER BY + 2 DESC, 1 DESC + LIMIT 10; + l_quantity | count +--------------------------------------------------------------------- + 48.00 | 13 + 47.00 | 13 + 37.00 | 13 + 33.00 | 13 + 26.00 | 13 + 25.00 | 13 + 23.00 | 13 + 21.00 | 13 + 15.00 | 13 + 12.00 | 13 +(10 rows) + +-- count DISTINCT is part of an expression which includes another aggregate +SELECT * + FROM ( + SELECT + sum(((l_partkey * l_tax) / 100)) / + count(DISTINCT + CASE + WHEN l_shipmode = 'TRUCK' THEN l_partkey + ELSE l_suppkey + END) as avg, + l_shipmode + FROM + lineitem_hash + GROUP BY + l_shipmode) sub + ORDER BY + 1 DESC, 2 DESC + LIMIT 10; + avg | l_shipmode +--------------------------------------------------------------------- + 44.82904609027336300064 | MAIL + 44.80704536679536679537 | SHIP + 44.68891732736572890026 | AIR + 44.34106724470134874759 | REG AIR + 43.12739987269255251432 | FOB + 43.07299253636938646426 | RAIL + 40.50298377916903813318 | TRUCK +(7 rows) + +-- count DISTINCT CASE WHEN expression +SELECT * + FROM ( + SELECT + count(DISTINCT + CASE + WHEN l_shipmode = 'TRUCK' THEN l_linenumber + WHEN l_shipmode = 'AIR' THEN l_linenumber + 10 + ELSE 2 + END) as avg + FROM + lineitem_hash + GROUP BY l_shipdate) sub + ORDER BY 1 DESC + LIMIT 10; + avg +--------------------------------------------------------------------- + 7 + 6 + 6 + 6 + 6 + 6 + 6 + 6 + 5 + 5 +(10 rows) + +-- COUNT DISTINCT (c1, c2) +SELECT * + FROM + (SELECT + l_shipmode, + count(DISTINCT (l_shipdate, l_tax)) + FROM + lineitem_hash + GROUP BY + l_shipmode) t + ORDER BY + 2 DESC,1 DESC + LIMIT 10; + l_shipmode | count +--------------------------------------------------------------------- + TRUCK | 1689 + MAIL | 1683 + FOB | 1655 + AIR | 1650 + SHIP | 1644 + RAIL | 1636 + REG AIR | 1607 +(7 rows) + +-- distinct on non-var (type cast/field select) columns are also +-- supported if grouped on distribution column +-- random is added to prevent flattening by postgresql +SELECT + l_orderkey, count(a::int), count(distinct a::int) + FROM ( + SELECT l_orderkey, l_orderkey * 1.5 a, random() b + FROM lineitem_hash) sub + GROUP BY 1 + ORDER BY 1 DESC + LIMIT 5; + l_orderkey | count | count +--------------------------------------------------------------------- + 14947 | 2 | 1 + 14946 | 2 | 1 + 14945 | 6 | 1 + 14944 | 2 | 1 + 14919 | 1 | 1 +(5 rows) + +SELECT user_id, + count(sub.a::int), + count(DISTINCT sub.a::int), + count(DISTINCT (sub).a) +FROM + (SELECT user_id, + unnest(ARRAY[user_id * 1.5])a, + random() b + FROM users_table + ) sub +GROUP BY 1 +ORDER BY 1 DESC +LIMIT 5; + user_id | count | count | count +--------------------------------------------------------------------- + 6 | 11 | 1 | 1 + 5 | 27 | 1 | 1 + 4 | 24 | 1 | 1 + 3 | 18 | 1 | 1 + 2 | 19 | 1 | 1 +(5 rows) + +CREATE TYPE test_item AS +( + id INTEGER, + duration INTEGER +); +CREATE TABLE test_count_distinct_array (key int, value int , value_arr test_item[]); +SELECT create_distributed_table('test_count_distinct_array', 'key'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO test_count_distinct_array SELECT i, i, ARRAY[(i,i)::test_item] FROM generate_Series(0, 1000) i; +SELECT + key, + count(DISTINCT value), + count(DISTINCT (item)."id"), + count(DISTINCT (item)."id" * 3) +FROM + ( + SELECT key, unnest(value_arr) as item, value FROM test_count_distinct_array + ) as sub +GROUP BY 1 +ORDER BY 1 DESC +LIMIT 5; + key | count | count | count +--------------------------------------------------------------------- + 1000 | 1 | 1 | 1 + 999 | 1 | 1 | 1 + 998 | 1 | 1 | 1 + 997 | 1 | 1 | 1 + 996 | 1 | 1 | 1 +(5 rows) + +DROP TABLE test_count_distinct_array; +DROP TYPE test_item; +-- other distinct aggregate are not supported +SELECT * + FROM ( + SELECT + l_linenumber, sum(DISTINCT l_partkey) + FROM lineitem_hash + GROUP BY l_linenumber) sub + ORDER BY 2 DESC, 1 DESC + LIMIT 10; +ERROR: cannot compute aggregate (distinct) +DETAIL: Only count(distinct) aggregate is supported in subqueries +SELECT * + FROM ( + SELECT + l_linenumber, avg(DISTINCT l_partkey) + FROM lineitem_hash + GROUP BY l_linenumber) sub + ORDER BY 2 DESC, 1 DESC + LIMIT 10; +ERROR: cannot compute aggregate (distinct) +DETAIL: Only count(distinct) aggregate is supported in subqueries +-- whole row references, oid, and ctid are not supported in count distinct +-- test table does not have oid or ctid enabled, so tests for them are skipped +SELECT * + FROM ( + SELECT + l_linenumber, count(DISTINCT lineitem_hash) + FROM lineitem_hash + GROUP BY l_linenumber) sub + ORDER BY 2 DESC, 1 DESC + LIMIT 10; +ERROR: cannot compute count (distinct) +DETAIL: Non-column references are not supported yet +SELECT * + FROM ( + SELECT + l_linenumber, count(DISTINCT lineitem_hash.*) + FROM lineitem_hash + GROUP BY l_linenumber) sub + ORDER BY 2 DESC, 1 DESC + LIMIT 10; +ERROR: cannot compute count (distinct) +DETAIL: Non-column references are not supported yet +-- count distinct pushdown is enabled +SELECT * + FROM ( + SELECT + l_shipdate, + count(DISTINCT + CASE + WHEN l_shipmode = 'TRUCK' THEN l_partkey + ELSE NULL + END) as distinct_part, + extract(year from l_shipdate) as year + FROM + lineitem_hash + GROUP BY l_shipdate, year) sub + WHERE year = 1995 + ORDER BY 2 DESC, 1 + LIMIT 10; + l_shipdate | distinct_part | year +--------------------------------------------------------------------- + 11-29-1995 | 5 | 1995 + 03-24-1995 | 4 | 1995 + 09-18-1995 | 4 | 1995 + 01-17-1995 | 3 | 1995 + 04-02-1995 | 3 | 1995 + 05-23-1995 | 3 | 1995 + 08-11-1995 | 3 | 1995 + 09-27-1995 | 3 | 1995 + 10-27-1995 | 3 | 1995 + 10-30-1995 | 3 | 1995 +(10 rows) + +RESET citus.task_executor_type; +-- count distinct pushdown is enabled +SELECT * + FROM ( + SELECT + l_shipdate, + count(DISTINCT + CASE + WHEN l_shipmode = 'TRUCK' THEN l_partkey + ELSE NULL + END) as distinct_part, + extract(year from l_shipdate) as year + FROM + lineitem_hash + GROUP BY l_shipdate, year) sub + WHERE year = 1995 + ORDER BY 2 DESC, 1 + LIMIT 10; + l_shipdate | distinct_part | year +--------------------------------------------------------------------- + 11-29-1995 | 5 | 1995 + 03-24-1995 | 4 | 1995 + 09-18-1995 | 4 | 1995 + 01-17-1995 | 3 | 1995 + 04-02-1995 | 3 | 1995 + 05-23-1995 | 3 | 1995 + 08-11-1995 | 3 | 1995 + 09-27-1995 | 3 | 1995 + 10-27-1995 | 3 | 1995 + 10-30-1995 | 3 | 1995 +(10 rows) + +SELECT * + FROM ( + SELECT + l_shipdate, + count(DISTINCT + CASE + WHEN l_shipmode = 'TRUCK' THEN l_partkey + ELSE NULL + END) as distinct_part, + extract(year from l_shipdate) as year + FROM + lineitem_hash + GROUP BY l_shipdate) sub + WHERE year = 1995 + ORDER BY 2 DESC, 1 + LIMIT 10; + l_shipdate | distinct_part | year +--------------------------------------------------------------------- + 11-29-1995 | 5 | 1995 + 03-24-1995 | 4 | 1995 + 09-18-1995 | 4 | 1995 + 01-17-1995 | 3 | 1995 + 04-02-1995 | 3 | 1995 + 05-23-1995 | 3 | 1995 + 08-11-1995 | 3 | 1995 + 09-27-1995 | 3 | 1995 + 10-27-1995 | 3 | 1995 + 10-30-1995 | 3 | 1995 +(10 rows) + +DROP TABLE lineitem_hash; diff --git a/src/test/regress/expected/multi_select_distinct.out b/src/test/regress/expected/multi_select_distinct.out index 506ce215f..bfb189095 100644 --- a/src/test/regress/expected/multi_select_distinct.out +++ b/src/test/regress/expected/multi_select_distinct.out @@ -946,7 +946,7 @@ EXPLAIN (COSTS FALSE) GROUP BY l_orderkey ORDER BY 2 LIMIT 15; - QUERY PLAN + QUERY PLAN --------------------------------------------------------------------- Limit -> Sort @@ -958,12 +958,10 @@ EXPLAIN (COSTS FALSE) Tasks Shown: One of 4 -> Task Node: host=localhost port=xxxxx dbname=regression - -> GroupAggregate + -> HashAggregate Group Key: l_orderkey - -> Sort - Sort Key: l_orderkey - -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part -(15 rows) + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(13 rows) -- check the plan if the hash aggreate is disabled. SET enable_hashagg TO off; @@ -973,7 +971,7 @@ EXPLAIN (COSTS FALSE) GROUP BY l_orderkey ORDER BY 2 LIMIT 15; - QUERY PLAN + QUERY PLAN --------------------------------------------------------------------- Limit -> Unique @@ -984,12 +982,10 @@ EXPLAIN (COSTS FALSE) Tasks Shown: One of 4 -> Task Node: host=localhost port=xxxxx dbname=regression - -> GroupAggregate + -> HashAggregate Group Key: l_orderkey - -> Sort - Sort Key: l_orderkey - -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part -(14 rows) + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(12 rows) SET enable_hashagg TO on; -- distinct on non-partition column with aggregate diff --git a/src/test/regress/expected/multi_select_distinct_1.out b/src/test/regress/expected/multi_select_distinct_1.out new file mode 100644 index 000000000..506ce215f --- /dev/null +++ b/src/test/regress/expected/multi_select_distinct_1.out @@ -0,0 +1,1569 @@ +-- +-- MULTI_SELECT_DISTINCT +-- +-- Tests select distinct, and select distinct on features. +-- +ANALYZE lineitem_hash_part; +-- function calls are supported +SELECT DISTINCT l_orderkey, now() FROM lineitem_hash_part LIMIT 0; + l_orderkey | now +--------------------------------------------------------------------- +(0 rows) + +SELECT DISTINCT l_orderkey, avg(l_linenumber) +FROM lineitem_hash_part +GROUP BY l_orderkey +HAVING avg(l_linenumber) = (select avg(distinct l_linenumber)) +LIMIT 10; +ERROR: Subqueries in HAVING cannot refer to outer query +SELECT DISTINCT l_orderkey +FROM lineitem_hash_part +GROUP BY l_orderkey +HAVING (select avg(distinct l_linenumber) = l_orderkey) +LIMIT 10; +ERROR: Subqueries in HAVING cannot refer to outer query +SELECT DISTINCT l_partkey, 1 + (random() * 0)::int FROM lineitem_hash_part ORDER BY 1 DESC LIMIT 3; + l_partkey | ?column? +--------------------------------------------------------------------- + 199973 | 1 + 199946 | 1 + 199943 | 1 +(3 rows) + +-- const expressions are supported +SELECT DISTINCT l_orderkey, 1+1 FROM lineitem_hash_part ORDER BY 1 LIMIT 5; + l_orderkey | ?column? +--------------------------------------------------------------------- + 1 | 2 + 2 | 2 + 3 | 2 + 4 | 2 + 5 | 2 +(5 rows) + +-- non const expressions are also supported +SELECT DISTINCT l_orderkey, l_partkey + 1 FROM lineitem_hash_part ORDER BY 1, 2 LIMIT 5; + l_orderkey | ?column? +--------------------------------------------------------------------- + 1 | 2133 + 1 | 15636 + 1 | 24028 + 1 | 63701 + 1 | 67311 +(5 rows) + +-- column expressions are supported +SELECT DISTINCT l_orderkey, l_shipinstruct || l_shipmode FROM lineitem_hash_part ORDER BY 2 , 1 LIMIT 5; + l_orderkey | ?column? +--------------------------------------------------------------------- + 32 | COLLECT CODAIR + 39 | COLLECT CODAIR + 66 | COLLECT CODAIR + 70 | COLLECT CODAIR + 98 | COLLECT CODAIR +(5 rows) + +-- function calls with const input are supported +SELECT DISTINCT l_orderkey, strpos('AIR', 'A') FROM lineitem_hash_part ORDER BY 1,2 LIMIT 5; + l_orderkey | strpos +--------------------------------------------------------------------- + 1 | 1 + 2 | 1 + 3 | 1 + 4 | 1 + 5 | 1 +(5 rows) + +-- function calls with non-const input are supported +SELECT DISTINCT l_orderkey, strpos(l_shipmode, 'I') + FROM lineitem_hash_part + WHERE strpos(l_shipmode, 'I') > 1 + ORDER BY 2, 1 + LIMIT 5; + l_orderkey | strpos +--------------------------------------------------------------------- + 1 | 2 + 3 | 2 + 5 | 2 + 32 | 2 + 33 | 2 +(5 rows) + +-- row types are supported +SELECT DISTINCT (l_orderkey, l_partkey) AS pair FROM lineitem_hash_part ORDER BY 1 LIMIT 5; + pair +--------------------------------------------------------------------- + (1,2132) + (1,15635) + (1,24027) + (1,63700) + (1,67310) +(5 rows) + +-- distinct on partition column +-- verify counts match with respect to count(distinct) +CREATE TEMP TABLE temp_orderkeys AS SELECT DISTINCT l_orderkey FROM lineitem_hash_part; +SELECT COUNT(*) FROM temp_orderkeys; + count +--------------------------------------------------------------------- + 2985 +(1 row) + +SELECT COUNT(DISTINCT l_orderkey) FROM lineitem_hash_part; + count +--------------------------------------------------------------------- + 2985 +(1 row) + +SELECT DISTINCT l_orderkey FROM lineitem_hash_part WHERE l_orderkey < 500 and l_partkey < 5000 order by 1; + l_orderkey +--------------------------------------------------------------------- + 1 + 3 + 32 + 35 + 39 + 65 + 129 + 130 + 134 + 164 + 194 + 228 + 261 + 290 + 320 + 321 + 354 + 418 +(18 rows) + +-- distinct on non-partition column +SELECT DISTINCT l_partkey FROM lineitem_hash_part WHERE l_orderkey > 5 and l_orderkey < 20 order by 1; + l_partkey +--------------------------------------------------------------------- + 79251 + 94780 + 139636 + 145243 + 151894 + 157238 + 163073 + 182052 +(8 rows) + +SELECT DISTINCT l_shipmode FROM lineitem_hash_part ORDER BY 1 DESC; + l_shipmode +--------------------------------------------------------------------- + TRUCK + SHIP + REG AIR + RAIL + MAIL + FOB + AIR +(7 rows) + +-- distinct with multiple columns +SELECT DISTINCT l_orderkey, o_orderdate + FROM lineitem_hash_part JOIN orders_hash_part ON (l_orderkey = o_orderkey) + WHERE l_orderkey < 10 + ORDER BY l_orderkey; + l_orderkey | o_orderdate +--------------------------------------------------------------------- + 1 | 01-02-1996 + 2 | 12-01-1996 + 3 | 10-14-1993 + 4 | 10-11-1995 + 5 | 07-30-1994 + 6 | 02-21-1992 + 7 | 01-10-1996 +(7 rows) + +-- distinct on partition column with aggregate +-- this is the same as the one without distinct due to group by +SELECT DISTINCT l_orderkey, count(*) + FROM lineitem_hash_part + WHERE l_orderkey < 200 + GROUP BY 1 + HAVING count(*) > 5 + ORDER BY 2 DESC, 1; + l_orderkey | count +--------------------------------------------------------------------- + 7 | 7 + 68 | 7 + 129 | 7 + 164 | 7 + 194 | 7 + 1 | 6 + 3 | 6 + 32 | 6 + 35 | 6 + 39 | 6 + 67 | 6 + 69 | 6 + 70 | 6 + 71 | 6 + 134 | 6 + 135 | 6 + 163 | 6 + 192 | 6 + 197 | 6 +(19 rows) + +-- explain the query to see actual plan +EXPLAIN (COSTS FALSE) + SELECT DISTINCT l_orderkey, count(*) + FROM lineitem_hash_part + WHERE l_orderkey < 200 + GROUP BY 1 + HAVING count(*) > 5 + ORDER BY 2 DESC, 1; + QUERY PLAN +--------------------------------------------------------------------- + Sort + Sort Key: remote_scan.count DESC, remote_scan.l_orderkey + -> HashAggregate + Group Key: remote_scan.count, remote_scan.l_orderkey + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Group Key: l_orderkey + Filter: (count(*) > 5) + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part + Filter: (l_orderkey < 200) +(14 rows) + +-- check the plan if the hash aggreate is disabled +SET enable_hashagg TO off; +EXPLAIN (COSTS FALSE) + SELECT DISTINCT l_orderkey, count(*) + FROM lineitem_hash_part + WHERE l_orderkey < 200 + GROUP BY 1 + HAVING count(*) > 5 + ORDER BY 2 DESC, 1; + QUERY PLAN +--------------------------------------------------------------------- + Unique + -> Sort + Sort Key: remote_scan.count DESC, remote_scan.l_orderkey + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Group Key: l_orderkey + Filter: (count(*) > 5) + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part + Filter: (l_orderkey < 200) +(13 rows) + +SET enable_hashagg TO on; +-- distinct on aggregate of group by columns, we try to check whether we handle +-- queries which does not have any group by column in distinct columns properly. +SELECT DISTINCT count(*) + FROM lineitem_hash_part + GROUP BY l_suppkey, l_linenumber + ORDER BY 1; + count +--------------------------------------------------------------------- + 1 + 2 + 3 + 4 +(4 rows) + +-- explain the query to see actual plan. We expect to see Aggregate node having +-- group by key on count(*) column, since columns in the Group By doesn't guarantee +-- the uniqueness of the result. +EXPLAIN (COSTS FALSE) + SELECT DISTINCT count(*) + FROM lineitem_hash_part + GROUP BY l_suppkey, l_linenumber + ORDER BY 1; + QUERY PLAN +--------------------------------------------------------------------- + Unique + -> Sort + Sort Key: (COALESCE((pg_catalog.sum(remote_scan.count))::bigint, '0'::bigint)) + -> HashAggregate + Group Key: remote_scan.worker_column_2, remote_scan.worker_column_3 + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Group Key: l_suppkey, l_linenumber + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(13 rows) + +-- check the plan if the hash aggreate is disabled. We expect to see sort+unique +-- instead of aggregate plan node to handle distinct. +SET enable_hashagg TO off; +EXPLAIN (COSTS FALSE) + SELECT DISTINCT count(*) + FROM lineitem_hash_part + GROUP BY l_suppkey, l_linenumber + ORDER BY 1; + QUERY PLAN +--------------------------------------------------------------------- + Unique + -> Sort + Sort Key: (COALESCE((pg_catalog.sum(remote_scan.count))::bigint, '0'::bigint)) + -> GroupAggregate + Group Key: remote_scan.worker_column_2, remote_scan.worker_column_3 + -> Sort + Sort Key: remote_scan.worker_column_2, remote_scan.worker_column_3 + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Group Key: l_suppkey, l_linenumber + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(15 rows) + +SET enable_hashagg TO on; +-- Now we have only part of group clause columns in distinct, yet it is still not +-- enough to use Group By columns to guarantee uniqueness of result list. +SELECT DISTINCT l_suppkey, count(*) + FROM lineitem_hash_part + GROUP BY l_suppkey, l_linenumber + ORDER BY 1 + LIMIT 10; + l_suppkey | count +--------------------------------------------------------------------- + 1 | 1 + 2 | 1 + 3 | 1 + 4 | 1 + 5 | 1 + 7 | 1 + 10 | 1 + 12 | 1 + 13 | 1 + 14 | 1 +(10 rows) + +-- explain the query to see actual plan. Similar to the explain of the query above. +EXPLAIN (COSTS FALSE) + SELECT DISTINCT l_suppkey, count(*) + FROM lineitem_hash_part + GROUP BY l_suppkey, l_linenumber + ORDER BY 1 + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Unique + -> Sort + Sort Key: remote_scan.l_suppkey, (COALESCE((pg_catalog.sum(remote_scan.count))::bigint, '0'::bigint)) + -> HashAggregate + Group Key: remote_scan.l_suppkey, remote_scan.worker_column_3 + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Group Key: l_suppkey, l_linenumber + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(14 rows) + +-- check the plan if the hash aggreate is disabled. Similar to the explain of +-- the query above. +SET enable_hashagg TO off; +EXPLAIN (COSTS FALSE) + SELECT DISTINCT l_suppkey, count(*) + FROM lineitem_hash_part + GROUP BY l_suppkey, l_linenumber + ORDER BY 1 + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Unique + -> Sort + Sort Key: remote_scan.l_suppkey, (COALESCE((pg_catalog.sum(remote_scan.count))::bigint, '0'::bigint)) + -> GroupAggregate + Group Key: remote_scan.l_suppkey, remote_scan.worker_column_3 + -> Sort + Sort Key: remote_scan.l_suppkey, remote_scan.worker_column_3 + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Group Key: l_suppkey, l_linenumber + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(16 rows) + +SET enable_hashagg TO on; +-- Similar to the above query, not with count but avg. Only difference with the +-- above query is that, we create run two aggregate functions in workers. +SELECT DISTINCT l_suppkey, avg(l_partkey) + FROM lineitem_hash_part + GROUP BY l_suppkey, l_linenumber + ORDER BY 1,2 + LIMIT 10; + l_suppkey | avg +--------------------------------------------------------------------- + 1 | 190000.000000000000 + 2 | 172450.000000000000 + 3 | 112469.000000000000 + 3 | 134976.000000000000 + 4 | 112470.000000000000 + 4 | 142461.000000000000 + 5 | 182450.000000000000 + 7 | 137493.000000000000 + 10 | 150009.000000000000 + 12 | 17510.0000000000000000 +(10 rows) + +-- explain the query to see actual plan. Similar to the explain of the query above. +-- Only aggregate functions will be changed. +EXPLAIN (COSTS FALSE) + SELECT DISTINCT l_suppkey, avg(l_partkey) + FROM lineitem_hash_part + GROUP BY l_suppkey, l_linenumber + ORDER BY 1,2 + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Unique + -> Sort + Sort Key: remote_scan.l_suppkey, ((pg_catalog.sum(remote_scan.avg) / pg_catalog.sum(remote_scan.avg_1))) + -> HashAggregate + Group Key: remote_scan.l_suppkey, remote_scan.worker_column_4 + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Group Key: l_suppkey, l_linenumber + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(14 rows) + +-- check the plan if the hash aggreate is disabled. This explain errors out due +-- to a bug right now, expectation must be corrected after fixing it. +SET enable_hashagg TO off; +EXPLAIN (COSTS FALSE) + SELECT DISTINCT l_suppkey, avg(l_partkey) + FROM lineitem_hash_part + GROUP BY l_suppkey, l_linenumber + ORDER BY 1,2 + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Unique + -> Sort + Sort Key: remote_scan.l_suppkey, ((pg_catalog.sum(remote_scan.avg) / pg_catalog.sum(remote_scan.avg_1))) + -> GroupAggregate + Group Key: remote_scan.l_suppkey, remote_scan.worker_column_4 + -> Sort + Sort Key: remote_scan.l_suppkey, remote_scan.worker_column_4 + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Group Key: l_suppkey, l_linenumber + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(16 rows) + +SET enable_hashagg TO on; +-- Similar to the above query but with distinct on +SELECT DISTINCT ON (l_suppkey) avg(l_partkey) + FROM lineitem_hash_part + GROUP BY l_suppkey, l_linenumber + ORDER BY l_suppkey,1 + LIMIT 10; + avg +--------------------------------------------------------------------- + 190000.000000000000 + 172450.000000000000 + 112469.000000000000 + 112470.000000000000 + 182450.000000000000 + 137493.000000000000 + 150009.000000000000 + 17510.0000000000000000 + 87504.000000000000 + 77506.000000000000 +(10 rows) + +-- explain the query to see actual plan. We expect to see sort+unique to handle +-- distinct on. +EXPLAIN (COSTS FALSE) + SELECT DISTINCT ON (l_suppkey) avg(l_partkey) + FROM lineitem_hash_part + GROUP BY l_suppkey, l_linenumber + ORDER BY l_suppkey,1 + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Unique + -> Sort + Sort Key: remote_scan.worker_column_3, ((pg_catalog.sum(remote_scan.avg) / pg_catalog.sum(remote_scan.avg_1))) + -> HashAggregate + Group Key: remote_scan.worker_column_3, remote_scan.worker_column_4 + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Group Key: l_suppkey, l_linenumber + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(14 rows) + +-- check the plan if the hash aggreate is disabled. We expect to see sort+unique to +-- handle distinct on. +SET enable_hashagg TO off; +EXPLAIN (COSTS FALSE) + SELECT DISTINCT ON (l_suppkey) avg(l_partkey) + FROM lineitem_hash_part + GROUP BY l_suppkey, l_linenumber + ORDER BY l_suppkey,1 + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Unique + -> Sort + Sort Key: remote_scan.worker_column_3, ((pg_catalog.sum(remote_scan.avg) / pg_catalog.sum(remote_scan.avg_1))) + -> GroupAggregate + Group Key: remote_scan.worker_column_3, remote_scan.worker_column_4 + -> Sort + Sort Key: remote_scan.worker_column_3, remote_scan.worker_column_4 + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Group Key: l_suppkey, l_linenumber + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(16 rows) + +SET enable_hashagg TO on; +-- distinct with expression and aggregation +SELECT DISTINCT avg(ceil(l_partkey / 2)) + FROM lineitem_hash_part + GROUP BY l_suppkey, l_linenumber + ORDER BY 1 + LIMIT 10; + avg +--------------------------------------------------------------------- + 9 + 39 + 74 + 87 + 89 + 91 + 97 + 102 + 111 + 122 +(10 rows) + +-- explain the query to see actual plan +EXPLAIN (COSTS FALSE) + SELECT DISTINCT avg(ceil(l_partkey / 2)) + FROM lineitem_hash_part + GROUP BY l_suppkey, l_linenumber + ORDER BY 1 + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Unique + -> Sort + Sort Key: ((sum(remote_scan.avg) / (pg_catalog.sum(remote_scan.avg_1))::double precision)) + -> HashAggregate + Group Key: remote_scan.worker_column_3, remote_scan.worker_column_4 + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Group Key: l_suppkey, l_linenumber + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(14 rows) + +-- check the plan if the hash aggreate is disabled. This explain errors out due +-- to a bug right now, expectation must be corrected after fixing it. +SET enable_hashagg TO off; +EXPLAIN (COSTS FALSE) + SELECT DISTINCT avg(ceil(l_partkey / 2)) + FROM lineitem_hash_part + GROUP BY l_suppkey, l_linenumber + ORDER BY 1 + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Unique + -> Sort + Sort Key: ((sum(remote_scan.avg) / (pg_catalog.sum(remote_scan.avg_1))::double precision)) + -> GroupAggregate + Group Key: remote_scan.worker_column_3, remote_scan.worker_column_4 + -> Sort + Sort Key: remote_scan.worker_column_3, remote_scan.worker_column_4 + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Group Key: l_suppkey, l_linenumber + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(16 rows) + +SET enable_hashagg TO on; +-- expression among aggregations. +SELECT DISTINCT sum(l_suppkey) + count(l_partkey) AS dis + FROM lineitem_hash_part + GROUP BY l_suppkey, l_linenumber + ORDER BY 1 + LIMIT 10; + dis +--------------------------------------------------------------------- + 2 + 3 + 4 + 5 + 6 + 8 + 11 + 13 + 14 + 15 +(10 rows) + +-- explain the query to see actual plan +EXPLAIN (COSTS FALSE) + SELECT DISTINCT sum(l_suppkey) + count(l_partkey) AS dis + FROM lineitem_hash_part + GROUP BY l_suppkey, l_linenumber + ORDER BY 1 + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Unique + -> Sort + Sort Key: (((pg_catalog.sum(remote_scan.dis))::bigint + COALESCE((pg_catalog.sum(remote_scan.dis_1))::bigint, '0'::bigint))) + -> HashAggregate + Group Key: remote_scan.worker_column_3, remote_scan.worker_column_4 + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Group Key: l_suppkey, l_linenumber + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(14 rows) + +-- check the plan if the hash aggreate is disabled. This explain errors out due +-- to a bug right now, expectation must be corrected after fixing it. +SET enable_hashagg TO off; +EXPLAIN (COSTS FALSE) + SELECT DISTINCT sum(l_suppkey) + count(l_partkey) AS dis + FROM lineitem_hash_part + GROUP BY l_suppkey, l_linenumber + ORDER BY 1 + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Unique + -> Sort + Sort Key: (((pg_catalog.sum(remote_scan.dis))::bigint + COALESCE((pg_catalog.sum(remote_scan.dis_1))::bigint, '0'::bigint))) + -> GroupAggregate + Group Key: remote_scan.worker_column_3, remote_scan.worker_column_4 + -> Sort + Sort Key: remote_scan.worker_column_3, remote_scan.worker_column_4 + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Group Key: l_suppkey, l_linenumber + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(16 rows) + +SET enable_hashagg TO on; +-- distinct on all columns, note Group By columns guarantees uniqueness of the +-- result list. +SELECT DISTINCT * + FROM lineitem_hash_part + GROUP BY 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16 + ORDER BY 1,2 + LIMIT 10; + l_orderkey | l_partkey | l_suppkey | l_linenumber | l_quantity | l_extendedprice | l_discount | l_tax | l_returnflag | l_linestatus | l_shipdate | l_commitdate | l_receiptdate | l_shipinstruct | l_shipmode | l_comment +--------------------------------------------------------------------- + 1 | 2132 | 4633 | 4 | 28.00 | 28955.64 | 0.09 | 0.06 | N | O | 04-21-1996 | 03-30-1996 | 05-16-1996 | NONE | AIR | lites. fluffily even de + 1 | 15635 | 638 | 6 | 32.00 | 49620.16 | 0.07 | 0.02 | N | O | 01-30-1996 | 02-07-1996 | 02-03-1996 | DELIVER IN PERSON | MAIL | arefully slyly ex + 1 | 24027 | 1534 | 5 | 24.00 | 22824.48 | 0.10 | 0.04 | N | O | 03-30-1996 | 03-14-1996 | 04-01-1996 | NONE | FOB | pending foxes. slyly re + 1 | 63700 | 3701 | 3 | 8.00 | 13309.60 | 0.10 | 0.02 | N | O | 01-29-1996 | 03-05-1996 | 01-31-1996 | TAKE BACK RETURN | REG AIR | riously. regular, express dep + 1 | 67310 | 7311 | 2 | 36.00 | 45983.16 | 0.09 | 0.06 | N | O | 04-12-1996 | 02-28-1996 | 04-20-1996 | TAKE BACK RETURN | MAIL | ly final dependencies: slyly bold + 1 | 155190 | 7706 | 1 | 17.00 | 21168.23 | 0.04 | 0.02 | N | O | 03-13-1996 | 02-12-1996 | 03-22-1996 | DELIVER IN PERSON | TRUCK | egular courts above the + 2 | 106170 | 1191 | 1 | 38.00 | 44694.46 | 0.00 | 0.05 | N | O | 01-28-1997 | 01-14-1997 | 02-02-1997 | TAKE BACK RETURN | RAIL | ven requests. deposits breach a + 3 | 4297 | 1798 | 1 | 45.00 | 54058.05 | 0.06 | 0.00 | R | F | 02-02-1994 | 01-04-1994 | 02-23-1994 | NONE | AIR | ongside of the furiously brave acco + 3 | 19036 | 6540 | 2 | 49.00 | 46796.47 | 0.10 | 0.00 | R | F | 11-09-1993 | 12-20-1993 | 11-24-1993 | TAKE BACK RETURN | RAIL | unusual accounts. eve + 3 | 29380 | 1883 | 4 | 2.00 | 2618.76 | 0.01 | 0.06 | A | F | 12-04-1993 | 01-07-1994 | 01-01-1994 | NONE | TRUCK | y. fluffily pending d +(10 rows) + +-- explain the query to see actual plan. We expect to see only one aggregation +-- node since group by columns guarantees the uniqueness. +EXPLAIN (COSTS FALSE) + SELECT DISTINCT * + FROM lineitem_hash_part + GROUP BY 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16 + ORDER BY 1,2 + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Sort + Sort Key: remote_scan.l_orderkey, remote_scan.l_partkey + -> HashAggregate + Group Key: remote_scan.l_orderkey, remote_scan.l_partkey, remote_scan.l_suppkey, remote_scan.l_linenumber, remote_scan.l_quantity, remote_scan.l_extendedprice, remote_scan.l_discount, remote_scan.l_tax, remote_scan.l_returnflag, remote_scan.l_linestatus, remote_scan.l_shipdate, remote_scan.l_commitdate, remote_scan.l_receiptdate, remote_scan.l_shipinstruct, remote_scan.l_shipmode, remote_scan.l_comment + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Limit + -> Unique + -> Group + Group Key: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment + -> Sort + Sort Key: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(17 rows) + +-- check the plan if the hash aggreate is disabled. We expect to see only one +-- aggregation node since group by columns guarantees the uniqueness. +SET enable_hashagg TO off; +EXPLAIN (COSTS FALSE) + SELECT DISTINCT * + FROM lineitem_hash_part + GROUP BY 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16 + ORDER BY 1,2 + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Unique + -> Sort + Sort Key: remote_scan.l_orderkey, remote_scan.l_partkey, remote_scan.l_suppkey, remote_scan.l_linenumber, remote_scan.l_quantity, remote_scan.l_extendedprice, remote_scan.l_discount, remote_scan.l_tax, remote_scan.l_returnflag, remote_scan.l_linestatus, remote_scan.l_shipdate, remote_scan.l_commitdate, remote_scan.l_receiptdate, remote_scan.l_shipinstruct, remote_scan.l_shipmode, remote_scan.l_comment + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Limit + -> Unique + -> Group + Group Key: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment + -> Sort + Sort Key: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(16 rows) + +SET enable_hashagg TO on; +-- distinct on count distinct +SELECT DISTINCT count(DISTINCT l_partkey), count(DISTINCT l_shipmode) + FROM lineitem_hash_part + GROUP BY l_orderkey + ORDER BY 1,2; + count | count +--------------------------------------------------------------------- + 1 | 1 + 2 | 1 + 2 | 2 + 3 | 1 + 3 | 2 + 3 | 3 + 4 | 1 + 4 | 2 + 4 | 3 + 4 | 4 + 5 | 2 + 5 | 3 + 5 | 4 + 5 | 5 + 6 | 2 + 6 | 3 + 6 | 4 + 6 | 5 + 6 | 6 + 7 | 2 + 7 | 3 + 7 | 4 + 7 | 5 + 7 | 6 + 7 | 7 +(25 rows) + +-- explain the query to see actual plan. We expect to see aggregation plan for +-- the outer distinct. +EXPLAIN (COSTS FALSE) + SELECT DISTINCT count(DISTINCT l_partkey), count(DISTINCT l_shipmode) + FROM lineitem_hash_part + GROUP BY l_orderkey + ORDER BY 1,2; + QUERY PLAN +--------------------------------------------------------------------- + Sort + Sort Key: remote_scan.count, remote_scan.count_1 + -> HashAggregate + Group Key: remote_scan.count, remote_scan.count_1 + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> GroupAggregate + Group Key: l_orderkey + -> Sort + Sort Key: l_orderkey + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(14 rows) + +-- check the plan if the hash aggreate is disabled. We expect to see sort + unique +-- plans for the outer distinct. +SET enable_hashagg TO off; +EXPLAIN (COSTS FALSE) + SELECT DISTINCT count(DISTINCT l_partkey), count(DISTINCT l_shipmode) + FROM lineitem_hash_part + GROUP BY l_orderkey + ORDER BY 1,2; + QUERY PLAN +--------------------------------------------------------------------- + Unique + -> Sort + Sort Key: remote_scan.count, remote_scan.count_1 + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> GroupAggregate + Group Key: l_orderkey + -> Sort + Sort Key: l_orderkey + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(13 rows) + +SET enable_hashagg TO on; +-- distinct on aggregation with filter and expression +SELECT DISTINCT ceil(count(case when l_partkey > 100000 THEN 1 ELSE 0 END) / 2) AS count + FROM lineitem_hash_part + GROUP BY l_suppkey + ORDER BY 1; + count +--------------------------------------------------------------------- + 0 + 1 + 2 + 3 + 4 +(5 rows) + +-- explain the query to see actual plan +EXPLAIN (COSTS FALSE) + SELECT DISTINCT ceil(count(case when l_partkey > 100000 THEN 1 ELSE 0 END) / 2) AS count + FROM lineitem_hash_part + GROUP BY l_suppkey + ORDER BY 1; + QUERY PLAN +--------------------------------------------------------------------- + Unique + -> Sort + Sort Key: (ceil(((COALESCE((pg_catalog.sum(remote_scan.count))::bigint, '0'::bigint) / 2))::double precision)) + -> HashAggregate + Group Key: remote_scan.worker_column_2 + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Group Key: l_suppkey + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(13 rows) + +-- check the plan if the hash aggreate is disabled +SET enable_hashagg TO off; +EXPLAIN (COSTS FALSE) + SELECT DISTINCT ceil(count(case when l_partkey > 100000 THEN 1 ELSE 0 END) / 2) AS count + FROM lineitem_hash_part + GROUP BY l_suppkey + ORDER BY 1; + QUERY PLAN +--------------------------------------------------------------------- + Unique + -> Sort + Sort Key: (ceil(((COALESCE((pg_catalog.sum(remote_scan.count))::bigint, '0'::bigint) / 2))::double precision)) + -> GroupAggregate + Group Key: remote_scan.worker_column_2 + -> Sort + Sort Key: remote_scan.worker_column_2 + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Group Key: l_suppkey + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(15 rows) + +SET enable_hashagg TO on; +-- explain the query to see actual plan with array_agg aggregation. +EXPLAIN (COSTS FALSE) + SELECT DISTINCT array_agg(l_linenumber), array_length(array_agg(l_linenumber), 1) + FROM lineitem_hash_part + GROUP BY l_orderkey + ORDER BY 2 + LIMIT 15; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Sort + Sort Key: remote_scan.array_length + -> HashAggregate + Group Key: remote_scan.array_length, remote_scan.array_agg + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> GroupAggregate + Group Key: l_orderkey + -> Sort + Sort Key: l_orderkey + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(15 rows) + +-- check the plan if the hash aggreate is disabled. +SET enable_hashagg TO off; +EXPLAIN (COSTS FALSE) + SELECT DISTINCT array_agg(l_linenumber), array_length(array_agg(l_linenumber), 1) + FROM lineitem_hash_part + GROUP BY l_orderkey + ORDER BY 2 + LIMIT 15; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Unique + -> Sort + Sort Key: remote_scan.array_length, remote_scan.array_agg + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> GroupAggregate + Group Key: l_orderkey + -> Sort + Sort Key: l_orderkey + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(14 rows) + +SET enable_hashagg TO on; +-- distinct on non-partition column with aggregate +-- this is the same as non-distinct version due to group by +SELECT DISTINCT l_partkey, count(*) + FROM lineitem_hash_part + GROUP BY 1 + HAVING count(*) > 2 + ORDER BY 1; + l_partkey | count +--------------------------------------------------------------------- + 1051 | 3 + 1927 | 3 + 6983 | 3 + 15283 | 3 + 87761 | 3 + 136884 | 3 + 149926 | 3 + 160895 | 3 + 177771 | 3 + 188804 | 3 + 199146 | 3 +(11 rows) + +-- explain the query to see actual plan +EXPLAIN (COSTS FALSE) + SELECT DISTINCT l_partkey, count(*) + FROM lineitem_hash_part + GROUP BY 1 + HAVING count(*) > 2 + ORDER BY 1; + QUERY PLAN +--------------------------------------------------------------------- + Unique + -> Sort + Sort Key: remote_scan.l_partkey, (COALESCE((pg_catalog.sum(remote_scan.count))::bigint, '0'::bigint)) + -> HashAggregate + Group Key: remote_scan.l_partkey + Filter: (COALESCE((pg_catalog.sum(remote_scan.worker_column_3))::bigint, '0'::bigint) > 2) + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate + Group Key: l_partkey + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(14 rows) + +-- distinct on non-partition column and avg +SELECT DISTINCT l_partkey, avg(l_linenumber) + FROM lineitem_hash_part + WHERE l_partkey < 500 + GROUP BY 1 + HAVING avg(l_linenumber) > 2 + ORDER BY 1; + l_partkey | avg +--------------------------------------------------------------------- + 18 | 7.0000000000000000 + 79 | 6.0000000000000000 + 149 | 4.5000000000000000 + 175 | 5.0000000000000000 + 179 | 6.0000000000000000 + 182 | 3.0000000000000000 + 222 | 4.0000000000000000 + 278 | 3.0000000000000000 + 299 | 7.0000000000000000 + 308 | 7.0000000000000000 + 309 | 5.0000000000000000 + 321 | 3.0000000000000000 + 337 | 6.0000000000000000 + 364 | 3.0000000000000000 + 403 | 4.0000000000000000 +(15 rows) + +-- distinct on multiple non-partition columns +SELECT DISTINCT l_partkey, l_suppkey + FROM lineitem_hash_part + WHERE l_shipmode = 'AIR' AND l_orderkey < 100 + ORDER BY 1, 2; + l_partkey | l_suppkey +--------------------------------------------------------------------- + 2132 | 4633 + 4297 | 1798 + 37531 | 35 + 44161 | 6666 + 44706 | 4707 + 67831 | 5350 + 85811 | 8320 + 94368 | 6878 + 108338 | 849 + 108570 | 8571 + 137267 | 4807 + 137469 | 9983 + 173489 | 3490 + 196156 | 1195 + 197921 | 441 +(15 rows) + +EXPLAIN (COSTS FALSE) + SELECT DISTINCT l_partkey, l_suppkey + FROM lineitem_hash_part + WHERE l_shipmode = 'AIR' AND l_orderkey < 100 + ORDER BY 1, 2; + QUERY PLAN +--------------------------------------------------------------------- + Sort + Sort Key: remote_scan.l_partkey, remote_scan.l_suppkey + -> HashAggregate + Group Key: remote_scan.l_partkey, remote_scan.l_suppkey + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Unique + -> Sort + Sort Key: l_partkey, l_suppkey + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part + Filter: ((l_orderkey < 100) AND (l_shipmode = 'AIR'::bpchar)) +(14 rows) + +-- distinct on partition column +SELECT DISTINCT ON (l_orderkey) l_orderkey, l_partkey, l_suppkey + FROM lineitem_hash_part + WHERE l_orderkey < 35 + ORDER BY 1, 2, 3; + l_orderkey | l_partkey | l_suppkey +--------------------------------------------------------------------- + 1 | 2132 | 4633 + 2 | 106170 | 1191 + 3 | 4297 | 1798 + 4 | 88035 | 5560 + 5 | 37531 | 35 + 6 | 139636 | 2150 + 7 | 79251 | 1759 + 32 | 2743 | 7744 + 33 | 33918 | 3919 + 34 | 88362 | 871 +(10 rows) + +EXPLAIN (COSTS FALSE) + SELECT DISTINCT ON (l_orderkey) l_orderkey, l_partkey, l_suppkey + FROM lineitem_hash_part + WHERE l_orderkey < 35 + ORDER BY 1, 2, 3; + QUERY PLAN +--------------------------------------------------------------------- + Unique + -> Sort + Sort Key: remote_scan.l_orderkey, remote_scan.l_partkey, remote_scan.l_suppkey + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Unique + -> Sort + Sort Key: l_orderkey, l_partkey, l_suppkey + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part + Filter: (l_orderkey < 35) +(13 rows) + +-- distinct on non-partition column +-- note order by is required here +-- otherwise query results will be different since +-- distinct on clause is on non-partition column +SELECT DISTINCT ON (l_partkey) l_partkey, l_orderkey + FROM lineitem_hash_part + ORDER BY 1,2 + LIMIT 20; + l_partkey | l_orderkey +--------------------------------------------------------------------- + 18 | 12005 + 79 | 5121 + 91 | 2883 + 149 | 807 + 175 | 4102 + 179 | 2117 + 182 | 548 + 195 | 2528 + 204 | 10048 + 222 | 9413 + 245 | 9446 + 278 | 1287 + 299 | 1122 + 308 | 11137 + 309 | 2374 + 318 | 321 + 321 | 5984 + 337 | 10403 + 350 | 13698 + 358 | 4323 +(20 rows) + +EXPLAIN (COSTS FALSE) + SELECT DISTINCT ON (l_partkey) l_partkey, l_orderkey + FROM lineitem_hash_part + ORDER BY 1,2 + LIMIT 20; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Unique + -> Sort + Sort Key: remote_scan.l_partkey, remote_scan.l_orderkey + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Limit + -> Unique + -> Sort + Sort Key: l_partkey, l_orderkey + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(14 rows) + +-- distinct on with joins +-- each customer's first order key +SELECT DISTINCT ON (o_custkey) o_custkey, l_orderkey + FROM lineitem_hash_part JOIN orders_hash_part ON (l_orderkey = o_orderkey) + WHERE o_custkey < 15 + ORDER BY 1,2; + o_custkey | l_orderkey +--------------------------------------------------------------------- + 1 | 9154 + 2 | 10563 + 4 | 320 + 5 | 11682 + 7 | 10402 + 8 | 102 + 10 | 1602 + 11 | 12800 + 13 | 994 + 14 | 11011 +(10 rows) + +SELECT coordinator_plan($Q$ +EXPLAIN (COSTS FALSE) + SELECT DISTINCT ON (o_custkey) o_custkey, l_orderkey + FROM lineitem_hash_part JOIN orders_hash_part ON (l_orderkey = o_orderkey) + WHERE o_custkey < 15 + ORDER BY 1,2; +$Q$); + coordinator_plan +--------------------------------------------------------------------- + Unique + -> Sort + Sort Key: remote_scan.o_custkey, remote_scan.l_orderkey + -> Custom Scan (Citus Adaptive) + Task Count: 4 +(5 rows) + +-- explain without order by +-- notice master plan has order by on distinct on column +SELECT coordinator_plan($Q$ +EXPLAIN (COSTS FALSE) + SELECT DISTINCT ON (o_custkey) o_custkey, l_orderkey + FROM lineitem_hash_part JOIN orders_hash_part ON (l_orderkey = o_orderkey) + WHERE o_custkey < 15; +$Q$); + coordinator_plan +--------------------------------------------------------------------- + Unique + -> Sort + Sort Key: remote_scan.o_custkey + -> Custom Scan (Citus Adaptive) + Task Count: 4 +(5 rows) + +-- each customer's each order's first l_partkey +SELECT DISTINCT ON (o_custkey, l_orderkey) o_custkey, l_orderkey, l_linenumber, l_partkey + FROM lineitem_hash_part JOIN orders_hash_part ON (l_orderkey = o_orderkey) + WHERE o_custkey < 20 + ORDER BY 1,2,3; + o_custkey | l_orderkey | l_linenumber | l_partkey +--------------------------------------------------------------------- + 1 | 9154 | 1 | 86513 + 1 | 14656 | 1 | 59539 + 2 | 10563 | 1 | 147459 + 4 | 320 | 1 | 4415 + 4 | 739 | 1 | 84489 + 4 | 10688 | 1 | 45037 + 4 | 10788 | 1 | 50814 + 4 | 13728 | 1 | 86216 + 5 | 11682 | 1 | 31634 + 5 | 11746 | 1 | 180724 + 5 | 14308 | 1 | 157430 + 7 | 10402 | 1 | 53661 + 7 | 13031 | 1 | 112161 + 7 | 14145 | 1 | 138729 + 7 | 14404 | 1 | 143034 + 8 | 102 | 1 | 88914 + 8 | 164 | 1 | 91309 + 8 | 13601 | 1 | 40504 + 10 | 1602 | 1 | 182806 + 10 | 9862 | 1 | 86241 + 10 | 11431 | 1 | 62112 + 10 | 13124 | 1 | 29414 + 11 | 12800 | 1 | 152806 + 13 | 994 | 1 | 64486 + 13 | 1603 | 1 | 38191 + 13 | 4704 | 1 | 77934 + 13 | 9927 | 1 | 875 + 14 | 11011 | 1 | 172485 + 17 | 896 | 1 | 38675 + 17 | 5507 | 1 | 9600 + 19 | 353 | 1 | 119305 + 19 | 1504 | 1 | 81389 + 19 | 1669 | 1 | 78373 + 19 | 5893 | 1 | 133707 + 19 | 9954 | 1 | 92138 + 19 | 14885 | 1 | 36154 +(36 rows) + +-- explain without order by +SELECT coordinator_plan($Q$ +EXPLAIN (COSTS FALSE) + SELECT DISTINCT ON (o_custkey, l_orderkey) o_custkey, l_orderkey, l_linenumber, l_partkey + FROM lineitem_hash_part JOIN orders_hash_part ON (l_orderkey = o_orderkey) + WHERE o_custkey < 20; +$Q$); + coordinator_plan +--------------------------------------------------------------------- + Unique + -> Sort + Sort Key: remote_scan.o_custkey, remote_scan.l_orderkey + -> Custom Scan (Citus Adaptive) + Task Count: 4 +(5 rows) + +-- each customer's each order's last l_partkey +SELECT DISTINCT ON (o_custkey, l_orderkey) o_custkey, l_orderkey, l_linenumber, l_partkey + FROM lineitem_hash_part JOIN orders_hash_part ON (l_orderkey = o_orderkey) + WHERE o_custkey < 15 + ORDER BY 1,2,3 DESC; + o_custkey | l_orderkey | l_linenumber | l_partkey +--------------------------------------------------------------------- + 1 | 9154 | 7 | 173448 + 1 | 14656 | 1 | 59539 + 2 | 10563 | 4 | 110741 + 4 | 320 | 2 | 192158 + 4 | 739 | 5 | 187523 + 4 | 10688 | 2 | 132574 + 4 | 10788 | 4 | 196473 + 4 | 13728 | 3 | 12450 + 5 | 11682 | 3 | 177152 + 5 | 11746 | 7 | 193807 + 5 | 14308 | 3 | 140916 + 7 | 10402 | 2 | 64514 + 7 | 13031 | 6 | 7761 + 7 | 14145 | 6 | 130723 + 7 | 14404 | 7 | 35349 + 8 | 102 | 4 | 61158 + 8 | 164 | 7 | 3037 + 8 | 13601 | 5 | 12470 + 10 | 1602 | 1 | 182806 + 10 | 9862 | 5 | 135675 + 10 | 11431 | 7 | 8563 + 10 | 13124 | 3 | 67055 + 11 | 12800 | 5 | 179110 + 13 | 994 | 4 | 130471 + 13 | 1603 | 2 | 65209 + 13 | 4704 | 3 | 63081 + 13 | 9927 | 6 | 119356 + 14 | 11011 | 7 | 95939 +(28 rows) + +-- subqueries +SELECT DISTINCT l_orderkey, l_partkey + FROM ( + SELECT l_orderkey, l_partkey + FROM lineitem_hash_part + ) q + ORDER BY 1,2 + LIMIT 10; + l_orderkey | l_partkey +--------------------------------------------------------------------- + 1 | 2132 + 1 | 15635 + 1 | 24027 + 1 | 63700 + 1 | 67310 + 1 | 155190 + 2 | 106170 + 3 | 4297 + 3 | 19036 + 3 | 29380 +(10 rows) + +EXPLAIN (COSTS FALSE) + SELECT DISTINCT l_orderkey, l_partkey + FROM ( + SELECT l_orderkey, l_partkey + FROM lineitem_hash_part + ) q + ORDER BY 1,2 + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Sort + Sort Key: remote_scan.l_orderkey, remote_scan.l_partkey + -> HashAggregate + Group Key: remote_scan.l_orderkey, remote_scan.l_partkey + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Limit + -> Sort + Sort Key: l_orderkey, l_partkey + -> HashAggregate + Group Key: l_orderkey, l_partkey + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(16 rows) + +SELECT DISTINCT l_orderkey, cnt + FROM ( + SELECT l_orderkey, count(*) as cnt + FROM lineitem_hash_part + GROUP BY 1 + ) q + ORDER BY 1,2 + LIMIT 10; + l_orderkey | cnt +--------------------------------------------------------------------- + 1 | 6 + 2 | 1 + 3 | 6 + 4 | 1 + 5 | 3 + 6 | 1 + 7 | 7 + 32 | 6 + 33 | 4 + 34 | 3 +(10 rows) + +EXPLAIN (COSTS FALSE) + SELECT DISTINCT l_orderkey, cnt + FROM ( + SELECT l_orderkey, count(*) as cnt + FROM lineitem_hash_part + GROUP BY 1 + ) q + ORDER BY 1,2 + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Sort + Sort Key: remote_scan.l_orderkey, remote_scan.cnt + -> HashAggregate + Group Key: remote_scan.l_orderkey, remote_scan.cnt + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Limit + -> Sort + Sort Key: lineitem_hash_part.l_orderkey, (count(*)) + -> HashAggregate + Group Key: lineitem_hash_part.l_orderkey, count(*) + -> HashAggregate + Group Key: lineitem_hash_part.l_orderkey + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(18 rows) + +-- distinct on partition column +-- random() is added to inner query to prevent flattening +SELECT DISTINCT ON (l_orderkey) l_orderkey, l_partkey + FROM ( + SELECT l_orderkey, l_partkey, (random()*10)::int + 2 as r + FROM lineitem_hash_part + ) q + WHERE r > 1 + ORDER BY 1,2 + LIMIT 10; + l_orderkey | l_partkey +--------------------------------------------------------------------- + 1 | 2132 + 2 | 106170 + 3 | 4297 + 4 | 88035 + 5 | 37531 + 6 | 139636 + 7 | 79251 + 32 | 2743 + 33 | 33918 + 34 | 88362 +(10 rows) + +EXPLAIN (COSTS FALSE) + SELECT DISTINCT ON (l_orderkey) l_orderkey, l_partkey + FROM ( + SELECT l_orderkey, l_partkey, (random()*10)::int + 2 as r + FROM lineitem_hash_part + ) q + WHERE r > 1 + ORDER BY 1,2 + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Unique + -> Sort + Sort Key: remote_scan.l_orderkey, remote_scan.l_partkey + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Limit + -> Unique + -> Sort + Sort Key: q.l_orderkey, q.l_partkey + -> Subquery Scan on q + Filter: (q.r > 1) + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(16 rows) + +-- distinct on non-partition column +SELECT DISTINCT ON (l_partkey) l_orderkey, l_partkey + FROM ( + SELECT l_orderkey, l_partkey, (random()*10)::int + 2 as r + FROM lineitem_hash_part + ) q + WHERE r > 1 + ORDER BY 2,1 + LIMIT 10; + l_orderkey | l_partkey +--------------------------------------------------------------------- + 12005 | 18 + 5121 | 79 + 2883 | 91 + 807 | 149 + 4102 | 175 + 2117 | 179 + 548 | 182 + 2528 | 195 + 10048 | 204 + 9413 | 222 +(10 rows) + +EXPLAIN (COSTS FALSE) + SELECT DISTINCT ON (l_partkey) l_orderkey, l_partkey + FROM ( + SELECT l_orderkey, l_partkey, (random()*10)::int + 2 as r + FROM lineitem_hash_part + ) q + WHERE r > 1 + ORDER BY 2,1 + LIMIT 10; + QUERY PLAN +--------------------------------------------------------------------- + Limit + -> Unique + -> Sort + Sort Key: remote_scan.l_partkey, remote_scan.l_orderkey + -> Custom Scan (Citus Adaptive) + Task Count: 4 + Tasks Shown: One of 4 + -> Task + Node: host=localhost port=xxxxx dbname=regression + -> Limit + -> Unique + -> Sort + Sort Key: q.l_partkey, q.l_orderkey + -> Subquery Scan on q + Filter: (q.r > 1) + -> Seq Scan on lineitem_hash_part_360041 lineitem_hash_part +(16 rows) +