diff --git a/src/backend/distributed/planner/cte_inline.c b/src/backend/distributed/planner/cte_inline.c index 28d2a38c7..42fe06102 100644 --- a/src/backend/distributed/planner/cte_inline.c +++ b/src/backend/distributed/planner/cte_inline.c @@ -66,6 +66,9 @@ static void InlineCTEsInQueryTree(Query *query); static bool QueryTreeContainsInlinableCteWalker(Node *node); +/* controlled via GUC */ +bool EnableCTEInlining = true; + /* * RecursivelyInlineCtesInQueryTree gets a query and recursively traverses the * tree from top to bottom. On each level, the CTEs that are eligable for diff --git a/src/backend/distributed/planner/distributed_planner.c b/src/backend/distributed/planner/distributed_planner.c index f97266741..5d017c37d 100644 --- a/src/backend/distributed/planner/distributed_planner.c +++ b/src/backend/distributed/planner/distributed_planner.c @@ -727,6 +727,19 @@ static PlannedStmt * InlineCtesAndCreateDistributedPlannedStmt(uint64 planId, DistributedPlanningContext *planContext) { + if (!EnableCTEInlining) + { + /* + * In Postgres 12+, users can adjust whether to inline/not inline CTEs + * by [NOT] MATERIALIZED keywords. However, in PG 11, that's not possible. + * So, with this we provide a way to prevent CTE inlining on Postgres 11. + * + * The main use-case for this is not to have divergent test outputs between + * PG 11 vs PG 12, so not very much intended for users. + */ + return NULL; + } + /* * We'll inline the CTEs and try distributed planning, preserve the original * query in case the planning fails and we fallback to recursive planning of diff --git a/src/backend/distributed/shared_library_init.c b/src/backend/distributed/shared_library_init.c index ebdf44198..4e993225e 100644 --- a/src/backend/distributed/shared_library_init.c +++ b/src/backend/distributed/shared_library_init.c @@ -30,6 +30,7 @@ #include "distributed/commands/multi_copy.h" #include "distributed/commands/utility_hook.h" #include "distributed/connection_management.h" +#include "distributed/cte_inline.h" #include "distributed/distributed_deadlock_detection.h" #include "distributed/intermediate_result_pruning.h" #include "distributed/local_executor.h" @@ -761,6 +762,20 @@ RegisterCitusConfigVariables(void) GUC_STANDARD, NULL, NULL, NULL); + DefineCustomBoolVariable( + "citus.enable_cte_inlining", + gettext_noop("When set to false, CTE inlining feature is disabled"), + gettext_noop("This feature is not intended for users. It is developed " + "to get consistent regression test outputs between Postgres 11" + "and Postgres 12. In Postgres 12+, the user can control the behaviour" + "by [NOT] MATERIALIZED keyword on CTEs. However, in PG 11, we cannot do " + "that."), + &EnableCTEInlining, + true, + PGC_SUSET, + GUC_NO_SHOW_ALL, + NULL, NULL, NULL); + DefineCustomEnumVariable( "citus.propagate_set_commands", gettext_noop("Sets which SET commands are propagated to workers."), diff --git a/src/include/distributed/cte_inline.h b/src/include/distributed/cte_inline.h index f9fd8fa9d..09cac7bdb 100644 --- a/src/include/distributed/cte_inline.h +++ b/src/include/distributed/cte_inline.h @@ -13,6 +13,8 @@ #include "nodes/parsenodes.h" +extern bool EnableCTEInlining; + extern void RecursivelyInlineCtesInQueryTree(Query *query); extern bool QueryTreeContainsInlinableCTE(Query *queryTree); diff --git a/src/test/regress/expected/intermediate_result_pruning.out b/src/test/regress/expected/intermediate_result_pruning.out index c95fec934..3da52fee2 100644 --- a/src/test/regress/expected/intermediate_result_pruning.out +++ b/src/test/regress/expected/intermediate_result_pruning.out @@ -32,6 +32,8 @@ SELECT create_reference_table('ref_table'); (1 row) +-- prevent PG 11 - PG 12 outputs to diverge +SET citus.enable_cte_inlining TO false; -- load some data INSERT INTO table_1 VALUES (1, '1'), (2, '2'), (3, '3'), (4, '4'); INSERT INTO table_2 VALUES (3, '3'), (4, '4'), (5, '5'), (6, '6'); diff --git a/src/test/regress/expected/limit_intermediate_size.out b/src/test/regress/expected/limit_intermediate_size.out index de7c758bf..335eae25a 100644 --- a/src/test/regress/expected/limit_intermediate_size.out +++ b/src/test/regress/expected/limit_intermediate_size.out @@ -1,5 +1,7 @@ SET citus.enable_repartition_joins to ON; SET citus.task_executor_type to 'task-tracker'; +-- prevent PG 11 - PG 12 outputs to diverge +SET citus.enable_cte_inlining TO false; SET citus.max_intermediate_result_size TO 2; -- should fail because the copy size is ~4kB for each cte WITH cte AS @@ -296,4 +298,3 @@ SELECT * FROM cte ORDER BY 1,2,3,4,5 LIMIT 10; 1 | Wed Nov 22 22:51:43.132261 2017 | 4 | 1 | 2 (10 rows) -SET citus.task_executor_type to 'adaptive'; diff --git a/src/test/regress/expected/multi_explain.out b/src/test/regress/expected/multi_explain.out index 251315e31..9da8e24e0 100644 --- a/src/test/regress/expected/multi_explain.out +++ b/src/test/regress/expected/multi_explain.out @@ -1251,6 +1251,8 @@ Custom Scan (Citus INSERT ... SELECT via coordinator) -> Function Scan on generate_series s -> Function Scan on generate_series s_1 -- explain with recursive planning +-- prevent PG 11 - PG 12 outputs to diverge +SET citus.enable_cte_inlining TO false; EXPLAIN (COSTS OFF, VERBOSE true) WITH keys AS ( SELECT DISTINCT l_orderkey FROM lineitem_hash_part @@ -1300,6 +1302,7 @@ Custom Scan (Citus Adaptive) -> Function Scan on pg_catalog.read_intermediate_result intermediate_result_1 Output: intermediate_result_1.l_orderkey Function Call: read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) +SET citus.enable_cte_inlining TO true; SELECT true AS valid FROM explain_json($$ WITH result AS ( SELECT l_quantity, count(*) count_quantity FROM lineitem diff --git a/src/test/regress/expected/multi_insert_select.out b/src/test/regress/expected/multi_insert_select.out index 9f74e8588..1f504a725 100644 --- a/src/test/regress/expected/multi_insert_select.out +++ b/src/test/regress/expected/multi_insert_select.out @@ -680,7 +680,10 @@ DEBUG: Router planner cannot handle multi-shard select queries (9 rows) -- We support CTEs +-- but prefer to prevent inlining of the CTE +-- in order not to diverge from pg 11 vs pg 12 BEGIN; +SET LOCAL citus.enable_cte_inlining TO false; WITH fist_table_agg AS (SELECT max(value_1)+1 as v1_agg, user_id FROM raw_events_first GROUP BY user_id) INSERT INTO agg_events @@ -705,12 +708,10 @@ INSERT INTO agg_events raw_events_first.user_id, (SELECT * FROM sub_cte) FROM raw_events_first; +DEBUG: CTE sub_cte is going to be inlined via distributed planning DEBUG: Subqueries without relations are not allowed in distributed INSERT ... SELECT queries DEBUG: Collecting INSERT ... SELECT results on coordinator DEBUG: Router planner cannot handle multi-shard select queries -DEBUG: generating subplan XXX_1 for CTE sub_cte: SELECT 1 -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT user_id, (SELECT sub_cte."?column?" FROM (SELECT intermediate_result."?column?" FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result("?column?" integer)) sub_cte) FROM public.raw_events_first -DEBUG: Router planner cannot handle multi-shard select queries ERROR: could not run distributed query with subquery outside the FROM, WHERE and HAVING clauses HINT: Consider using an equality filter on the distributed table's partition column. -- We support set operations via the coordinator diff --git a/src/test/regress/expected/multi_insert_select_conflict.out b/src/test/regress/expected/multi_insert_select_conflict.out index e1310c46b..f543ceb6f 100644 --- a/src/test/regress/expected/multi_insert_select_conflict.out +++ b/src/test/regress/expected/multi_insert_select_conflict.out @@ -172,6 +172,8 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT col_1, col_2 10 | 0 (10 rows) +-- prevent PG 11 - PG 12 outputs to diverge +SET citus.enable_cte_inlining TO false; -- Get the select part from cte and do nothing on conflict WITH cte AS( SELECT col_1, col_2 FROM source_table_1 @@ -205,6 +207,7 @@ SELECT * FROM target_table ORDER BY 1; 10 | 0 (10 rows) +SET citus.enable_cte_inlining TO true; -- Test with multiple CTEs WITH cte AS( SELECT col_1, col_2 FROM source_table_1 @@ -214,10 +217,8 @@ WITH cte AS( INSERT INTO target_table ((SELECT * FROM cte) UNION (SELECT * FROM cte_2)) ON CONFLICT(col_1) DO UPDATE SET col_2 = EXCLUDED.col_2 + 1; DEBUG: distributed INSERT ... SELECT can only select from distributed tables DEBUG: Collecting INSERT ... SELECT results on coordinator -DEBUG: generating subplan XXX_1 for CTE cte: SELECT col_1, col_2 FROM on_conflict.source_table_1 -DEBUG: generating subplan XXX_2 for CTE cte_2: SELECT col_1, col_2 FROM on_conflict.source_table_2 -DEBUG: generating subplan XXX_3 for subquery SELECT cte.col_1, cte.col_2 FROM (SELECT intermediate_result.col_1, intermediate_result.col_2 FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(col_1 integer, col_2 integer)) cte UNION SELECT cte_2.col_1, cte_2.col_2 FROM (SELECT intermediate_result.col_1, intermediate_result.col_2 FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(col_1 integer, col_2 integer)) cte_2 -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT col_1, col_2 FROM (SELECT intermediate_result.col_1, intermediate_result.col_2 FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(col_1 integer, col_2 integer)) citus_insert_select_subquery +DEBUG: CTE cte is going to be inlined via distributed planning +DEBUG: CTE cte_2 is going to be inlined via distributed planning SELECT * FROM target_table ORDER BY 1; col_1 | col_2 --------------------------------------------------------------------- @@ -233,6 +234,8 @@ SELECT * FROM target_table ORDER BY 1; 10 | 11 (10 rows) +-- prevent PG 11 - PG 12 outputs to diverge +SET citus.enable_cte_inlining TO false; WITH inserted_table AS ( WITH cte AS( SELECT col_1, col_2, col_3 FROM source_table_1 @@ -287,6 +290,7 @@ FROM cte, source_table_1 WHERE cte.col_1 = source_table_1.col_1 ON CONFLICT DO NOTHING; ERROR: cannot perform distributed planning for the given modification DETAIL: Select query cannot be pushed down to the worker. +SET citus.enable_cte_inlining TO true; -- Tests with foreign key to reference table CREATE TABLE test_ref_table (key int PRIMARY KEY); SELECT create_reference_table('test_ref_table'); @@ -499,6 +503,8 @@ SELECT * FROM target_table ORDER BY 1; 10 | 0 (10 rows) +-- prevent PG 11 - PG 12 outputs to diverge +SET citus.enable_cte_inlining TO false; WITH cte AS( SELECT col_1, col_2, col_3 FROM source_table_1 ), cte_2 AS( diff --git a/src/test/regress/expected/multi_router_planner.out b/src/test/regress/expected/multi_router_planner.out index b20da4f38..a4db688af 100644 --- a/src/test/regress/expected/multi_router_planner.out +++ b/src/test/regress/expected/multi_router_planner.out @@ -6,6 +6,9 @@ SET citus.next_shard_id TO 840000; -- router planner, so we're disabling it in this file. We've bunch of -- other tests that triggers fast-path-router planner SET citus.enable_fast_path_router_planner TO false; +-- prevent PG 11 - PG 12 outputs to diverge +-- and CTE inlining is not relevant to router plannery anyway +SET citus.enable_cte_inlining TO false; CREATE TABLE articles_hash ( id bigint NOT NULL, author_id bigint NOT NULL, diff --git a/src/test/regress/expected/multi_task_assignment_policy.out b/src/test/regress/expected/multi_task_assignment_policy.out index 67f29166d..e44d8ee9a 100644 --- a/src/test/regress/expected/multi_task_assignment_policy.out +++ b/src/test/regress/expected/multi_task_assignment_policy.out @@ -307,6 +307,8 @@ SELECT create_distributed_table('task_assignment_test_table_2', 'test_id'); SET citus.task_assignment_policy TO 'round-robin'; -- Run the query two times to make sure that it hits two different workers -- on consecutive runs +-- prevent PG 11 - PG 12 outputs to diverge +SET citus.enable_cte_inlining TO false; INSERT INTO explain_outputs SELECT parse_explain_output($cmd$ EXPLAIN WITH q1 AS (SELECT * FROM task_assignment_test_table_2) SELECT * FROM q1 diff --git a/src/test/regress/expected/non_colocated_subquery_joins.out b/src/test/regress/expected/non_colocated_subquery_joins.out index 171781aaa..705b54bd2 100644 --- a/src/test/regress/expected/non_colocated_subquery_joins.out +++ b/src/test/regress/expected/non_colocated_subquery_joins.out @@ -542,15 +542,17 @@ SELECT count(*) FROM q1, (SELECT users_table, events_table WHERE users_table.user_id = events_table.value_2 AND event_type IN (1,2,3,4)) as bar WHERE bar.user_id = q1.user_id ;$$); -DEBUG: generating subplan XXX_1 for CTE q1: SELECT user_id FROM public.users_table -DEBUG: generating subplan XXX_2 for subquery SELECT users_table.user_id, random() AS random FROM public.users_table, public.events_table WHERE ((users_table.user_id OPERATOR(pg_catalog.=) events_table.value_2) AND (events_table.event_type OPERATOR(pg_catalog.=) ANY (ARRAY[1, 2, 3, 4]))) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.user_id FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer)) q1, (SELECT intermediate_result.user_id, intermediate_result.random FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer, random double precision)) bar WHERE (bar.user_id OPERATOR(pg_catalog.=) q1.user_id) +DEBUG: CTE q1 is going to be inlined via distributed planning +DEBUG: generating subplan XXX_1 for subquery SELECT users_table.user_id, random() AS random FROM public.users_table, public.events_table WHERE ((users_table.user_id OPERATOR(pg_catalog.=) events_table.value_2) AND (events_table.event_type OPERATOR(pg_catalog.=) ANY (ARRAY[1, 2, 3, 4]))) +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT users_table.user_id FROM public.users_table) q1, (SELECT intermediate_result.user_id, intermediate_result.random FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer, random double precision)) bar WHERE (bar.user_id OPERATOR(pg_catalog.=) q1.user_id) valid --------------------------------------------------------------------- t (1 row) -- subquery joins should work fine when used with CTEs +-- prevent PG 11 - PG 12 outputs to diverge +SET citus.enable_cte_inlining TO false; SELECT true AS valid FROM explain_json_2($$ WITH q1 AS (SELECT user_id FROM users_table) SELECT count(*) FROM q1, (SELECT @@ -566,6 +568,7 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS c t (1 row) +SET citus.enable_cte_inlining TO true; -- should work fine within UNIONs SELECT true AS valid FROM explain_json_2($$ (SELECT users_table.user_id FROM users_table, events_table WHERE users_table.user_id = events_table.value_2 AND event_type IN (1,2,3,4)) UNION @@ -811,13 +814,12 @@ SELECT true AS valid FROM explain_json_2($$ WHERE non_colocated_subquery.value_2 != non_colocated_subquery_2.cnt $$); -DEBUG: generating subplan XXX_1 for CTE non_colocated_subquery: SELECT foo.value_2 FROM (SELECT users_table.value_2 FROM public.users_table, public.events_table WHERE ((users_table.user_id OPERATOR(pg_catalog.=) events_table.user_id) AND (events_table.event_type OPERATOR(pg_catalog.=) ANY (ARRAY[1, 2, 3, 4])))) foo, (SELECT users_table.value_2 FROM public.users_table, public.events_table WHERE ((users_table.user_id OPERATOR(pg_catalog.=) events_table.user_id) AND (events_table.event_type OPERATOR(pg_catalog.=) ANY (ARRAY[5, 6, 7, 8])))) bar WHERE (foo.value_2 OPERATOR(pg_catalog.=) bar.value_2) +DEBUG: CTE non_colocated_subquery is going to be inlined via distributed planning +DEBUG: CTE non_colocated_subquery_2 is going to be inlined via distributed planning DEBUG: generating subplan XXX_1 for subquery SELECT users_table.value_2 FROM public.users_table, public.events_table WHERE ((users_table.user_id OPERATOR(pg_catalog.=) events_table.user_id) AND (events_table.event_type OPERATOR(pg_catalog.=) ANY (ARRAY[5, 6, 7, 8]))) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT foo.value_2 FROM (SELECT users_table.value_2 FROM public.users_table, public.events_table WHERE ((users_table.user_id OPERATOR(pg_catalog.=) events_table.user_id) AND (events_table.event_type OPERATOR(pg_catalog.=) ANY (ARRAY[1, 2, 3, 4])))) foo, (SELECT intermediate_result.value_2 FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value_2 integer)) bar WHERE (foo.value_2 OPERATOR(pg_catalog.=) bar.value_2) -DEBUG: generating subplan XXX_2 for CTE non_colocated_subquery_2: SELECT count(*) AS cnt FROM public.events_table WHERE (event_type OPERATOR(pg_catalog.=) ANY (SELECT events_table_1.event_type FROM public.events_table events_table_1 WHERE (events_table_1.user_id OPERATOR(pg_catalog.<) 4))) -DEBUG: generating subplan XXX_1 for subquery SELECT event_type FROM public.events_table WHERE (user_id OPERATOR(pg_catalog.<) 4) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS cnt FROM public.events_table WHERE (event_type OPERATOR(pg_catalog.=) ANY (SELECT intermediate_result.event_type FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(event_type integer))) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT non_colocated_subquery.value_2, non_colocated_subquery_2.cnt FROM (SELECT intermediate_result.value_2 FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value_2 integer)) non_colocated_subquery, (SELECT intermediate_result.cnt FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(cnt bigint)) non_colocated_subquery_2 WHERE (non_colocated_subquery.value_2 OPERATOR(pg_catalog.<>) non_colocated_subquery_2.cnt) +DEBUG: generating subplan XXX_2 for subquery SELECT event_type FROM public.events_table WHERE (user_id OPERATOR(pg_catalog.<) 4) +DEBUG: generating subplan XXX_3 for subquery SELECT count(*) AS cnt FROM public.events_table WHERE (event_type OPERATOR(pg_catalog.=) ANY (SELECT intermediate_result.event_type FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(event_type integer))) +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT non_colocated_subquery.value_2, non_colocated_subquery_2.cnt FROM (SELECT foo.value_2 FROM (SELECT users_table.value_2 FROM public.users_table, public.events_table WHERE ((users_table.user_id OPERATOR(pg_catalog.=) events_table.user_id) AND (events_table.event_type OPERATOR(pg_catalog.=) ANY (ARRAY[1, 2, 3, 4])))) foo, (SELECT intermediate_result.value_2 FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value_2 integer)) bar WHERE (foo.value_2 OPERATOR(pg_catalog.=) bar.value_2)) non_colocated_subquery, (SELECT intermediate_result.cnt FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(cnt bigint)) non_colocated_subquery_2 WHERE (non_colocated_subquery.value_2 OPERATOR(pg_catalog.<>) non_colocated_subquery_2.cnt) valid --------------------------------------------------------------------- t diff --git a/src/test/regress/expected/subquery_and_cte.out b/src/test/regress/expected/subquery_and_cte.out index ea6b28828..878486d33 100644 --- a/src/test/regress/expected/subquery_and_cte.out +++ b/src/test/regress/expected/subquery_and_cte.out @@ -2,6 +2,8 @@ -- test recursive planning functionality with subqueries and CTEs -- =================================================================== SET search_path TO subquery_and_ctes; +-- prevent PG 11 - PG 12 outputs to diverge +SET citus.enable_cte_inlining TO false; CREATE TABLE users_table_local AS SELECT * FROM users_table; CREATE TABLE dist_table (id int, value int); SELECT create_distributed_table('dist_table', 'id', colocate_with => 'users_table'); diff --git a/src/test/regress/expected/subquery_in_where.out b/src/test/regress/expected/subquery_in_where.out index 96cf8e7c9..62aefb80d 100644 --- a/src/test/regress/expected/subquery_in_where.out +++ b/src/test/regress/expected/subquery_in_where.out @@ -5,6 +5,8 @@ CREATE SCHEMA subquery_in_where; SET search_path TO subquery_in_where, public; SET client_min_messages TO DEBUG1; --CTEs can be used as a recurring tuple with subqueries in WHERE +-- prevent PG 11 - PG 12 outputs to diverge +SET citus.enable_cte_inlining TO false; WITH event_id AS (SELECT user_id AS events_user_id, time AS events_time, @@ -22,6 +24,7 @@ DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS c 101 (1 row) +SET citus.enable_cte_inlining TO true; --Correlated subqueries can not be used in WHERE clause WITH event_id AS (SELECT user_id AS events_user_id, @@ -33,6 +36,7 @@ FROM event_id WHERE events_user_id IN (SELECT user_id FROM users_table WHERE users_table.time = events_time); +DEBUG: CTE event_id is going to be inlined via distributed planning DEBUG: generating subplan XXX_1 for CTE event_id: SELECT user_id AS events_user_id, "time" AS events_time, event_type FROM public.events_table DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.events_user_id, intermediate_result.events_time, intermediate_result.event_type FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(events_user_id integer, events_time timestamp without time zone, event_type integer)) event_id WHERE (events_user_id OPERATOR(pg_catalog.=) ANY (SELECT users_table.user_id FROM public.users_table WHERE (users_table."time" OPERATOR(pg_catalog.=) event_id.events_time))) ERROR: cannot pushdown the subquery @@ -71,14 +75,15 @@ WHERE events_user_id IN ( FROM users_table ORDER BY 1 limit 10)); +DEBUG: CTE event_id is going to be inlined via distributed planning DEBUG: push down of limit count: 10 DEBUG: generating subplan XXX_1 for subquery SELECT user_id FROM public.users_table ORDER BY user_id LIMIT 10 DEBUG: push down of limit count: 10 DEBUG: generating subplan XXX_2 for subquery SELECT value_1 FROM public.users_table ORDER BY value_1 LIMIT 10 DEBUG: generating subplan XXX_3 for subquery SELECT intermediate_result.user_id FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer) UNION ALL SELECT intermediate_result.value_1 FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(value_1 integer) -DEBUG: generating subplan XXX_4 for CTE event_id: SELECT user_id AS events_user_id, "time" AS events_time, event_type FROM public.events_table -DEBUG: generating subplan XXX_5 for subquery SELECT events_user_id, events_time, event_type FROM (SELECT intermediate_result.events_user_id, intermediate_result.events_time, intermediate_result.event_type FROM read_intermediate_result('XXX_4'::text, 'binary'::citus_copy_format) intermediate_result(events_user_id integer, events_time timestamp without time zone, event_type integer)) event_id ORDER BY events_user_id, events_time, event_type LIMIT 10 -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.events_user_id, intermediate_result.events_time, intermediate_result.event_type FROM read_intermediate_result('XXX_5'::text, 'binary'::citus_copy_format) intermediate_result(events_user_id integer, events_time timestamp without time zone, event_type integer)) sub_table WHERE (events_user_id OPERATOR(pg_catalog.=) ANY (SELECT intermediate_result.user_id FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer))) +DEBUG: push down of limit count: 10 +DEBUG: generating subplan XXX_4 for subquery SELECT events_user_id, events_time, event_type FROM (SELECT events_table.user_id AS events_user_id, events_table."time" AS events_time, events_table.event_type FROM public.events_table) event_id ORDER BY events_user_id, events_time, event_type LIMIT 10 +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.events_user_id, intermediate_result.events_time, intermediate_result.event_type FROM read_intermediate_result('XXX_4'::text, 'binary'::citus_copy_format) intermediate_result(events_user_id integer, events_time timestamp without time zone, event_type integer)) sub_table WHERE (events_user_id OPERATOR(pg_catalog.=) ANY (SELECT intermediate_result.user_id FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer))) count --------------------------------------------------------------------- 10 @@ -129,14 +134,15 @@ IN LIMIT 10 ); -DEBUG: generating subplan XXX_1 for CTE event_id: SELECT user_id AS events_user_id, "time" AS events_time, event_type FROM public.events_table -DEBUG: generating subplan XXX_2 for subquery SELECT events_user_id FROM (SELECT intermediate_result.events_user_id, intermediate_result.events_time, intermediate_result.event_type FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(events_user_id integer, events_time timestamp without time zone, event_type integer)) event_id ORDER BY events_user_id LIMIT 10 +DEBUG: CTE event_id is going to be inlined via distributed planning DEBUG: push down of limit count: 10 -DEBUG: generating subplan XXX_3 for subquery SELECT user_id FROM public.users_table ORDER BY user_id LIMIT 10 +DEBUG: generating subplan XXX_1 for subquery SELECT events_user_id FROM (SELECT events_table.user_id AS events_user_id, events_table."time" AS events_time, events_table.event_type FROM public.events_table) event_id ORDER BY events_user_id LIMIT 10 DEBUG: push down of limit count: 10 -DEBUG: generating subplan XXX_4 for subquery SELECT value_1 FROM public.users_table ORDER BY value_1 LIMIT 10 -DEBUG: generating subplan XXX_5 for subquery SELECT intermediate_result.user_id FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer) UNION ALL SELECT intermediate_result.value_1 FROM read_intermediate_result('XXX_4'::text, 'binary'::citus_copy_format) intermediate_result(value_1 integer) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT user_id FROM (SELECT intermediate_result.user_id FROM read_intermediate_result('XXX_5'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer)) sub_table WHERE (user_id OPERATOR(pg_catalog.=) ANY (SELECT intermediate_result.events_user_id FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(events_user_id integer))) +DEBUG: generating subplan XXX_2 for subquery SELECT user_id FROM public.users_table ORDER BY user_id LIMIT 10 +DEBUG: push down of limit count: 10 +DEBUG: generating subplan XXX_3 for subquery SELECT value_1 FROM public.users_table ORDER BY value_1 LIMIT 10 +DEBUG: generating subplan XXX_4 for subquery SELECT intermediate_result.user_id FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer) UNION ALL SELECT intermediate_result.value_1 FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(value_1 integer) +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT user_id FROM (SELECT intermediate_result.user_id FROM read_intermediate_result('XXX_4'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer)) sub_table WHERE (user_id OPERATOR(pg_catalog.=) ANY (SELECT intermediate_result.events_user_id FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(events_user_id integer))) user_id --------------------------------------------------------------------- 1 @@ -324,12 +330,12 @@ SELECT COUNT(*) FROM cte; -DEBUG: generating subplan XXX_1 for CTE cte: SELECT user_id, "time", value_1, value_2, value_3, value_4 FROM (SELECT users_table.user_id, users_table."time", users_table.value_1, users_table.value_2, users_table.value_3, users_table.value_4 FROM public.users_table ORDER BY users_table.user_id, users_table.value_2 DESC LIMIT 10) sub_table WHERE (user_id OPERATOR(pg_catalog.=) ANY (SELECT events_table.value_2 FROM public.events_table)) +DEBUG: CTE cte is going to be inlined via distributed planning DEBUG: push down of limit count: 10 DEBUG: generating subplan XXX_1 for subquery SELECT user_id, "time", value_1, value_2, value_3, value_4 FROM public.users_table ORDER BY user_id, value_2 DESC LIMIT 10 DEBUG: generating subplan XXX_2 for subquery SELECT value_2 FROM public.events_table -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT user_id, "time", value_1, value_2, value_3, value_4 FROM (SELECT intermediate_result.user_id, intermediate_result."time", intermediate_result.value_1, intermediate_result.value_2, intermediate_result.value_3, intermediate_result.value_4 FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer, "time" timestamp without time zone, value_1 integer, value_2 integer, value_3 double precision, value_4 bigint)) sub_table WHERE (user_id OPERATOR(pg_catalog.=) ANY (SELECT intermediate_result.value_2 FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(value_2 integer))) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.user_id, intermediate_result."time", intermediate_result.value_1, intermediate_result.value_2, intermediate_result.value_3, intermediate_result.value_4 FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer, "time" timestamp without time zone, value_1 integer, value_2 integer, value_3 double precision, value_4 bigint)) cte +DEBUG: generating subplan XXX_3 for subquery SELECT user_id, "time", value_1, value_2, value_3, value_4 FROM (SELECT intermediate_result.user_id, intermediate_result."time", intermediate_result.value_1, intermediate_result.value_2, intermediate_result.value_3, intermediate_result.value_4 FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer, "time" timestamp without time zone, value_1 integer, value_2 integer, value_3 double precision, value_4 bigint)) sub_table WHERE (user_id OPERATOR(pg_catalog.=) ANY (SELECT intermediate_result.value_2 FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(value_2 integer))) +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS count FROM (SELECT intermediate_result.user_id, intermediate_result."time", intermediate_result.value_1, intermediate_result.value_2, intermediate_result.value_3, intermediate_result.value_4 FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer, "time" timestamp without time zone, value_1 integer, value_2 integer, value_3 double precision, value_4 bigint)) cte count --------------------------------------------------------------------- 10 diff --git a/src/test/regress/expected/subquery_view.out b/src/test/regress/expected/subquery_view.out index fcb5d0eba..502f19f3e 100644 --- a/src/test/regress/expected/subquery_view.out +++ b/src/test/regress/expected/subquery_view.out @@ -392,18 +392,17 @@ FROM SELECT * FROM subquery_and_ctes ORDER BY 3 DESC, 1 DESC, 2 DESC, 4 DESC LIMIT 5; -DEBUG: generating subplan XXX_1 for CTE cte: WITH local_cte AS (SELECT users_table_local.user_id, users_table_local."time", users_table_local.value_1, users_table_local.value_2, users_table_local.value_3, users_table_local.value_4 FROM subquery_view.users_table_local), dist_cte AS (SELECT events_table.user_id FROM public.events_table, (SELECT DISTINCT users_table.value_2 FROM public.users_table OFFSET 0) foo WHERE ((events_table.user_id OPERATOR(pg_catalog.=) foo.value_2) AND (events_table.user_id OPERATOR(pg_catalog.=) ANY (SELECT DISTINCT users_table.value_1 FROM public.users_table ORDER BY users_table.value_1 LIMIT 3)))) SELECT dist_cte.user_id FROM (local_cte JOIN dist_cte ON ((dist_cte.user_id OPERATOR(pg_catalog.=) local_cte.user_id))) -DEBUG: generating subplan XXX_1 for CTE local_cte: SELECT user_id, "time", value_1, value_2, value_3, value_4 FROM subquery_view.users_table_local -DEBUG: generating subplan XXX_2 for CTE dist_cte: SELECT events_table.user_id FROM public.events_table, (SELECT DISTINCT users_table.value_2 FROM public.users_table OFFSET 0) foo WHERE ((events_table.user_id OPERATOR(pg_catalog.=) foo.value_2) AND (events_table.user_id OPERATOR(pg_catalog.=) ANY (SELECT DISTINCT users_table.value_1 FROM public.users_table ORDER BY users_table.value_1 LIMIT 3))) +DEBUG: CTE cte is going to be inlined via distributed planning +DEBUG: CTE local_cte is going to be inlined via distributed planning +DEBUG: CTE dist_cte is going to be inlined via distributed planning +DEBUG: generating subplan XXX_1 for subquery SELECT user_id, "time", value_1, value_2, value_3, value_4 FROM subquery_view.users_table_local DEBUG: push down of limit count: 3 -DEBUG: generating subplan XXX_1 for subquery SELECT DISTINCT value_1 FROM public.users_table ORDER BY value_1 LIMIT 3 -DEBUG: generating subplan XXX_2 for subquery SELECT DISTINCT value_2 FROM public.users_table OFFSET 0 -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT events_table.user_id FROM public.events_table, (SELECT intermediate_result.value_2 FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(value_2 integer)) foo WHERE ((events_table.user_id OPERATOR(pg_catalog.=) foo.value_2) AND (events_table.user_id OPERATOR(pg_catalog.=) ANY (SELECT intermediate_result.value_1 FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(value_1 integer)))) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT dist_cte.user_id FROM ((SELECT intermediate_result.user_id, intermediate_result."time", intermediate_result.value_1, intermediate_result.value_2, intermediate_result.value_3, intermediate_result.value_4 FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer, "time" timestamp without time zone, value_1 integer, value_2 integer, value_3 double precision, value_4 bigint)) local_cte JOIN (SELECT intermediate_result.user_id FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer)) dist_cte ON ((dist_cte.user_id OPERATOR(pg_catalog.=) local_cte.user_id))) +DEBUG: generating subplan XXX_2 for subquery SELECT DISTINCT value_1 FROM public.users_table ORDER BY value_1 LIMIT 3 +DEBUG: generating subplan XXX_3 for subquery SELECT DISTINCT value_2 FROM public.users_table OFFSET 0 DEBUG: push down of limit count: 5 -DEBUG: generating subplan XXX_2 for subquery SELECT DISTINCT users_table.user_id FROM public.users_table, public.events_table WHERE ((users_table.user_id OPERATOR(pg_catalog.=) events_table.user_id) AND (events_table.event_type OPERATOR(pg_catalog.=) ANY (ARRAY[1, 2, 3, 4]))) ORDER BY users_table.user_id DESC LIMIT 5 -DEBUG: generating subplan XXX_3 for subquery SELECT count(*) AS cnt FROM (SELECT intermediate_result.user_id FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer)) cte, (SELECT intermediate_result.user_id FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer)) foo WHERE (foo.user_id OPERATOR(pg_catalog.=) cte.user_id) -DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT cnt, user_id, "time", value_1, value_2, value_3, value_4 FROM (SELECT foo.cnt, users_table.user_id, users_table."time", users_table.value_1, users_table.value_2, users_table.value_3, users_table.value_4 FROM (SELECT intermediate_result.cnt FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(cnt bigint)) foo, public.users_table WHERE (foo.cnt OPERATOR(pg_catalog.>) users_table.value_2)) subquery_and_ctes ORDER BY "time" DESC, cnt DESC, user_id DESC, value_1 DESC LIMIT 5 +DEBUG: generating subplan XXX_4 for subquery SELECT DISTINCT users_table.user_id FROM public.users_table, public.events_table WHERE ((users_table.user_id OPERATOR(pg_catalog.=) events_table.user_id) AND (events_table.event_type OPERATOR(pg_catalog.=) ANY (ARRAY[1, 2, 3, 4]))) ORDER BY users_table.user_id DESC LIMIT 5 +DEBUG: generating subplan XXX_5 for subquery SELECT count(*) AS cnt FROM (SELECT dist_cte.user_id FROM ((SELECT intermediate_result.user_id, intermediate_result."time", intermediate_result.value_1, intermediate_result.value_2, intermediate_result.value_3, intermediate_result.value_4 FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer, "time" timestamp without time zone, value_1 integer, value_2 integer, value_3 double precision, value_4 bigint)) local_cte JOIN (SELECT events_table.user_id FROM public.events_table, (SELECT intermediate_result.value_2 FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result(value_2 integer)) foo_1 WHERE ((events_table.user_id OPERATOR(pg_catalog.=) foo_1.value_2) AND (events_table.user_id OPERATOR(pg_catalog.=) ANY (SELECT intermediate_result.value_1 FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result(value_1 integer))))) dist_cte ON ((dist_cte.user_id OPERATOR(pg_catalog.=) local_cte.user_id)))) cte, (SELECT intermediate_result.user_id FROM read_intermediate_result('XXX_4'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer)) foo WHERE (foo.user_id OPERATOR(pg_catalog.=) cte.user_id) +DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT cnt, user_id, "time", value_1, value_2, value_3, value_4 FROM (SELECT foo.cnt, users_table.user_id, users_table."time", users_table.value_1, users_table.value_2, users_table.value_3, users_table.value_4 FROM (SELECT intermediate_result.cnt FROM read_intermediate_result('XXX_5'::text, 'binary'::citus_copy_format) intermediate_result(cnt bigint)) foo, public.users_table WHERE (foo.cnt OPERATOR(pg_catalog.>) users_table.value_2)) subquery_and_ctes ORDER BY "time" DESC, cnt DESC, user_id DESC, value_1 DESC LIMIT 5 DEBUG: push down of limit count: 5 cnt | user_id | time | value_1 | value_2 | value_3 | value_4 --------------------------------------------------------------------- @@ -437,6 +436,8 @@ SELECT time, event_type, value_2, value_3 FROM events_table WHERE foo.user_id = events_table.value_2; +-- prevent PG 11 - PG 12 outputs to diverge +SET citus.enable_cte_inlining TO false; SELECT * FROM subquery_and_ctes_second ORDER BY 3 DESC, 2 DESC, 1 DESC LIMIT 5; @@ -458,6 +459,7 @@ DEBUG: push down of limit count: 5 Thu Nov 23 09:33:16.992454 2017 | 3 | 4 | 1 (5 rows) +SET citus.enable_cte_inlining TO true; CREATE VIEW deep_subquery AS SELECT count(*) FROM diff --git a/src/test/regress/expected/with_join.out b/src/test/regress/expected/with_join.out index 12d855c3e..ba2ffaa6a 100644 --- a/src/test/regress/expected/with_join.out +++ b/src/test/regress/expected/with_join.out @@ -152,6 +152,9 @@ LIMIT (5 rows) -- cte LEFT JOIN distributed_table should error out +-- as long as the CTE is recursively planned +-- prevent PG 11 - PG 12 outputs to diverge +SET citus.enable_cte_inlining TO false; WITH cte AS ( SELECT * FROM users_table WHERE user_id = 1 ORDER BY value_1 ) @@ -167,6 +170,7 @@ LIMIT 5; ERROR: cannot pushdown the subquery DETAIL: Complex subqueries and CTEs cannot be in the outer part of the outer join +SET citus.enable_cte_inlining TO true; -- cte RIGHT JOIN distributed_table should work WITH cte AS ( SELECT * FROM users_table WHERE user_id = 1 ORDER BY value_1 @@ -214,6 +218,8 @@ LIMIT (5 rows) -- distributed_table RIGHT JOIN cte should error out +-- prevent PG 11 - PG 12 outputs to diverge +SET citus.enable_cte_inlining TO false; WITH cte AS ( SELECT * FROM users_table WHERE value_1 = 1 ORDER BY value_1 ) @@ -245,6 +251,7 @@ LIMIT 5; ERROR: cannot pushdown the subquery DETAIL: Complex subqueries and CTEs cannot be in the outer part of the outer join +SET citus.enable_cte_inlining TO false; -- Joins with reference tables are planned as router queries WITH cte AS ( SELECT value_2, max(user_id) AS user_id FROM users_table WHERE value_2 = 1 GROUP BY value_2 HAVING count(*) > 1 diff --git a/src/test/regress/expected/with_set_operations.out b/src/test/regress/expected/with_set_operations.out index d685c4246..c3ccbd4bd 100644 --- a/src/test/regress/expected/with_set_operations.out +++ b/src/test/regress/expected/with_set_operations.out @@ -2,6 +2,8 @@ -- test recursive planning functionality with Set Operations and CTEs -- =================================================================== SET client_min_messages TO DEBUG1; +-- prevent PG 11 - PG 12 outputs to diverge +SET citus.enable_cte_inlining TO false; -- use ctes inside unions on the top level WITH cte_1 AS (SELECT user_id FROM users_table), diff --git a/src/test/regress/sql/intermediate_result_pruning.sql b/src/test/regress/sql/intermediate_result_pruning.sql index 22a61d4e4..8aee49fcc 100644 --- a/src/test/regress/sql/intermediate_result_pruning.sql +++ b/src/test/regress/sql/intermediate_result_pruning.sql @@ -19,6 +19,8 @@ SELECT create_distributed_table('table_3', 'key'); CREATE TABLE ref_table (key int, value text); SELECT create_reference_table('ref_table'); +-- prevent PG 11 - PG 12 outputs to diverge +SET citus.enable_cte_inlining TO false; -- load some data INSERT INTO table_1 VALUES (1, '1'), (2, '2'), (3, '3'), (4, '4'); diff --git a/src/test/regress/sql/limit_intermediate_size.sql b/src/test/regress/sql/limit_intermediate_size.sql index 34f88dbf1..6207faef1 100644 --- a/src/test/regress/sql/limit_intermediate_size.sql +++ b/src/test/regress/sql/limit_intermediate_size.sql @@ -1,6 +1,9 @@ SET citus.enable_repartition_joins to ON; SET citus.task_executor_type to 'task-tracker'; +-- prevent PG 11 - PG 12 outputs to diverge +SET citus.enable_cte_inlining TO false; + SET citus.max_intermediate_result_size TO 2; -- should fail because the copy size is ~4kB for each cte WITH cte AS @@ -229,5 +232,3 @@ WITH cte AS ( cte2.user_id = cte3.user_id AND cte2.user_id = 1 ) SELECT * FROM cte ORDER BY 1,2,3,4,5 LIMIT 10; - -SET citus.task_executor_type to 'adaptive'; diff --git a/src/test/regress/sql/multi_explain.sql b/src/test/regress/sql/multi_explain.sql index a9ec11eab..d67d69d87 100644 --- a/src/test/regress/sql/multi_explain.sql +++ b/src/test/regress/sql/multi_explain.sql @@ -559,6 +559,8 @@ INSERT INTO lineitem_hash_part ( SELECT s FROM generate_series(5,10) s); -- explain with recursive planning +-- prevent PG 11 - PG 12 outputs to diverge +SET citus.enable_cte_inlining TO false; EXPLAIN (COSTS OFF, VERBOSE true) WITH keys AS ( SELECT DISTINCT l_orderkey FROM lineitem_hash_part @@ -569,6 +571,8 @@ series AS ( SELECT l_orderkey FROM series JOIN keys ON (s = l_orderkey) ORDER BY s; +SET citus.enable_cte_inlining TO true; + SELECT true AS valid FROM explain_json($$ WITH result AS ( SELECT l_quantity, count(*) count_quantity FROM lineitem diff --git a/src/test/regress/sql/multi_insert_select.sql b/src/test/regress/sql/multi_insert_select.sql index 1f5464064..bb9ea4619 100644 --- a/src/test/regress/sql/multi_insert_select.sql +++ b/src/test/regress/sql/multi_insert_select.sql @@ -515,7 +515,10 @@ INSERT INTO agg_events (value_1_agg, user_id) SELECT user_id, value_1_agg FROM agg_events ORDER BY 1,2; -- We support CTEs +-- but prefer to prevent inlining of the CTE +-- in order not to diverge from pg 11 vs pg 12 BEGIN; +SET LOCAL citus.enable_cte_inlining TO false; WITH fist_table_agg AS (SELECT max(value_1)+1 as v1_agg, user_id FROM raw_events_first GROUP BY user_id) INSERT INTO agg_events diff --git a/src/test/regress/sql/multi_insert_select_conflict.sql b/src/test/regress/sql/multi_insert_select_conflict.sql index 521c1afd0..08619d9e1 100644 --- a/src/test/regress/sql/multi_insert_select_conflict.sql +++ b/src/test/regress/sql/multi_insert_select_conflict.sql @@ -103,12 +103,16 @@ WITH inserted_table AS ( ON CONFLICT(col_1) DO UPDATE SET col_2 = 0 RETURNING * ) SELECT * FROM inserted_table ORDER BY 1; +-- prevent PG 11 - PG 12 outputs to diverge +SET citus.enable_cte_inlining TO false; + -- Get the select part from cte and do nothing on conflict WITH cte AS( SELECT col_1, col_2 FROM source_table_1 ) INSERT INTO target_table SELECT * FROM cte ON CONFLICT DO NOTHING; + -- Get the select part from cte and update on conflict WITH cte AS( SELECT col_1, col_2 FROM source_table_1 @@ -116,6 +120,8 @@ WITH cte AS( INSERT INTO target_table SELECT * FROM cte ON CONFLICT(col_1) DO UPDATE SET col_2 = EXCLUDED.col_2 + 1; SELECT * FROM target_table ORDER BY 1; +SET citus.enable_cte_inlining TO true; + -- Test with multiple CTEs WITH cte AS( SELECT col_1, col_2 FROM source_table_1 @@ -125,6 +131,9 @@ WITH cte AS( INSERT INTO target_table ((SELECT * FROM cte) UNION (SELECT * FROM cte_2)) ON CONFLICT(col_1) DO UPDATE SET col_2 = EXCLUDED.col_2 + 1; SELECT * FROM target_table ORDER BY 1; +-- prevent PG 11 - PG 12 outputs to diverge +SET citus.enable_cte_inlining TO false; + WITH inserted_table AS ( WITH cte AS( SELECT col_1, col_2, col_3 FROM source_table_1 @@ -159,6 +168,8 @@ SELECT FROM cte, source_table_1 WHERE cte.col_1 = source_table_1.col_1 ON CONFLICT DO NOTHING; +SET citus.enable_cte_inlining TO true; + -- Tests with foreign key to reference table CREATE TABLE test_ref_table (key int PRIMARY KEY); SELECT create_reference_table('test_ref_table'); @@ -288,6 +299,9 @@ FROM ( ON CONFLICT(col_1) DO UPDATE SET col_2 = 0; SELECT * FROM target_table ORDER BY 1; +-- prevent PG 11 - PG 12 outputs to diverge +SET citus.enable_cte_inlining TO false; + WITH cte AS( SELECT col_1, col_2, col_3 FROM source_table_1 ), cte_2 AS( diff --git a/src/test/regress/sql/multi_router_planner.sql b/src/test/regress/sql/multi_router_planner.sql index 18d14f544..a9783b105 100644 --- a/src/test/regress/sql/multi_router_planner.sql +++ b/src/test/regress/sql/multi_router_planner.sql @@ -10,6 +10,10 @@ SET citus.next_shard_id TO 840000; -- other tests that triggers fast-path-router planner SET citus.enable_fast_path_router_planner TO false; +-- prevent PG 11 - PG 12 outputs to diverge +-- and CTE inlining is not relevant to router plannery anyway +SET citus.enable_cte_inlining TO false; + CREATE TABLE articles_hash ( id bigint NOT NULL, author_id bigint NOT NULL, diff --git a/src/test/regress/sql/multi_task_assignment_policy.sql b/src/test/regress/sql/multi_task_assignment_policy.sql index 03ead23c2..855b1ef82 100644 --- a/src/test/regress/sql/multi_task_assignment_policy.sql +++ b/src/test/regress/sql/multi_task_assignment_policy.sql @@ -240,6 +240,8 @@ SET citus.task_assignment_policy TO 'round-robin'; -- Run the query two times to make sure that it hits two different workers -- on consecutive runs +-- prevent PG 11 - PG 12 outputs to diverge +SET citus.enable_cte_inlining TO false; INSERT INTO explain_outputs SELECT parse_explain_output($cmd$ EXPLAIN WITH q1 AS (SELECT * FROM task_assignment_test_table_2) SELECT * FROM q1 diff --git a/src/test/regress/sql/non_colocated_subquery_joins.sql b/src/test/regress/sql/non_colocated_subquery_joins.sql index 9b74ba28f..925974ca9 100644 --- a/src/test/regress/sql/non_colocated_subquery_joins.sql +++ b/src/test/regress/sql/non_colocated_subquery_joins.sql @@ -437,6 +437,8 @@ SELECT count(*) FROM q1, (SELECT users_table.user_id = events_table.value_2 AND event_type IN (1,2,3,4)) as bar WHERE bar.user_id = q1.user_id ;$$); -- subquery joins should work fine when used with CTEs +-- prevent PG 11 - PG 12 outputs to diverge +SET citus.enable_cte_inlining TO false; SELECT true AS valid FROM explain_json_2($$ WITH q1 AS (SELECT user_id FROM users_table) SELECT count(*) FROM q1, (SELECT @@ -445,6 +447,7 @@ SELECT true AS valid FROM explain_json_2($$ users_table, events_table WHERE users_table.user_id = events_table.user_id AND event_type IN (1,2,3,4)) as bar WHERE bar.user_id = q1.user_id ;$$); +SET citus.enable_cte_inlining TO true; -- should work fine within UNIONs diff --git a/src/test/regress/sql/subquery_and_cte.sql b/src/test/regress/sql/subquery_and_cte.sql index 272384176..1752a2bb6 100644 --- a/src/test/regress/sql/subquery_and_cte.sql +++ b/src/test/regress/sql/subquery_and_cte.sql @@ -3,6 +3,9 @@ -- =================================================================== SET search_path TO subquery_and_ctes; +-- prevent PG 11 - PG 12 outputs to diverge +SET citus.enable_cte_inlining TO false; + CREATE TABLE users_table_local AS SELECT * FROM users_table; CREATE TABLE dist_table (id int, value int); diff --git a/src/test/regress/sql/subquery_in_where.sql b/src/test/regress/sql/subquery_in_where.sql index 993f0e481..df637ce19 100644 --- a/src/test/regress/sql/subquery_in_where.sql +++ b/src/test/regress/sql/subquery_in_where.sql @@ -7,6 +7,8 @@ SET search_path TO subquery_in_where, public; SET client_min_messages TO DEBUG1; --CTEs can be used as a recurring tuple with subqueries in WHERE +-- prevent PG 11 - PG 12 outputs to diverge +SET citus.enable_cte_inlining TO false; WITH event_id AS (SELECT user_id AS events_user_id, time AS events_time, @@ -16,6 +18,7 @@ SELECT Count(*) FROM event_id WHERE events_user_id IN (SELECT user_id FROM users_table); +SET citus.enable_cte_inlining TO true; --Correlated subqueries can not be used in WHERE clause WITH event_id diff --git a/src/test/regress/sql/subquery_view.sql b/src/test/regress/sql/subquery_view.sql index fd7c8e662..91541a189 100644 --- a/src/test/regress/sql/subquery_view.sql +++ b/src/test/regress/sql/subquery_view.sql @@ -339,11 +339,14 @@ SELECT time, event_type, value_2, value_3 FROM WHERE foo.user_id = events_table.value_2; - +-- prevent PG 11 - PG 12 outputs to diverge +SET citus.enable_cte_inlining TO false; SELECT * FROM subquery_and_ctes_second ORDER BY 3 DESC, 2 DESC, 1 DESC LIMIT 5; +SET citus.enable_cte_inlining TO true; + CREATE VIEW deep_subquery AS SELECT count(*) FROM diff --git a/src/test/regress/sql/with_join.sql b/src/test/regress/sql/with_join.sql index 0157199c2..85c2c122a 100644 --- a/src/test/regress/sql/with_join.sql +++ b/src/test/regress/sql/with_join.sql @@ -127,6 +127,9 @@ LIMIT 5; -- cte LEFT JOIN distributed_table should error out +-- as long as the CTE is recursively planned +-- prevent PG 11 - PG 12 outputs to diverge +SET citus.enable_cte_inlining TO false; WITH cte AS ( SELECT * FROM users_table WHERE user_id = 1 ORDER BY value_1 ) @@ -141,6 +144,8 @@ ORDER BY LIMIT 5; +SET citus.enable_cte_inlining TO true; + -- cte RIGHT JOIN distributed_table should work WITH cte AS ( SELECT * FROM users_table WHERE user_id = 1 ORDER BY value_1 @@ -172,6 +177,8 @@ LIMIT 5; -- distributed_table RIGHT JOIN cte should error out +-- prevent PG 11 - PG 12 outputs to diverge +SET citus.enable_cte_inlining TO false; WITH cte AS ( SELECT * FROM users_table WHERE value_1 = 1 ORDER BY value_1 ) @@ -201,6 +208,8 @@ ORDER BY LIMIT 5; +SET citus.enable_cte_inlining TO false; + -- Joins with reference tables are planned as router queries WITH cte AS ( SELECT value_2, max(user_id) AS user_id FROM users_table WHERE value_2 = 1 GROUP BY value_2 HAVING count(*) > 1 diff --git a/src/test/regress/sql/with_set_operations.sql b/src/test/regress/sql/with_set_operations.sql index e6b5caecd..bbfe707a1 100644 --- a/src/test/regress/sql/with_set_operations.sql +++ b/src/test/regress/sql/with_set_operations.sql @@ -4,6 +4,9 @@ SET client_min_messages TO DEBUG1; +-- prevent PG 11 - PG 12 outputs to diverge +SET citus.enable_cte_inlining TO false; + -- use ctes inside unions on the top level WITH cte_1 AS (SELECT user_id FROM users_table),