external
/
citus
mirror of https://github.com/citusdata/citus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
citus/src/test/regress/expected/multi_subquery_complex_quer...

2901 lines
105 KiB
Plaintext
Raw Blame History

--
-- multi subquery complex queries aims to expand existing subquery pushdown
-- regression tests to cover more caeses
-- the tables that are used depends to multi_insert_select_behavioral_analytics_create_table.sql
--
-- We don't need shard id sequence here, so commented out to prevent conflicts with concurrent tests
-- SET citus.next_shard_id TO 1400000;
--
-- UNIONs and JOINs mixed
--
SELECT ("final_query"."event_types") as types, count(*) AS sumOfEventType
FROM
( SELECT *, random()
FROM
( SELECT "t"."user_id", "t"."time", unnest("t"."collected_events") AS "event_types"
FROM
( SELECT "t1"."user_id", min("t1"."time") AS "time", array_agg(("t1"."event") ORDER BY TIME ASC, event DESC) AS collected_events
FROM (
(SELECT
*
FROM
(SELECT
"events"."user_id", "events"."time", 0 AS event
FROM
events_table as "events"
WHERE
event_type IN (1, 2) ) events_subquery_1)
UNION
(SELECT
*
FROM
(SELECT
"events"."user_id", "events"."time", 1 AS event
FROM
events_table as "events"
WHERE
event_type IN (3, 4) ) events_subquery_2)
UNION
(SELECT
*
FROM
(SELECT
"events"."user_id", "events"."time", 2 AS event
FROM
events_table as "events"
WHERE
event_type IN (5, 6) ) events_subquery_3)
UNION
(SELECT
*
FROM
(SELECT
"events"."user_id", "events"."time", 3 AS event
FROM
events_table as "events"
WHERE
event_type IN (1, 6)) events_subquery_4)) t1
GROUP BY "t1"."user_id") AS t) "q"
INNER JOIN
(SELECT
"users"."user_id"
FROM
users_table as "users"
WHERE
value_1 > 0 and value_1 < 4) AS t
ON (t.user_id = q.user_id)) as final_query
GROUP BY
types
ORDER BY
types;
types | sumofeventtype
---------------------------------------------------------------------
0 | 449
1 | 433
2 | 75
3 | 268
(4 rows)
-- same query with target entries shuffled inside UNIONs
SELECT ("final_query"."event_types") as types, count(*) AS sumOfEventType
FROM
( SELECT *, random()
FROM
( SELECT "t"."user_id", "t"."time", unnest("t"."collected_events") AS "event_types"
FROM
( SELECT "t1"."user_id", min("t1"."time") AS "time", array_agg(("t1"."event") ORDER BY TIME ASC, event DESC) AS collected_events
FROM (
(SELECT *
FROM
(SELECT
"events"."time", 0 AS event, "events"."user_id"
FROM
events_table as "events"
WHERE
event_type IN (1, 2) ) events_subquery_1)
UNION
(SELECT *
FROM
(SELECT
"events"."time", 1 AS event, "events"."user_id"
FROM
events_table as "events"
WHERE
event_type IN (3, 4) ) events_subquery_2)
UNION
(SELECT *
FROM
(SELECT
"events"."time", 2 AS event, "events"."user_id"
FROM
events_table as "events"
WHERE
event_type IN (5, 6) ) events_subquery_3)
UNION
(SELECT *
FROM
(SELECT
"events"."time", 3 AS event, "events"."user_id"
FROM
events_table as "events"
WHERE
event_type IN (1, 6)) events_subquery_4)) t1
GROUP BY "t1"."user_id") AS t) "q"
INNER JOIN
(SELECT
"users"."user_id"
FROM
users_table as "users"
WHERE
value_1 > 0 and value_1 < 4) AS t
ON (t.user_id = q.user_id)) as final_query
GROUP BY
types
ORDER BY
types;
types | sumofeventtype
---------------------------------------------------------------------
0 | 449
1 | 433
2 | 75
3 | 268
(4 rows)
-- supported through recursive planning since events_subquery_2 doesn't have partition key on the target list
-- within the shuffled target list
SELECT ("final_query"."event_types") as types, count(*) AS sumOfEventType
FROM
( SELECT *, random()
FROM
( SELECT "t"."user_id", "t"."time", unnest("t"."collected_events") AS "event_types"
FROM
( SELECT "t1"."user_id", min("t1"."time") AS "time", array_agg(("t1"."event") ORDER BY TIME ASC, event DESC) AS collected_events
FROM (
(SELECT *
FROM
(SELECT
"events"."time", 0 AS event, "events"."user_id"
FROM
events_table as "events"
WHERE
event_type IN (1, 2) ) events_subquery_1)
UNION
(SELECT *
FROM
(SELECT
"events"."time", 1 AS event, "events"."user_id" * 2
FROM
events_table as "events"
WHERE
event_type IN (3, 4) ) events_subquery_2)
UNION
(SELECT *
FROM
(SELECT
"events"."time", 2 AS event, "events"."user_id"
FROM
events_table as "events"
WHERE
event_type IN (5, 6) ) events_subquery_3)
UNION
(SELECT *
FROM
(SELECT
"events"."time", 3 AS event, "events"."user_id"
FROM
events_table as "events"
WHERE
event_type IN (1, 6)) events_subquery_4)) t1
GROUP BY "t1"."user_id") AS t) "q"
INNER JOIN
(SELECT
"users"."user_id"
FROM
users_table as "users"
WHERE
value_1 > 0 and value_1 < 4) AS t
ON (t.user_id = q.user_id)) as final_query
GROUP BY
types
ORDER BY
types;
types | sumofeventtype
---------------------------------------------------------------------
0 | 449
1 | 234
2 | 75
3 | 268
(4 rows)
-- supported through recursive planning since events_subquery_2 doesn't have partition key on the target list
SELECT ("final_query"."event_types") as types, count(*) AS sumOfEventType
FROM
( SELECT *, random()
FROM
( SELECT "t"."user_id", "t"."time", unnest("t"."collected_events") AS "event_types"
FROM
( SELECT "t1"."user_id", min("t1"."time") AS "time", array_agg(("t1"."event") ORDER BY TIME ASC, event DESC) AS collected_events
FROM (
(SELECT *
FROM
(SELECT
"events"."time", 0 AS event, "events"."user_id"
FROM
events_table as "events"
WHERE
event_type IN (1, 2) ) events_subquery_1)
UNION
(SELECT *
FROM
(SELECT
"events"."time", 1 AS event, "events"."value_2" as user_id
FROM
events_table as "events"
WHERE
event_type IN (3, 4) ) events_subquery_2)
UNION
(SELECT *
FROM
(SELECT
"events"."time", 2 AS event, "events"."user_id"
FROM
events_table as "events"
WHERE
event_type IN (5, 6) ) events_subquery_3)
UNION
(SELECT *
FROM
(SELECT
"events"."time", 3 AS event, "events"."user_id"
FROM
events_table as "events"
WHERE
event_type IN (1, 6)) events_subquery_4)) t1
GROUP BY "t1"."user_id") AS t) "q"
INNER JOIN
(SELECT
"users"."user_id"
FROM
users_table as "users"
WHERE
value_1 > 0 and value_1 < 4) AS t
ON (t.user_id = q.user_id)) as final_query
GROUP BY
types
ORDER BY
types;
types | sumofeventtype
---------------------------------------------------------------------
0 | 449
1 | 369
2 | 75
3 | 268
(4 rows)
-- we can support arbitrary subqueries within UNIONs
SELECT ("final_query"."event_types") as types, count(*) AS sumOfEventType
FROM
( SELECT
*, random()
FROM
(SELECT
"t"."user_id", "t"."time", unnest("t"."collected_events") AS "event_types"
FROM
( SELECT
"t1"."user_id", min("t1"."time") AS "time", array_agg(("t1"."event") ORDER BY TIME ASC, event DESC) AS collected_events
FROM (
(SELECT
*
FROM
(SELECT
"events"."time", 0 AS event, "events"."user_id"
FROM
events_table as "events"
WHERE
event_type IN (1, 2) ) events_subquery_1)
UNION
(SELECT *
FROM
(
SELECT * FROM
(
SELECT
max("events"."time"),
0 AS event,
"events"."user_id"
FROM
events_table as "events", users_table as "users"
WHERE
events.user_id = users.user_id AND
event_type IN (1, 2)
GROUP BY "events"."user_id"
) as events_subquery_5
) events_subquery_2)
UNION
(SELECT *
FROM
(SELECT
"events"."time", 2 AS event, "events"."user_id"
FROM
events_table as "events"
WHERE
event_type IN (3, 4) ) events_subquery_3)
UNION
(SELECT *
FROM
(SELECT
"events"."time", 3 AS event, "events"."user_id"
FROM
events_table as "events"
WHERE
event_type IN (5, 6)) events_subquery_4)
) t1
GROUP BY "t1"."user_id") AS t) "q"
INNER JOIN
(SELECT
"users"."user_id"
FROM
users_table as "users"
WHERE
value_1 > 0 and value_1 < 4) AS t
ON (t.user_id = q.user_id)) as final_query
GROUP BY
types
ORDER BY
types;
types | sumofeventtype
---------------------------------------------------------------------
0 | 449
2 | 433
3 | 75
(3 rows)
SET citus.enable_repartition_joins to ON;
SET client_min_messages TO DEBUG1;
-- recursively planned since events_subquery_5 is not joined on partition key
SELECT ("final_query"."event_types") as types, count(*) AS sumOfEventType
FROM
( SELECT
*, random()
FROM
(SELECT
"t"."user_id", "t"."time", unnest("t"."collected_events") AS "event_types"
FROM
( SELECT
"t1"."user_id", min("t1"."time") AS "time", array_agg(("t1"."event") ORDER BY TIME ASC, event DESC) AS collected_events
FROM (
(SELECT
*
FROM
(SELECT
"events"."time", 0 AS event, "events"."user_id"
FROM
events_table as "events"
WHERE
event_type IN (1, 2) ) events_subquery_1)
UNION
(SELECT *
FROM
(
SELECT * FROM
(
SELECT
max("events"."time"),
0 AS event,
"events"."user_id"
FROM
events_table as "events", users_table as "users"
WHERE
events.user_id = users.value_2 AND
event_type IN (1, 2)
GROUP BY "events"."user_id"
) as events_subquery_5
) events_subquery_2)
UNION
(SELECT *
FROM
(SELECT
"events"."time", 2 AS event, "events"."user_id"
FROM
events_table as "events"
WHERE
event_type IN (3, 4) ) events_subquery_3)
UNION
(SELECT *
FROM
(SELECT
"events"."time", 3 AS event, "events"."user_id"
FROM
events_table as "events"
WHERE
event_type IN (5, 6)) events_subquery_4)
) t1
GROUP BY "t1"."user_id") AS t) "q"
INNER JOIN
(SELECT
"users"."user_id"
FROM
users_table as "users"
WHERE
value_1 > 0 and value_1 < 4) AS t
ON (t.user_id = q.user_id)) as final_query
GROUP BY
types
ORDER BY
types;
DEBUG: generating subplan XXX_1 for subquery SELECT max(events."time") AS max, 0 AS event, events.user_id FROM public.events_table events, public.users_table users WHERE ((events.user_id OPERATOR(pg_catalog.=) users.value_2) AND (events.event_type OPERATOR(pg_catalog.=) ANY (ARRAY[1, 2]))) GROUP BY events.user_id
DEBUG: generating subplan XXX_2 for subquery SELECT "time", event, user_id FROM (SELECT events."time", 0 AS event, events.user_id FROM public.events_table events WHERE (events.event_type OPERATOR(pg_catalog.=) ANY (ARRAY[1, 2]))) events_subquery_1
DEBUG: generating subplan XXX_3 for subquery SELECT "time", event, user_id FROM (SELECT events."time", 2 AS event, events.user_id FROM public.events_table events WHERE (events.event_type OPERATOR(pg_catalog.=) ANY (ARRAY[3, 4]))) events_subquery_3
DEBUG: generating subplan XXX_4 for subquery SELECT "time", event, user_id FROM (SELECT events."time", 3 AS event, events.user_id FROM public.events_table events WHERE (events.event_type OPERATOR(pg_catalog.=) ANY (ARRAY[5, 6]))) events_subquery_4
DEBUG: generating subplan XXX_5 for subquery SELECT intermediate_result."time", intermediate_result.event, intermediate_result.user_id FROM read_intermediate_result('XXX_2'::text, 'binary'::citus_copy_format) intermediate_result("time" timestamp without time zone, event integer, user_id integer) UNION SELECT events_subquery_2.max, events_subquery_2.event, events_subquery_2.user_id FROM (SELECT events_subquery_5.max, events_subquery_5.event, events_subquery_5.user_id FROM (SELECT intermediate_result.max, intermediate_result.event, intermediate_result.user_id FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(max timestamp without time zone, event integer, user_id integer)) events_subquery_5) events_subquery_2 UNION SELECT intermediate_result."time", intermediate_result.event, intermediate_result.user_id FROM read_intermediate_result('XXX_3'::text, 'binary'::citus_copy_format) intermediate_result("time" timestamp without time zone, event integer, user_id integer) UNION SELECT intermediate_result."time", intermediate_result.event, intermediate_result.user_id FROM read_intermediate_result('XXX_4'::text, 'binary'::citus_copy_format) intermediate_result("time" timestamp without time zone, event integer, user_id integer)
DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT event_types AS types, count(*) AS sumofeventtype FROM (SELECT q.user_id, q."time", q.event_types, t.user_id, random() AS random FROM ((SELECT t_1.user_id, t_1."time", unnest(t_1.collected_events) AS event_types FROM (SELECT t1.user_id, min(t1."time") AS "time", array_agg(t1.event ORDER BY t1."time", t1.event DESC) AS collected_events FROM (SELECT intermediate_result."time", intermediate_result.event, intermediate_result.user_id FROM read_intermediate_result('XXX_5'::text, 'binary'::citus_copy_format) intermediate_result("time" timestamp without time zone, event integer, user_id integer)) t1 GROUP BY t1.user_id) t_1) q JOIN (SELECT users.user_id FROM public.users_table users WHERE ((users.value_1 OPERATOR(pg_catalog.>) 0) AND (users.value_1 OPERATOR(pg_catalog.<) 4))) t ON ((t.user_id OPERATOR(pg_catalog.=) q.user_id)))) final_query(user_id, "time", event_types, user_id_1, random) GROUP BY event_types ORDER BY event_types
types | sumofeventtype
---------------------------------------------------------------------
0 | 449
2 | 433
3 | 75
(3 rows)
RESET client_min_messages;
SET citus.enable_repartition_joins to OFF;
-- recursively planned since the join is not equi join
SET client_min_messages TO DEBUG1;
SELECT ("final_query"."event_types") as types, count(*) AS sumOfEventType
FROM
( SELECT *, random()
FROM
( SELECT "t"."user_id", "t"."time", unnest("t"."collected_events") AS "event_types"
FROM
( SELECT "t1"."user_id", min("t1"."time") AS "time", array_agg(("t1"."event") ORDER BY TIME ASC, event DESC) AS collected_events
FROM (
(SELECT
*
FROM
(SELECT
"events"."user_id", "events"."time", 0 AS event
FROM
events_table as "events"
WHERE
event_type IN (1, 2) ) events_subquery_1)
UNION
(SELECT
*
FROM
(SELECT
"events"."user_id", "events"."time", 1 AS event
FROM
events_table as "events"
WHERE
event_type IN (3, 4) ) events_subquery_2)
UNION
(SELECT
*
FROM
(SELECT
"events"."user_id", "events"."time", 2 AS event
FROM
events_table as "events"
WHERE
event_type IN (5, 6) ) events_subquery_3)
UNION
(SELECT
*
FROM
(SELECT
"events"."user_id", "events"."time", 3 AS event
FROM
events_table as "events"
WHERE
event_type IN (4, 5)) events_subquery_4)) t1
GROUP BY "t1"."user_id") AS t) "q"
INNER JOIN
(SELECT
"users"."user_id"
FROM
users_table as "users"
WHERE
value_1 > 0 and value_1 < 4) AS t
ON (t.user_id != q.user_id)) as final_query
GROUP BY
types
ORDER BY
types;
DEBUG: generating subplan XXX_1 for subquery SELECT user_id, "time", unnest(collected_events) AS event_types FROM (SELECT t1.user_id, min(t1."time") AS "time", array_agg(t1.event ORDER BY t1."time", t1.event DESC) AS collected_events FROM (SELECT events_subquery_1.user_id, events_subquery_1."time", events_subquery_1.event FROM (SELECT events.user_id, events."time", 0 AS event FROM public.events_table events WHERE (events.event_type OPERATOR(pg_catalog.=) ANY (ARRAY[1, 2]))) events_subquery_1 UNION SELECT events_subquery_2.user_id, events_subquery_2."time", events_subquery_2.event FROM (SELECT events.user_id, events."time", 1 AS event FROM public.events_table events WHERE (events.event_type OPERATOR(pg_catalog.=) ANY (ARRAY[3, 4]))) events_subquery_2 UNION SELECT events_subquery_3.user_id, events_subquery_3."time", events_subquery_3.event FROM (SELECT events.user_id, events."time", 2 AS event FROM public.events_table events WHERE (events.event_type OPERATOR(pg_catalog.=) ANY (ARRAY[5, 6]))) events_subquery_3 UNION SELECT events_subquery_4.user_id, events_subquery_4."time", events_subquery_4.event FROM (SELECT events.user_id, events."time", 3 AS event FROM public.events_table events WHERE (events.event_type OPERATOR(pg_catalog.=) ANY (ARRAY[4, 5]))) events_subquery_4) t1 GROUP BY t1.user_id) t
DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT event_types AS types, count(*) AS sumofeventtype FROM (SELECT q.user_id, q."time", q.event_types, t.user_id, random() AS random FROM ((SELECT intermediate_result.user_id, intermediate_result."time", intermediate_result.event_types FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer, "time" timestamp without time zone, event_types integer)) q JOIN (SELECT users.user_id FROM public.users_table users WHERE ((users.value_1 OPERATOR(pg_catalog.>) 0) AND (users.value_1 OPERATOR(pg_catalog.<) 4))) t ON ((t.user_id OPERATOR(pg_catalog.<>) q.user_id)))) final_query(user_id, "time", event_types, user_id_1, random) GROUP BY event_types ORDER BY event_types
types | sumofeventtype
---------------------------------------------------------------------
0 | 2088
1 | 2163
2 | 397
3 | 1397
(4 rows)
RESET client_min_messages;
-- not supported since subquery 3 includes a JOIN with non-equi join
SELECT ("final_query"."event_types") as types, count(*) AS sumOfEventType
FROM
( SELECT *, random()
FROM
( SELECT "t"."user_id", "t"."time", unnest("t"."collected_events") AS "event_types"
FROM
( SELECT "t1"."user_id", min("t1"."time") AS "time", array_agg(("t1"."event") ORDER BY TIME ASC, event DESC) AS collected_events
FROM (
(SELECT
*
FROM
(SELECT
"events"."user_id", "events"."time", 0 AS event
FROM
events_table as "events"
WHERE
event_type IN (1, 2) ) events_subquery_1)
UNION
(SELECT
*
FROM
(SELECT
"events"."user_id", "events"."time", 1 AS event
FROM
events_table as "events"
WHERE
event_type IN (3, 4) ) events_subquery_2)
UNION
(SELECT
*
FROM
(SELECT
"events"."user_id", "events"."time", 2 AS event
FROM
events_table as "events", users_table as "users"
WHERE
event_type IN (5, 6) AND users.user_id != events.user_id ) events_subquery_3)
UNION
(SELECT
*
FROM
(SELECT
"events"."user_id", "events"."time", 3 AS event
FROM
events_table as "events"
WHERE
event_type IN (4, 5)) events_subquery_4)) t1
GROUP BY "t1"."user_id") AS t) "q"
INNER JOIN
(SELECT
"users"."user_id"
FROM
users_table as "users"
WHERE
value_1 > 0 and value_1 < 4) AS t
ON (t.user_id = q.user_id)) as final_query
GROUP BY
types
ORDER BY
types;
ERROR: complex joins are only supported when all distributed tables are joined on their distribution columns with equal operator
-- similar query with more union statements (to enable UNION tree become larger)
SELECT ("final_query"."event_types") as types, count(*) AS sumOfEventType
FROM
( SELECT *, random()
FROM
( SELECT "t"."user_id", "t"."time", unnest("t"."collected_events") AS "event_types"
FROM
( SELECT "t1"."user_id", min("t1"."time") AS "time", array_agg(("t1"."event") ORDER BY TIME ASC, event DESC) AS collected_events
FROM (
(SELECT *
FROM
(SELECT
"events"."user_id", "events"."time", 0 AS event
FROM
events_table as "events"
WHERE
event_type IN (1, 2) ) events_subquery_1)
UNION
(SELECT *
FROM
(SELECT
"events"."user_id", "events"."time", 1 AS event
FROM
events_table as "events"
WHERE
event_type IN (2, 3) ) events_subquery_2)
UNION
(SELECT *
FROM
(SELECT
"events"."user_id", "events"."time", 2 AS event
FROM
events_table as "events"
WHERE
event_type IN (3, 4) ) events_subquery_3)
UNION
(SELECT *
FROM
(SELECT
"events"."user_id", "events"."time", 3 AS event
FROM
events_table as "events"
WHERE
event_type IN (4, 5)) events_subquery_4)
UNION
(SELECT *
FROM
(SELECT
"events"."user_id", "events"."time", 4 AS event
FROM
events_table as "events"
WHERE
event_type IN (5, 6)) events_subquery_5)
UNION
(SELECT *
FROM
(SELECT
"events"."user_id", "events"."time", 5 AS event
FROM
events_table as "events"
WHERE
event_type IN (6, 1)) events_subquery_6)
UNION
(SELECT *
FROM
(SELECT
"events"."user_id", "events"."time", 6 AS event
FROM
events_table as "events"
WHERE
event_type IN (2, 5)) events_subquery_6)
) t1
GROUP BY "t1"."user_id") AS t) "q"
INNER JOIN
(SELECT "users"."user_id"
FROM users_table as "users"
WHERE value_1 > 0 and value_1 < 4) AS t ON (t.user_id = q.user_id)) as final_query
GROUP BY types
ORDER BY types;
types | sumofeventtype
---------------------------------------------------------------------
0 | 449
1 | 434
2 | 433
3 | 255
4 | 75
5 | 268
6 | 256
(7 rows)
--
-- UNION ALL Queries
--
SELECT ("final_query"."event_types") as types, count(*) AS sumOfEventType
FROM
( SELECT *, random()
FROM
( SELECT "t"."user_id", "t"."time", unnest("t"."collected_events") AS "event_types"
FROM
( SELECT "t1"."user_id", min("t1"."time") AS "time", array_agg(("t1"."event") ORDER BY TIME ASC, event DESC) AS collected_events
FROM (
(SELECT *
FROM
(SELECT
"events"."user_id", "events"."time", 0 AS event
FROM
events_table as "events"
WHERE
event_type IN (1, 2) ) events_subquery_1)
UNION ALL
(SELECT *
FROM
(SELECT
"events"."user_id", "events"."time", 1 AS event
FROM
events_table as "events"
WHERE
event_type IN (3, 4) ) events_subquery_2)
UNION ALL
(SELECT *
FROM
(SELECT
"events"."user_id", "events"."time", 2 AS event
FROM
events_table as "events"
WHERE
event_type IN (5, 6) ) events_subquery_3)
UNION ALL
(SELECT *
FROM
(SELECT
"events"."user_id", "events"."time", 3 AS event
FROM
events_table as "events"
WHERE
event_type IN (6, 1)) events_subquery_4)) t1
GROUP BY "t1"."user_id") AS t) "q"
INNER JOIN
(SELECT "users"."user_id"
FROM users_table as "users"
WHERE value_1 > 0 and value_1 < 4) AS t ON (t.user_id = q.user_id)) as final_query
GROUP BY types
ORDER BY types;
types | sumofeventtype
---------------------------------------------------------------------
0 | 449
1 | 433
2 | 75
3 | 268
(4 rows)
-- same query target list entries shuffled
SELECT ("final_query"."event_types") as types, count(*) AS sumOfEventType
FROM
( SELECT *, random()
FROM
( SELECT "t"."user_id", "t"."time", unnest("t"."collected_events") AS "event_types"
FROM
( SELECT "t1"."user_id", min("t1"."time") AS "time", array_agg(("t1"."event") ORDER BY TIME ASC, event DESC) AS collected_events
FROM (
(SELECT *
FROM
(SELECT
"events"."time", 0 AS event, "events"."user_id"
FROM
events_table as "events"
WHERE
event_type IN (1, 2) ) events_subquery_1)
UNION ALL
(SELECT *
FROM
(SELECT
"events"."time", 1 AS event, "events"."user_id"
FROM
events_table as "events"
WHERE
event_type IN (3, 4) ) events_subquery_2)
UNION ALL
(SELECT *
FROM
(SELECT
"events"."time", 2 AS event, "events"."user_id"
FROM
events_table as "events"
WHERE
event_type IN (5, 6) ) events_subquery_3)
UNION ALL
(SELECT *
FROM
(SELECT
"events"."time", 3 AS event, "events"."user_id"
FROM
events_table as "events"
WHERE
event_type IN (1, 6)) events_subquery_4)) t1
GROUP BY "t1"."user_id") AS t) "q"
INNER JOIN
(SELECT
"users"."user_id"
FROM
users_table as "users"
WHERE
value_1 > 0 and value_1 < 4) AS t
ON (t.user_id = q.user_id)) as final_query
GROUP BY types
ORDER BY types;
types | sumofeventtype
---------------------------------------------------------------------
0 | 449
1 | 433
2 | 75
3 | 268
(4 rows)
-- supported through recursive planning since subquery 3 does not have partition key
SELECT ("final_query"."event_types") as types, count(*) AS sumOfEventType
FROM
( SELECT *, random()
FROM
( SELECT "t"."user_id", "t"."time", unnest("t"."collected_events") AS "event_types"
FROM
( SELECT "t1"."user_id", min("t1"."time") AS "time", array_agg(("t1"."event") ORDER BY TIME ASC, event DESC) AS collected_events
FROM (
(SELECT *
FROM
(SELECT
"events"."user_id", "events"."time", 0 AS event
FROM
events_table as "events"
WHERE
event_type IN (1, 2) ) events_subquery_1)
UNION ALL
(SELECT *
FROM
(SELECT
"events"."user_id", "events"."time", 1 AS event
FROM
events_table as "events"
WHERE
event_type IN (3, 4) ) events_subquery_2)
UNION ALL
(SELECT *
FROM
(SELECT
"events"."value_2", "events"."time", 2 AS event
FROM
events_table as "events"
WHERE
event_type IN (5, 6) ) events_subquery_3)
UNION ALL
(SELECT *
FROM
(SELECT
"events"."user_id", "events"."time", 3 AS event
FROM
events_table as "events"
WHERE
event_type IN (1, 6)) events_subquery_4)) t1
GROUP BY "t1"."user_id") AS t) "q"
INNER JOIN
(SELECT "users"."user_id"
FROM users_table as "users"
WHERE value_1 > 0 and value_1 < 4) AS t ON (t.user_id = q.user_id)) as final_query
GROUP BY types
ORDER BY types;
types | sumofeventtype
---------------------------------------------------------------------
0 | 449
1 | 433
2 | 62
3 | 268
(4 rows)
-- supported through recursive planning since events_subquery_4 does not have partition key on the
-- target list
SELECT ("final_query"."event_types") as types, count(*) AS sumOfEventType
FROM
( SELECT *, random()
FROM
( SELECT "t"."user_id", "t"."time", unnest("t"."collected_events") AS "event_types"
FROM
( SELECT "t1"."user_id", min("t1"."time") AS "time", array_agg(("t1"."event") ORDER BY TIME ASC, event DESC) AS collected_events
FROM (
(SELECT *
FROM
(SELECT
"events"."time", 0 AS event, "events"."user_id"
FROM
events_table as "events"
WHERE
event_type IN (1, 2) ) events_subquery_1)
UNION ALL
(SELECT *
FROM
(SELECT
"events"."time", 1 AS event, "events"."user_id"
FROM
events_table as "events"
WHERE
event_type IN (3, 4) ) events_subquery_2)
UNION ALL
(SELECT *
FROM
(SELECT
"events"."time", 2 AS event, "events"."user_id"
FROM
events_table as "events"
WHERE
event_type IN (5, 6) ) events_subquery_3)
UNION ALL
(SELECT *
FROM
(SELECT
"events"."time", 3 AS event, "events"."user_id" * 2
FROM
events_table as "events"
WHERE
event_type IN (1, 6)) events_subquery_4)) t1
GROUP BY "t1"."user_id") AS t) "q"
INNER JOIN
(SELECT
"users"."user_id"
FROM
users_table as "users"
WHERE
value_1 > 0 and value_1 < 4) AS t
ON (t.user_id = q.user_id)) as final_query
GROUP BY types
ORDER BY types;
types | sumofeventtype
---------------------------------------------------------------------
0 | 449
1 | 433
2 | 75
3 | 124
(4 rows)
-- union all with inner and left joins
SELECT user_id, count(*) as cnt
FROM
(SELECT first_query.user_id, random()
FROM
( SELECT
"t"."user_id", "t"."time", unnest("t"."collected_events") AS "event_types"
FROM
( SELECT
"t1"."user_id", min("t1"."time") AS "time", array_agg(("t1"."event") ORDER BY TIME ASC, event DESC) AS collected_events
FROM (
(SELECT *
FROM
(SELECT
"events"."user_id", "events"."time", 0 AS event
FROM
events_table as "events"
WHERE
event_type IN (1, 2) ) events_subquery_1)
UNION ALL
(SELECT *
FROM
(SELECT
"events"."user_id", "events"."time", 1 AS event
FROM
events_table as "events"
WHERE
event_type IN (3, 4) ) events_subquery_2)
UNION ALL
(SELECT *
FROM
(SELECT
"events"."user_id", "events"."time", 2 AS event
FROM
events_table as "events"
WHERE
event_type IN (5, 6) ) events_subquery_3)
UNION ALL
(SELECT *
FROM
(SELECT
"events"."user_id", "events"."time", 3 AS event
FROM
events_table as "events"
WHERE
event_type IN (1, 6)) events_subquery_4)) t1
GROUP BY "t1"."user_id") AS t) "first_query"
INNER JOIN
(SELECT "t"."user_id"
FROM
(SELECT
"users"."user_id"
FROM
users_table as "users"
WHERE
value_1 > 0 and value_1 < 4) AS t
LEFT OUTER JOIN
(
SELECT
DISTINCT "events"."user_id" as user_id
FROM
events_table as "events"
WHERE
event_type IN (0, 6)
GROUP BY
user_id
) as t2
ON (t2.user_id = t.user_id) WHERE t2.user_id is NULL) as second_query
ON ("first_query".user_id = "second_query".user_id)) as final_query
GROUP BY
user_id ORDER BY cnt DESC, user_id DESC
LIMIT 10;
user_id | cnt
---------------------------------------------------------------------
3 | 275
6 | 72
(2 rows)
-- recursively planned since the join between t and t2 is not equi join
-- union all with inner and left joins
SET client_min_messages TO DEBUG1;
SELECT user_id, count(*) as cnt
FROM
(SELECT first_query.user_id, random()
FROM
( SELECT
"t"."user_id", "t"."time", unnest("t"."collected_events") AS "event_types"
FROM
( SELECT
"t1"."user_id", min("t1"."time") AS "time", array_agg(("t1"."event") ORDER BY TIME ASC, event DESC) AS collected_events
FROM (
(SELECT *
FROM
(SELECT
"events"."user_id", "events"."time", 0 AS event
FROM
events_table as "events"
WHERE
event_type IN (1, 2) ) events_subquery_1)
UNION ALL
(SELECT *
FROM
(SELECT
"events"."user_id", "events"."time", 1 AS event
FROM
events_table as "events"
WHERE
event_type IN (3, 4) ) events_subquery_2)
UNION ALL
(SELECT *
FROM
(SELECT
"events"."user_id", "events"."time", 2 AS event
FROM
events_table as "events"
WHERE
event_type IN (5, 6) ) events_subquery_3)
UNION ALL
(SELECT *
FROM
(SELECT
"events"."user_id", "events"."time", 3 AS event
FROM
events_table as "events"
WHERE
event_type IN (1, 6)) events_subquery_4)) t1
GROUP BY "t1"."user_id") AS t) "first_query"
INNER JOIN
(SELECT "t"."user_id"
FROM
(SELECT
"users"."user_id"
FROM
users_table as "users"
WHERE
value_1 > 0 and value_1 < 4) AS t
LEFT OUTER JOIN
(
SELECT
DISTINCT "events"."user_id" as user_id
FROM
events_table as "events"
WHERE
event_type IN (0, 6)
GROUP BY
user_id
) as t2
ON (t2.user_id > t.user_id) WHERE t2.user_id is NULL) as second_query
ON ("first_query".user_id = "second_query".user_id)) as final_query
GROUP BY
user_id ORDER BY cnt DESC, user_id DESC
LIMIT 10;
DEBUG: generating subplan XXX_1 for subquery SELECT DISTINCT user_id FROM public.events_table events WHERE (event_type OPERATOR(pg_catalog.=) ANY (ARRAY[0, 6])) GROUP BY user_id
DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT user_id, count(*) AS cnt FROM (SELECT first_query.user_id, random() AS random FROM ((SELECT t.user_id, t."time", unnest(t.collected_events) AS event_types FROM (SELECT t1.user_id, min(t1."time") AS "time", array_agg(t1.event ORDER BY t1."time", t1.event DESC) AS collected_events FROM (SELECT events_subquery_1.user_id, events_subquery_1."time", events_subquery_1.event FROM (SELECT events.user_id, events."time", 0 AS event FROM public.events_table events WHERE (events.event_type OPERATOR(pg_catalog.=) ANY (ARRAY[1, 2]))) events_subquery_1 UNION ALL SELECT events_subquery_2.user_id, events_subquery_2."time", events_subquery_2.event FROM (SELECT events.user_id, events."time", 1 AS event FROM public.events_table events WHERE (events.event_type OPERATOR(pg_catalog.=) ANY (ARRAY[3, 4]))) events_subquery_2 UNION ALL SELECT events_subquery_3.user_id, events_subquery_3."time", events_subquery_3.event FROM (SELECT events.user_id, events."time", 2 AS event FROM public.events_table events WHERE (events.event_type OPERATOR(pg_catalog.=) ANY (ARRAY[5, 6]))) events_subquery_3 UNION ALL SELECT events_subquery_4.user_id, events_subquery_4."time", events_subquery_4.event FROM (SELECT events.user_id, events."time", 3 AS event FROM public.events_table events WHERE (events.event_type OPERATOR(pg_catalog.=) ANY (ARRAY[1, 6]))) events_subquery_4) t1 GROUP BY t1.user_id) t) first_query JOIN (SELECT t.user_id FROM ((SELECT users.user_id FROM public.users_table users WHERE ((users.value_1 OPERATOR(pg_catalog.>) 0) AND (users.value_1 OPERATOR(pg_catalog.<) 4))) t LEFT JOIN (SELECT intermediate_result.user_id FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer)) t2 ON ((t2.user_id OPERATOR(pg_catalog.>) t.user_id))) WHERE (t2.user_id IS NULL)) second_query ON ((first_query.user_id OPERATOR(pg_catalog.=) second_query.user_id)))) final_query GROUP BY user_id ORDER BY (count(*)) DESC, user_id DESC LIMIT 10
DEBUG: push down of limit count: 10
user_id | cnt
---------------------------------------------------------------------
5 | 324
6 | 72
(2 rows)
RESET client_min_messages;
--
-- Union, inner join and left join
--
SELECT user_id, count(*) as cnt
FROM
(SELECT first_query.user_id, random()
FROM
( SELECT
"t"."user_id", "t"."time", unnest("t"."collected_events") AS "event_types"
FROM
( SELECT
"t1"."user_id", min("t1"."time") AS "time", array_agg(("t1"."event") ORDER BY TIME ASC, event DESC) AS collected_events
FROM (
(SELECT *
FROM
(SELECT
"events"."user_id", "events"."time", 0 AS event
FROM
events_table as "events"
WHERE
event_type IN (1, 2) ) events_subquery_1)
UNION
(SELECT *
FROM
(SELECT
"events"."user_id", "events"."time", 1 AS event
FROM
events_table as "events"
WHERE
event_type IN (3, 4) ) events_subquery_2)
UNION
(SELECT *
FROM
(SELECT
"events"."user_id", "events"."time", 2 AS event
FROM
events_table as "events"
WHERE
event_type IN (5, 6) ) events_subquery_3)
UNION
(SELECT *
FROM
(SELECT
"events"."user_id", "events"."time", 3 AS event
FROM
events_table as "events"
WHERE
event_type IN (1, 6)) events_subquery_4)) t1
GROUP BY "t1"."user_id") AS t) "first_query"
INNER JOIN
(SELECT "t"."user_id"
FROM
(SELECT
"users"."user_id"
FROM
users_table as "users"
WHERE
value_1 > 0 and value_1 < 4) AS t
LEFT OUTER JOIN
(
SELECT
DISTINCT "events"."user_id" as user_id
FROM
events_table as "events"
WHERE
event_type IN (0, 6)
GROUP BY
user_id
) as t2
ON (t2.user_id = t.user_id) WHERE t2.user_id is NULL) as second_query
ON ("first_query".user_id = "second_query".user_id)) as final_query
GROUP BY
user_id ORDER BY cnt DESC, user_id DESC
LIMIT 10;
user_id | cnt
---------------------------------------------------------------------
3 | 275
6 | 72
(2 rows)
-- Simple LATERAL JOINs with GROUP BYs in each side
-- need to set subquery_pushdown due to limit for next 2 queries
SET citus.subquery_pushdown to ON;
NOTICE: Setting citus.subquery_pushdown flag is discouraged becuase it forces the planner to pushdown certain queries, skipping relevant correctness checks.
DETAIL: When enabled, the planner skips many correctness checks for subqueries and pushes down the queries to shards as-is. It means that the queries are likely to return wrong results unless the user is absolutely sure that pushing down the subquery is safe. This GUC is maintained only for backward compatibility, no new users are supposed to use it. The planner is capable of pushing down as much computation as possible to the shards depending on the query.
SELECT *
FROM
(SELECT "some_users_data".user_id, lastseen
FROM
(SELECT user_id, max(time) AS lastseen
FROM
(SELECT user_id, time
FROM
(SELECT
user_id, time
FROM
events_table as "events"
WHERE
user_id > 1 and user_id < 4) "events_1"
ORDER BY
time DESC
LIMIT 1000) "recent_events_1"
GROUP BY
user_id
ORDER BY
max(time) DESC) "some_recent_users"
JOIN LATERAL
(SELECT
"users".user_id
FROM
users_table as "users"
WHERE
"users"."user_id" = "some_recent_users"."user_id" AND
users.value_2 > 1 and users.value_2 < 3
LIMIT 1) "some_users_data"
ON TRUE
ORDER BY
lastseen DESC
LIMIT 50) "some_users"
order BY
user_id
LIMIT 50;
user_id | lastseen
---------------------------------------------------------------------
2 | Thu Nov 23 17:26:14.563216 2017
3 | Thu Nov 23 18:08:26.550729 2017
(2 rows)
-- same query with subuqery joins in topmost select
SELECT "some_users_data".user_id, lastseen
FROM
(SELECT user_id, max(time) AS lastseen
FROM
(SELECT user_id, time
FROM
(SELECT
user_id, time
FROM
events_table as "events"
WHERE
user_id > 1 and user_id < 4) "events_1"
ORDER BY
time DESC
LIMIT 1000) "recent_events_1"
GROUP BY
user_id
ORDER BY
max(TIME) DESC) "some_recent_users"
JOIN LATERAL
(SELECT
"users".user_id
FROM
users_table as "users"
WHERE
"users"."user_id" = "some_recent_users"."user_id" AND
users.value_2 > 1 and users.value_2 < 3
ORDER BY 1 LIMIT 1) "some_users_data"
ON TRUE
ORDER BY
user_id
limit 50;
user_id | lastseen
---------------------------------------------------------------------
2 | Thu Nov 23 17:26:14.563216 2017
3 | Thu Nov 23 18:08:26.550729 2017
(2 rows)
-- reset subquery_pushdown
SET citus.subquery_pushdown to OFF;
-- mixture of recursively planned subqueries and correlated subqueries
SELECT "some_users_data".user_id, lastseen
FROM
(SELECT user_id, max(time) AS lastseen
FROM
(SELECT user_id, time
FROM
(SELECT
user_id, time
FROM
events_table as "events"
WHERE
user_id > 1 and user_id < 4) "events_1"
ORDER BY
time DESC
LIMIT 1000) "recent_events_1"
GROUP BY
user_id
ORDER BY
max(TIME) DESC) "some_recent_users"
JOIN LATERAL
(SELECT
"users".user_id
FROM
users_table as "users"
WHERE
"users"."value_1" = "some_recent_users"."user_id" AND
users.value_2 > 1 and users.value_2 < 3
ORDER BY 1 LIMIT 1) "some_users_data"
ON TRUE
ORDER BY
user_id
limit 50;
ERROR: cannot push down this subquery
DETAIL: Limit clause is currently unsupported when a lateral subquery references a column from complex subqueries, CTEs or local tables
SELECT "some_users_data".user_id, lastseen
FROM
(SELECT 2 * user_id as user_id, max(time) AS lastseen
FROM
(SELECT user_id, time
FROM
(SELECT
user_id, time
FROM
events_table as "events"
WHERE
user_id > 1 and user_id < 4) "events_1"
ORDER BY
time DESC
LIMIT 1000) "recent_events_1"
GROUP BY
user_id
ORDER BY
max(TIME) DESC) "some_recent_users"
JOIN LATERAL
(SELECT
"users".user_id
FROM
users_table as "users"
WHERE
"users"."user_id" = "some_recent_users"."user_id" AND
users.value_2 > 1 and users.value_2 < 3
ORDER BY 1 LIMIT 1) "some_users_data"
ON TRUE
ORDER BY
user_id
limit 50;
ERROR: cannot push down this subquery
DETAIL: Limit clause is currently unsupported when a lateral subquery references a column from complex subqueries, CTEs or local tables
-- LATERAL JOINs used with INNER JOINs
SET citus.subquery_pushdown to ON;
NOTICE: Setting citus.subquery_pushdown flag is discouraged becuase it forces the planner to pushdown certain queries, skipping relevant correctness checks.
DETAIL: When enabled, the planner skips many correctness checks for subqueries and pushes down the queries to shards as-is. It means that the queries are likely to return wrong results unless the user is absolutely sure that pushing down the subquery is safe. This GUC is maintained only for backward compatibility, no new users are supposed to use it. The planner is capable of pushing down as much computation as possible to the shards depending on the query.
SELECT user_id, lastseen
FROM
(SELECT
"some_users_data".user_id, lastseen
FROM
(SELECT
filter_users_1.user_id, time AS lastseen
FROM
(SELECT
user_where_1_1.user_id
FROM
(SELECT
"users"."user_id"
FROM
users_table as "users"
WHERE
user_id > 1 and user_id < 3 and value_1 > 2) user_where_1_1
INNER JOIN
(SELECT
"users"."user_id"
FROM
users_table as "users"
WHERE
user_id > 1 and user_id < 3 and value_2 > 3) user_where_1_join_1
ON ("user_where_1_1".user_id = "user_where_1_join_1".user_id))
filter_users_1
JOIN LATERAL
(SELECT
user_id, time
FROM
events_table as "events"
WHERE
user_id > 1 and user_id < 3 AND
user_id = filter_users_1.user_id
ORDER BY
time DESC
LIMIT 1) "last_events_1"
ON TRUE
ORDER BY
time DESC
LIMIT 10) "some_recent_users"
JOIN LATERAL
(SELECT
"users".user_id
FROM
users_table as "users"
WHERE
"users"."user_id" = "some_recent_users"."user_id" AND
"users"."value_2" > 4
ORDER BY 1 LIMIT 1) "some_users_data"
ON TRUE
ORDER BY
lastseen DESC
LIMIT 10) "some_users"
ORDER BY
user_id DESC, lastseen DESC
LIMIT 10;
user_id | lastseen
---------------------------------------------------------------------
2 | Thu Nov 23 17:26:14.563216 2017
2 | Thu Nov 23 17:26:14.563216 2017
2 | Thu Nov 23 17:26:14.563216 2017
2 | Thu Nov 23 17:26:14.563216 2017
2 | Thu Nov 23 17:26:14.563216 2017
2 | Thu Nov 23 17:26:14.563216 2017
2 | Thu Nov 23 17:26:14.563216 2017
2 | Thu Nov 23 17:26:14.563216 2017
2 | Thu Nov 23 17:26:14.563216 2017
2 | Thu Nov 23 17:26:14.563216 2017
(10 rows)
--
-- A similar query with topmost select is dropped
-- and replaced by aggregation. Notice the heavy use of limit
--
SELECT "some_users_data".user_id, MAX(lastseen), count(*)
FROM
(SELECT
filter_users_1.user_id, time AS lastseen
FROM
(SELECT
user_where_1_1.user_id
FROM
(SELECT
"users"."user_id"
FROM
users_table as "users"
WHERE
user_id > 1 and user_id < 3 and value_1 > 2) user_where_1_1
INNER JOIN
(SELECT
"users"."user_id"
FROM
users_table as "users"
WHERE
user_id > 1 and user_id < 3 and value_2 > 3) user_where_1_join_1
ON ("user_where_1_1".user_id = "user_where_1_join_1".user_id)) filter_users_1
JOIN LATERAL
(SELECT
user_id, time
FROM
events_table as "events"
WHERE
user_id > 1 and user_id < 3 and user_id = filter_users_1.user_id
ORDER BY
time DESC
LIMIT 1) "last_events_1" ON true
ORDER BY time DESC
LIMIT 10) "some_recent_users"
JOIN LATERAL
(SELECT
"users".user_id
FROM
users_table as "users"
WHERE
"users"."user_id" = "some_recent_users"."user_id" AND
"users"."value_2" > 4
ORDER BY 1 LIMIT 1) "some_users_data" ON true
GROUP BY 1
ORDER BY 2, 1 DESC
LIMIT 10;
user_id | max | count
---------------------------------------------------------------------
2 | Thu Nov 23 17:26:14.563216 2017 | 10
(1 row)
SET citus.subquery_pushdown to OFF;
-- not supported since the inner JOIN is not equi join and LATERAL JOIN prevents recursive planning
SET client_min_messages TO DEBUG2;
SELECT user_id, lastseen
FROM
(SELECT
"some_users_data".user_id, lastseen
FROM
(SELECT
filter_users_1.user_id, time AS lastseen
FROM
(SELECT
user_where_1_1.user_id
FROM
(SELECT
"users"."user_id"
FROM
users_table as "users"
WHERE
user_id > 1 and user_id < 4 and value_1 > 2) user_where_1_1
INNER JOIN
(SELECT
"users"."user_id"
FROM
users_table as "users"
WHERE
user_id > 1 and user_id < 4 and value_2 > 3) user_where_1_join_1
ON ("user_where_1_1".user_id != "user_where_1_join_1".user_id)) filter_users_1
JOIN LATERAL
(SELECT
user_id, time
FROM
events_table as "events"
WHERE
user_id > 1 and user_id < 4 and user_id = filter_users_1.user_id
ORDER BY
time DESC
LIMIT 1) "last_events_1" ON true
ORDER BY time DESC
LIMIT 10) "some_recent_users"
JOIN LATERAL
(SELECT
"users".user_id
FROM
users_table as "users"
WHERE
"users"."user_id" = "some_recent_users"."user_id" AND
"users"."value_2" > 4
ORDER BY 1 LIMIT 1) "some_users_data" ON true
ORDER BY
lastseen DESC
LIMIT 10) "some_users"
ORDER BY
user_id DESC, lastseen DESC
LIMIT 10;
DEBUG: Router planner cannot handle multi-shard select queries
DEBUG: Router planner cannot handle multi-shard select queries
DEBUG: generating subplan XXX_1 for subquery SELECT user_id FROM public.users_table users WHERE ((user_id OPERATOR(pg_catalog.>) 1) AND (user_id OPERATOR(pg_catalog.<) 4) AND (value_2 OPERATOR(pg_catalog.>) 3))
DEBUG: Router planner cannot handle multi-shard select queries
DEBUG: push down of limit count: 10
DEBUG: generating subplan XXX_2 for subquery SELECT filter_users_1.user_id, last_events_1."time" AS lastseen FROM ((SELECT user_where_1_1.user_id FROM ((SELECT users.user_id FROM public.users_table users WHERE ((users.user_id OPERATOR(pg_catalog.>) 1) AND (users.user_id OPERATOR(pg_catalog.<) 4) AND (users.value_1 OPERATOR(pg_catalog.>) 2))) user_where_1_1 JOIN (SELECT intermediate_result.user_id FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer)) user_where_1_join_1 ON ((user_where_1_1.user_id OPERATOR(pg_catalog.<>) user_where_1_join_1.user_id)))) filter_users_1 JOIN LATERAL (SELECT events.user_id, events."time" FROM public.events_table events WHERE ((events.user_id OPERATOR(pg_catalog.>) 1) AND (events.user_id OPERATOR(pg_catalog.<) 4) AND (events.user_id OPERATOR(pg_catalog.=) filter_users_1.user_id)) ORDER BY events."time" DESC LIMIT 1) last_events_1 ON (true)) ORDER BY last_events_1."time" DESC LIMIT 10
DEBUG: Router planner cannot handle multi-shard select queries
ERROR: cannot push down this subquery
DETAIL: Limit clause is currently unsupported when a lateral subquery references a column from complex subqueries, CTEs or local tables
SET citus.enable_repartition_joins to ON;
SET client_min_messages TO DEBUG1;
-- recursively planner since the inner JOIN is not on the partition key
SELECT user_id, lastseen
FROM
(SELECT
"some_users_data".user_id, lastseen
FROM
(SELECT
filter_users_1.user_id, time AS lastseen
FROM
(SELECT
user_where_1_1.user_id
FROM
(SELECT
"users"."user_id"
FROM
users_table as "users"
WHERE
user_id > 1 and user_id < 4 and value_1 > 2) user_where_1_1
INNER JOIN
(SELECT
"users"."user_id", "users"."value_1"
FROM
users_table as "users"
WHERE
user_id > 1 and user_id < 4 and value_2 > 3) user_where_1_join_1
ON ("user_where_1_1".user_id = "user_where_1_join_1".value_1)) filter_users_1
JOIN LATERAL
(SELECT
user_id, time
FROM
events_table as "events"
WHERE
user_id > 1 and user_id < 4 and user_id = filter_users_1.user_id
ORDER BY
time DESC
LIMIT 1) "last_events_1" ON true
ORDER BY time DESC
LIMIT 10) "some_recent_users"
JOIN LATERAL
(SELECT
"users".user_id
FROM
users_table as "users"
WHERE
"users"."user_id" = "some_recent_users"."user_id" AND
"users"."value_2" > 4
ORDER BY 1 LIMIT 1) "some_users_data" ON true
ORDER BY
lastseen DESC
LIMIT 10) "some_users"
ORDER BY
user_id DESC, lastseen DESC
LIMIT 10;
DEBUG: generating subplan XXX_1 for subquery SELECT user_id, value_1 FROM public.users_table users WHERE ((user_id OPERATOR(pg_catalog.>) 1) AND (user_id OPERATOR(pg_catalog.<) 4) AND (value_2 OPERATOR(pg_catalog.>) 3))
DEBUG: push down of limit count: 10
DEBUG: generating subplan XXX_2 for subquery SELECT filter_users_1.user_id, last_events_1."time" AS lastseen FROM ((SELECT user_where_1_1.user_id FROM ((SELECT users.user_id FROM public.users_table users WHERE ((users.user_id OPERATOR(pg_catalog.>) 1) AND (users.user_id OPERATOR(pg_catalog.<) 4) AND (users.value_1 OPERATOR(pg_catalog.>) 2))) user_where_1_1 JOIN (SELECT intermediate_result.user_id, intermediate_result.value_1 FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer, value_1 integer)) user_where_1_join_1 ON ((user_where_1_1.user_id OPERATOR(pg_catalog.=) user_where_1_join_1.value_1)))) filter_users_1 JOIN LATERAL (SELECT events.user_id, events."time" FROM public.events_table events WHERE ((events.user_id OPERATOR(pg_catalog.>) 1) AND (events.user_id OPERATOR(pg_catalog.<) 4) AND (events.user_id OPERATOR(pg_catalog.=) filter_users_1.user_id)) ORDER BY events."time" DESC LIMIT 1) last_events_1 ON (true)) ORDER BY last_events_1."time" DESC LIMIT 10
ERROR: cannot push down this subquery
DETAIL: Limit clause is currently unsupported when a lateral subquery references a column from complex subqueries, CTEs or local tables
SET citus.enable_repartition_joins to OFF;
RESET client_min_messages;
-- not supported since upper LATERAL JOIN is not equi join
SELECT user_id, lastseen
FROM
(SELECT
"some_users_data".user_id, lastseen
FROM
(SELECT
filter_users_1.user_id, time AS lastseen
FROM
(SELECT
user_where_1_1.user_id
FROM
(SELECT
"users"."user_id"
FROM
users_table as "users"
WHERE
user_id > 1 and user_id < 3 and value_1 > 2) user_where_1_1
INNER JOIN
(SELECT
"users"."user_id", "users"."value_1"
FROM
users_table as "users"
WHERE
user_id > 1 and user_id < 3 and value_2 > 3) user_where_1_join_1
ON ("user_where_1_1".user_id = "user_where_1_join_1".user_id)) filter_users_1
JOIN LATERAL
(SELECT
user_id, time
FROM
events_table as "events"
WHERE
user_id > 1 and user_id < 3 and user_id != filter_users_1.user_id
ORDER BY
time DESC
LIMIT 1) "last_events_1" ON true
ORDER BY time DESC
LIMIT 10) "some_recent_users"
JOIN LATERAL
(SELECT
"users".user_id
FROM
users_table as "users"
WHERE
"users"."user_id" = "some_recent_users"."user_id" AND
"users"."value_2" > 4
LIMIT 1) "some_users_data" ON true
ORDER BY
lastseen DESC
LIMIT 10) "some_users"
ORDER BY
user_id DESC, lastseen DESC
LIMIT 10;
ERROR: complex joins are only supported when all distributed tables are co-located and joined on their distribution columns
-- complex lateral join between inner join and correlated subquery
SELECT user_id, lastseen
FROM
(SELECT
"some_users_data".user_id, lastseen
FROM
(SELECT
filter_users_1.user_id, time AS lastseen
FROM
(SELECT
user_where_1_1.user_id
FROM
(SELECT
"users"."user_id"
FROM
users_table as "users"
WHERE
user_id > 1 and user_id < 3 and value_1 > 2) user_where_1_1
INNER JOIN
(SELECT
"users"."user_id", "users"."value_1"
FROM
users_table as "users"
WHERE
user_id > 1 and user_id < 3 and value_2 > 3) user_where_1_join_1
ON ("user_where_1_1".user_id = "user_where_1_join_1".user_id)) filter_users_1
JOIN LATERAL
(SELECT
user_id, time
FROM
events_table as "events"
WHERE
user_id > 1 and user_id < 3 and user_id = filter_users_1.user_id
ORDER BY
time DESC
LIMIT 1) "last_events_1" ON true
ORDER BY time DESC
LIMIT 10) "some_recent_users"
JOIN LATERAL
(SELECT
"users".user_id
FROM
users_table as "users"
WHERE
"users"."value_1" = "some_recent_users"."user_id" AND
"users"."value_2" > 4
ORDER BY 1 LIMIT 1) "some_users_data" ON true
ORDER BY
lastseen DESC
LIMIT 10) "some_users"
ORDER BY
user_id DESC, lastseen DESC
LIMIT 10;
ERROR: cannot push down this subquery
DETAIL: Limit clause is currently unsupported when a lateral subquery references a column from complex subqueries, CTEs or local tables
-- NESTED INNER JOINs
SELECT
count(*) AS value, "generated_group_field"
FROM
(SELECT
DISTINCT "pushedDownQuery"."real_user_id", "generated_group_field"
FROM
(SELECT
"eventQuery"."real_user_id", "eventQuery"."time", random(), ("eventQuery"."value_2") AS "generated_group_field"
FROM
(SELECT
*
FROM
(SELECT
"events"."time", "events"."user_id", "events"."value_2"
FROM
events_table as "events"
WHERE
user_id > 1 and user_id < 4 AND event_type IN (4, 5) ) "temp_data_queries"
INNER JOIN
(SELECT
user_where_1_1.real_user_id
FROM
(SELECT
"users"."user_id" as real_user_id
FROM
users_table as "users"
WHERE
user_id > 1 and user_id < 4 and value_2 > 3 ) user_where_1_1
INNER JOIN
(SELECT
"users"."user_id"
FROM
users_table as "users"
WHERE
user_id > 1 and user_id < 4 and value_3 > 3 ) user_where_1_join_1
ON ("user_where_1_1".real_user_id = "user_where_1_join_1".user_id)) "user_filters_1"
ON ("temp_data_queries".user_id = "user_filters_1".real_user_id)) "eventQuery") "pushedDownQuery") "pushedDownQuery"
GROUP BY
"generated_group_field"
ORDER BY
generated_group_field DESC, value DESC;
value | generated_group_field
---------------------------------------------------------------------
1 | 5
2 | 2
2 | 1
1 | 0
(4 rows)
SET citus.enable_repartition_joins to ON;
SET client_min_messages TO DEBUG1;
-- recursively planned since the first inner join is not on the partition key
SELECT
count(*) AS value, "generated_group_field"
FROM
(SELECT
DISTINCT "pushedDownQuery"."real_user_id", "generated_group_field"
FROM
(SELECT
"eventQuery"."real_user_id", "eventQuery"."time", random(), ("eventQuery"."value_2") AS "generated_group_field"
FROM
(SELECT
*
FROM
(SELECT
"events"."time", "events"."user_id", "events"."value_2"
FROM
events_table as "events"
WHERE
user_id > 1 and user_id < 4 AND event_type IN (4, 5) ) "temp_data_queries"
INNER JOIN
(SELECT
user_where_1_1.real_user_id
FROM
(SELECT
"users"."user_id" as real_user_id
FROM
users_table as "users"
WHERE
user_id > 1 and user_id < 4 and value_2 > 3 ) user_where_1_1
INNER JOIN
(SELECT
"users"."user_id", "users"."value_2"
FROM
users_table as "users"
WHERE
user_id > 1 and user_id < 4 and value_3 > 3 ) user_where_1_join_1
ON ("user_where_1_1".real_user_id = "user_where_1_join_1".value_2)) "user_filters_1"
ON ("temp_data_queries".user_id = "user_filters_1".real_user_id)) "eventQuery") "pushedDownQuery") "pushedDownQuery"
GROUP BY
"generated_group_field"
ORDER BY
generated_group_field DESC, value DESC;
DEBUG: generating subplan XXX_1 for subquery SELECT user_id, value_2 FROM public.users_table users WHERE ((user_id OPERATOR(pg_catalog.>) 1) AND (user_id OPERATOR(pg_catalog.<) 4) AND (value_3 OPERATOR(pg_catalog.>) (3)::double precision))
DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS value, generated_group_field FROM (SELECT DISTINCT "pushedDownQuery_1".real_user_id, "pushedDownQuery_1".generated_group_field FROM (SELECT "eventQuery".real_user_id, "eventQuery"."time", random() AS random, "eventQuery".value_2 AS generated_group_field FROM (SELECT temp_data_queries."time", temp_data_queries.user_id, temp_data_queries.value_2, user_filters_1.real_user_id FROM ((SELECT events."time", events.user_id, events.value_2 FROM public.events_table events WHERE ((events.user_id OPERATOR(pg_catalog.>) 1) AND (events.user_id OPERATOR(pg_catalog.<) 4) AND (events.event_type OPERATOR(pg_catalog.=) ANY (ARRAY[4, 5])))) temp_data_queries JOIN (SELECT user_where_1_1.real_user_id FROM ((SELECT users.user_id AS real_user_id FROM public.users_table users WHERE ((users.user_id OPERATOR(pg_catalog.>) 1) AND (users.user_id OPERATOR(pg_catalog.<) 4) AND (users.value_2 OPERATOR(pg_catalog.>) 3))) user_where_1_1 JOIN (SELECT intermediate_result.user_id, intermediate_result.value_2 FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer, value_2 integer)) user_where_1_join_1 ON ((user_where_1_1.real_user_id OPERATOR(pg_catalog.=) user_where_1_join_1.value_2)))) user_filters_1 ON ((temp_data_queries.user_id OPERATOR(pg_catalog.=) user_filters_1.real_user_id)))) "eventQuery") "pushedDownQuery_1") "pushedDownQuery" GROUP BY generated_group_field ORDER BY generated_group_field DESC, (count(*)) DESC
value | generated_group_field
---------------------------------------------------------------------
1 | 5
2 | 2
2 | 1
1 | 0
(4 rows)
-- recursive planning kicked-in since the non-equi join is among subqueries
SELECT
count(*) AS value, "generated_group_field"
FROM
(SELECT
DISTINCT "pushedDownQuery"."real_user_id", "generated_group_field"
FROM
(SELECT
"eventQuery"."real_user_id", "eventQuery"."time", random(), ("eventQuery"."value_2") AS "generated_group_field"
FROM
(SELECT
*
FROM
(SELECT
"events"."time", "events"."user_id", "events"."value_2"
FROM
events_table as "events"
WHERE
user_id > 1 and user_id < 4 AND event_type IN (4, 5) ) "temp_data_queries"
INNER JOIN
(SELECT
user_where_1_1.real_user_id
FROM
(SELECT
"users"."user_id" as real_user_id
FROM
users_table as "users"
WHERE
user_id > 1 and user_id < 4 and value_2 > 3 ) user_where_1_1
INNER JOIN
(SELECT
"users"."user_id", "users"."value_2"
FROM
users_table as "users"
WHERE
user_id > 1 and user_id < 4 and value_3 > 3 ) user_where_1_join_1
ON ("user_where_1_1".real_user_id >= "user_where_1_join_1".user_id)) "user_filters_1"
ON ("temp_data_queries".user_id = "user_filters_1".real_user_id)) "eventQuery") "pushedDownQuery") "pushedDownQuery"
GROUP BY
"generated_group_field"
ORDER BY
generated_group_field DESC, value DESC;
DEBUG: generating subplan XXX_1 for subquery SELECT user_id, value_2 FROM public.users_table users WHERE ((user_id OPERATOR(pg_catalog.>) 1) AND (user_id OPERATOR(pg_catalog.<) 4) AND (value_3 OPERATOR(pg_catalog.>) (3)::double precision))
DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT count(*) AS value, generated_group_field FROM (SELECT DISTINCT "pushedDownQuery_1".real_user_id, "pushedDownQuery_1".generated_group_field FROM (SELECT "eventQuery".real_user_id, "eventQuery"."time", random() AS random, "eventQuery".value_2 AS generated_group_field FROM (SELECT temp_data_queries."time", temp_data_queries.user_id, temp_data_queries.value_2, user_filters_1.real_user_id FROM ((SELECT events."time", events.user_id, events.value_2 FROM public.events_table events WHERE ((events.user_id OPERATOR(pg_catalog.>) 1) AND (events.user_id OPERATOR(pg_catalog.<) 4) AND (events.event_type OPERATOR(pg_catalog.=) ANY (ARRAY[4, 5])))) temp_data_queries JOIN (SELECT user_where_1_1.real_user_id FROM ((SELECT users.user_id AS real_user_id FROM public.users_table users WHERE ((users.user_id OPERATOR(pg_catalog.>) 1) AND (users.user_id OPERATOR(pg_catalog.<) 4) AND (users.value_2 OPERATOR(pg_catalog.>) 3))) user_where_1_1 JOIN (SELECT intermediate_result.user_id, intermediate_result.value_2 FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer, value_2 integer)) user_where_1_join_1 ON ((user_where_1_1.real_user_id OPERATOR(pg_catalog.>=) user_where_1_join_1.user_id)))) user_filters_1 ON ((temp_data_queries.user_id OPERATOR(pg_catalog.=) user_filters_1.real_user_id)))) "eventQuery") "pushedDownQuery_1") "pushedDownQuery" GROUP BY generated_group_field ORDER BY generated_group_field DESC, (count(*)) DESC
value | generated_group_field
---------------------------------------------------------------------
1 | 5
2 | 2
2 | 1
1 | 0
(4 rows)
SET citus.enable_repartition_joins to OFF;
RESET client_min_messages;
-- single level inner joins
SELECT
"value_3", count(*) AS cnt
FROM
(SELECT
"value_3", "user_id", random()
FROM
(SELECT
users_in_segment_1.user_id, value_3
FROM
(SELECT
user_id, value_3 * 2 as value_3
FROM
(SELECT
user_id, value_3
FROM
(SELECT
"users"."user_id", value_3
FROM
users_table as "users"
WHERE
user_id > 1 and user_id < 4 and value_2 > 2
) simple_user_where_1
) all_buckets_1
) users_in_segment_1
JOIN
(SELECT
"users"."user_id"
FROM
users_table as "users"
WHERE
user_id > 1 and user_id < 4 and value_2 > 3
) some_users_data
ON ("users_in_segment_1".user_id = "some_users_data".user_id)
) segmentalias_1) "tempQuery"
GROUP BY "value_3"
ORDER BY cnt, value_3 DESC LIMIT 10;
value_3 | cnt
---------------------------------------------------------------------
0 | 7
10 | 21
4 | 21
8 | 28
6 | 28
2 | 35
(6 rows)
SET citus.enable_repartition_joins to ON;
SET client_min_messages TO DEBUG1;
-- although there is no column equality at all
-- still recursive planning plans "some_users_data"
-- and the query becomes OK
SELECT
"value_3", count(*) AS cnt
FROM
(SELECT
"value_3", "user_id", random()
FROM
(SELECT
users_in_segment_1.user_id, value_3
FROM
(SELECT
user_id, value_3 * 2 as value_3
FROM
(SELECT
user_id, value_3
FROM
(SELECT
"users"."user_id", value_3
FROM
users_table as "users"
WHERE
user_id > 1 and user_id < 4 and value_2 > 2
) simple_user_where_1
) all_buckets_1
) users_in_segment_1
JOIN
(SELECT
"users"."user_id"
FROM
users_table as "users"
WHERE
user_id > 1 and user_id < 4 and value_2 > 3
) some_users_data
ON (true)
) segmentalias_1) "tempQuery"
GROUP BY "value_3"
ORDER BY cnt, value_3 DESC LIMIT 10;
DEBUG: generating subplan XXX_1 for subquery SELECT user_id FROM public.users_table users WHERE ((user_id OPERATOR(pg_catalog.>) 1) AND (user_id OPERATOR(pg_catalog.<) 4) AND (value_2 OPERATOR(pg_catalog.>) 3))
DEBUG: Plan XXX query after replacing subqueries and CTEs: SELECT value_3, count(*) AS cnt FROM (SELECT segmentalias_1.value_3, segmentalias_1.user_id, random() AS random FROM (SELECT users_in_segment_1.user_id, users_in_segment_1.value_3 FROM ((SELECT all_buckets_1.user_id, (all_buckets_1.value_3 OPERATOR(pg_catalog.*) (2)::double precision) AS value_3 FROM (SELECT simple_user_where_1.user_id, simple_user_where_1.value_3 FROM (SELECT users.user_id, users.value_3 FROM public.users_table users WHERE ((users.user_id OPERATOR(pg_catalog.>) 1) AND (users.user_id OPERATOR(pg_catalog.<) 4) AND (users.value_2 OPERATOR(pg_catalog.>) 2))) simple_user_where_1) all_buckets_1) users_in_segment_1 JOIN (SELECT intermediate_result.user_id FROM read_intermediate_result('XXX_1'::text, 'binary'::citus_copy_format) intermediate_result(user_id integer)) some_users_data ON (true))) segmentalias_1) "tempQuery" GROUP BY value_3 ORDER BY (count(*)), value_3 DESC LIMIT 10
value_3 | cnt
---------------------------------------------------------------------
0 | 14
10 | 42
4 | 42
8 | 56
6 | 56
2 | 70
(6 rows)
SET citus.enable_repartition_joins to OFF;
RESET client_min_messages;
-- nested LATERAL JOINs
SET citus.subquery_pushdown to ON;
NOTICE: Setting citus.subquery_pushdown flag is discouraged becuase it forces the planner to pushdown certain queries, skipping relevant correctness checks.
DETAIL: When enabled, the planner skips many correctness checks for subqueries and pushes down the queries to shards as-is. It means that the queries are likely to return wrong results unless the user is absolutely sure that pushing down the subquery is safe. This GUC is maintained only for backward compatibility, no new users are supposed to use it. The planner is capable of pushing down as much computation as possible to the shards depending on the query.
SELECT *
FROM
(SELECT "some_users_data".user_id, "some_recent_users".value_3
FROM
(SELECT
filter_users_1.user_id, value_3
FROM
(SELECT
"users"."user_id"
FROM
users_table as "users"
WHERE
user_id > 1 and user_id < 4 and users.value_2 = 2) filter_users_1
JOIN LATERAL
(SELECT
user_id, value_3
FROM
events_table as "events"
WHERE
user_id > 1 and user_id < 4 AND
("events".user_id = "filter_users_1".user_id)
ORDER BY
value_3 DESC
LIMIT 1) "last_events_1" ON true
ORDER BY value_3 DESC
LIMIT 10) "some_recent_users"
JOIN LATERAL
(SELECT
"users".user_id
FROM
users_table as "users"
WHERE
"users"."user_id" = "some_recent_users"."user_id" AND
users.value_2 > 2
LIMIT 1) "some_users_data" ON true
ORDER BY
value_3 DESC
LIMIT 10) "some_users"
ORDER BY
value_3 DESC, user_id ASC
LIMIT 10;
user_id | value_3
---------------------------------------------------------------------
2 | 5
2 | 5
2 | 5
2 | 5
3 | 5
3 | 5
3 | 5
3 | 5
3 | 5
(9 rows)
-- nested lateral join at top most level
SELECT "some_users_data".user_id, "some_recent_users".value_3
FROM
(SELECT
filter_users_1.user_id, value_3
FROM
(SELECT
"users"."user_id"
FROM
users_table as "users"
WHERE
user_id > 1 and user_id < 4 and users.value_2 = 2
) filter_users_1
JOIN LATERAL
(SELECT
user_id, value_3
FROM
events_table as "events"
WHERE
user_id > 1 and user_id < 4 AND
("events".user_id = "filter_users_1".user_id)
ORDER BY
value_3 DESC
LIMIT 1
) "last_events_1" ON true
ORDER BY value_3 DESC
LIMIT 10
) "some_recent_users"
JOIN LATERAL
(SELECT
"users".user_id
FROM
users_table as "users"
WHERE
"users"."user_id" = "some_recent_users"."user_id" AND
users.value_2 > 2
LIMIT 1
) "some_users_data" ON true
ORDER BY
value_3 DESC, user_id ASC
LIMIT 10;
user_id | value_3
---------------------------------------------------------------------
2 | 5
2 | 5
2 | 5
2 | 5
3 | 5
3 | 5
3 | 5
3 | 5
3 | 5
(9 rows)
-- longer nested lateral joins
SELECT *
FROM
(SELECT "some_users_data".user_id, "some_recent_users".value_3
FROM
(SELECT filter_users_1.user_id, value_3
FROM
(SELECT
"users"."user_id"
FROM
users_table as "users"
WHERE
user_id > 1 and user_id < 4 and users.value_2 = 2) filter_users_1
JOIN LATERAL
(SELECT
user_id, value_3
FROM
events_table as "events"
WHERE
user_id > 1 and user_id < 4 AND
("events".user_id = "filter_users_1".user_id)
ORDER BY
value_3 DESC
LIMIT 1) "last_events_1" ON true
ORDER BY
value_3 DESC
LIMIT 10) "some_recent_users"
JOIN LATERAL
(SELECT
"users".user_id
FROM
users_table as "users"
WHERE
"users"."user_id" = "some_recent_users"."user_id" AND
users.value_2 > 2
LIMIT 1) "some_users_data" ON true
ORDER BY
value_3 DESC
LIMIT 10) "some_users"
ORDER BY
value_3 DESC, user_id DESC
LIMIT 10;
user_id | value_3
---------------------------------------------------------------------
3 | 5
3 | 5
3 | 5
3 | 5
3 | 5
2 | 5
2 | 5
2 | 5
2 | 5
(9 rows)
-- longer nested lateral join wth top level join
SELECT "some_users_data".user_id, "some_recent_users".value_3
FROM
(SELECT filter_users_1.user_id, value_3
FROM
(SELECT
"users"."user_id"
FROM
users_table as "users"
WHERE
user_id > 1 and user_id < 4 and users.value_2 = 2
) filter_users_1
JOIN LATERAL
(SELECT
user_id, value_3
FROM
events_table as "events"
WHERE
user_id > 1 and user_id < 4
AND
("events".user_id = "filter_users_1".user_id)
ORDER BY
value_3 DESC
LIMIT 1
) "last_events_1" ON TRUE
ORDER BY value_3 DESC
LIMIT 10
) "some_recent_users"
JOIN LATERAL
(SELECT
"users".user_id
FROM
users_table as "users"
WHERE
"users"."user_id" = "some_recent_users"."user_id" AND
users.value_2 > 2
LIMIT 1
) "some_users_data" ON TRUE
ORDER BY value_3 DESC, user_id DESC
LIMIT 10;
user_id | value_3
---------------------------------------------------------------------
3 | 5
3 | 5
3 | 5
3 | 5
3 | 5
2 | 5
2 | 5
2 | 5
2 | 5
(9 rows)
SET citus.subquery_pushdown to OFF;
-- LEFT JOINs used with INNER JOINs
SELECT
count(*) AS cnt, "generated_group_field"
FROM
(SELECT
"eventQuery"."user_id", random(), generated_group_field
FROM
(SELECT
"multi_group_wrapper_1".*, generated_group_field, random()
FROM
(SELECT *
FROM
(SELECT
"events"."time", "events"."user_id" as event_user_id
FROM
events_table as "events"
WHERE
user_id > 4) "temp_data_queries"
INNER JOIN
(SELECT
"users"."user_id"
FROM
users_table as "users"
WHERE
user_id > 4 and value_2 = 5) "user_filters_1"
ON ("temp_data_queries".event_user_id = "user_filters_1".user_id)) AS "multi_group_wrapper_1"
LEFT JOIN
(SELECT
"users"."user_id" AS "user_id", value_2 AS "generated_group_field"
FROM
users_table as "users") "left_group_by_1"
ON ("left_group_by_1".user_id = "multi_group_wrapper_1".event_user_id)) "eventQuery") "pushedDownQuery"
group BY
"generated_group_field"
ORDER BY
cnt DESC, generated_group_field ASC
LIMIT 10;
cnt | generated_group_field
---------------------------------------------------------------------
336 | 2
210 | 1
210 | 3
126 | 4
126 | 5
84 | 0
(6 rows)
-- single table subquery, no JOINS involved
SELECT
count(*) AS cnt, user_id
FROM
(SELECT
"eventQuery"."user_id", random()
FROM
(SELECT
"events"."user_id"
FROM
events_table "events"
WHERE
event_type IN (1, 2)) "eventQuery") "pushedDownQuery"
GROUP BY
"user_id"
ORDER BY
cnt DESC, user_id DESC
LIMIT 10;
cnt | user_id
---------------------------------------------------------------------
11 | 3
10 | 2
8 | 4
6 | 5
4 | 6
4 | 1
(6 rows)
-- lateral joins in the nested manner
SET citus.subquery_pushdown to ON;
NOTICE: Setting citus.subquery_pushdown flag is discouraged becuase it forces the planner to pushdown certain queries, skipping relevant correctness checks.
DETAIL: When enabled, the planner skips many correctness checks for subqueries and pushes down the queries to shards as-is. It means that the queries are likely to return wrong results unless the user is absolutely sure that pushing down the subquery is safe. This GUC is maintained only for backward compatibility, no new users are supposed to use it. The planner is capable of pushing down as much computation as possible to the shards depending on the query.
SELECT *
FROM
(SELECT
"some_users_data".user_id, value_2
FROM
(SELECT user_id, max(value_2) AS value_2
FROM
(SELECT user_id, value_2
FROM
(SELECT
user_id, value_2
FROM
events_table as "events"
WHERE
user_id > 1 and user_id < 3) "events_1"
ORDER BY
value_2 DESC
LIMIT 10000) "recent_events_1"
GROUP BY
user_id
ORDER BY
max(value_2) DESC) "some_recent_users"
JOIN LATERAL
(SELECT
"users".user_id
FROM
users_table as "users"
WHERE
"users"."user_id" = "some_recent_users"."user_id" AND
value_2 > 4
LIMIT 1) "some_users_data" ON true
ORDER BY
value_2 DESC
LIMIT 10) "some_users"
ORDER BY
value_2 DESC, user_id DESC
LIMIT 10;
user_id | value_2
---------------------------------------------------------------------
2 | 5
(1 row)
SET citus.subquery_pushdown to OFF;
-- on side of the lateral join can be recursively plannen, then pushed down
SELECT *
FROM
(SELECT
"some_users_data".user_id, value_2
FROM
(SELECT user_id, max(value_2) AS value_2
FROM
(SELECT user_id, value_2
FROM
(SELECT
user_id, value_2
FROM
events_table as "events"
WHERE
user_id > 1 and user_id < 3) "events_1"
ORDER BY
value_2 DESC
LIMIT 10000) "recent_events_1"
GROUP BY
user_id
ORDER BY
max(value_2) DESC) "some_recent_users"
JOIN LATERAL
(SELECT
"users".user_id
FROM
users_table as "users"
WHERE
"users"."value_2" = "some_recent_users"."user_id" AND
value_2 > 4
ORDER BY 1 LIMIT 1) "some_users_data" ON true
ORDER BY
value_2 DESC
LIMIT 10) "some_users"
ORDER BY
value_2 DESC, user_id DESC
LIMIT 10;
ERROR: cannot push down this subquery
DETAIL: Limit clause is currently unsupported when a lateral subquery references a column from complex subqueries, CTEs or local tables
-- lets test some unsupported set operations
-- not supported since we use INTERSECT
SELECT ("final_query"."event_types") as types, count(*) AS sumOfEventType
FROM
( SELECT *, random()
FROM
( SELECT "t"."user_id", "t"."time", unnest("t"."collected_events") AS "event_types"
FROM
( SELECT "t1"."user_id", min("t1"."time") AS "time", array_agg(("t1"."event") ORDER BY TIME ASC, event DESC) AS collected_events
FROM (
(SELECT
*
FROM
(SELECT
"events"."user_id", "events"."time", 0 AS event
FROM
events_table as "events"
WHERE
event_type IN (1, 2) ) events_subquery_1)
UNION
(SELECT
*
FROM
(SELECT
"events"."user_id", "events"."time", 1 AS event
FROM
events_table as "events"
WHERE
event_type IN (3, 4) ) events_subquery_2)
UNION
(SELECT
*
FROM
(SELECT
"events"."user_id", "events"."time", 2 AS event
FROM
events_table as "events"
WHERE
event_type IN (5, 6) ) events_subquery_3)
INTERSECT
(SELECT
*
FROM
(SELECT
"events"."user_id", "events"."time", 3 AS event
FROM
events_table as "events"
WHERE
event_type IN (4, 5)) events_subquery_4)) t1
GROUP BY "t1"."user_id") AS t) "q"
INNER JOIN
(SELECT
"users"."user_id"
FROM
users_table as "users"
WHERE
value_1 > 0 and value_1 < 4) AS t
ON (t.user_id = q.user_id)) as final_query
GROUP BY
types
ORDER BY
types;
types | sumofeventtype
---------------------------------------------------------------------
0 | 449
1 | 433
(2 rows)
-- supported through recursive planning
SELECT ("final_query"."event_types") as types, count(*) AS sumOfEventType
FROM
( SELECT *
FROM
( SELECT "t"."user_id", "t"."time", unnest("t"."collected_events") AS "event_types"
FROM
( SELECT "t1"."user_id", min("t1"."time") AS "time", array_agg(("t1"."event") ORDER BY TIME ASC, event DESC) AS collected_events
FROM (
(SELECT
*
FROM
(SELECT
"events"."user_id", "events"."time", 0 AS event
FROM
events_table as "events"
WHERE
event_type IN (1, 2) ) events_subquery_1)
UNION
(SELECT
*
FROM
(SELECT
"events"."user_id", "events"."time", 1 AS event
FROM
events_table as "events"
WHERE
event_type IN (3, 4) ) events_subquery_2)
UNION
(SELECT
*
FROM
(SELECT
"events"."user_id", "events"."time", 2 AS event
FROM
events_table as "events"
WHERE
event_type IN (5, 6) ) events_subquery_3)
UNION
(SELECT
*
FROM
(SELECT
"events"."user_id", "events"."time", 3 AS event
FROM
events_table as "events"
WHERE
event_type IN (4, 5)) events_subquery_4) ORDER BY 1, 2 OFFSET 3) t1
GROUP BY "t1"."user_id") AS t) "q"
INNER JOIN
(SELECT
"users"."user_id"
FROM
users_table as "users"
WHERE
value_1 > 0 and value_1 < 4) AS t
ON (t.user_id = q.user_id)) as final_query
GROUP BY
types
ORDER BY
types;
types | sumofeventtype
---------------------------------------------------------------------
0 | 449
1 | 425
2 | 75
3 | 251
(4 rows)
-- not supported due to non relation rte
SELECT ("final_query"."event_types") as types, count(*) AS sumOfEventType
FROM
( SELECT *, random()
FROM
( SELECT "t"."user_id", "t"."time", unnest("t"."collected_events") AS "event_types"
FROM
( SELECT "t1"."user_id", min("t1"."time") AS "time", array_agg(("t1"."event") ORDER BY TIME ASC, event DESC) AS collected_events
FROM (
(SELECT
*
FROM
(SELECT
"events"."user_id", "events"."time", 0 AS event
FROM
events_table as "events"
WHERE
event_type IN (1, 2) ) events_subquery_1)
UNION
(SELECT
*
FROM
(SELECT
"events"."user_id", "events"."time", 1 AS event
FROM
events_table as "events"
WHERE
event_type IN (3, 4) ) events_subquery_2)
UNION
(SELECT
*
FROM
(SELECT
"events"."user_id", "events"."time", 2 AS event
FROM
events_table as "events"
WHERE
event_type IN (5, 6) ) events_subquery_3)
UNION
(SELECT
*
FROM
(SELECT
1 as user_id, now(), 3 AS event
) events_subquery_4)) t1
GROUP BY "t1"."user_id") AS t) "q"
INNER JOIN
(SELECT
"users"."user_id"
FROM
users_table as "users"
WHERE
value_1 > 0 and value_1 < 4) AS t
ON (t.user_id = q.user_id)) as final_query
GROUP BY
types
ORDER BY
types;
types | sumofeventtype
---------------------------------------------------------------------
0 | 449
1 | 433
2 | 75
3 | 4
(4 rows)
-- similar to the above, but constant rte is on the right side of the query
SELECT ("final_query"."event_types") as types, count(*) AS sumOfEventType
FROM
( SELECT *, random()
FROM
( SELECT "t"."user_id", "t"."time", unnest("t"."collected_events") AS "event_types"
FROM
( SELECT "t1"."user_id", min("t1"."time") AS "time", array_agg(("t1"."event") ORDER BY TIME ASC, event DESC) AS collected_events
FROM (
(SELECT
*
FROM
(SELECT
"events"."user_id", "events"."time", 0 AS event
FROM
events_table as "events"
WHERE
event_type IN (1, 2) ) events_subquery_1)
UNION
(SELECT
*
FROM
(SELECT
"events"."user_id", "events"."time", 1 AS event
FROM
events_table as "events"
WHERE
event_type IN (3, 4) ) events_subquery_2)
UNION
(SELECT
*
FROM
(SELECT
"events"."user_id", "events"."time", 2 AS event
FROM
events_table as "events"
WHERE
event_type IN (5, 6) ) events_subquery_3)
UNION
(SELECT
*
FROM
(SELECT
1 as user_id, now(), 3 AS event
) events_subquery_4)) t1
GROUP BY "t1"."user_id") AS t) "q"
INNER JOIN
(SELECT 1 as user_id) AS t
ON (t.user_id = q.user_id)) as final_query
GROUP BY
types
ORDER BY
types;
types | sumofeventtype
---------------------------------------------------------------------
0 | 4
1 | 8
2 | 1
3 | 1
(4 rows)
-- we've fixed a bug related to joins w/wout alias
-- while implementing top window functions
-- thus adding some tests related to that (i.e., next 3 tests)
WITH users_events AS
(
SELECT
user_id
FROM
users_table
)
SELECT
uid,
event_type,
value_2,
value_3
FROM (
(SELECT
user_id as uid
FROM
users_events
) users
JOIN
events_table
ON
users.uid = events_table.event_type
) a
ORDER BY
1,2,3,4
LIMIT 5;
uid | event_type | value_2 | value_3
---------------------------------------------------------------------
1 | 1 | 0 | 2
1 | 1 | 0 | 2
1 | 1 | 0 | 2
1 | 1 | 0 | 2
1 | 1 | 0 | 2
(5 rows)
-- the following queries are almost the same,
-- the only difference is the final GROUP BY
SELECT a.user_id, avg(b.value_2) as subquery_avg
FROM
(SELECT
user_id
FROM
users_table
WHERE
(value_1 > 2)
GROUP BY
user_id
HAVING
count(distinct value_1) > 2
) as a
LEFT JOIN
(SELECT
DISTINCT ON (value_2) value_2 , user_id, value_3
FROM
users_table
WHERE
(value_1 > 3)
ORDER BY
1,2,3
) AS b
USING (user_id)
GROUP BY user_id
ORDER BY 1, 2;
user_id | subquery_avg
---------------------------------------------------------------------
1 | 2.3333333333333333
3 | 5.0000000000000000
4 | 1.00000000000000000000
5 |
(4 rows)
-- see the comment for the above query
SELECT a.user_id, avg(b.value_2) as subquery_avg
FROM
(SELECT
user_id
FROM
users_table
WHERE
(value_1 > 2)
GROUP BY
user_id
HAVING
count(distinct value_1) > 2
) as a
LEFT JOIN
(SELECT
DISTINCT ON (value_2) value_2 , user_id, value_3
FROM
users_table
WHERE
(value_1 > 3)
ORDER BY
1,2,3
) AS b
USING (user_id)
GROUP BY a.user_id
ORDER BY 1, 2;
user_id | subquery_avg
---------------------------------------------------------------------
1 | 2.3333333333333333
3 | 5.0000000000000000
4 | 1.00000000000000000000
5 |
(4 rows)
-- queries where column aliases are used
-- the query is not very complex. join is given an alias with aliases
-- for each output column
SELECT k1
FROM (
SELECT k1, random()
FROM (users_table JOIN events_table USING (user_id)) k (k1, k2, k3)) l
ORDER BY k1
LIMIT 5;
k1
---------------------------------------------------------------------
1
1
1
1
1
(5 rows)
SELECT DISTINCT k1
FROM (
SELECT k1, random()
FROM (users_table JOIN events_table USING (user_id)) k (k1, k2, k3)) l
ORDER BY k1
LIMIT 5;
k1
---------------------------------------------------------------------
1
2
3
4
5
(5 rows)
SELECT x1, x3, value_2
FROM (users_table u FULL JOIN events_table e ON (u.user_id = e.user_id)) k(x1, x2, x3, x4, x5)
ORDER BY 1, 2, 3
LIMIT 5;
x1 | x3 | value_2
---------------------------------------------------------------------
1 | 1 | 1
1 | 1 | 1
1 | 1 | 1
1 | 1 | 2
1 | 1 | 2
(5 rows)
SELECT x1, x3, value_2
FROM (users_table u FULL JOIN events_table e USING (user_id)) k(x1, x2, x3, x4, x5)
ORDER BY 1, 2, 3
LIMIT 5;
x1 | x3 | value_2
---------------------------------------------------------------------
1 | 1 | 1
1 | 1 | 1
1 | 1 | 1
1 | 1 | 2
1 | 1 | 2
(5 rows)
SELECT c_custkey
FROM (users_table LEFT OUTER JOIN events_table ON (users_table.user_id = events_table.user_id)) AS test(c_custkey, c_nationkey)
INNER JOIN users_table as u2 ON (test.c_custkey = u2.user_id)
ORDER BY 1 DESC
LIMIT 10;
c_custkey
---------------------------------------------------------------------
6
6
6
6
6
6
6
6
6
6
(10 rows)
SELECT c_custkey, date_trunc('minute', max(c_nationkey))
FROM (users_table LEFT OUTER JOIN events_table ON (users_table.user_id = events_table.user_id)) AS test(c_custkey, c_nationkey)
INNER JOIN users_table as u2 ON (test.c_custkey = u2.user_id)
GROUP BY 1
ORDER BY 2, 1
LIMIT 10;
c_custkey | date_trunc
---------------------------------------------------------------------
2 | Thu Nov 23 13:52:00 2017
6 | Thu Nov 23 14:43:00 2017
4 | Thu Nov 23 15:32:00 2017
5 | Thu Nov 23 16:48:00 2017
3 | Thu Nov 23 17:18:00 2017
1 | Thu Nov 23 17:30:00 2017
(6 rows)
SELECT c_custkey, date_trunc('minute', max(c_nationkey))
FROM (users_table LEFT OUTER JOIN events_table ON (users_table.user_id = events_table.user_id)) AS test(c_custkey, c_nationkey)
INNER JOIN users_table as u2 ON (test.c_custkey = u2.user_id)
GROUP BY 1
HAVING extract(minute from max(c_nationkey)) >= 45
ORDER BY 2, 1
LIMIT 10;
c_custkey | date_trunc
---------------------------------------------------------------------
2 | Thu Nov 23 13:52:00 2017
5 | Thu Nov 23 16:48:00 2017
(2 rows)
SELECT user_id
FROM (users_table JOIN events_table USING (user_id)) AS test(user_id, c_nationkey)
FULL JOIN users_table AS u2 USING (user_id)
ORDER BY 1 DESC
LIMIT 10;
user_id
---------------------------------------------------------------------
6
6
6
6
6
6
6
6
6
6
(10 rows)
-- nested joins
SELECT bar, value_3_table.value_3
FROM ((users_table
JOIN (events_table INNER JOIN users_reference_table foo ON (events_table.user_id = foo.value_2)) AS deeper_join(user_id_deep)
ON (users_table.user_id = deeper_join.user_id_deep)) AS test(c_custkey, c_nationkey)
LEFT JOIN users_table AS u2 ON (test.c_custkey = u2.user_id)) outer_test(bar,foo)
JOIN LATERAL (SELECT value_3 FROM events_table WHERE user_id = bar) as value_3_table ON true
GROUP BY 1,2
ORDER BY 2 DESC, 1 DESC
LIMIT 10;
bar | value_3
---------------------------------------------------------------------
3 | 5
2 | 5
1 | 5
5 | 4
4 | 4
3 | 4
2 | 4
1 | 4
5 | 3
4 | 3
(10 rows)
-- lateral joins
SELECT bar,
value_3_table.value_3
FROM ((users_table
JOIN (events_table
INNER JOIN users_reference_table foo ON (events_table.user_id = foo.value_2)) AS deeper_join(user_id_deep) ON (users_table.user_id = deeper_join.user_id_deep)) AS test(c_custkey, c_nationkey)
LEFT JOIN users_table AS u2 ON (test.c_custkey = u2.user_id)) outer_test(bar, foo)
JOIN LATERAL
(SELECT value_3
FROM events_table
WHERE user_id = bar) AS value_3_table ON TRUE
GROUP BY 1, 2
ORDER BY 2 DESC, 1 DESC
LIMIT 10;
bar | value_3
---------------------------------------------------------------------
3 | 5
2 | 5
1 | 5
5 | 4
4 | 4
3 | 4
2 | 4
1 | 4
5 | 3
4 | 3
(10 rows)
--Joins inside subqueries are the sources of the values in the target list:
SELECT bar, foo.value_3, c_custkey, test_2.time_2 FROM
(
SELECT bar, value_3_table.value_3, random()
FROM ((users_table
JOIN (events_table INNER JOIN users_reference_table foo ON (events_table.user_id = foo.value_2)) AS deeper_join(user_id_deep)
ON (users_table.user_id = deeper_join.user_id_deep)) AS test(c_custkey, c_nationkey)
LEFT JOIN users_table AS u2 ON (test.c_custkey = u2.user_id)) outer_test(bar,foo)
JOIN LATERAL (SELECT value_3 FROM events_table WHERE user_id = bar) as value_3_table ON true
GROUP BY 1,2
) as foo, (users_table
JOIN (events_table INNER JOIN users_reference_table foo ON (events_table.user_id = foo.value_2)) AS deeper_join_2(user_id_deep)
ON (users_table.user_id = deeper_join_2.user_id_deep)) AS test_2(c_custkey, time_2) WHERE foo.bar = test_2.c_custkey
ORDER BY 2 DESC, 1 DESC, 3 DESC, 4 DESC
LIMIT 10;
bar | value_3 | c_custkey | time_2
---------------------------------------------------------------------
3 | 5 | 3 | Thu Nov 23 17:18:51.048758 2017
3 | 5 | 3 | Thu Nov 23 17:18:51.048758 2017
3 | 5 | 3 | Thu Nov 23 17:18:51.048758 2017
3 | 5 | 3 | Thu Nov 23 17:18:51.048758 2017
3 | 5 | 3 | Thu Nov 23 17:18:51.048758 2017
3 | 5 | 3 | Thu Nov 23 17:18:51.048758 2017
3 | 5 | 3 | Thu Nov 23 17:18:51.048758 2017
3 | 5 | 3 | Thu Nov 23 17:18:51.048758 2017
3 | 5 | 3 | Thu Nov 23 17:18:51.048758 2017
3 | 5 | 3 | Thu Nov 23 17:18:51.048758 2017
(10 rows)
Reference in New Issue View Git Blame Copy Permalink