From 705d1bf0e03a077ebb77490b658379e92df7223e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philip=20Dub=C3=A9?= Date: Wed, 7 Aug 2019 18:06:37 +0000 Subject: [PATCH 01/13] Use PG_JOB_CACHE_DIR --- src/backend/distributed/commands/transmit.c | 2 +- src/backend/distributed/shared_library_init.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/distributed/commands/transmit.c b/src/backend/distributed/commands/transmit.c index ea9e951b3..bfa434b31 100644 --- a/src/backend/distributed/commands/transmit.c +++ b/src/backend/distributed/commands/transmit.c @@ -411,7 +411,7 @@ VerifyTransmitStmt(CopyStmt *copyStatement) { ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - (errmsg("path must be in the pgsql_job_cache directory")))); + (errmsg("path must be in the " PG_JOB_CACHE_DIR " directory")))); } if (copyStatement->filename != NULL) diff --git a/src/backend/distributed/shared_library_init.c b/src/backend/distributed/shared_library_init.c index 3b2084218..abdf0a01e 100644 --- a/src/backend/distributed/shared_library_init.c +++ b/src/backend/distributed/shared_library_init.c @@ -343,7 +343,7 @@ CreateRequiredDirectories(void) const char *subdirs[] = { "pg_foreign_file", "pg_foreign_file/cached", - "base/pgsql_job_cache" + "base/" PG_JOB_CACHE_DIR }; for (dirNo = 0; dirNo < lengthof(subdirs); dirNo++) From b2ea806d8a8547453827cb736b41c4c60d859ec5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philip=20Dub=C3=A9?= Date: Wed, 7 Aug 2019 18:07:26 +0000 Subject: [PATCH 02/13] extra_float_digits=0 --- src/test/regress/pg_regress_multi.pl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/regress/pg_regress_multi.pl b/src/test/regress/pg_regress_multi.pl index f22648d65..9b9546fa1 100755 --- a/src/test/regress/pg_regress_multi.pl +++ b/src/test/regress/pg_regress_multi.pl @@ -289,6 +289,7 @@ push(@pgOptions, '-c', "listen_addresses=${host}"); # not required, and we don't necessarily have access to the default directory push(@pgOptions, '-c', "unix_socket_directories="); push(@pgOptions, '-c', "fsync=off"); +push(@pgOptions, '-c', "extra_float_digits=0"); my $sharedPreloadLibraries = "citus"; From 667c67891e3617868b45ccdee038c8514b374f94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philip=20Dub=C3=A9?= Date: Wed, 7 Aug 2019 18:14:02 +0000 Subject: [PATCH 03/13] intermediate_results: COSTS OFF --- .../regress/expected/intermediate_results.out | 24 +++++++++---------- src/test/regress/sql/intermediate_results.sql | 6 ++--- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/test/regress/expected/intermediate_results.out b/src/test/regress/expected/intermediate_results.out index f0abe1b10..d0dc1a6c3 100644 --- a/src/test/regress/expected/intermediate_results.out +++ b/src/test/regress/expected/intermediate_results.out @@ -208,10 +208,10 @@ SELECT create_intermediate_result('squares', 'SELECT s, s*s FROM generate_series 632 (1 row) -EXPLAIN SELECT * FROM read_intermediate_result('squares', 'binary') AS res (x int, x2 int); - QUERY PLAN ------------------------------------------------------------------------------------ - Function Scan on read_intermediate_result res (cost=0.00..4.55 rows=632 width=8) +EXPLAIN (COSTS OFF) SELECT * FROM read_intermediate_result('squares', 'binary') AS res (x int, x2 int); + QUERY PLAN +----------------------------------------------- + Function Scan on read_intermediate_result res (1 row) -- less accurate results for variable types @@ -221,10 +221,10 @@ SELECT create_intermediate_result('hellos', $$SELECT s, 'hello-'||s FROM generat 63 (1 row) -EXPLAIN SELECT * FROM read_intermediate_result('hellos', 'binary') AS res (x int, y text); - QUERY PLAN ------------------------------------------------------------------------------------ - Function Scan on read_intermediate_result res (cost=0.00..0.32 rows=30 width=36) +EXPLAIN (COSTS OFF) SELECT * FROM read_intermediate_result('hellos', 'binary') AS res (x int, y text); + QUERY PLAN +----------------------------------------------- + Function Scan on read_intermediate_result res (1 row) -- not very accurate results for text encoding @@ -234,10 +234,10 @@ SELECT create_intermediate_result('stored_squares', 'SELECT square FROM stored_s 4 (1 row) -EXPLAIN SELECT * FROM read_intermediate_result('stored_squares', 'text') AS res (s intermediate_results.square_type); - QUERY PLAN ----------------------------------------------------------------------------------- - Function Scan on read_intermediate_result res (cost=0.00..0.01 rows=1 width=32) +EXPLAIN (COSTS OFF) SELECT * FROM read_intermediate_result('stored_squares', 'text') AS res (s intermediate_results.square_type); + QUERY PLAN +----------------------------------------------- + Function Scan on read_intermediate_result res (1 row) END; diff --git a/src/test/regress/sql/intermediate_results.sql b/src/test/regress/sql/intermediate_results.sql index 67e307f6f..512fcd4cb 100644 --- a/src/test/regress/sql/intermediate_results.sql +++ b/src/test/regress/sql/intermediate_results.sql @@ -122,15 +122,15 @@ END; BEGIN; -- accurate row count estimates for primitive types SELECT create_intermediate_result('squares', 'SELECT s, s*s FROM generate_series(1,632) s'); -EXPLAIN SELECT * FROM read_intermediate_result('squares', 'binary') AS res (x int, x2 int); +EXPLAIN (COSTS OFF) SELECT * FROM read_intermediate_result('squares', 'binary') AS res (x int, x2 int); -- less accurate results for variable types SELECT create_intermediate_result('hellos', $$SELECT s, 'hello-'||s FROM generate_series(1,63) s$$); -EXPLAIN SELECT * FROM read_intermediate_result('hellos', 'binary') AS res (x int, y text); +EXPLAIN (COSTS OFF) SELECT * FROM read_intermediate_result('hellos', 'binary') AS res (x int, y text); -- not very accurate results for text encoding SELECT create_intermediate_result('stored_squares', 'SELECT square FROM stored_squares'); -EXPLAIN SELECT * FROM read_intermediate_result('stored_squares', 'text') AS res (s intermediate_results.square_type); +EXPLAIN (COSTS OFF) SELECT * FROM read_intermediate_result('stored_squares', 'text') AS res (s intermediate_results.square_type); END; -- pipe query output into a result file and create a table to check the result From 871dabdc631461daa8e425d7523d15737393d04d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philip=20Dub=C3=A9?= Date: Wed, 7 Aug 2019 18:15:51 +0000 Subject: [PATCH 04/13] Force CTE materialization in pg12 --- .../expected/limit_intermediate_size.out | 7 +- src/test/regress/expected/multi_explain.out | 87 ++++++++++--------- .../expected/multi_index_statements.out | 8 +- .../expected/multi_index_statements_0.out | 8 +- .../multi_null_minmax_value_pruning.out | 26 +++--- .../multi_null_minmax_value_pruning_0.out | 8 +- .../expected/replicated_partitioned_table.out | 8 +- .../replicated_partitioned_table_0.out | 8 +- src/test/regress/expected/with_dml.out | 5 +- .../regress/sql/limit_intermediate_size.sql | 7 +- src/test/regress/sql/multi_explain.sql | 5 +- .../regress/sql/multi_index_statements.sql | 2 +- .../sql/multi_null_minmax_value_pruning.sql | 2 +- .../sql/replicated_partitioned_table.sql | 2 +- src/test/regress/sql/with_dml.sql | 3 +- 15 files changed, 100 insertions(+), 86 deletions(-) diff --git a/src/test/regress/expected/limit_intermediate_size.out b/src/test/regress/expected/limit_intermediate_size.out index e54fd4c86..8f77b73f4 100644 --- a/src/test/regress/expected/limit_intermediate_size.out +++ b/src/test/regress/expected/limit_intermediate_size.out @@ -89,8 +89,8 @@ UNION ERROR: the intermediate result size exceeds citus.max_intermediate_result_size (currently 0 kB) DETAIL: Citus restricts the size of intermediate results of complex subqueries and CTEs to avoid accidentally pulling large result sets into once place. HINT: To run the current query, set citus.max_intermediate_result_size to a higher value or -1 to disable. --- this should fail since the cte-subplan exceeds the limit even if the --- cte2 and cte3 does not +-- this fails since cte-subplan exceeds limit even if cte2 and cte3 don't +-- WHERE EXISTS forces materialization in pg12 SET citus.max_intermediate_result_size TO 4; WITH cte AS ( WITH cte2 AS ( @@ -100,8 +100,9 @@ WITH cte AS ( SELECT * FROM events_table ) SELECT * FROM cte2, cte3 WHERE cte2.user_id = cte3.user_id AND cte2.user_id = 1 + AND EXISTS (select * from cte2, cte3) ) -SELECT * FROM cte; +SELECT * FROM cte WHERE EXISTS (select * from cte); ERROR: the intermediate result size exceeds citus.max_intermediate_result_size (currently 4 kB) DETAIL: Citus restricts the size of intermediate results of complex subqueries and CTEs to avoid accidentally pulling large result sets into once place. HINT: To run the current query, set citus.max_intermediate_result_size to a higher value or -1 to disable. diff --git a/src/test/regress/expected/multi_explain.out b/src/test/regress/expected/multi_explain.out index a358ea3a5..198ebacfa 100644 --- a/src/test/regress/expected/multi_explain.out +++ b/src/test/regress/expected/multi_explain.out @@ -46,7 +46,7 @@ Sort Sort Key: COALESCE((pg_catalog.sum((COALESCE((pg_catalog.sum(remote_scan.count_quantity))::bigint, '0'::bigint))))::bigint, '0'::bigint), remote_scan.l_quantity -> HashAggregate Group Key: remote_scan.l_quantity - -> Custom Scan (Citus Real-Time) + -> Custom Scan (Citus Adaptive) Task Count: 2 Tasks Shown: One of 2 -> Task @@ -65,7 +65,7 @@ Sort Group Key: remote_scan.l_quantity -> Sort Sort Key: remote_scan.l_quantity - -> Custom Scan (Citus Real-Time) + -> Custom Scan (Citus Adaptive) Task Count: 2 Tasks Shown: One of 2 -> Task @@ -96,7 +96,7 @@ EXPLAIN (COSTS FALSE, FORMAT JSON) { "Node Type": "Custom Scan", "Parent Relationship": "Outer", - "Custom Plan Provider": "Citus Real-Time", + "Custom Plan Provider": "Citus Adaptive", "Parallel Aware": false, "Distributed Query": { "Job": { @@ -171,7 +171,7 @@ EXPLAIN (COSTS FALSE, FORMAT XML) Custom Scan Outer - Citus Real-Time + Citus Adaptive false @@ -241,7 +241,7 @@ EXPLAIN (COSTS FALSE, FORMAT YAML) Plans: - Node Type: "Custom Scan" Parent Relationship: "Outer" - Custom Plan Provider: "Citus Real-Time" + Custom Plan Provider: "Citus Adaptive" Parallel Aware: false Distributed Query: Job: @@ -272,7 +272,7 @@ Sort Sort Key: COALESCE((pg_catalog.sum((COALESCE((pg_catalog.sum(remote_scan.count_quantity))::bigint, '0'::bigint))))::bigint, '0'::bigint), remote_scan.l_quantity -> HashAggregate Group Key: remote_scan.l_quantity - -> Custom Scan (Citus Real-Time) + -> Custom Scan (Citus Adaptive) Task Count: 2 Tasks Shown: One of 2 -> Task @@ -285,7 +285,7 @@ EXPLAIN (COSTS FALSE, VERBOSE TRUE) SELECT sum(l_quantity) / avg(l_quantity) FROM lineitem; Aggregate Output: (sum(remote_scan."?column?") / (sum(remote_scan."?column?_1") / pg_catalog.sum(remote_scan."?column?_2"))) - -> Custom Scan (Citus Real-Time) + -> Custom Scan (Citus Adaptive) Output: remote_scan."?column?", remote_scan."?column?_1", remote_scan."?column?_2" Task Count: 2 Tasks Shown: One of 2 @@ -303,7 +303,7 @@ EXPLAIN (COSTS FALSE) Limit -> Sort Sort Key: remote_scan.l_quantity - -> Custom Scan (Citus Real-Time) + -> Custom Scan (Citus Adaptive) Task Count: 2 Tasks Shown: One of 2 -> Task @@ -320,7 +320,7 @@ Limit -- Test insert EXPLAIN (COSTS FALSE) INSERT INTO lineitem VALUES (1,0), (2, 0), (3, 0), (4, 0); -Custom Scan (Citus Router) +Custom Scan (Citus Adaptive) Task Count: 1 Tasks Shown: All -> Task @@ -332,7 +332,7 @@ EXPLAIN (COSTS FALSE) UPDATE lineitem SET l_suppkey = 12 WHERE l_orderkey = 1 AND l_partkey = 0; -Custom Scan (Citus Router) +Custom Scan (Citus Adaptive) Task Count: 1 Tasks Shown: All -> Task @@ -345,7 +345,7 @@ Custom Scan (Citus Router) EXPLAIN (COSTS FALSE) DELETE FROM lineitem WHERE l_orderkey = 1 AND l_partkey = 0; -Custom Scan (Citus Router) +Custom Scan (Citus Adaptive) Task Count: 1 Tasks Shown: All -> Task @@ -359,20 +359,20 @@ EXPLAIN (COSTS FALSE) UPDATE lineitem SET l_suppkey = 12 WHERE l_orderkey = 1 AND l_orderkey = 0; -Custom Scan (Citus Router) +Custom Scan (Citus Adaptive) Task Count: 0 Tasks Shown: All -- Test zero-shard delete EXPLAIN (COSTS FALSE) DELETE FROM lineitem WHERE l_orderkey = 1 AND l_orderkey = 0; -Custom Scan (Citus Router) +Custom Scan (Citus Adaptive) Task Count: 0 Tasks Shown: All -- Test single-shard SELECT EXPLAIN (COSTS FALSE) SELECT l_quantity FROM lineitem WHERE l_orderkey = 5; -Custom Scan (Citus Router) +Custom Scan (Citus Adaptive) Task Count: 1 Tasks Shown: All -> Task @@ -389,7 +389,7 @@ t EXPLAIN (COSTS FALSE) CREATE TABLE explain_result AS SELECT * FROM lineitem; -Custom Scan (Citus Real-Time) +Custom Scan (Citus Adaptive) Task Count: 2 Tasks Shown: One of 2 -> Task @@ -402,7 +402,7 @@ EXPLAIN (COSTS FALSE, VERBOSE TRUE) Aggregate Output: (sum(remote_scan."?column?") / (sum(remote_scan."?column?_1") / pg_catalog.sum(remote_scan."?column?_2"))) Filter: (sum(remote_scan.worker_column_4) > '100'::numeric) - -> Custom Scan (Citus Real-Time) + -> Custom Scan (Citus Adaptive) Output: remote_scan."?column?", remote_scan."?column?_1", remote_scan."?column?_2", remote_scan.worker_column_4 Task Count: 2 Tasks Shown: One of 2 @@ -421,7 +421,7 @@ HashAggregate Output: remote_scan.l_quantity Group Key: remote_scan.l_quantity Filter: ((remote_scan.worker_column_2)::double precision > ('100'::double precision * random())) - -> Custom Scan (Citus Real-Time) + -> Custom Scan (Citus Adaptive) Output: remote_scan.l_quantity, remote_scan.worker_column_2 Task Count: 2 Tasks Shown: One of 2 @@ -459,7 +459,7 @@ FROM tenant_id, user_id) AS subquery; Aggregate - -> Custom Scan (Citus Real-Time) + -> Custom Scan (Citus Adaptive) Task Count: 4 Tasks Shown: One of 4 -> Task @@ -541,7 +541,7 @@ GROUP BY hasdone; HashAggregate Group Key: remote_scan.hasdone - -> Custom Scan (Citus Real-Time) + -> Custom Scan (Citus Adaptive) Task Count: 4 Tasks Shown: One of 4 -> Task @@ -709,7 +709,7 @@ LIMIT Limit -> Sort Sort Key: remote_scan.user_lastseen DESC - -> Custom Scan (Citus Real-Time) + -> Custom Scan (Citus Adaptive) Task Count: 4 Tasks Shown: One of 4 -> Task @@ -738,7 +738,7 @@ SET citus.explain_all_tasks TO on; EXPLAIN (COSTS FALSE) SELECT avg(l_linenumber) FROM lineitem WHERE l_orderkey > 9030; Aggregate - -> Custom Scan (Citus Real-Time) + -> Custom Scan (Citus Adaptive) Task Count: 1 Tasks Shown: All -> Task @@ -757,7 +757,7 @@ t EXPLAIN (COSTS FALSE) UPDATE lineitem_hash_part SET l_suppkey = 12; -Custom Scan (Citus Router) +Custom Scan (Citus Adaptive) Task Count: 4 Tasks Shown: All -> Task @@ -781,7 +781,7 @@ EXPLAIN (COSTS FALSE) UPDATE lineitem_hash_part SET l_suppkey = 12 WHERE l_orderkey = 1 OR l_orderkey = 3; -Custom Scan (Citus Router) +Custom Scan (Citus Adaptive) Task Count: 2 Tasks Shown: All -> Task @@ -797,7 +797,7 @@ Custom Scan (Citus Router) -- Test multi shard delete EXPLAIN (COSTS FALSE) DELETE FROM lineitem_hash_part; -Custom Scan (Citus Router) +Custom Scan (Citus Adaptive) Task Count: 4 Tasks Shown: All -> Task @@ -823,7 +823,7 @@ EXPLAIN (COSTS FALSE) SET l_suppkey = 12 FROM orders_hash_part WHERE orders_hash_part.o_orderkey = lineitem_hash_part.l_orderkey; -Custom Scan (Citus Router) +Custom Scan (Citus Adaptive) Task Count: 4 Tasks Shown: One of 4 -> Task @@ -839,7 +839,7 @@ EXPLAIN (COSTS FALSE) DELETE FROM lineitem_hash_part USING orders_hash_part WHERE orders_hash_part.o_orderkey = lineitem_hash_part.l_orderkey; -Custom Scan (Citus Router) +Custom Scan (Citus Adaptive) Task Count: 4 Tasks Shown: One of 4 -> Task @@ -1055,7 +1055,7 @@ Aggregate RESET citus.task_executor_type; PREPARE router_executor_query AS SELECT l_quantity FROM lineitem WHERE l_orderkey = 5; EXPLAIN EXECUTE router_executor_query; -Custom Scan (Citus Router) (cost=0.00..0.00 rows=0 width=0) +Custom Scan (Citus Adaptive) (cost=0.00..0.00 rows=0 width=0) Task Count: 1 Tasks Shown: All -> Task @@ -1066,7 +1066,7 @@ PREPARE real_time_executor_query AS SELECT avg(l_linenumber) FROM lineitem WHERE l_orderkey > 9030; EXPLAIN (COSTS FALSE) EXECUTE real_time_executor_query; Aggregate - -> Custom Scan (Citus Real-Time) + -> Custom Scan (Citus Adaptive) Task Count: 1 Tasks Shown: All -> Task @@ -1078,7 +1078,7 @@ Aggregate -- at least make sure to fail without crashing PREPARE router_executor_query_param(int) AS SELECT l_quantity FROM lineitem WHERE l_orderkey = $1; EXPLAIN EXECUTE router_executor_query_param(5); -Custom Scan (Citus Router) (cost=0.00..0.00 rows=0 width=0) +Custom Scan (Citus Adaptive) (cost=0.00..0.00 rows=0 width=0) Task Count: 1 Tasks Shown: All -> Task @@ -1098,11 +1098,11 @@ INSERT INTO lineitem_hash_part SELECT o_orderkey FROM orders_hash_part LIMIT 3; Custom Scan (Citus INSERT ... SELECT via coordinator) -> Limit - -> Custom Scan (Citus Real-Time) + -> Custom Scan (Citus Adaptive) Task Count: 4 Tasks Shown: One of 4 -> Task - Node: host=localhost port=57638 dbname=regression + Node: host=localhost port=57637 dbname=regression -> Limit -> Seq Scan on orders_hash_part_360045 orders_hash_part SELECT true AS valid FROM explain_json($$ @@ -1115,11 +1115,11 @@ INSERT INTO lineitem_hash_part (l_orderkey, l_quantity) SELECT o_orderkey, 5 FROM orders_hash_part LIMIT 3; Custom Scan (Citus INSERT ... SELECT via coordinator) -> Limit - -> Custom Scan (Citus Real-Time) + -> Custom Scan (Citus Adaptive) Task Count: 4 Tasks Shown: One of 4 -> Task - Node: host=localhost port=57638 dbname=regression + Node: host=localhost port=57637 dbname=regression -> Limit -> Seq Scan on orders_hash_part_360045 orders_hash_part EXPLAIN (COSTS OFF) @@ -1127,18 +1127,27 @@ INSERT INTO lineitem_hash_part (l_orderkey) SELECT s FROM generate_series(1,5) s; Custom Scan (Citus INSERT ... SELECT via coordinator) -> Function Scan on generate_series s +-- WHERE EXISTS forces pg12 to materialize cte EXPLAIN (COSTS OFF) WITH cte1 AS (SELECT s FROM generate_series(1,10) s) INSERT INTO lineitem_hash_part -WITH cte1 AS (SELECT * FROM cte1 LIMIT 5) -SELECT s FROM cte1; +WITH cte1 AS (SELECT * FROM cte1 WHERE EXISTS (SELECT * FROM cte1) LIMIT 5) +SELECT s FROM cte1 WHERE EXISTS (SELECT * FROM cte1); Custom Scan (Citus INSERT ... SELECT via coordinator) - -> CTE Scan on cte1 + -> Result + One-Time Filter: $3 CTE cte1 -> Function Scan on generate_series s CTE cte1 -> Limit - -> CTE Scan on cte1 cte1_1 + InitPlan 2 (returns $1) + -> CTE Scan on cte1 cte1_1 + -> Result + One-Time Filter: $1 + -> CTE Scan on cte1 cte1_2 + InitPlan 4 (returns $3) + -> CTE Scan on cte1 cte1_3 + -> CTE Scan on cte1 EXPLAIN (COSTS OFF) INSERT INTO lineitem_hash_part ( SELECT s FROM generate_series(1,5) s) UNION @@ -1159,13 +1168,13 @@ series AS ( ) SELECT l_orderkey FROM series JOIN keys ON (s = l_orderkey) ORDER BY s; -Custom Scan (Citus Router) +Custom Scan (Citus Adaptive) Output: remote_scan.l_orderkey -> Distributed Subplan 57_1 -> HashAggregate Output: remote_scan.l_orderkey Group Key: remote_scan.l_orderkey - -> Custom Scan (Citus Real-Time) + -> Custom Scan (Citus Adaptive) Output: remote_scan.l_orderkey Task Count: 4 Tasks Shown: One of 4 diff --git a/src/test/regress/expected/multi_index_statements.out b/src/test/regress/expected/multi_index_statements.out index 4ad80ad4e..0a6ed4614 100644 --- a/src/test/regress/expected/multi_index_statements.out +++ b/src/test/regress/expected/multi_index_statements.out @@ -4,10 +4,10 @@ -- Check that we can run CREATE INDEX and DROP INDEX statements on distributed -- tables. SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int AS major_version; - major_version ---------------- - 11 +SELECT substring(:'server_version', '\d+')::int > 10 AS server_version_above_ten; + server_version_above_ten +-------------------------- + t (1 row) -- diff --git a/src/test/regress/expected/multi_index_statements_0.out b/src/test/regress/expected/multi_index_statements_0.out index 70eb4e7cf..daa6d71b4 100644 --- a/src/test/regress/expected/multi_index_statements_0.out +++ b/src/test/regress/expected/multi_index_statements_0.out @@ -4,10 +4,10 @@ -- Check that we can run CREATE INDEX and DROP INDEX statements on distributed -- tables. SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int AS major_version; - major_version ---------------- - 10 +SELECT substring(:'server_version', '\d+')::int > 10 AS server_version_above_ten; + server_version_above_ten +-------------------------- + f (1 row) -- diff --git a/src/test/regress/expected/multi_null_minmax_value_pruning.out b/src/test/regress/expected/multi_null_minmax_value_pruning.out index e40e4893b..fe294b5ae 100644 --- a/src/test/regress/expected/multi_null_minmax_value_pruning.out +++ b/src/test/regress/expected/multi_null_minmax_value_pruning.out @@ -5,10 +5,10 @@ -- and that we don't partition or join prune shards that have null values. -- print major version number for version-specific tests SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int AS server_version; - server_version ----------------- - 11 +SELECT substring(:'server_version', '\d+')::int > 10 AS server_version_above_ten; + server_version_above_ten +-------------------------- + t (1 row) SET client_min_messages TO DEBUG2; @@ -39,7 +39,7 @@ DEBUG: Router planner does not support append-partitioned tables. LOG: join order: [ "lineitem" ] QUERY PLAN ----------------------------------------------------------------------- - Custom Scan (Citus Real-Time) + Custom Scan (Citus Adaptive) Task Count: 2 Tasks Shown: All -> Task @@ -72,7 +72,7 @@ DEBUG: join prunable for intervals [8997,14947] and [1,5986] QUERY PLAN -------------------------------------------------------------------------------- Aggregate - -> Custom Scan (Citus Real-Time) + -> Custom Scan (Citus Adaptive) Task Count: 2 Tasks Shown: All -> Task @@ -103,7 +103,7 @@ DEBUG: Router planner does not support append-partitioned tables. LOG: join order: [ "lineitem" ] QUERY PLAN ------------------------------------------------------------------------------- - Custom Scan (Citus Real-Time) + Custom Scan (Citus Adaptive) Task Count: 2 Tasks Shown: All -> Task @@ -175,17 +175,17 @@ DEBUG: Router planner does not support append-partitioned tables. LOG: join order: [ "lineitem" ] QUERY PLAN ------------------------------------------------------------------------------- - Custom Scan (Citus Real-Time) + Custom Scan (Citus Adaptive) Task Count: 2 Tasks Shown: All - -> Task - Node: host=localhost port=57638 dbname=regression - -> Index Scan using lineitem_pkey_290000 on lineitem_290000 lineitem - Index Cond: (l_orderkey = 9030) -> Task Node: host=localhost port=57637 dbname=regression -> Index Scan using lineitem_pkey_290001 on lineitem_290001 lineitem Index Cond: (l_orderkey = 9030) + -> Task + Node: host=localhost port=57638 dbname=regression + -> Index Scan using lineitem_pkey_290000 on lineitem_290000 lineitem + Index Cond: (l_orderkey = 9030) (11 rows) EXPLAIN (COSTS FALSE) @@ -248,7 +248,7 @@ LOG: join order: [ "lineitem" ] DEBUG: Plan is router executable QUERY PLAN ------------------------------------------------------------------------------- - Custom Scan (Citus Router) + Custom Scan (Citus Adaptive) Task Count: 1 Tasks Shown: All -> Task diff --git a/src/test/regress/expected/multi_null_minmax_value_pruning_0.out b/src/test/regress/expected/multi_null_minmax_value_pruning_0.out index dd1238349..a047dd7c2 100644 --- a/src/test/regress/expected/multi_null_minmax_value_pruning_0.out +++ b/src/test/regress/expected/multi_null_minmax_value_pruning_0.out @@ -5,10 +5,10 @@ -- and that we don't partition or join prune shards that have null values. -- print major version number for version-specific tests SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int AS server_version; - server_version ----------------- - 10 +SELECT substring(:'server_version', '\d+')::int > 10 AS server_version_above_ten; + server_version_above_ten +-------------------------- + f (1 row) SET client_min_messages TO DEBUG2; diff --git a/src/test/regress/expected/replicated_partitioned_table.out b/src/test/regress/expected/replicated_partitioned_table.out index df990cb02..72ff280a1 100644 --- a/src/test/regress/expected/replicated_partitioned_table.out +++ b/src/test/regress/expected/replicated_partitioned_table.out @@ -8,10 +8,10 @@ SET citus.shard_count TO 4; SET citus.shard_replication_factor TO 2; -- print major version number for version-specific tests SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int AS server_version; - server_version ----------------- - 11 +SELECT substring(:'server_version', '\d+')::int < 11 AS server_version_is_10; + server_version_is_10 +---------------------- + f (1 row) CREATE TABLE collections ( diff --git a/src/test/regress/expected/replicated_partitioned_table_0.out b/src/test/regress/expected/replicated_partitioned_table_0.out index 015b644a7..08206f63f 100644 --- a/src/test/regress/expected/replicated_partitioned_table_0.out +++ b/src/test/regress/expected/replicated_partitioned_table_0.out @@ -8,10 +8,10 @@ SET citus.shard_count TO 4; SET citus.shard_replication_factor TO 2; -- print major version number for version-specific tests SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int AS server_version; - server_version ----------------- - 10 +SELECT substring(:'server_version', '\d+')::int < 11 AS server_version_is_10; + server_version_is_10 +---------------------- + t (1 row) CREATE TABLE collections ( diff --git a/src/test/regress/expected/with_dml.out b/src/test/regress/expected/with_dml.out index 45a482f6e..b1d09b315 100644 --- a/src/test/regress/expected/with_dml.out +++ b/src/test/regress/expected/with_dml.out @@ -66,9 +66,10 @@ FROM ids_to_delete, (SELECT tenant_id FROM distributed_table WHERE tenant_id::int < 60) as some_tenants WHERE some_tenants.tenant_id = ids_to_delete.tenant_id - AND distributed_table.tenant_id = some_tenants.tenant_id; + AND distributed_table.tenant_id = some_tenants.tenant_id + AND EXISTS (SELECT * FROM ids_to_delete); DEBUG: generating subplan 12_1 for CTE ids_to_delete: SELECT tenant_id FROM with_dml.distributed_table WHERE (dept OPERATOR(pg_catalog.=) 5) -DEBUG: Plan 12 query after replacing subqueries and CTEs: UPDATE with_dml.distributed_table SET dept = (distributed_table.dept OPERATOR(pg_catalog.+) 1) FROM (SELECT intermediate_result.tenant_id FROM read_intermediate_result('12_1'::text, 'binary'::citus_copy_format) intermediate_result(tenant_id text)) ids_to_delete, (SELECT distributed_table_1.tenant_id FROM with_dml.distributed_table distributed_table_1 WHERE ((distributed_table_1.tenant_id)::integer OPERATOR(pg_catalog.<) 60)) some_tenants WHERE ((some_tenants.tenant_id OPERATOR(pg_catalog.=) ids_to_delete.tenant_id) AND (distributed_table.tenant_id OPERATOR(pg_catalog.=) some_tenants.tenant_id)) +DEBUG: Plan 12 query after replacing subqueries and CTEs: UPDATE with_dml.distributed_table SET dept = (distributed_table.dept OPERATOR(pg_catalog.+) 1) FROM (SELECT intermediate_result.tenant_id FROM read_intermediate_result('12_1'::text, 'binary'::citus_copy_format) intermediate_result(tenant_id text)) ids_to_delete, (SELECT distributed_table_1.tenant_id FROM with_dml.distributed_table distributed_table_1 WHERE ((distributed_table_1.tenant_id)::integer OPERATOR(pg_catalog.<) 60)) some_tenants WHERE ((some_tenants.tenant_id OPERATOR(pg_catalog.=) ids_to_delete.tenant_id) AND (distributed_table.tenant_id OPERATOR(pg_catalog.=) some_tenants.tenant_id) AND (EXISTS (SELECT ids_to_delete_1.tenant_id FROM (SELECT intermediate_result.tenant_id FROM read_intermediate_result('12_1'::text, 'binary'::citus_copy_format) intermediate_result(tenant_id text)) ids_to_delete_1))) -- this query errors out since we've some hard -- errors in the INSERT ... SELECT pushdown -- which prevents to fallback to recursive planning diff --git a/src/test/regress/sql/limit_intermediate_size.sql b/src/test/regress/sql/limit_intermediate_size.sql index 64326847e..4cf8b52ce 100644 --- a/src/test/regress/sql/limit_intermediate_size.sql +++ b/src/test/regress/sql/limit_intermediate_size.sql @@ -84,8 +84,8 @@ UNION )a ORDER BY 1,2,3,4,5 LIMIT 10; --- this should fail since the cte-subplan exceeds the limit even if the --- cte2 and cte3 does not +-- this fails since cte-subplan exceeds limit even if cte2 and cte3 don't +-- WHERE EXISTS forces materialization in pg12 SET citus.max_intermediate_result_size TO 4; WITH cte AS ( WITH cte2 AS ( @@ -95,8 +95,9 @@ WITH cte AS ( SELECT * FROM events_table ) SELECT * FROM cte2, cte3 WHERE cte2.user_id = cte3.user_id AND cte2.user_id = 1 + AND EXISTS (select * from cte2, cte3) ) -SELECT * FROM cte; +SELECT * FROM cte WHERE EXISTS (select * from cte); SET citus.max_intermediate_result_size TO 3; diff --git a/src/test/regress/sql/multi_explain.sql b/src/test/regress/sql/multi_explain.sql index 29123d2a8..12682e9c6 100644 --- a/src/test/regress/sql/multi_explain.sql +++ b/src/test/regress/sql/multi_explain.sql @@ -532,11 +532,12 @@ EXPLAIN (COSTS OFF) INSERT INTO lineitem_hash_part (l_orderkey) SELECT s FROM generate_series(1,5) s; +-- WHERE EXISTS forces pg12 to materialize cte EXPLAIN (COSTS OFF) WITH cte1 AS (SELECT s FROM generate_series(1,10) s) INSERT INTO lineitem_hash_part -WITH cte1 AS (SELECT * FROM cte1 LIMIT 5) -SELECT s FROM cte1; +WITH cte1 AS (SELECT * FROM cte1 WHERE EXISTS (SELECT * FROM cte1) LIMIT 5) +SELECT s FROM cte1 WHERE EXISTS (SELECT * FROM cte1); EXPLAIN (COSTS OFF) INSERT INTO lineitem_hash_part diff --git a/src/test/regress/sql/multi_index_statements.sql b/src/test/regress/sql/multi_index_statements.sql index 89fb7bc54..148f7e860 100644 --- a/src/test/regress/sql/multi_index_statements.sql +++ b/src/test/regress/sql/multi_index_statements.sql @@ -6,7 +6,7 @@ -- tables. SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int AS major_version; +SELECT substring(:'server_version', '\d+')::int > 10 AS server_version_above_ten; -- -- CREATE TEST TABLES diff --git a/src/test/regress/sql/multi_null_minmax_value_pruning.sql b/src/test/regress/sql/multi_null_minmax_value_pruning.sql index 3a1c0694a..89fc26731 100644 --- a/src/test/regress/sql/multi_null_minmax_value_pruning.sql +++ b/src/test/regress/sql/multi_null_minmax_value_pruning.sql @@ -8,7 +8,7 @@ -- print major version number for version-specific tests SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int AS server_version; +SELECT substring(:'server_version', '\d+')::int > 10 AS server_version_above_ten; SET client_min_messages TO DEBUG2; SET citus.explain_all_tasks TO on; diff --git a/src/test/regress/sql/replicated_partitioned_table.sql b/src/test/regress/sql/replicated_partitioned_table.sql index eba4b8d35..44c787adf 100644 --- a/src/test/regress/sql/replicated_partitioned_table.sql +++ b/src/test/regress/sql/replicated_partitioned_table.sql @@ -11,7 +11,7 @@ SET citus.shard_replication_factor TO 2; -- print major version number for version-specific tests SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int AS server_version; +SELECT substring(:'server_version', '\d+')::int < 11 AS server_version_is_10; CREATE TABLE collections ( diff --git a/src/test/regress/sql/with_dml.sql b/src/test/regress/sql/with_dml.sql index db1f42324..7b8a235b7 100644 --- a/src/test/regress/sql/with_dml.sql +++ b/src/test/regress/sql/with_dml.sql @@ -53,7 +53,8 @@ FROM ids_to_delete, (SELECT tenant_id FROM distributed_table WHERE tenant_id::int < 60) as some_tenants WHERE some_tenants.tenant_id = ids_to_delete.tenant_id - AND distributed_table.tenant_id = some_tenants.tenant_id; + AND distributed_table.tenant_id = some_tenants.tenant_id + AND EXISTS (SELECT * FROM ids_to_delete); -- this query errors out since we've some hard -- errors in the INSERT ... SELECT pushdown From a523a5b773918f916d9cb5d184d83b4d3237ac68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philip=20Dub=C3=A9?= Date: Thu, 25 Jul 2019 02:20:04 +0000 Subject: [PATCH 05/13] multi_null_minmax_value_pruning: no versioning & coordinator_plan --- .../multi_null_minmax_value_pruning.out | 91 ++--- .../multi_null_minmax_value_pruning_0.out | 312 ------------------ .../sql/multi_null_minmax_value_pruning.sql | 12 +- 3 files changed, 37 insertions(+), 378 deletions(-) delete mode 100644 src/test/regress/expected/multi_null_minmax_value_pruning_0.out diff --git a/src/test/regress/expected/multi_null_minmax_value_pruning.out b/src/test/regress/expected/multi_null_minmax_value_pruning.out index fe294b5ae..852215fef 100644 --- a/src/test/regress/expected/multi_null_minmax_value_pruning.out +++ b/src/test/regress/expected/multi_null_minmax_value_pruning.out @@ -3,14 +3,6 @@ -- -- This test checks that we can handle null min/max values in shard statistics -- and that we don't partition or join prune shards that have null values. --- print major version number for version-specific tests -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int > 10 AS server_version_above_ten; - server_version_above_ten --------------------------- - t -(1 row) - SET client_min_messages TO DEBUG2; SET citus.explain_all_tasks TO on; -- to avoid differing explain output - executor doesn't matter, @@ -33,34 +25,19 @@ SELECT shardminvalue, shardmaxvalue from pg_dist_shard WHERE shardid = 290001; -- Check that partition and join pruning works when min/max values exist -- Adding l_orderkey = 1 to make the query not router executable +SELECT coordinator_plan($Q$ EXPLAIN (COSTS FALSE) SELECT l_orderkey, l_linenumber, l_shipdate FROM lineitem WHERE l_orderkey = 9030 or l_orderkey = 1; +$Q$); DEBUG: Router planner does not support append-partitioned tables. +CONTEXT: PL/pgSQL function coordinator_plan(text) line 3 at FOR over EXECUTE statement LOG: join order: [ "lineitem" ] - QUERY PLAN ------------------------------------------------------------------------ +CONTEXT: PL/pgSQL function coordinator_plan(text) line 3 at FOR over EXECUTE statement + coordinator_plan +------------------------------ Custom Scan (Citus Adaptive) Task Count: 2 - Tasks Shown: All - -> Task - Node: host=localhost port=57637 dbname=regression - -> Bitmap Heap Scan on lineitem_290001 lineitem - Recheck Cond: ((l_orderkey = 9030) OR (l_orderkey = 1)) - -> BitmapOr - -> Bitmap Index Scan on lineitem_pkey_290001 - Index Cond: (l_orderkey = 9030) - -> Bitmap Index Scan on lineitem_pkey_290001 - Index Cond: (l_orderkey = 1) - -> Task - Node: host=localhost port=57638 dbname=regression - -> Bitmap Heap Scan on lineitem_290000 lineitem - Recheck Cond: ((l_orderkey = 9030) OR (l_orderkey = 1)) - -> BitmapOr - -> Bitmap Index Scan on lineitem_pkey_290000 - Index Cond: (l_orderkey = 9030) - -> Bitmap Index Scan on lineitem_pkey_290000 - Index Cond: (l_orderkey = 1) -(21 rows) +(2 rows) EXPLAIN (COSTS FALSE) SELECT sum(l_linenumber), avg(l_linenumber) FROM lineitem, orders @@ -97,24 +74,19 @@ DEBUG: join prunable for intervals [8997,14947] and [1,5986] -- partition or join pruning for the shard with null min value. Since it is not -- supported with single-repartition join, dual-repartition has been used. UPDATE pg_dist_shard SET shardminvalue = NULL WHERE shardid = 290000; +SELECT coordinator_plan($Q$ EXPLAIN (COSTS FALSE) SELECT l_orderkey, l_linenumber, l_shipdate FROM lineitem WHERE l_orderkey = 9030; +$Q$); DEBUG: Router planner does not support append-partitioned tables. +CONTEXT: PL/pgSQL function coordinator_plan(text) line 3 at FOR over EXECUTE statement LOG: join order: [ "lineitem" ] - QUERY PLAN -------------------------------------------------------------------------------- +CONTEXT: PL/pgSQL function coordinator_plan(text) line 3 at FOR over EXECUTE statement + coordinator_plan +------------------------------ Custom Scan (Citus Adaptive) Task Count: 2 - Tasks Shown: All - -> Task - Node: host=localhost port=57637 dbname=regression - -> Index Scan using lineitem_pkey_290001 on lineitem_290001 lineitem - Index Cond: (l_orderkey = 9030) - -> Task - Node: host=localhost port=57638 dbname=regression - -> Index Scan using lineitem_pkey_290000 on lineitem_290000 lineitem - Index Cond: (l_orderkey = 9030) -(11 rows) +(2 rows) EXPLAIN (COSTS FALSE) SELECT sum(l_linenumber), avg(l_linenumber) FROM lineitem, orders @@ -169,24 +141,19 @@ HINT: Since you enabled citus.enable_repartition_joins Citus chose to use task- -- don't apply partition or join pruning for this other shard either. Since it -- is not supported with single-repartition join, dual-repartition has been used. UPDATE pg_dist_shard SET shardmaxvalue = NULL WHERE shardid = 290001; +SELECT coordinator_plan($Q$ EXPLAIN (COSTS FALSE) SELECT l_orderkey, l_linenumber, l_shipdate FROM lineitem WHERE l_orderkey = 9030; +$Q$); DEBUG: Router planner does not support append-partitioned tables. +CONTEXT: PL/pgSQL function coordinator_plan(text) line 3 at FOR over EXECUTE statement LOG: join order: [ "lineitem" ] - QUERY PLAN -------------------------------------------------------------------------------- +CONTEXT: PL/pgSQL function coordinator_plan(text) line 3 at FOR over EXECUTE statement + coordinator_plan +------------------------------ Custom Scan (Citus Adaptive) Task Count: 2 - Tasks Shown: All - -> Task - Node: host=localhost port=57637 dbname=regression - -> Index Scan using lineitem_pkey_290001 on lineitem_290001 lineitem - Index Cond: (l_orderkey = 9030) - -> Task - Node: host=localhost port=57638 dbname=regression - -> Index Scan using lineitem_pkey_290000 on lineitem_290000 lineitem - Index Cond: (l_orderkey = 9030) -(11 rows) +(2 rows) EXPLAIN (COSTS FALSE) SELECT sum(l_linenumber), avg(l_linenumber) FROM lineitem, orders @@ -241,21 +208,21 @@ HINT: Since you enabled citus.enable_repartition_joins Citus chose to use task- -- should apply partition and join pruning for this shard now. Since it is not -- supported with single-repartition join, dual-repartition has been used. UPDATE pg_dist_shard SET shardminvalue = '0' WHERE shardid = 290000; +SELECT coordinator_plan($Q$ EXPLAIN (COSTS FALSE) SELECT l_orderkey, l_linenumber, l_shipdate FROM lineitem WHERE l_orderkey = 9030; +$Q$); DEBUG: Router planner does not support append-partitioned tables. +CONTEXT: PL/pgSQL function coordinator_plan(text) line 3 at FOR over EXECUTE statement LOG: join order: [ "lineitem" ] +CONTEXT: PL/pgSQL function coordinator_plan(text) line 3 at FOR over EXECUTE statement DEBUG: Plan is router executable - QUERY PLAN -------------------------------------------------------------------------------- +CONTEXT: PL/pgSQL function coordinator_plan(text) line 3 at FOR over EXECUTE statement + coordinator_plan +------------------------------ Custom Scan (Citus Adaptive) Task Count: 1 - Tasks Shown: All - -> Task - Node: host=localhost port=57637 dbname=regression - -> Index Scan using lineitem_pkey_290001 on lineitem_290001 lineitem - Index Cond: (l_orderkey = 9030) -(7 rows) +(2 rows) EXPLAIN (COSTS FALSE) SELECT sum(l_linenumber), avg(l_linenumber) FROM lineitem, orders diff --git a/src/test/regress/expected/multi_null_minmax_value_pruning_0.out b/src/test/regress/expected/multi_null_minmax_value_pruning_0.out deleted file mode 100644 index a047dd7c2..000000000 --- a/src/test/regress/expected/multi_null_minmax_value_pruning_0.out +++ /dev/null @@ -1,312 +0,0 @@ --- --- MULTI_NULL_MINMAX_VALUE_PRUNING --- --- This test checks that we can handle null min/max values in shard statistics --- and that we don't partition or join prune shards that have null values. --- print major version number for version-specific tests -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int > 10 AS server_version_above_ten; - server_version_above_ten --------------------------- - f -(1 row) - -SET client_min_messages TO DEBUG2; -SET citus.explain_all_tasks TO on; --- to avoid differing explain output - executor doesn't matter, --- because were testing pruning here. -RESET citus.task_executor_type; --- Change configuration to treat lineitem and orders tables as large -SET citus.log_multi_join_order to true; -SET citus.enable_repartition_joins to ON; -SELECT shardminvalue, shardmaxvalue from pg_dist_shard WHERE shardid = 290000; - shardminvalue | shardmaxvalue ----------------+--------------- - 1 | 5986 -(1 row) - -SELECT shardminvalue, shardmaxvalue from pg_dist_shard WHERE shardid = 290001; - shardminvalue | shardmaxvalue ----------------+--------------- - 8997 | 14947 -(1 row) - --- Check that partition and join pruning works when min/max values exist --- Adding l_orderkey = 1 to make the query not router executable -EXPLAIN (COSTS FALSE) -SELECT l_orderkey, l_linenumber, l_shipdate FROM lineitem WHERE l_orderkey = 9030 or l_orderkey = 1; -DEBUG: Router planner does not support append-partitioned tables. -LOG: join order: [ "lineitem" ] - QUERY PLAN ------------------------------------------------------------------------ - Custom Scan (Citus Adaptive) - Task Count: 2 - Tasks Shown: All - -> Task - Node: host=localhost port=57637 dbname=regression - -> Bitmap Heap Scan on lineitem_290001 lineitem - Recheck Cond: ((l_orderkey = 9030) OR (l_orderkey = 1)) - -> BitmapOr - -> Bitmap Index Scan on lineitem_pkey_290001 - Index Cond: (l_orderkey = 9030) - -> Bitmap Index Scan on lineitem_pkey_290001 - Index Cond: (l_orderkey = 1) - -> Task - Node: host=localhost port=57638 dbname=regression - -> Bitmap Heap Scan on lineitem_290000 lineitem - Recheck Cond: ((l_orderkey = 9030) OR (l_orderkey = 1)) - -> BitmapOr - -> Bitmap Index Scan on lineitem_pkey_290000 - Index Cond: (l_orderkey = 9030) - -> Bitmap Index Scan on lineitem_pkey_290000 - Index Cond: (l_orderkey = 1) -(21 rows) - -EXPLAIN (COSTS FALSE) -SELECT sum(l_linenumber), avg(l_linenumber) FROM lineitem, orders - WHERE l_orderkey = o_orderkey; -DEBUG: Router planner does not support append-partitioned tables. -LOG: join order: [ "lineitem" ][ local partition join "orders" ] -DEBUG: join prunable for intervals [1,5986] and [8997,14947] -DEBUG: join prunable for intervals [8997,14947] and [1,5986] - QUERY PLAN --------------------------------------------------------------------------------- - Aggregate - -> Custom Scan (Citus Adaptive) - Task Count: 2 - Tasks Shown: All - -> Task - Node: host=localhost port=57638 dbname=regression - -> Aggregate - -> Hash Join - Hash Cond: (lineitem.l_orderkey = orders.o_orderkey) - -> Seq Scan on lineitem_290000 lineitem - -> Hash - -> Seq Scan on orders_290002 orders - -> Task - Node: host=localhost port=57637 dbname=regression - -> Aggregate - -> Hash Join - Hash Cond: (lineitem.l_orderkey = orders.o_orderkey) - -> Seq Scan on lineitem_290001 lineitem - -> Hash - -> Seq Scan on orders_290003 orders -(20 rows) - --- Now set the minimum value for a shard to null. Then check that we don't apply --- partition or join pruning for the shard with null min value. Since it is not --- supported with single-repartition join, dual-repartition has been used. -UPDATE pg_dist_shard SET shardminvalue = NULL WHERE shardid = 290000; -EXPLAIN (COSTS FALSE) -SELECT l_orderkey, l_linenumber, l_shipdate FROM lineitem WHERE l_orderkey = 9030; -DEBUG: Router planner does not support append-partitioned tables. -LOG: join order: [ "lineitem" ] - QUERY PLAN -------------------------------------------------------------------------------- - Custom Scan (Citus Adaptive) - Task Count: 2 - Tasks Shown: All - -> Task - Node: host=localhost port=57637 dbname=regression - -> Index Scan using lineitem_pkey_290001 on lineitem_290001 lineitem - Index Cond: (l_orderkey = 9030) - -> Task - Node: host=localhost port=57638 dbname=regression - -> Index Scan using lineitem_pkey_290000 on lineitem_290000 lineitem - Index Cond: (l_orderkey = 9030) -(11 rows) - -EXPLAIN (COSTS FALSE) -SELECT sum(l_linenumber), avg(l_linenumber) FROM lineitem, orders - WHERE l_partkey = o_custkey; -DEBUG: Router planner does not support append-partitioned tables. -LOG: join order: [ "lineitem" ][ dual partition join "orders" ] -DEBUG: join prunable for task partitionId 0 and 1 -DEBUG: join prunable for task partitionId 0 and 2 -DEBUG: join prunable for task partitionId 0 and 3 -DEBUG: join prunable for task partitionId 1 and 0 -DEBUG: join prunable for task partitionId 1 and 2 -DEBUG: join prunable for task partitionId 1 and 3 -DEBUG: join prunable for task partitionId 2 and 0 -DEBUG: join prunable for task partitionId 2 and 1 -DEBUG: join prunable for task partitionId 2 and 3 -DEBUG: join prunable for task partitionId 3 and 0 -DEBUG: join prunable for task partitionId 3 and 1 -DEBUG: join prunable for task partitionId 3 and 2 -DEBUG: pruning merge fetch taskId 1 -DETAIL: Creating dependency on merge taskId 3 -DEBUG: pruning merge fetch taskId 2 -DETAIL: Creating dependency on merge taskId 3 -DEBUG: pruning merge fetch taskId 4 -DETAIL: Creating dependency on merge taskId 6 -DEBUG: pruning merge fetch taskId 5 -DETAIL: Creating dependency on merge taskId 6 -DEBUG: pruning merge fetch taskId 7 -DETAIL: Creating dependency on merge taskId 9 -DEBUG: pruning merge fetch taskId 8 -DETAIL: Creating dependency on merge taskId 9 -DEBUG: pruning merge fetch taskId 10 -DETAIL: Creating dependency on merge taskId 12 -DEBUG: pruning merge fetch taskId 11 -DETAIL: Creating dependency on merge taskId 12 -DEBUG: cannot use real time executor with repartition jobs -HINT: Since you enabled citus.enable_repartition_joins Citus chose to use task-tracker. - QUERY PLAN -------------------------------------------------------------------- - Aggregate - -> Custom Scan (Citus Task-Tracker) - Task Count: 4 - Tasks Shown: None, not supported for re-partition queries - -> MapMergeJob - Map Task Count: 2 - Merge Task Count: 4 - -> MapMergeJob - Map Task Count: 2 - Merge Task Count: 4 -(10 rows) - --- Next, set the maximum value for another shard to null. Then check that we --- don't apply partition or join pruning for this other shard either. Since it --- is not supported with single-repartition join, dual-repartition has been used. -UPDATE pg_dist_shard SET shardmaxvalue = NULL WHERE shardid = 290001; -EXPLAIN (COSTS FALSE) -SELECT l_orderkey, l_linenumber, l_shipdate FROM lineitem WHERE l_orderkey = 9030; -DEBUG: Router planner does not support append-partitioned tables. -LOG: join order: [ "lineitem" ] - QUERY PLAN -------------------------------------------------------------------------------- - Custom Scan (Citus Adaptive) - Task Count: 2 - Tasks Shown: All - -> Task - Node: host=localhost port=57638 dbname=regression - -> Index Scan using lineitem_pkey_290000 on lineitem_290000 lineitem - Index Cond: (l_orderkey = 9030) - -> Task - Node: host=localhost port=57637 dbname=regression - -> Index Scan using lineitem_pkey_290001 on lineitem_290001 lineitem - Index Cond: (l_orderkey = 9030) -(11 rows) - -EXPLAIN (COSTS FALSE) -SELECT sum(l_linenumber), avg(l_linenumber) FROM lineitem, orders - WHERE l_partkey = o_custkey; -DEBUG: Router planner does not support append-partitioned tables. -LOG: join order: [ "lineitem" ][ dual partition join "orders" ] -DEBUG: join prunable for task partitionId 0 and 1 -DEBUG: join prunable for task partitionId 0 and 2 -DEBUG: join prunable for task partitionId 0 and 3 -DEBUG: join prunable for task partitionId 1 and 0 -DEBUG: join prunable for task partitionId 1 and 2 -DEBUG: join prunable for task partitionId 1 and 3 -DEBUG: join prunable for task partitionId 2 and 0 -DEBUG: join prunable for task partitionId 2 and 1 -DEBUG: join prunable for task partitionId 2 and 3 -DEBUG: join prunable for task partitionId 3 and 0 -DEBUG: join prunable for task partitionId 3 and 1 -DEBUG: join prunable for task partitionId 3 and 2 -DEBUG: pruning merge fetch taskId 1 -DETAIL: Creating dependency on merge taskId 3 -DEBUG: pruning merge fetch taskId 2 -DETAIL: Creating dependency on merge taskId 3 -DEBUG: pruning merge fetch taskId 4 -DETAIL: Creating dependency on merge taskId 6 -DEBUG: pruning merge fetch taskId 5 -DETAIL: Creating dependency on merge taskId 6 -DEBUG: pruning merge fetch taskId 7 -DETAIL: Creating dependency on merge taskId 9 -DEBUG: pruning merge fetch taskId 8 -DETAIL: Creating dependency on merge taskId 9 -DEBUG: pruning merge fetch taskId 10 -DETAIL: Creating dependency on merge taskId 12 -DEBUG: pruning merge fetch taskId 11 -DETAIL: Creating dependency on merge taskId 12 -DEBUG: cannot use real time executor with repartition jobs -HINT: Since you enabled citus.enable_repartition_joins Citus chose to use task-tracker. - QUERY PLAN -------------------------------------------------------------------- - Aggregate - -> Custom Scan (Citus Task-Tracker) - Task Count: 4 - Tasks Shown: None, not supported for re-partition queries - -> MapMergeJob - Map Task Count: 2 - Merge Task Count: 4 - -> MapMergeJob - Map Task Count: 2 - Merge Task Count: 4 -(10 rows) - --- Last, set the minimum value to 0 and check that we don't treat it as null. We --- should apply partition and join pruning for this shard now. Since it is not --- supported with single-repartition join, dual-repartition has been used. -UPDATE pg_dist_shard SET shardminvalue = '0' WHERE shardid = 290000; -EXPLAIN (COSTS FALSE) -SELECT l_orderkey, l_linenumber, l_shipdate FROM lineitem WHERE l_orderkey = 9030; -DEBUG: Router planner does not support append-partitioned tables. -LOG: join order: [ "lineitem" ] -DEBUG: Plan is router executable - QUERY PLAN -------------------------------------------------------------------------------- - Custom Scan (Citus Adaptive) - Task Count: 1 - Tasks Shown: All - -> Task - Node: host=localhost port=57637 dbname=regression - -> Index Scan using lineitem_pkey_290001 on lineitem_290001 lineitem - Index Cond: (l_orderkey = 9030) -(7 rows) - -EXPLAIN (COSTS FALSE) -SELECT sum(l_linenumber), avg(l_linenumber) FROM lineitem, orders - WHERE l_partkey = o_custkey; -DEBUG: Router planner does not support append-partitioned tables. -LOG: join order: [ "lineitem" ][ dual partition join "orders" ] -DEBUG: join prunable for task partitionId 0 and 1 -DEBUG: join prunable for task partitionId 0 and 2 -DEBUG: join prunable for task partitionId 0 and 3 -DEBUG: join prunable for task partitionId 1 and 0 -DEBUG: join prunable for task partitionId 1 and 2 -DEBUG: join prunable for task partitionId 1 and 3 -DEBUG: join prunable for task partitionId 2 and 0 -DEBUG: join prunable for task partitionId 2 and 1 -DEBUG: join prunable for task partitionId 2 and 3 -DEBUG: join prunable for task partitionId 3 and 0 -DEBUG: join prunable for task partitionId 3 and 1 -DEBUG: join prunable for task partitionId 3 and 2 -DEBUG: pruning merge fetch taskId 1 -DETAIL: Creating dependency on merge taskId 3 -DEBUG: pruning merge fetch taskId 2 -DETAIL: Creating dependency on merge taskId 3 -DEBUG: pruning merge fetch taskId 4 -DETAIL: Creating dependency on merge taskId 6 -DEBUG: pruning merge fetch taskId 5 -DETAIL: Creating dependency on merge taskId 6 -DEBUG: pruning merge fetch taskId 7 -DETAIL: Creating dependency on merge taskId 9 -DEBUG: pruning merge fetch taskId 8 -DETAIL: Creating dependency on merge taskId 9 -DEBUG: pruning merge fetch taskId 10 -DETAIL: Creating dependency on merge taskId 12 -DEBUG: pruning merge fetch taskId 11 -DETAIL: Creating dependency on merge taskId 12 -DEBUG: cannot use real time executor with repartition jobs -HINT: Since you enabled citus.enable_repartition_joins Citus chose to use task-tracker. - QUERY PLAN -------------------------------------------------------------------- - Aggregate - -> Custom Scan (Citus Task-Tracker) - Task Count: 4 - Tasks Shown: None, not supported for re-partition queries - -> MapMergeJob - Map Task Count: 2 - Merge Task Count: 4 - -> MapMergeJob - Map Task Count: 2 - Merge Task Count: 4 -(10 rows) - --- Set minimum and maximum values for two shards back to their original values -UPDATE pg_dist_shard SET shardminvalue = '1' WHERE shardid = 290000; -UPDATE pg_dist_shard SET shardmaxvalue = '14947' WHERE shardid = 290001; -SET client_min_messages TO NOTICE; diff --git a/src/test/regress/sql/multi_null_minmax_value_pruning.sql b/src/test/regress/sql/multi_null_minmax_value_pruning.sql index 89fc26731..843898daa 100644 --- a/src/test/regress/sql/multi_null_minmax_value_pruning.sql +++ b/src/test/regress/sql/multi_null_minmax_value_pruning.sql @@ -6,10 +6,6 @@ -- and that we don't partition or join prune shards that have null values. --- print major version number for version-specific tests -SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int > 10 AS server_version_above_ten; - SET client_min_messages TO DEBUG2; SET citus.explain_all_tasks TO on; -- to avoid differing explain output - executor doesn't matter, @@ -26,8 +22,10 @@ SELECT shardminvalue, shardmaxvalue from pg_dist_shard WHERE shardid = 290001; -- Check that partition and join pruning works when min/max values exist -- Adding l_orderkey = 1 to make the query not router executable +SELECT coordinator_plan($Q$ EXPLAIN (COSTS FALSE) SELECT l_orderkey, l_linenumber, l_shipdate FROM lineitem WHERE l_orderkey = 9030 or l_orderkey = 1; +$Q$); EXPLAIN (COSTS FALSE) SELECT sum(l_linenumber), avg(l_linenumber) FROM lineitem, orders @@ -39,8 +37,10 @@ SELECT sum(l_linenumber), avg(l_linenumber) FROM lineitem, orders UPDATE pg_dist_shard SET shardminvalue = NULL WHERE shardid = 290000; +SELECT coordinator_plan($Q$ EXPLAIN (COSTS FALSE) SELECT l_orderkey, l_linenumber, l_shipdate FROM lineitem WHERE l_orderkey = 9030; +$Q$); EXPLAIN (COSTS FALSE) SELECT sum(l_linenumber), avg(l_linenumber) FROM lineitem, orders @@ -52,8 +52,10 @@ SELECT sum(l_linenumber), avg(l_linenumber) FROM lineitem, orders UPDATE pg_dist_shard SET shardmaxvalue = NULL WHERE shardid = 290001; +SELECT coordinator_plan($Q$ EXPLAIN (COSTS FALSE) SELECT l_orderkey, l_linenumber, l_shipdate FROM lineitem WHERE l_orderkey = 9030; +$Q$); EXPLAIN (COSTS FALSE) SELECT sum(l_linenumber), avg(l_linenumber) FROM lineitem, orders @@ -65,8 +67,10 @@ SELECT sum(l_linenumber), avg(l_linenumber) FROM lineitem, orders UPDATE pg_dist_shard SET shardminvalue = '0' WHERE shardid = 290000; +SELECT coordinator_plan($Q$ EXPLAIN (COSTS FALSE) SELECT l_orderkey, l_linenumber, l_shipdate FROM lineitem WHERE l_orderkey = 9030; +$Q$); EXPLAIN (COSTS FALSE) SELECT sum(l_linenumber), avg(l_linenumber) FROM lineitem, orders From 546b71ac18a7527386c51509dbe23d93febf2aba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philip=20Dub=C3=A9?= Date: Thu, 25 Jul 2019 21:21:00 +0000 Subject: [PATCH 06/13] multi_router_planner: be terse for ctes with false wheres --- src/test/regress/expected/multi_router_planner.out | 5 +++-- src/test/regress/sql/multi_router_planner.sql | 5 +++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/test/regress/expected/multi_router_planner.out b/src/test/regress/expected/multi_router_planner.out index d28d41244..926f4a158 100644 --- a/src/test/regress/expected/multi_router_planner.out +++ b/src/test/regress/expected/multi_router_planner.out @@ -1573,12 +1573,13 @@ DETAIL: distribution column value: 1 (0 rows) -- CTEs with where false +-- terse because distribution column inference varies between pg11 & pg12 +\set VERBOSITY terse WITH id_author AS ( SELECT id, author_id FROM articles_hash WHERE author_id = 1), id_title AS (SELECT id, title from articles_hash WHERE author_id = 1 and 1=0) SELECT * FROM id_author, id_title WHERE id_author.id = id_title.id; DEBUG: Creating router plan DEBUG: Plan is router executable -DETAIL: distribution column value: 1 id | author_id | id | title ----+-----------+----+------- (0 rows) @@ -1588,11 +1589,11 @@ id_title AS (SELECT id, title from articles_hash WHERE author_id = 1) SELECT * FROM id_author, id_title WHERE id_author.id = id_title.id and 1=0; DEBUG: Creating router plan DEBUG: Plan is router executable -DETAIL: distribution column value: 1 id | author_id | id | title ----+-----------+----+------- (0 rows) +\set VERBOSITY DEFAULT WITH RECURSIVE hierarchy as ( SELECT *, 1 AS level FROM company_employees diff --git a/src/test/regress/sql/multi_router_planner.sql b/src/test/regress/sql/multi_router_planner.sql index dcc4a1f38..74145a19a 100644 --- a/src/test/regress/sql/multi_router_planner.sql +++ b/src/test/regress/sql/multi_router_planner.sql @@ -681,6 +681,9 @@ INTERSECT (SELECT * FROM articles_hash WHERE author_id = 2 and 1=0); -- CTEs with where false +-- terse because distribution column inference varies between pg11 & pg12 +\set VERBOSITY terse + WITH id_author AS ( SELECT id, author_id FROM articles_hash WHERE author_id = 1), id_title AS (SELECT id, title from articles_hash WHERE author_id = 1 and 1=0) SELECT * FROM id_author, id_title WHERE id_author.id = id_title.id; @@ -689,6 +692,8 @@ WITH id_author AS ( SELECT id, author_id FROM articles_hash WHERE author_id = 1) id_title AS (SELECT id, title from articles_hash WHERE author_id = 1) SELECT * FROM id_author, id_title WHERE id_author.id = id_title.id and 1=0; +\set VERBOSITY DEFAULT + WITH RECURSIVE hierarchy as ( SELECT *, 1 AS level FROM company_employees From 7992077fd98744939d8f4e94fdfd486680df5671 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philip=20Dub=C3=A9?= Date: Thu, 25 Jul 2019 23:58:58 +0000 Subject: [PATCH 07/13] multi_modifying_xacts: don't differ in output if reference table select tries broken worker first --- src/test/regress/expected/multi_modifying_xacts.out | 2 ++ src/test/regress/expected/multi_modifying_xacts_9.out | 2 ++ src/test/regress/sql/multi_modifying_xacts.sql | 2 ++ 3 files changed, 6 insertions(+) diff --git a/src/test/regress/expected/multi_modifying_xacts.out b/src/test/regress/expected/multi_modifying_xacts.out index 3dc2db011..9495941c7 100644 --- a/src/test/regress/expected/multi_modifying_xacts.out +++ b/src/test/regress/expected/multi_modifying_xacts.out @@ -1186,11 +1186,13 @@ COPY reference_failure_test FROM STDIN WITH (FORMAT 'csv'); ERROR: connection error: localhost:57637 COMMIT; -- show that no data go through the table and shard states are good +SET client_min_messages to 'ERROR'; SELECT * FROM reference_failure_test; key | value -----+------- (0 rows) +RESET client_min_messages; -- all placements should be healthy SELECT s.logicalrelid::regclass::text, sp.shardstate, count(*) FROM pg_dist_shard_placement AS sp, diff --git a/src/test/regress/expected/multi_modifying_xacts_9.out b/src/test/regress/expected/multi_modifying_xacts_9.out index c59189392..cb6f1d756 100644 --- a/src/test/regress/expected/multi_modifying_xacts_9.out +++ b/src/test/regress/expected/multi_modifying_xacts_9.out @@ -1188,11 +1188,13 @@ COPY reference_failure_test FROM STDIN WITH (FORMAT 'csv'); ERROR: connection error: localhost:57637 COMMIT; -- show that no data go through the table and shard states are good +SET client_min_messages to 'ERROR'; SELECT * FROM reference_failure_test; key | value -----+------- (0 rows) +RESET client_min_messages; -- all placements should be healthy SELECT s.logicalrelid::regclass::text, sp.shardstate, count(*) FROM pg_dist_shard_placement AS sp, diff --git a/src/test/regress/sql/multi_modifying_xacts.sql b/src/test/regress/sql/multi_modifying_xacts.sql index 2d462e6bc..0250dc5fe 100644 --- a/src/test/regress/sql/multi_modifying_xacts.sql +++ b/src/test/regress/sql/multi_modifying_xacts.sql @@ -923,7 +923,9 @@ COPY reference_failure_test FROM STDIN WITH (FORMAT 'csv'); COMMIT; -- show that no data go through the table and shard states are good +SET client_min_messages to 'ERROR'; SELECT * FROM reference_failure_test; +RESET client_min_messages; -- all placements should be healthy From 0d9e5bde9c007e2821bf2fcde50bbdfcba6bf2e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philip=20Dub=C3=A9?= Date: Fri, 26 Jul 2019 00:12:28 +0000 Subject: [PATCH 08/13] window_functions: 'ORDER BY time' when using lag(time) & coordinator_plan --- .../regress/expected/window_functions.out | 34 +++++++------------ .../regress/expected/window_functions_0.out | 24 ++++--------- src/test/regress/sql/window_functions.sql | 4 ++- 3 files changed, 22 insertions(+), 40 deletions(-) diff --git a/src/test/regress/expected/window_functions.out b/src/test/regress/expected/window_functions.out index f9e004cc8..7efbe8ee3 100644 --- a/src/test/regress/expected/window_functions.out +++ b/src/test/regress/expected/window_functions.out @@ -302,7 +302,7 @@ SELECT COUNT(*) OVER (PARTITION BY user_id, user_id + 1), rank() OVER (PARTITION BY user_id) as cnt1, COUNT(*) OVER (PARTITION BY user_id, abs(value_1 - value_2)) as cnt2, - date_trunc('min', lag(time) OVER (PARTITION BY user_id)) as datee, + date_trunc('min', lag(time) OVER (PARTITION BY user_id ORDER BY time)) as datee, rank() OVER my_win as rnnk, avg(CASE WHEN user_id > 4 @@ -759,6 +759,7 @@ ORDER BY 1 | 3.2857142857142857 | 1.00000000000000000000 (6 rows) +SELECT coordinator_plan($Q$ EXPLAIN (COSTS FALSE) SELECT user_id, @@ -770,27 +771,16 @@ GROUP BY 1 ORDER BY 3 DESC, 2 DESC, 1 DESC; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------ +$Q$); + coordinator_plan +------------------------------------------------------------------------------------ Sort Sort Key: remote_scan.avg_1 DESC, remote_scan.avg DESC, remote_scan.user_id DESC -> HashAggregate Group Key: remote_scan.user_id - -> Custom Scan (Citus Real-Time) + -> Custom Scan (Citus Adaptive) Task Count: 4 - Tasks Shown: One of 4 - -> Task - Node: host=localhost port=57637 dbname=regression - -> WindowAgg - -> Sort - Sort Key: users_table.user_id, (min(users_table.user_id)), (avg(users_table.value_1)) - -> WindowAgg - -> Sort - Sort Key: users_table.user_id, (max(users_table.user_id)), (min(users_table.value_2)) - -> HashAggregate - Group Key: users_table.user_id - -> Seq Scan on users_table_1400256 users_table -(18 rows) +(6 rows) SELECT user_id, @@ -963,7 +953,7 @@ ORDER BY user_id, avg(value_1) DESC; Sort Key: remote_scan.user_id, (pg_catalog.sum(((pg_catalog.sum(remote_scan.avg) / pg_catalog.sum(remote_scan.avg_1)))) / pg_catalog.sum(remote_scan.rank)) DESC -> HashAggregate Group Key: remote_scan.user_id, remote_scan.worker_column_5 - -> Custom Scan (Citus Real-Time) + -> Custom Scan (Citus Adaptive) Task Count: 4 Tasks Shown: One of 4 -> Task @@ -1039,7 +1029,7 @@ LIMIT 5; Sort Key: remote_scan.user_id, (pg_catalog.sum(((pg_catalog.sum(remote_scan.avg) / pg_catalog.sum(remote_scan.avg_1)))) / pg_catalog.sum(remote_scan.rank)) DESC -> HashAggregate Group Key: remote_scan.user_id, remote_scan.worker_column_5 - -> Custom Scan (Citus Real-Time) + -> Custom Scan (Citus Adaptive) Task Count: 4 Tasks Shown: One of 4 -> Task @@ -1069,7 +1059,7 @@ LIMIT 5; Sort Key: remote_scan.user_id, (pg_catalog.sum(((pg_catalog.sum(remote_scan.avg) / pg_catalog.sum(remote_scan.avg_1)))) / pg_catalog.sum(remote_scan.rank)) DESC -> HashAggregate Group Key: remote_scan.user_id, remote_scan.worker_column_5 - -> Custom Scan (Citus Real-Time) + -> Custom Scan (Citus Adaptive) Task Count: 4 Tasks Shown: One of 4 -> Task @@ -1099,7 +1089,7 @@ LIMIT 5; Sort Key: remote_scan.user_id, (pg_catalog.sum(((pg_catalog.sum(remote_scan.avg) / pg_catalog.sum(remote_scan.avg_1)))) / pg_catalog.sum(remote_scan.rank)) DESC -> HashAggregate Group Key: remote_scan.user_id, remote_scan.worker_column_5 - -> Custom Scan (Citus Real-Time) + -> Custom Scan (Citus Adaptive) Task Count: 4 Tasks Shown: One of 4 -> Task @@ -1129,7 +1119,7 @@ LIMIT 5; Sort Key: remote_scan.user_id, (pg_catalog.sum(((pg_catalog.sum(remote_scan.avg) / pg_catalog.sum(remote_scan.avg_1)))) / pg_catalog.sum(remote_scan.rank)) DESC -> HashAggregate Group Key: remote_scan.user_id, remote_scan.worker_column_5 - -> Custom Scan (Citus Real-Time) + -> Custom Scan (Citus Adaptive) Task Count: 4 Tasks Shown: One of 4 -> Task diff --git a/src/test/regress/expected/window_functions_0.out b/src/test/regress/expected/window_functions_0.out index 185765ef4..de4d2f234 100644 --- a/src/test/regress/expected/window_functions_0.out +++ b/src/test/regress/expected/window_functions_0.out @@ -302,7 +302,7 @@ SELECT COUNT(*) OVER (PARTITION BY user_id, user_id + 1), rank() OVER (PARTITION BY user_id) as cnt1, COUNT(*) OVER (PARTITION BY user_id, abs(value_1 - value_2)) as cnt2, - date_trunc('min', lag(time) OVER (PARTITION BY user_id)) as datee, + date_trunc('min', lag(time) OVER (PARTITION BY user_id ORDER BY time)) as datee, rank() OVER my_win as rnnk, avg(CASE WHEN user_id > 4 @@ -558,6 +558,7 @@ ORDER BY 1 | 3.2857142857142857 | 1.00000000000000000000 (6 rows) +SELECT coordinator_plan($Q$ EXPLAIN (COSTS FALSE) SELECT user_id, @@ -569,27 +570,16 @@ GROUP BY 1 ORDER BY 3 DESC, 2 DESC, 1 DESC; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------ +$Q$); + coordinator_plan +------------------------------------------------------------------------------------ Sort Sort Key: remote_scan.avg_1 DESC, remote_scan.avg DESC, remote_scan.user_id DESC -> HashAggregate Group Key: remote_scan.user_id - -> Custom Scan (Citus Real-Time) + -> Custom Scan (Citus Adaptive) Task Count: 4 - Tasks Shown: One of 4 - -> Task - Node: host=localhost port=57637 dbname=regression - -> WindowAgg - -> Sort - Sort Key: users_table.user_id, (min(users_table.user_id)), (avg(users_table.value_1)) - -> WindowAgg - -> Sort - Sort Key: users_table.user_id, (max(users_table.user_id)), (min(users_table.value_2)) - -> HashAggregate - Group Key: users_table.user_id - -> Seq Scan on users_table_1400256 users_table -(18 rows) +(6 rows) SELECT user_id, diff --git a/src/test/regress/sql/window_functions.sql b/src/test/regress/sql/window_functions.sql index 45e37372e..69f208d89 100644 --- a/src/test/regress/sql/window_functions.sql +++ b/src/test/regress/sql/window_functions.sql @@ -170,7 +170,7 @@ SELECT COUNT(*) OVER (PARTITION BY user_id, user_id + 1), rank() OVER (PARTITION BY user_id) as cnt1, COUNT(*) OVER (PARTITION BY user_id, abs(value_1 - value_2)) as cnt2, - date_trunc('min', lag(time) OVER (PARTITION BY user_id)) as datee, + date_trunc('min', lag(time) OVER (PARTITION BY user_id ORDER BY time)) as datee, rank() OVER my_win as rnnk, avg(CASE WHEN user_id > 4 @@ -336,6 +336,7 @@ GROUP BY ORDER BY 3 DESC, 2 DESC, 1 DESC; +SELECT coordinator_plan($Q$ EXPLAIN (COSTS FALSE) SELECT user_id, @@ -347,6 +348,7 @@ GROUP BY 1 ORDER BY 3 DESC, 2 DESC, 1 DESC; +$Q$); SELECT user_id, From 9065ef429cf223356517b64cde945c2696310228 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philip=20Dub=C3=A9?= Date: Fri, 26 Jul 2019 03:26:42 +0000 Subject: [PATCH 09/13] foreign_key_to_reference_table: terse to avoid differing order of drop cascade details --- src/test/regress/bin/diff | 4 +- .../foreign_key_to_reference_table.out | 45 +++++++++---------- .../foreign_key_to_reference_table_0.out | 21 +++++---- .../sql/foreign_key_to_reference_table.sql | 15 ++++--- 4 files changed, 44 insertions(+), 41 deletions(-) diff --git a/src/test/regress/bin/diff b/src/test/regress/bin/diff index 50c819450..67fab9e10 100755 --- a/src/test/regress/bin/diff +++ b/src/test/regress/bin/diff @@ -37,13 +37,13 @@ then sed -Ef $BASEDIR/normalize.sed < $file1 > $file1.modified sed -Ef $BASEDIR/normalize.sed < $file2 > $file2.modified - $DIFF $args $file1.modified $file2.modified + $DIFF -w $args $file1.modified $file2.modified exitcode=$? rm -f $file1.modified $file2.modified else # if test is not in normalized_tests.lst, just diff without normalizing. - $DIFF $args $file1 $file2 + $DIFF -w $args $file1 $file2 exitcode=$? fi diff --git a/src/test/regress/expected/foreign_key_to_reference_table.out b/src/test/regress/expected/foreign_key_to_reference_table.out index 8b73fa63d..a98ff1120 100644 --- a/src/test/regress/expected/foreign_key_to_reference_table.out +++ b/src/test/regress/expected/foreign_key_to_reference_table.out @@ -702,8 +702,8 @@ INSERT INTO referenced_table(test_column2) SELECT x FROM generate_series(1,1000) INSERT INTO referencing_table(id) SELECT x FROM generate_series(1,1000) AS f(x); -- Fails for non existing value inserts (serial is already incremented) INSERT INTO referencing_table(id) SELECT x FROM generate_series(1,10) AS f(x); -ERROR: insert or update on table "referencing_table_7000197" violates foreign key constraint "fkey_ref_7000197" -DETAIL: Key (ref_id)=(1001) is not present in table "referenced_table_7000196". +ERROR: insert or update on table "referencing_table_7000200" violates foreign key constraint "fkey_ref_7000200" +DETAIL: Key (ref_id)=(1003) is not present in table "referenced_table_7000196". DROP TABLE referenced_table CASCADE; NOTICE: drop cascades to constraint fkey_ref on table referencing_table DROP TABLE referencing_table CASCADE; @@ -844,17 +844,17 @@ INSERT INTO referenced_table SELECT x, x+1 FROM generate_series(0,1000) AS f(x); INSERT INTO referenced_table2 SELECT x, x+1 FROM generate_series(500,1500) AS f(x); -- should fail INSERT INTO referencing_table SELECT x, x+1 FROM generate_series(0,1500) AS f(x); -ERROR: insert or update on table "referencing_table_7000227" violates foreign key constraint "foreign_key_2_7000227" -DETAIL: Key (id)=(5) is not present in table "referenced_table2_7000225". +ERROR: insert or update on table "referencing_table_7000229" violates foreign key constraint "foreign_key_2_7000229" +DETAIL: Key (id)=(0) is not present in table "referenced_table2_7000225". -- should fail INSERT INTO referencing_table SELECT x, x+1 FROM generate_series(0,400) AS f(x); -ERROR: insert or update on table "referencing_table_7000227" violates foreign key constraint "foreign_key_2_7000227" -DETAIL: Key (id)=(5) is not present in table "referenced_table2_7000225". +ERROR: insert or update on table "referencing_table_7000229" violates foreign key constraint "foreign_key_2_7000229" +DETAIL: Key (id)=(0) is not present in table "referenced_table2_7000225". -- should fail INSERT INTO referencing_table SELECT x, x+1 FROM generate_series(1000,1400) AS f(x); -ERROR: insert or update on table "referencing_table_7000227" violates foreign key constraint "fkey_ref_7000227" -DETAIL: Key (id)=(1005) is not present in table "referenced_table_7000224". --- should success +ERROR: insert or update on table "referencing_table_7000231" violates foreign key constraint "fkey_ref_7000231" +DETAIL: Key (id)=(1001) is not present in table "referenced_table_7000224". +-- should succeed INSERT INTO referencing_table SELECT x, x+1 FROM generate_series(600,900) AS f(x); SELECT count(*) FROM referencing_table; count @@ -909,6 +909,7 @@ SELECT count(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_tab 16 (1 row) +\set VERBOSITY terse DROP TABLE referenced_table CASCADE; NOTICE: drop cascades to constraint referencing_table_id_fkey on table referencing_table DROP TABLE referenced_table2 CASCADE; @@ -971,17 +972,14 @@ INSERT INTO referenced_table SELECT x, x+1 FROM generate_series(0,1000) AS f(x); INSERT INTO referenced_table2 SELECT x, x+1 FROM generate_series(500,1500) AS f(x); -- should fail INSERT INTO referencing_table SELECT x, x+1 FROM generate_series(0,1500) AS f(x); -ERROR: insert or update on table "referencing_table_7000251" violates foreign key constraint "foreign_key_2_7000251" -DETAIL: Key (ref_id)=(7) is not present in table "referenced_table2_7000245". +ERROR: insert or update on table "referencing_table_7000249" violates foreign key constraint "foreign_key_2_7000249" -- should fail INSERT INTO referencing_table SELECT x, x+1 FROM generate_series(0,400) AS f(x); -ERROR: insert or update on table "referencing_table_7000251" violates foreign key constraint "foreign_key_2_7000251" -DETAIL: Key (ref_id)=(7) is not present in table "referenced_table2_7000245". +ERROR: insert or update on table "referencing_table_7000249" violates foreign key constraint "foreign_key_2_7000249" -- should fail INSERT INTO referencing_table SELECT x, x+1 FROM generate_series(1000,1400) AS f(x); ERROR: insert or update on table "referencing_table_7000251" violates foreign key constraint "fkey_ref_7000251" -DETAIL: Key (id)=(1001) is not present in table "referenced_table_7000244". --- should success +-- should succeed INSERT INTO referencing_table SELECT x, x+501 FROM generate_series(0,1000) AS f(x); SELECT count(*) FROM referencing_table; count @@ -1044,6 +1042,7 @@ NOTICE: drop cascades to constraint referencing_table_id_fkey on table referenc DROP TABLE referenced_table2 CASCADE; NOTICE: drop cascades to constraint foreign_key_2 on table referencing_table DROP TABLE referencing_table CASCADE; +\set VERBOSITY default -- two distributed tables are referencing to one reference table and -- in the same time the distributed table 2 is referencing to -- distributed table 1. Thus, we have a triangular @@ -1111,15 +1110,15 @@ SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' A INSERT INTO referenced_table SELECT x, x+1 FROM generate_series(0,1000) AS f(x); -- should fail INSERT INTO referencing_table2 SELECT x, x+1 FROM generate_series(0,100) AS f(x); -ERROR: insert or update on table "referencing_table2_7000274" violates foreign key constraint "fkey_ref_to_dist_7000274" -DETAIL: Key (id)=(5) is not present in table "referencing_table_7000266". --- should success +ERROR: insert or update on table "referencing_table2_7000276" violates foreign key constraint "fkey_ref_to_dist_7000276" +DETAIL: Key (id)=(0) is not present in table "referencing_table_7000268". +-- should succeed INSERT INTO referencing_table SELECT x, x+1 FROM generate_series(0,400) AS f(x); -- should fail INSERT INTO referencing_table2 SELECT x, x+1 FROM generate_series(200,500) AS f(x); ERROR: insert or update on table "referencing_table2_7000274" violates foreign key constraint "fkey_ref_to_dist_7000274" DETAIL: Key (id)=(403) is not present in table "referencing_table_7000266". --- should success +-- should succeed INSERT INTO referencing_table2 SELECT x, x+1 FROM generate_series(0,300) AS f(x); DELETE FROM referenced_table WHERE test_column < 200; SELECT count(*) FROM referencing_table; @@ -1141,13 +1140,13 @@ SELECT count(*) FROM referencing_table2; 1 (1 row) +\set VERBOSITY terse DROP TABLE referenced_table CASCADE; NOTICE: drop cascades to 2 other objects -DETAIL: drop cascades to constraint fkey_ref on table referencing_table2 -drop cascades to constraint fkey_ref on table referencing_table DROP TABLE referencing_table CASCADE; NOTICE: drop cascades to constraint fkey_ref_to_dist on table referencing_table2 DROP TABLE referencing_table2 CASCADE; +\set VERBOSITY default -- Check if the above fkeys are created with create_distributed_table CREATE TABLE referenced_table(test_column int, test_column2 int UNIQUE, PRIMARY KEY(test_column)); CREATE TABLE referencing_table(id int PRIMARY KEY, ref_id int, FOREIGN KEY (id) REFERENCES referenced_table(test_column) ON DELETE CASCADE); @@ -1179,13 +1178,13 @@ SELECT count(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_tab 24 (1 row) +\set VERBOSITY terse DROP TABLE referenced_table CASCADE; NOTICE: drop cascades to 2 other objects -DETAIL: drop cascades to constraint referencing_table2_ref_id_fkey on table referencing_table2 -drop cascades to constraint referencing_table_id_fkey on table referencing_table DROP TABLE referencing_table CASCADE; NOTICE: drop cascades to constraint referencing_table2_id_fkey on table referencing_table2 DROP TABLE referencing_table2 CASCADE; +\set VERBOSITY default -- In this test we have a chained relationship in form of -- distributed table (referencing_referencing_table) has a foreign key with two columns -- to another distributed table (referencing_table) diff --git a/src/test/regress/expected/foreign_key_to_reference_table_0.out b/src/test/regress/expected/foreign_key_to_reference_table_0.out index a5e6e4e66..19a2a1092 100644 --- a/src/test/regress/expected/foreign_key_to_reference_table_0.out +++ b/src/test/regress/expected/foreign_key_to_reference_table_0.out @@ -854,7 +854,7 @@ DETAIL: Key (id)=(5) is not present in table "referenced_table2_7000225". INSERT INTO referencing_table SELECT x, x+1 FROM generate_series(1000,1400) AS f(x); ERROR: insert or update on table "referencing_table_7000227" violates foreign key constraint "fkey_ref_7000227" DETAIL: Key (id)=(1005) is not present in table "referenced_table_7000224". --- should success +-- should succeed INSERT INTO referencing_table SELECT x, x+1 FROM generate_series(600,900) AS f(x); SELECT count(*) FROM referencing_table; count @@ -909,6 +909,7 @@ SELECT count(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_tab 16 (1 row) +\set VERBOSITY terse DROP TABLE referenced_table CASCADE; NOTICE: drop cascades to constraint referencing_table_id_fkey on table referencing_table DROP TABLE referenced_table2 CASCADE; @@ -972,16 +973,13 @@ INSERT INTO referenced_table2 SELECT x, x+1 FROM generate_series(500,1500) AS f( -- should fail INSERT INTO referencing_table SELECT x, x+1 FROM generate_series(0,1500) AS f(x); ERROR: insert or update on table "referencing_table_7000251" violates foreign key constraint "foreign_key_2_7000251" -DETAIL: Key (ref_id)=(7) is not present in table "referenced_table2_7000245". -- should fail INSERT INTO referencing_table SELECT x, x+1 FROM generate_series(0,400) AS f(x); ERROR: insert or update on table "referencing_table_7000251" violates foreign key constraint "foreign_key_2_7000251" -DETAIL: Key (ref_id)=(7) is not present in table "referenced_table2_7000245". -- should fail INSERT INTO referencing_table SELECT x, x+1 FROM generate_series(1000,1400) AS f(x); ERROR: insert or update on table "referencing_table_7000251" violates foreign key constraint "fkey_ref_7000251" -DETAIL: Key (id)=(1001) is not present in table "referenced_table_7000244". --- should success +-- should succeed INSERT INTO referencing_table SELECT x, x+501 FROM generate_series(0,1000) AS f(x); SELECT count(*) FROM referencing_table; count @@ -1044,6 +1042,7 @@ NOTICE: drop cascades to constraint referencing_table_id_fkey on table referenc DROP TABLE referenced_table2 CASCADE; NOTICE: drop cascades to constraint foreign_key_2 on table referencing_table DROP TABLE referencing_table CASCADE; +\set VERBOSITY default -- two distributed tables are referencing to one reference table and -- in the same time the distributed table 2 is referencing to -- distributed table 1. Thus, we have a triangular @@ -1113,13 +1112,13 @@ INSERT INTO referenced_table SELECT x, x+1 FROM generate_series(0,1000) AS f(x); INSERT INTO referencing_table2 SELECT x, x+1 FROM generate_series(0,100) AS f(x); ERROR: insert or update on table "referencing_table2_7000274" violates foreign key constraint "fkey_ref_to_dist_7000274" DETAIL: Key (id)=(5) is not present in table "referencing_table_7000266". --- should success +-- should succeed INSERT INTO referencing_table SELECT x, x+1 FROM generate_series(0,400) AS f(x); -- should fail INSERT INTO referencing_table2 SELECT x, x+1 FROM generate_series(200,500) AS f(x); ERROR: insert or update on table "referencing_table2_7000274" violates foreign key constraint "fkey_ref_to_dist_7000274" DETAIL: Key (id)=(403) is not present in table "referencing_table_7000266". --- should success +-- should succeed INSERT INTO referencing_table2 SELECT x, x+1 FROM generate_series(0,300) AS f(x); DELETE FROM referenced_table WHERE test_column < 200; SELECT count(*) FROM referencing_table; @@ -1141,13 +1140,13 @@ SELECT count(*) FROM referencing_table2; 1 (1 row) +\set VERBOSITY terse DROP TABLE referenced_table CASCADE; NOTICE: drop cascades to 2 other objects -DETAIL: drop cascades to constraint fkey_ref on table referencing_table2 -drop cascades to constraint fkey_ref on table referencing_table DROP TABLE referencing_table CASCADE; NOTICE: drop cascades to constraint fkey_ref_to_dist on table referencing_table2 DROP TABLE referencing_table2 CASCADE; +\set VERBOSITY default -- Check if the above fkeys are created with create_distributed_table CREATE TABLE referenced_table(test_column int, test_column2 int UNIQUE, PRIMARY KEY(test_column)); CREATE TABLE referencing_table(id int PRIMARY KEY, ref_id int, FOREIGN KEY (id) REFERENCES referenced_table(test_column) ON DELETE CASCADE); @@ -1179,13 +1178,13 @@ SELECT count(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_tab 24 (1 row) +\set VERBOSITY terse DROP TABLE referenced_table CASCADE; NOTICE: drop cascades to 2 other objects -DETAIL: drop cascades to constraint referencing_table2_ref_id_fkey on table referencing_table2 -drop cascades to constraint referencing_table_id_fkey on table referencing_table DROP TABLE referencing_table CASCADE; NOTICE: drop cascades to constraint referencing_table2_id_fkey on table referencing_table2 DROP TABLE referencing_table2 CASCADE; +\set VERBOSITY default -- In this test we have a chained relationship in form of -- distributed table (referencing_referencing_table) has a foreign key with two columns -- to another distributed table (referencing_table) diff --git a/src/test/regress/sql/foreign_key_to_reference_table.sql b/src/test/regress/sql/foreign_key_to_reference_table.sql index 07924bdb5..a60842f5d 100644 --- a/src/test/regress/sql/foreign_key_to_reference_table.sql +++ b/src/test/regress/sql/foreign_key_to_reference_table.sql @@ -428,7 +428,7 @@ INSERT INTO referencing_table SELECT x, x+1 FROM generate_series(0,1500) AS f(x) INSERT INTO referencing_table SELECT x, x+1 FROM generate_series(0,400) AS f(x); -- should fail INSERT INTO referencing_table SELECT x, x+1 FROM generate_series(1000,1400) AS f(x); --- should success +-- should succeed INSERT INTO referencing_table SELECT x, x+1 FROM generate_series(600,900) AS f(x); SELECT count(*) FROM referencing_table; @@ -451,6 +451,7 @@ SELECT create_distributed_table('referencing_table', 'id'); SELECT count(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%'; +\set VERBOSITY terse DROP TABLE referenced_table CASCADE; DROP TABLE referenced_table2 CASCADE; DROP TABLE referencing_table CASCADE; @@ -484,7 +485,7 @@ INSERT INTO referencing_table SELECT x, x+1 FROM generate_series(0,1500) AS f(x) INSERT INTO referencing_table SELECT x, x+1 FROM generate_series(0,400) AS f(x); -- should fail INSERT INTO referencing_table SELECT x, x+1 FROM generate_series(1000,1400) AS f(x); --- should success +-- should succeed INSERT INTO referencing_table SELECT x, x+501 FROM generate_series(0,1000) AS f(x); SELECT count(*) FROM referencing_table; @@ -514,6 +515,7 @@ SELECT count(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_tab DROP TABLE referenced_table CASCADE; DROP TABLE referenced_table2 CASCADE; DROP TABLE referencing_table CASCADE; +\set VERBOSITY default -- two distributed tables are referencing to one reference table and @@ -542,11 +544,11 @@ SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' A INSERT INTO referenced_table SELECT x, x+1 FROM generate_series(0,1000) AS f(x); -- should fail INSERT INTO referencing_table2 SELECT x, x+1 FROM generate_series(0,100) AS f(x); --- should success +-- should succeed INSERT INTO referencing_table SELECT x, x+1 FROM generate_series(0,400) AS f(x); -- should fail INSERT INTO referencing_table2 SELECT x, x+1 FROM generate_series(200,500) AS f(x); --- should success +-- should succeed INSERT INTO referencing_table2 SELECT x, x+1 FROM generate_series(0,300) AS f(x); DELETE FROM referenced_table WHERE test_column < 200; @@ -555,9 +557,11 @@ SELECT count(*) FROM referencing_table2; DELETE FROM referencing_table WHERE id > 200; SELECT count(*) FROM referencing_table2; +\set VERBOSITY terse DROP TABLE referenced_table CASCADE; DROP TABLE referencing_table CASCADE; DROP TABLE referencing_table2 CASCADE; +\set VERBOSITY default -- Check if the above fkeys are created with create_distributed_table CREATE TABLE referenced_table(test_column int, test_column2 int UNIQUE, PRIMARY KEY(test_column)); @@ -572,10 +576,11 @@ COMMIT; SELECT count(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%'; +\set VERBOSITY terse DROP TABLE referenced_table CASCADE; DROP TABLE referencing_table CASCADE; DROP TABLE referencing_table2 CASCADE; - +\set VERBOSITY default -- In this test we have a chained relationship in form of -- distributed table (referencing_referencing_table) has a foreign key with two columns From 66ce2d2d2d587c3b5abd13bc8dc9c20fcca1d2c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philip=20Dub=C3=A9?= Date: Fri, 26 Jul 2019 19:07:06 +0000 Subject: [PATCH 10/13] Materialize c1 to keep subplan ids in sync --- src/test/regress/expected/multi_view.out | 4 ++-- src/test/regress/expected/multi_view_0.out | 4 ++-- src/test/regress/sql/multi_view.sql | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/test/regress/expected/multi_view.out b/src/test/regress/expected/multi_view.out index fc4970032..7a3f93002 100644 --- a/src/test/regress/expected/multi_view.out +++ b/src/test/regress/expected/multi_view.out @@ -698,7 +698,7 @@ SELECT * FROM distinct_value_1 ORDER BY 1 DESC LIMIT 5; -- CTEs are supported even if they are on views CREATE VIEW cte_view_1 AS -WITH c1 AS (SELECT * FROM users_table WHERE value_1 = 3) SELECT * FROM c1 WHERE value_2 < 4; +WITH c1 AS (SELECT * FROM users_table WHERE value_1 = 3) SELECT * FROM c1 WHERE value_2 < 4 AND EXISTS (SELECT * FROM c1); SELECT * FROM cte_view_1 ORDER BY 1,2,3,4,5 LIMIT 5; user_id | time | value_1 | value_2 | value_3 | value_4 ---------+---------------------------------+---------+---------+---------+--------- @@ -875,7 +875,7 @@ EXPLAIN (COSTS FALSE) SELECT et.* FROM recent_10_users JOIN events_table et USIN -> Sort Sort Key: remote_scan."time" DESC -> Custom Scan (Citus Real-Time) - -> Distributed Subplan 96_1 + -> Distributed Subplan 98_1 -> Limit -> Sort Sort Key: max((max(remote_scan.lastseen))) DESC diff --git a/src/test/regress/expected/multi_view_0.out b/src/test/regress/expected/multi_view_0.out index b2db7666b..28e38c7fc 100644 --- a/src/test/regress/expected/multi_view_0.out +++ b/src/test/regress/expected/multi_view_0.out @@ -698,7 +698,7 @@ SELECT * FROM distinct_value_1 ORDER BY 1 DESC LIMIT 5; -- CTEs are supported even if they are on views CREATE VIEW cte_view_1 AS -WITH c1 AS (SELECT * FROM users_table WHERE value_1 = 3) SELECT * FROM c1 WHERE value_2 < 4; +WITH c1 AS (SELECT * FROM users_table WHERE value_1 = 3) SELECT * FROM c1 WHERE value_2 < 4 AND EXISTS (SELECT * FROM c1); SELECT * FROM cte_view_1 ORDER BY 1,2,3,4,5 LIMIT 5; user_id | time | value_1 | value_2 | value_3 | value_4 ---------+---------------------------------+---------+---------+---------+--------- @@ -877,7 +877,7 @@ EXPLAIN (COSTS FALSE) SELECT et.* FROM recent_10_users JOIN events_table et USIN -> Sort Sort Key: remote_scan."time" DESC -> Custom Scan (Citus Real-Time) - -> Distributed Subplan 96_1 + -> Distributed Subplan 98_1 -> Limit -> Sort Sort Key: max((max(remote_scan.lastseen))) DESC diff --git a/src/test/regress/sql/multi_view.sql b/src/test/regress/sql/multi_view.sql index 7c3870c7d..c324644a7 100644 --- a/src/test/regress/sql/multi_view.sql +++ b/src/test/regress/sql/multi_view.sql @@ -360,7 +360,7 @@ SELECT * FROM distinct_value_1 ORDER BY 1 DESC LIMIT 5; -- CTEs are supported even if they are on views CREATE VIEW cte_view_1 AS -WITH c1 AS (SELECT * FROM users_table WHERE value_1 = 3) SELECT * FROM c1 WHERE value_2 < 4; +WITH c1 AS (SELECT * FROM users_table WHERE value_1 = 3) SELECT * FROM c1 WHERE value_2 < 4 AND EXISTS (SELECT * FROM c1); SELECT * FROM cte_view_1 ORDER BY 1,2,3,4,5 LIMIT 5; From 5e835e7565d824fd4110e230b92be00433b42653 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philip=20Dub=C3=A9?= Date: Tue, 30 Jul 2019 00:42:01 +0000 Subject: [PATCH 11/13] Fix multi_repair_shards. There's already a group/shardid entry, pg11 gives us back the inserted one, pg12 gives us the preexisting one --- src/test/regress/expected/multi_repair_shards.out | 8 +++----- src/test/regress/sql/multi_repair_shards.sql | 9 +++------ 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/src/test/regress/expected/multi_repair_shards.out b/src/test/regress/expected/multi_repair_shards.out index 89617f1e3..17a9c45eb 100644 --- a/src/test/regress/expected/multi_repair_shards.out +++ b/src/test/regress/expected/multi_repair_shards.out @@ -66,13 +66,11 @@ SELECT master_copy_shard_placement(:newshardid, 'localhost', :worker_1_port, 'lo INSERT INTO customer_engagements VALUES (4, '04-01-2015', 'fourth event'); ROLLBACK; --- add a fake healthy placement for the tests -INSERT INTO pg_dist_placement (groupid, shardid, shardstate, shardlength) - VALUES (:worker_2_group, :newshardid, 1, 0); +-- deactivate placement +UPDATE pg_dist_placement SET shardstate = 1 WHERE groupid = :worker_2_group and shardid = :newshardid; SELECT master_copy_shard_placement(:newshardid, 'localhost', :worker_1_port, 'localhost', :worker_2_port); ERROR: target placement must be in inactive state -DELETE FROM pg_dist_placement - WHERE groupid = :worker_2_group AND shardid = :newshardid AND shardstate = 1; +UPDATE pg_dist_placement SET shardstate = 3 WHERE groupid = :worker_2_group and shardid = :newshardid; -- also try to copy from an inactive placement SELECT master_copy_shard_placement(:newshardid, 'localhost', :worker_2_port, 'localhost', :worker_1_port); ERROR: source placement must be in finalized state diff --git a/src/test/regress/sql/multi_repair_shards.sql b/src/test/regress/sql/multi_repair_shards.sql index 2d087d78f..c33202962 100644 --- a/src/test/regress/sql/multi_repair_shards.sql +++ b/src/test/regress/sql/multi_repair_shards.sql @@ -62,15 +62,12 @@ SELECT master_copy_shard_placement(:newshardid, 'localhost', :worker_1_port, 'lo INSERT INTO customer_engagements VALUES (4, '04-01-2015', 'fourth event'); ROLLBACK; --- add a fake healthy placement for the tests - -INSERT INTO pg_dist_placement (groupid, shardid, shardstate, shardlength) - VALUES (:worker_2_group, :newshardid, 1, 0); +-- deactivate placement +UPDATE pg_dist_placement SET shardstate = 1 WHERE groupid = :worker_2_group and shardid = :newshardid; SELECT master_copy_shard_placement(:newshardid, 'localhost', :worker_1_port, 'localhost', :worker_2_port); -DELETE FROM pg_dist_placement - WHERE groupid = :worker_2_group AND shardid = :newshardid AND shardstate = 1; +UPDATE pg_dist_placement SET shardstate = 3 WHERE groupid = :worker_2_group and shardid = :newshardid; -- also try to copy from an inactive placement SELECT master_copy_shard_placement(:newshardid, 'localhost', :worker_2_port, 'localhost', :worker_1_port); From e0f19fb58cc82d2dfe850c65f1e9404ba9472ead Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philip=20Dub=C3=A9?= Date: Tue, 30 Jul 2019 21:53:01 +0000 Subject: [PATCH 12/13] multi_partitioning_1.out --- .../{multi_partitioning_9.out => multi_partitioning_1.out} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/test/regress/expected/{multi_partitioning_9.out => multi_partitioning_1.out} (100%) diff --git a/src/test/regress/expected/multi_partitioning_9.out b/src/test/regress/expected/multi_partitioning_1.out similarity index 100% rename from src/test/regress/expected/multi_partitioning_9.out rename to src/test/regress/expected/multi_partitioning_1.out From 5459c01956aa2fe4dd17a590166c9acf9cff79e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philip=20Dub=C3=A9?= Date: Tue, 30 Jul 2019 22:34:19 +0000 Subject: [PATCH 13/13] multi_partitioning_utils: version_above_ten --- src/test/regress/expected/multi_partitioning_utils.out | 8 ++++---- src/test/regress/expected/multi_partitioning_utils_0.out | 8 ++++---- src/test/regress/sql/multi_partitioning_utils.sql | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/test/regress/expected/multi_partitioning_utils.out b/src/test/regress/expected/multi_partitioning_utils.out index 7e010f6d8..7f7901ce5 100644 --- a/src/test/regress/expected/multi_partitioning_utils.out +++ b/src/test/regress/expected/multi_partitioning_utils.out @@ -1,9 +1,9 @@ -- This test has different output per major version SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int as server_major_version; - server_major_version ----------------------- - 11 +SELECT substring(:'server_version', '\d+')::int > 10 AS version_above_ten; + version_above_ten +------------------- + t (1 row) -- =================================================================== diff --git a/src/test/regress/expected/multi_partitioning_utils_0.out b/src/test/regress/expected/multi_partitioning_utils_0.out index 6dc89e12d..81e36696b 100644 --- a/src/test/regress/expected/multi_partitioning_utils_0.out +++ b/src/test/regress/expected/multi_partitioning_utils_0.out @@ -1,9 +1,9 @@ -- This test has different output per major version SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int as server_major_version; - server_major_version ----------------------- - 10 +SELECT substring(:'server_version', '\d+')::int > 10 AS version_above_ten; + version_above_ten +------------------- + f (1 row) -- =================================================================== diff --git a/src/test/regress/sql/multi_partitioning_utils.sql b/src/test/regress/sql/multi_partitioning_utils.sql index df19ff569..6eea8acbc 100644 --- a/src/test/regress/sql/multi_partitioning_utils.sql +++ b/src/test/regress/sql/multi_partitioning_utils.sql @@ -1,6 +1,6 @@ -- This test has different output per major version SHOW server_version \gset -SELECT substring(:'server_version', '\d+')::int as server_major_version; +SELECT substring(:'server_version', '\d+')::int > 10 AS version_above_ten; -- =================================================================== -- create test functions