From c603fd5d7a1403430366de9fabd4d21dcc5971d5 Mon Sep 17 00:00:00 2001 From: Sait Talha Nisanci Date: Tue, 19 May 2020 11:27:11 +0300 Subject: [PATCH] Update check-multi-mx tests for adaptive executor Basically repartition joins are enabled where necessary. For parallel tests max adaptive executor pool size is decresed to 2, otherwise we would get too many clients error. --- src/test/regress/expected/local_shard_execution.out | 13 ++++++++++--- src/test/regress/expected/multi_mx_explain.out | 10 +++++----- .../regress/expected/multi_mx_reference_table.out | 1 + .../expected/multi_mx_repartition_join_w1.out | 2 ++ .../expected/multi_mx_repartition_join_w2.out | 2 ++ .../expected/multi_mx_repartition_udt_prepare.out | 3 ++- .../expected/multi_mx_repartition_udt_w1.out | 2 ++ .../expected/multi_mx_repartition_udt_w2.out | 2 ++ .../regress/expected/multi_mx_schema_support.out | 9 +++++---- src/test/regress/sql/multi_mx_reference_table.sql | 1 + .../regress/sql/multi_mx_repartition_join_w1.sql | 2 ++ .../regress/sql/multi_mx_repartition_join_w2.sql | 2 ++ .../sql/multi_mx_repartition_udt_prepare.sql | 1 + .../regress/sql/multi_mx_repartition_udt_w1.sql | 2 ++ .../regress/sql/multi_mx_repartition_udt_w2.sql | 2 ++ src/test/regress/sql/multi_mx_schema_support.sql | 2 +- 16 files changed, 42 insertions(+), 14 deletions(-) diff --git a/src/test/regress/expected/local_shard_execution.out b/src/test/regress/expected/local_shard_execution.out index 000eefbf6..6a606202a 100644 --- a/src/test/regress/expected/local_shard_execution.out +++ b/src/test/regress/expected/local_shard_execution.out @@ -1373,9 +1373,13 @@ BEGIN; NOTICE: executing the command locally: DELETE FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE (key OPERATOR(pg_catalog.=) 500) SET LOCAL citus.task_executor_type = 'task-tracker'; SELECT count(*) FROM distributed_table; -ERROR: cannot execute command because a local execution has accessed a placement in the transaction -DETAIL: Some parallel commands cannot be executed if a previous command has already been executed locally -HINT: Try re-running the transaction with "SET LOCAL citus.enable_local_execution TO OFF;" +NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE true +NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE true + count +--------------------------------------------------------------------- + 100 +(1 row) + ROLLBACK; -- local execution should not be executed locally -- becase a task-tracker query has already been executed @@ -1383,6 +1387,8 @@ BEGIN; SET LOCAL citus.task_executor_type = 'task-tracker'; SET LOCAL client_min_messages TO INFO; SELECT count(*) FROM distributed_table; +NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470001 distributed_table WHERE true +NOTICE: executing the command locally: SELECT count(*) AS count FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE true count --------------------------------------------------------------------- 101 @@ -1390,6 +1396,7 @@ BEGIN; SET LOCAL client_min_messages TO LOG; DELETE FROM distributed_table WHERE key = 500; +NOTICE: executing the command locally: DELETE FROM local_shard_execution.distributed_table_1470003 distributed_table WHERE (key OPERATOR(pg_catalog.=) 500) ROLLBACK; -- probably not a realistic case since views are not very -- well supported with MX diff --git a/src/test/regress/expected/multi_mx_explain.out b/src/test/regress/expected/multi_mx_explain.out index fe2f78a29..2a368b78d 100644 --- a/src/test/regress/expected/multi_mx_explain.out +++ b/src/test/regress/expected/multi_mx_explain.out @@ -487,7 +487,7 @@ SET citus.explain_all_tasks TO off; EXPLAIN (COSTS FALSE) SELECT avg(l_linenumber) FROM lineitem_mx WHERE l_orderkey > 9030; Aggregate - -> Custom Scan (Citus Task-Tracker) + -> Custom Scan (Citus Adaptive) Task Count: 16 Tasks Shown: One of 16 -> Task @@ -503,7 +503,7 @@ EXPLAIN (COSTS FALSE) AND o_custkey = c_custkey AND l_suppkey = s_suppkey; Aggregate - -> Custom Scan (Citus Task-Tracker) + -> Custom Scan (Citus Adaptive) Task Count: 16 Tasks Shown: One of 16 -> Task @@ -539,7 +539,7 @@ EXPLAIN (COSTS FALSE, FORMAT JSON) { "Node Type": "Custom Scan", "Parent Relationship": "Outer", - "Custom Plan Provider": "Citus Task-Tracker", + "Custom Plan Provider": "Citus Adaptive", "Parallel Aware": false, "Distributed Query": { "Job": { @@ -675,7 +675,7 @@ EXPLAIN (COSTS FALSE, FORMAT XML) Custom Scan Outer - Citus Task-Tracker + Citus Adaptive false @@ -807,7 +807,7 @@ EXPLAIN (COSTS FALSE, FORMAT YAML) Plans: - Node Type: "Custom Scan" Parent Relationship: "Outer" - Custom Plan Provider: "Citus Task-Tracker" + Custom Plan Provider: "Citus Adaptive" Parallel Aware: false Distributed Query: Job: diff --git a/src/test/regress/expected/multi_mx_reference_table.out b/src/test/regress/expected/multi_mx_reference_table.out index e540182f3..223f2c65c 100644 --- a/src/test/regress/expected/multi_mx_reference_table.out +++ b/src/test/regress/expected/multi_mx_reference_table.out @@ -879,6 +879,7 @@ LOG: join order: [ "colocated_table_test" ][ reference join "reference_table_te (2 rows) SET citus.task_executor_type to "task-tracker"; +SET citus.enable_repartition_joins to ON; SELECT colocated_table_test.value_2 FROM diff --git a/src/test/regress/expected/multi_mx_repartition_join_w1.out b/src/test/regress/expected/multi_mx_repartition_join_w1.out index c3072c5f3..eb5fa9afb 100644 --- a/src/test/regress/expected/multi_mx_repartition_join_w1.out +++ b/src/test/regress/expected/multi_mx_repartition_join_w1.out @@ -4,6 +4,8 @@ -- tests use the same sequence ids but the queries should not fail. \c - - - :worker_1_port SET citus.task_executor_type TO "task-tracker"; +SET citus.max_adaptive_executor_pool_size TO 2; +SET citus.enable_repartition_joins to ON; CREATE TEMP TABLE t1 AS SELECT l1.l_comment diff --git a/src/test/regress/expected/multi_mx_repartition_join_w2.out b/src/test/regress/expected/multi_mx_repartition_join_w2.out index a643cd90d..fe3d46533 100644 --- a/src/test/regress/expected/multi_mx_repartition_join_w2.out +++ b/src/test/regress/expected/multi_mx_repartition_join_w2.out @@ -4,6 +4,8 @@ -- tests use the same sequence ids but the queries should not fail. \c - - - :worker_2_port SET citus.task_executor_type TO "task-tracker"; +SET citus.max_adaptive_executor_pool_size TO 2; +SET citus.enable_repartition_joins to ON; CREATE TEMP TABLE t1 AS SELECT l1.l_comment diff --git a/src/test/regress/expected/multi_mx_repartition_udt_prepare.out b/src/test/regress/expected/multi_mx_repartition_udt_prepare.out index 19776411c..9502e8fcc 100644 --- a/src/test/regress/expected/multi_mx_repartition_udt_prepare.out +++ b/src/test/regress/expected/multi_mx_repartition_udt_prepare.out @@ -153,6 +153,7 @@ INSERT INTO repartition_udt_other values (11, '(2,2)'::test_udt, 'foo'); INSERT INTO repartition_udt_other values (12, '(2,3)'::test_udt, 'foo'); SET client_min_messages = LOG; SET citus.task_executor_type = 'task-tracker'; +SET citus.enable_repartition_joins to ON; -- Query that should result in a repartition -- join on int column, and be empty. SELECT * FROM repartition_udt JOIN repartition_udt_other @@ -170,7 +171,7 @@ SELECT * FROM repartition_udt JOIN repartition_udt_other LOG: join order: [ "repartition_udt" ][ dual partition join "repartition_udt_other" ] QUERY PLAN --------------------------------------------------------------------- - Custom Scan (Citus Task-Tracker) + Custom Scan (Citus Adaptive) Task Count: 4 Tasks Shown: None, not supported for re-partition queries -> MapMergeJob diff --git a/src/test/regress/expected/multi_mx_repartition_udt_w1.out b/src/test/regress/expected/multi_mx_repartition_udt_w1.out index 2095866e8..8fd2ae4fb 100644 --- a/src/test/regress/expected/multi_mx_repartition_udt_w1.out +++ b/src/test/regress/expected/multi_mx_repartition_udt_w1.out @@ -5,6 +5,8 @@ SET client_min_messages = LOG; -- Query that should result in a repartition join on UDT column. SET citus.task_executor_type = 'task-tracker'; +SET citus.max_adaptive_executor_pool_size TO 2; +SET citus.enable_repartition_joins to ON; SET citus.log_multi_join_order = true; -- Query that should result in a repartition -- join on int column, and be empty diff --git a/src/test/regress/expected/multi_mx_repartition_udt_w2.out b/src/test/regress/expected/multi_mx_repartition_udt_w2.out index 547c62c5b..d48cb410a 100644 --- a/src/test/regress/expected/multi_mx_repartition_udt_w2.out +++ b/src/test/regress/expected/multi_mx_repartition_udt_w2.out @@ -5,6 +5,8 @@ SET client_min_messages = LOG; -- Query that should result in a repartition join on UDT column. SET citus.task_executor_type = 'task-tracker'; +SET citus.max_adaptive_executor_pool_size TO 2; +SET citus.enable_repartition_joins to ON; SET citus.log_multi_join_order = true; -- Query that should result in a repartition -- join on int column, and be empty. diff --git a/src/test/regress/expected/multi_mx_schema_support.out b/src/test/regress/expected/multi_mx_schema_support.out index f7669fb70..c9cb22697 100644 --- a/src/test/regress/expected/multi_mx_schema_support.out +++ b/src/test/regress/expected/multi_mx_schema_support.out @@ -160,7 +160,7 @@ SELECT * FROM nation_hash WHERE n_nationkey OPERATOR(===) 1; (1 row) SELECT * FROM citus_mx_test_schema.nation_hash_collation_search_path ORDER BY 1; - n_nationkey | n_name | n_regionkey | n_comment + n_nationkey | n_name | n_regionkey | n_comment --------------------------------------------------------------------- 0 | ALGERIA | 0 | haggle. carefully final deposits detect slyly agai 1 | ARGENTINA | 1 | al foxes promise slyly according to the regular accounts. bold requests alon @@ -171,7 +171,7 @@ SELECT * FROM citus_mx_test_schema.nation_hash_collation_search_path ORDER BY 1; (6 rows) SELECT n_comment FROM citus_mx_test_schema.nation_hash_collation_search_path ORDER BY n_comment COLLATE citus_mx_test_schema.english; - n_comment + n_comment --------------------------------------------------------------------- al foxes promise slyly according to the regular accounts. bold requests alon eas hang ironic, silent packages. slyly regular packages are furiously over the tithes. fluffily bold @@ -183,7 +183,7 @@ SELECT n_comment FROM citus_mx_test_schema.nation_hash_collation_search_path ORD SET search_path TO citus_mx_test_schema; SELECT * FROM nation_hash_collation_search_path ORDER BY 1 DESC; - n_nationkey | n_name | n_regionkey | n_comment + n_nationkey | n_name | n_regionkey | n_comment --------------------------------------------------------------------- 5 | ETHIOPIA | 0 | ven packages wake quickly. regu 4 | EGYPT | 4 | y above the carefully unusual theodolites. final dugouts are quickly across the furiously regular d @@ -194,7 +194,7 @@ SELECT * FROM nation_hash_collation_search_path ORDER BY 1 DESC; (6 rows) SELECT n_comment FROM nation_hash_collation_search_path ORDER BY n_comment COLLATE english; - n_comment + n_comment --------------------------------------------------------------------- al foxes promise slyly according to the regular accounts. bold requests alon eas hang ironic, silent packages. slyly regular packages are furiously over the tithes. fluffily bold @@ -218,6 +218,7 @@ SELECT * FROM nation_hash_composite_types WHERE test_col = '(a,a)'::new_composit 0 | ALGERIA | 0 | haggle. carefully final deposits detect slyly agai | (a,a) (1 row) +SET citus.enable_repartition_joins to ON; -- check when search_path is public, -- join of two tables which are in different schemas, -- join on partition column diff --git a/src/test/regress/sql/multi_mx_reference_table.sql b/src/test/regress/sql/multi_mx_reference_table.sql index 35b18ad39..658793a79 100644 --- a/src/test/regress/sql/multi_mx_reference_table.sql +++ b/src/test/regress/sql/multi_mx_reference_table.sql @@ -537,6 +537,7 @@ WHERE ORDER BY 1; SET citus.task_executor_type to "task-tracker"; +SET citus.enable_repartition_joins to ON; SELECT colocated_table_test.value_2 FROM diff --git a/src/test/regress/sql/multi_mx_repartition_join_w1.sql b/src/test/regress/sql/multi_mx_repartition_join_w1.sql index a9e0b6e10..c76de0516 100644 --- a/src/test/regress/sql/multi_mx_repartition_join_w1.sql +++ b/src/test/regress/sql/multi_mx_repartition_join_w1.sql @@ -5,6 +5,8 @@ \c - - - :worker_1_port SET citus.task_executor_type TO "task-tracker"; +SET citus.max_adaptive_executor_pool_size TO 2; +SET citus.enable_repartition_joins to ON; CREATE TEMP TABLE t1 AS SELECT l1.l_comment diff --git a/src/test/regress/sql/multi_mx_repartition_join_w2.sql b/src/test/regress/sql/multi_mx_repartition_join_w2.sql index d27fb57a2..538d8728f 100644 --- a/src/test/regress/sql/multi_mx_repartition_join_w2.sql +++ b/src/test/regress/sql/multi_mx_repartition_join_w2.sql @@ -5,6 +5,8 @@ \c - - - :worker_2_port SET citus.task_executor_type TO "task-tracker"; +SET citus.max_adaptive_executor_pool_size TO 2; +SET citus.enable_repartition_joins to ON; CREATE TEMP TABLE t1 AS SELECT l1.l_comment diff --git a/src/test/regress/sql/multi_mx_repartition_udt_prepare.sql b/src/test/regress/sql/multi_mx_repartition_udt_prepare.sql index 6bc5c16d6..bce22ac2c 100644 --- a/src/test/regress/sql/multi_mx_repartition_udt_prepare.sql +++ b/src/test/regress/sql/multi_mx_repartition_udt_prepare.sql @@ -183,6 +183,7 @@ INSERT INTO repartition_udt_other values (12, '(2,3)'::test_udt, 'foo'); SET client_min_messages = LOG; SET citus.task_executor_type = 'task-tracker'; +SET citus.enable_repartition_joins to ON; -- Query that should result in a repartition -- join on int column, and be empty. diff --git a/src/test/regress/sql/multi_mx_repartition_udt_w1.sql b/src/test/regress/sql/multi_mx_repartition_udt_w1.sql index 0710d98b4..ff4aba820 100644 --- a/src/test/regress/sql/multi_mx_repartition_udt_w1.sql +++ b/src/test/regress/sql/multi_mx_repartition_udt_w1.sql @@ -6,6 +6,8 @@ SET client_min_messages = LOG; -- Query that should result in a repartition join on UDT column. SET citus.task_executor_type = 'task-tracker'; +SET citus.max_adaptive_executor_pool_size TO 2; +SET citus.enable_repartition_joins to ON; SET citus.log_multi_join_order = true; -- Query that should result in a repartition diff --git a/src/test/regress/sql/multi_mx_repartition_udt_w2.sql b/src/test/regress/sql/multi_mx_repartition_udt_w2.sql index fd39c8b86..9c6fca314 100644 --- a/src/test/regress/sql/multi_mx_repartition_udt_w2.sql +++ b/src/test/regress/sql/multi_mx_repartition_udt_w2.sql @@ -6,6 +6,8 @@ SET client_min_messages = LOG; -- Query that should result in a repartition join on UDT column. SET citus.task_executor_type = 'task-tracker'; +SET citus.max_adaptive_executor_pool_size TO 2; +SET citus.enable_repartition_joins to ON; SET citus.log_multi_join_order = true; -- Query that should result in a repartition diff --git a/src/test/regress/sql/multi_mx_schema_support.sql b/src/test/regress/sql/multi_mx_schema_support.sql index 40f6c10e4..cb4b1912d 100644 --- a/src/test/regress/sql/multi_mx_schema_support.sql +++ b/src/test/regress/sql/multi_mx_schema_support.sql @@ -101,7 +101,7 @@ SELECT * FROM citus_mx_test_schema.nation_hash_composite_types WHERE test_col = SET search_path TO citus_mx_test_schema; SELECT * FROM nation_hash_composite_types WHERE test_col = '(a,a)'::new_composite_type ORDER BY 1::int DESC; - +SET citus.enable_repartition_joins to ON; -- check when search_path is public, -- join of two tables which are in different schemas, -- join on partition column