mirror of https://github.com/citusdata/citus.git
209 lines
8.5 KiB
Plaintext
209 lines
8.5 KiB
Plaintext
--
|
|
-- MULTI_LARGE_TABLE_TASK_ASSIGNMENT
|
|
--
|
|
-- Tests which cover task assignment for MapMerge jobs for single range repartition
|
|
-- and dual hash repartition joins. The tests also cover task assignment propagation
|
|
-- from a sql task to its depended tasks. Note that we set the executor type to task
|
|
-- tracker executor here, as we cannot run repartition jobs with real time executor.
|
|
BEGIN;
|
|
SET client_min_messages TO DEBUG3;
|
|
DEBUG: CommitTransactionCommand
|
|
SET citus.large_table_shard_count TO 2;
|
|
DEBUG: StartTransactionCommand
|
|
DEBUG: ProcessUtility
|
|
DEBUG: CommitTransactionCommand
|
|
SET citus.task_executor_type TO 'task-tracker';
|
|
DEBUG: StartTransactionCommand
|
|
DEBUG: ProcessUtility
|
|
DEBUG: CommitTransactionCommand
|
|
-- Single range repartition join to test anchor-shard based task assignment and
|
|
-- assignment propagation to merge and data-fetch tasks.
|
|
SELECT
|
|
count(*)
|
|
FROM
|
|
orders, customer
|
|
WHERE
|
|
o_custkey = c_custkey;
|
|
DEBUG: StartTransactionCommand
|
|
DEBUG: assigned task 4 to node localhost:57637
|
|
DEBUG: assigned task 2 to node localhost:57638
|
|
DEBUG: join prunable for intervals [1,1000] and [1001,2000]
|
|
DEBUG: join prunable for intervals [1,1000] and [6001,7000]
|
|
DEBUG: join prunable for intervals [1001,2000] and [6001,7000]
|
|
DEBUG: join prunable for intervals [1001,2000] and [1,1000]
|
|
DEBUG: join prunable for intervals [6001,7000] and [1001,2000]
|
|
DEBUG: join prunable for intervals [6001,7000] and [1,1000]
|
|
DEBUG: pruning merge fetch taskId 1
|
|
DETAIL: Creating dependency on merge taskId 5
|
|
DEBUG: pruning merge fetch taskId 4
|
|
DETAIL: Creating dependency on merge taskId 8
|
|
DEBUG: pruning merge fetch taskId 7
|
|
DETAIL: Creating dependency on merge taskId 11
|
|
DEBUG: assigned task 3 to node localhost:57637
|
|
DEBUG: assigned task 9 to node localhost:57638
|
|
DEBUG: assigned task 6 to node localhost:57637
|
|
DEBUG: CommitTransactionCommand
|
|
count
|
|
-------
|
|
2984
|
|
(1 row)
|
|
|
|
-- Single range repartition join, along with a join with a small table containing
|
|
-- more than one shard. This situation results in multiple sql tasks depending on
|
|
-- the same merge task, and tests our constraint group creation and assignment
|
|
-- propagation. Here 'orders' is considered the small table.
|
|
SET citus.large_table_shard_count TO 3;
|
|
DEBUG: StartTransactionCommand
|
|
DEBUG: ProcessUtility
|
|
DEBUG: CommitTransactionCommand
|
|
SELECT
|
|
count(*)
|
|
FROM
|
|
orders, customer, lineitem
|
|
WHERE
|
|
o_custkey = c_custkey AND
|
|
o_orderkey = l_orderkey;
|
|
DEBUG: StartTransactionCommand
|
|
DEBUG: assigned task 15 to node localhost:57637
|
|
DEBUG: assigned task 18 to node localhost:57638
|
|
DEBUG: assigned task 9 to node localhost:57637
|
|
DEBUG: assigned task 12 to node localhost:57638
|
|
DEBUG: assigned task 3 to node localhost:57637
|
|
DEBUG: assigned task 6 to node localhost:57638
|
|
DEBUG: join prunable for intervals [1,2496] and [13921,14947]
|
|
DEBUG: join prunable for intervals [1,2496] and [11554,13920]
|
|
DEBUG: join prunable for intervals [1,2496] and [8997,11554]
|
|
DEBUG: join prunable for intervals [1,2496] and [4965,5986]
|
|
DEBUG: join prunable for intervals [1,2496] and [2497,4964]
|
|
DEBUG: join prunable for intervals [2497,4964] and [13921,14947]
|
|
DEBUG: join prunable for intervals [2497,4964] and [11554,13920]
|
|
DEBUG: join prunable for intervals [2497,4964] and [8997,11554]
|
|
DEBUG: join prunable for intervals [2497,4964] and [4965,5986]
|
|
DEBUG: join prunable for intervals [2497,4964] and [1,2496]
|
|
DEBUG: join prunable for intervals [4965,5986] and [13921,14947]
|
|
DEBUG: join prunable for intervals [4965,5986] and [11554,13920]
|
|
DEBUG: join prunable for intervals [4965,5986] and [8997,11554]
|
|
DEBUG: join prunable for intervals [4965,5986] and [2497,4964]
|
|
DEBUG: join prunable for intervals [4965,5986] and [1,2496]
|
|
DEBUG: join prunable for intervals [8997,11554] and [13921,14947]
|
|
DEBUG: join prunable for intervals [8997,11554] and [4965,5986]
|
|
DEBUG: join prunable for intervals [8997,11554] and [2497,4964]
|
|
DEBUG: join prunable for intervals [8997,11554] and [1,2496]
|
|
DEBUG: join prunable for intervals [11554,13920] and [13921,14947]
|
|
DEBUG: join prunable for intervals [11554,13920] and [4965,5986]
|
|
DEBUG: join prunable for intervals [11554,13920] and [2497,4964]
|
|
DEBUG: join prunable for intervals [11554,13920] and [1,2496]
|
|
DEBUG: join prunable for intervals [13921,14947] and [11554,13920]
|
|
DEBUG: join prunable for intervals [13921,14947] and [8997,11554]
|
|
DEBUG: join prunable for intervals [13921,14947] and [4965,5986]
|
|
DEBUG: join prunable for intervals [13921,14947] and [2497,4964]
|
|
DEBUG: join prunable for intervals [13921,14947] and [1,2496]
|
|
DEBUG: pruning merge fetch taskId 1
|
|
DETAIL: Creating dependency on merge taskId 19
|
|
DEBUG: pruning merge fetch taskId 4
|
|
DETAIL: Creating dependency on merge taskId 26
|
|
DEBUG: pruning merge fetch taskId 7
|
|
DETAIL: Creating dependency on merge taskId 33
|
|
DEBUG: pruning merge fetch taskId 10
|
|
DETAIL: Creating dependency on merge taskId 40
|
|
DEBUG: pruning merge fetch taskId 13
|
|
DETAIL: Creating dependency on merge taskId 40
|
|
DEBUG: pruning merge fetch taskId 16
|
|
DETAIL: Creating dependency on merge taskId 47
|
|
DEBUG: pruning merge fetch taskId 19
|
|
DETAIL: Creating dependency on merge taskId 47
|
|
DEBUG: pruning merge fetch taskId 22
|
|
DETAIL: Creating dependency on merge taskId 54
|
|
DEBUG: assigned task 3 to node localhost:57637
|
|
DEBUG: assigned task 6 to node localhost:57638
|
|
DEBUG: assigned task 9 to node localhost:57637
|
|
DEBUG: assigned task 12 to node localhost:57638
|
|
DEBUG: assigned task 18 to node localhost:57637
|
|
DEBUG: assigned task 24 to node localhost:57638
|
|
DEBUG: propagating assignment from merge task 40 to constrained sql task 15
|
|
DEBUG: propagating assignment from merge task 47 to constrained sql task 21
|
|
DEBUG: CommitTransactionCommand
|
|
count
|
|
-------
|
|
11998
|
|
(1 row)
|
|
|
|
SET citus.large_table_shard_count TO 2;
|
|
DEBUG: StartTransactionCommand
|
|
DEBUG: ProcessUtility
|
|
DEBUG: CommitTransactionCommand
|
|
-- The next test, dual hash repartition join, uses the current jobId to assign
|
|
-- tasks in a round-robin fashion. We therefore need to ensure that jobIds start
|
|
-- with an odd number here to get consistent test output.
|
|
SELECT case when (currval('pg_dist_jobid_seq') % 2) = 0
|
|
then nextval('pg_dist_jobid_seq') % 2
|
|
else 1 end;
|
|
DEBUG: StartTransactionCommand
|
|
DEBUG: CommitTransactionCommand
|
|
case
|
|
------
|
|
1
|
|
(1 row)
|
|
|
|
-- Dual hash repartition join which tests the separate hash repartition join
|
|
-- task assignment algorithm.
|
|
SELECT
|
|
count(*)
|
|
FROM
|
|
lineitem, customer
|
|
WHERE
|
|
l_partkey = c_nationkey;
|
|
DEBUG: StartTransactionCommand
|
|
DEBUG: assigned task 12 to node localhost:57637
|
|
DEBUG: assigned task 10 to node localhost:57638
|
|
DEBUG: assigned task 8 to node localhost:57637
|
|
DEBUG: assigned task 6 to node localhost:57638
|
|
DEBUG: assigned task 4 to node localhost:57637
|
|
DEBUG: assigned task 2 to node localhost:57638
|
|
DEBUG: assigned task 6 to node localhost:57637
|
|
DEBUG: assigned task 4 to node localhost:57638
|
|
DEBUG: assigned task 2 to node localhost:57637
|
|
DEBUG: join prunable for task partitionId 0 and 1
|
|
DEBUG: join prunable for task partitionId 0 and 2
|
|
DEBUG: join prunable for task partitionId 0 and 3
|
|
DEBUG: join prunable for task partitionId 1 and 0
|
|
DEBUG: join prunable for task partitionId 1 and 2
|
|
DEBUG: join prunable for task partitionId 1 and 3
|
|
DEBUG: join prunable for task partitionId 2 and 0
|
|
DEBUG: join prunable for task partitionId 2 and 1
|
|
DEBUG: join prunable for task partitionId 2 and 3
|
|
DEBUG: join prunable for task partitionId 3 and 0
|
|
DEBUG: join prunable for task partitionId 3 and 1
|
|
DEBUG: join prunable for task partitionId 3 and 2
|
|
DEBUG: pruning merge fetch taskId 1
|
|
DETAIL: Creating dependency on merge taskId 13
|
|
DEBUG: pruning merge fetch taskId 2
|
|
DETAIL: Creating dependency on merge taskId 7
|
|
DEBUG: pruning merge fetch taskId 4
|
|
DETAIL: Creating dependency on merge taskId 20
|
|
DEBUG: pruning merge fetch taskId 5
|
|
DETAIL: Creating dependency on merge taskId 11
|
|
DEBUG: pruning merge fetch taskId 7
|
|
DETAIL: Creating dependency on merge taskId 27
|
|
DEBUG: pruning merge fetch taskId 8
|
|
DETAIL: Creating dependency on merge taskId 15
|
|
DEBUG: pruning merge fetch taskId 10
|
|
DETAIL: Creating dependency on merge taskId 34
|
|
DEBUG: pruning merge fetch taskId 11
|
|
DETAIL: Creating dependency on merge taskId 19
|
|
DEBUG: assigned task 3 to node localhost:57637
|
|
DEBUG: assigned task 6 to node localhost:57638
|
|
DEBUG: assigned task 9 to node localhost:57637
|
|
DEBUG: assigned task 12 to node localhost:57638
|
|
DEBUG: CommitTransactionCommand
|
|
count
|
|
-------
|
|
125
|
|
(1 row)
|
|
|
|
-- Reset client logging level to its previous value
|
|
SET client_min_messages TO NOTICE;
|
|
DEBUG: StartTransactionCommand
|
|
DEBUG: ProcessUtility
|
|
COMMIT;
|