mirror of https://github.com/citusdata/citus.git
861 lines
32 KiB
Plaintext
861 lines
32 KiB
Plaintext
--
|
|
-- BACKGROUND_REBALANCE_PARALLEL
|
|
--
|
|
-- Test to check if the background tasks scheduled by the background rebalancer
|
|
-- have the correct dependencies
|
|
--
|
|
-- Test to verify that we do not allow parallel rebalancer moves involving a
|
|
-- particular node (either as source or target) more than
|
|
-- citus.max_background_task_executors_per_node, and that we can change the GUC on
|
|
-- the fly, and that will affect the ongoing balance as it should
|
|
--
|
|
-- Test to verify that there's a hard dependency when a specific node is first being
|
|
-- used as a source for a move, and then later as a target.
|
|
--
|
|
CREATE SCHEMA background_rebalance_parallel;
|
|
SET search_path TO background_rebalance_parallel;
|
|
SET citus.next_shard_id TO 85674000;
|
|
SET citus.shard_replication_factor TO 1;
|
|
SET client_min_messages TO ERROR;
|
|
ALTER SEQUENCE pg_dist_background_job_job_id_seq RESTART 17777;
|
|
ALTER SEQUENCE pg_dist_background_task_task_id_seq RESTART 1000;
|
|
ALTER SEQUENCE pg_catalog.pg_dist_colocationid_seq RESTART 50050;
|
|
SELECT nextval('pg_catalog.pg_dist_groupid_seq') AS last_group_id_cls \gset
|
|
SELECT nextval('pg_catalog.pg_dist_node_nodeid_seq') AS last_node_id_cls \gset
|
|
ALTER SEQUENCE pg_catalog.pg_dist_groupid_seq RESTART 50;
|
|
ALTER SEQUENCE pg_catalog.pg_dist_node_nodeid_seq RESTART 50;
|
|
SELECT 1 FROM master_remove_node('localhost', :worker_1_port);
|
|
?column?
|
|
---------------------------------------------------------------------
|
|
1
|
|
(1 row)
|
|
|
|
SELECT 1 FROM master_remove_node('localhost', :worker_2_port);
|
|
?column?
|
|
---------------------------------------------------------------------
|
|
1
|
|
(1 row)
|
|
|
|
SELECT 1 FROM master_add_node('localhost', :worker_1_port);
|
|
?column?
|
|
---------------------------------------------------------------------
|
|
1
|
|
(1 row)
|
|
|
|
SELECT 1 FROM master_add_node('localhost', :worker_2_port);
|
|
?column?
|
|
---------------------------------------------------------------------
|
|
1
|
|
(1 row)
|
|
|
|
ALTER SYSTEM SET citus.background_task_queue_interval TO '1s';
|
|
SELECT pg_reload_conf();
|
|
pg_reload_conf
|
|
---------------------------------------------------------------------
|
|
t
|
|
(1 row)
|
|
|
|
-- Colocation group 1: create two tables table1_colg1, table2_colg1 and in a colocation group
|
|
CREATE TABLE table1_colg1 (a int PRIMARY KEY);
|
|
SELECT create_distributed_table('table1_colg1', 'a', shard_count => 4, colocate_with => 'none');
|
|
create_distributed_table
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
CREATE TABLE table2_colg1 (b int PRIMARY KEY);
|
|
SELECT create_distributed_table('table2_colg1', 'b', colocate_with => 'table1_colg1');
|
|
create_distributed_table
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
-- Colocation group 2: create two tables table1_colg2, table2_colg2 and in a colocation group
|
|
CREATE TABLE table1_colg2 (a int PRIMARY KEY);
|
|
SELECT create_distributed_table('table1_colg2', 'a', shard_count => 4, colocate_with => 'none');
|
|
create_distributed_table
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
CREATE TABLE table2_colg2 (b int primary key);
|
|
SELECT create_distributed_table('table2_colg2', 'b', colocate_with => 'table1_colg2');
|
|
create_distributed_table
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
-- Colocation group 3: create two tables table1_colg3, table2_colg3 and in a colocation group
|
|
CREATE TABLE table1_colg3 (a int PRIMARY KEY);
|
|
SELECT create_distributed_table('table1_colg3', 'a', shard_count => 4, colocate_with => 'none');
|
|
create_distributed_table
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
CREATE TABLE table2_colg3 (b int primary key);
|
|
SELECT create_distributed_table('table2_colg3', 'b', colocate_with => 'table1_colg3');
|
|
create_distributed_table
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
-- Add two new nodes so that we can rebalance
|
|
SELECT 1 FROM citus_add_node('localhost', :worker_3_port);
|
|
?column?
|
|
---------------------------------------------------------------------
|
|
1
|
|
(1 row)
|
|
|
|
SELECT 1 FROM citus_add_node('localhost', :worker_4_port);
|
|
?column?
|
|
---------------------------------------------------------------------
|
|
1
|
|
(1 row)
|
|
|
|
SELECT * FROM get_rebalance_table_shards_plan() ORDER BY shardid;
|
|
table_name | shardid | shard_size | sourcename | sourceport | targetname | targetport
|
|
---------------------------------------------------------------------
|
|
table1_colg1 | 85674000 | 0 | localhost | 57637 | localhost | 57640
|
|
table1_colg1 | 85674001 | 0 | localhost | 57638 | localhost | 57639
|
|
table2_colg1 | 85674004 | 0 | localhost | 57637 | localhost | 57640
|
|
table2_colg1 | 85674005 | 0 | localhost | 57638 | localhost | 57639
|
|
table1_colg2 | 85674008 | 0 | localhost | 57637 | localhost | 57640
|
|
table1_colg2 | 85674009 | 0 | localhost | 57638 | localhost | 57639
|
|
table2_colg2 | 85674012 | 0 | localhost | 57637 | localhost | 57640
|
|
table2_colg2 | 85674013 | 0 | localhost | 57638 | localhost | 57639
|
|
table1_colg3 | 85674016 | 0 | localhost | 57637 | localhost | 57640
|
|
table1_colg3 | 85674017 | 0 | localhost | 57638 | localhost | 57639
|
|
table2_colg3 | 85674020 | 0 | localhost | 57637 | localhost | 57640
|
|
table2_colg3 | 85674021 | 0 | localhost | 57638 | localhost | 57639
|
|
(12 rows)
|
|
|
|
SELECT * FROM citus_rebalance_start();
|
|
citus_rebalance_start
|
|
---------------------------------------------------------------------
|
|
17777
|
|
(1 row)
|
|
|
|
SELECT citus_rebalance_wait();
|
|
citus_rebalance_wait
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
-- PART 1
|
|
-- Test to check if the background tasks scheduled by the background rebalancer
|
|
-- have the correct dependencies
|
|
-- Check that a move is dependent on
|
|
-- any other move scheduled earlier in its colocation group.
|
|
SELECT S.shardid, P.colocationid
|
|
FROM pg_dist_shard S, pg_dist_partition P
|
|
WHERE S.logicalrelid = P.logicalrelid ORDER BY S.shardid ASC;
|
|
shardid | colocationid
|
|
---------------------------------------------------------------------
|
|
85674000 | 50050
|
|
85674001 | 50050
|
|
85674002 | 50050
|
|
85674003 | 50050
|
|
85674004 | 50050
|
|
85674005 | 50050
|
|
85674006 | 50050
|
|
85674007 | 50050
|
|
85674008 | 50051
|
|
85674009 | 50051
|
|
85674010 | 50051
|
|
85674011 | 50051
|
|
85674012 | 50051
|
|
85674013 | 50051
|
|
85674014 | 50051
|
|
85674015 | 50051
|
|
85674016 | 50052
|
|
85674017 | 50052
|
|
85674018 | 50052
|
|
85674019 | 50052
|
|
85674020 | 50052
|
|
85674021 | 50052
|
|
85674022 | 50052
|
|
85674023 | 50052
|
|
(24 rows)
|
|
|
|
SELECT D.task_id,
|
|
(SELECT T.command FROM pg_dist_background_task T WHERE T.task_id = D.task_id),
|
|
D.depends_on,
|
|
(SELECT T.command FROM pg_dist_background_task T WHERE T.task_id = D.depends_on)
|
|
FROM pg_dist_background_task_depend D WHERE job_id = 17777 ORDER BY D.task_id, D.depends_on ASC;
|
|
task_id | command | depends_on | command
|
|
---------------------------------------------------------------------
|
|
1001 | SELECT pg_catalog.citus_move_shard_placement(85674000,50,53,'auto') | 1000 | SELECT pg_catalog.citus_move_shard_placement(85674001,51,52,'auto')
|
|
1003 | SELECT pg_catalog.citus_move_shard_placement(85674008,50,53,'auto') | 1002 | SELECT pg_catalog.citus_move_shard_placement(85674009,51,52,'auto')
|
|
1005 | SELECT pg_catalog.citus_move_shard_placement(85674016,50,53,'auto') | 1004 | SELECT pg_catalog.citus_move_shard_placement(85674017,51,52,'auto')
|
|
(3 rows)
|
|
|
|
-- Check that if there is a reference table that needs to be synched to a node,
|
|
-- any move without a dependency must depend on the move task for reference table.
|
|
SELECT 1 FROM citus_drain_node('localhost',:worker_4_port);
|
|
?column?
|
|
---------------------------------------------------------------------
|
|
1
|
|
(1 row)
|
|
|
|
SELECT public.wait_for_resource_cleanup();
|
|
wait_for_resource_cleanup
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT 1 FROM citus_disable_node('localhost', :worker_4_port, synchronous:=true);
|
|
?column?
|
|
---------------------------------------------------------------------
|
|
1
|
|
(1 row)
|
|
|
|
-- Drain worker_3 so that we can move only one colocation group to worker_3
|
|
-- to create an unbalance that would cause parallel rebalancing.
|
|
SELECT 1 FROM citus_drain_node('localhost',:worker_3_port);
|
|
?column?
|
|
---------------------------------------------------------------------
|
|
1
|
|
(1 row)
|
|
|
|
SELECT citus_set_node_property('localhost', :worker_3_port, 'shouldhaveshards', true);
|
|
citus_set_node_property
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
CALL citus_cleanup_orphaned_resources();
|
|
CREATE TABLE ref_table(a int PRIMARY KEY);
|
|
SELECT create_reference_table('ref_table');
|
|
create_reference_table
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
-- Move all the shards of Colocation group 3 to worker_3.
|
|
SELECT
|
|
master_move_shard_placement(shardid, 'localhost', nodeport, 'localhost', :worker_3_port, 'block_writes')
|
|
FROM
|
|
pg_dist_shard NATURAL JOIN pg_dist_shard_placement
|
|
WHERE
|
|
logicalrelid = 'table1_colg3'::regclass AND nodeport <> :worker_3_port
|
|
ORDER BY
|
|
shardid;
|
|
master_move_shard_placement
|
|
---------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
(4 rows)
|
|
|
|
CALL citus_cleanup_orphaned_resources();
|
|
-- Activate and new nodes so that we can rebalance.
|
|
SELECT 1 FROM citus_activate_node('localhost', :worker_4_port);
|
|
?column?
|
|
---------------------------------------------------------------------
|
|
1
|
|
(1 row)
|
|
|
|
SELECT citus_set_node_property('localhost', :worker_4_port, 'shouldhaveshards', true);
|
|
citus_set_node_property
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT 1 FROM citus_add_node('localhost', :worker_5_port);
|
|
?column?
|
|
---------------------------------------------------------------------
|
|
1
|
|
(1 row)
|
|
|
|
SELECT 1 FROM citus_add_node('localhost', :worker_6_port);
|
|
?column?
|
|
---------------------------------------------------------------------
|
|
1
|
|
(1 row)
|
|
|
|
SELECT * FROM citus_rebalance_start();
|
|
citus_rebalance_start
|
|
---------------------------------------------------------------------
|
|
17778
|
|
(1 row)
|
|
|
|
SELECT citus_rebalance_wait();
|
|
citus_rebalance_wait
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT S.shardid, P.colocationid
|
|
FROM pg_dist_shard S, pg_dist_partition P
|
|
WHERE S.logicalrelid = P.logicalrelid ORDER BY S.shardid ASC;
|
|
shardid | colocationid
|
|
---------------------------------------------------------------------
|
|
85674000 | 50050
|
|
85674001 | 50050
|
|
85674002 | 50050
|
|
85674003 | 50050
|
|
85674004 | 50050
|
|
85674005 | 50050
|
|
85674006 | 50050
|
|
85674007 | 50050
|
|
85674008 | 50051
|
|
85674009 | 50051
|
|
85674010 | 50051
|
|
85674011 | 50051
|
|
85674012 | 50051
|
|
85674013 | 50051
|
|
85674014 | 50051
|
|
85674015 | 50051
|
|
85674016 | 50052
|
|
85674017 | 50052
|
|
85674018 | 50052
|
|
85674019 | 50052
|
|
85674020 | 50052
|
|
85674021 | 50052
|
|
85674022 | 50052
|
|
85674023 | 50052
|
|
85674024 | 50053
|
|
(25 rows)
|
|
|
|
SELECT D.task_id,
|
|
(SELECT T.command FROM pg_dist_background_task T WHERE T.task_id = D.task_id),
|
|
D.depends_on,
|
|
(SELECT T.command FROM pg_dist_background_task T WHERE T.task_id = D.depends_on)
|
|
FROM pg_dist_background_task_depend D WHERE job_id = 17778 ORDER BY D.task_id, D.depends_on ASC;
|
|
task_id | command | depends_on | command
|
|
---------------------------------------------------------------------
|
|
1007 | SELECT pg_catalog.citus_move_shard_placement(85674016,52,53,'auto') | 1006 | SELECT pg_catalog.replicate_reference_tables('auto')
|
|
1008 | SELECT pg_catalog.citus_move_shard_placement(85674003,51,54,'auto') | 1006 | SELECT pg_catalog.replicate_reference_tables('auto')
|
|
1009 | SELECT pg_catalog.citus_move_shard_placement(85674000,50,55,'auto') | 1008 | SELECT pg_catalog.citus_move_shard_placement(85674003,51,54,'auto')
|
|
1010 | SELECT pg_catalog.citus_move_shard_placement(85674017,52,53,'auto') | 1007 | SELECT pg_catalog.citus_move_shard_placement(85674016,52,53,'auto')
|
|
1011 | SELECT pg_catalog.citus_move_shard_placement(85674008,51,54,'auto') | 1006 | SELECT pg_catalog.replicate_reference_tables('auto')
|
|
1012 | SELECT pg_catalog.citus_move_shard_placement(85674001,50,55,'auto') | 1009 | SELECT pg_catalog.citus_move_shard_placement(85674000,50,55,'auto')
|
|
(6 rows)
|
|
|
|
-- PART 2
|
|
-- Test to verify that we do not allow parallel rebalancer moves involving a
|
|
-- particular node (either as source or target)
|
|
-- more than citus.max_background_task_executors_per_node
|
|
-- and that we can change the GUC on the fly
|
|
-- citus_task_wait calls are used to ensure consistent pg_dist_background_task query
|
|
-- output i.e. to avoid flakiness
|
|
-- First let's restart the scenario
|
|
DROP SCHEMA background_rebalance_parallel CASCADE;
|
|
TRUNCATE pg_dist_background_job CASCADE;
|
|
TRUNCATE pg_dist_background_task CASCADE;
|
|
TRUNCATE pg_dist_background_task_depend;
|
|
SELECT public.wait_for_resource_cleanup();
|
|
wait_for_resource_cleanup
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
select citus_remove_node('localhost', :worker_2_port);
|
|
citus_remove_node
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
select citus_remove_node('localhost', :worker_3_port);
|
|
citus_remove_node
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
select citus_remove_node('localhost', :worker_4_port);
|
|
citus_remove_node
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
select citus_remove_node('localhost', :worker_5_port);
|
|
citus_remove_node
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
select citus_remove_node('localhost', :worker_6_port);
|
|
citus_remove_node
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
CREATE SCHEMA background_rebalance_parallel;
|
|
SET search_path TO background_rebalance_parallel;
|
|
-- Create 8 tables in 4 colocation groups, and populate them
|
|
CREATE TABLE table1_colg1 (a int PRIMARY KEY);
|
|
SELECT create_distributed_table('table1_colg1', 'a', shard_count => 3, colocate_with => 'none');
|
|
create_distributed_table
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
INSERT INTO table1_colg1 SELECT i FROM generate_series(0, 100)i;
|
|
CREATE TABLE table2_colg1 (b int PRIMARY KEY);
|
|
SELECT create_distributed_table('table2_colg1', 'b', colocate_with => 'table1_colg1');
|
|
create_distributed_table
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
INSERT INTO table2_colg1 SELECT i FROM generate_series(0, 100)i;
|
|
CREATE TABLE table1_colg2 (a int PRIMARY KEY);
|
|
SELECT create_distributed_table('table1_colg2', 'a', shard_count => 3, colocate_with => 'none');
|
|
create_distributed_table
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
INSERT INTO table1_colg2 SELECT i FROM generate_series(0, 100)i;
|
|
CREATE TABLE table2_colg2 (b int PRIMARY KEY);
|
|
SELECT create_distributed_table('table2_colg2', 'b', colocate_with => 'table1_colg2');
|
|
create_distributed_table
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
INSERT INTO table2_colg2 SELECT i FROM generate_series(0, 100)i;
|
|
CREATE TABLE table1_colg3 (a int PRIMARY KEY);
|
|
SELECT create_distributed_table('table1_colg3', 'a', shard_count => 3, colocate_with => 'none');
|
|
create_distributed_table
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
INSERT INTO table1_colg3 SELECT i FROM generate_series(0, 100)i;
|
|
CREATE TABLE table2_colg3 (b int primary key);
|
|
SELECT create_distributed_table('table2_colg3', 'b', colocate_with => 'table1_colg3');
|
|
create_distributed_table
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
INSERT INTO table2_colg3 SELECT i FROM generate_series(0, 100)i;
|
|
CREATE TABLE table1_colg4 (a int PRIMARY KEY);
|
|
SELECT create_distributed_table('table1_colg4', 'a', shard_count => 3, colocate_with => 'none');
|
|
create_distributed_table
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
INSERT INTO table1_colg4 SELECT i FROM generate_series(0, 100)i;
|
|
CREATE TABLE table2_colg4 (b int PRIMARY KEY);
|
|
SELECT create_distributed_table('table2_colg4', 'b', colocate_with => 'table1_colg4');
|
|
create_distributed_table
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
INSERT INTO table2_colg4 SELECT i FROM generate_series(0, 100)i;
|
|
-- Add nodes so that we can rebalance
|
|
SELECT citus_add_node('localhost', :worker_2_port);
|
|
citus_add_node
|
|
---------------------------------------------------------------------
|
|
56
|
|
(1 row)
|
|
|
|
SELECT citus_add_node('localhost', :worker_3_port);
|
|
citus_add_node
|
|
---------------------------------------------------------------------
|
|
57
|
|
(1 row)
|
|
|
|
SELECT citus_rebalance_start AS job_id from citus_rebalance_start() \gset
|
|
-- see dependent tasks to understand which tasks remain runnable because of
|
|
-- citus.max_background_task_executors_per_node
|
|
-- and which tasks are actually blocked from colocation group dependencies
|
|
SELECT task_id, depends_on
|
|
FROM pg_dist_background_task_depend
|
|
WHERE job_id in (:job_id)
|
|
ORDER BY 1, 2 ASC;
|
|
task_id | depends_on
|
|
---------------------------------------------------------------------
|
|
1014 | 1013
|
|
1016 | 1015
|
|
1018 | 1017
|
|
1020 | 1019
|
|
(4 rows)
|
|
|
|
-- default citus.max_background_task_executors_per_node is 1
|
|
-- show that first exactly one task per node is running
|
|
-- among the tasks that are not blocked
|
|
SELECT citus_task_wait(1013, desired_status => 'running');
|
|
citus_task_wait
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT job_id, task_id, status, nodes_involved
|
|
FROM pg_dist_background_task WHERE job_id in (:job_id) ORDER BY task_id;
|
|
job_id | task_id | status | nodes_involved
|
|
---------------------------------------------------------------------
|
|
17779 | 1013 | running | {50,56}
|
|
17779 | 1014 | blocked | {50,57}
|
|
17779 | 1015 | runnable | {50,56}
|
|
17779 | 1016 | blocked | {50,57}
|
|
17779 | 1017 | runnable | {50,56}
|
|
17779 | 1018 | blocked | {50,57}
|
|
17779 | 1019 | runnable | {50,56}
|
|
17779 | 1020 | blocked | {50,57}
|
|
(8 rows)
|
|
|
|
-- increase citus.max_background_task_executors_per_node
|
|
SELECT citus_task_wait(1013, desired_status => 'done');
|
|
citus_task_wait
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
ALTER SYSTEM SET citus.max_background_task_executors_per_node = 2;
|
|
SELECT pg_reload_conf();
|
|
pg_reload_conf
|
|
---------------------------------------------------------------------
|
|
t
|
|
(1 row)
|
|
|
|
SELECT citus_task_wait(1014, desired_status => 'running');
|
|
citus_task_wait
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT citus_task_wait(1015, desired_status => 'running');
|
|
citus_task_wait
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
-- show that at most 2 tasks per node are running
|
|
-- among the tasks that are not blocked
|
|
SELECT job_id, task_id, status, nodes_involved
|
|
FROM pg_dist_background_task WHERE job_id in (:job_id) ORDER BY task_id;
|
|
job_id | task_id | status | nodes_involved
|
|
---------------------------------------------------------------------
|
|
17779 | 1013 | done | {50,56}
|
|
17779 | 1014 | running | {50,57}
|
|
17779 | 1015 | running | {50,56}
|
|
17779 | 1016 | blocked | {50,57}
|
|
17779 | 1017 | runnable | {50,56}
|
|
17779 | 1018 | blocked | {50,57}
|
|
17779 | 1019 | runnable | {50,56}
|
|
17779 | 1020 | blocked | {50,57}
|
|
(8 rows)
|
|
|
|
-- decrease to default (1)
|
|
ALTER SYSTEM RESET citus.max_background_task_executors_per_node;
|
|
SELECT pg_reload_conf();
|
|
pg_reload_conf
|
|
---------------------------------------------------------------------
|
|
t
|
|
(1 row)
|
|
|
|
SELECT citus_task_wait(1015, desired_status => 'done');
|
|
citus_task_wait
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT citus_task_wait(1014, desired_status => 'done');
|
|
citus_task_wait
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT citus_task_wait(1016, desired_status => 'running');
|
|
citus_task_wait
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
-- show that exactly one task per node is running
|
|
-- among the tasks that are not blocked
|
|
SELECT job_id, task_id, status, nodes_involved
|
|
FROM pg_dist_background_task WHERE job_id in (:job_id) ORDER BY task_id;
|
|
job_id | task_id | status | nodes_involved
|
|
---------------------------------------------------------------------
|
|
17779 | 1013 | done | {50,56}
|
|
17779 | 1014 | done | {50,57}
|
|
17779 | 1015 | done | {50,56}
|
|
17779 | 1016 | running | {50,57}
|
|
17779 | 1017 | runnable | {50,56}
|
|
17779 | 1018 | blocked | {50,57}
|
|
17779 | 1019 | runnable | {50,56}
|
|
17779 | 1020 | blocked | {50,57}
|
|
(8 rows)
|
|
|
|
SELECT citus_rebalance_stop();
|
|
citus_rebalance_stop
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
-- PART 3
|
|
-- Test to verify that there's a hard dependency when A specific node is first being used as a
|
|
-- source for a move, and then later as a target.
|
|
-- First let's restart the scenario
|
|
DROP SCHEMA background_rebalance_parallel CASCADE;
|
|
TRUNCATE pg_dist_background_job CASCADE;
|
|
TRUNCATE pg_dist_background_task CASCADE;
|
|
TRUNCATE pg_dist_background_task_depend;
|
|
SELECT public.wait_for_resource_cleanup();
|
|
wait_for_resource_cleanup
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
select citus_remove_node('localhost', :worker_1_port);
|
|
citus_remove_node
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
select citus_remove_node('localhost', :worker_2_port);
|
|
citus_remove_node
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
select citus_remove_node('localhost', :worker_3_port);
|
|
citus_remove_node
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
CREATE SCHEMA background_rebalance_parallel;
|
|
SET search_path TO background_rebalance_parallel;
|
|
SET citus.next_shard_id TO 85674051;
|
|
ALTER SEQUENCE pg_catalog.pg_dist_node_nodeid_seq RESTART 61;
|
|
-- add the first node
|
|
-- nodeid here is 61
|
|
select citus_add_node('localhost', :worker_1_port);
|
|
citus_add_node
|
|
---------------------------------------------------------------------
|
|
61
|
|
(1 row)
|
|
|
|
-- create, populate and distribute 6 tables, each with 1 shard, none colocated with each other
|
|
CREATE TABLE table1_colg1 (a int PRIMARY KEY);
|
|
SELECT create_distributed_table('table1_colg1', 'a', shard_count => 1, colocate_with => 'none');
|
|
create_distributed_table
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
INSERT INTO table1_colg1 SELECT i FROM generate_series(0, 100)i;
|
|
CREATE TABLE table1_colg2 (a int PRIMARY KEY);
|
|
SELECT create_distributed_table('table1_colg2', 'a', shard_count => 1, colocate_with => 'none');
|
|
create_distributed_table
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
INSERT INTO table1_colg2 SELECT i FROM generate_series(0, 100)i;
|
|
CREATE TABLE table1_colg3 (a int PRIMARY KEY);
|
|
SELECT create_distributed_table('table1_colg3', 'a', shard_count => 1, colocate_with => 'none');
|
|
create_distributed_table
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
INSERT INTO table1_colg3 SELECT i FROM generate_series(0, 100)i;
|
|
CREATE TABLE table1_colg4 (a int PRIMARY KEY);
|
|
SELECT create_distributed_table('table1_colg4', 'a', shard_count => 1, colocate_with => 'none');
|
|
create_distributed_table
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
INSERT INTO table1_colg4 SELECT i FROM generate_series(0, 100)i;
|
|
CREATE TABLE table1_colg5 (a int PRIMARY KEY);
|
|
SELECT create_distributed_table('table1_colg5', 'a', shard_count => 1, colocate_with => 'none');
|
|
create_distributed_table
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
INSERT INTO table1_colg5 SELECT i FROM generate_series(0, 100)i;
|
|
CREATE TABLE table1_colg6 (a int PRIMARY KEY);
|
|
SELECT create_distributed_table('table1_colg6', 'a', shard_count => 1, colocate_with => 'none');
|
|
create_distributed_table
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
INSERT INTO table1_colg6 SELECT i FROM generate_series(0, 100)i;
|
|
-- add two other nodes
|
|
-- nodeid here is 62
|
|
select citus_add_node('localhost', :worker_2_port);
|
|
citus_add_node
|
|
---------------------------------------------------------------------
|
|
62
|
|
(1 row)
|
|
|
|
-- nodeid here is 63
|
|
select citus_add_node('localhost', :worker_3_port);
|
|
citus_add_node
|
|
---------------------------------------------------------------------
|
|
63
|
|
(1 row)
|
|
|
|
CREATE OR REPLACE FUNCTION shard_placement_rebalance_array(
|
|
worker_node_list json[],
|
|
shard_placement_list json[],
|
|
threshold float4 DEFAULT 0,
|
|
max_shard_moves int DEFAULT 1000000,
|
|
drain_only bool DEFAULT false,
|
|
improvement_threshold float4 DEFAULT 0.5
|
|
)
|
|
RETURNS json[]
|
|
AS 'citus'
|
|
LANGUAGE C STRICT VOLATILE;
|
|
-- we are simulating the following from shard_rebalancer_unit.sql
|
|
-- the following steps are all according to this scenario
|
|
-- where the third move should be dependent of the first two
|
|
-- because the third move's target is the source of the first two
|
|
SELECT unnest(shard_placement_rebalance_array(
|
|
ARRAY['{"node_name": "hostname1", "disallowed_shards": "1,2,3,5,6"}',
|
|
'{"node_name": "hostname2", "disallowed_shards": "4"}',
|
|
'{"node_name": "hostname3", "disallowed_shards": "4"}'
|
|
]::json[],
|
|
ARRAY['{"shardid":1, "nodename":"hostname1"}',
|
|
'{"shardid":2, "nodename":"hostname1"}',
|
|
'{"shardid":3, "nodename":"hostname2"}',
|
|
'{"shardid":4, "nodename":"hostname2"}',
|
|
'{"shardid":5, "nodename":"hostname3"}',
|
|
'{"shardid":6, "nodename":"hostname3"}'
|
|
]::json[]
|
|
));
|
|
unnest
|
|
---------------------------------------------------------------------
|
|
{"updatetype":1,"shardid":1,"sourcename":"hostname1","sourceport":5432,"targetname":"hostname2","targetport":5432}
|
|
{"updatetype":1,"shardid":2,"sourcename":"hostname1","sourceport":5432,"targetname":"hostname3","targetport":5432}
|
|
{"updatetype":1,"shardid":4,"sourcename":"hostname2","sourceport":5432,"targetname":"hostname1","targetport":5432}
|
|
(3 rows)
|
|
|
|
-- manually balance the cluster such that we have
|
|
-- a balanced cluster like above with 1,2,3,4,5,6 and hostname1/2/3
|
|
-- shardid 85674051 (1) nodeid 61 (hostname1)
|
|
-- shardid 85674052 (2) nodeid 61 (hostname1)
|
|
-- shardid 85674053 (3) nodeid 62 (hostname2)
|
|
-- shardid 85674054 (4) nodeid 62 (hostname2)
|
|
-- shardid 85674055 (5) nodeid 63 (hostname3)
|
|
-- shardid 85674056 (6) nodeid 63 (hostname3)
|
|
SELECT pg_catalog.citus_move_shard_placement(85674053,61,62,'auto');
|
|
citus_move_shard_placement
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT pg_catalog.citus_move_shard_placement(85674054,61,62,'auto');
|
|
citus_move_shard_placement
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT pg_catalog.citus_move_shard_placement(85674055,61,63,'auto');
|
|
citus_move_shard_placement
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT pg_catalog.citus_move_shard_placement(85674056,61,63,'auto');
|
|
citus_move_shard_placement
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
-- now create another rebalance strategy in order to simulate moves
|
|
-- which use as target a node that has been previously used as source
|
|
CREATE OR REPLACE FUNCTION test_shard_allowed_on_node(shardid bigint, nodeid int)
|
|
RETURNS boolean AS
|
|
$$
|
|
-- analogous to '{"node_name": "hostname1", "disallowed_shards": "1,2,3,5,6"}'
|
|
select case when (shardid != 85674054 and nodeid = 61)
|
|
then false
|
|
-- analogous to '{"node_name": "hostname2", "disallowed_shards": "4"}'
|
|
-- AND '{"node_name": "hostname2", "disallowed_shards": "4"}'
|
|
when (shardid = 85674054 and nodeid != 61)
|
|
then false
|
|
else true
|
|
end;
|
|
$$ LANGUAGE sql;
|
|
-- insert the new test rebalance strategy
|
|
INSERT INTO
|
|
pg_catalog.pg_dist_rebalance_strategy(
|
|
name,
|
|
default_strategy,
|
|
shard_cost_function,
|
|
node_capacity_function,
|
|
shard_allowed_on_node_function,
|
|
default_threshold,
|
|
minimum_threshold,
|
|
improvement_threshold
|
|
) VALUES (
|
|
'test_source_then_target',
|
|
false,
|
|
'citus_shard_cost_1',
|
|
'citus_node_capacity_1',
|
|
'background_rebalance_parallel.test_shard_allowed_on_node',
|
|
0,
|
|
0,
|
|
0
|
|
);
|
|
SELECT * FROM get_rebalance_table_shards_plan(rebalance_strategy := 'test_source_then_target');
|
|
table_name | shardid | shard_size | sourcename | sourceport | targetname | targetport
|
|
---------------------------------------------------------------------
|
|
table1_colg1 | 85674051 | 0 | localhost | 57637 | localhost | 57638
|
|
table1_colg2 | 85674052 | 0 | localhost | 57637 | localhost | 57639
|
|
table1_colg4 | 85674054 | 0 | localhost | 57638 | localhost | 57637
|
|
(3 rows)
|
|
|
|
SELECT citus_rebalance_start AS job_id from citus_rebalance_start(rebalance_strategy := 'test_source_then_target') \gset
|
|
-- check that the third move is blocked and depends on the first two
|
|
SELECT job_id, task_id, status, nodes_involved
|
|
FROM pg_dist_background_task WHERE job_id in (:job_id) ORDER BY task_id;
|
|
job_id | task_id | status | nodes_involved
|
|
---------------------------------------------------------------------
|
|
17780 | 1021 | runnable | {61,62}
|
|
17780 | 1022 | runnable | {61,63}
|
|
17780 | 1023 | blocked | {62,61}
|
|
(3 rows)
|
|
|
|
SELECT D.task_id,
|
|
(SELECT T.command FROM pg_dist_background_task T WHERE T.task_id = D.task_id),
|
|
D.depends_on,
|
|
(SELECT T.command FROM pg_dist_background_task T WHERE T.task_id = D.depends_on)
|
|
FROM pg_dist_background_task_depend D WHERE job_id in (:job_id) ORDER BY D.task_id, D.depends_on ASC;
|
|
task_id | command | depends_on | command
|
|
---------------------------------------------------------------------
|
|
1023 | SELECT pg_catalog.citus_move_shard_placement(85674054,62,61,'auto') | 1021 | SELECT pg_catalog.citus_move_shard_placement(85674051,61,62,'auto')
|
|
1023 | SELECT pg_catalog.citus_move_shard_placement(85674054,62,61,'auto') | 1022 | SELECT pg_catalog.citus_move_shard_placement(85674052,61,63,'auto')
|
|
(2 rows)
|
|
|
|
SELECT citus_rebalance_stop();
|
|
citus_rebalance_stop
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
DELETE FROM pg_catalog.pg_dist_rebalance_strategy WHERE name='test_source_then_target';
|
|
DROP SCHEMA background_rebalance_parallel CASCADE;
|
|
TRUNCATE pg_dist_background_job CASCADE;
|
|
TRUNCATE pg_dist_background_task CASCADE;
|
|
TRUNCATE pg_dist_background_task_depend;
|
|
SELECT public.wait_for_resource_cleanup();
|
|
wait_for_resource_cleanup
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
select citus_remove_node('localhost', :worker_3_port);
|
|
citus_remove_node
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
-- keep the rest of the tests inact that depends node/group ids
|
|
ALTER SEQUENCE pg_catalog.pg_dist_groupid_seq RESTART :last_group_id_cls;
|
|
ALTER SEQUENCE pg_catalog.pg_dist_node_nodeid_seq RESTART :last_node_id_cls;
|