mirror of https://github.com/citusdata/citus.git
371 lines
13 KiB
Plaintext
371 lines
13 KiB
Plaintext
CREATE SCHEMA background_rebalance;
|
|
SET search_path TO background_rebalance;
|
|
SET citus.next_shard_id TO 85674000;
|
|
SET citus.shard_replication_factor TO 1;
|
|
ALTER SYSTEM SET citus.background_task_queue_interval TO '1s';
|
|
SELECT pg_reload_conf();
|
|
pg_reload_conf
|
|
---------------------------------------------------------------------
|
|
t
|
|
(1 row)
|
|
|
|
CREATE TABLE t1 (a int PRIMARY KEY);
|
|
SELECT create_distributed_table('t1', 'a', shard_count => 4, colocate_with => 'none');
|
|
create_distributed_table
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
-- verify the rebalance works - no-op - when the shards are balanced. Noop is shown by wait complaining there is nothing
|
|
-- to wait on.
|
|
SELECT 1 FROM citus_rebalance_start();
|
|
NOTICE: No moves available for rebalancing
|
|
?column?
|
|
---------------------------------------------------------------------
|
|
1
|
|
(1 row)
|
|
|
|
SELECT citus_rebalance_wait();
|
|
WARNING: no ongoing rebalance that can be waited on
|
|
citus_rebalance_wait
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT citus_move_shard_placement(85674000, 'localhost', :worker_1_port, 'localhost', :worker_2_port, shard_transfer_mode => 'block_writes');
|
|
citus_move_shard_placement
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
-- rebalance a table in the background
|
|
SELECT 1 FROM citus_rebalance_start();
|
|
NOTICE: Scheduled 1 moves as job xxx
|
|
DETAIL: Rebalance scheduled as background job
|
|
HINT: To monitor progress, run: SELECT * FROM citus_rebalance_status();
|
|
?column?
|
|
---------------------------------------------------------------------
|
|
1
|
|
(1 row)
|
|
|
|
SELECT citus_rebalance_wait();
|
|
citus_rebalance_wait
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT citus_move_shard_placement(85674000, 'localhost', :worker_1_port, 'localhost', :worker_2_port, shard_transfer_mode => 'block_writes');
|
|
citus_move_shard_placement
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
CREATE TABLE t2 (a int);
|
|
SELECT create_distributed_table('t2', 'a' , colocate_with => 't1');
|
|
create_distributed_table
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
-- show that we get an error when a table in the colocation group can't be moved non-blocking
|
|
SELECT 1 FROM citus_rebalance_start();
|
|
ERROR: cannot use logical replication to transfer shards of the relation t2 since it doesn't have a REPLICA IDENTITY or PRIMARY KEY
|
|
DETAIL: UPDATE and DELETE commands on the shard will error out during logical replication unless there is a REPLICA IDENTITY or PRIMARY KEY.
|
|
HINT: If you wish to continue without a replica identity set the shard_transfer_mode to 'force_logical' or 'block_writes'.
|
|
SELECT 1 FROM citus_rebalance_start(shard_transfer_mode => 'block_writes');
|
|
NOTICE: Scheduled 1 moves as job xxx
|
|
DETAIL: Rebalance scheduled as background job
|
|
HINT: To monitor progress, run: SELECT * FROM citus_rebalance_status();
|
|
?column?
|
|
---------------------------------------------------------------------
|
|
1
|
|
(1 row)
|
|
|
|
SELECT citus_rebalance_wait();
|
|
citus_rebalance_wait
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
DROP TABLE t2;
|
|
SELECT citus_move_shard_placement(85674000, 'localhost', :worker_1_port, 'localhost', :worker_2_port, shard_transfer_mode => 'block_writes');
|
|
citus_move_shard_placement
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
-- show we can stop a rebalance, the stop causes the move to not have happened, eg, our move back below fails.
|
|
SELECT 1 FROM citus_rebalance_start();
|
|
NOTICE: Scheduled 1 moves as job xxx
|
|
DETAIL: Rebalance scheduled as background job
|
|
HINT: To monitor progress, run: SELECT * FROM citus_rebalance_status();
|
|
?column?
|
|
---------------------------------------------------------------------
|
|
1
|
|
(1 row)
|
|
|
|
SELECT citus_rebalance_stop();
|
|
citus_rebalance_stop
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
-- waiting on this rebalance is racy, as it sometimes sees no rebalance is ongoing while other times it actually sees it ongoing
|
|
-- we simply sleep a bit here
|
|
SELECT pg_sleep(1);
|
|
pg_sleep
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
-- failing move due to a stopped rebalance, first clean orphans to make the error stable
|
|
SET client_min_messages TO WARNING;
|
|
CALL citus_cleanup_orphaned_resources();
|
|
RESET client_min_messages;
|
|
SELECT citus_move_shard_placement(85674000, 'localhost', :worker_1_port, 'localhost', :worker_2_port, shard_transfer_mode => 'block_writes');
|
|
WARNING: shard is already present on node localhost:xxxxx
|
|
DETAIL: Move may have already completed.
|
|
citus_move_shard_placement
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
-- show we can't start the rebalancer twice
|
|
SELECT 1 FROM citus_rebalance_start();
|
|
NOTICE: Scheduled 1 moves as job xxx
|
|
DETAIL: Rebalance scheduled as background job
|
|
HINT: To monitor progress, run: SELECT * FROM citus_rebalance_status();
|
|
?column?
|
|
---------------------------------------------------------------------
|
|
1
|
|
(1 row)
|
|
|
|
SELECT 1 FROM citus_rebalance_start();
|
|
ERROR: A rebalance is already running as job xxx
|
|
DETAIL: A rebalance was already scheduled as background job
|
|
HINT: To monitor progress, run: SELECT * FROM citus_rebalance_status();
|
|
SELECT citus_rebalance_wait();
|
|
citus_rebalance_wait
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
-- show that the old rebalancer cannot be started with a background rebalance in progress
|
|
SELECT citus_move_shard_placement(85674000, 'localhost', :worker_1_port, 'localhost', :worker_2_port, shard_transfer_mode => 'block_writes');
|
|
citus_move_shard_placement
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT 1 FROM citus_rebalance_start();
|
|
NOTICE: Scheduled 1 moves as job xxx
|
|
DETAIL: Rebalance scheduled as background job
|
|
HINT: To monitor progress, run: SELECT * FROM citus_rebalance_status();
|
|
?column?
|
|
---------------------------------------------------------------------
|
|
1
|
|
(1 row)
|
|
|
|
SELECT rebalance_table_shards();
|
|
ERROR: A rebalance is already running as job xxx
|
|
DETAIL: A rebalance was already scheduled as background job
|
|
HINT: To monitor progress, run: SELECT * FROM citus_rebalance_status();
|
|
SELECT citus_rebalance_wait();
|
|
citus_rebalance_wait
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
DROP TABLE t1;
|
|
-- make sure a non-super user can stop rebalancing
|
|
CREATE USER non_super_user_rebalance WITH LOGIN;
|
|
GRANT ALL ON SCHEMA background_rebalance TO non_super_user_rebalance;
|
|
SET ROLE non_super_user_rebalance;
|
|
CREATE TABLE non_super_user_t1 (a int PRIMARY KEY);
|
|
SELECT create_distributed_table('non_super_user_t1', 'a', shard_count => 4, colocate_with => 'none');
|
|
create_distributed_table
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT citus_move_shard_placement(85674008, 'localhost', :worker_1_port, 'localhost', :worker_2_port, shard_transfer_mode => 'block_writes');
|
|
citus_move_shard_placement
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT 1 FROM citus_rebalance_start();
|
|
NOTICE: Scheduled 1 moves as job xxx
|
|
DETAIL: Rebalance scheduled as background job
|
|
HINT: To monitor progress, run: SELECT * FROM citus_rebalance_status();
|
|
?column?
|
|
---------------------------------------------------------------------
|
|
1
|
|
(1 row)
|
|
|
|
SELECT citus_rebalance_stop();
|
|
citus_rebalance_stop
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
RESET ROLE;
|
|
CREATE TABLE ref_no_pk(a int);
|
|
SELECT create_reference_table('ref_no_pk');
|
|
create_reference_table
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
CREATE TABLE ref_with_pk(a int primary key);
|
|
SELECT create_reference_table('ref_with_pk');
|
|
create_reference_table
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
-- Add coordinator so there's a node which doesn't have the reference tables
|
|
SELECT 1 FROM citus_add_node('localhost', :master_port, groupId=>0);
|
|
NOTICE: localhost:xxxxx is the coordinator and already contains metadata, skipping syncing the metadata
|
|
?column?
|
|
---------------------------------------------------------------------
|
|
1
|
|
(1 row)
|
|
|
|
-- fails
|
|
BEGIN;
|
|
SELECT 1 FROM citus_rebalance_start();
|
|
ERROR: cannot use logical replication to transfer shards of the relation ref_no_pk since it doesn't have a REPLICA IDENTITY or PRIMARY KEY
|
|
DETAIL: UPDATE and DELETE commands on the shard will error out during logical replication unless there is a REPLICA IDENTITY or PRIMARY KEY.
|
|
HINT: If you wish to continue without a replica identity set the shard_transfer_mode to 'force_logical' or 'block_writes'.
|
|
ROLLBACK;
|
|
-- success
|
|
BEGIN;
|
|
SELECT 1 FROM citus_rebalance_start(shard_transfer_mode := 'force_logical');
|
|
NOTICE: Scheduled 1 moves as job xxx
|
|
DETAIL: Rebalance scheduled as background job
|
|
HINT: To monitor progress, run: SELECT * FROM citus_rebalance_status();
|
|
?column?
|
|
---------------------------------------------------------------------
|
|
1
|
|
(1 row)
|
|
|
|
ROLLBACK;
|
|
-- success
|
|
BEGIN;
|
|
SELECT 1 FROM citus_rebalance_start(shard_transfer_mode := 'block_writes');
|
|
NOTICE: Scheduled 1 moves as job xxx
|
|
DETAIL: Rebalance scheduled as background job
|
|
HINT: To monitor progress, run: SELECT * FROM citus_rebalance_status();
|
|
?column?
|
|
---------------------------------------------------------------------
|
|
1
|
|
(1 row)
|
|
|
|
ROLLBACK;
|
|
-- fails
|
|
SELECT 1 FROM citus_rebalance_start();
|
|
ERROR: cannot use logical replication to transfer shards of the relation ref_no_pk since it doesn't have a REPLICA IDENTITY or PRIMARY KEY
|
|
DETAIL: UPDATE and DELETE commands on the shard will error out during logical replication unless there is a REPLICA IDENTITY or PRIMARY KEY.
|
|
HINT: If you wish to continue without a replica identity set the shard_transfer_mode to 'force_logical' or 'block_writes'.
|
|
-- succeeds
|
|
SELECT 1 FROM citus_rebalance_start(shard_transfer_mode := 'force_logical');
|
|
NOTICE: Scheduled 1 moves as job xxx
|
|
DETAIL: Rebalance scheduled as background job
|
|
HINT: To monitor progress, run: SELECT * FROM citus_rebalance_status();
|
|
?column?
|
|
---------------------------------------------------------------------
|
|
1
|
|
(1 row)
|
|
|
|
-- wait for success
|
|
SELECT citus_rebalance_wait();
|
|
citus_rebalance_wait
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT state, details from citus_rebalance_status();
|
|
state | details
|
|
---------------------------------------------------------------------
|
|
finished | {"tasks": [], "task_state_counts": {"done": 2}}
|
|
(1 row)
|
|
|
|
SELECT public.wait_for_resource_cleanup();
|
|
wait_for_resource_cleanup
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
-- Remove coordinator again to allow rerunning of this test
|
|
SELECT 1 FROM citus_remove_node('localhost', :master_port);
|
|
?column?
|
|
---------------------------------------------------------------------
|
|
1
|
|
(1 row)
|
|
|
|
SELECT public.wait_until_metadata_sync(30000);
|
|
wait_until_metadata_sync
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
-- make sure a non-super user can rebalance when there are reference tables to replicate
|
|
CREATE TABLE ref_table(a int primary key);
|
|
SELECT create_reference_table('ref_table');
|
|
create_reference_table
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
-- add a new node to trigger replicate_reference_tables task
|
|
SELECT 1 FROM citus_add_node('localhost', :worker_3_port);
|
|
?column?
|
|
---------------------------------------------------------------------
|
|
1
|
|
(1 row)
|
|
|
|
SET ROLE non_super_user_rebalance;
|
|
SELECT 1 FROM citus_rebalance_start(shard_transfer_mode := 'force_logical');
|
|
NOTICE: Scheduled 1 moves as job xxx
|
|
DETAIL: Rebalance scheduled as background job
|
|
HINT: To monitor progress, run: SELECT * FROM citus_rebalance_status();
|
|
?column?
|
|
---------------------------------------------------------------------
|
|
1
|
|
(1 row)
|
|
|
|
-- wait for success
|
|
SELECT citus_rebalance_wait();
|
|
citus_rebalance_wait
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT state, details from citus_rebalance_status();
|
|
state | details
|
|
---------------------------------------------------------------------
|
|
finished | {"tasks": [], "task_state_counts": {"done": 2}}
|
|
(1 row)
|
|
|
|
RESET ROLE;
|
|
-- reset the the number of nodes by removing the previously added node
|
|
SELECT 1 FROM citus_drain_node('localhost', :worker_3_port);
|
|
NOTICE: Moving shard xxxxx from localhost:xxxxx to localhost:xxxxx ...
|
|
?column?
|
|
---------------------------------------------------------------------
|
|
1
|
|
(1 row)
|
|
|
|
CALL citus_cleanup_orphaned_resources();
|
|
NOTICE: cleaned up 1 orphaned resources
|
|
SELECT 1 FROM citus_remove_node('localhost', :worker_3_port);
|
|
?column?
|
|
---------------------------------------------------------------------
|
|
1
|
|
(1 row)
|
|
|
|
SET client_min_messages TO WARNING;
|
|
DROP SCHEMA background_rebalance CASCADE;
|
|
DROP USER non_super_user_rebalance;
|