external
/
citus
mirror of https://github.com/citusdata/citus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix background rebalance when reference table has no PK (#6682) 

DESCRIPTION: Fix background rebalance when reference table has no PK

For the background rebalance we would always fail if a reference table
that was not replicated to all nodes would not have a PK (or replica
identity). Even when we used force_logical or block_writes as the shard
transfer mode. This fixes that and adds some regression tests.

Fixes #6680
pull/6683/head
Jelte Fennema 2023-01-31 12:18:29 +01:00 committed by GitHub
parent d919506076
commit 14c31fbb07
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 129 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/backend/distributed/operations/shard_rebalancer.c
View File

@ -1937,7 +1937,10 @@ RebalanceTableShardsBackground(RebalanceOptions *options, Oid shardReplicationMo
if (HasNodesWithMissingReferenceTables(&referenceTableIdList))
{
VerifyTablesHaveReplicaIdentity(referenceTableIdList);
if (shardTransferMode == TRANSFER_MODE_AUTOMATIC)
{
VerifyTablesHaveReplicaIdentity(referenceTableIdList);
}
/*
* Reference tables need to be copied to (newly-added) nodes, this needs to be the

94
src/test/regress/expected/background_rebalance.out
View File

@ -210,6 +210,100 @@ SELECT citus_rebalance_stop();
(1 row)
RESET ROLE;
CREATE TABLE ref_no_pk(a int);
SELECT create_reference_table('ref_no_pk');
create_reference_table
---------------------------------------------------------------------
(1 row)
CREATE TABLE ref_with_pk(a int primary key);
SELECT create_reference_table('ref_with_pk');
create_reference_table
---------------------------------------------------------------------
(1 row)
-- Add coordinator so there's a node which doesn't have the reference tables
SELECT 1 FROM citus_add_node('localhost', :master_port, groupId=>0);
NOTICE: localhost:xxxxx is the coordinator and already contains metadata, skipping syncing the metadata
?column?
---------------------------------------------------------------------
1
(1 row)
-- fails
BEGIN;
SELECT 1 FROM citus_rebalance_start();
ERROR: cannot use logical replication to transfer shards of the relation ref_no_pk since it doesn't have a REPLICA IDENTITY or PRIMARY KEY
DETAIL: UPDATE and DELETE commands on the shard will error out during logical replication unless there is a REPLICA IDENTITY or PRIMARY KEY.
HINT: If you wish to continue without a replica identity set the shard_transfer_mode to 'force_logical' or 'block_writes'.
ROLLBACK;
-- success
BEGIN;
SELECT 1 FROM citus_rebalance_start(shard_transfer_mode := 'force_logical');
NOTICE: Scheduled 1 moves as job xxx
DETAIL: Rebalance scheduled as background job
HINT: To monitor progress, run: SELECT * FROM citus_rebalance_status();
?column?
---------------------------------------------------------------------
1
(1 row)
ROLLBACK;
-- success
BEGIN;
SELECT 1 FROM citus_rebalance_start(shard_transfer_mode := 'block_writes');
NOTICE: Scheduled 1 moves as job xxx
DETAIL: Rebalance scheduled as background job
HINT: To monitor progress, run: SELECT * FROM citus_rebalance_status();
?column?
---------------------------------------------------------------------
1
(1 row)
ROLLBACK;
-- fails
SELECT 1 FROM citus_rebalance_start();
ERROR: cannot use logical replication to transfer shards of the relation ref_no_pk since it doesn't have a REPLICA IDENTITY or PRIMARY KEY
DETAIL: UPDATE and DELETE commands on the shard will error out during logical replication unless there is a REPLICA IDENTITY or PRIMARY KEY.
HINT: If you wish to continue without a replica identity set the shard_transfer_mode to 'force_logical' or 'block_writes'.
-- succeeds
SELECT 1 FROM citus_rebalance_start(shard_transfer_mode := 'force_logical');
NOTICE: Scheduled 1 moves as job xxx
DETAIL: Rebalance scheduled as background job
HINT: To monitor progress, run: SELECT * FROM citus_rebalance_status();
?column?
---------------------------------------------------------------------
1
(1 row)
-- wait for success
SELECT citus_rebalance_wait();
citus_rebalance_wait
---------------------------------------------------------------------
(1 row)
SELECT state, details from citus_rebalance_status();
state | details
---------------------------------------------------------------------
finished | {"tasks": [], "task_state_counts": {"done": 2}}
(1 row)
-- Remove coordinator again to allow rerunning of this test
SELECT 1 FROM citus_remove_node('localhost', :master_port);
?column?
---------------------------------------------------------------------
1
(1 row)
SELECT public.wait_until_metadata_sync(30000);
wait_until_metadata_sync
---------------------------------------------------------------------
(1 row)
SET client_min_messages TO WARNING;
DROP SCHEMA background_rebalance CASCADE;
DROP USER non_super_user_rebalance;

32
src/test/regress/sql/background_rebalance.sql
View File

@ -61,7 +61,6 @@ SELECT citus_rebalance_wait();
DROP TABLE t1;
-- make sure a non-super user can stop rebalancing
CREATE USER non_super_user_rebalance WITH LOGIN;
GRANT ALL ON SCHEMA background_rebalance TO non_super_user_rebalance;
@ -77,6 +76,37 @@ SELECT citus_rebalance_stop();
RESET ROLE;
CREATE TABLE ref_no_pk(a int);
SELECT create_reference_table('ref_no_pk');
CREATE TABLE ref_with_pk(a int primary key);
SELECT create_reference_table('ref_with_pk');
-- Add coordinator so there's a node which doesn't have the reference tables
SELECT 1 FROM citus_add_node('localhost', :master_port, groupId=>0);
-- fails
BEGIN;
SELECT 1 FROM citus_rebalance_start();
ROLLBACK;
-- success
BEGIN;
SELECT 1 FROM citus_rebalance_start(shard_transfer_mode := 'force_logical');
ROLLBACK;
-- success
BEGIN;
SELECT 1 FROM citus_rebalance_start(shard_transfer_mode := 'block_writes');
ROLLBACK;
-- fails
SELECT 1 FROM citus_rebalance_start();
-- succeeds
SELECT 1 FROM citus_rebalance_start(shard_transfer_mode := 'force_logical');
-- wait for success
SELECT citus_rebalance_wait();
SELECT state, details from citus_rebalance_status();
-- Remove coordinator again to allow rerunning of this test
SELECT 1 FROM citus_remove_node('localhost', :master_port);
SELECT public.wait_until_metadata_sync(30000);
SET client_min_messages TO WARNING;
DROP SCHEMA background_rebalance CASCADE;