external
/
citus
mirror of https://github.com/citusdata/citus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix metadata sync fails in operations_schedule

pull/5506/head
Halil Ozan Akgul 2021-12-03 15:34:14 +03:00
parent ae134c209f
commit 7570d52386
7 changed files with 50 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/test/regress/expected/multi_colocated_shard_rebalance.out
View File

@ -489,7 +489,7 @@ SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.table2_
-- make sure that we've created the foreign keys
SELECT "Constraint", "Definition" FROM table_fkeys
WHERE "Constraint" LIKE 'table2_group%' OR "Constraint" LIKE 'table1_group%';
WHERE "Constraint" SIMILAR TO 'table2_group%\d' OR "Constraint" SIMILAR TO 'table1_group%\d';
Constraint | Definition
---------------------------------------------------------------------
table2_group1_table1_id_fkey_13000028 | FOREIGN KEY (table1_id) REFERENCES table1_group1_13000022(id)

18
src/test/regress/expected/multi_move_mx.out
View File

@ -2,12 +2,6 @@
-- MULTI_MOVE_MX
--
ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 1550000;
SELECT start_metadata_sync_to_node('localhost', :worker_2_port);
start_metadata_sync_to_node
---------------------------------------------------------------------
(1 row)
-- Create mx test tables
SET citus.shard_count TO 4;
SET citus.shard_replication_factor TO 1;
@ -220,15 +214,3 @@ HINT: Connect to the coordinator and run it again.
DROP TABLE mx_table_1;
DROP TABLE mx_table_2;
DROP TABLE mx_table_3;
SELECT stop_metadata_sync_to_node('localhost', :worker_2_port);
NOTICE: dropping metadata on the node (localhost,57638)
stop_metadata_sync_to_node
---------------------------------------------------------------------
(1 row)
\c - - - :worker_2_port
DELETE FROM pg_dist_node;
DELETE FROM pg_dist_partition;
DELETE FROM pg_dist_shard;
DELETE FROM pg_dist_shard_placement;

57
src/test/regress/expected/shard_rebalancer.out
View File

@ -21,6 +21,7 @@ CREATE TABLE postgres_table_test(a int primary key);
-- reference tables are replicated to the coordinator
SELECT 1 FROM master_add_node('localhost', :master_port, groupId=>0);
NOTICE: Replicating reference table "ref_table_test" to the node localhost:xxxxx
NOTICE: localhost:xxxxx is the coordinator and already contains metadata, skipping syncing the metadata
?column?
---------------------------------------------------------------------
1
@ -766,6 +767,12 @@ SELECT count(*) FROM rebalance_test_table;
DROP TABLE rebalance_test_table;
-- Test schema support
SELECT citus_set_node_property('localhost', :master_port, 'shouldhaveshards', true);
citus_set_node_property
---------------------------------------------------------------------
(1 row)
CREATE SCHEMA test_schema_support;
SELECT COUNT(*) FROM pg_dist_shard_placement;
count
@ -779,9 +786,7 @@ CREATE TABLE test_schema_support.nation_hash (
n_regionkey integer not null,
n_comment varchar(152)
);
SET citus.shard_count TO 4;
SET citus.shard_replication_factor TO 1;
SELECT create_distributed_table('test_schema_support.nation_hash', 'n_nationkey', 'hash');
SELECT create_distributed_table('test_schema_support.nation_hash', 'n_nationkey', shard_count:=4);
create_distributed_table
---------------------------------------------------------------------
@ -793,7 +798,7 @@ CREATE TABLE test_schema_support.nation_hash2 (
n_regionkey integer not null,
n_comment varchar(152)
);
SELECT create_distributed_table('test_schema_support.nation_hash2', 'n_nationkey', 'hash');
SELECT create_distributed_table('test_schema_support.nation_hash2', 'n_nationkey', colocate_with:='None', shard_count:=4);
create_distributed_table
---------------------------------------------------------------------
@ -806,11 +811,11 @@ UPDATE pg_dist_partition SET repmodel='c' WHERE logicalrelid IN
SELECT COUNT(*) FROM pg_dist_shard_placement;
count
---------------------------------------------------------------------
8
16
(1 row)
SET search_path TO public;
SELECT replicate_table_shards('test_schema_support.nation_hash', shard_replication_factor:=2, max_shard_copies:=1, shard_transfer_mode:='block_writes');
SELECT replicate_table_shards('test_schema_support.nation_hash', shard_replication_factor:=3, shard_transfer_mode:='block_writes');
replicate_table_shards
---------------------------------------------------------------------
@ -820,12 +825,12 @@ SELECT replicate_table_shards('test_schema_support.nation_hash', shard_replicati
SELECT COUNT(*) FROM pg_dist_shard_placement;
count
---------------------------------------------------------------------
10
20
(1 row)
-- Test with search_path is set
SET search_path TO test_schema_support;
SELECT replicate_table_shards('nation_hash2', shard_replication_factor:=2, shard_transfer_mode:='block_writes');
SELECT replicate_table_shards('nation_hash2', shard_replication_factor:=3, shard_transfer_mode:='block_writes');
replicate_table_shards
---------------------------------------------------------------------
@ -835,11 +840,17 @@ SELECT replicate_table_shards('nation_hash2', shard_replication_factor:=2, shard
SELECT COUNT(*) FROM pg_dist_shard_placement;
count
---------------------------------------------------------------------
16
24
(1 row)
DROP TABLE test_schema_support.nation_hash;
DROP TABLE test_schema_support.nation_hash2;
SELECT citus_set_node_property('localhost', :master_port, 'shouldhaveshards', false);
citus_set_node_property
---------------------------------------------------------------------
(1 row)
-- Test rebalancer with schema
-- Next few operations is to create imbalanced distributed table
CREATE TABLE test_schema_support.imbalanced_table_local (
@ -1066,14 +1077,14 @@ CALL citus_cleanup_orphaned_shards();
select * from pg_dist_placement ORDER BY placementid;
placementid | shardid | shardstate | shardlength | groupid
---------------------------------------------------------------------
150 | 123023 | 1 | 0 | 14
153 | 123024 | 1 | 0 | 14
156 | 123027 | 1 | 0 | 14
157 | 123028 | 1 | 0 | 14
158 | 123021 | 1 | 0 | 16
159 | 123025 | 1 | 0 | 16
160 | 123022 | 1 | 0 | 16
161 | 123026 | 1 | 0 | 16
158 | 123023 | 1 | 0 | 14
161 | 123024 | 1 | 0 | 14
164 | 123027 | 1 | 0 | 14
165 | 123028 | 1 | 0 | 14
166 | 123021 | 1 | 0 | 16
167 | 123025 | 1 | 0 | 16
168 | 123022 | 1 | 0 | 16
169 | 123026 | 1 | 0 | 16
(8 rows)
-- Move all shards to worker1 again
@ -1439,7 +1450,7 @@ SELECT table_schema, table_name, row_estimate, total_bytes
LEFT JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE relkind = 'r'
) a
WHERE table_schema = 'public'
WHERE table_schema = 'public' AND table_name SIMILAR TO '%\d{2,}'
) a ORDER BY table_name;
table_schema | table_name | row_estimate | total_bytes
---------------------------------------------------------------------
@ -1460,7 +1471,7 @@ SELECT table_schema, table_name, row_estimate, total_bytes
LEFT JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE relkind = 'r'
) a
WHERE table_schema = 'public'
WHERE table_schema = 'public' AND table_name SIMILAR TO '%\d{2,}'
) a ORDER BY table_name;
table_schema | table_name | row_estimate | total_bytes
---------------------------------------------------------------------
@ -1563,7 +1574,7 @@ SELECT table_schema, table_name, row_estimate, total_bytes
LEFT JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE relkind = 'r'
) a
WHERE table_schema = 'public'
WHERE table_schema = 'public' AND table_name SIMILAR TO '%\d{2,}'
) a ORDER BY table_name;
table_schema | table_name | row_estimate | total_bytes
---------------------------------------------------------------------
@ -1584,7 +1595,7 @@ SELECT table_schema, table_name, row_estimate, total_bytes
LEFT JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE relkind = 'r'
) a
WHERE table_schema = 'public'
WHERE table_schema = 'public' AND table_name SIMILAR TO '%\d{2,}'
) a ORDER BY table_name;
table_schema | table_name | row_estimate | total_bytes
---------------------------------------------------------------------
@ -1658,7 +1669,7 @@ SELECT table_schema, table_name, row_estimate, total_bytes
LEFT JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE relkind = 'r'
) a
WHERE table_schema = 'public'
WHERE table_schema = 'public' AND table_name SIMILAR TO '%\d{2,}'
) a ORDER BY table_name;
table_schema | table_name | row_estimate | total_bytes
---------------------------------------------------------------------
@ -1683,7 +1694,7 @@ SELECT table_schema, table_name, row_estimate, total_bytes
LEFT JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE relkind = 'r'
) a
WHERE table_schema = 'public'
WHERE table_schema = 'public' AND table_name SIMILAR TO '%\d{2,}'
) a ORDER BY table_name;
table_schema | table_name | row_estimate | total_bytes
---------------------------------------------------------------------

3
src/test/regress/operations_schedule
View File

@ -2,11 +2,10 @@ test: multi_test_helpers multi_test_helpers_superuser
test: multi_cluster_management
test: multi_test_catalog_views
test: shard_rebalancer_unit
test: turn_mx_off
test: shard_rebalancer
test: turn_mx_on
test: foreign_key_to_reference_shard_rebalance
test: multi_move_mx
test: shard_move_deferred_delete
test: multi_colocated_shard_rebalance
test: ignoring_orphaned_shards
test: check_mx

2
src/test/regress/sql/multi_colocated_shard_rebalance.sql
View File

@ -259,7 +259,7 @@ SELECT "Column", "Type", "Modifiers" FROM table_desc WHERE relid='public.table2_
-- make sure that we've created the foreign keys
SELECT "Constraint", "Definition" FROM table_fkeys
WHERE "Constraint" LIKE 'table2_group%' OR "Constraint" LIKE 'table1_group%';
WHERE "Constraint" SIMILAR TO 'table2_group%\d' OR "Constraint" SIMILAR TO 'table1_group%\d';
\c - - - :master_port

8
src/test/regress/sql/multi_move_mx.sql
View File

@ -3,8 +3,6 @@
--
ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART 1550000;
SELECT start_metadata_sync_to_node('localhost', :worker_2_port);
-- Create mx test tables
SET citus.shard_count TO 4;
SET citus.shard_replication_factor TO 1;
@ -134,9 +132,3 @@ LIMIT 1 OFFSET 1;
DROP TABLE mx_table_1;
DROP TABLE mx_table_2;
DROP TABLE mx_table_3;
SELECT stop_metadata_sync_to_node('localhost', :worker_2_port);
\c - - - :worker_2_port
DELETE FROM pg_dist_node;
DELETE FROM pg_dist_partition;
DELETE FROM pg_dist_shard;
DELETE FROM pg_dist_shard_placement;

25
src/test/regress/sql/shard_rebalancer.sql
View File

@ -538,6 +538,7 @@ DROP TABLE rebalance_test_table;
-- Test schema support
SELECT citus_set_node_property('localhost', :master_port, 'shouldhaveshards', true);
CREATE SCHEMA test_schema_support;
@ -550,9 +551,7 @@ CREATE TABLE test_schema_support.nation_hash (
n_comment varchar(152)
);
SET citus.shard_count TO 4;
SET citus.shard_replication_factor TO 1;
SELECT create_distributed_table('test_schema_support.nation_hash', 'n_nationkey', 'hash');
SELECT create_distributed_table('test_schema_support.nation_hash', 'n_nationkey', shard_count:=4);
CREATE TABLE test_schema_support.nation_hash2 (
n_nationkey integer not null,
@ -561,7 +560,7 @@ CREATE TABLE test_schema_support.nation_hash2 (
n_comment varchar(152)
);
SELECT create_distributed_table('test_schema_support.nation_hash2', 'n_nationkey', 'hash');
SELECT create_distributed_table('test_schema_support.nation_hash2', 'n_nationkey', colocate_with:='None', shard_count:=4);
-- Mark tables as coordinator replicated in order to be able to test replicate_table_shards
UPDATE pg_dist_partition SET repmodel='c' WHERE logicalrelid IN
@ -571,14 +570,14 @@ UPDATE pg_dist_partition SET repmodel='c' WHERE logicalrelid IN
SELECT COUNT(*) FROM pg_dist_shard_placement;
SET search_path TO public;
SELECT replicate_table_shards('test_schema_support.nation_hash', shard_replication_factor:=2, max_shard_copies:=1, shard_transfer_mode:='block_writes');
SELECT replicate_table_shards('test_schema_support.nation_hash', shard_replication_factor:=3, shard_transfer_mode:='block_writes');
-- Confirm replication, both tables replicated due to colocation
SELECT COUNT(*) FROM pg_dist_shard_placement;
-- Test with search_path is set
SET search_path TO test_schema_support;
SELECT replicate_table_shards('nation_hash2', shard_replication_factor:=2, shard_transfer_mode:='block_writes');
SELECT replicate_table_shards('nation_hash2', shard_replication_factor:=3, shard_transfer_mode:='block_writes');
-- Confirm replication
SELECT COUNT(*) FROM pg_dist_shard_placement;
@ -586,6 +585,8 @@ SELECT COUNT(*) FROM pg_dist_shard_placement;
DROP TABLE test_schema_support.nation_hash;
DROP TABLE test_schema_support.nation_hash2;
SELECT citus_set_node_property('localhost', :master_port, 'shouldhaveshards', false);
-- Test rebalancer with schema
-- Next few operations is to create imbalanced distributed table
@ -861,7 +862,7 @@ SELECT table_schema, table_name, row_estimate, total_bytes
LEFT JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE relkind = 'r'
) a
WHERE table_schema = 'public'
WHERE table_schema = 'public' AND table_name SIMILAR TO '%\d{2,}'
) a ORDER BY table_name;
\c - - - :worker_2_port
SELECT table_schema, table_name, row_estimate, total_bytes
@ -876,7 +877,7 @@ SELECT table_schema, table_name, row_estimate, total_bytes
LEFT JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE relkind = 'r'
) a
WHERE table_schema = 'public'
WHERE table_schema = 'public' AND table_name SIMILAR TO '%\d{2,}'
) a ORDER BY table_name;
\c - - - :master_port
@ -922,7 +923,7 @@ SELECT table_schema, table_name, row_estimate, total_bytes
LEFT JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE relkind = 'r'
) a
WHERE table_schema = 'public'
WHERE table_schema = 'public' AND table_name SIMILAR TO '%\d{2,}'
) a ORDER BY table_name;
\c - - - :worker_2_port
SELECT table_schema, table_name, row_estimate, total_bytes
@ -937,7 +938,7 @@ SELECT table_schema, table_name, row_estimate, total_bytes
LEFT JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE relkind = 'r'
) a
WHERE table_schema = 'public'
WHERE table_schema = 'public' AND table_name SIMILAR TO '%\d{2,}'
) a ORDER BY table_name;
\c - - - :master_port
@ -963,7 +964,7 @@ SELECT table_schema, table_name, row_estimate, total_bytes
LEFT JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE relkind = 'r'
) a
WHERE table_schema = 'public'
WHERE table_schema = 'public' AND table_name SIMILAR TO '%\d{2,}'
) a ORDER BY table_name;
\c - - - :worker_2_port
SELECT table_schema, table_name, row_estimate, total_bytes
@ -978,7 +979,7 @@ SELECT table_schema, table_name, row_estimate, total_bytes
LEFT JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE relkind = 'r'
) a
WHERE table_schema = 'public'
WHERE table_schema = 'public' AND table_name SIMILAR TO '%\d{2,}'
) a ORDER BY table_name;
\c - - - :master_port