From 5c2fb06322866def2ffeba964ac34d202ffc396b Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Tue, 21 Dec 2021 11:45:33 +0300 Subject: [PATCH 1/2] Fix metadata sync fails on multi_sequence_default --- .../expected/multi_sequence_default.out | 71 +++++++++++++------ src/test/regress/multi_1_schedule | 2 +- .../regress/sql/multi_sequence_default.sql | 30 +++++--- 3 files changed, 71 insertions(+), 32 deletions(-) diff --git a/src/test/regress/expected/multi_sequence_default.out b/src/test/regress/expected/multi_sequence_default.out index 903feaeeb..e00310b0e 100644 --- a/src/test/regress/expected/multi_sequence_default.out +++ b/src/test/regress/expected/multi_sequence_default.out @@ -10,6 +10,7 @@ CREATE SCHEMA sequence_default; SET search_path = sequence_default, public; -- test both distributed and citus local tables SELECT 1 FROM citus_add_node('localhost', :master_port, groupId => 0); +NOTICE: localhost:xxxxx is the coordinator and already contains metadata, skipping syncing the metadata ?column? --------------------------------------------------------------------- 1 @@ -64,13 +65,9 @@ ERROR: cannot add a column involving DEFAULT nextval('..') because the table is HINT: You can first call ALTER TABLE .. ADD COLUMN .. smallint/int/bigint Then set the default by ALTER TABLE .. ALTER COLUMN .. SET DEFAULT nextval('..') ALTER TABLE seq_test_0 ADD COLUMN z serial; -ERROR: Cannot add a column involving serial pseudotypes because the table is not empty -HINT: You can first call ALTER TABLE .. ADD COLUMN .. smallint/int/bigint -Then set the default by ALTER TABLE .. ALTER COLUMN .. SET DEFAULT nextval('..') +ERROR: cannot execute ADD COLUMN commands involving serial pseudotypes when metadata is synchronized to workers ALTER TABLE seq_test_0_local_table ADD COLUMN z serial; -ERROR: Cannot add a column involving serial pseudotypes because the table is not empty -HINT: You can first call ALTER TABLE .. ADD COLUMN .. smallint/int/bigint -Then set the default by ALTER TABLE .. ALTER COLUMN .. SET DEFAULT nextval('..') +ERROR: cannot execute ADD COLUMN commands involving serial pseudotypes when metadata is synchronized to workers -- follow hint ALTER TABLE seq_test_0 ADD COLUMN z int; ALTER TABLE seq_test_0 ALTER COLUMN z SET DEFAULT nextval('seq_0'); @@ -127,30 +124,63 @@ SELECT * FROM seq_test_0_local_table ORDER BY 1, 2 LIMIT 5; --------------------------------------------------------------------- integer | 1 | 1 | 2147483647 | 1 | no | 1 --- cannot change the type of a sequence used in a distributed table --- even if metadata is not synced to workers +-- cannot alter a sequence used in a distributed table +-- since the metadata is synced to workers ALTER SEQUENCE seq_0 AS bigint; -ERROR: Altering a sequence used in a distributed table is currently not supported. +ERROR: Altering a distributed sequence is currently not supported. ALTER SEQUENCE seq_0_local_table AS bigint; -ERROR: Altering a sequence used in a local table that is added to metadata is currently not supported. +ERROR: Altering a distributed sequence is currently not supported. -- we can change other things like increment -- if metadata is not synced to workers -ALTER SEQUENCE seq_0 INCREMENT BY 2; -ALTER SEQUENCE seq_0_local_table INCREMENT BY 2; -\d seq_0 - Sequence "sequence_default.seq_0" +BEGIN; +SELECT stop_metadata_sync_to_node('localhost', :worker_1_port); +NOTICE: dropping metadata on the node (localhost,57637) + stop_metadata_sync_to_node +--------------------------------------------------------------------- + +(1 row) + +SELECT stop_metadata_sync_to_node('localhost', :worker_2_port); +NOTICE: dropping metadata on the node (localhost,57638) + stop_metadata_sync_to_node +--------------------------------------------------------------------- + +(1 row) + +CREATE SEQUENCE seq_13; +CREATE SEQUENCE seq_13_local_table; +CREATE TABLE seq_test_13 (x int, y int); +CREATE TABLE seq_test_13_local_table (x int, y int); +SELECT create_distributed_table('seq_test_13','x'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +SELECT citus_add_local_table_to_metadata('seq_test_13_local_table'); + citus_add_local_table_to_metadata +--------------------------------------------------------------------- + +(1 row) + +ALTER TABLE seq_test_13 ADD COLUMN z int DEFAULT nextval('seq_13'); +ALTER TABLE seq_test_13_local_table ADD COLUMN z int DEFAULT nextval('seq_13_local_table'); +ALTER SEQUENCE seq_13 INCREMENT BY 2; +ALTER SEQUENCE seq_13_local_table INCREMENT BY 2; +\d seq_13 + Sequence "sequence_default.seq_13" Type | Start | Minimum | Maximum | Increment | Cycles? | Cache --------------------------------------------------------------------- integer | 1 | 1 | 2147483647 | 2 | no | 1 -\d seq_0_local_table - Sequence "sequence_default.seq_0_local_table" +\d seq_13_local_table + Sequence "sequence_default.seq_13_local_table" Type | Start | Minimum | Maximum | Increment | Cycles? | Cache --------------------------------------------------------------------- integer | 1 | 1 | 2147483647 | 2 | no | 1 -- check that we can add serial pseudo-type columns --- when metadata is not yet synced to workers +-- when metadata is not synced to workers TRUNCATE seq_test_0; ALTER TABLE seq_test_0 ADD COLUMN w00 smallserial; ALTER TABLE seq_test_0 ADD COLUMN w01 serial2; @@ -165,6 +195,7 @@ ALTER TABLE seq_test_0_local_table ADD COLUMN w10 serial; ALTER TABLE seq_test_0_local_table ADD COLUMN w11 serial4; ALTER TABLE seq_test_0_local_table ADD COLUMN w20 bigserial; ALTER TABLE seq_test_0_local_table ADD COLUMN w21 serial8; +ROLLBACK; -- check alter column type precaution ALTER TABLE seq_test_0 ALTER COLUMN z TYPE bigint; ERROR: cannot execute ALTER COLUMN TYPE .. command because the column involves a default coming from a sequence @@ -962,12 +993,6 @@ SELECT run_command_on_workers('DROP SCHEMA IF EXISTS sequence_default CASCADE'); (localhost,57638,t,"DROP SCHEMA") (2 rows) -SELECT stop_metadata_sync_to_node('localhost', :worker_1_port); - stop_metadata_sync_to_node ---------------------------------------------------------------------- - -(1 row) - SELECT master_remove_node('localhost', :master_port); master_remove_node --------------------------------------------------------------------- diff --git a/src/test/regress/multi_1_schedule b/src/test/regress/multi_1_schedule index 25b3db268..6f181b608 100644 --- a/src/test/regress/multi_1_schedule +++ b/src/test/regress/multi_1_schedule @@ -35,9 +35,9 @@ test: alter_database_owner test: multi_test_catalog_views test: multi_table_ddl +test: multi_sequence_default test: check_mx test: turn_mx_off -test: multi_sequence_default test: multi_name_lengths test: turn_mx_on test: multi_name_resolution diff --git a/src/test/regress/sql/multi_sequence_default.sql b/src/test/regress/sql/multi_sequence_default.sql index 4b0ac1ca3..67663accb 100644 --- a/src/test/regress/sql/multi_sequence_default.sql +++ b/src/test/regress/sql/multi_sequence_default.sql @@ -46,20 +46,33 @@ SELECT * FROM seq_test_0_local_table ORDER BY 1, 2 LIMIT 5; -- in this case column z is of type int \d seq_0 \d seq_0_local_table --- cannot change the type of a sequence used in a distributed table --- even if metadata is not synced to workers +-- cannot alter a sequence used in a distributed table +-- since the metadata is synced to workers ALTER SEQUENCE seq_0 AS bigint; ALTER SEQUENCE seq_0_local_table AS bigint; + -- we can change other things like increment -- if metadata is not synced to workers -ALTER SEQUENCE seq_0 INCREMENT BY 2; -ALTER SEQUENCE seq_0_local_table INCREMENT BY 2; -\d seq_0 -\d seq_0_local_table +BEGIN; +SELECT stop_metadata_sync_to_node('localhost', :worker_1_port); +SELECT stop_metadata_sync_to_node('localhost', :worker_2_port); +CREATE SEQUENCE seq_13; +CREATE SEQUENCE seq_13_local_table; +CREATE TABLE seq_test_13 (x int, y int); +CREATE TABLE seq_test_13_local_table (x int, y int); +SELECT create_distributed_table('seq_test_13','x'); +SELECT citus_add_local_table_to_metadata('seq_test_13_local_table'); +ALTER TABLE seq_test_13 ADD COLUMN z int DEFAULT nextval('seq_13'); +ALTER TABLE seq_test_13_local_table ADD COLUMN z int DEFAULT nextval('seq_13_local_table'); + +ALTER SEQUENCE seq_13 INCREMENT BY 2; +ALTER SEQUENCE seq_13_local_table INCREMENT BY 2; +\d seq_13 +\d seq_13_local_table -- check that we can add serial pseudo-type columns --- when metadata is not yet synced to workers +-- when metadata is not synced to workers TRUNCATE seq_test_0; ALTER TABLE seq_test_0 ADD COLUMN w00 smallserial; ALTER TABLE seq_test_0 ADD COLUMN w01 serial2; @@ -76,6 +89,8 @@ ALTER TABLE seq_test_0_local_table ADD COLUMN w11 serial4; ALTER TABLE seq_test_0_local_table ADD COLUMN w20 bigserial; ALTER TABLE seq_test_0_local_table ADD COLUMN w21 serial8; +ROLLBACK; + -- check alter column type precaution ALTER TABLE seq_test_0 ALTER COLUMN z TYPE bigint; ALTER TABLE seq_test_0 ALTER COLUMN z TYPE smallint; @@ -468,6 +483,5 @@ DROP TABLE sequence_default.seq_test_7_par; SET client_min_messages TO error; -- suppress cascading objects dropping DROP SCHEMA sequence_default CASCADE; SELECT run_command_on_workers('DROP SCHEMA IF EXISTS sequence_default CASCADE'); -SELECT stop_metadata_sync_to_node('localhost', :worker_1_port); SELECT master_remove_node('localhost', :master_port); SET search_path TO public; From 70e68d5312e1af0abcd5211a5bcfe1c530acd3f8 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Tue, 21 Dec 2021 14:19:10 +0300 Subject: [PATCH 2/2] Fix metadata sync fails on multi_name_lengths --- src/test/regress/expected/multi_name_lengths.out | 13 +++++++++---- src/test/regress/multi_1_schedule | 3 --- src/test/regress/sql/multi_name_lengths.sql | 13 +++++++++---- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/test/regress/expected/multi_name_lengths.out b/src/test/regress/expected/multi_name_lengths.out index 5ef151042..e100ba7d8 100644 --- a/src/test/regress/expected/multi_name_lengths.out +++ b/src/test/regress/expected/multi_name_lengths.out @@ -219,8 +219,9 @@ NOTICE: identifier "append_zero_shard_table_12345678901234567890123456789012345 -- Verify that CREATE INDEX on already distributed table has proper shard names. CREATE INDEX tmp_idx_12345678901234567890123456789012345678901234567890 ON name_lengths(col2); \c - - :public_worker_1_host :worker_1_port +SET citus.override_table_visibility TO FALSE; SELECT "relname", "Column", "Type", "Definition" FROM index_attrs WHERE - relname LIKE 'tmp_idx_%' ORDER BY 1 DESC, 2 DESC, 3 DESC, 4 DESC; + relname SIMILAR TO 'tmp_idx_%\_\d{6}' ORDER BY 1 DESC, 2 DESC, 3 DESC, 4 DESC; relname | Column | Type | Definition --------------------------------------------------------------------- tmp_idx_123456789012345678901234567890123456789_5e470afa_225003 | col2 | integer | col2 @@ -237,8 +238,9 @@ ALTER INDEX tmp_idx_123456789012345678901234567890123456789012345678901234567890 NOTICE: identifier "tmp_idx_123456789012345678901234567890123456789012345678901234567890" will be truncated to "tmp_idx_1234567890123456789012345678901234567890123456789012345" NOTICE: identifier "tmp_idx_newname_123456789012345678901234567890123456789012345678901234567890" will be truncated to "tmp_idx_newname_12345678901234567890123456789012345678901234567" \c - - :public_worker_1_host :worker_1_port +SET citus.override_table_visibility TO FALSE; SELECT "relname", "Column", "Type", "Definition" FROM index_attrs WHERE - relname LIKE 'tmp_idx_%' ORDER BY 1 DESC, 2 DESC, 3 DESC, 4 DESC; + relname SIMILAR TO 'tmp_idx_%\_\d{6}' ORDER BY 1 DESC, 2 DESC, 3 DESC, 4 DESC; relname | Column | Type | Definition --------------------------------------------------------------------- tmp_idx_newname_1234567890123456789012345678901_c54e849b_225003 | col2 | integer | col2 @@ -337,6 +339,7 @@ SELECT create_distributed_table('sneaky_name_lengths', 'col1', 'hash'); (1 row) \c - - :public_worker_1_host :worker_1_port +SET citus.override_table_visibility TO FALSE; SELECT c1.relname AS unique_index_name FROM pg_class c1 JOIN pg_index i ON i.indexrelid = c1.oid @@ -369,6 +372,7 @@ SELECT create_distributed_table('too_long_12345678901234567890123456789012345678 (1 row) \c - - :public_worker_1_host :worker_1_port +SET citus.override_table_visibility TO FALSE; \dt *225000000000* List of relations Schema | Name | Type | Owner @@ -401,7 +405,8 @@ WHERE logicalrelid = U&'elephant_!0441!043B!043E!043D!0441!043B!043E!043D!0441!0 (1 row) \c - - :public_worker_1_host :worker_1_port -\dt public.elephant_* +SET citus.override_table_visibility TO FALSE; +\dt public.elephant_*[0-9]+ List of relations Schema | Name | Type | Owner --------------------------------------------------------------------- @@ -409,7 +414,7 @@ WHERE logicalrelid = U&'elephant_!0441!043B!043E!043D!0441!043B!043E!043D!0441!0 public | elephant_слонслонслонсло_c8b737c2_2250000000003 | table | postgres (2 rows) -\di public.elephant_* +\di public.elephant_*[0-9]+ List of relations Schema | Name | Type | Owner | Table --------------------------------------------------------------------- diff --git a/src/test/regress/multi_1_schedule b/src/test/regress/multi_1_schedule index 6f181b608..1f00ca06d 100644 --- a/src/test/regress/multi_1_schedule +++ b/src/test/regress/multi_1_schedule @@ -36,10 +36,7 @@ test: alter_database_owner test: multi_test_catalog_views test: multi_table_ddl test: multi_sequence_default -test: check_mx -test: turn_mx_off test: multi_name_lengths -test: turn_mx_on test: multi_name_resolution test: multi_metadata_access test: multi_metadata_attributes diff --git a/src/test/regress/sql/multi_name_lengths.sql b/src/test/regress/sql/multi_name_lengths.sql index 3328572d4..df0a68f33 100644 --- a/src/test/regress/sql/multi_name_lengths.sql +++ b/src/test/regress/sql/multi_name_lengths.sql @@ -163,8 +163,9 @@ CREATE INDEX append_zero_shard_table_idx_123456789012345678901234567890123456789 CREATE INDEX tmp_idx_12345678901234567890123456789012345678901234567890 ON name_lengths(col2); \c - - :public_worker_1_host :worker_1_port +SET citus.override_table_visibility TO FALSE; SELECT "relname", "Column", "Type", "Definition" FROM index_attrs WHERE - relname LIKE 'tmp_idx_%' ORDER BY 1 DESC, 2 DESC, 3 DESC, 4 DESC; + relname SIMILAR TO 'tmp_idx_%\_\d{6}' ORDER BY 1 DESC, 2 DESC, 3 DESC, 4 DESC; \c - - :master_host :master_port -- Verify that a new index name > 63 characters is auto-truncated @@ -175,8 +176,9 @@ CREATE INDEX tmp_idx_12345678901234567890123456789012345678901234567890123456789 ALTER INDEX tmp_idx_123456789012345678901234567890123456789012345678901234567890 RENAME TO tmp_idx_newname_123456789012345678901234567890123456789012345678901234567890; \c - - :public_worker_1_host :worker_1_port +SET citus.override_table_visibility TO FALSE; SELECT "relname", "Column", "Type", "Definition" FROM index_attrs WHERE - relname LIKE 'tmp_idx_%' ORDER BY 1 DESC, 2 DESC, 3 DESC, 4 DESC; + relname SIMILAR TO 'tmp_idx_%\_\d{6}' ORDER BY 1 DESC, 2 DESC, 3 DESC, 4 DESC; \c - - :master_host :master_port SET citus.shard_count TO 2; @@ -236,6 +238,7 @@ CREATE TABLE sneaky_name_lengths ( SELECT create_distributed_table('sneaky_name_lengths', 'col1', 'hash'); \c - - :public_worker_1_host :worker_1_port +SET citus.override_table_visibility TO FALSE; SELECT c1.relname AS unique_index_name FROM pg_class c1 @@ -263,6 +266,7 @@ CREATE TABLE too_long_12345678901234567890123456789012345678901234567890 ( SELECT create_distributed_table('too_long_12345678901234567890123456789012345678901234567890', 'col1', 'hash'); \c - - :public_worker_1_host :worker_1_port +SET citus.override_table_visibility TO FALSE; \dt *225000000000* \c - - :master_host :master_port @@ -283,8 +287,9 @@ FROM pg_dist_shard WHERE logicalrelid = U&'elephant_!0441!043B!043E!043D!0441!043B!043E!043D!0441!043B!043E!043D!0441!043B!043E!043D!0441!043B!043E!043D!0441!043B!043E!043D' UESCAPE '!'::regclass; \c - - :public_worker_1_host :worker_1_port -\dt public.elephant_* -\di public.elephant_* +SET citus.override_table_visibility TO FALSE; +\dt public.elephant_*[0-9]+ +\di public.elephant_*[0-9]+ \c - - :master_host :master_port SET citus.shard_count TO 2;