Merge pull request #5553 from citusdata/fix_metadata_sync_fails_on_multi_sequence_default

Fix metadata sync fails on multi_sequence_default and multi_name_lengths

src/test/regress/expected/multi_name_lengths.out

@@ -219,8 +219,9 @@ NOTICE:  identifier "append_zero_shard_table_12345678901234567890123456789012345
 -- Verify that CREATE INDEX on already distributed table has proper shard names.
 CREATE INDEX tmp_idx_12345678901234567890123456789012345678901234567890 ON name_lengths(col2);
 \c - - :public_worker_1_host :worker_1_port
+SET citus.override_table_visibility TO FALSE;
 SELECT "relname", "Column", "Type", "Definition" FROM index_attrs WHERE
-    relname LIKE 'tmp_idx_%' ORDER BY 1 DESC, 2 DESC, 3 DESC, 4 DESC;
+    relname SIMILAR TO 'tmp_idx_%\_\d{6}' ORDER BY 1 DESC, 2 DESC, 3 DESC, 4 DESC;
                              relname                             | Column |  Type   | Definition
 ---------------------------------------------------------------------
  tmp_idx_123456789012345678901234567890123456789_5e470afa_225003 | col2   | integer | col2
@@ -237,8 +238,9 @@ ALTER INDEX tmp_idx_123456789012345678901234567890123456789012345678901234567890
 NOTICE:  identifier "tmp_idx_123456789012345678901234567890123456789012345678901234567890" will be truncated to "tmp_idx_1234567890123456789012345678901234567890123456789012345"
 NOTICE:  identifier "tmp_idx_newname_123456789012345678901234567890123456789012345678901234567890" will be truncated to "tmp_idx_newname_12345678901234567890123456789012345678901234567"
 \c - - :public_worker_1_host :worker_1_port
+SET citus.override_table_visibility TO FALSE;
 SELECT "relname", "Column", "Type", "Definition" FROM index_attrs WHERE
-    relname LIKE 'tmp_idx_%' ORDER BY 1 DESC, 2 DESC, 3 DESC, 4 DESC;
+    relname SIMILAR TO 'tmp_idx_%\_\d{6}' ORDER BY 1 DESC, 2 DESC, 3 DESC, 4 DESC;
                              relname                             | Column |  Type   | Definition
 ---------------------------------------------------------------------
  tmp_idx_newname_1234567890123456789012345678901_c54e849b_225003 | col2   | integer | col2
@@ -337,6 +339,7 @@ SELECT create_distributed_table('sneaky_name_lengths', 'col1', 'hash');
 (1 row)
 
 \c - - :public_worker_1_host :worker_1_port
+SET citus.override_table_visibility TO FALSE;
 SELECT c1.relname AS unique_index_name
 FROM pg_class c1
     JOIN pg_index i ON i.indexrelid = c1.oid
@@ -369,6 +372,7 @@ SELECT create_distributed_table('too_long_12345678901234567890123456789012345678
 (1 row)
 
 \c - - :public_worker_1_host :worker_1_port
+SET citus.override_table_visibility TO FALSE;
 \dt *225000000000*
                                       List of relations
  Schema |                              Name                               | Type  |  Owner
@@ -401,7 +405,8 @@ WHERE logicalrelid = U&'elephant_!0441!043B!043E!043D!0441!043B!043E!043D!0441!0
 (1 row)
 
 \c - - :public_worker_1_host :worker_1_port
-\dt public.elephant_*
+SET citus.override_table_visibility TO FALSE;
+\dt public.elephant_*[0-9]+
                               List of relations
  Schema |                      Name                       | Type  |  Owner
 ---------------------------------------------------------------------
@@ -409,7 +414,7 @@ WHERE logicalrelid = U&'elephant_!0441!043B!043E!043D!0441!043B!043E!043D!0441!0
  public | elephant_слонслонслонсло_c8b737c2_2250000000003 | table | postgres
 (2 rows)
 
-\di public.elephant_*
+\di public.elephant_*[0-9]+
                                                        List of relations
  Schema |                      Name                       | Type  |  Owner   |                      Table
 ---------------------------------------------------------------------

src/test/regress/expected/multi_sequence_default.out

@@ -10,6 +10,7 @@ CREATE SCHEMA sequence_default;
 SET search_path = sequence_default, public;
 -- test both distributed and citus local tables
 SELECT 1 FROM citus_add_node('localhost', :master_port, groupId => 0);
+NOTICE:  localhost:xxxxx is the coordinator and already contains metadata, skipping syncing the metadata
  ?column?
 ---------------------------------------------------------------------
         1
@@ -64,13 +65,9 @@ ERROR:  cannot add a column involving DEFAULT nextval('..') because the table is
 HINT:  You can first call ALTER TABLE .. ADD COLUMN .. smallint/int/bigint
 Then set the default by ALTER TABLE .. ALTER COLUMN .. SET DEFAULT nextval('..')
 ALTER TABLE seq_test_0 ADD COLUMN z serial;
-ERROR:  Cannot add a column involving serial pseudotypes because the table is not empty
+ERROR:  cannot execute ADD COLUMN commands involving serial pseudotypes when metadata is synchronized to workers
-HINT:  You can first call ALTER TABLE .. ADD COLUMN .. smallint/int/bigint
-Then set the default by ALTER TABLE .. ALTER COLUMN .. SET DEFAULT nextval('..')
 ALTER TABLE seq_test_0_local_table ADD COLUMN z serial;
-ERROR:  Cannot add a column involving serial pseudotypes because the table is not empty
+ERROR:  cannot execute ADD COLUMN commands involving serial pseudotypes when metadata is synchronized to workers
-HINT:  You can first call ALTER TABLE .. ADD COLUMN .. smallint/int/bigint
-Then set the default by ALTER TABLE .. ALTER COLUMN .. SET DEFAULT nextval('..')
 -- follow hint
 ALTER TABLE seq_test_0 ADD COLUMN z int;
 ALTER TABLE seq_test_0 ALTER COLUMN z SET DEFAULT nextval('seq_0');
@@ -127,30 +124,63 @@ SELECT * FROM seq_test_0_local_table ORDER BY 1, 2 LIMIT 5;
 ---------------------------------------------------------------------
  integer |     1 |       1 | 2147483647 |         1 | no      |     1
 
--- cannot change the type of a sequence used in a distributed table
+-- cannot alter a sequence used in a distributed table
--- even if metadata is not synced to workers
+-- since the metadata is synced to workers
 ALTER SEQUENCE seq_0 AS bigint;
-ERROR:  Altering a sequence used in a distributed table is currently not supported.
+ERROR:  Altering a distributed sequence is currently not supported.
 ALTER SEQUENCE seq_0_local_table AS bigint;
-ERROR:  Altering a sequence used in a local table that is added to metadata is currently not supported.
+ERROR:  Altering a distributed sequence is currently not supported.
 -- we can change other things like increment
 -- if metadata is not synced to workers
-ALTER SEQUENCE seq_0 INCREMENT BY 2;
+BEGIN;
-ALTER SEQUENCE seq_0_local_table INCREMENT BY 2;
+SELECT stop_metadata_sync_to_node('localhost', :worker_1_port);
-\d seq_0
+NOTICE:  dropping metadata on the node (localhost,57637)
-                  Sequence "sequence_default.seq_0"
+ stop_metadata_sync_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
+SELECT stop_metadata_sync_to_node('localhost', :worker_2_port);
+NOTICE:  dropping metadata on the node (localhost,57638)
+ stop_metadata_sync_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE SEQUENCE seq_13;
+CREATE SEQUENCE seq_13_local_table;
+CREATE TABLE seq_test_13 (x int, y int);
+CREATE TABLE seq_test_13_local_table (x int, y int);
+SELECT create_distributed_table('seq_test_13','x');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+SELECT citus_add_local_table_to_metadata('seq_test_13_local_table');
+ citus_add_local_table_to_metadata
+---------------------------------------------------------------------
+
+(1 row)
+
+ALTER TABLE seq_test_13 ADD COLUMN z int DEFAULT nextval('seq_13');
+ALTER TABLE seq_test_13_local_table ADD COLUMN z int DEFAULT nextval('seq_13_local_table');
+ALTER SEQUENCE seq_13 INCREMENT BY 2;
+ALTER SEQUENCE seq_13_local_table INCREMENT BY 2;
+\d seq_13
+                  Sequence "sequence_default.seq_13"
   Type   | Start | Minimum |  Maximum   | Increment | Cycles? | Cache
 ---------------------------------------------------------------------
  integer |     1 |       1 | 2147483647 |         2 | no      |     1
 
-\d seq_0_local_table
+\d seq_13_local_table
-            Sequence "sequence_default.seq_0_local_table"
+            Sequence "sequence_default.seq_13_local_table"
   Type   | Start | Minimum |  Maximum   | Increment | Cycles? | Cache
 ---------------------------------------------------------------------
  integer |     1 |       1 | 2147483647 |         2 | no      |     1
 
 -- check that we can add serial pseudo-type columns
--- when metadata is not yet synced to workers
+-- when metadata is not synced to workers
 TRUNCATE seq_test_0;
 ALTER TABLE seq_test_0 ADD COLUMN w00 smallserial;
 ALTER TABLE seq_test_0 ADD COLUMN w01 serial2;
@@ -165,6 +195,7 @@ ALTER TABLE seq_test_0_local_table ADD COLUMN w10 serial;
 ALTER TABLE seq_test_0_local_table ADD COLUMN w11 serial4;
 ALTER TABLE seq_test_0_local_table ADD COLUMN w20 bigserial;
 ALTER TABLE seq_test_0_local_table ADD COLUMN w21 serial8;
+ROLLBACK;
 -- check alter column type precaution
 ALTER TABLE seq_test_0 ALTER COLUMN z TYPE bigint;
 ERROR:  cannot execute ALTER COLUMN TYPE .. command because the column involves a default coming from a sequence
@@ -962,12 +993,6 @@ SELECT run_command_on_workers('DROP SCHEMA IF EXISTS sequence_default CASCADE');
  (localhost,57638,t,"DROP SCHEMA")
 (2 rows)
 
-SELECT stop_metadata_sync_to_node('localhost', :worker_1_port);
- stop_metadata_sync_to_node
----------------------------------------------------------------------
-
-(1 row)
-
 SELECT master_remove_node('localhost', :master_port);
  master_remove_node
 ---------------------------------------------------------------------

src/test/regress/multi_1_schedule

@@ -35,11 +35,8 @@ test: alter_database_owner
 
 test: multi_test_catalog_views
 test: multi_table_ddl
-test: check_mx
-test: turn_mx_off
 test: multi_sequence_default
 test: multi_name_lengths
-test: turn_mx_on
 test: multi_name_resolution
 test: multi_metadata_access
 test: multi_metadata_attributes

src/test/regress/sql/multi_name_lengths.sql

@@ -163,8 +163,9 @@ CREATE INDEX append_zero_shard_table_idx_123456789012345678901234567890123456789
 CREATE INDEX tmp_idx_12345678901234567890123456789012345678901234567890 ON name_lengths(col2);
 
 \c - - :public_worker_1_host :worker_1_port
+SET citus.override_table_visibility TO FALSE;
 SELECT "relname", "Column", "Type", "Definition" FROM index_attrs WHERE
-    relname LIKE 'tmp_idx_%' ORDER BY 1 DESC, 2 DESC, 3 DESC, 4 DESC;
+    relname SIMILAR TO 'tmp_idx_%\_\d{6}' ORDER BY 1 DESC, 2 DESC, 3 DESC, 4 DESC;
 \c - - :master_host :master_port
 
 -- Verify that a new index name > 63 characters is auto-truncated
@@ -175,8 +176,9 @@ CREATE INDEX tmp_idx_12345678901234567890123456789012345678901234567890123456789
 ALTER INDEX tmp_idx_123456789012345678901234567890123456789012345678901234567890 RENAME TO tmp_idx_newname_123456789012345678901234567890123456789012345678901234567890;
 
 \c - - :public_worker_1_host :worker_1_port
+SET citus.override_table_visibility TO FALSE;
 SELECT "relname", "Column", "Type", "Definition" FROM index_attrs WHERE
-    relname LIKE 'tmp_idx_%' ORDER BY 1 DESC, 2 DESC, 3 DESC, 4 DESC;
+    relname SIMILAR TO 'tmp_idx_%\_\d{6}' ORDER BY 1 DESC, 2 DESC, 3 DESC, 4 DESC;
 \c - - :master_host :master_port
 
 SET citus.shard_count TO 2;
@@ -236,6 +238,7 @@ CREATE TABLE sneaky_name_lengths (
 SELECT create_distributed_table('sneaky_name_lengths', 'col1', 'hash');
 
 \c - - :public_worker_1_host :worker_1_port
+SET citus.override_table_visibility TO FALSE;
 
 SELECT c1.relname AS unique_index_name
 FROM pg_class c1
@@ -263,6 +266,7 @@ CREATE TABLE too_long_12345678901234567890123456789012345678901234567890 (
 SELECT create_distributed_table('too_long_12345678901234567890123456789012345678901234567890', 'col1', 'hash');
 
 \c - - :public_worker_1_host :worker_1_port
+SET citus.override_table_visibility TO FALSE;
 \dt *225000000000*
 \c - - :master_host :master_port
 
@@ -283,8 +287,9 @@ FROM pg_dist_shard
 WHERE logicalrelid = U&'elephant_!0441!043B!043E!043D!0441!043B!043E!043D!0441!043B!043E!043D!0441!043B!043E!043D!0441!043B!043E!043D!0441!043B!043E!043D' UESCAPE '!'::regclass;
 
 \c - - :public_worker_1_host :worker_1_port
-\dt public.elephant_*
+SET citus.override_table_visibility TO FALSE;
-\di public.elephant_*
+\dt public.elephant_*[0-9]+
+\di public.elephant_*[0-9]+
 \c - - :master_host :master_port
 
 SET citus.shard_count TO 2;

src/test/regress/sql/multi_sequence_default.sql

@@ -46,20 +46,33 @@ SELECT * FROM seq_test_0_local_table ORDER BY 1, 2 LIMIT 5;
 -- in this case column z is of type int
 \d seq_0
 \d seq_0_local_table
--- cannot change the type of a sequence used in a distributed table
+-- cannot alter a sequence used in a distributed table
--- even if metadata is not synced to workers
+-- since the metadata is synced to workers
 ALTER SEQUENCE seq_0 AS bigint;
 ALTER SEQUENCE seq_0_local_table AS bigint;
+
 -- we can change other things like increment
 -- if metadata is not synced to workers
-ALTER SEQUENCE seq_0 INCREMENT BY 2;
+BEGIN;
-ALTER SEQUENCE seq_0_local_table INCREMENT BY 2;
+SELECT stop_metadata_sync_to_node('localhost', :worker_1_port);
-\d seq_0
+SELECT stop_metadata_sync_to_node('localhost', :worker_2_port);
-\d seq_0_local_table
+CREATE SEQUENCE seq_13;
+CREATE SEQUENCE seq_13_local_table;
+CREATE TABLE seq_test_13 (x int, y int);
+CREATE TABLE seq_test_13_local_table (x int, y int);
+SELECT create_distributed_table('seq_test_13','x');
+SELECT citus_add_local_table_to_metadata('seq_test_13_local_table');
+ALTER TABLE seq_test_13 ADD COLUMN z int DEFAULT nextval('seq_13');
+ALTER TABLE seq_test_13_local_table ADD COLUMN z int DEFAULT nextval('seq_13_local_table');
+
+ALTER SEQUENCE seq_13 INCREMENT BY 2;
+ALTER SEQUENCE seq_13_local_table INCREMENT BY 2;
+\d seq_13
+\d seq_13_local_table
 
 
 -- check that we can add serial pseudo-type columns
--- when metadata is not yet synced to workers
+-- when metadata is not synced to workers
 TRUNCATE seq_test_0;
 ALTER TABLE seq_test_0 ADD COLUMN w00 smallserial;
 ALTER TABLE seq_test_0 ADD COLUMN w01 serial2;
@@ -76,6 +89,8 @@ ALTER TABLE seq_test_0_local_table ADD COLUMN w11 serial4;
 ALTER TABLE seq_test_0_local_table ADD COLUMN w20 bigserial;
 ALTER TABLE seq_test_0_local_table ADD COLUMN w21 serial8;
 
+ROLLBACK;
+
 -- check alter column type precaution
 ALTER TABLE seq_test_0 ALTER COLUMN z TYPE bigint;
 ALTER TABLE seq_test_0 ALTER COLUMN z TYPE smallint;
@@ -468,6 +483,5 @@ DROP TABLE sequence_default.seq_test_7_par;
 SET client_min_messages TO error; -- suppress cascading objects dropping
 DROP SCHEMA sequence_default CASCADE;
 SELECT run_command_on_workers('DROP SCHEMA IF EXISTS sequence_default CASCADE');
-SELECT stop_metadata_sync_to_node('localhost', :worker_1_port);
 SELECT master_remove_node('localhost', :master_port);
 SET search_path TO public;