From d33650d1c12280371f2c0beff373fb1c48ddb75b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96nder=20Kalac=C4=B1?= Date: Mon, 27 Dec 2021 12:33:34 +0100 Subject: [PATCH] Record if any partitioned Citus tables during upgrade (#5555) With Citus 11, the default behavior is to sync the metadata. However, partitioned tables created pre-Citus 11 might have index names that are not compatiable with metadata syncing. See https://github.com/citusdata/citus/issues/4962 for the details. With this commit, we record the existence of partitioned tables such that we can fix it later if any exists. --- .../distributed/sql/citus--10.2-4--11.0-1.sql | 14 ++ src/test/regress/expected/multi_extension.out | 133 +++++++++++------- src/test/regress/sql/multi_extension.sql | 22 +++ 3 files changed, 116 insertions(+), 53 deletions(-) diff --git a/src/backend/distributed/sql/citus--10.2-4--11.0-1.sql b/src/backend/distributed/sql/citus--10.2-4--11.0-1.sql index f4ba74584..1419ce1b2 100644 --- a/src/backend/distributed/sql/citus--10.2-4--11.0-1.sql +++ b/src/backend/distributed/sql/citus--10.2-4--11.0-1.sql @@ -30,3 +30,17 @@ BEGIN END IF; END; $$; + +-- Here we keep track of partitioned tables that exists before Citus 11 +-- where we need to call fix_all_partition_shard_index_names() before +-- metadata is synced. Note that after citus-11, we automatically +-- adjust the indexes so we only need to fix existing indexes +DO LANGUAGE plpgsql +$$ +DECLARE + partitioned_table_exists bool :=false; +BEGIN + SELECT count(*) > 0 INTO partitioned_table_exists FROM pg_dist_partition p JOIN pg_class c ON p.logicalrelid = c.oid WHERE c.relkind = 'p'; + UPDATE pg_dist_node_metadata SET metadata=jsonb_set(metadata, '{partitioned_citus_table_exists_pre_11}', to_jsonb(partitioned_table_exists), true); +END; +$$; diff --git a/src/test/regress/expected/multi_extension.out b/src/test/regress/expected/multi_extension.out index 3643c8c75..ec3b806cc 100644 --- a/src/test/regress/expected/multi_extension.out +++ b/src/test/regress/expected/multi_extension.out @@ -425,20 +425,20 @@ SELECT prosrc FROM pg_proc WHERE proname = 'master_update_table_statistics' ORDE ALTER EXTENSION citus UPDATE TO '9.4-2'; -- should see the old source code SELECT prosrc FROM pg_proc WHERE proname = 'master_update_table_statistics' ORDER BY 1; - prosrc + prosrc --------------------------------------------------------------------- - + - DECLARE + - colocated_tables regclass[]; + - BEGIN + - SELECT get_colocated_table_array(relation) INTO colocated_tables;+ - PERFORM + - master_update_shard_statistics(shardid) + - FROM + - pg_dist_shard + - WHERE + - logicalrelid = ANY (colocated_tables); + - END; + + + + DECLARE + + colocated_tables regclass[]; + + BEGIN + + SELECT get_colocated_table_array(relation) INTO colocated_tables;+ + PERFORM + + master_update_shard_statistics(shardid) + + FROM + + pg_dist_shard + + WHERE + + logicalrelid = ANY (colocated_tables); + + END; + (1 row) @@ -466,20 +466,20 @@ SELECT * FROM multi_extension.print_extension_changes(); ALTER EXTENSION citus UPDATE TO '9.4-1'; -- should see the old source code SELECT prosrc FROM pg_proc WHERE proname = 'master_update_table_statistics' ORDER BY 1; - prosrc + prosrc --------------------------------------------------------------------- - + - DECLARE + - colocated_tables regclass[]; + - BEGIN + - SELECT get_colocated_table_array(relation) INTO colocated_tables;+ - PERFORM + - master_update_shard_statistics(shardid) + - FROM + - pg_dist_shard + - WHERE + - logicalrelid = ANY (colocated_tables); + - END; + + + + DECLARE + + colocated_tables regclass[]; + + BEGIN + + SELECT get_colocated_table_array(relation) INTO colocated_tables;+ + PERFORM + + master_update_shard_statistics(shardid) + + FROM + + pg_dist_shard + + WHERE + + logicalrelid = ANY (colocated_tables); + + END; + (1 row) @@ -573,20 +573,20 @@ SELECT prosrc FROM pg_proc WHERE proname = 'master_update_table_statistics' ORDE ALTER EXTENSION citus UPDATE TO '9.5-2'; -- should see the old source code SELECT prosrc FROM pg_proc WHERE proname = 'master_update_table_statistics' ORDER BY 1; - prosrc + prosrc --------------------------------------------------------------------- - + - DECLARE + - colocated_tables regclass[]; + - BEGIN + - SELECT get_colocated_table_array(relation) INTO colocated_tables;+ - PERFORM + - master_update_shard_statistics(shardid) + - FROM + - pg_dist_shard + - WHERE + - logicalrelid = ANY (colocated_tables); + - END; + + + + DECLARE + + colocated_tables regclass[]; + + BEGIN + + SELECT get_colocated_table_array(relation) INTO colocated_tables;+ + PERFORM + + master_update_shard_statistics(shardid) + + FROM + + pg_dist_shard + + WHERE + + logicalrelid = ANY (colocated_tables); + + END; + (1 row) @@ -614,20 +614,20 @@ SELECT * FROM multi_extension.print_extension_changes(); ALTER EXTENSION citus UPDATE TO '9.5-1'; -- should see the old source code SELECT prosrc FROM pg_proc WHERE proname = 'master_update_table_statistics' ORDER BY 1; - prosrc + prosrc --------------------------------------------------------------------- - + - DECLARE + - colocated_tables regclass[]; + - BEGIN + - SELECT get_colocated_table_array(relation) INTO colocated_tables;+ - PERFORM + - master_update_shard_statistics(shardid) + - FROM + - pg_dist_shard + - WHERE + - logicalrelid = ANY (colocated_tables); + - END; + + + + DECLARE + + colocated_tables regclass[]; + + BEGIN + + SELECT get_colocated_table_array(relation) INTO colocated_tables;+ + PERFORM + + master_update_shard_statistics(shardid) + + FROM + + pg_dist_shard + + WHERE + + logicalrelid = ANY (colocated_tables); + + END; + (1 row) @@ -955,8 +955,35 @@ ERROR: cstore_fdw tables are deprecated as of Citus 11.0 HINT: Install Citus 10.2 and convert your cstore_fdw tables to the columnar access method before upgrading further CONTEXT: PL/pgSQL function inline_code_block line XX at RAISE DELETE FROM pg_dist_shard WHERE shardid = 1; +-- partitioned table count is tracked on Citus 11 upgrade +CREATE TABLE e_transactions(order_id varchar(255) NULL, transaction_id int) PARTITION BY LIST(transaction_id); +CREATE TABLE orders_2020_07_01 +PARTITION OF e_transactions FOR VALUES IN (1,2,3); +INSERT INTO pg_dist_partition VALUES ('e_transactions'::regclass,'h', NULL, 7, 's'); +SELECT + (metadata->>'partitioned_citus_table_exists_pre_11')::boolean as partitioned_citus_table_exists_pre_11, + (metadata->>'partitioned_citus_table_exists_pre_11') IS NULL as is_null +FROM + pg_dist_node_metadata; + partitioned_citus_table_exists_pre_11 | is_null +--------------------------------------------------------------------- + | t +(1 row) + -- Test downgrade to 10.2-4 from 11.0-1 ALTER EXTENSION citus UPDATE TO '11.0-1'; +SELECT + (metadata->>'partitioned_citus_table_exists_pre_11')::boolean as partitioned_citus_table_exists_pre_11, + (metadata->>'partitioned_citus_table_exists_pre_11') IS NULL as is_null +FROM + pg_dist_node_metadata; + partitioned_citus_table_exists_pre_11 | is_null +--------------------------------------------------------------------- + t | f +(1 row) + +DELETE FROM pg_dist_partition WHERE logicalrelid = 'e_transactions'::regclass; +DROP TABLE e_transactions; ALTER EXTENSION citus UPDATE TO '10.2-4'; -- Should be empty result since upgrade+downgrade should be a no-op SELECT * FROM multi_extension.print_extension_changes(); @@ -967,7 +994,7 @@ SELECT * FROM multi_extension.print_extension_changes(); -- Snapshot of state at 11.0-1 ALTER EXTENSION citus UPDATE TO '11.0-1'; SELECT * FROM multi_extension.print_extension_changes(); - previous_object | current_object + previous_object | current_object --------------------------------------------------------------------- function citus_disable_node(text,integer) void | function master_append_table_to_shard(bigint,text,text,integer) real | diff --git a/src/test/regress/sql/multi_extension.sql b/src/test/regress/sql/multi_extension.sql index d237bf5a1..a5150d2c2 100644 --- a/src/test/regress/sql/multi_extension.sql +++ b/src/test/regress/sql/multi_extension.sql @@ -420,8 +420,30 @@ INSERT INTO pg_dist_shard (logicalrelid, shardid, shardstorage) VALUES ('pg_dist ALTER EXTENSION citus UPDATE TO '11.0-1'; DELETE FROM pg_dist_shard WHERE shardid = 1; +-- partitioned table count is tracked on Citus 11 upgrade +CREATE TABLE e_transactions(order_id varchar(255) NULL, transaction_id int) PARTITION BY LIST(transaction_id); +CREATE TABLE orders_2020_07_01 +PARTITION OF e_transactions FOR VALUES IN (1,2,3); +INSERT INTO pg_dist_partition VALUES ('e_transactions'::regclass,'h', NULL, 7, 's'); + +SELECT + (metadata->>'partitioned_citus_table_exists_pre_11')::boolean as partitioned_citus_table_exists_pre_11, + (metadata->>'partitioned_citus_table_exists_pre_11') IS NULL as is_null +FROM + pg_dist_node_metadata; + -- Test downgrade to 10.2-4 from 11.0-1 ALTER EXTENSION citus UPDATE TO '11.0-1'; + +SELECT + (metadata->>'partitioned_citus_table_exists_pre_11')::boolean as partitioned_citus_table_exists_pre_11, + (metadata->>'partitioned_citus_table_exists_pre_11') IS NULL as is_null +FROM + pg_dist_node_metadata; + +DELETE FROM pg_dist_partition WHERE logicalrelid = 'e_transactions'::regclass; +DROP TABLE e_transactions; + ALTER EXTENSION citus UPDATE TO '10.2-4'; -- Should be empty result since upgrade+downgrade should be a no-op SELECT * FROM multi_extension.print_extension_changes();