From b729a2b519343be713612cd9e361128ccbe7e58d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Halil=20Ozan=20Akg=C3=BCl?= Date: Fri, 14 Jul 2023 17:19:45 +0300 Subject: [PATCH] Add 11.3-2 backporting changes (#7062) This PR moves `citus_shard_sizes` changes from #7003, and #7018 to a new Citus version 11.3-2 This PR backports the changes to 12.0 release branch, there is another PR, #7051 for 11.3 release branch, and one, #7050, for main branch. --- .../distributed/sql/citus--11.3-1--11.3-2.sql | 9 +++ ...--12.0-1.sql => citus--11.3-2--12.0-1.sql} | 7 +-- .../sql/downgrades/citus--11.3-2--11.3-1.sql | 13 +++++ ...--11.3-1.sql => citus--12.0-1--11.3-2.sql} | 10 +--- .../{12.0-1.sql => 11.3-2.sql} | 0 .../sql/udfs/citus_shards/11.3-2.sql | 46 ++++++++++++++++ .../sql/udfs/citus_tables/11.3-2.sql | 55 +++++++++++++++++++ src/test/regress/expected/multi_extension.out | 24 ++++++-- src/test/regress/sql/multi_extension.sql | 18 ++++-- 9 files changed, 160 insertions(+), 22 deletions(-) create mode 100644 src/backend/distributed/sql/citus--11.3-1--11.3-2.sql rename src/backend/distributed/sql/{citus--11.3-1--12.0-1.sql => citus--11.3-2--12.0-1.sql} (91%) create mode 100644 src/backend/distributed/sql/downgrades/citus--11.3-2--11.3-1.sql rename src/backend/distributed/sql/downgrades/{citus--12.0-1--11.3-1.sql => citus--12.0-1--11.3-2.sql} (85%) rename src/backend/distributed/sql/udfs/citus_shard_sizes/{12.0-1.sql => 11.3-2.sql} (100%) create mode 100644 src/backend/distributed/sql/udfs/citus_shards/11.3-2.sql create mode 100644 src/backend/distributed/sql/udfs/citus_tables/11.3-2.sql diff --git a/src/backend/distributed/sql/citus--11.3-1--11.3-2.sql b/src/backend/distributed/sql/citus--11.3-1--11.3-2.sql new file mode 100644 index 000000000..e7f0864f1 --- /dev/null +++ b/src/backend/distributed/sql/citus--11.3-1--11.3-2.sql @@ -0,0 +1,9 @@ +DROP VIEW citus_shards; +DROP VIEW IF EXISTS pg_catalog.citus_tables; +DROP VIEW IF EXISTS public.citus_tables; +DROP FUNCTION citus_shard_sizes; + +#include "udfs/citus_shard_sizes/11.3-2.sql" + +#include "udfs/citus_shards/11.3-2.sql" +#include "udfs/citus_tables/11.3-2.sql" diff --git a/src/backend/distributed/sql/citus--11.3-1--12.0-1.sql b/src/backend/distributed/sql/citus--11.3-2--12.0-1.sql similarity index 91% rename from src/backend/distributed/sql/citus--11.3-1--12.0-1.sql rename to src/backend/distributed/sql/citus--11.3-2--12.0-1.sql index fac95dbd4..ea0de6534 100644 --- a/src/backend/distributed/sql/citus--11.3-1--12.0-1.sql +++ b/src/backend/distributed/sql/citus--11.3-2--12.0-1.sql @@ -24,13 +24,8 @@ GRANT SELECT ON pg_catalog.pg_dist_schema TO public; #include "udfs/citus_internal_unregister_tenant_schema_globally/12.0-1.sql" #include "udfs/citus_drop_trigger/12.0-1.sql" -DROP VIEW citus_shards; -DROP VIEW IF EXISTS pg_catalog.citus_tables; -DROP VIEW IF EXISTS public.citus_tables; -DROP FUNCTION citus_shard_sizes; -#include "udfs/citus_shard_sizes/12.0-1.sql" - #include "udfs/citus_tables/12.0-1.sql" +DROP VIEW citus_shards; #include "udfs/citus_shards/12.0-1.sql" #include "udfs/citus_schemas/12.0-1.sql" diff --git a/src/backend/distributed/sql/downgrades/citus--11.3-2--11.3-1.sql b/src/backend/distributed/sql/downgrades/citus--11.3-2--11.3-1.sql new file mode 100644 index 000000000..78dacd59a --- /dev/null +++ b/src/backend/distributed/sql/downgrades/citus--11.3-2--11.3-1.sql @@ -0,0 +1,13 @@ +DROP VIEW IF EXISTS public.citus_tables; +DROP VIEW IF EXISTS pg_catalog.citus_tables; + +DROP VIEW pg_catalog.citus_shards; +DROP FUNCTION pg_catalog.citus_shard_sizes; +#include "../udfs/citus_shard_sizes/10.0-1.sql" +-- citus_shards/11.1-1.sql tries to create citus_shards in pg_catalog but it is not allowed. +-- Here we use citus_shards/10.0-1.sql to properly create the view in citus schema and +-- then alter it to pg_catalog, so citus_shards/11.1-1.sql can REPLACE it without any errors. +#include "../udfs/citus_shards/10.0-1.sql" + +#include "../udfs/citus_tables/11.1-1.sql" +#include "../udfs/citus_shards/11.1-1.sql" diff --git a/src/backend/distributed/sql/downgrades/citus--12.0-1--11.3-1.sql b/src/backend/distributed/sql/downgrades/citus--12.0-1--11.3-2.sql similarity index 85% rename from src/backend/distributed/sql/downgrades/citus--12.0-1--11.3-1.sql rename to src/backend/distributed/sql/downgrades/citus--12.0-1--11.3-2.sql index c391837f4..b410a4069 100644 --- a/src/backend/distributed/sql/downgrades/citus--12.0-1--11.3-1.sql +++ b/src/backend/distributed/sql/downgrades/citus--12.0-1--11.3-2.sql @@ -51,15 +51,9 @@ DROP VIEW IF EXISTS public.citus_tables; DROP VIEW IF EXISTS pg_catalog.citus_tables; DROP VIEW pg_catalog.citus_shards; -DROP FUNCTION pg_catalog.citus_shard_sizes; -#include "../udfs/citus_shard_sizes/10.0-1.sql" --- citus_shards/11.1-1.sql tries to create citus_shards in pg_catalog but it is not allowed. --- Here we use citus_shards/10.0-1.sql to properly create the view in citus schema and --- then alter it to pg_catalog, so citus_shards/11.1-1.sql can REPLACE it without any errors. -#include "../udfs/citus_shards/10.0-1.sql" -#include "../udfs/citus_tables/11.1-1.sql" -#include "../udfs/citus_shards/11.1-1.sql" +#include "../udfs/citus_tables/11.3-2.sql" +#include "../udfs/citus_shards/11.3-2.sql" DROP TABLE pg_catalog.pg_dist_schema; diff --git a/src/backend/distributed/sql/udfs/citus_shard_sizes/12.0-1.sql b/src/backend/distributed/sql/udfs/citus_shard_sizes/11.3-2.sql similarity index 100% rename from src/backend/distributed/sql/udfs/citus_shard_sizes/12.0-1.sql rename to src/backend/distributed/sql/udfs/citus_shard_sizes/11.3-2.sql diff --git a/src/backend/distributed/sql/udfs/citus_shards/11.3-2.sql b/src/backend/distributed/sql/udfs/citus_shards/11.3-2.sql new file mode 100644 index 000000000..3b08a5463 --- /dev/null +++ b/src/backend/distributed/sql/udfs/citus_shards/11.3-2.sql @@ -0,0 +1,46 @@ +CREATE OR REPLACE VIEW citus.citus_shards AS +SELECT + pg_dist_shard.logicalrelid AS table_name, + pg_dist_shard.shardid, + shard_name(pg_dist_shard.logicalrelid, pg_dist_shard.shardid) as shard_name, + CASE WHEN partkey IS NOT NULL THEN 'distributed' WHEN repmodel = 't' THEN 'reference' ELSE 'local' END AS citus_table_type, + colocationid AS colocation_id, + pg_dist_node.nodename, + pg_dist_node.nodeport, + size as shard_size +FROM + pg_dist_shard +JOIN + pg_dist_placement +ON + pg_dist_shard.shardid = pg_dist_placement.shardid +JOIN + pg_dist_node +ON + pg_dist_placement.groupid = pg_dist_node.groupid +JOIN + pg_dist_partition +ON + pg_dist_partition.logicalrelid = pg_dist_shard.logicalrelid +LEFT JOIN + (SELECT shard_id, max(size) as size from citus_shard_sizes() GROUP BY shard_id) as shard_sizes +ON + pg_dist_shard.shardid = shard_sizes.shard_id +WHERE + pg_dist_placement.shardstate = 1 +AND + -- filter out tables owned by extensions + pg_dist_partition.logicalrelid NOT IN ( + SELECT + objid + FROM + pg_depend + WHERE + classid = 'pg_class'::regclass AND refclassid = 'pg_extension'::regclass AND deptype = 'e' + ) +ORDER BY + pg_dist_shard.logicalrelid::text, shardid +; + +ALTER VIEW citus.citus_shards SET SCHEMA pg_catalog; +GRANT SELECT ON pg_catalog.citus_shards TO public; diff --git a/src/backend/distributed/sql/udfs/citus_tables/11.3-2.sql b/src/backend/distributed/sql/udfs/citus_tables/11.3-2.sql new file mode 100644 index 000000000..ead0b4923 --- /dev/null +++ b/src/backend/distributed/sql/udfs/citus_tables/11.3-2.sql @@ -0,0 +1,55 @@ +DO $$ +declare +citus_tables_create_query text; +BEGIN +citus_tables_create_query=$CTCQ$ + CREATE OR REPLACE VIEW %I.citus_tables AS + SELECT + logicalrelid AS table_name, + CASE WHEN partkey IS NOT NULL THEN 'distributed' ELSE + CASE when repmodel = 't' THEN 'reference' ELSE 'local' END + END AS citus_table_type, + coalesce(column_to_column_name(logicalrelid, partkey), '') AS distribution_column, + colocationid AS colocation_id, + pg_size_pretty(table_sizes.table_size) AS table_size, + (select count(*) from pg_dist_shard where logicalrelid = p.logicalrelid) AS shard_count, + pg_get_userbyid(relowner) AS table_owner, + amname AS access_method + FROM + pg_dist_partition p + JOIN + pg_class c ON (p.logicalrelid = c.oid) + LEFT JOIN + pg_am a ON (a.oid = c.relam) + JOIN + ( + SELECT ds.logicalrelid AS table_id, SUM(css.size) AS table_size + FROM citus_shard_sizes() css, pg_dist_shard ds + WHERE css.shard_id = ds.shardid + GROUP BY ds.logicalrelid + ) table_sizes ON (table_sizes.table_id = p.logicalrelid) + WHERE + -- filter out tables owned by extensions + logicalrelid NOT IN ( + SELECT + objid + FROM + pg_depend + WHERE + classid = 'pg_class'::regclass AND refclassid = 'pg_extension'::regclass AND deptype = 'e' + ) + ORDER BY + logicalrelid::text; +$CTCQ$; + +IF EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = 'public') THEN + EXECUTE format(citus_tables_create_query, 'public'); + GRANT SELECT ON public.citus_tables TO public; +ELSE + EXECUTE format(citus_tables_create_query, 'citus'); + ALTER VIEW citus.citus_tables SET SCHEMA pg_catalog; + GRANT SELECT ON pg_catalog.citus_tables TO public; +END IF; + +END; +$$; diff --git a/src/test/regress/expected/multi_extension.out b/src/test/regress/expected/multi_extension.out index b1b2046a2..35d9aa26a 100644 --- a/src/test/regress/expected/multi_extension.out +++ b/src/test/regress/expected/multi_extension.out @@ -1331,7 +1331,23 @@ SELECT * FROM multi_extension.print_extension_changes(); | view citus_stat_tenants_local (11 rows) --- Test downgrade to 11.3-1 from 12.0-1 +-- Test downgrade to 11.3-1 from 11.3-2 +ALTER EXTENSION citus UPDATE TO '11.3-2'; +ALTER EXTENSION citus UPDATE TO '11.3-1'; +-- Should be empty result since upgrade+downgrade should be a no-op +SELECT * FROM multi_extension.print_extension_changes(); + previous_object | current_object +--------------------------------------------------------------------- +(0 rows) + +-- Snapshot of state at 11.3-2 +ALTER EXTENSION citus UPDATE TO '11.3-2'; +SELECT * FROM multi_extension.print_extension_changes(); + previous_object | current_object +--------------------------------------------------------------------- +(0 rows) + +-- Test downgrade to 11.3-2 from 12.0-1 ALTER EXTENSION citus UPDATE TO '12.0-1'; CREATE TABLE null_shard_key (x int, y int); SET citus.shard_replication_factor TO 1; @@ -1341,15 +1357,15 @@ SELECT create_distributed_table('null_shard_key', null); (1 row) --- Show that we cannot downgrade to 11.3-1 becuase the cluster has a +-- Show that we cannot downgrade to 11.3-2 becuase the cluster has a -- distributed table with single-shard. -ALTER EXTENSION citus UPDATE TO '11.3-1'; +ALTER EXTENSION citus UPDATE TO '11.3-2'; ERROR: cannot downgrade Citus because there are distributed tables without a shard key. DETAIL: To downgrade Citus to an older version, you should first convert those tables to Postgres tables by executing SELECT undistribute_table("%s"). HINT: You can find the distributed tables without a shard key in the cluster by using the following query: "SELECT * FROM citus_tables WHERE distribution_column = '' AND colocation_id > 0". CONTEXT: PL/pgSQL function inline_code_block line XX at RAISE DROP TABLE null_shard_key; -ALTER EXTENSION citus UPDATE TO '11.3-1'; +ALTER EXTENSION citus UPDATE TO '11.3-2'; -- Should be empty result since upgrade+downgrade should be a no-op SELECT * FROM multi_extension.print_extension_changes(); previous_object | current_object diff --git a/src/test/regress/sql/multi_extension.sql b/src/test/regress/sql/multi_extension.sql index 72d939867..85847006b 100644 --- a/src/test/regress/sql/multi_extension.sql +++ b/src/test/regress/sql/multi_extension.sql @@ -591,20 +591,30 @@ SELECT * FROM multi_extension.print_extension_changes(); ALTER EXTENSION citus UPDATE TO '11.3-1'; SELECT * FROM multi_extension.print_extension_changes(); --- Test downgrade to 11.3-1 from 12.0-1 +-- Test downgrade to 11.3-1 from 11.3-2 +ALTER EXTENSION citus UPDATE TO '11.3-2'; +ALTER EXTENSION citus UPDATE TO '11.3-1'; +-- Should be empty result since upgrade+downgrade should be a no-op +SELECT * FROM multi_extension.print_extension_changes(); + +-- Snapshot of state at 11.3-2 +ALTER EXTENSION citus UPDATE TO '11.3-2'; +SELECT * FROM multi_extension.print_extension_changes(); + +-- Test downgrade to 11.3-2 from 12.0-1 ALTER EXTENSION citus UPDATE TO '12.0-1'; CREATE TABLE null_shard_key (x int, y int); SET citus.shard_replication_factor TO 1; SELECT create_distributed_table('null_shard_key', null); --- Show that we cannot downgrade to 11.3-1 becuase the cluster has a +-- Show that we cannot downgrade to 11.3-2 becuase the cluster has a -- distributed table with single-shard. -ALTER EXTENSION citus UPDATE TO '11.3-1'; +ALTER EXTENSION citus UPDATE TO '11.3-2'; DROP TABLE null_shard_key; -ALTER EXTENSION citus UPDATE TO '11.3-1'; +ALTER EXTENSION citus UPDATE TO '11.3-2'; -- Should be empty result since upgrade+downgrade should be a no-op SELECT * FROM multi_extension.print_extension_changes();