From e7f13978b58dede496acb6f002448c28b36893e5 Mon Sep 17 00:00:00 2001 From: Marco Slot Date: Thu, 17 Dec 2020 18:34:34 +0100 Subject: [PATCH] Add a view for simple (time) partitions and their access methods --- .../distributed/operations/partitioning.c | 134 ++++++++++++++++++ .../distributed/sql/citus--9.5-1--10.0-1.sql | 3 + .../sql/downgrades/citus--10.0-1--9.5-1.sql | 3 + .../sql/udfs/time_partition_range/10.0-1.sql | 10 ++ .../sql/udfs/time_partition_range/latest.sql | 10 ++ .../sql/udfs/time_partitions/10.0-1.sql | 18 +++ .../sql/udfs/time_partitions/latest.sql | 18 +++ src/test/regress/expected/multi_extension.out | 4 +- .../regress/expected/multi_extension_0.out | 4 +- .../regress/expected/multi_partitioning.out | 63 +++++++- .../expected/upgrade_list_citus_objects.out | 4 +- .../expected/upgrade_list_citus_objects_0.out | 4 +- src/test/regress/sql/multi_partitioning.sql | 27 +++- 13 files changed, 295 insertions(+), 7 deletions(-) create mode 100644 src/backend/distributed/operations/partitioning.c create mode 100644 src/backend/distributed/sql/udfs/time_partition_range/10.0-1.sql create mode 100644 src/backend/distributed/sql/udfs/time_partition_range/latest.sql create mode 100644 src/backend/distributed/sql/udfs/time_partitions/10.0-1.sql create mode 100644 src/backend/distributed/sql/udfs/time_partitions/latest.sql diff --git a/src/backend/distributed/operations/partitioning.c b/src/backend/distributed/operations/partitioning.c new file mode 100644 index 000000000..c7d989d5a --- /dev/null +++ b/src/backend/distributed/operations/partitioning.c @@ -0,0 +1,134 @@ +/*------------------------------------------------------------------------- + * + * partitioning.c + * Functions for dealing with partitioned tables. + * + * Copyright (c) Microsoft Corporation. All rights reserved. + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" +#include "fmgr.h" +#include "funcapi.h" + +#include "access/htup.h" +#include "access/htup_details.h" +#include "distributed/metadata_cache.h" +#include "distributed/metadata_utility.h" +#include "nodes/parsenodes.h" +#include "nodes/pg_list.h" +#include "utils/builtins.h" +#include "utils/elog.h" +#include "utils/lsyscache.h" +#include "utils/syscache.h" + + +/* exports for SQL callable functions */ +PG_FUNCTION_INFO_V1(time_partition_range); + + +/* + * time_partition_range returns the lower and upper bound of partition + * key values for the partition of a time-partitioned table. + */ +Datum +time_partition_range(PG_FUNCTION_ARGS) +{ + Oid relationId = PG_GETARG_OID(0); + + CheckCitusVersion(ERROR); + + /* create tuple descriptor for return value */ + TupleDesc metadataDescriptor = NULL; + TypeFuncClass resultTypeClass = get_call_result_type(fcinfo, NULL, + &metadataDescriptor); + if (resultTypeClass != TYPEFUNC_COMPOSITE) + { + ereport(ERROR, (errmsg("return type must be a row type"))); + } + + /* get the pg_class record */ + HeapTuple tuple = SearchSysCache1(RELOID, relationId); + if (!HeapTupleIsValid(tuple)) + { + ereport(ERROR, (errmsg("relation with OID %u does not exist", relationId))); + } + + /* get the pg_class record */ + bool isNull = false; + Datum partitionBoundDatum = SysCacheGetAttr(RELOID, tuple, Anum_pg_class_relpartbound, + &isNull); + if (isNull) + { + ereport(ERROR, (errmsg("relation \"%s\" is not a partition", + get_rel_name(relationId)))); + } + + PartitionBoundSpec *partitionBoundSpec = + (PartitionBoundSpec *) stringToNode(TextDatumGetCString(partitionBoundDatum)); + + if (!IsA(partitionBoundSpec, PartitionBoundSpec)) + { + ereport(ERROR, (errmsg("expected PartitionBoundSpec"))); + } + + if (partitionBoundSpec->strategy != PARTITION_STRATEGY_RANGE) + { + ereport(ERROR, (errmsg("relation \"%s\" is not a range partition", + get_rel_name(relationId)), + errdetail("time_partition_range can only be used for " + "partitions of range-partitioned tables with a single " + "partition column"))); + } + + Datum values[2]; + bool isNulls[2]; + + memset(values, 0, sizeof(values)); + memset(isNulls, false, sizeof(isNulls)); + + if (partitionBoundSpec->is_default) + { + /* return NULL for default partition */ + isNulls[0] = true; + isNulls[1] = true; + } + else + { + if (list_length(partitionBoundSpec->lowerdatums) != 1 || + list_length(partitionBoundSpec->upperdatums) != 1) + { + ereport(ERROR, (errmsg("relation \"%s\" is a partition with multiple " + "partition columns", + get_rel_name(relationId)), + errdetail("time_partition_range can only be used for " + "partitions of range-partitioned tables with a " + "single partition column"))); + } + + PartitionRangeDatum *lowerBoundDatum = + castNode(PartitionRangeDatum, linitial(partitionBoundSpec->lowerdatums)); + PartitionRangeDatum *upperBoundDatum = + castNode(PartitionRangeDatum, linitial(partitionBoundSpec->upperdatums)); + + Const *lowerConst = castNode(Const, lowerBoundDatum->value); + Const *upperConst = castNode(Const, upperBoundDatum->value); + + char *lowerConstStr = DatumToString(lowerConst->constvalue, + lowerConst->consttype); + + char *upperConstStr = DatumToString(upperConst->constvalue, + upperConst->consttype); + + values[0] = CStringGetTextDatum(lowerConstStr); + values[1] = CStringGetTextDatum(upperConstStr); + } + + HeapTuple metadataTuple = heap_form_tuple(metadataDescriptor, values, isNulls); + Datum metadataDatum = HeapTupleGetDatum(metadataTuple); + + ReleaseSysCache(tuple); + + PG_RETURN_DATUM(metadataDatum); +} diff --git a/src/backend/distributed/sql/citus--9.5-1--10.0-1.sql b/src/backend/distributed/sql/citus--9.5-1--10.0-1.sql index d805453d1..be5d7d866 100644 --- a/src/backend/distributed/sql/citus--9.5-1--10.0-1.sql +++ b/src/backend/distributed/sql/citus--9.5-1--10.0-1.sql @@ -10,3 +10,6 @@ DROP FUNCTION IF EXISTS pg_catalog.citus_total_relation_size(regclass); #include "udfs/create_citus_local_table/10.0-1.sql" #include "../../columnar/sql/columnar--9.5-1--10.0-1.sql" + +#include "udfs/time_partition_range/10.0-1.sql" +#include "udfs/time_partitions/10.0-1.sql" diff --git a/src/backend/distributed/sql/downgrades/citus--10.0-1--9.5-1.sql b/src/backend/distributed/sql/downgrades/citus--10.0-1--9.5-1.sql index 601a208ac..6274422bc 100644 --- a/src/backend/distributed/sql/downgrades/citus--10.0-1--9.5-1.sql +++ b/src/backend/distributed/sql/downgrades/citus--10.0-1--9.5-1.sql @@ -10,6 +10,9 @@ DROP FUNCTION pg_catalog.citus_total_relation_size(regclass,boolean); DROP FUNCTION pg_catalog.undistribute_table(regclass,boolean); DROP FUNCTION pg_catalog.create_citus_local_table(regclass,boolean); +DROP VIEW pg_catalog.time_partitions; +DROP FUNCTION pg_catalog.time_partition_range(regclass); + #include "../udfs/citus_total_relation_size/7.0-1.sql" #include "../udfs/upgrade_to_reference_table/8.0-1.sql" #include "../udfs/undistribute_table/9.5-1.sql" diff --git a/src/backend/distributed/sql/udfs/time_partition_range/10.0-1.sql b/src/backend/distributed/sql/udfs/time_partition_range/10.0-1.sql new file mode 100644 index 000000000..0271cc95d --- /dev/null +++ b/src/backend/distributed/sql/udfs/time_partition_range/10.0-1.sql @@ -0,0 +1,10 @@ +CREATE OR REPLACE FUNCTION pg_catalog.time_partition_range( + table_name regclass, + OUT lower_bound text, + OUT upper_bound text) +RETURNS record +LANGUAGE C STRICT +AS 'MODULE_PATHNAME', $$time_partition_range$$; + +COMMENT ON FUNCTION pg_catalog.time_partition_range(regclass) +IS 'returns the start and end of partition boundaries'; diff --git a/src/backend/distributed/sql/udfs/time_partition_range/latest.sql b/src/backend/distributed/sql/udfs/time_partition_range/latest.sql new file mode 100644 index 000000000..0271cc95d --- /dev/null +++ b/src/backend/distributed/sql/udfs/time_partition_range/latest.sql @@ -0,0 +1,10 @@ +CREATE OR REPLACE FUNCTION pg_catalog.time_partition_range( + table_name regclass, + OUT lower_bound text, + OUT upper_bound text) +RETURNS record +LANGUAGE C STRICT +AS 'MODULE_PATHNAME', $$time_partition_range$$; + +COMMENT ON FUNCTION pg_catalog.time_partition_range(regclass) +IS 'returns the start and end of partition boundaries'; diff --git a/src/backend/distributed/sql/udfs/time_partitions/10.0-1.sql b/src/backend/distributed/sql/udfs/time_partitions/10.0-1.sql new file mode 100644 index 000000000..3f2e1995e --- /dev/null +++ b/src/backend/distributed/sql/udfs/time_partitions/10.0-1.sql @@ -0,0 +1,18 @@ +CREATE VIEW citus.time_partitions AS +SELECT partrelid AS parent_table, attname AS partition_column, relid AS partition, lower_bound AS from_value, upper_bound AS to_value, amname AS access_method +FROM ( + SELECT partrelid::regclass AS partrelid, attname, c.oid::regclass AS relid, lower_bound, upper_bound, amname + FROM pg_class c + JOIN pg_inherits i ON (c.oid = inhrelid) + JOIN pg_partitioned_table p ON (inhparent = partrelid) + JOIN pg_attribute a ON (partrelid = attrelid AND ARRAY[attnum] <@ string_to_array(partattrs::text, ' ')::int2[]) + JOIN pg_type t ON (atttypid = t.oid) + JOIN pg_namespace tn ON (t.typnamespace = tn.oid) + LEFT JOIN pg_am am ON (c.relam = am.oid), + pg_catalog.time_partition_range(c.oid) + WHERE c.relpartbound IS NOT NULL AND p.partstrat = 'r' AND p.partnatts = 1 +) partitions +ORDER BY partrelid::text, lower_bound; + +ALTER VIEW citus.time_partitions SET SCHEMA pg_catalog; +GRANT SELECT ON pg_catalog.time_partitions TO public; diff --git a/src/backend/distributed/sql/udfs/time_partitions/latest.sql b/src/backend/distributed/sql/udfs/time_partitions/latest.sql new file mode 100644 index 000000000..3f2e1995e --- /dev/null +++ b/src/backend/distributed/sql/udfs/time_partitions/latest.sql @@ -0,0 +1,18 @@ +CREATE VIEW citus.time_partitions AS +SELECT partrelid AS parent_table, attname AS partition_column, relid AS partition, lower_bound AS from_value, upper_bound AS to_value, amname AS access_method +FROM ( + SELECT partrelid::regclass AS partrelid, attname, c.oid::regclass AS relid, lower_bound, upper_bound, amname + FROM pg_class c + JOIN pg_inherits i ON (c.oid = inhrelid) + JOIN pg_partitioned_table p ON (inhparent = partrelid) + JOIN pg_attribute a ON (partrelid = attrelid AND ARRAY[attnum] <@ string_to_array(partattrs::text, ' ')::int2[]) + JOIN pg_type t ON (atttypid = t.oid) + JOIN pg_namespace tn ON (t.typnamespace = tn.oid) + LEFT JOIN pg_am am ON (c.relam = am.oid), + pg_catalog.time_partition_range(c.oid) + WHERE c.relpartbound IS NOT NULL AND p.partstrat = 'r' AND p.partnatts = 1 +) partitions +ORDER BY partrelid::text, lower_bound; + +ALTER VIEW citus.time_partitions SET SCHEMA pg_catalog; +GRANT SELECT ON pg_catalog.time_partitions TO public; diff --git a/src/test/regress/expected/multi_extension.out b/src/test/regress/expected/multi_extension.out index 3624039ce..491e249b4 100644 --- a/src/test/regress/expected/multi_extension.out +++ b/src/test/regress/expected/multi_extension.out @@ -456,6 +456,7 @@ SELECT * FROM print_extension_changes(); | function citus_total_relation_size(regclass,boolean) | function columnar.columnar_handler(internal) | function create_citus_local_table(regclass,boolean) + | function time_partition_range(regclass) | function undistribute_table(regclass,boolean) | schema columnar | sequence columnar.storageid_seq @@ -463,7 +464,8 @@ SELECT * FROM print_extension_changes(); | table columnar.columnar_stripes | table columnar.options | view citus_tables -(18 rows) + | view time_partitions +(20 rows) DROP TABLE prev_objects, extension_diff; -- show running version diff --git a/src/test/regress/expected/multi_extension_0.out b/src/test/regress/expected/multi_extension_0.out index dfd4d61c8..e50a7fe6b 100644 --- a/src/test/regress/expected/multi_extension_0.out +++ b/src/test/regress/expected/multi_extension_0.out @@ -452,6 +452,7 @@ SELECT * FROM print_extension_changes(); | function citus_internal.columnar_ensure_objects_exist() | function citus_total_relation_size(regclass,boolean) | function create_citus_local_table(regclass,boolean) + | function time_partition_range(regclass) | function undistribute_table(regclass,boolean) | schema columnar | sequence columnar.storageid_seq @@ -459,7 +460,8 @@ SELECT * FROM print_extension_changes(); | table columnar.columnar_stripes | table columnar.options | view citus_tables -(14 rows) + | view time_partitions +(16 rows) DROP TABLE prev_objects, extension_diff; -- show running version diff --git a/src/test/regress/expected/multi_partitioning.out b/src/test/regress/expected/multi_partitioning.out index 0c148a188..8873e64d8 100644 --- a/src/test/regress/expected/multi_partitioning.out +++ b/src/test/regress/expected/multi_partitioning.out @@ -1947,6 +1947,17 @@ ALTER TABLE partitioning_test ATTACH PARTITION partitioning_test_2010 -- Attach a table which has a different constraint ALTER TABLE partitioning_test ATTACH PARTITION partitioning_test_2011 FOR VALUES FROM ('2011-01-01') TO ('2012-01-01'); +SELECT parent_table, partition_column, partition, from_value, to_value FROM time_partitions; + parent_table | partition_column | partition | from_value | to_value +--------------------------------------------------------------------- + "schema-test" | time | "schema-test_2009" | 01-01-2009 | 01-01-2010 + partitioning_test | time | partitioning_test_2008 | 01-01-2008 | 01-01-2009 + partitioning_test | time | partitioning_test_2009 | 01-01-2009 | 01-01-2010 + partitioning_test | time | partitioning_test_2010 | 01-01-2010 | 01-01-2011 + partitioning_test | time | partitioning_test_2011 | 01-01-2011 | 01-01-2012 + public.non_distributed_partitioned_table | a | public.non_distributed_partitioned_table_1 | 0 | 10 +(6 rows) + ALTER TABLE partitioning_test DETACH PARTITION partitioning_test_2008; ALTER TABLE partitioning_test DETACH PARTITION partitioning_test_2009; ALTER TABLE partitioning_test DETACH PARTITION partitioning_test_2010; @@ -1954,9 +1965,57 @@ ALTER TABLE partitioning_test DETACH PARTITION partitioning_test_2011; DROP TABLE partitioning_test, partitioning_test_2008, partitioning_test_2009, partitioning_test_2010, partitioning_test_2011, reference_table, reference_table_2; -DROP SCHEMA partitioning_schema CASCADE; -NOTICE: drop cascades to table "schema-test" RESET SEARCH_PATH; +-- not timestamp partitioned +CREATE TABLE not_time_partitioned (x int, y int) PARTITION BY RANGE (x); +CREATE TABLE not_time_partitioned_p0 PARTITION OF not_time_partitioned DEFAULT; +CREATE TABLE not_time_partitioned_p1 PARTITION OF not_time_partitioned FOR VALUES FROM (1) TO (2); +SELECT parent_table, partition_column, partition, from_value, to_value FROM time_partitions; + parent_table | partition_column | partition | from_value | to_value +--------------------------------------------------------------------- + non_distributed_partitioned_table | a | non_distributed_partitioned_table_1 | 0 | 10 + not_time_partitioned | x | not_time_partitioned_p1 | 1 | 2 + not_time_partitioned | x | not_time_partitioned_p0 | | + partitioning_schema."schema-test" | time | partitioning_schema."schema-test_2009" | 01-01-2009 | 01-01-2010 +(4 rows) + +SELECT * FROM time_partition_range('not_time_partitioned_p1'); + lower_bound | upper_bound +--------------------------------------------------------------------- + 1 | 2 +(1 row) + +DROP TABLE not_time_partitioned; +-- multi-column partitioned +CREATE TABLE multi_column_partitioned (x date, y date) PARTITION BY RANGE (x, y); +CREATE TABLE multi_column_partitioned_p1 PARTITION OF multi_column_partitioned FOR VALUES FROM ('2020-01-01', '2020-01-01') TO ('2020-12-31','2020-12-31'); +SELECT parent_table, partition_column, partition, from_value, to_value FROM time_partitions; + parent_table | partition_column | partition | from_value | to_value +--------------------------------------------------------------------- + non_distributed_partitioned_table | a | non_distributed_partitioned_table_1 | 0 | 10 + partitioning_schema."schema-test" | time | partitioning_schema."schema-test_2009" | 01-01-2009 | 01-01-2010 +(2 rows) + +SELECT * FROM time_partition_range('multi_column_partitioned_p1'); +ERROR: relation "multi_column_partitioned_p1" is a partition with multiple partition columns +DETAIL: time_partition_range can only be used for partitions of range-partitioned tables with a single partition column +DROP TABLE multi_column_partitioned; +-- not-range-partitioned +CREATE TABLE list_partitioned (x date, y date) PARTITION BY LIST (x); +CREATE TABLE list_partitioned_p1 PARTITION OF list_partitioned FOR VALUES IN ('2020-01-01'); +SELECT parent_table, partition_column, partition, from_value, to_value FROM time_partitions; + parent_table | partition_column | partition | from_value | to_value +--------------------------------------------------------------------- + non_distributed_partitioned_table | a | non_distributed_partitioned_table_1 | 0 | 10 + partitioning_schema."schema-test" | time | partitioning_schema."schema-test_2009" | 01-01-2009 | 01-01-2010 +(2 rows) + +SELECT * FROM time_partition_range('list_partitioned_p1'); +ERROR: relation "list_partitioned_p1" is not a range partition +DETAIL: time_partition_range can only be used for partitions of range-partitioned tables with a single partition column +DROP TABLE list_partitioned; +DROP SCHEMA partitioning_schema CASCADE; +NOTICE: drop cascades to table partitioning_schema."schema-test" DROP TABLE IF EXISTS partitioning_hash_test, partitioning_hash_join_test, diff --git a/src/test/regress/expected/upgrade_list_citus_objects.out b/src/test/regress/expected/upgrade_list_citus_objects.out index 3ac1cd586..ef407ae35 100644 --- a/src/test/regress/expected/upgrade_list_citus_objects.out +++ b/src/test/regress/expected/upgrade_list_citus_objects.out @@ -152,6 +152,7 @@ ORDER BY 1; function shard_name(regclass,bigint) function start_metadata_sync_to_node(text,integer) function stop_metadata_sync_to_node(text,integer) + function time_partition_range(regclass) function truncate_local_data_after_distributing_table(regclass) function undistribute_table(regclass,boolean) function update_distributed_table_colocation(regclass,text) @@ -217,5 +218,6 @@ ORDER BY 1; view citus_tables view citus_worker_stat_activity view pg_dist_shard_placement -(201 rows) + view time_partitions +(203 rows) diff --git a/src/test/regress/expected/upgrade_list_citus_objects_0.out b/src/test/regress/expected/upgrade_list_citus_objects_0.out index f50d5acb1..2c89e4025 100644 --- a/src/test/regress/expected/upgrade_list_citus_objects_0.out +++ b/src/test/regress/expected/upgrade_list_citus_objects_0.out @@ -148,6 +148,7 @@ ORDER BY 1; function shard_name(regclass,bigint) function start_metadata_sync_to_node(text,integer) function stop_metadata_sync_to_node(text,integer) + function time_partition_range(regclass) function truncate_local_data_after_distributing_table(regclass) function undistribute_table(regclass,boolean) function update_distributed_table_colocation(regclass,text) @@ -213,5 +214,6 @@ ORDER BY 1; view citus_tables view citus_worker_stat_activity view pg_dist_shard_placement -(197 rows) + view time_partitions +(199 rows) diff --git a/src/test/regress/sql/multi_partitioning.sql b/src/test/regress/sql/multi_partitioning.sql index c369da5d6..57b69a6a4 100644 --- a/src/test/regress/sql/multi_partitioning.sql +++ b/src/test/regress/sql/multi_partitioning.sql @@ -1153,6 +1153,8 @@ ALTER TABLE partitioning_test ATTACH PARTITION partitioning_test_2010 ALTER TABLE partitioning_test ATTACH PARTITION partitioning_test_2011 FOR VALUES FROM ('2011-01-01') TO ('2012-01-01'); +SELECT parent_table, partition_column, partition, from_value, to_value FROM time_partitions; + ALTER TABLE partitioning_test DETACH PARTITION partitioning_test_2008; ALTER TABLE partitioning_test DETACH PARTITION partitioning_test_2009; ALTER TABLE partitioning_test DETACH PARTITION partitioning_test_2010; @@ -1162,8 +1164,31 @@ DROP TABLE partitioning_test, partitioning_test_2008, partitioning_test_2009, partitioning_test_2010, partitioning_test_2011, reference_table, reference_table_2; -DROP SCHEMA partitioning_schema CASCADE; RESET SEARCH_PATH; + +-- not timestamp partitioned +CREATE TABLE not_time_partitioned (x int, y int) PARTITION BY RANGE (x); +CREATE TABLE not_time_partitioned_p0 PARTITION OF not_time_partitioned DEFAULT; +CREATE TABLE not_time_partitioned_p1 PARTITION OF not_time_partitioned FOR VALUES FROM (1) TO (2); +SELECT parent_table, partition_column, partition, from_value, to_value FROM time_partitions; +SELECT * FROM time_partition_range('not_time_partitioned_p1'); +DROP TABLE not_time_partitioned; + +-- multi-column partitioned +CREATE TABLE multi_column_partitioned (x date, y date) PARTITION BY RANGE (x, y); +CREATE TABLE multi_column_partitioned_p1 PARTITION OF multi_column_partitioned FOR VALUES FROM ('2020-01-01', '2020-01-01') TO ('2020-12-31','2020-12-31'); +SELECT parent_table, partition_column, partition, from_value, to_value FROM time_partitions; +SELECT * FROM time_partition_range('multi_column_partitioned_p1'); +DROP TABLE multi_column_partitioned; + +-- not-range-partitioned +CREATE TABLE list_partitioned (x date, y date) PARTITION BY LIST (x); +CREATE TABLE list_partitioned_p1 PARTITION OF list_partitioned FOR VALUES IN ('2020-01-01'); +SELECT parent_table, partition_column, partition, from_value, to_value FROM time_partitions; +SELECT * FROM time_partition_range('list_partitioned_p1'); +DROP TABLE list_partitioned; + +DROP SCHEMA partitioning_schema CASCADE; DROP TABLE IF EXISTS partitioning_hash_test, partitioning_hash_join_test,