external
/
citus
mirror of https://github.com/citusdata/citus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add truncating according to interval and minor fixes

velioglu/create_timeseries_table
Burak Velioglu 2021-08-20 01:53:26 +03:00
parent ef724edf31
commit e9bb732b22
No known key found for this signature in database
GPG Key ID: F6827E620F6549C6
2 changed files with 61 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
src/backend/timeseries/sql/udfs/create_missing_partitions/10.2-1.sql
View File

@ -13,19 +13,14 @@ DECLARE
current_partition_count int;
BEGIN
/*
* TODO: Check whether the table is timeseries table with INTERVAL partition range
* and timestamptz partition column
*/
/*
* First check whether such timeseries table exists. If not, error out.
*
* Then check if start_from is given. If it is, create the partition for that time
* and let remaining of the function fill the gap from start_from to to_date.
*
* TODO: That part is implemented by assuming that there is no partition exist for
* the given table, in other words that function will be called via create_timeseries_table
* only.
*
* TODO: Handle date trunc according to given interval of the timeseries table
* only. That will be handled while adding starting time for old data ingestion (Task 1.3)
*/
SELECT partitioninterval
@ -33,11 +28,32 @@ BEGIN
FROM citus_timeseries.citus_timeseries_tables
WHERE logicalrelid = table_name;
IF NOT found THEN
RAISE '% must be timeseries table', table_name;
END IF;
IF start_from IS NOT NULL THEN
RAISE NOTICE 'IN START FROM';
/*
* Decide on the current_range_from_value of the initial partition according to interval of the timeseries table.
* Since we will create all other partitions by adding intervals, truncating given start time will provide
* more intuitive interval ranges.
*/
IF table_partition_interval < INTERVAL '1 hour' THEN
current_range_from_value = date_trunc('minute', start_from);
ELSIF table_partition_interval < INTERVAL '1 day' THEN
current_range_from_value = date_trunc('hour', start_from);
ELSIF table_partition_interval < INTERVAL '1 week' THEN
current_range_from_value = date_trunc('day', start_from);
ELSIF table_partition_interval < INTERVAL '1 month' THEN
current_range_from_value = date_trunc('week', start_from);
ELSIF table_partition_interval < INTERVAL '1 year' THEN
current_range_from_value = date_trunc('month', start_from);
ELSE
current_range_from_value = date_trunc('year', start_from);
END IF;
current_partition_count := 0;
current_range_from_value := start_from;
current_range_to_value := start_from + table_partition_interval;
current_range_to_value := current_range_from_value + table_partition_interval;
current_partition_name := table_name::text || '_' || current_partition_count::text;
EXECUTE format('CREATE TABLE %I PARTITION OF %I FOR VALUES FROM (''%I'') TO (''%I'')', current_partition_name, table_name::text, current_range_from_value::text, current_range_to_value::text);
@ -59,9 +75,7 @@ BEGIN
FROM pg_catalog.time_partitions
WHERE parent_table = table_name;
RAISE NOTICE 'current_range_from_value %', current_range_from_value;
WHILE current_range_from_value < to_date LOOP
WHILE current_range_to_value < to_date LOOP
current_range_from_value := current_range_to_value;
current_range_to_value := current_range_to_value + table_partition_interval;
current_partition_name := table_name::text || '_' || current_partition_count::text;
@ -69,14 +83,13 @@ BEGIN
-- TODO: Create that using attach partition, which can be implemented with another UDF to make sure that it gets light locks.
BEGIN
EXECUTE format('CREATE TABLE %I PARTITION OF %I FOR VALUES FROM (''%I'') TO (''%I'')', current_partition_name, table_name::text, current_range_from_value::text, current_range_to_value::text);
EXCEPTION WHEN OTHERS THEN
raise notice 'Partition has already been created for the range from % to %', current_range_from_value::text, current_range_to_value::text;
END;
current_partition_count := current_partition_count + 1;
EXCEPTION WHEN OTHERS THEN
raise DEBUG3 'Partition has already been created for the range from % to %', current_range_from_value::text, current_range_to_value::text;
END;
END LOOP;
return true;
RETURN true;
END;
$$;
COMMENT ON FUNCTION pg_catalog.create_missing_partitions(

48
src/backend/timeseries/sql/udfs/create_missing_partitions/latest.sql
View File

@ -13,19 +13,14 @@ DECLARE
current_partition_count int;
BEGIN
/*
* TODO: Check whether the table is timeseries table with INTERVAL partition range
* and timestamptz partition column
*/
/*
* First check whether such timeseries table exists. If not, error out.
*
* Then check if start_from is given. If it is, create the partition for that time
* and let remaining of the function fill the gap from start_from to to_date.
*
* TODO: That part is implemented by assuming that there is no partition exist for
* the given table, in other words that function will be called via create_timeseries_table
* only.
*
* TODO: Handle date trunc according to given interval of the timeseries table
* only. That will be handled while adding starting time for old data ingestion (Task 1.3)
*/
SELECT partitioninterval
@ -33,11 +28,32 @@ BEGIN
FROM citus_timeseries.citus_timeseries_tables
WHERE logicalrelid = table_name;
IF NOT found THEN
RAISE '% must be timeseries table', table_name;
END IF;
IF start_from IS NOT NULL THEN
RAISE NOTICE 'IN START FROM';
/*
* Decide on the current_range_from_value of the initial partition according to interval of the timeseries table.
* Since we will create all other partitions by adding intervals, truncating given start time will provide
* more intuitive interval ranges.
*/
IF table_partition_interval < INTERVAL '1 hour' THEN
current_range_from_value = date_trunc('minute', start_from);
ELSIF table_partition_interval < INTERVAL '1 day' THEN
current_range_from_value = date_trunc('hour', start_from);
ELSIF table_partition_interval < INTERVAL '1 week' THEN
current_range_from_value = date_trunc('day', start_from);
ELSIF table_partition_interval < INTERVAL '1 month' THEN
current_range_from_value = date_trunc('week', start_from);
ELSIF table_partition_interval < INTERVAL '1 year' THEN
current_range_from_value = date_trunc('month', start_from);
ELSE
current_range_from_value = date_trunc('year', start_from);
END IF;
current_partition_count := 0;
current_range_from_value := start_from;
current_range_to_value := start_from + table_partition_interval;
current_range_to_value := current_range_from_value + table_partition_interval;
current_partition_name := table_name::text || '_' || current_partition_count::text;
EXECUTE format('CREATE TABLE %I PARTITION OF %I FOR VALUES FROM (''%I'') TO (''%I'')', current_partition_name, table_name::text, current_range_from_value::text, current_range_to_value::text);
@ -52,7 +68,6 @@ BEGIN
INTO current_range_from_value, current_range_to_value
FROM pg_catalog.time_partitions
WHERE parent_table = table_name
ORDER BY from_value::timestamptz;
SELECT count(*)
@ -60,25 +75,22 @@ BEGIN
FROM pg_catalog.time_partitions
WHERE parent_table = table_name;
RAISE NOTICE 'current_range_from_value %', current_range_from_value;
WHILE current_range_from_value < to_date LOOP
WHILE current_range_to_value < to_date LOOP
current_range_from_value := current_range_to_value;
current_range_to_value := current_range_to_value + table_partition_interval;
current_partition_name := table_name::text || '_' || current_partition_count::text;
-- TODO: Create that using attach partition, which can be implemented with another UDF to make sure that it gets light locks.
-- TODO: Add exception handling or partition check
BEGIN
EXECUTE format('CREATE TABLE %I PARTITION OF %I FOR VALUES FROM (''%I'') TO (''%I'')', current_partition_name, table_name::text, current_range_from_value::text, current_range_to_value::text);
EXCEPTION WHEN OTHERS THEN
raise notice 'oops %', sqlstate;
raise DEBUG3 'Partition has already been created for the range from % to %', current_range_from_value::text, current_range_to_value::text;
END;
current_partition_count := current_partition_count + 1;
END LOOP;
return true;
RETURN true;
END;
$$;
COMMENT ON FUNCTION pg_catalog.create_missing_partitions(