external
/
citus
mirror of https://github.com/citusdata/citus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update version as 11.1-1 and makefile for testing purpose

pull/5911/head
Yanwen Jin 2022-05-17 16:56:16 -07:00
parent 9e7f22251f
commit 99016251e8
11 changed files with 101 additions and 329 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/backend/columnar/Makefile
View File

@ -10,10 +10,11 @@ OBJS += \
MODULE_big = citus_columnar
EXTENSION = citus_columnar
DATA = sql/columnar--9.5-1--10.0-1.sql
template_sql_files = $(patsubst $(citus_abs_srcdir)/%,%,$(wildcard $(citus_abs_srcdir)/sql/*.sql))
#DATA = $(wildcard sql/*--*.sql)
# $(wildcard sql/downgrades/*--*.sql)
DATA = $(template_sql_files)
PG_CPPFLAGS += -I$(libpq_srcdir) -I$(safestringlib_srcdir)/include

2
src/backend/columnar/citus_columnar.control
View File

@ -1,6 +1,6 @@
# Columnar extension
comment = 'Citus Columnar extension'
default_version = '11.2-1'
default_version = '11.1-1'
module_pathname = '$libdir/citus_columnar'
relocatable = false
schema = pg_catalog

23
src/backend/columnar/columnar_tableam.c
View File

@ -58,7 +58,6 @@
#include "columnar/columnar_tableam.h"
#include "columnar/columnar_version_compat.h"
#include "distributed/listutils.h"
#include "distributed/deparser.h"
/*
* Timing parameters for truncate locking heuristics.
@ -2159,7 +2158,7 @@ ColumnarProcessUtility(PlannedStmt *pstmt,
if (get_extension_oid("citus", true) != InvalidOid)
{
ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("must upgrade citus to version 11.2-1 first")));
errmsg("must upgrade citus to version 11.1-1 first")));
}
}
}
@ -2970,3 +2969,23 @@ InstalledExtensionVersionColumnar(void)
return installedExtensionVersion;
}
/*
* GetExtensionOption returns DefElem * node with "defname" from "options" list
*/
DefElem *
GetExtensionOption(List *extensionOptions, const char *defname)
{
DefElem *defElement = NULL;
foreach_ptr(defElement, extensionOptions)
{
if (IsA(defElement, DefElem) &&
strncmp(defElement->defname, defname, NAMEDATALEN) == 0)
{
return defElement;
}
}
return NULL;
}

23
src/backend/columnar/sql/citus_columnar--11.1-0--11.2-1.sql
View File

@ -1,23 +0,0 @@
-- add columnar objects back
ALTER EXTENSION citus_columnar ADD SCHEMA columnar;
ALTER EXTENSION citus_columnar ADD SEQUENCE columnar.storageid_seq;
ALTER EXTENSION citus_columnar ADD TABLE columnar.options;
ALTER EXTENSION citus_columnar ADD TABLE columnar.stripe;
ALTER EXTENSION citus_columnar ADD TABLE columnar.chunk_group;
ALTER EXTENSION citus_columnar ADD TABLE columnar.chunk;
DO $proc$
BEGIN
-- columnar functions
IF substring(current_Setting('server_version'), '\d+')::int >= 12 THEN
EXECUTE $$
ALTER EXTENSION citus_columnar ADD FUNCTION columnar.columnar_handler;
ALTER EXTENSION citus_columnar ADD ACCESS METHOD columnar;
ALTER EXTENSION citus_columnar ADD FUNCTION pg_catalog.alter_columnar_table_set;
ALTER EXTENSION citus_columnar ADD FUNCTION pg_catalog.alter_columnar_table_reset;
$$;
END IF;
END$proc$;
ALTER EXTENSION citus_columnar ADD FUNCTION citus_internal.upgrade_columnar_storage;
ALTER EXTENSION citus_columnar ADD FUNCTION citus_internal.downgrade_columnar_storage;
ALTER EXTENSION citus_columnar ADD FUNCTION citus_internal.columnar_ensure_am_depends_catalog;

266
src/backend/columnar/sql/citus_columnar--11.2-1.sql
View File

@ -1,266 +0,0 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "CREATE EXTENSION citus_columnar" to load this file. \quit
-- columnar--9.5-1--10.0-1.sql
CREATE SCHEMA IF NOT EXISTS columnar;
SET search_path TO columnar;
CREATE SEQUENCE IF NOT EXISTS storageid_seq MINVALUE 10000000000 NO CYCLE;
CREATE TABLE IF NOT EXISTS options (
regclass regclass NOT NULL PRIMARY KEY,
chunk_group_row_limit int NOT NULL,
stripe_row_limit int NOT NULL,
compression_level int NOT NULL,
compression name NOT NULL
) WITH (user_catalog_table = true);
COMMENT ON TABLE options IS 'columnar table specific options, maintained by alter_columnar_table_set';
CREATE TABLE IF NOT EXISTS stripe (
storage_id bigint NOT NULL,
stripe_num bigint NOT NULL,
file_offset bigint NOT NULL,
data_length bigint NOT NULL,
column_count int NOT NULL,
chunk_row_count int NOT NULL,
row_count bigint NOT NULL,
chunk_group_count int NOT NULL,
first_row_number bigint NOT NULL,
PRIMARY KEY (storage_id, stripe_num),
CONSTRAINT stripe_first_row_number_idx UNIQUE (storage_id, first_row_number)
) WITH (user_catalog_table = true);
COMMENT ON TABLE stripe IS 'Columnar per stripe metadata';
CREATE TABLE IF NOT EXISTS chunk_group (
storage_id bigint NOT NULL,
stripe_num bigint NOT NULL,
chunk_group_num int NOT NULL,
row_count bigint NOT NULL,
PRIMARY KEY (storage_id, stripe_num, chunk_group_num)
);
COMMENT ON TABLE chunk_group IS 'Columnar chunk group metadata';
CREATE TABLE IF NOT EXISTS chunk (
storage_id bigint NOT NULL,
stripe_num bigint NOT NULL,
attr_num int NOT NULL,
chunk_group_num int NOT NULL,
minimum_value bytea,
maximum_value bytea,
value_stream_offset bigint NOT NULL,
value_stream_length bigint NOT NULL,
exists_stream_offset bigint NOT NULL,
exists_stream_length bigint NOT NULL,
value_compression_type int NOT NULL,
value_compression_level int NOT NULL,
value_decompressed_length bigint NOT NULL,
value_count bigint NOT NULL,
PRIMARY KEY (storage_id, stripe_num, attr_num, chunk_group_num)
) WITH (user_catalog_table = true);
COMMENT ON TABLE chunk IS 'Columnar per chunk metadata';
DO $proc$
BEGIN
-- from version 12 and up we have support for tableam's if installed on pg11 we can't
-- create the objects here. Instead we rely on citus_finish_pg_upgrade to be called by the
-- user instead to add the missing objects
IF substring(current_Setting('server_version'), '\d+')::int >= 12 THEN
EXECUTE $$
--#include "udfs/columnar_handler/10.0-1.sql"
CREATE OR REPLACE FUNCTION columnar.columnar_handler(internal)
RETURNS table_am_handler
LANGUAGE C
AS 'MODULE_PATHNAME', 'columnar_handler';
COMMENT ON FUNCTION columnar.columnar_handler(internal)
IS 'internal function returning the handler for columnar tables';
-- postgres 11.8 does not support the syntax for table am, also it is seemingly trying
-- to parse the upgrade file and erroring on unknown syntax.
-- normally this section would not execute on postgres 11 anyway. To trick it to pass on
-- 11.8 we wrap the statement in a plpgsql block together with an EXECUTE. This is valid
-- syntax on 11.8 and will execute correctly in 12
DO $create_table_am$
BEGIN
EXECUTE 'CREATE ACCESS METHOD columnar TYPE TABLE HANDLER columnar.columnar_handler';
END $create_table_am$;
--#include "udfs/alter_columnar_table_set/10.0-1.sql"
CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_set(
table_name regclass,
chunk_group_row_limit int DEFAULT NULL,
stripe_row_limit int DEFAULT NULL,
compression name DEFAULT null,
compression_level int DEFAULT NULL)
RETURNS void
LANGUAGE C
AS 'MODULE_PATHNAME', 'alter_columnar_table_set';
COMMENT ON FUNCTION pg_catalog.alter_columnar_table_set(
table_name regclass,
chunk_group_row_limit int,
stripe_row_limit int,
compression name,
compression_level int)
IS 'set one or more options on a columnar table, when set to NULL no change is made';
--#include "udfs/alter_columnar_table_reset/10.0-1.sql"
CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_reset(
table_name regclass,
chunk_group_row_limit bool DEFAULT false,
stripe_row_limit bool DEFAULT false,
compression bool DEFAULT false,
compression_level bool DEFAULT false)
RETURNS void
LANGUAGE C
AS 'MODULE_PATHNAME', 'alter_columnar_table_reset';
COMMENT ON FUNCTION pg_catalog.alter_columnar_table_reset(
table_name regclass,
chunk_group_row_limit bool,
stripe_row_limit bool,
compression bool,
compression_level bool)
IS 'reset on or more options on a columnar table to the system defaults';
$$;
END IF;
END$proc$;
-- add citus_internal schema
CREATE SCHEMA IF NOT EXISTS citus_internal;
-- (this function being dropped in 10.0.3)->#include "udfs/columnar_ensure_objects_exist/10.0-1.sql"
RESET search_path;
-- columnar--10.0.-1 --10.0.2
GRANT USAGE ON SCHEMA columnar TO PUBLIC;
GRANT SELECT ON ALL tables IN SCHEMA columnar TO PUBLIC ;
-- columnar--10.0-3--10.1-1.sql
-- Drop foreign keys between columnar metadata tables.
-- columnar--10.1-1--10.2-1.sql
-- For a proper mapping between tid & (stripe, row_num), add a new column to
-- columnar.stripe and define a BTREE index on this column.
-- Also include storage_id column for per-relation scans.
-- Populate first_row_number column of columnar.stripe table.
--
-- For simplicity, we calculate MAX(row_count) value across all the stripes
-- of all the columanar tables and then use it to populate first_row_number
-- column. This would introduce some gaps however we are okay with that since
-- it's already the case with regular INSERT/COPY's.
DO $$
DECLARE
max_row_count bigint;
-- this should be equal to columnar_storage.h/COLUMNAR_FIRST_ROW_NUMBER
COLUMNAR_FIRST_ROW_NUMBER constant bigint := 1;
BEGIN
SELECT MAX(row_count) INTO max_row_count FROM columnar.stripe;
UPDATE columnar.stripe SET first_row_number = COLUMNAR_FIRST_ROW_NUMBER +
(stripe_num - 1) * max_row_count;
END;
$$;
--#include "udfs/upgrade_columnar_storage/10.2-1.sql"
CREATE OR REPLACE FUNCTION citus_internal.upgrade_columnar_storage(rel regclass)
RETURNS VOID
STRICT
LANGUAGE c AS 'MODULE_PATHNAME', $$upgrade_columnar_storage$$;
COMMENT ON FUNCTION citus_internal.upgrade_columnar_storage(regclass)
IS 'function to upgrade the columnar storage, if necessary';
--#include "udfs/downgrade_columnar_storage/10.2-1.sql"
CREATE OR REPLACE FUNCTION citus_internal.downgrade_columnar_storage(rel regclass)
RETURNS VOID
STRICT
LANGUAGE c AS 'MODULE_PATHNAME', $$downgrade_columnar_storage$$;
COMMENT ON FUNCTION citus_internal.downgrade_columnar_storage(regclass)
IS 'function to downgrade the columnar storage, if necessary';
-- upgrade storage for all columnar relations
SELECT citus_internal.upgrade_columnar_storage(c.oid) FROM pg_class c, pg_am a
WHERE c.relam = a.oid AND amname = 'columnar';
-- columnar--10.2-1--10.2-2.sql
-- revoke read access for columnar.chunk from unprivileged
-- user as it contains chunk min/max values
REVOKE SELECT ON columnar.chunk FROM PUBLIC;
-- columnar--10.2-2--10.2-3.sql
-- Since stripe_first_row_number_idx is required to scan a columnar table, we
-- need to make sure that it is created before doing anything with columnar
-- tables during pg upgrades.
--
-- However, a plain btree index is not a dependency of a table, so pg_upgrade
-- cannot guarantee that stripe_first_row_number_idx gets created when
-- creating columnar.stripe, unless we make it a unique "constraint".
--
-- To do that, drop stripe_first_row_number_idx and create a unique
-- constraint with the same name to keep the code change at minimum.
-- columnar--10.2-3--10.2-4.sql
CREATE OR REPLACE FUNCTION citus_internal.columnar_ensure_am_depends_catalog()
RETURNS void
LANGUAGE plpgsql
SET search_path = pg_catalog
AS $func$
BEGIN
INSERT INTO pg_depend
SELECT -- Define a dependency edge from "columnar table access method" ..
'pg_am'::regclass::oid as classid,
(select oid from pg_am where amname = 'columnar') as objid,
0 as objsubid,
-- ... to each object that is registered to pg_class and that lives
-- in "columnar" schema. That contains catalog tables, indexes
-- created on them and the sequences created in "columnar" schema.
--
-- Given the possibility of user might have created their own objects
-- in columnar schema, we explicitly specify list of objects that we
-- are interested in.
'pg_class'::regclass::oid as refclassid,
columnar_schema_members.relname::regclass::oid as refobjid,
0 as refobjsubid,
'n' as deptype
FROM (VALUES ('columnar.chunk'),
('columnar.chunk_group'),
('columnar.chunk_group_pkey'),
('columnar.chunk_pkey'),
('columnar.options'),
('columnar.options_pkey'),
('columnar.storageid_seq'),
('columnar.stripe'),
('columnar.stripe_first_row_number_idx'),
('columnar.stripe_pkey')
) columnar_schema_members(relname)
-- Avoid inserting duplicate entries into pg_depend.
EXCEPT TABLE pg_depend;
END;
$func$;
COMMENT ON FUNCTION citus_internal.columnar_ensure_am_depends_catalog()
IS 'internal function responsible for creating dependencies from columnar '
'table access method to the rel objects in columnar schema';
SELECT citus_internal.columnar_ensure_am_depends_catalog();

27
src/backend/columnar/sql/downgrades/citus_columnar--11.2-1--11.1-0.sql
View File

@ -1,27 +0,0 @@
-- detach relations from citus_columnar
ALTER EXTENSION citus_columnar DROP SCHEMA columnar;
ALTER EXTENSION citus_columnar DROP SEQUENCE columnar.storageid_seq;
-- columnar tables
ALTER EXTENSION citus_columnar DROP TABLE columnar.options;
ALTER EXTENSION citus_columnar DROP TABLE columnar.stripe;
ALTER EXTENSION citus_columnar DROP TABLE columnar.chunk_group;
ALTER EXTENSION citus_columnar DROP TABLE columnar.chunk;
DO $proc$
BEGIN
-- columnar functions
IF substring(current_Setting('server_version'), '\d+')::int >= 12 THEN
EXECUTE $$
ALTER EXTENSION citus_columnar DROP FUNCTION columnar.columnar_handler;
ALTER EXTENSION citus_columnar DROP ACCESS METHOD columnar;
ALTER EXTENSION citus_columnar DROP FUNCTION pg_catalog.alter_columnar_table_set;
ALTER EXTENSION citus_columnar DROP FUNCTION pg_catalog.alter_columnar_table_reset;
$$;
END IF;
END$proc$;
-- functions under citus_internal for columnar
ALTER EXTENSION citus_columnar DROP FUNCTION citus_internal.upgrade_columnar_storage;
ALTER EXTENSION citus_columnar DROP FUNCTION citus_internal.downgrade_columnar_storage;
ALTER EXTENSION citus_columnar DROP FUNCTION citus_internal.columnar_ensure_am_depends_catalog;

2
src/backend/distributed/citus.control
View File

@ -1,6 +1,6 @@
# Citus extension
comment = 'Citus distributed database'
default_version = '11.2-1'
default_version = '11.1-1'
module_pathname = '$libdir/citus'
relocatable = false
schema = pg_catalog

14
src/backend/distributed/commands/utility_hook.c
View File

@ -617,13 +617,13 @@ ProcessUtilityInternal(PlannedStmt *pstmt,
{
const char *newVersion = defGetString(newVersionValue);
/*alter extension citus update to 11.2-1, and no citus_columnar installed */
if (strcmp(newVersion, "11.2-1") == 0 && citusColumnarOid == InvalidOid)
/*alter extension citus update to 11.1-1, and no citus_columnar installed */
if (strcmp(newVersion, "11.1-1") == 0 && citusColumnarOid == InvalidOid)
{
/*it's upgrade citus to 11.2-1 */
/*it's upgrade citus to 11.1-1 */
CreateExtensionWithVersion("citus_columnar", "11.1-0");
}
else if (strcmp(curExtensionVersion, "11.2-1") == 0 && citusColumnarOid !=
else if (strcmp(curExtensionVersion, "11.1-1") == 0 && citusColumnarOid !=
InvalidOid)
{
/*downgrade citus_columnar to Y */
@ -643,16 +643,16 @@ ProcessUtilityInternal(PlannedStmt *pstmt,
if (newVersionValue)
{
const char *newVersion = defGetString(newVersionValue);
if (strcmp(newVersion, "11.2-1") == 0 && citusColumnarOid != InvalidOid)
if (strcmp(newVersion, "11.1-1") == 0 && citusColumnarOid != InvalidOid)
{
/*after "ALTER EXTENSION citus" updates citus_columnar Y to version Z. */
char *curColumnarVersion = get_extension_version(citusColumnarOid);
if (strcmp(curColumnarVersion, "11.1-0") == 0)
{
AlterExtensionUpdateStmt("citus_columnar", "11.2-1");
AlterExtensionUpdateStmt("citus_columnar", "11.1-1");
}
}
else if (strcmp(newVersion, "11.1-1") == 0 && citusColumnarOid !=
else if (strcmp(newVersion, "11.0-2") == 0 && citusColumnarOid !=
InvalidOid)
{
/*after "ALTER EXTENSION citus" drops citus_columnar extension */

41
src/backend/distributed/sql/citus--11.0-2--11.1-1.sql
View File

@ -6,3 +6,44 @@ DROP FUNCTION pg_catalog.worker_hash_partition_table(bigint, integer, text, text
DROP FUNCTION pg_catalog.worker_merge_files_into_table(bigint, integer, text[], text[]);
DROP FUNCTION pg_catalog.worker_range_partition_table(bigint, integer, text, text, oid, anyarray);
DROP FUNCTION pg_catalog.worker_repartition_cleanup(bigint);
-- bump version to 11.1-1 as version 'Z'
-- drop columnar objects if they exists in citus extension
DO $check_citus$
BEGIN
IF EXISTS (SELECT 1 FROM pg_catalog.pg_extension AS e
INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid)
INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid)
WHERE e.extname='citus' and p.proname = 'columnar_handler'
) THEN
ALTER EXTENSION citus DROP SCHEMA columnar;
ALTER EXTENSION citus DROP SEQUENCE columnar.storageid_seq;
-- columnar tables
ALTER EXTENSION citus DROP TABLE columnar.options;
ALTER EXTENSION citus DROP TABLE columnar.stripe;
ALTER EXTENSION citus DROP TABLE columnar.chunk_group;
ALTER EXTENSION citus DROP TABLE columnar.chunk;
DO $proc$
BEGIN
-- columnar functions
IF substring(current_Setting('server_version'), '\d+')::int >= 12 THEN
EXECUTE $$
ALTER EXTENSION citus DROP FUNCTION columnar.columnar_handler;
ALTER EXTENSION citus DROP ACCESS METHOD columnar;
ALTER EXTENSION citus DROP FUNCTION pg_catalog.alter_columnar_table_set;
ALTER EXTENSION citus DROP FUNCTION pg_catalog.alter_columnar_table_reset;
$$;
END IF;
END$proc$;
-- functions under citus_internal for columnar
ALTER EXTENSION citus DROP FUNCTION citus_internal.upgrade_columnar_storage;
ALTER EXTENSION citus DROP FUNCTION citus_internal.downgrade_columnar_storage;
ALTER EXTENSION citus DROP FUNCTION citus_internal.columnar_ensure_am_depends_catalog;
END IF;
END $check_citus$;

27
src/backend/distributed/sql/downgrades/citus--11.1-1--11.0-2.sql
View File

@ -44,4 +44,29 @@ CREATE FUNCTION pg_catalog.worker_repartition_cleanup(bigint)
RETURNS void
LANGUAGE c
STRICT
AS 'MODULE_PATHNAME', $function$worker_repartition_cleanup$function$
AS 'MODULE_PATHNAME', $function$worker_repartition_cleanup$function$;
-- add relations to citus
ALTER EXTENSION citus ADD SCHEMA columnar;
ALTER EXTENSION citus ADD SEQUENCE columnar.storageid_seq;
ALTER EXTENSION citus ADD TABLE columnar.options;
ALTER EXTENSION citus ADD TABLE columnar.stripe;
ALTER EXTENSION citus ADD TABLE columnar.chunk_group;
ALTER EXTENSION citus ADD TABLE columnar.chunk;
DO $proc$
BEGIN
-- columnar functions
IF substring(current_Setting('server_version'), '\d+')::int >= 12 THEN
EXECUTE $$
ALTER EXTENSION citus ADD FUNCTION columnar.columnar_handler;
ALTER EXTENSION citus ADD ACCESS METHOD columnar;
ALTER EXTENSION citus ADD FUNCTION pg_catalog.alter_columnar_table_set;
ALTER EXTENSION citus ADD FUNCTION pg_catalog.alter_columnar_table_reset;
$$;
END IF;
END$proc$;
ALTER EXTENSION citus ADD FUNCTION citus_internal.upgrade_columnar_storage;
ALTER EXTENSION citus ADD FUNCTION citus_internal.downgrade_columnar_storage;
ALTER EXTENSION citus ADD FUNCTION citus_internal.columnar_ensure_am_depends_catalog;

2
src/include/columnar/columnar_tableam.h
View File

@ -59,5 +59,7 @@ extern int64 ColumnarScanChunkGroupsFiltered(ColumnarScanDesc columnarScanDesc);
extern bool ColumnarSupportsIndexAM(char *indexAMName);
extern bool IsColumnarTableAmTable(Oid relationId);
extern DefElem * GetExtensionOption(List *extensionOptions,
const char *defname);
#endif /* COLUMNAR_TABLEAM_H */