external
/
citus
mirror of https://github.com/citusdata/citus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Squashed commit of the following: 

commit b45a3dc25a
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Tue Jun 14 17:22:01 2022 -0700

    Add comments

commit f78a25f72b
Merge: 1a88d2eb5 f52755a0a
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Tue Jun 14 17:08:00 2022 -0700

    Merge branch 'dev/yanwjin/upgrade1' of github.com:citusdata/citus into dev/yanwjin/upgrade1

commit 1a88d2eb53
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Tue Jun 14 16:32:34 2022 -0700

    Clean up after merge

commit f52755a0a9
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Tue Jun 14 16:32:34 2022 -0700

    Clean up after merge

commit 06babdbdbe
Merge: e3bfb6f74 9c0e00fb4
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Tue Jun 14 16:21:19 2022 -0700

    Merge branch 'dev/yanwjin/upgrade1' of github.com:citusdata/citus into dev/yanwjin/upgrade1

commit e3bfb6f741
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Tue Jun 14 16:20:59 2022 -0700

    clean up after merge

commit 9c0e00fb49
Author: ywj <mouwan.jyw@outlook.com>
Date:   Tue Jun 14 16:03:31 2022 -0700

    Delete citus--11.1-1--11.2-1.sql

    delete unnecessary file

commit 013cff8d5f
Author: ywj <mouwan.jyw@outlook.com>
Date:   Tue Jun 14 16:03:09 2022 -0700

    Delete citus--11.2-1--11.1-1.sql

    delete unnecessary files

commit 5041fa6597
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Tue Jun 14 15:58:21 2022 -0700

    Fix some missing parts

commit e99eb59dd9
Merge: 2494604c5 03c83f0ae
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Tue Jun 14 15:19:45 2022 -0700

    Merge remote-tracking branch 'origin/separate-extensions' into dev/yanwjin/upgrade1

commit 03c83f0ae8
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Fri Jun 10 13:51:03 2022 -0700

    Test: use columnar_internal instead of citus_internal in citus_finish_pg_upgrade forcolumnar

commit 99be185f52
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Fri Jun 10 13:16:14 2022 -0700

    Test: citus_finish_pg_upgrade

commit 40e4dab3de
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Fri Jun 10 13:11:46 2022 -0700

    Test: update latest.sql in citus_finish_pg_upgrade

commit f7b548dc10
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Fri Jun 10 12:55:14 2022 -0700

    Test: add citus_finish_pg_upgrade 11.1-1

commit cf48ab0872
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Thu Jun 9 12:57:02 2022 -0700

    Revert "Test: disable dependencies set for columnar table access method in citus_finish_pg_upgrade latest"

    This reverts commit eab2b56fab.

commit eab2b56fab
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Thu Jun 9 12:50:45 2022 -0700

    Test: disable dependencies set for columnar table access method in citus_finish_pg_upgrade latest

commit 6c9c5b50df
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Thu Jun 9 11:00:59 2022 -0700

    Fix test:upgrade_columnar_metapage_after

commit 057ad83619
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Wed Jun 8 17:42:46 2022 -0700

    remove unused variable

commit da83829974
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Wed Jun 8 17:38:10 2022 -0700

    remove unused variable

commit f95d4be924
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Wed Jun 8 17:31:01 2022 -0700

    remove unused variables

commit 64e1ec7488
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Wed Jun 8 17:16:36 2022 -0700

    Fix upgrade & downgrade errors

commit 1f9fb0511c
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Tue Jun 7 15:09:45 2022 -0700

    test: removed duplicate declaration

commit 7174fb87a0
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Tue Jun 7 12:54:07 2022 -0700

    Testing purpose: test 'alter extension update'

commit 1960e978da
Author: Jeff Davis <jeff@j-davis.com>
Date:   Thu May 26 07:45:53 2022 -0700

    fixup columnar_ensure

commit 335c2ab5a8
Author: Jeff Davis <jeff@j-davis.com>
Date:   Wed May 25 13:55:03 2022 -0700

    tmp3

commit e06abe3534
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Wed May 25 13:55:32 2022 -0700

    install citus_columnar for citus version >= 11.1

commit 345b465b18
Author: Jeff Davis <jeff@j-davis.com>
Date:   Wed May 25 12:43:12 2022 -0700

    tmp2

commit 07a6d32885
Author: Jeff Davis <jeff@j-davis.com>
Date:   Wed May 25 09:47:44 2022 -0700

    works kinda

commit 72fd456870
Author: Jeff Davis <jeff@j-davis.com>
Date:   Wed May 25 09:14:36 2022 -0700

    rm 11.2

commit 25b3e1d1c1
Author: Jeff Davis <jeff@j-davis.com>
Date:   Wed May 25 09:00:17 2022 -0700

    tmp1

commit 9c8c52445b
Author: Jeff Davis <jeff@j-davis.com>
Date:   Wed May 25 08:42:11 2022 -0700

    Add auxiliary control file

commit 5b67bfcc1a
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Tue May 24 15:55:45 2022 -0700

    modify columnar_drop test

commit a1d7b08f45
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Fri May 20 11:39:58 2022 -0700

    add citus_columnar upgrade & downgrade script with new version 11.1-1

commit 60cc332abc
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Tue May 17 17:09:59 2022 -0700

    Modify columnar sql files in Makefile

commit 4d287c9581
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Tue May 17 16:56:16 2022 -0700

    Update version as 11.1-1 and makefile for testing purpose

commit 51ae3ced76
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Tue May 17 11:41:04 2022 -0700

    Testing

commit a1adc06e74
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Mon May 16 13:48:18 2022 -0700

    Delete whitespace at end of lines

commit 32bd222e11
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Mon May 16 13:36:07 2022 -0700

    Fix whitespace in columnar

commit 12927b153c
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Mon May 16 12:46:58 2022 -0700

    Fix whitespace

commit e869fb3be1
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Mon May 16 12:34:35 2022 -0700

    Fix whitespace style issues

commit 98bcf174c7
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Mon May 16 12:06:30 2022 -0700

    Fix some build errors

commit 502fb5ae57
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Tue Apr 26 14:53:51 2022 -0700

    Combine sql changes in columnar

commit b2681564d5
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Fri Apr 22 14:30:13 2022 -0700

    Support upgrade and downgrade for current citus customer

commit d5e25bcb3a
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Tue Apr 12 12:24:46 2022 -0700

    Have citus_columnar as an extension

commit 2494604c59
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Tue May 24 15:55:45 2022 -0700

    modify columnar_drop test

commit 60d4b9f254
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Fri May 20 11:39:58 2022 -0700

    add citus_columnar upgrade & downgrade script with new version 11.1-1

commit 35bd5ed5f6
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Tue May 17 17:09:59 2022 -0700

    Modify columnar sql files in Makefile

commit 99016251e8
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Tue May 17 16:56:16 2022 -0700

    Update version as 11.1-1 and makefile for testing purpose

commit 9e7f22251f
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Tue May 17 11:41:04 2022 -0700

    Testing

commit 7b90b52308
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Mon May 16 13:48:18 2022 -0700

    Delete whitespace at end of lines

commit ddaacfc0de
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Mon May 16 13:36:07 2022 -0700

    Fix whitespace in columnar

commit 671d45c45f
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Mon May 16 12:46:58 2022 -0700

    Fix whitespace

commit d9057d1f4b
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Mon May 16 12:34:35 2022 -0700

    Fix whitespace style issues

commit e45d9eb386
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Mon May 16 12:06:30 2022 -0700

    Fix some build errors

commit 2d6ba8e4ee
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Tue Apr 26 14:53:51 2022 -0700

    Combine sql changes in columnar

commit 5b01917e12
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Fri Apr 22 14:30:13 2022 -0700

    Support upgrade and downgrade for current citus customer

commit 2bc491303f
Author: Yanwen Jin <yanwjin@microsoft.com>
Date:   Tue Apr 12 12:24:46 2022 -0700

    Have citus_columnar as an extension
separate-extensions
Jeff Davis 2022-06-15 11:24:48 -07:00
parent e244e9ffb6
commit 4a488ced29
33 changed files with 1420 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/backend/columnar/Makefile
View File

@ -8,10 +8,22 @@ OBJS += \
$(patsubst $(citus_abs_srcdir)/%.c,%.o,$(foreach dir,$(SUBDIRS), $(sort $(wildcard $(citus_abs_srcdir)/$(dir)/*.c))))
MODULE_big = citus_columnar
EXTENSION = citus_columnar
columnar_sql_files = $(patsubst $(citus_abs_srcdir)/%,%,$(wildcard $(citus_abs_srcdir)/sql/*.sql))
columnar_downgrade_sql_files = $(patsubst $(citus_abs_srcdir)/%,%,$(wildcard $(citus_abs_srcdir)/sql/downgrades/*.sql))
DATA = $(columnar_sql_files) \
$(columnar_downgrade_sql_files)
PG_CPPFLAGS += -I$(libpq_srcdir) -I$(safestringlib_srcdir)/include
ifdef USE_PGXS
PG_CONFIG = pg_config
PGXS := $(shell $(PG_CONFIG) --pgxs)
include $(PGXS)
else
include $(citus_top_builddir)/Makefile.global
.PHONY: install-all
install-all: install
endif

6
src/backend/columnar/citus_columnar.control Normal file
View File

@ -0,0 +1,6 @@
# Columnar extension
comment = 'Citus Columnar extension'
default_version = '11.1-1'
module_pathname = '$libdir/citus_columnar'
relocatable = false
schema = pg_catalog

78
src/backend/columnar/columnar_tableam.c
View File

@ -28,6 +28,7 @@
#include "catalog/pg_extension.h"
#include "catalog/storage.h"
#include "catalog/storage_xlog.h"
#include "commands/defrem.h"
#include "commands/progress.h"
#include "commands/vacuum.h"
#include "commands/extension.h"
@ -2363,6 +2364,63 @@ ColumnarProcessUtility(PlannedStmt *pstmt,
(errmsg("columnar storage parameters specified on non-columnar table")));
}
if (IsA(parsetree, CreateExtensionStmt))
{
CreateExtensionStmt *createExtensionStmt = castNode(CreateExtensionStmt,
parsetree);
if (get_extension_oid("citus_columnar", true) == InvalidOid)
{
if (strcmp(createExtensionStmt->extname, "citus_columnar") == 0)
{
DefElem *newVersionValue = GetExtensionOption(
createExtensionStmt->options,
"new_version");
/*we are not allowed to install citus_columnar as version 11.1-0 by cx*/
if (newVersionValue)
{
const char *newVersion = defGetString(newVersionValue);
if (strcmp(newVersion, "11.1-0") == 0)
{
ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg(
"unsupported citus_columnar version 11.1-0")));
}
}
/*latest citus requires install columnar first, existing citus can only be an older version */
if (get_extension_oid("citus", true) != InvalidOid)
{
ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg(
"must upgrade citus to version 11.1-1 first")));
}
}
}
}
if (IsA(parsetree, AlterExtensionStmt))
{
AlterExtensionStmt *alterExtensionStmt = castNode(AlterExtensionStmt, parsetree);
if (strcmp(alterExtensionStmt->extname, "citus_columnar") == 0)
{
DefElem *newVersionValue = GetExtensionOption(alterExtensionStmt->options,
"new_version");
/*we are not allowed cx to downgrade citus_columnar to 11.1-0*/
if (newVersionValue)
{
const char *newVersion = defGetString(newVersionValue);
if (strcmp(newVersion, "11.1-0") == 0)
{
ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("unsupported citus_columnar version 11.1-0")));
}
}
}
}
PrevProcessUtilityHook_compat(pstmt, queryString, false, context,
params, queryEnv, dest, completionTag);
@ -2984,3 +3042,23 @@ InstalledExtensionVersionColumnar(void)
return installedExtensionVersion;
}
/*
* GetExtensionOption returns DefElem * node with "defname" from "options" list
*/
DefElem *
GetExtensionOption(List *extensionOptions, const char *defname)
{
DefElem *defElement = NULL;
foreach_ptr(defElement, extensionOptions)
{
if (IsA(defElement, DefElem) &&
strncmp(defElement->defname, defname, NAMEDATALEN) == 0)
{
return defElement;
}
}
return NULL;
}

238
src/backend/columnar/sql/citus_columnar--11.1-0--11.1-1.sql Normal file
View File

@ -0,0 +1,238 @@
-- add columnar objects back
ALTER EXTENSION citus_columnar ADD SCHEMA columnar;
ALTER EXTENSION citus_columnar ADD SEQUENCE columnar.storageid_seq;
ALTER EXTENSION citus_columnar ADD TABLE columnar.options;
ALTER EXTENSION citus_columnar ADD TABLE columnar.stripe;
ALTER EXTENSION citus_columnar ADD TABLE columnar.chunk_group;
ALTER EXTENSION citus_columnar ADD TABLE columnar.chunk;
ALTER EXTENSION citus_columnar ADD FUNCTION columnar.columnar_handler;
ALTER EXTENSION citus_columnar ADD ACCESS METHOD columnar;
ALTER EXTENSION citus_columnar ADD FUNCTION pg_catalog.alter_columnar_table_set;
ALTER EXTENSION citus_columnar ADD FUNCTION pg_catalog.alter_columnar_table_reset;
ALTER EXTENSION citus_columnar ADD FUNCTION citus_internal.upgrade_columnar_storage;
ALTER EXTENSION citus_columnar ADD FUNCTION citus_internal.downgrade_columnar_storage;
ALTER EXTENSION citus_columnar ADD FUNCTION citus_internal.columnar_ensure_am_depends_catalog;
CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_set(
table_name regclass,
chunk_group_row_limit int DEFAULT NULL,
stripe_row_limit int DEFAULT NULL,
compression name DEFAULT null,
compression_level int DEFAULT NULL)
RETURNS void
LANGUAGE plpgsql AS
$alter_columnar_table_set$
declare
noop BOOLEAN := true;
cmd TEXT := 'ALTER TABLE ' || table_name::text || ' SET (';
begin
if (chunk_group_row_limit is not null) then
if (not noop) then cmd := cmd || ', '; end if;
cmd := cmd || 'columnar.chunk_group_row_limit=' || chunk_group_row_limit;
noop := false;
end if;
if (stripe_row_limit is not null) then
if (not noop) then cmd := cmd || ', '; end if;
cmd := cmd || 'columnar.stripe_row_limit=' || stripe_row_limit;
noop := false;
end if;
if (compression is not null) then
if (not noop) then cmd := cmd || ', '; end if;
cmd := cmd || 'columnar.compression=' || compression;
noop := false;
end if;
if (compression_level is not null) then
if (not noop) then cmd := cmd || ', '; end if;
cmd := cmd || 'columnar.compression_level=' || compression_level;
noop := false;
end if;
cmd := cmd || ')';
if (not noop) then
execute cmd;
end if;
return;
end;
$alter_columnar_table_set$;
COMMENT ON FUNCTION pg_catalog.alter_columnar_table_set(
table_name regclass,
chunk_group_row_limit int,
stripe_row_limit int,
compression name,
compression_level int)
IS 'set one or more options on a columnar table, when set to NULL no change is made';
CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_reset(
table_name regclass,
chunk_group_row_limit bool DEFAULT false,
stripe_row_limit bool DEFAULT false,
compression bool DEFAULT false,
compression_level bool DEFAULT false)
RETURNS void
LANGUAGE plpgsql AS
$alter_columnar_table_reset$
declare
noop BOOLEAN := true;
cmd TEXT := 'ALTER TABLE ' || table_name::text || ' RESET (';
begin
if (chunk_group_row_limit) then
if (not noop) then cmd := cmd || ', '; end if;
cmd := cmd || 'columnar.chunk_group_row_limit';
noop := false;
end if;
if (stripe_row_limit) then
if (not noop) then cmd := cmd || ', '; end if;
cmd := cmd || 'columnar.stripe_row_limit';
noop := false;
end if;
if (compression) then
if (not noop) then cmd := cmd || ', '; end if;
cmd := cmd || 'columnar.compression';
noop := false;
end if;
if (compression_level) then
if (not noop) then cmd := cmd || ', '; end if;
cmd := cmd || 'columnar.compression_level';
noop := false;
end if;
cmd := cmd || ')';
if (not noop) then
execute cmd;
end if;
return;
end;
$alter_columnar_table_reset$;
COMMENT ON FUNCTION pg_catalog.alter_columnar_table_reset(
table_name regclass,
chunk_group_row_limit bool,
stripe_row_limit bool,
compression bool,
compression_level bool)
IS 'reset on or more options on a columnar table to the system defaults';
-- rename columnar schema to columnar_internal and tighten security
REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA columnar FROM PUBLIC;
ALTER SCHEMA columnar RENAME TO columnar_internal;
REVOKE ALL PRIVILEGES ON SCHEMA columnar_internal FROM PUBLIC;
-- move citus_internal functions to columnar_internal
ALTER FUNCTION citus_internal.upgrade_columnar_storage(regclass) SET SCHEMA columnar_internal;
ALTER FUNCTION citus_internal.downgrade_columnar_storage(regclass) SET SCHEMA columnar_internal;
ALTER FUNCTION citus_internal.columnar_ensure_am_depends_catalog() SET SCHEMA columnar_internal;
-- create columnar schema with public usage privileges
CREATE SCHEMA columnar;
GRANT USAGE ON SCHEMA columnar TO PUBLIC;
-- update UDF to account for columnar_internal schema
CREATE OR REPLACE FUNCTION columnar_internal.columnar_ensure_am_depends_catalog()
RETURNS void
LANGUAGE plpgsql
SET search_path = pg_catalog
AS $func$
BEGIN
INSERT INTO pg_depend
WITH columnar_schema_members(relid) AS (
SELECT pg_class.oid AS relid FROM pg_class
WHERE relnamespace =
COALESCE(
(SELECT pg_namespace.oid FROM pg_namespace WHERE nspname = 'columnar_internal'),
(SELECT pg_namespace.oid FROM pg_namespace WHERE nspname = 'columnar')
)
AND relname IN ('chunk',
'chunk_group',
'chunk_group_pkey',
'chunk_pkey',
'options',
'options_pkey',
'storageid_seq',
'stripe',
'stripe_first_row_number_idx',
'stripe_pkey')
)
SELECT -- Define a dependency edge from "columnar table access method" ..
'pg_am'::regclass::oid as classid,
(select oid from pg_am where amname = 'columnar') as objid,
0 as objsubid,
-- ... to each object that is registered to pg_class and that lives
-- in "columnar" schema. That contains catalog tables, indexes
-- created on them and the sequences created in "columnar" schema.
--
-- Given the possibility of user might have created their own objects
-- in columnar schema, we explicitly specify list of objects that we
-- are interested in.
'pg_class'::regclass::oid as refclassid,
columnar_schema_members.relid as refobjid,
0 as refobjsubid,
'n' as deptype
FROM columnar_schema_members
-- Avoid inserting duplicate entries into pg_depend.
EXCEPT TABLE pg_depend;
END;
$func$;
COMMENT ON FUNCTION columnar_internal.columnar_ensure_am_depends_catalog()
IS 'internal function responsible for creating dependencies from columnar '
'table access method to the rel objects in columnar schema';
-- add utility function
CREATE FUNCTION columnar.get_storage_id(regclass) RETURNS bigint
LANGUAGE C STRICT
AS 'citus_columnar', $$columnar_relation_storageid$$;
-- create views for columnar table information
CREATE VIEW columnar.storage WITH (security_barrier) AS
SELECT c.oid::regclass AS relation,
columnar.get_storage_id(c.oid) AS storage_id
FROM pg_class c, pg_am am
WHERE c.relam = am.oid AND am.amname = 'columnar'
AND pg_has_role(c.relowner, 'USAGE');
COMMENT ON VIEW columnar.storage IS 'Columnar relation ID to storage ID mapping.';
GRANT SELECT ON columnar.storage TO PUBLIC;
CREATE VIEW columnar.options WITH (security_barrier) AS
SELECT regclass AS relation, chunk_group_row_limit,
stripe_row_limit, compression, compression_level
FROM columnar_internal.options o, pg_class c
WHERE o.regclass = c.oid
AND pg_has_role(c.relowner, 'USAGE');
COMMENT ON VIEW columnar.options
IS 'Columnar options for tables on which the current user has ownership privileges.';
GRANT SELECT ON columnar.options TO PUBLIC;
CREATE VIEW columnar.stripe WITH (security_barrier) AS
SELECT relation, storage.storage_id, stripe_num, file_offset, data_length,
column_count, chunk_row_count, row_count, chunk_group_count, first_row_number
FROM columnar_internal.stripe stripe, columnar.storage storage
WHERE stripe.storage_id = storage.storage_id;
COMMENT ON VIEW columnar.stripe
IS 'Columnar stripe information for tables on which the current user has ownership privileges.';
GRANT SELECT ON columnar.stripe TO PUBLIC;
CREATE VIEW columnar.chunk_group WITH (security_barrier) AS
SELECT relation, storage.storage_id, stripe_num, chunk_group_num, row_count
FROM columnar_internal.chunk_group cg, columnar.storage storage
WHERE cg.storage_id = storage.storage_id;
COMMENT ON VIEW columnar.chunk_group
IS 'Columnar chunk group information for tables on which the current user has ownership privileges.';
GRANT SELECT ON columnar.chunk_group TO PUBLIC;
CREATE VIEW columnar.chunk WITH (security_barrier) AS
SELECT relation, storage.storage_id, stripe_num, attr_num, chunk_group_num,
minimum_value, maximum_value, value_stream_offset, value_stream_length,
exists_stream_offset, exists_stream_length, value_compression_type,
value_compression_level, value_decompressed_length, value_count
FROM columnar_internal.chunk chunk, columnar.storage storage
WHERE chunk.storage_id = storage.storage_id;
COMMENT ON VIEW columnar.chunk
IS 'Columnar chunk information for tables on which the current user has ownership privileges.';
GRANT SELECT ON columnar.chunk TO PUBLIC;

1
src/backend/columnar/sql/citus_columnar--11.1-0.sql Normal file
View File

@ -0,0 +1 @@
-- fake sql file 'Y'

435
src/backend/columnar/sql/citus_columnar--11.1-1.sql Normal file
View File

@ -0,0 +1,435 @@
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "CREATE EXTENSION citus_columnar" to load this file. \quit
-- columnar--9.5-1--10.0-1.sql
CREATE SCHEMA IF NOT EXISTS columnar;
SET search_path TO columnar;
CREATE SEQUENCE IF NOT EXISTS storageid_seq MINVALUE 10000000000 NO CYCLE;
CREATE TABLE IF NOT EXISTS options (
regclass regclass NOT NULL PRIMARY KEY,
chunk_group_row_limit int NOT NULL,
stripe_row_limit int NOT NULL,
compression_level int NOT NULL,
compression name NOT NULL
) WITH (user_catalog_table = true);
COMMENT ON TABLE options IS 'columnar table specific options, maintained by alter_columnar_table_set';
CREATE TABLE IF NOT EXISTS stripe (
storage_id bigint NOT NULL,
stripe_num bigint NOT NULL,
file_offset bigint NOT NULL,
data_length bigint NOT NULL,
column_count int NOT NULL,
chunk_row_count int NOT NULL,
row_count bigint NOT NULL,
chunk_group_count int NOT NULL,
first_row_number bigint NOT NULL,
PRIMARY KEY (storage_id, stripe_num),
CONSTRAINT stripe_first_row_number_idx UNIQUE (storage_id, first_row_number)
) WITH (user_catalog_table = true);
COMMENT ON TABLE stripe IS 'Columnar per stripe metadata';
CREATE TABLE IF NOT EXISTS chunk_group (
storage_id bigint NOT NULL,
stripe_num bigint NOT NULL,
chunk_group_num int NOT NULL,
row_count bigint NOT NULL,
PRIMARY KEY (storage_id, stripe_num, chunk_group_num)
);
COMMENT ON TABLE chunk_group IS 'Columnar chunk group metadata';
CREATE TABLE IF NOT EXISTS chunk (
storage_id bigint NOT NULL,
stripe_num bigint NOT NULL,
attr_num int NOT NULL,
chunk_group_num int NOT NULL,
minimum_value bytea,
maximum_value bytea,
value_stream_offset bigint NOT NULL,
value_stream_length bigint NOT NULL,
exists_stream_offset bigint NOT NULL,
exists_stream_length bigint NOT NULL,
value_compression_type int NOT NULL,
value_compression_level int NOT NULL,
value_decompressed_length bigint NOT NULL,
value_count bigint NOT NULL,
PRIMARY KEY (storage_id, stripe_num, attr_num, chunk_group_num)
) WITH (user_catalog_table = true);
COMMENT ON TABLE chunk IS 'Columnar per chunk metadata';
DO $proc$
BEGIN
-- from version 12 and up we have support for tableam's if installed on pg11 we can't
-- create the objects here. Instead we rely on citus_finish_pg_upgrade to be called by the
-- user instead to add the missing objects
IF substring(current_Setting('server_version'), '\d+')::int >= 12 THEN
EXECUTE $$
--#include "udfs/columnar_handler/10.0-1.sql"
CREATE OR REPLACE FUNCTION columnar.columnar_handler(internal)
RETURNS table_am_handler
LANGUAGE C
AS 'MODULE_PATHNAME', 'columnar_handler';
COMMENT ON FUNCTION columnar.columnar_handler(internal)
IS 'internal function returning the handler for columnar tables';
-- postgres 11.8 does not support the syntax for table am, also it is seemingly trying
-- to parse the upgrade file and erroring on unknown syntax.
-- normally this section would not execute on postgres 11 anyway. To trick it to pass on
-- 11.8 we wrap the statement in a plpgsql block together with an EXECUTE. This is valid
-- syntax on 11.8 and will execute correctly in 12
DO $create_table_am$
BEGIN
EXECUTE 'CREATE ACCESS METHOD columnar TYPE TABLE HANDLER columnar.columnar_handler';
END $create_table_am$;
--#include "udfs/alter_columnar_table_set/10.0-1.sql"
CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_set(
table_name regclass,
chunk_group_row_limit int DEFAULT NULL,
stripe_row_limit int DEFAULT NULL,
compression name DEFAULT null,
compression_level int DEFAULT NULL)
RETURNS void
LANGUAGE C
AS 'MODULE_PATHNAME', 'alter_columnar_table_set';
COMMENT ON FUNCTION pg_catalog.alter_columnar_table_set(
table_name regclass,
chunk_group_row_limit int,
stripe_row_limit int,
compression name,
compression_level int)
IS 'set one or more options on a columnar table, when set to NULL no change is made';
--#include "udfs/alter_columnar_table_reset/10.0-1.sql"
CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_reset(
table_name regclass,
chunk_group_row_limit bool DEFAULT false,
stripe_row_limit bool DEFAULT false,
compression bool DEFAULT false,
compression_level bool DEFAULT false)
RETURNS void
LANGUAGE C
AS 'MODULE_PATHNAME', 'alter_columnar_table_reset';
COMMENT ON FUNCTION pg_catalog.alter_columnar_table_reset(
table_name regclass,
chunk_group_row_limit bool,
stripe_row_limit bool,
compression bool,
compression_level bool)
IS 'reset on or more options on a columnar table to the system defaults';
$$;
END IF;
END$proc$;
-- (this function being dropped in 10.0.3)->#include "udfs/columnar_ensure_objects_exist/10.0-1.sql"
RESET search_path;
-- columnar--10.0.-1 --10.0.2
GRANT USAGE ON SCHEMA columnar TO PUBLIC;
GRANT SELECT ON ALL tables IN SCHEMA columnar TO PUBLIC ;
-- columnar--10.0-3--10.1-1.sql
-- Drop foreign keys between columnar metadata tables.
-- columnar--10.1-1--10.2-1.sql
-- For a proper mapping between tid & (stripe, row_num), add a new column to
-- columnar.stripe and define a BTREE index on this column.
-- Also include storage_id column for per-relation scans.
-- Populate first_row_number column of columnar.stripe table.
--
-- For simplicity, we calculate MAX(row_count) value across all the stripes
-- of all the columanar tables and then use it to populate first_row_number
-- column. This would introduce some gaps however we are okay with that since
-- it's already the case with regular INSERT/COPY's.
DO $$
DECLARE
max_row_count bigint;
-- this should be equal to columnar_storage.h/COLUMNAR_FIRST_ROW_NUMBER
COLUMNAR_FIRST_ROW_NUMBER constant bigint := 1;
BEGIN
SELECT MAX(row_count) INTO max_row_count FROM columnar.stripe;
UPDATE columnar.stripe SET first_row_number = COLUMNAR_FIRST_ROW_NUMBER +
(stripe_num - 1) * max_row_count;
END;
$$;
-- columnar--10.2-1--10.2-2.sql
-- revoke read access for columnar.chunk from unprivileged
-- user as it contains chunk min/max values
REVOKE SELECT ON columnar.chunk FROM PUBLIC;
-- columnar--10.2-2--10.2-3.sql
-- Since stripe_first_row_number_idx is required to scan a columnar table, we
-- need to make sure that it is created before doing anything with columnar
-- tables during pg upgrades.
--
-- However, a plain btree index is not a dependency of a table, so pg_upgrade
-- cannot guarantee that stripe_first_row_number_idx gets created when
-- creating columnar.stripe, unless we make it a unique "constraint".
--
-- To do that, drop stripe_first_row_number_idx and create a unique
-- constraint with the same name to keep the code change at minimum.
-- columnar--10.2-3--10.2-4.sql
-- columnar--11.0-2--11.1-1.sql
CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_set(
table_name regclass,
chunk_group_row_limit int DEFAULT NULL,
stripe_row_limit int DEFAULT NULL,
compression name DEFAULT null,
compression_level int DEFAULT NULL)
RETURNS void
LANGUAGE plpgsql AS
$alter_columnar_table_set$
declare
noop BOOLEAN := true;
cmd TEXT := 'ALTER TABLE ' || table_name::text || ' SET (';
begin
if (chunk_group_row_limit is not null) then
if (not noop) then cmd := cmd || ', '; end if;
cmd := cmd || 'columnar.chunk_group_row_limit=' || chunk_group_row_limit;
noop := false;
end if;
if (stripe_row_limit is not null) then
if (not noop) then cmd := cmd || ', '; end if;
cmd := cmd || 'columnar.stripe_row_limit=' || stripe_row_limit;
noop := false;
end if;
if (compression is not null) then
if (not noop) then cmd := cmd || ', '; end if;
cmd := cmd || 'columnar.compression=' || compression;
noop := false;
end if;
if (compression_level is not null) then
if (not noop) then cmd := cmd || ', '; end if;
cmd := cmd || 'columnar.compression_level=' || compression_level;
noop := false;
end if;
cmd := cmd || ')';
if (not noop) then
execute cmd;
end if;
return;
end;
$alter_columnar_table_set$;
COMMENT ON FUNCTION pg_catalog.alter_columnar_table_set(
table_name regclass,
chunk_group_row_limit int,
stripe_row_limit int,
compression name,
compression_level int)
IS 'set one or more options on a columnar table, when set to NULL no change is made';
CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_reset(
table_name regclass,
chunk_group_row_limit bool DEFAULT false,
stripe_row_limit bool DEFAULT false,
compression bool DEFAULT false,
compression_level bool DEFAULT false)
RETURNS void
LANGUAGE plpgsql AS
$alter_columnar_table_reset$
declare
noop BOOLEAN := true;
cmd TEXT := 'ALTER TABLE ' || table_name::text || ' RESET (';
begin
if (chunk_group_row_limit) then
if (not noop) then cmd := cmd || ', '; end if;
cmd := cmd || 'columnar.chunk_group_row_limit';
noop := false;
end if;
if (stripe_row_limit) then
if (not noop) then cmd := cmd || ', '; end if;
cmd := cmd || 'columnar.stripe_row_limit';
noop := false;
end if;
if (compression) then
if (not noop) then cmd := cmd || ', '; end if;
cmd := cmd || 'columnar.compression';
noop := false;
end if;
if (compression_level) then
if (not noop) then cmd := cmd || ', '; end if;
cmd := cmd || 'columnar.compression_level';
noop := false;
end if;
cmd := cmd || ')';
if (not noop) then
execute cmd;
end if;
return;
end;
$alter_columnar_table_reset$;
COMMENT ON FUNCTION pg_catalog.alter_columnar_table_reset(
table_name regclass,
chunk_group_row_limit bool,
stripe_row_limit bool,
compression bool,
compression_level bool)
IS 'reset on or more options on a columnar table to the system defaults';
-- rename columnar schema to columnar_internal and tighten security
REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA columnar FROM PUBLIC;
ALTER SCHEMA columnar RENAME TO columnar_internal;
REVOKE ALL PRIVILEGES ON SCHEMA columnar_internal FROM PUBLIC;
-- create columnar schema with public usage privileges
CREATE SCHEMA columnar;
GRANT USAGE ON SCHEMA columnar TO PUBLIC;
--#include "udfs/upgrade_columnar_storage/10.2-1.sql"
CREATE OR REPLACE FUNCTION columnar_internal.upgrade_columnar_storage(rel regclass)
RETURNS VOID
STRICT
LANGUAGE c AS 'MODULE_PATHNAME', $$upgrade_columnar_storage$$;
COMMENT ON FUNCTION columnar_internal.upgrade_columnar_storage(regclass)
IS 'function to upgrade the columnar storage, if necessary';
--#include "udfs/downgrade_columnar_storage/10.2-1.sql"
CREATE OR REPLACE FUNCTION columnar_internal.downgrade_columnar_storage(rel regclass)
RETURNS VOID
STRICT
LANGUAGE c AS 'MODULE_PATHNAME', $$downgrade_columnar_storage$$;
COMMENT ON FUNCTION columnar_internal.downgrade_columnar_storage(regclass)
IS 'function to downgrade the columnar storage, if necessary';
-- update UDF to account for columnar_internal schema
CREATE OR REPLACE FUNCTION columnar_internal.columnar_ensure_am_depends_catalog()
RETURNS void
LANGUAGE plpgsql
SET search_path = pg_catalog
AS $func$
BEGIN
INSERT INTO pg_depend
WITH columnar_schema_members(relid) AS (
SELECT pg_class.oid AS relid FROM pg_class
WHERE relnamespace =
COALESCE(
(SELECT pg_namespace.oid FROM pg_namespace WHERE nspname = 'columnar_internal'),
(SELECT pg_namespace.oid FROM pg_namespace WHERE nspname = 'columnar')
)
AND relname IN ('chunk',
'chunk_group',
'chunk_group_pkey',
'chunk_pkey',
'options',
'options_pkey',
'storageid_seq',
'stripe',
'stripe_first_row_number_idx',
'stripe_pkey')
)
SELECT -- Define a dependency edge from "columnar table access method" ..
'pg_am'::regclass::oid as classid,
(select oid from pg_am where amname = 'columnar') as objid,
0 as objsubid,
-- ... to each object that is registered to pg_class and that lives
-- in "columnar" schema. That contains catalog tables, indexes
-- created on them and the sequences created in "columnar" schema.
--
-- Given the possibility of user might have created their own objects
-- in columnar schema, we explicitly specify list of objects that we
-- are interested in.
'pg_class'::regclass::oid as refclassid,
columnar_schema_members.relid as refobjid,
0 as refobjsubid,
'n' as deptype
FROM columnar_schema_members
-- Avoid inserting duplicate entries into pg_depend.
EXCEPT TABLE pg_depend;
END;
$func$;
COMMENT ON FUNCTION columnar_internal.columnar_ensure_am_depends_catalog()
IS 'internal function responsible for creating dependencies from columnar '
'table access method to the rel objects in columnar schema';
SELECT columnar_internal.columnar_ensure_am_depends_catalog();
-- add utility function
CREATE FUNCTION columnar.get_storage_id(regclass) RETURNS bigint
LANGUAGE C STRICT
AS 'citus_columnar', $$columnar_relation_storageid$$;
-- create views for columnar table information
CREATE VIEW columnar.storage WITH (security_barrier) AS
SELECT c.oid::regclass AS relation,
columnar.get_storage_id(c.oid) AS storage_id
FROM pg_class c, pg_am am
WHERE c.relam = am.oid AND am.amname = 'columnar'
AND pg_has_role(c.relowner, 'USAGE');
COMMENT ON VIEW columnar.storage IS 'Columnar relation ID to storage ID mapping.';
GRANT SELECT ON columnar.storage TO PUBLIC;
CREATE VIEW columnar.options WITH (security_barrier) AS
SELECT regclass AS relation, chunk_group_row_limit,
stripe_row_limit, compression, compression_level
FROM columnar_internal.options o, pg_class c
WHERE o.regclass = c.oid
AND pg_has_role(c.relowner, 'USAGE');
COMMENT ON VIEW columnar.options
IS 'Columnar options for tables on which the current user has ownership privileges.';
GRANT SELECT ON columnar.options TO PUBLIC;
CREATE VIEW columnar.stripe WITH (security_barrier) AS
SELECT relation, storage.storage_id, stripe_num, file_offset, data_length,
column_count, chunk_row_count, row_count, chunk_group_count, first_row_number
FROM columnar_internal.stripe stripe, columnar.storage storage
WHERE stripe.storage_id = storage.storage_id;
COMMENT ON VIEW columnar.stripe
IS 'Columnar stripe information for tables on which the current user has ownership privileges.';
GRANT SELECT ON columnar.stripe TO PUBLIC;
CREATE VIEW columnar.chunk_group WITH (security_barrier) AS
SELECT relation, storage.storage_id, stripe_num, chunk_group_num, row_count
FROM columnar_internal.chunk_group cg, columnar.storage storage
WHERE cg.storage_id = storage.storage_id;
COMMENT ON VIEW columnar.chunk_group
IS 'Columnar chunk group information for tables on which the current user has ownership privileges.';
GRANT SELECT ON columnar.chunk_group TO PUBLIC;
CREATE VIEW columnar.chunk WITH (security_barrier) AS
SELECT relation, storage.storage_id, stripe_num, attr_num, chunk_group_num,
minimum_value, maximum_value, value_stream_offset, value_stream_length,
exists_stream_offset, exists_stream_length, value_compression_type,
value_compression_level, value_decompressed_length, value_count
FROM columnar_internal.chunk chunk, columnar.storage storage
WHERE chunk.storage_id = storage.storage_id;
COMMENT ON VIEW columnar.chunk
IS 'Columnar chunk information for tables on which the current user has ownership privileges.';
GRANT SELECT ON columnar.chunk TO PUBLIC;

2
src/backend/columnar/sql/columnar--10.1-1--10.2-1.sql
View File

@ -28,5 +28,5 @@ $$;
#include "udfs/downgrade_columnar_storage/10.2-1.sql"
-- upgrade storage for all columnar relations
SELECT citus_internal.upgrade_columnar_storage(c.oid) FROM pg_class c, pg_am a
PERFORM citus_internal.upgrade_columnar_storage(c.oid) FROM pg_class c, pg_am a
WHERE c.relam = a.oid AND amname = 'columnar';

2
src/backend/columnar/sql/columnar--10.2-3--10.2-4.sql
View File

@ -2,4 +2,4 @@
#include "udfs/columnar_ensure_am_depends_catalog/10.2-4.sql"
SELECT citus_internal.columnar_ensure_am_depends_catalog();
PERFORM citus_internal.columnar_ensure_am_depends_catalog();

116
src/backend/columnar/sql/downgrades/citus_columnar--11.1-1--11.1-0.sql Normal file
View File

@ -0,0 +1,116 @@
CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_set(
table_name regclass,
chunk_group_row_limit int DEFAULT NULL,
stripe_row_limit int DEFAULT NULL,
compression name DEFAULT null,
compression_level int DEFAULT NULL)
RETURNS void
LANGUAGE C
AS 'MODULE_PATHNAME', 'alter_columnar_table_set';
COMMENT ON FUNCTION pg_catalog.alter_columnar_table_set(
table_name regclass,
chunk_group_row_limit int,
stripe_row_limit int,
compression name,
compression_level int)
IS 'set one or more options on a columnar table, when set to NULL no change is made';
CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_reset(
table_name regclass,
chunk_group_row_limit bool DEFAULT false,
stripe_row_limit bool DEFAULT false,
compression bool DEFAULT false,
compression_level bool DEFAULT false)
RETURNS void
LANGUAGE C
AS 'MODULE_PATHNAME', 'alter_columnar_table_reset';
COMMENT ON FUNCTION pg_catalog.alter_columnar_table_reset(
table_name regclass,
chunk_group_row_limit bool,
stripe_row_limit bool,
compression bool,
compression_level bool)
IS 'reset on or more options on a columnar table to the system defaults';
CREATE OR REPLACE FUNCTION columnar_internal.columnar_ensure_am_depends_catalog()
RETURNS void
LANGUAGE plpgsql
SET search_path = pg_catalog
AS $func$
BEGIN
INSERT INTO pg_depend
SELECT -- Define a dependency edge from "columnar table access method" ..
'pg_am'::regclass::oid as classid,
(select oid from pg_am where amname = 'columnar') as objid,
0 as objsubid,
-- ... to each object that is registered to pg_class and that lives
-- in "columnar" schema. That contains catalog tables, indexes
-- created on them and the sequences created in "columnar" schema.
--
-- Given the possibility of user might have created their own objects
-- in columnar schema, we explicitly specify list of objects that we
-- are interested in.
'pg_class'::regclass::oid as refclassid,
columnar_schema_members.relname::regclass::oid as refobjid,
0 as refobjsubid,
'n' as deptype
FROM (VALUES ('columnar.chunk'),
('columnar.chunk_group'),
('columnar.chunk_group_pkey'),
('columnar.chunk_pkey'),
('columnar.options'),
('columnar.options_pkey'),
('columnar.storageid_seq'),
('columnar.stripe'),
('columnar.stripe_first_row_number_idx'),
('columnar.stripe_pkey')
) columnar_schema_members(relname)
-- Avoid inserting duplicate entries into pg_depend.
EXCEPT TABLE pg_depend;
END;
$func$;
COMMENT ON FUNCTION columnar_internal.columnar_ensure_am_depends_catalog()
IS 'internal function responsible for creating dependencies from columnar '
'table access method to the rel objects in columnar schema';
DROP VIEW columnar.options;
DROP VIEW columnar.stripe;
DROP VIEW columnar.chunk_group;
DROP VIEW columnar.chunk;
DROP VIEW columnar.storage;
DROP FUNCTION columnar.get_storage_id(regclass);
DROP SCHEMA columnar;
-- move columnar_internal functions back to citus_internal
ALTER FUNCTION columnar_internal.upgrade_columnar_storage(regclass) SET SCHEMA citus_internal;
ALTER FUNCTION columnar_internal.downgrade_columnar_storage(regclass) SET SCHEMA citus_internal;
ALTER FUNCTION columnar_internal.columnar_ensure_am_depends_catalog() SET SCHEMA citus_internal;
ALTER SCHEMA columnar_internal RENAME TO columnar;
GRANT USAGE ON SCHEMA columnar TO PUBLIC;
GRANT SELECT ON columnar.options TO PUBLIC;
GRANT SELECT ON columnar.stripe TO PUBLIC;
GRANT SELECT ON columnar.chunk_group TO PUBLIC;
-- detach relations from citus_columnar
ALTER EXTENSION citus_columnar DROP SCHEMA columnar;
ALTER EXTENSION citus_columnar DROP SEQUENCE columnar.storageid_seq;
-- columnar tables
ALTER EXTENSION citus_columnar DROP TABLE columnar.options;
ALTER EXTENSION citus_columnar DROP TABLE columnar.stripe;
ALTER EXTENSION citus_columnar DROP TABLE columnar.chunk_group;
ALTER EXTENSION citus_columnar DROP TABLE columnar.chunk;
ALTER EXTENSION citus_columnar DROP FUNCTION columnar.columnar_handler;
ALTER EXTENSION citus_columnar DROP ACCESS METHOD columnar;
ALTER EXTENSION citus_columnar DROP FUNCTION pg_catalog.alter_columnar_table_set;
ALTER EXTENSION citus_columnar DROP FUNCTION pg_catalog.alter_columnar_table_reset;
-- functions under citus_internal for columnar
ALTER EXTENSION citus_columnar DROP FUNCTION citus_internal.upgrade_columnar_storage;
ALTER EXTENSION citus_columnar DROP FUNCTION citus_internal.downgrade_columnar_storage;
ALTER EXTENSION citus_columnar DROP FUNCTION citus_internal.columnar_ensure_am_depends_catalog;

1
src/backend/distributed/citus--11.1-1.control Normal file
View File

@ -0,0 +1 @@
requires = 'citus_columnar'

78
src/backend/distributed/commands/extension.c
View File

@ -11,6 +11,7 @@
#include "postgres.h"
#include "access/genam.h"
#include "access/xact.h"
#include "citus_version.h"
#include "catalog/pg_extension_d.h"
#include "commands/defrem.h"
@ -930,3 +931,80 @@ AlterExtensionUpdateStmtObjectAddress(Node *node, bool missing_ok)
return address;
}
/*
* CreateExtensionWithVersion builds and execute create extension statements
* per given extension name and extension verision
*/
void
CreateExtensionWithVersion(char *extname, char *extVersion)
{
CreateExtensionStmt *createExtensionStmt = makeNode(CreateExtensionStmt);
/* set location to -1 as it is unknown */
int location = -1;
/* set extension name and if_not_exists fields */
createExtensionStmt->extname = extname;
createExtensionStmt->if_not_exists = true;
if (extVersion == NULL)
{
createExtensionStmt->options = NIL;
}
else
{
Node *extensionVersionArg = (Node *) makeString(extVersion);
DefElem *extensionVersionElement = makeDefElem("new_version", extensionVersionArg,
location);
createExtensionStmt->options = lappend(createExtensionStmt->options,
extensionVersionElement);
}
CreateExtension(NULL, createExtensionStmt);
CommandCounterIncrement();
}
/*
* GetExtensionVersionNumber convert extension version to real value
*/
double
GetExtensionVersionNumber(char *extVersion)
{
char *strtokPosition = NULL;
char *versionVal = strtok_r(extVersion, "-", &strtokPosition);
double versionNumber = strtod(versionVal, NULL);
return versionNumber;
}
/*
* AlterExtensionUpdateStmt builds and execute Alter extension statements
* per given extension name and updates extension verision
*/
void
AlterExtensionUpdateStmt(char *extname, char *extVersion)
{
AlterExtensionStmt *alterExtensionStmt = makeNode(AlterExtensionStmt);
/* set location to -1 as it is unknown */
int location = -1;
alterExtensionStmt->extname = extname;
if (extVersion == NULL)
{
ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT),
errmsg("alter extension \"%s\" should not be empty",
extVersion)));
}
Node *extensionVersionArg = (Node *) makeString(extVersion);
DefElem *extensionVersionElement = makeDefElem("new_version", extensionVersionArg,
location);
alterExtensionStmt->options = lappend(alterExtensionStmt->options,
extensionVersionElement);
ExecAlterExtensionStmt(NULL, alterExtensionStmt);
CommandCounterIncrement();
}

110
src/backend/distributed/commands/utility_hook.c
View File

@ -38,8 +38,10 @@
#endif
#include "catalog/catalog.h"
#include "catalog/dependency.h"
#include "citus_version.h"
#include "commands/dbcommands.h"
#include "commands/defrem.h"
#include "commands/extension.h"
#include "commands/tablecmds.h"
#include "distributed/adaptive_executor.h"
#include "distributed/backend_data.h"
@ -72,8 +74,10 @@
#include "lib/stringinfo.h"
#include "nodes/parsenodes.h"
#include "nodes/pg_list.h"
#include "nodes/makefuncs.h"
#include "tcop/utility.h"
#include "utils/builtins.h"
#include "utils/fmgroids.h"
#include "utils/lsyscache.h"
#include "utils/syscache.h"
@ -194,6 +198,35 @@ multi_ProcessUtility(PlannedStmt *pstmt,
ErrorIfUnstableCreateOrAlterExtensionStmt(parsetree);
}
if (IsA(parsetree, CreateExtensionStmt))
{
/*CREATE EXTENSION CITUS (version Z) */
CreateExtensionStmt *createExtensionStmt = castNode(CreateExtensionStmt,
parsetree);
if (strcmp(createExtensionStmt->extname, "citus") == 0)
{
double versionNumber = strtod(CITUS_MAJORVERSION, NULL);
DefElem *newVersionValue = GetExtensionOption(createExtensionStmt->options,
"new_version");
/*create extension citus version xxx*/
if (newVersionValue)
{
char *newVersion = strdup(defGetString(newVersionValue));
versionNumber = GetExtensionVersionNumber(newVersion);
}
/*citus version >= 11.1 requires install citus_columnar first*/
if (versionNumber * 100 >= 1110.0)
{
if (get_extension_oid("citus_columnar", true) == InvalidOid)
{
CreateExtensionWithVersion("citus_columnar", NULL);
}
}
}
}
if (!CitusHasBeenLoaded())
{
/*
@ -616,11 +649,88 @@ ProcessUtilityInternal(PlannedStmt *pstmt,
if (isAlterExtensionUpdateCitusStmt)
{
citusCanBeUpdatedToAvailableVersion = !InstalledAndAvailableVersionsSame();
/*upgrade citus: alter extension citus update to 'xxx' */
DefElem *newVersionValue = GetExtensionOption(
((AlterExtensionStmt *) parsetree)->options, "new_version");
Oid citusColumnarOid = get_extension_oid("citus_columnar", true);
if (newVersionValue)
{
char *newVersion = defGetString(newVersionValue);
double newVersionNumber = GetExtensionVersionNumber(strdup(newVersion));
/*alter extension citus update to version >= 11.1-1, and no citus_columnar installed */
if (newVersionNumber * 100 >= 1110 && citusColumnarOid == InvalidOid)
{
/*it's upgrade citus to 11.1-1 or further version */
CreateExtensionWithVersion("citus_columnar", "11.1-0");
}
else if (newVersionNumber * 100 < 1110 && citusColumnarOid != InvalidOid)
{
/*downgrade citus, need downgrade citus_columnar to Y */
AlterExtensionUpdateStmt("citus_columnar", "11.1-0");
}
}
else
{
/*alter exention citus update without specifying the version*/
double versionNumber = strtod(CITUS_MAJORVERSION, NULL);
if (versionNumber * 100 >= 1110)
{
if (citusColumnarOid == InvalidOid)
{
CreateExtensionWithVersion("citus_columnar", "11.1-0");
}
}
}
}
PrevProcessUtility_compat(pstmt, queryString, false, context,
params, queryEnv, dest, completionTag);
if (isAlterExtensionUpdateCitusStmt)
{
DefElem *newVersionValue = GetExtensionOption(
((AlterExtensionStmt *) parsetree)->options, "new_version");
Oid citusColumnarOid = get_extension_oid("citus_columnar", true);
if (newVersionValue)
{
char *newVersion = defGetString(newVersionValue);
double newVersionNumber = GetExtensionVersionNumber(strdup(newVersion));
if (newVersionNumber * 100 >= 1110 && citusColumnarOid != InvalidOid)
{
/*upgrade citus, after "ALTER EXTENSION citus update to xxx" updates citus_columnar Y to version Z. */
char *curColumnarVersion = get_extension_version(citusColumnarOid);
if (strcmp(curColumnarVersion, "11.1-0") == 0)
{
AlterExtensionUpdateStmt("citus_columnar", "11.1-1");
}
}
else if (newVersionNumber * 100 < 1110 && citusColumnarOid != InvalidOid)
{
/*downgrade citus, after "ALTER EXTENSION citus update to xxx" drops citus_columnar extension */
char *curColumnarVersion = get_extension_version(citusColumnarOid);
if (strcmp(curColumnarVersion, "11.1-0") == 0)
{
RemoveExtensionById(citusColumnarOid);
}
}
}
else
{
/*alter extension citus update, need upgrade citus_columnar from Y to Z*/
double versionNumber = strtod(CITUS_MAJORVERSION, NULL);
if (versionNumber * 100 >= 1110)
{
char *curColumnarVersion = get_extension_version(citusColumnarOid);
if (strcmp(curColumnarVersion, "11.1-0") == 0)
{
AlterExtensionUpdateStmt("citus_columnar", "11.1-1");
}
}
}
}
/*
* if we are running ALTER EXTENSION citus UPDATE (to "<version>") command, we may need
* to mark existing objects as distributed depending on the "version" parameter if

15
src/backend/distributed/sql/citus--10.0-1--10.0-2.sql
View File

@ -1,5 +1,18 @@
-- citus--10.0-1--10.0-2
#include "../../columnar/sql/columnar--10.0-1--10.0-2.sql"
--#include "../../columnar/sql/columnar--10.0-1--10.0-2.sql"
DO $$ begin raise log '%', 'begin 10.0-1--10.0-2'; end; $$;
DO $check_columnar$
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_extension AS e
INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid)
INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid)
WHERE e.extname='citus_columnar' and p.proname = 'columnar_handler'
) THEN
#include "../../columnar/sql/columnar--10.0-1--10.0-2.sql"
END IF;
END;
$check_columnar$;
GRANT SELECT ON public.citus_tables TO public;
DO $$ begin raise log '%', ' 10.0-1--10.0-2'; end; $$;

15
src/backend/distributed/sql/citus--10.0-4--10.1-1.sql
View File

@ -3,13 +3,25 @@
-- add the current database to the distributed objects if not already in there.
-- this is to reliably propagate some of the alter database commands that might be
-- supported.
INSERT INTO citus.pg_dist_object SELECT
'pg_catalog.pg_database'::regclass::oid AS oid,
(SELECT oid FROM pg_database WHERE datname = current_database()) as objid,
0 as objsubid
ON CONFLICT DO NOTHING;
#include "../../columnar/sql/columnar--10.0-3--10.1-1.sql"
--#include "../../columnar/sql/columnar--10.0-3--10.1-1.sql"
DO $check_columnar$
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_extension AS e
INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid)
INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid)
WHERE e.extname='citus_columnar' and p.proname = 'columnar_handler'
) THEN
#include "../../columnar/sql/columnar--10.0-3--10.1-1.sql"
END IF;
END;
$check_columnar$;
#include "udfs/create_distributed_table/10.1-1.sql";
#include "udfs/worker_partitioned_relation_total_size/10.1-1.sql"
#include "udfs/worker_partitioned_relation_size/10.1-1.sql"
@ -48,4 +60,5 @@ WHERE repmodel = 'c'
DROP TRIGGER pg_dist_rebalance_strategy_enterprise_check_trigger ON pg_catalog.pg_dist_rebalance_strategy;
DROP FUNCTION citus_internal.pg_dist_rebalance_strategy_enterprise_check();
DO $$ begin raise log '%', '10.0-4--10.1-1'; end; $$;
#include "udfs/citus_cleanup_orphaned_shards/10.1-1.sql"

17
src/backend/distributed/sql/citus--10.1-1--10.2-1.sql
View File

@ -9,7 +9,19 @@ GRANT ALL ON FUNCTION pg_catalog.worker_record_sequence_dependency(regclass,regc
ALTER TABLE pg_catalog.pg_dist_placement ADD CONSTRAINT placement_shardid_groupid_unique_index UNIQUE (shardid, groupid);
#include "udfs/stop_metadata_sync_to_node/10.2-1.sql"
#include "../../columnar/sql/columnar--10.1-1--10.2-1.sql"
--#include "../../columnar/sql/columnar--10.1-1--10.2-1.sql"
DO $check_columnar$
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_extension AS e
INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid)
INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid)
WHERE e.extname='citus_columnar' and p.proname = 'columnar_handler'
) THEN
#include "../../columnar/sql/columnar--10.1-1--10.2-1.sql"
END IF;
END;
$check_columnar$;
#include "udfs/citus_internal_add_partition_metadata/10.2-1.sql";
#include "udfs/citus_internal_add_shard_metadata/10.2-1.sql";
#include "udfs/citus_internal_add_placement_metadata/10.2-1.sql";
@ -21,6 +33,8 @@ ALTER TABLE pg_catalog.pg_dist_placement ADD CONSTRAINT placement_shardid_groupi
#include "udfs/get_missing_time_partition_ranges/10.2-1.sql"
#include "udfs/worker_nextval/10.2-1.sql"
DO $$ begin raise log '%', 'begin 10.1-1--10.2-1'; end; $$;
DROP FUNCTION pg_catalog.citus_drop_all_shards(regclass, text, text);
CREATE FUNCTION pg_catalog.citus_drop_all_shards(logicalrelid regclass,
schema_name text,
@ -34,3 +48,4 @@ COMMENT ON FUNCTION pg_catalog.citus_drop_all_shards(regclass, text, text, boole
#include "udfs/citus_drop_trigger/10.2-1.sql";
#include "udfs/citus_prepare_pg_upgrade/10.2-1.sql"
#include "udfs/citus_finish_pg_upgrade/10.2-1.sql"
DO $$ begin raise log '%', '10.1-1--10.2-1'; end; $$;

15
src/backend/distributed/sql/citus--10.2-1--10.2-2.sql
View File

@ -2,4 +2,17 @@
-- bump version to 10.2-2
#include "../../columnar/sql/columnar--10.2-1--10.2-2.sql"
--#include "../../columnar/sql/columnar--10.2-1--10.2-2.sql"
DO $$ begin raise log '%', 'begin 10.2-1--10.2-2'; end; $$;
DO $check_columnar$
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_extension AS e
INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid)
INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid)
WHERE e.extname='citus_columnar' and p.proname = 'columnar_handler'
) THEN
#include "../../columnar/sql/columnar--10.2-1--10.2-2.sql"
END IF;
END;
$check_columnar$;
DO $$ begin raise log '%', '10.2-1--10.2-2'; end; $$;

13
src/backend/distributed/sql/citus--10.2-2--10.2-3.sql
View File

@ -2,4 +2,15 @@
-- bump version to 10.2-3
#include "../../columnar/sql/columnar--10.2-2--10.2-3.sql"
--#include "../../columnar/sql/columnar--10.2-2--10.2-3.sql"
DO $check_columnar$
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_extension AS e
INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid)
INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid)
WHERE e.extname='citus_columnar' and p.proname = 'columnar_handler'
) THEN
#include "../../columnar/sql/columnar--10.2-2--10.2-3.sql"
END IF;
END;
$check_columnar$;

15
src/backend/distributed/sql/citus--10.2-3--10.2-4.sql
View File

@ -2,9 +2,22 @@
-- bump version to 10.2-4
#include "../../columnar/sql/columnar--10.2-3--10.2-4.sql"
--#include "../../columnar/sql/columnar--10.2-3--10.2-4.sql"
DO $$ begin raise log '%', 'begin 10.2-3--10.2-4'; end; $$;
DO $check_columnar$
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_extension AS e
INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid)
INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid)
WHERE e.extname='citus_columnar' and p.proname = 'columnar_handler'
) THEN
#include "../../columnar/sql/columnar--10.2-3--10.2-4.sql"
END IF;
END;
$check_columnar$;
#include "udfs/fix_partition_shard_index_names/10.2-4.sql"
#include "udfs/fix_all_partition_shard_index_names/10.2-4.sql"
#include "udfs/worker_fix_partition_shard_index_names/10.2-4.sql"
#include "udfs/citus_finish_pg_upgrade/10.2-4.sql"
DO $$ begin raise log '%', '10.2-3--10.2-4'; end; $$;

35
src/backend/distributed/sql/citus--11.0-2--11.1-1.sql
View File

@ -1,3 +1,4 @@
DO $$ begin raise log '%', 'begin 11.0-2--11.1-1'; end; $$;
DROP FUNCTION pg_catalog.worker_create_schema(bigint,text);
DROP FUNCTION pg_catalog.worker_cleanup_job_schema_cache();
DROP FUNCTION pg_catalog.worker_fetch_foreign_file(text, text, bigint, text[], integer[]);
@ -7,4 +8,36 @@ DROP FUNCTION pg_catalog.worker_merge_files_into_table(bigint, integer, text[],
DROP FUNCTION pg_catalog.worker_range_partition_table(bigint, integer, text, text, oid, anyarray);
DROP FUNCTION pg_catalog.worker_repartition_cleanup(bigint);
#include "../../columnar/sql/columnar--11.0-2--11.1-1.sql"
-- If upgrading citus, the columnar objects are already being a part of the
-- citus extension, and must be detached so that they can be attached
-- to the citus_columnar extension.
DO $check_citus$
BEGIN
IF EXISTS (SELECT 1 FROM pg_catalog.pg_extension AS e
INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid)
INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid)
WHERE e.extname='citus' and p.proname = 'columnar_handler'
) THEN
ALTER EXTENSION citus DROP SCHEMA columnar;
ALTER EXTENSION citus DROP SEQUENCE columnar.storageid_seq;
-- columnar tables
ALTER EXTENSION citus DROP TABLE columnar.options;
ALTER EXTENSION citus DROP TABLE columnar.stripe;
ALTER EXTENSION citus DROP TABLE columnar.chunk_group;
ALTER EXTENSION citus DROP TABLE columnar.chunk;
ALTER EXTENSION citus DROP FUNCTION columnar.columnar_handler;
ALTER EXTENSION citus DROP ACCESS METHOD columnar;
ALTER EXTENSION citus DROP FUNCTION pg_catalog.alter_columnar_table_set;
ALTER EXTENSION citus DROP FUNCTION pg_catalog.alter_columnar_table_reset;
-- functions under citus_internal for columnar
ALTER EXTENSION citus DROP FUNCTION citus_internal.upgrade_columnar_storage;
ALTER EXTENSION citus DROP FUNCTION citus_internal.downgrade_columnar_storage;
ALTER EXTENSION citus DROP FUNCTION citus_internal.columnar_ensure_am_depends_catalog;
END IF;
END $check_citus$;
#include "udfs/citus_finish_pg_upgrade/11.1-1.sql"
DO $$ begin raise log '%', ' 11.0-2--11.1-1'; end; $$;

26
src/backend/distributed/sql/citus--9.5-1--10.0-4.sql
View File

@ -6,7 +6,7 @@
-- cat citus--9.5-1--10.0-1.sql citus--10.0-1--10.0-2.sql citus--10.0-2--10.0-3.sql > citus--9.5-1--10.0-4.sql
-- copy of citus--9.5-1--10.0-1
DO $$ begin raise log '%', 'begin 9.5-1--10.0-4'; end; $$;
DROP FUNCTION pg_catalog.upgrade_to_reference_table(regclass);
DROP FUNCTION IF EXISTS pg_catalog.citus_total_relation_size(regclass);
@ -35,7 +35,18 @@ DROP FUNCTION IF EXISTS pg_catalog.citus_total_relation_size(regclass);
#include "udfs/worker_change_sequence_dependency/10.0-1.sql"
#include "udfs/remove_local_tables_from_metadata/10.0-1.sql"
--#include "../../columnar/sql/columnar--9.5-1--10.0-1.sql"
DO $check_columnar$
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_extension AS e
INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid)
INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid)
WHERE e.extname='citus_columnar' and p.proname = 'columnar_handler'
) THEN
#include "../../columnar/sql/columnar--9.5-1--10.0-1.sql"
END IF;
END;
$check_columnar$;
#include "udfs/time_partition_range/10.0-1.sql"
#include "udfs/time_partitions/10.0-1.sql"
@ -172,7 +183,19 @@ GRANT SELECT ON pg_catalog.citus_worker_stat_activity TO PUBLIC;
-- copy of citus--10.0-1--10.0-2
--#include "../../columnar/sql/columnar--10.0-1--10.0-2.sql"
DO $check_columnar$
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_extension AS e
INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid)
INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid)
WHERE e.extname='citus_columnar' and p.proname = 'columnar_handler'
) THEN
#include "../../columnar/sql/columnar--10.0-1--10.0-2.sql"
END IF;
END;
$check_columnar$;
-- copy of citus--10.0-2--10.0-3
@ -215,3 +238,4 @@ COMMENT ON FUNCTION pg_catalog.citus_get_active_worker_nodes()
RESET search_path;
DO $$ begin raise log '%', ' 9.5-1--10.0-4'; end; $$;

17
src/backend/distributed/sql/downgrades/citus--11.1-1--11.0-2.sql
View File

@ -46,4 +46,19 @@ CREATE FUNCTION pg_catalog.worker_repartition_cleanup(bigint)
STRICT
AS 'MODULE_PATHNAME', $function$worker_repartition_cleanup$function$;
#include "../../../columnar/sql/downgrades/columnar--11.1-1--11.0-2.sql"
-- add relations to citus
ALTER EXTENSION citus ADD SCHEMA columnar;
ALTER EXTENSION citus ADD SEQUENCE columnar.storageid_seq;
ALTER EXTENSION citus ADD TABLE columnar.options;
ALTER EXTENSION citus ADD TABLE columnar.stripe;
ALTER EXTENSION citus ADD TABLE columnar.chunk_group;
ALTER EXTENSION citus ADD TABLE columnar.chunk;
ALTER EXTENSION citus ADD FUNCTION columnar.columnar_handler;
ALTER EXTENSION citus ADD ACCESS METHOD columnar;
ALTER EXTENSION citus ADD FUNCTION pg_catalog.alter_columnar_table_set;
ALTER EXTENSION citus ADD FUNCTION pg_catalog.alter_columnar_table_reset;
ALTER EXTENSION citus ADD FUNCTION citus_internal.upgrade_columnar_storage;
ALTER EXTENSION citus ADD FUNCTION citus_internal.downgrade_columnar_storage;
ALTER EXTENSION citus ADD FUNCTION citus_internal.columnar_ensure_am_depends_catalog;

151
src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/11.1-1.sql generated Normal file
View File

@ -0,0 +1,151 @@
CREATE OR REPLACE FUNCTION pg_catalog.citus_finish_pg_upgrade()
RETURNS void
LANGUAGE plpgsql
SET search_path = pg_catalog
AS $cppu$
DECLARE
table_name regclass;
command text;
trigger_name text;
BEGIN
IF substring(current_Setting('server_version'), '\d+')::int >= 14 THEN
EXECUTE $cmd$
-- disable propagation to prevent EnsureCoordinator errors
-- the aggregate created here does not depend on Citus extension (yet)
-- since we add the dependency with the next command
SET citus.enable_ddl_propagation TO OFF;
CREATE AGGREGATE array_cat_agg(anycompatiblearray) (SFUNC = array_cat, STYPE = anycompatiblearray);
COMMENT ON AGGREGATE array_cat_agg(anycompatiblearray)
IS 'concatenate input arrays into a single array';
RESET citus.enable_ddl_propagation;
$cmd$;
ELSE
EXECUTE $cmd$
SET citus.enable_ddl_propagation TO OFF;
CREATE AGGREGATE array_cat_agg(anyarray) (SFUNC = array_cat, STYPE = anyarray);
COMMENT ON AGGREGATE array_cat_agg(anyarray)
IS 'concatenate input arrays into a single array';
RESET citus.enable_ddl_propagation;
$cmd$;
END IF;
--
-- Citus creates the array_cat_agg but because of a compatibility
-- issue between pg13-pg14, we drop and create it during upgrade.
-- And as Citus creates it, there needs to be a dependency to the
-- Citus extension, so we create that dependency here.
-- We are not using:
-- ALTER EXENSION citus DROP/CREATE AGGREGATE array_cat_agg
-- because we don't have an easy way to check if the aggregate
-- exists with anyarray type or anycompatiblearray type.
INSERT INTO pg_depend
SELECT
'pg_proc'::regclass::oid as classid,
(SELECT oid FROM pg_proc WHERE proname = 'array_cat_agg') as objid,
0 as objsubid,
'pg_extension'::regclass::oid as refclassid,
(select oid from pg_extension where extname = 'citus') as refobjid,
0 as refobjsubid ,
'e' as deptype;
--
-- restore citus catalog tables
--
INSERT INTO pg_catalog.pg_dist_partition SELECT * FROM public.pg_dist_partition;
INSERT INTO pg_catalog.pg_dist_shard SELECT * FROM public.pg_dist_shard;
INSERT INTO pg_catalog.pg_dist_placement SELECT * FROM public.pg_dist_placement;
INSERT INTO pg_catalog.pg_dist_node_metadata SELECT * FROM public.pg_dist_node_metadata;
INSERT INTO pg_catalog.pg_dist_node SELECT * FROM public.pg_dist_node;
INSERT INTO pg_catalog.pg_dist_local_group SELECT * FROM public.pg_dist_local_group;
INSERT INTO pg_catalog.pg_dist_transaction SELECT * FROM public.pg_dist_transaction;
INSERT INTO pg_catalog.pg_dist_colocation SELECT * FROM public.pg_dist_colocation;
-- enterprise catalog tables
INSERT INTO pg_catalog.pg_dist_authinfo SELECT * FROM public.pg_dist_authinfo;
INSERT INTO pg_catalog.pg_dist_poolinfo SELECT * FROM public.pg_dist_poolinfo;
INSERT INTO pg_catalog.pg_dist_rebalance_strategy SELECT
name,
default_strategy,
shard_cost_function::regprocedure::regproc,
node_capacity_function::regprocedure::regproc,
shard_allowed_on_node_function::regprocedure::regproc,
default_threshold,
minimum_threshold,
improvement_threshold
FROM public.pg_dist_rebalance_strategy;
--
-- drop backup tables
--
DROP TABLE public.pg_dist_authinfo;
DROP TABLE public.pg_dist_colocation;
DROP TABLE public.pg_dist_local_group;
DROP TABLE public.pg_dist_node;
DROP TABLE public.pg_dist_node_metadata;
DROP TABLE public.pg_dist_partition;
DROP TABLE public.pg_dist_placement;
DROP TABLE public.pg_dist_poolinfo;
DROP TABLE public.pg_dist_shard;
DROP TABLE public.pg_dist_transaction;
DROP TABLE public.pg_dist_rebalance_strategy;
--
-- reset sequences
--
PERFORM setval('pg_catalog.pg_dist_shardid_seq', (SELECT MAX(shardid)+1 AS max_shard_id FROM pg_dist_shard), false);
PERFORM setval('pg_catalog.pg_dist_placement_placementid_seq', (SELECT MAX(placementid)+1 AS max_placement_id FROM pg_dist_placement), false);
PERFORM setval('pg_catalog.pg_dist_groupid_seq', (SELECT MAX(groupid)+1 AS max_group_id FROM pg_dist_node), false);
PERFORM setval('pg_catalog.pg_dist_node_nodeid_seq', (SELECT MAX(nodeid)+1 AS max_node_id FROM pg_dist_node), false);
PERFORM setval('pg_catalog.pg_dist_colocationid_seq', (SELECT MAX(colocationid)+1 AS max_colocation_id FROM pg_dist_colocation), false);
--
-- register triggers
--
FOR table_name IN SELECT logicalrelid FROM pg_catalog.pg_dist_partition
LOOP
trigger_name := 'truncate_trigger_' || table_name::oid;
command := 'create trigger ' || trigger_name || ' after truncate on ' || table_name || ' execute procedure pg_catalog.citus_truncate_trigger()';
EXECUTE command;
command := 'update pg_trigger set tgisinternal = true where tgname = ' || quote_literal(trigger_name);
EXECUTE command;
END LOOP;
--
-- set dependencies
--
INSERT INTO pg_depend
SELECT
'pg_class'::regclass::oid as classid,
p.logicalrelid::regclass::oid as objid,
0 as objsubid,
'pg_extension'::regclass::oid as refclassid,
(select oid from pg_extension where extname = 'citus') as refobjid,
0 as refobjsubid ,
'n' as deptype
FROM pg_catalog.pg_dist_partition p;
-- set dependencies for columnar table access method
PERFORM columnar_internal.columnar_ensure_am_depends_catalog();
-- restore pg_dist_object from the stable identifiers
TRUNCATE pg_catalog.pg_dist_object;
INSERT INTO pg_catalog.pg_dist_object (classid, objid, objsubid, distribution_argument_index, colocationid)
SELECT
address.classid,
address.objid,
address.objsubid,
naming.distribution_argument_index,
naming.colocationid
FROM
public.pg_dist_object naming,
pg_catalog.pg_get_object_address(naming.type, naming.object_names, naming.object_args) address;
DROP TABLE public.pg_dist_object;
END;
$cppu$;
COMMENT ON FUNCTION pg_catalog.citus_finish_pg_upgrade()
IS 'perform tasks to restore citus settings from a location that has been prepared before pg_upgrade';

2
src/backend/distributed/sql/udfs/citus_finish_pg_upgrade/latest.sql
View File

@ -128,7 +128,7 @@ BEGIN
FROM pg_catalog.pg_dist_partition p;
-- set dependencies for columnar table access method
PERFORM citus_internal.columnar_ensure_am_depends_catalog();
PERFORM columnar_internal.columnar_ensure_am_depends_catalog();
-- restore pg_dist_object from the stable identifiers
TRUNCATE pg_catalog.pg_dist_object;

2
src/include/columnar/columnar_tableam.h
View File

@ -59,5 +59,7 @@ extern int64 ColumnarScanChunkGroupsFiltered(ColumnarScanDesc columnarScanDesc);
extern bool ColumnarSupportsIndexAM(char *indexAMName);
extern bool IsColumnarTableAmTable(Oid relationId);
extern DefElem * GetExtensionOption(List *extensionOptions,
const char *defname);
#endif /* COLUMNAR_TABLEAM_H */

4
src/include/distributed/commands.h
View File

@ -206,7 +206,9 @@ extern ObjectAddress AlterExtensionSchemaStmtObjectAddress(Node *stmt,
bool missing_ok);
extern ObjectAddress AlterExtensionUpdateStmtObjectAddress(Node *stmt,
bool missing_ok);
extern void CreateExtensionWithVersion(char *extname, char *extVersion);
extern void AlterExtensionUpdateStmt(char *extname, char *extVersion);
extern double GetExtensionVersionNumber(char *extVersion);
/* foreign_constraint.c - forward declarations */
extern bool ConstraintIsAForeignKeyToReferenceTable(char *constraintName,

2
src/test/regress/Makefile
View File

@ -186,7 +186,7 @@ check-operations: all
-- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/operations_schedule $(EXTRA_TESTS)
check-columnar:
$(pg_regress_multi_check) --load-extension=citus \
$(pg_regress_multi_check) --load-extension=citus_columnar --load-extension=citus \
-- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/columnar_schedule $(EXTRA_TESTS)
check-columnar-isolation: all $(isolation_test_files)

6
src/test/regress/expected/columnar_drop.out
View File

@ -42,13 +42,13 @@ NOTICE: Citus partially supports CREATE DATABASE for distributed databases
DETAIL: Citus does not propagate CREATE DATABASE command to workers
HINT: You can manually create a database and its extensions on workers.
\c db_to_drop
CREATE EXTENSION citus;
CREATE EXTENSION citus_columnar;
SELECT oid::text databaseoid FROM pg_database WHERE datname = current_database() \gset
CREATE TABLE test_table(data int) USING columnar;
DROP EXTENSION citus CASCADE;
DROP EXTENSION citus_columnar CASCADE;
NOTICE: drop cascades to table test_table
-- test database drop
CREATE EXTENSION citus;
CREATE EXTENSION citus_columnar;
SELECT oid::text databaseoid FROM pg_database WHERE datname = current_database() \gset
CREATE TABLE test_table(data int) USING columnar;
\c :datname

28
src/test/regress/expected/multi_extension.out
View File

@ -103,6 +103,7 @@ ORDER BY 1, 2;
-- DROP EXTENSION pre-created by the regression suite
DROP EXTENSION citus;
DROP EXTENSION citus_columnar;
\c
-- these tests switch between citus versions and call ddl's that require pg_dist_object to be created
SET citus.enable_metadata_sync TO 'false';
@ -1053,8 +1054,14 @@ SELECT * FROM multi_extension.print_extension_changes();
-- Snapshot of state at 11.1-1
ALTER EXTENSION citus UPDATE TO '11.1-1';
SELECT * FROM multi_extension.print_extension_changes();
previous_object | current_object
previous_object | current_object
---------------------------------------------------------------------
access method columnar |
function alter_columnar_table_reset(regclass,boolean,boolean,boolean,boolean) void |
function alter_columnar_table_set(regclass,integer,integer,name,integer) void |
function citus_internal.columnar_ensure_am_depends_catalog() void |
function citus_internal.downgrade_columnar_storage(regclass) void |
function citus_internal.upgrade_columnar_storage(regclass) void |
function columnar.columnar_handler(internal) table_am_handler |
function worker_cleanup_job_schema_cache() void |
function worker_create_schema(bigint,text) void |
@ -1064,25 +1071,13 @@ SELECT * FROM multi_extension.print_extension_changes();
function worker_merge_files_into_table(bigint,integer,text[],text[]) void |
function worker_range_partition_table(bigint,integer,text,text,oid,anyarray) void |
function worker_repartition_cleanup(bigint) void |
schema columnar |
sequence columnar.storageid_seq |
table columnar.chunk |
table columnar.chunk_group |
table columnar.options |
table columnar.stripe |
| function columnar.get_storage_id(regclass) bigint
| function columnar_internal.columnar_handler(internal) table_am_handler
| schema columnar_internal
| sequence columnar_internal.storageid_seq
| table columnar_internal.chunk
| table columnar_internal.chunk_group
| table columnar_internal.options
| table columnar_internal.stripe
| view columnar.chunk
| view columnar.chunk_group
| view columnar.options
| view columnar.storage
| view columnar.stripe
(27 rows)
(21 rows)
-- Test downgrade script (result should be empty)
ALTER EXTENSION citus UPDATE TO '11.0-2';
@ -1119,6 +1114,7 @@ ORDER BY 1, 2;
RESET citus.enable_version_checks;
RESET columnar.enable_version_checks;
DROP EXTENSION citus;
DROP EXTENSION citus_columnar;
CREATE EXTENSION citus VERSION '8.0-1';
ERROR: specified version incompatible with loaded Citus library
DETAIL: Loaded library requires 11.1, but 8.0-1 was specified.
@ -1195,11 +1191,13 @@ NOTICE: version "9.1-1" of extension "citus" is already installed
ALTER EXTENSION citus UPDATE;
-- re-create in newest version
DROP EXTENSION citus;
DROP EXTENSION citus_columnar;
\c
CREATE EXTENSION citus;
-- test cache invalidation in workers
\c - - - :worker_1_port
DROP EXTENSION citus;
DROP EXTENSION citus_columnar;
SET citus.enable_version_checks TO 'false';
SET columnar.enable_version_checks TO 'false';
CREATE EXTENSION citus VERSION '8.0-1';

2
src/test/regress/expected/upgrade_columnar_metapage_after.out
View File

@ -94,7 +94,7 @@ SELECT version_major, version_minor, reserved_stripe_id, reserved_row_number
(1 row)
-- table is already upgraded, make sure that upgrade_columnar_metapage is no-op
SELECT citus_internal.upgrade_columnar_storage(c.oid)
SELECT columnar_internal.upgrade_columnar_storage(c.oid)
FROM pg_class c, pg_am a
WHERE c.relam = a.oid AND amname = 'columnar' and relname = 'columnar_table_2';
upgrade_columnar_storage

22
src/test/regress/expected/upgrade_list_citus_objects.out
View File

@ -16,10 +16,7 @@ WHERE refclassid = 'pg_catalog.pg_extension'::pg_catalog.regclass
ORDER BY 1;
description
---------------------------------------------------------------------
access method columnar
event trigger citus_cascade_to_partition
function alter_columnar_table_reset(regclass,boolean,boolean,boolean,boolean)
function alter_columnar_table_set(regclass,integer,integer,name,integer)
function alter_distributed_table(regclass,text,integer,text,boolean)
function alter_old_partitions_set_access_method(regclass,timestamp with time zone,name)
function alter_role_if_exists(text,text)
@ -63,8 +60,6 @@ ORDER BY 1;
function citus_finish_citus_upgrade()
function citus_finish_pg_upgrade()
function citus_get_active_worker_nodes()
function citus_internal.columnar_ensure_am_depends_catalog()
function citus_internal.downgrade_columnar_storage(regclass)
function citus_internal.find_groupid_for_node(text,integer)
function citus_internal.pg_dist_node_trigger_func()
function citus_internal.pg_dist_rebalance_strategy_trigger_func()
@ -72,7 +67,6 @@ ORDER BY 1;
function citus_internal.refresh_isolation_tester_prepared_statement()
function citus_internal.replace_isolation_tester_func()
function citus_internal.restore_isolation_tester_func()
function citus_internal.upgrade_columnar_storage(regclass)
function citus_internal_add_colocation_metadata(integer,integer,integer,regtype,oid)
function citus_internal_add_object_metadata(text,text[],text[],integer,integer,boolean)
function citus_internal_add_partition_metadata(regclass,"char",text,integer,"char")
@ -129,8 +123,6 @@ ORDER BY 1;
function citus_version()
function column_name_to_column(regclass,text)
function column_to_column_name(regclass,text)
function columnar.get_storage_id(regclass)
function columnar_internal.columnar_handler(internal)
function coord_combine_agg(oid,cstring,anyelement)
function coord_combine_agg_ffunc(internal,oid,cstring,anyelement)
function coord_combine_agg_sfunc(internal,oid,cstring,anyelement)
@ -243,18 +235,11 @@ ORDER BY 1;
function worker_save_query_explain_analyze(text,jsonb)
schema citus
schema citus_internal
schema columnar
schema columnar_internal
sequence columnar_internal.storageid_seq
sequence pg_dist_colocationid_seq
sequence pg_dist_groupid_seq
sequence pg_dist_node_nodeid_seq
sequence pg_dist_placement_placementid_seq
sequence pg_dist_shardid_seq
table columnar_internal.chunk
table columnar_internal.chunk_group
table columnar_internal.options
table columnar_internal.stripe
table pg_dist_authinfo
table pg_dist_colocation
table pg_dist_local_group
@ -279,12 +264,7 @@ ORDER BY 1;
view citus_shards_on_worker
view citus_stat_activity
view citus_stat_statements
view columnar.chunk
view columnar.chunk_group
view columnar.options
view columnar.storage
view columnar.stripe
view pg_dist_shard_placement
view time_partitions
(270 rows)
(250 rows)

6
src/test/regress/sql/columnar_drop.sql
View File

@ -37,15 +37,15 @@ SELECT current_database() datname \gset
CREATE DATABASE db_to_drop;
\c db_to_drop
CREATE EXTENSION citus;
CREATE EXTENSION citus_columnar;
SELECT oid::text databaseoid FROM pg_database WHERE datname = current_database() \gset
CREATE TABLE test_table(data int) USING columnar;
DROP EXTENSION citus CASCADE;
DROP EXTENSION citus_columnar CASCADE;
-- test database drop
CREATE EXTENSION citus;
CREATE EXTENSION citus_columnar;
SELECT oid::text databaseoid FROM pg_database WHERE datname = current_database() \gset
CREATE TABLE test_table(data int) USING columnar;

4
src/test/regress/sql/multi_extension.sql
View File

@ -98,6 +98,7 @@ ORDER BY 1, 2;
-- DROP EXTENSION pre-created by the regression suite
DROP EXTENSION citus;
DROP EXTENSION citus_columnar;
\c
-- these tests switch between citus versions and call ddl's that require pg_dist_object to be created
@ -497,6 +498,7 @@ ORDER BY 1, 2;
RESET citus.enable_version_checks;
RESET columnar.enable_version_checks;
DROP EXTENSION citus;
DROP EXTENSION citus_columnar;
CREATE EXTENSION citus VERSION '8.0-1';
-- Test non-distributed queries work even in version mismatch
@ -561,6 +563,7 @@ ALTER EXTENSION citus UPDATE;
-- re-create in newest version
DROP EXTENSION citus;
DROP EXTENSION citus_columnar;
\c
CREATE EXTENSION citus;
@ -568,6 +571,7 @@ CREATE EXTENSION citus;
\c - - - :worker_1_port
DROP EXTENSION citus;
DROP EXTENSION citus_columnar;
SET citus.enable_version_checks TO 'false';
SET columnar.enable_version_checks TO 'false';
CREATE EXTENSION citus VERSION '8.0-1';

2
src/test/regress/sql/upgrade_columnar_metapage_after.sql
View File

@ -63,7 +63,7 @@ SELECT version_major, version_minor, reserved_stripe_id, reserved_row_number
FROM columnar_storage_info('no_data_columnar_table');
-- table is already upgraded, make sure that upgrade_columnar_metapage is no-op
SELECT citus_internal.upgrade_columnar_storage(c.oid)
SELECT columnar_internal.upgrade_columnar_storage(c.oid)
FROM pg_class c, pg_am a
WHERE c.relam = a.oid AND amname = 'columnar' and relname = 'columnar_table_2';