external
/
citus
mirror of https://github.com/citusdata/citus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Support upgrade and downgrade for current citus customer

pull/5911/head
Yanwen Jin 2022-04-22 14:30:13 -07:00
parent 2bc491303f
commit 5b01917e12
25 changed files with 363 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/backend/columnar/Makefile
View File

@ -9,7 +9,8 @@ OBJS += \
MODULE_big = citus_columnar
EXTENSION = citus_columnar
DATA = $(wildcard sql/*--*.sql)
DATA = $(wildcard sql/*--*.sql) \
$(wildcard sql/downgrades/*--*.sql)
PG_CPPFLAGS += -I$(libpq_srcdir) -I$(safestringlib_srcdir)/include
ifdef USE_PGXS

2
src/backend/columnar/citus_columnar.control
View File

@ -1,6 +1,6 @@
# Columnar extension
comment = 'Citus Columnar extension'
default_version = '11.1-2'
default_version = '11.2-1'
module_pathname = '$libdir/citus_columnar'
relocatable = false
schema = pg_catalog

48
src/backend/columnar/columnar_tableam.c
View File

@ -27,6 +27,7 @@
#include "catalog/pg_extension.h"
#include "catalog/storage.h"
#include "catalog/storage_xlog.h"
#include "commands/defrem.h"
#include "commands/progress.h"
#include "commands/vacuum.h"
#include "commands/extension.h"
@ -57,6 +58,7 @@
#include "columnar/columnar_tableam.h"
#include "columnar/columnar_version_compat.h"
#include "distributed/listutils.h"
#include "distributed/deparser.h"
/*
* Timing parameters for truncate locking heuristics.
@ -2135,6 +2137,52 @@ ColumnarProcessUtility(PlannedStmt *pstmt,
RelationClose(rel);
}
if (IsA(parsetree, CreateExtensionStmt))
{
CreateExtensionStmt *createExtensionStmt = castNode(CreateExtensionStmt,
parsetree);
if (strcmp(createExtensionStmt->extname, "citus_columnar") == 0)
{
DefElem *newVersionValue = GetExtensionOption(createExtensionStmt->options,
"new_version");
if (newVersionValue)
{
const char *newVersion = defGetString(newVersionValue);
if (strcmp(newVersion, "11.1-0") == 0)
{
ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("unsupported citus_columnar version 11.1-0")));
}
}
/*latest citus requires install columnar first, existing citus can only be an older version */
if (get_extension_oid("citus", true) != InvalidOid)
{
ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("must upgrade citus to version 11.2-1 first")));
}
}
}
if (IsA(parsetree, AlterExtensionStmt))
{
AlterExtensionStmt *alterExtensionStmt = castNode(AlterExtensionStmt, parsetree);
if (strcmp(alterExtensionStmt->extname, "citus_columnar") == 0)
{
DefElem *newVersionValue = GetExtensionOption(alterExtensionStmt->options,
"new_version");
if (newVersionValue)
{
const char *newVersion = defGetString(newVersionValue);
if (strcmp(newVersion, "11.1-0") == 0)
{
ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("unsupported citus_columnar version 11.1-0")));
}
}
}
}
PrevProcessUtilityHook_compat(pstmt, queryString, false, context,
params, queryEnv, dest, completionTag);
}

23
src/backend/columnar/sql/citus_columnar--11.1-0--11.2-1.sql Normal file
View File

@ -0,0 +1,23 @@
-- add columnar objects back
ALTER EXTENSION citus_columnar ADD SCHEMA columnar;
ALTER EXTENSION citus_columnar ADD SEQUENCE columnar.storageid_seq;
ALTER EXTENSION citus_columnar ADD TABLE columnar.options;
ALTER EXTENSION citus_columnar ADD TABLE columnar.stripe;
ALTER EXTENSION citus_columnar ADD TABLE columnar.chunk_group;
ALTER EXTENSION citus_columnar ADD TABLE columnar.chunk;
DO $proc$
BEGIN
-- columnar functions
IF substring(current_Setting('server_version'), '\d+')::int >= 12 THEN
EXECUTE $$
ALTER EXTENSION citus_columnar ADD FUNCTION columnar.columnar_handler;
ALTER EXTENSION citus_columnar ADD ACCESS METHOD columnar;
ALTER EXTENSION citus_columnar ADD FUNCTION pg_catalog.alter_columnar_table_set;
ALTER EXTENSION citus_columnar ADD FUNCTION pg_catalog.alter_columnar_table_reset;
$$;
END IF;
END$proc$;
ALTER EXTENSION citus_columnar ADD FUNCTION citus_internal.upgrade_columnar_storage;
ALTER EXTENSION citus_columnar ADD FUNCTION citus_internal.downgrade_columnar_storage;
ALTER EXTENSION citus_columnar ADD FUNCTION citus_internal.columnar_ensure_am_depends_catalog;

1
src/backend/columnar/sql/citus_columnar--11.1-0.sql Normal file
View File

@ -0,0 +1 @@
-- fake sql file 'Y'

0
src/backend/columnar/sql/citus_columnar--11.1-2.sql → src/backend/columnar/sql/citus_columnar--11.2-1.sql
View File

2
src/backend/columnar/sql/columnar--10.1-1--10.2-1.sql
View File

@ -28,5 +28,5 @@ $$;
#include "udfs/downgrade_columnar_storage/10.2-1.sql"
-- upgrade storage for all columnar relations
SELECT citus_internal.upgrade_columnar_storage(c.oid) FROM pg_class c, pg_am a
PERFORM citus_internal.upgrade_columnar_storage(c.oid) FROM pg_class c, pg_am a
WHERE c.relam = a.oid AND amname = 'columnar';

2
src/backend/columnar/sql/columnar--10.2-3--10.2-4.sql
View File

@ -2,4 +2,4 @@
#include "udfs/columnar_ensure_am_depends_catalog/10.2-4.sql"
SELECT citus_internal.columnar_ensure_am_depends_catalog();
PERFORM citus_internal.columnar_ensure_am_depends_catalog();

27
src/backend/columnar/sql/downgrades/citus_columnar--11.2-1--11.1-0.sql Normal file
View File

@ -0,0 +1,27 @@
-- detach relations from citus_columnar
ALTER EXTENSION citus_columnar DROP SCHEMA columnar;
ALTER EXTENSION citus_columnar DROP SEQUENCE columnar.storageid_seq;
-- columnar tables
ALTER EXTENSION citus_columnar DROP TABLE columnar.options;
ALTER EXTENSION citus_columnar DROP TABLE columnar.stripe;
ALTER EXTENSION citus_columnar DROP TABLE columnar.chunk_group;
ALTER EXTENSION citus_columnar DROP TABLE columnar.chunk;
DO $proc$
BEGIN
-- columnar functions
IF substring(current_Setting('server_version'), '\d+')::int >= 12 THEN
EXECUTE $$
ALTER EXTENSION citus_columnar DROP FUNCTION columnar.columnar_handler;
ALTER EXTENSION citus_columnar DROP ACCESS METHOD columnar;
ALTER EXTENSION citus_columnar DROP FUNCTION pg_catalog.alter_columnar_table_set;
ALTER EXTENSION citus_columnar DROP FUNCTION pg_catalog.alter_columnar_table_reset;
$$;
END IF;
END$proc$;
-- functions under citus_internal for columnar
ALTER EXTENSION citus_columnar DROP FUNCTION citus_internal.upgrade_columnar_storage;
ALTER EXTENSION citus_columnar DROP FUNCTION citus_internal.downgrade_columnar_storage;
ALTER EXTENSION citus_columnar DROP FUNCTION citus_internal.columnar_ensure_am_depends_catalog;

1
src/backend/distributed/Makefile
View File

@ -16,7 +16,6 @@ generated_sql_files = $(patsubst %,$(citus_abs_srcdir)/build/%,$(template_sql_fi
generated_downgrade_sql_files += $(patsubst %,$(citus_abs_srcdir)/build/sql/%,$(template_downgrade_sql_files))
# All citus--*.sql files that are used to upgrade between versions
DATA_built = $(generated_sql_files)
DATA = citus--11.1-2.control
# directories with source files
SUBDIRS = . commands connection ddl deparser executor metadata operations planner progress relay safeclib test transaction utils worker
# enterprise modules

4
src/backend/distributed/citus--11.1-2.control
View File

@ -1,4 +0,0 @@
# Citus extension 11.1-2
relocatable = false
schema = pg_catalog
requires = 'citus_columnar'

3
src/backend/distributed/citus.control
View File

@ -1,6 +1,7 @@
# Citus extension
comment = 'Citus distributed database'
default_version = '11.1-2'
default_version = '11.2-1'
module_pathname = '$libdir/citus'
relocatable = false
schema = pg_catalog
requires = 'citus_columnar'

66
src/backend/distributed/commands/extension.c
View File

@ -9,7 +9,7 @@
*/
#include "postgres.h"
#include "access/xact.h"
#include "citus_version.h"
#include "catalog/pg_extension_d.h"
#include "commands/defrem.h"
@ -842,3 +842,67 @@ AlterExtensionUpdateStmtObjectAddress(Node *node, bool missing_ok)
return address;
}
/*
* CreateExtensionWithVersion builds and execute create extension statements
* per given extension name and extension verision
*/
void
CreateExtensionWithVersion(const char *extname, const char *extVersion)
{
CreateExtensionStmt *createExtensionStmt = makeNode(CreateExtensionStmt);
/* set location to -1 as it is unknown */
int location = -1;
/* set extension name and if_not_exists fields */
createExtensionStmt->extname = extname;
createExtensionStmt->if_not_exists = true;
if (extVersion == NULL)
{
createExtensionStmt->options = NIL;
}
else
{
Node *extensionVersionArg = (Node *) makeString(extVersion);
DefElem *extensionVersionElement = makeDefElem("new_version", extensionVersionArg,
location);
createExtensionStmt->options = lappend(createExtensionStmt->options,
extensionVersionElement);
}
CreateExtension(NULL, createExtensionStmt);
CommandCounterIncrement();
}
/*
* AlterExtensionUpdateStmt builds and execute Alter extension statements
* per given extension name and updates extension verision
*/
void
AlterExtensionUpdateStmt(const char *extname, const char *extVersion)
{
AlterExtensionStmt *alterExtensionStmt = makeNode(AlterExtensionStmt);
/* set location to -1 as it is unknown */
int location = -1;
alterExtensionStmt->extname = extname;
if (extVersion == NULL)
{
ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT),
errmsg("alter extension \"%s\" should not be empty",
extVersion)));
}
Node *extensionVersionArg = (Node *) makeString(extVersion);
DefElem *extensionVersionElement = makeDefElem("new_version", extensionVersionArg,
location);
alterExtensionStmt->options = lappend(alterExtensionStmt->options,
extensionVersionElement);
ExecAlterExtensionStmt(NULL, alterExtensionStmt);
CommandCounterIncrement();
}

71
src/backend/distributed/commands/utility_hook.c
View File

@ -38,8 +38,10 @@
#endif
#include "catalog/catalog.h"
#include "catalog/dependency.h"
#include "citus_version.h"
#include "commands/dbcommands.h"
#include "commands/defrem.h"
#include "commands/extension.h"
#include "commands/tablecmds.h"
#include "distributed/adaptive_executor.h"
#include "distributed/colocation_utils.h"
@ -71,6 +73,7 @@
#include "lib/stringinfo.h"
#include "nodes/parsenodes.h"
#include "nodes/pg_list.h"
#include "nodes/makefuncs.h"
#include "tcop/utility.h"
#include "utils/builtins.h"
#include "utils/lsyscache.h"
@ -193,6 +196,20 @@ multi_ProcessUtility(PlannedStmt *pstmt,
ErrorIfUnstableCreateOrAlterExtensionStmt(parsetree);
}
if (IsA(parsetree, CreateExtensionStmt))
{
/*CREATE EXTENSION CITUS (version Z) */
CreateExtensionStmt *createExtensionStmt = castNode(CreateExtensionStmt,
parsetree);
if (strcmp(createExtensionStmt->extname, "citus") == 0)
{
if (get_extension_oid("citus_columnar", true) == InvalidOid)
{
CreateExtensionWithVersion("citus_columnar", NULL);
}
}
}
if (!CitusHasBeenLoaded())
{
/*
@ -589,11 +606,65 @@ ProcessUtilityInternal(PlannedStmt *pstmt,
if (isAlterExtensionUpdateCitusStmt)
{
citusCanBeUpdatedToAvailableVersion = !InstalledAndAvailableVersionsSame();
/*upgrade citus */
DefElem *newVersionValue = GetExtensionOption(
((AlterExtensionStmt *) parsetree)->options, "new_version");
Oid citusExtensionOid = get_extension_oid("citus", true);
char *curExtensionVersion = get_extension_version(citusExtensionOid);
Oid citusColumnarOid = get_extension_oid("citus_columnar", true);
if (newVersionValue)
{
const char *newVersion = defGetString(newVersionValue);
/*alter extension citus update to 11.2-1, and no citus_columnar installed */
if (strcmp(newVersion, "11.2-1") == 0 && citusColumnarOid == InvalidOid)
{
/*it's upgrade citus to 11.2-1 */
CreateExtensionWithVersion("citus_columnar", "11.1-0");
}
else if (strcmp(curExtensionVersion, "11.2-1") == 0 && citusColumnarOid !=
InvalidOid)
{
/*downgrade citus_columnar to Y */
AlterExtensionUpdateStmt("citus_columnar", "11.1-0");
}
}
}
standard_ProcessUtility_compat(pstmt, queryString, false, context,
params, queryEnv, dest, completionTag);
if (isAlterExtensionUpdateCitusStmt)
{
DefElem *newVersionValue = GetExtensionOption(
((AlterExtensionStmt *) parsetree)->options, "new_version");
Oid citusColumnarOid = get_extension_oid("citus_columnar", true);
if (newVersionValue)
{
const char *newVersion = defGetString(newVersionValue);
if (strcmp(newVersion, "11.2-1") == 0 && citusColumnarOid != InvalidOid)
{
/*after "ALTER EXTENSION citus" updates citus_columnar Y to version Z. */
char *curColumnarVersion = get_extension_version(citusColumnarOid);
if (strcmp(curColumnarVersion, "11.1-0") == 0)
{
AlterExtensionUpdateStmt("citus_columnar", "11.2-1");
}
}
else if (strcmp(newVersion, "11.1-1") == 0 && citusColumnarOid !=
InvalidOid)
{
/*after "ALTER EXTENSION citus" drops citus_columnar extension */
char *curColumnarVersion = get_extension_version(citusColumnarOid);
if (strcmp(curColumnarVersion, "11.1-0") == 0)
{
RemoveExtensionById(citusColumnarOid);
}
}
}
}
/*
* if we are running ALTER EXTENSION citus UPDATE (to "<version>") command, we may need
* to mark existing objects as distributed depending on the "version" parameter if

9
src/backend/distributed/sql/citus--10.0-1--10.0-2.sql
View File

@ -1,5 +1,12 @@
-- citus--10.0-1--10.0-2
#include "../../columnar/sql/columnar--10.0-1--10.0-2.sql"
--#include "../../columnar/sql/columnar--10.0-1--10.0-2.sql"
DO $check_columnar$
BEGIN
IF NOT EXISTS (select 1 from pg_extension where extname='citus_columnar') THEN
#include "../../columnar/sql/columnar--10.0-1--10.0-2.sql"
END IF;
END;
$check_columnar$;
GRANT SELECT ON public.citus_tables TO public;

9
src/backend/distributed/sql/citus--10.0-4--10.1-1.sql
View File

@ -9,7 +9,14 @@ INSERT INTO citus.pg_dist_object SELECT
0 as objsubid
ON CONFLICT DO NOTHING;
#include "../../columnar/sql/columnar--10.0-3--10.1-1.sql"
--#include "../../columnar/sql/columnar--10.0-3--10.1-1.sql"
DO $check_columnar$
BEGIN
IF NOT EXISTS (select 1 from pg_extension where extname='citus_columnar') THEN
#include "../../columnar/sql/columnar--10.0-3--10.1-1.sql"
END IF;
END;
$check_columnar$;
#include "udfs/create_distributed_table/10.1-1.sql";
#include "udfs/worker_partitioned_relation_total_size/10.1-1.sql"
#include "udfs/worker_partitioned_relation_size/10.1-1.sql"

10
src/backend/distributed/sql/citus--10.1-1--10.2-1.sql
View File

@ -9,7 +9,15 @@ GRANT ALL ON FUNCTION pg_catalog.worker_record_sequence_dependency(regclass,regc
ALTER TABLE pg_catalog.pg_dist_placement ADD CONSTRAINT placement_shardid_groupid_unique_index UNIQUE (shardid, groupid);
#include "udfs/stop_metadata_sync_to_node/10.2-1.sql"
#include "../../columnar/sql/columnar--10.1-1--10.2-1.sql"
--#include "../../columnar/sql/columnar--10.1-1--10.2-1.sql"
DO $check_columnar$
BEGIN
IF NOT EXISTS (select 1 from pg_extension where extname='citus_columnar') THEN
#include "../../columnar/sql/columnar--10.1-1--10.2-1.sql"
END IF;
END;
$check_columnar$;
#include "udfs/citus_internal_add_partition_metadata/10.2-1.sql";
#include "udfs/citus_internal_add_shard_metadata/10.2-1.sql";
#include "udfs/citus_internal_add_placement_metadata/10.2-1.sql";

9
src/backend/distributed/sql/citus--10.2-1--10.2-2.sql
View File

@ -2,4 +2,11 @@
-- bump version to 10.2-2
#include "../../columnar/sql/columnar--10.2-1--10.2-2.sql"
--#include "../../columnar/sql/columnar--10.2-1--10.2-2.sql"
DO $check_columnar$
BEGIN
IF NOT EXISTS (select 1 from pg_extension where extname='citus_columnar') THEN
#include "../../columnar/sql/columnar--10.2-1--10.2-2.sql"
END IF;
END;
$check_columnar$;

9
src/backend/distributed/sql/citus--10.2-2--10.2-3.sql
View File

@ -2,4 +2,11 @@
-- bump version to 10.2-3
#include "../../columnar/sql/columnar--10.2-2--10.2-3.sql"
--#include "../../columnar/sql/columnar--10.2-2--10.2-3.sql"
DO $check_columnar$
BEGIN
IF NOT EXISTS (select 1 from pg_extension where extname='citus_columnar') THEN
#include "../../columnar/sql/columnar--10.2-2--10.2-3.sql"
END IF;
END;
$check_columnar$;

9
src/backend/distributed/sql/citus--10.2-3--10.2-4.sql
View File

@ -2,7 +2,14 @@
-- bump version to 10.2-4
#include "../../columnar/sql/columnar--10.2-3--10.2-4.sql"
--#include "../../columnar/sql/columnar--10.2-3--10.2-4.sql"
DO $check_columnar$
BEGIN
IF NOT EXISTS (select 1 from pg_extension where extname='citus_columnar') THEN
#include "../../columnar/sql/columnar--10.2-3--10.2-4.sql"
END IF;
END;
$check_columnar$;
#include "udfs/fix_partition_shard_index_names/10.2-4.sql"
#include "udfs/fix_all_partition_shard_index_names/10.2-4.sql"

4
src/backend/distributed/sql/citus--11.1-1--11.1-2.sql
View File

@ -1,4 +0,0 @@
-- bump version to 11.1-2
--\echo Use "ALTER EXTENSION citus UPDATE TO '11.1-2'" to load this file. \quit
--do $$ begin raise log 'citus--11.1-1--11.1-2.sql'; end; $$;

38
src/backend/distributed/sql/citus--11.1-1--11.2-1.sql Normal file
View File

@ -0,0 +1,38 @@
-- bump version to 11.2-1
-- drop columnar objects if they exists in citus extension
DO $check_citus$
BEGIN
IF EXISTS (SELECT 1 FROM pg_catalog.pg_extension AS e
INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid)
INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid)
WHERE e.extname='citus' and p.proname = 'columnar_handler'
) THEN
ALTER EXTENSION citus DROP SCHEMA columnar;
ALTER EXTENSION citus DROP SEQUENCE columnar.storageid_seq;
-- columnar tables
ALTER EXTENSION citus DROP TABLE columnar.options;
ALTER EXTENSION citus DROP TABLE columnar.stripe;
ALTER EXTENSION citus DROP TABLE columnar.chunk_group;
ALTER EXTENSION citus DROP TABLE columnar.chunk;
DO $proc$
BEGIN
-- columnar functions
IF substring(current_Setting('server_version'), '\d+')::int >= 12 THEN
EXECUTE $$
ALTER EXTENSION citus DROP FUNCTION columnar.columnar_handler;
ALTER EXTENSION citus DROP ACCESS METHOD columnar;
ALTER EXTENSION citus DROP FUNCTION pg_catalog.alter_columnar_table_set;
ALTER EXTENSION citus DROP FUNCTION pg_catalog.alter_columnar_table_reset;
$$;
END IF;
END$proc$;
-- functions under citus_internal for columnar
ALTER EXTENSION citus DROP FUNCTION citus_internal.upgrade_columnar_storage;
ALTER EXTENSION citus DROP FUNCTION citus_internal.downgrade_columnar_storage;
ALTER EXTENSION citus DROP FUNCTION citus_internal.columnar_ensure_am_depends_catalog;
END IF;
END $check_citus$;

9
src/backend/distributed/sql/citus--9.5-1--10.0-4.sql
View File

@ -35,7 +35,14 @@ DROP FUNCTION IF EXISTS pg_catalog.citus_total_relation_size(regclass);
#include "udfs/worker_change_sequence_dependency/10.0-1.sql"
#include "udfs/remove_local_tables_from_metadata/10.0-1.sql"
#include "../../columnar/sql/columnar--9.5-1--10.0-1.sql"
--#include "../../columnar/sql/columnar--9.5-1--10.0-1.sql"
DO $check_columnar$
BEGIN
IF NOT EXISTS (select 1 from pg_extension where extname='citus_columnar') THEN
#include "../../columnar/sql/columnar--9.5-1--10.0-1.sql"
END IF;
END;
$check_columnar$;
#include "udfs/time_partition_range/10.0-1.sql"
#include "udfs/time_partitions/10.0-1.sql"

24
src/backend/distributed/sql/downgrades/citus--11.2-1--11.1-1.sql Normal file
View File

@ -0,0 +1,24 @@
-- add relations to citus
ALTER EXTENSION citus ADD SCHEMA columnar;
ALTER EXTENSION citus ADD SEQUENCE columnar.storageid_seq;
ALTER EXTENSION citus ADD TABLE columnar.options;
ALTER EXTENSION citus ADD TABLE columnar.stripe;
ALTER EXTENSION citus ADD TABLE columnar.chunk_group;
ALTER EXTENSION citus ADD TABLE columnar.chunk;
DO $proc$
BEGIN
-- columnar functions
IF substring(current_Setting('server_version'), '\d+')::int >= 12 THEN
EXECUTE $$
ALTER EXTENSION citus ADD FUNCTION columnar.columnar_handler;
ALTER EXTENSION citus ADD ACCESS METHOD columnar;
ALTER EXTENSION citus ADD FUNCTION pg_catalog.alter_columnar_table_set;
ALTER EXTENSION citus ADD FUNCTION pg_catalog.alter_columnar_table_reset;
$$;
END IF;
END$proc$;
ALTER EXTENSION citus ADD FUNCTION citus_internal.upgrade_columnar_storage;
ALTER EXTENSION citus ADD FUNCTION citus_internal.downgrade_columnar_storage;
ALTER EXTENSION citus ADD FUNCTION citus_internal.columnar_ensure_am_depends_catalog;

3
src/include/distributed/commands.h
View File

@ -227,7 +227,8 @@ extern ObjectAddress AlterExtensionSchemaStmtObjectAddress(Node *stmt,
bool missing_ok);
extern ObjectAddress AlterExtensionUpdateStmtObjectAddress(Node *stmt,
bool missing_ok);
extern void CreateExtensionWithVersion(const char *extname, const char *extVersion);
extern void AlterExtensionUpdateStmt(const char *extname, const char *extVersion);
/* foreign_constraint.c - forward declarations */
extern bool ConstraintIsAForeignKeyToReferenceTable(char *constraintName,