external
/
citus
mirror of https://github.com/citusdata/citus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Columnar: pg_upgrade support (#4354)

pull/4338/head
Hadi Moshayedi 2020-12-02 08:46:59 -08:00 committed by GitHub
parent 27113255e5
commit c2f60b6422
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 483 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
src/backend/columnar/cstore_metadata_tables.c
View File

@ -155,19 +155,26 @@ typedef FormData_cstore_options *Form_cstore_options;
/*
* InitColumnarOptions initialized the columnar table options. Meaning it writes the
* default options to the options table if not already existing.
*
* The return value indicates if options have actually been written.
*/
bool
void
InitColumnarOptions(Oid regclass)
{
/*
* When upgrading we retain options for all columnar tables by upgrading
* "cstore.options" catalog table, so we shouldn't do anything here.
*/
if (IsBinaryUpgrade)
{
return;
}
ColumnarOptions defaultOptions = {
.blockRowCount = cstore_block_row_count,
.stripeRowCount = cstore_stripe_row_count,
.compressionType = cstore_compression,
};
return WriteColumnarOptions(regclass, &defaultOptions, false);
WriteColumnarOptions(regclass, &defaultOptions, false);
}
@ -194,6 +201,12 @@ SetColumnarOptions(Oid regclass, ColumnarOptions *options)
static bool
WriteColumnarOptions(Oid regclass, ColumnarOptions *options, bool overwrite)
{
/*
* When upgrading we should retain the options from the previous
* cluster and don't write new options.
*/
Assert(!IsBinaryUpgrade);
bool written = false;
bool nulls[Natts_cstore_options] = { 0 };
@ -272,6 +285,12 @@ DeleteColumnarTableOptions(Oid regclass, bool missingOk)
{
bool result = false;
/*
* When upgrading we shouldn't delete or modify table options and
* retain options from the previous cluster.
*/
Assert(!IsBinaryUpgrade);
Relation columnarOptions = relation_open(ColumnarOptionsRelationId(),
RowExclusiveLock);
@ -720,6 +739,7 @@ ReadDataFileStripeList(uint64 storageId, Snapshot snapshot)
BTEqualStrategyNumber, F_OIDEQ, Int32GetDatum(storageId));
Oid cstoreStripesOid = CStoreStripesRelationId();
Relation cstoreStripes = heap_open(cstoreStripesOid, AccessShareLock);
Relation index = index_open(CStoreStripesIndexRelationId(), AccessShareLock);
TupleDesc tupleDescriptor = RelationGetDescr(cstoreStripes);
@ -1125,6 +1145,13 @@ ReadMetapage(RelFileNode relfilenode, bool missingOk)
static ColumnarMetapage *
InitMetapage(Relation relation)
{
/*
* If we init metapage during upgrade, we might override the
* pre-upgrade storage id which will render pre-upgrade data
* invisible.
*/
Assert(!IsBinaryUpgrade);
ColumnarMetapage *metapage = palloc0(sizeof(ColumnarMetapage));
metapage->storageId = GetNextStorageId();
metapage->versionMajor = CSTORE_VERSION_MAJOR;

2
src/backend/columnar/sql/columnar--9.5-1--10.0-1.sql
View File

@ -10,7 +10,7 @@ CREATE TABLE options (
block_row_count int NOT NULL,
stripe_row_count int NOT NULL,
compression name NOT NULL
);
) WITH (user_catalog_table = true);
COMMENT ON TABLE options IS 'columnar table specific options, maintained by alter_columnar_table_set';

2
src/include/columnar/cstore.h
View File

@ -285,7 +285,7 @@ extern StringInfo DecompressBuffer(StringInfo buffer, CompressionType compressio
extern char * CompressionTypeStr(CompressionType type);
/* cstore_metadata_tables.c */
extern bool InitColumnarOptions(Oid regclass);
extern void InitColumnarOptions(Oid regclass);
extern void SetColumnarOptions(Oid regclass, ColumnarOptions *options);
extern bool DeleteColumnarTableOptions(Oid regclass, bool missingOk);
extern bool ReadColumnarOptions(Oid regclass, ColumnarOptions *options);

2
src/test/regress/after_pg_upgrade_schedule
View File

@ -1 +1 @@
test: upgrade_basic_after upgrade_type_after upgrade_ref2ref_after upgrade_distributed_function_after upgrade_rebalance_strategy_after upgrade_list_citus_objects
test: upgrade_basic_after upgrade_columnar_after upgrade_type_after upgrade_ref2ref_after upgrade_distributed_function_after upgrade_rebalance_strategy_after upgrade_list_citus_objects

1
src/test/regress/before_pg_upgrade_schedule
View File

@ -2,4 +2,5 @@
test: multi_test_helpers multi_test_helpers_superuser
test: multi_test_catalog_views
test: upgrade_basic_before
test: upgrade_columnar_before
test: upgrade_type_before upgrade_ref2ref_before upgrade_distributed_function_before upgrade_rebalance_strategy_before

141
src/test/regress/expected/upgrade_columnar_after.out Normal file
View File

@ -0,0 +1,141 @@
SHOW server_version \gset
SELECT substring(:'server_version', '\d+')::int > 12 AS server_version_above_eleven
\gset
\if :server_version_above_eleven
\else
\q
\endif
SET search_path TO upgrade_columnar, public;
-- test we retained data
SELECT * FROM test_retains_data ORDER BY a;
a | b | c | d
---------------------------------------------------------------------
1 | abc | (1,4) | {1,2,3,4}
2 | pi | (3,192) | {3,1,4,1,5}
3 | earth | (4,22) | {1,2,7,5,6}
(3 rows)
SELECT count(*) FROM test_retains_data;
count
---------------------------------------------------------------------
3
(1 row)
SELECT a,c FROM test_retains_data ORDER BY a;
a | c
---------------------------------------------------------------------
1 | (1,4)
2 | (3,192)
3 | (4,22)
(3 rows)
SELECT b,d FROM test_retains_data ORDER BY a;
b | d
---------------------------------------------------------------------
abc | {1,2,3,4}
pi | {3,1,4,1,5}
earth | {1,2,7,5,6}
(3 rows)
SELECT * FROM test_retains_data ORDER BY a;
a | b | c | d
---------------------------------------------------------------------
1 | abc | (1,4) | {1,2,3,4}
2 | pi | (3,192) | {3,1,4,1,5}
3 | earth | (4,22) | {1,2,7,5,6}
(3 rows)
-- test we retained data with a once truncated table
SELECT * FROM test_truncated ORDER BY a;
a
---------------------------------------------------------------------
11
12
13
(3 rows)
-- test we retained data with a once vacuum fulled table
SELECT * FROM test_vacuum_full ORDER BY a;
a
---------------------------------------------------------------------
1
2
3
4
5
6
7
8
9
10
11
12
13
(13 rows)
-- test we retained data with a once alter typed table
SELECT * FROM test_alter_type ORDER BY a;
a
---------------------------------------------------------------------
1
10
11
12
13
2
3
4
5
6
7
8
9
(13 rows)
-- test we retained data with a once refreshed materialized view
SELECT * FROM matview ORDER BY a;
a | b
---------------------------------------------------------------------
0 | {abc,pi}
1 | {earth}
(2 rows)
-- test we retained options
SELECT * FROM cstore.options WHERE regclass = 'test_options_1'::regclass;
regclass | block_row_count | stripe_row_count | compression
---------------------------------------------------------------------
test_options_1 | 1000 | 5000 | pglz
(1 row)
VACUUM VERBOSE test_options_1;
INFO: statistics for "test_options_1":
storage id: xxxxx
total file size: 65536, total data size: 43136
total row count: 10000, stripe count: 2, average rows per stripe: 5000
block count: 20, containing data for dropped columns: 0, none compressed: 10, pglz compressed: 10
SELECT count(*), sum(a), sum(b) FROM test_options_1;
count | sum | sum
---------------------------------------------------------------------
10000 | 50005000 | 45010
(1 row)
SELECT * FROM cstore.options WHERE regclass = 'test_options_2'::regclass;
regclass | block_row_count | stripe_row_count | compression
---------------------------------------------------------------------
test_options_2 | 2000 | 6000 | none
(1 row)
VACUUM VERBOSE test_options_2;
INFO: statistics for "test_options_2":
storage id: xxxxx
total file size: 163840, total data size: 125636
total row count: 20000, stripe count: 4, average rows per stripe: 5000
block count: 30, containing data for dropped columns: 0, none compressed: 20, pglz compressed: 10
SELECT count(*), sum(a), sum(b) FROM test_options_2;
count | sum | sum
---------------------------------------------------------------------
20000 | 100010000 | 65015
(1 row)

6
src/test/regress/expected/upgrade_columnar_after_0.out Normal file
View File

@ -0,0 +1,6 @@
SHOW server_version \gset
SELECT substring(:'server_version', '\d+')::int > 12 AS server_version_above_eleven
\gset
\if :server_version_above_eleven
\else
\q

143
src/test/regress/expected/upgrade_columnar_before.out Normal file
View File

@ -0,0 +1,143 @@
SHOW server_version \gset
SELECT substring(:'server_version', '\d+')::int > 11 AS server_version_above_eleven
\gset
\if :server_version_above_eleven
\else
\q
\endif
CREATE SCHEMA upgrade_columnar;
SET search_path TO upgrade_columnar, public;
CREATE TYPE compfoo AS (f1 int, f2 text);
CREATE TABLE test_retains_data (a int, b text, c compfoo, d int[]) USING columnar;
INSERT INTO test_retains_data VALUES
(1, 'abc', (1, '4'), ARRAY[1,2,3,4]),
(2, 'pi', (3, '192'), ARRAY[3,1,4,1,5]),
(3, 'earth', (4, '22'), ARRAY[1,2,7,5,6]);
--
-- Verify that after upgrade we can read data for tables whose
-- relfilenode has changed before upgrade.
--
-- truncate
CREATE TABLE test_truncated (a int) USING columnar;
INSERT INTO test_truncated SELECT * FROM generate_series(1, 10);
SELECT count(*) FROM test_truncated;
count
---------------------------------------------------------------------
10
(1 row)
SELECT relfilenode AS relfilenode_pre_truncate
FROM pg_class WHERE oid = 'test_truncated'::regclass::oid \gset
TRUNCATE test_truncated;
SELECT relfilenode AS relfilenode_post_truncate
FROM pg_class WHERE oid = 'test_truncated'::regclass::oid \gset
SELECT :relfilenode_post_truncate <> :relfilenode_pre_truncate AS relfilenode_changed;
relfilenode_changed
---------------------------------------------------------------------
t
(1 row)
INSERT INTO test_truncated SELECT * FROM generate_series(11, 13);
SELECT count(*) FROM test_truncated;
count
---------------------------------------------------------------------
3
(1 row)
-- vacuum full
CREATE TABLE test_vacuum_full (a int) USING columnar;
INSERT INTO test_vacuum_full SELECT * FROM generate_series(1, 10);
SELECT count(*) FROM test_vacuum_full;
count
---------------------------------------------------------------------
10
(1 row)
SELECT relfilenode AS relfilenode_pre_vacuum_full
FROM pg_class WHERE oid = 'test_vacuum_full'::regclass::oid \gset
VACUUM FULL test_vacuum_full;
SELECT relfilenode AS relfilenode_post_vacuum_full
FROM pg_class WHERE oid = 'test_vacuum_full'::regclass::oid \gset
SELECT :relfilenode_post_vacuum_full <> :relfilenode_pre_vacuum_full AS relfilenode_changed;
relfilenode_changed
---------------------------------------------------------------------
t
(1 row)
INSERT INTO test_vacuum_full SELECT * FROM generate_series(11, 13);
SELECT count(*) FROM test_vacuum_full;
count
---------------------------------------------------------------------
13
(1 row)
-- alter column type
CREATE TABLE test_alter_type (a int) USING columnar;
INSERT INTO test_alter_type SELECT * FROM generate_series(1, 10);
SELECT count(*) FROM test_alter_type;
count
---------------------------------------------------------------------
10
(1 row)
SELECT relfilenode AS relfilenode_pre_alter
FROM pg_class WHERE oid = 'test_alter_type'::regclass::oid \gset
ALTER TABLE test_alter_type ALTER COLUMN a TYPE text;
SELECT relfilenode AS relfilenode_post_alter
FROM pg_class WHERE oid = 'test_alter_type'::regclass::oid \gset
SELECT :relfilenode_pre_alter <> :relfilenode_post_alter AS relfilenode_changed;
relfilenode_changed
---------------------------------------------------------------------
t
(1 row)
INSERT INTO test_alter_type SELECT * FROM generate_series(11, 13);
SELECT count(*) FROM test_alter_type;
count
---------------------------------------------------------------------
13
(1 row)
-- materialized view
CREATE MATERIALIZED VIEW matview(a, b) USING columnar AS
SELECT floor(a/3), array_agg(b) FROM test_retains_data GROUP BY 1;
SELECT relfilenode AS relfilenode_pre_refresh
FROM pg_class WHERE oid = 'matview'::regclass::oid \gset
REFRESH MATERIALIZED VIEW matview;
SELECT relfilenode AS relfilenode_post_refresh
FROM pg_class WHERE oid = 'matview'::regclass::oid \gset
SELECT :relfilenode_pre_alter <> :relfilenode_post_alter AS relfilenode_changed;
relfilenode_changed
---------------------------------------------------------------------
t
(1 row)
--
-- Test that we retain options
--
SET cstore.stripe_row_count TO 5000;
SET cstore.block_row_count TO 1000;
SET cstore.compression TO 'pglz';
CREATE TABLE test_options_1(a int, b int) USING columnar;
INSERT INTO test_options_1 SELECT i, floor(i/1000) FROM generate_series(1, 10000) i;
CREATE TABLE test_options_2(a int, b int) USING columnar;
INSERT INTO test_options_2 SELECT i, floor(i/1000) FROM generate_series(1, 10000) i;
SELECT alter_columnar_table_set('test_options_2', block_row_count => 2000);
alter_columnar_table_set
---------------------------------------------------------------------
(1 row)
SELECT alter_columnar_table_set('test_options_2', stripe_row_count => 6000);
alter_columnar_table_set
---------------------------------------------------------------------
(1 row)
SELECT alter_columnar_table_set('test_options_2', compression => 'none');
alter_columnar_table_set
---------------------------------------------------------------------
(1 row)
INSERT INTO test_options_2 SELECT i, floor(i/2000) FROM generate_series(1, 10000) i;

6
src/test/regress/expected/upgrade_columnar_before_0.out Normal file
View File

@ -0,0 +1,6 @@
SHOW server_version \gset
SELECT substring(:'server_version', '\d+')::int > 11 AS server_version_above_eleven
\gset
\if :server_version_above_eleven
\else
\q

41
src/test/regress/sql/upgrade_columnar_after.sql Normal file
View File

@ -0,0 +1,41 @@
SHOW server_version \gset
SELECT substring(:'server_version', '\d+')::int > 12 AS server_version_above_eleven
\gset
\if :server_version_above_eleven
\else
\q
\endif
SET search_path TO upgrade_columnar, public;
-- test we retained data
SELECT * FROM test_retains_data ORDER BY a;
SELECT count(*) FROM test_retains_data;
SELECT a,c FROM test_retains_data ORDER BY a;
SELECT b,d FROM test_retains_data ORDER BY a;
SELECT * FROM test_retains_data ORDER BY a;
-- test we retained data with a once truncated table
SELECT * FROM test_truncated ORDER BY a;
-- test we retained data with a once vacuum fulled table
SELECT * FROM test_vacuum_full ORDER BY a;
-- test we retained data with a once alter typed table
SELECT * FROM test_alter_type ORDER BY a;
-- test we retained data with a once refreshed materialized view
SELECT * FROM matview ORDER BY a;
-- test we retained options
SELECT * FROM cstore.options WHERE regclass = 'test_options_1'::regclass;
VACUUM VERBOSE test_options_1;
SELECT count(*), sum(a), sum(b) FROM test_options_1;
SELECT * FROM cstore.options WHERE regclass = 'test_options_2'::regclass;
VACUUM VERBOSE test_options_2;
SELECT count(*), sum(a), sum(b) FROM test_options_2;

111
src/test/regress/sql/upgrade_columnar_before.sql Normal file
View File

@ -0,0 +1,111 @@
SHOW server_version \gset
SELECT substring(:'server_version', '\d+')::int > 11 AS server_version_above_eleven
\gset
\if :server_version_above_eleven
\else
\q
\endif
CREATE SCHEMA upgrade_columnar;
SET search_path TO upgrade_columnar, public;
CREATE TYPE compfoo AS (f1 int, f2 text);
CREATE TABLE test_retains_data (a int, b text, c compfoo, d int[]) USING columnar;
INSERT INTO test_retains_data VALUES
(1, 'abc', (1, '4'), ARRAY[1,2,3,4]),
(2, 'pi', (3, '192'), ARRAY[3,1,4,1,5]),
(3, 'earth', (4, '22'), ARRAY[1,2,7,5,6]);
--
-- Verify that after upgrade we can read data for tables whose
-- relfilenode has changed before upgrade.
--
-- truncate
CREATE TABLE test_truncated (a int) USING columnar;
INSERT INTO test_truncated SELECT * FROM generate_series(1, 10);
SELECT count(*) FROM test_truncated;
SELECT relfilenode AS relfilenode_pre_truncate
FROM pg_class WHERE oid = 'test_truncated'::regclass::oid \gset
TRUNCATE test_truncated;
SELECT relfilenode AS relfilenode_post_truncate
FROM pg_class WHERE oid = 'test_truncated'::regclass::oid \gset
SELECT :relfilenode_post_truncate <> :relfilenode_pre_truncate AS relfilenode_changed;
INSERT INTO test_truncated SELECT * FROM generate_series(11, 13);
SELECT count(*) FROM test_truncated;
-- vacuum full
CREATE TABLE test_vacuum_full (a int) USING columnar;
INSERT INTO test_vacuum_full SELECT * FROM generate_series(1, 10);
SELECT count(*) FROM test_vacuum_full;
SELECT relfilenode AS relfilenode_pre_vacuum_full
FROM pg_class WHERE oid = 'test_vacuum_full'::regclass::oid \gset
VACUUM FULL test_vacuum_full;
SELECT relfilenode AS relfilenode_post_vacuum_full
FROM pg_class WHERE oid = 'test_vacuum_full'::regclass::oid \gset
SELECT :relfilenode_post_vacuum_full <> :relfilenode_pre_vacuum_full AS relfilenode_changed;
INSERT INTO test_vacuum_full SELECT * FROM generate_series(11, 13);
SELECT count(*) FROM test_vacuum_full;
-- alter column type
CREATE TABLE test_alter_type (a int) USING columnar;
INSERT INTO test_alter_type SELECT * FROM generate_series(1, 10);
SELECT count(*) FROM test_alter_type;
SELECT relfilenode AS relfilenode_pre_alter
FROM pg_class WHERE oid = 'test_alter_type'::regclass::oid \gset
ALTER TABLE test_alter_type ALTER COLUMN a TYPE text;
SELECT relfilenode AS relfilenode_post_alter
FROM pg_class WHERE oid = 'test_alter_type'::regclass::oid \gset
SELECT :relfilenode_pre_alter <> :relfilenode_post_alter AS relfilenode_changed;
INSERT INTO test_alter_type SELECT * FROM generate_series(11, 13);
SELECT count(*) FROM test_alter_type;
-- materialized view
CREATE MATERIALIZED VIEW matview(a, b) USING columnar AS
SELECT floor(a/3), array_agg(b) FROM test_retains_data GROUP BY 1;
SELECT relfilenode AS relfilenode_pre_refresh
FROM pg_class WHERE oid = 'matview'::regclass::oid \gset
REFRESH MATERIALIZED VIEW matview;
SELECT relfilenode AS relfilenode_post_refresh
FROM pg_class WHERE oid = 'matview'::regclass::oid \gset
SELECT :relfilenode_pre_alter <> :relfilenode_post_alter AS relfilenode_changed;
--
-- Test that we retain options
--
SET cstore.stripe_row_count TO 5000;
SET cstore.block_row_count TO 1000;
SET cstore.compression TO 'pglz';
CREATE TABLE test_options_1(a int, b int) USING columnar;
INSERT INTO test_options_1 SELECT i, floor(i/1000) FROM generate_series(1, 10000) i;
CREATE TABLE test_options_2(a int, b int) USING columnar;
INSERT INTO test_options_2 SELECT i, floor(i/1000) FROM generate_series(1, 10000) i;
SELECT alter_columnar_table_set('test_options_2', block_row_count => 2000);
SELECT alter_columnar_table_set('test_options_2', stripe_row_count => 6000);
SELECT alter_columnar_table_set('test_options_2', compression => 'none');
INSERT INTO test_options_2 SELECT i, floor(i/2000) FROM generate_series(1, 10000) i;