diff --git a/src/backend/columnar/cstore_metadata_tables.c b/src/backend/columnar/cstore_metadata_tables.c index a2bf8b85c..aa62cba00 100644 --- a/src/backend/columnar/cstore_metadata_tables.c +++ b/src/backend/columnar/cstore_metadata_tables.c @@ -155,19 +155,26 @@ typedef FormData_cstore_options *Form_cstore_options; /* * InitColumnarOptions initialized the columnar table options. Meaning it writes the * default options to the options table if not already existing. - * - * The return value indicates if options have actually been written. */ -bool +void InitColumnarOptions(Oid regclass) { + /* + * When upgrading we retain options for all columnar tables by upgrading + * "cstore.options" catalog table, so we shouldn't do anything here. + */ + if (IsBinaryUpgrade) + { + return; + } + ColumnarOptions defaultOptions = { .blockRowCount = cstore_block_row_count, .stripeRowCount = cstore_stripe_row_count, .compressionType = cstore_compression, }; - return WriteColumnarOptions(regclass, &defaultOptions, false); + WriteColumnarOptions(regclass, &defaultOptions, false); } @@ -194,6 +201,12 @@ SetColumnarOptions(Oid regclass, ColumnarOptions *options) static bool WriteColumnarOptions(Oid regclass, ColumnarOptions *options, bool overwrite) { + /* + * When upgrading we should retain the options from the previous + * cluster and don't write new options. + */ + Assert(!IsBinaryUpgrade); + bool written = false; bool nulls[Natts_cstore_options] = { 0 }; @@ -272,6 +285,12 @@ DeleteColumnarTableOptions(Oid regclass, bool missingOk) { bool result = false; + /* + * When upgrading we shouldn't delete or modify table options and + * retain options from the previous cluster. + */ + Assert(!IsBinaryUpgrade); + Relation columnarOptions = relation_open(ColumnarOptionsRelationId(), RowExclusiveLock); @@ -720,6 +739,7 @@ ReadDataFileStripeList(uint64 storageId, Snapshot snapshot) BTEqualStrategyNumber, F_OIDEQ, Int32GetDatum(storageId)); Oid cstoreStripesOid = CStoreStripesRelationId(); + Relation cstoreStripes = heap_open(cstoreStripesOid, AccessShareLock); Relation index = index_open(CStoreStripesIndexRelationId(), AccessShareLock); TupleDesc tupleDescriptor = RelationGetDescr(cstoreStripes); @@ -1125,6 +1145,13 @@ ReadMetapage(RelFileNode relfilenode, bool missingOk) static ColumnarMetapage * InitMetapage(Relation relation) { + /* + * If we init metapage during upgrade, we might override the + * pre-upgrade storage id which will render pre-upgrade data + * invisible. + */ + Assert(!IsBinaryUpgrade); + ColumnarMetapage *metapage = palloc0(sizeof(ColumnarMetapage)); metapage->storageId = GetNextStorageId(); metapage->versionMajor = CSTORE_VERSION_MAJOR; diff --git a/src/backend/columnar/sql/columnar--9.5-1--10.0-1.sql b/src/backend/columnar/sql/columnar--9.5-1--10.0-1.sql index 23a20796d..dae08dbc5 100644 --- a/src/backend/columnar/sql/columnar--9.5-1--10.0-1.sql +++ b/src/backend/columnar/sql/columnar--9.5-1--10.0-1.sql @@ -10,7 +10,7 @@ CREATE TABLE options ( block_row_count int NOT NULL, stripe_row_count int NOT NULL, compression name NOT NULL -); +) WITH (user_catalog_table = true); COMMENT ON TABLE options IS 'columnar table specific options, maintained by alter_columnar_table_set'; diff --git a/src/include/columnar/cstore.h b/src/include/columnar/cstore.h index 71be3a2c6..5ab12462b 100644 --- a/src/include/columnar/cstore.h +++ b/src/include/columnar/cstore.h @@ -285,7 +285,7 @@ extern StringInfo DecompressBuffer(StringInfo buffer, CompressionType compressio extern char * CompressionTypeStr(CompressionType type); /* cstore_metadata_tables.c */ -extern bool InitColumnarOptions(Oid regclass); +extern void InitColumnarOptions(Oid regclass); extern void SetColumnarOptions(Oid regclass, ColumnarOptions *options); extern bool DeleteColumnarTableOptions(Oid regclass, bool missingOk); extern bool ReadColumnarOptions(Oid regclass, ColumnarOptions *options); diff --git a/src/test/regress/after_pg_upgrade_schedule b/src/test/regress/after_pg_upgrade_schedule index 1d4d45039..8e3a87fbe 100644 --- a/src/test/regress/after_pg_upgrade_schedule +++ b/src/test/regress/after_pg_upgrade_schedule @@ -1 +1 @@ -test: upgrade_basic_after upgrade_type_after upgrade_ref2ref_after upgrade_distributed_function_after upgrade_rebalance_strategy_after upgrade_list_citus_objects +test: upgrade_basic_after upgrade_columnar_after upgrade_type_after upgrade_ref2ref_after upgrade_distributed_function_after upgrade_rebalance_strategy_after upgrade_list_citus_objects diff --git a/src/test/regress/before_pg_upgrade_schedule b/src/test/regress/before_pg_upgrade_schedule index 10b90dd3e..d340ce62c 100644 --- a/src/test/regress/before_pg_upgrade_schedule +++ b/src/test/regress/before_pg_upgrade_schedule @@ -2,4 +2,5 @@ test: multi_test_helpers multi_test_helpers_superuser test: multi_test_catalog_views test: upgrade_basic_before +test: upgrade_columnar_before test: upgrade_type_before upgrade_ref2ref_before upgrade_distributed_function_before upgrade_rebalance_strategy_before diff --git a/src/test/regress/expected/upgrade_columnar_after.out b/src/test/regress/expected/upgrade_columnar_after.out new file mode 100644 index 000000000..044fc6a66 --- /dev/null +++ b/src/test/regress/expected/upgrade_columnar_after.out @@ -0,0 +1,141 @@ +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int > 12 AS server_version_above_eleven +\gset +\if :server_version_above_eleven +\else +\q +\endif +SET search_path TO upgrade_columnar, public; +-- test we retained data +SELECT * FROM test_retains_data ORDER BY a; + a | b | c | d +--------------------------------------------------------------------- + 1 | abc | (1,4) | {1,2,3,4} + 2 | pi | (3,192) | {3,1,4,1,5} + 3 | earth | (4,22) | {1,2,7,5,6} +(3 rows) + +SELECT count(*) FROM test_retains_data; + count +--------------------------------------------------------------------- + 3 +(1 row) + +SELECT a,c FROM test_retains_data ORDER BY a; + a | c +--------------------------------------------------------------------- + 1 | (1,4) + 2 | (3,192) + 3 | (4,22) +(3 rows) + +SELECT b,d FROM test_retains_data ORDER BY a; + b | d +--------------------------------------------------------------------- + abc | {1,2,3,4} + pi | {3,1,4,1,5} + earth | {1,2,7,5,6} +(3 rows) + +SELECT * FROM test_retains_data ORDER BY a; + a | b | c | d +--------------------------------------------------------------------- + 1 | abc | (1,4) | {1,2,3,4} + 2 | pi | (3,192) | {3,1,4,1,5} + 3 | earth | (4,22) | {1,2,7,5,6} +(3 rows) + +-- test we retained data with a once truncated table +SELECT * FROM test_truncated ORDER BY a; + a +--------------------------------------------------------------------- + 11 + 12 + 13 +(3 rows) + +-- test we retained data with a once vacuum fulled table +SELECT * FROM test_vacuum_full ORDER BY a; + a +--------------------------------------------------------------------- + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12 + 13 +(13 rows) + +-- test we retained data with a once alter typed table +SELECT * FROM test_alter_type ORDER BY a; + a +--------------------------------------------------------------------- + 1 + 10 + 11 + 12 + 13 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 +(13 rows) + +-- test we retained data with a once refreshed materialized view +SELECT * FROM matview ORDER BY a; + a | b +--------------------------------------------------------------------- + 0 | {abc,pi} + 1 | {earth} +(2 rows) + +-- test we retained options +SELECT * FROM cstore.options WHERE regclass = 'test_options_1'::regclass; + regclass | block_row_count | stripe_row_count | compression +--------------------------------------------------------------------- + test_options_1 | 1000 | 5000 | pglz +(1 row) + +VACUUM VERBOSE test_options_1; +INFO: statistics for "test_options_1": +storage id: xxxxx +total file size: 65536, total data size: 43136 +total row count: 10000, stripe count: 2, average rows per stripe: 5000 +block count: 20, containing data for dropped columns: 0, none compressed: 10, pglz compressed: 10 + +SELECT count(*), sum(a), sum(b) FROM test_options_1; + count | sum | sum +--------------------------------------------------------------------- + 10000 | 50005000 | 45010 +(1 row) + +SELECT * FROM cstore.options WHERE regclass = 'test_options_2'::regclass; + regclass | block_row_count | stripe_row_count | compression +--------------------------------------------------------------------- + test_options_2 | 2000 | 6000 | none +(1 row) + +VACUUM VERBOSE test_options_2; +INFO: statistics for "test_options_2": +storage id: xxxxx +total file size: 163840, total data size: 125636 +total row count: 20000, stripe count: 4, average rows per stripe: 5000 +block count: 30, containing data for dropped columns: 0, none compressed: 20, pglz compressed: 10 + +SELECT count(*), sum(a), sum(b) FROM test_options_2; + count | sum | sum +--------------------------------------------------------------------- + 20000 | 100010000 | 65015 +(1 row) + diff --git a/src/test/regress/expected/upgrade_columnar_after_0.out b/src/test/regress/expected/upgrade_columnar_after_0.out new file mode 100644 index 000000000..3f4eaf233 --- /dev/null +++ b/src/test/regress/expected/upgrade_columnar_after_0.out @@ -0,0 +1,6 @@ +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int > 12 AS server_version_above_eleven +\gset +\if :server_version_above_eleven +\else +\q diff --git a/src/test/regress/expected/upgrade_columnar_before.out b/src/test/regress/expected/upgrade_columnar_before.out new file mode 100644 index 000000000..302d9fbb4 --- /dev/null +++ b/src/test/regress/expected/upgrade_columnar_before.out @@ -0,0 +1,143 @@ +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int > 11 AS server_version_above_eleven +\gset +\if :server_version_above_eleven +\else +\q +\endif +CREATE SCHEMA upgrade_columnar; +SET search_path TO upgrade_columnar, public; +CREATE TYPE compfoo AS (f1 int, f2 text); +CREATE TABLE test_retains_data (a int, b text, c compfoo, d int[]) USING columnar; +INSERT INTO test_retains_data VALUES + (1, 'abc', (1, '4'), ARRAY[1,2,3,4]), + (2, 'pi', (3, '192'), ARRAY[3,1,4,1,5]), + (3, 'earth', (4, '22'), ARRAY[1,2,7,5,6]); +-- +-- Verify that after upgrade we can read data for tables whose +-- relfilenode has changed before upgrade. +-- +-- truncate +CREATE TABLE test_truncated (a int) USING columnar; +INSERT INTO test_truncated SELECT * FROM generate_series(1, 10); +SELECT count(*) FROM test_truncated; + count +--------------------------------------------------------------------- + 10 +(1 row) + +SELECT relfilenode AS relfilenode_pre_truncate +FROM pg_class WHERE oid = 'test_truncated'::regclass::oid \gset +TRUNCATE test_truncated; +SELECT relfilenode AS relfilenode_post_truncate +FROM pg_class WHERE oid = 'test_truncated'::regclass::oid \gset +SELECT :relfilenode_post_truncate <> :relfilenode_pre_truncate AS relfilenode_changed; + relfilenode_changed +--------------------------------------------------------------------- + t +(1 row) + +INSERT INTO test_truncated SELECT * FROM generate_series(11, 13); +SELECT count(*) FROM test_truncated; + count +--------------------------------------------------------------------- + 3 +(1 row) + +-- vacuum full +CREATE TABLE test_vacuum_full (a int) USING columnar; +INSERT INTO test_vacuum_full SELECT * FROM generate_series(1, 10); +SELECT count(*) FROM test_vacuum_full; + count +--------------------------------------------------------------------- + 10 +(1 row) + +SELECT relfilenode AS relfilenode_pre_vacuum_full +FROM pg_class WHERE oid = 'test_vacuum_full'::regclass::oid \gset +VACUUM FULL test_vacuum_full; +SELECT relfilenode AS relfilenode_post_vacuum_full +FROM pg_class WHERE oid = 'test_vacuum_full'::regclass::oid \gset +SELECT :relfilenode_post_vacuum_full <> :relfilenode_pre_vacuum_full AS relfilenode_changed; + relfilenode_changed +--------------------------------------------------------------------- + t +(1 row) + +INSERT INTO test_vacuum_full SELECT * FROM generate_series(11, 13); +SELECT count(*) FROM test_vacuum_full; + count +--------------------------------------------------------------------- + 13 +(1 row) + +-- alter column type +CREATE TABLE test_alter_type (a int) USING columnar; +INSERT INTO test_alter_type SELECT * FROM generate_series(1, 10); +SELECT count(*) FROM test_alter_type; + count +--------------------------------------------------------------------- + 10 +(1 row) + +SELECT relfilenode AS relfilenode_pre_alter +FROM pg_class WHERE oid = 'test_alter_type'::regclass::oid \gset +ALTER TABLE test_alter_type ALTER COLUMN a TYPE text; +SELECT relfilenode AS relfilenode_post_alter +FROM pg_class WHERE oid = 'test_alter_type'::regclass::oid \gset +SELECT :relfilenode_pre_alter <> :relfilenode_post_alter AS relfilenode_changed; + relfilenode_changed +--------------------------------------------------------------------- + t +(1 row) + +INSERT INTO test_alter_type SELECT * FROM generate_series(11, 13); +SELECT count(*) FROM test_alter_type; + count +--------------------------------------------------------------------- + 13 +(1 row) + +-- materialized view +CREATE MATERIALIZED VIEW matview(a, b) USING columnar AS +SELECT floor(a/3), array_agg(b) FROM test_retains_data GROUP BY 1; +SELECT relfilenode AS relfilenode_pre_refresh +FROM pg_class WHERE oid = 'matview'::regclass::oid \gset +REFRESH MATERIALIZED VIEW matview; +SELECT relfilenode AS relfilenode_post_refresh +FROM pg_class WHERE oid = 'matview'::regclass::oid \gset +SELECT :relfilenode_pre_alter <> :relfilenode_post_alter AS relfilenode_changed; + relfilenode_changed +--------------------------------------------------------------------- + t +(1 row) + +-- +-- Test that we retain options +-- +SET cstore.stripe_row_count TO 5000; +SET cstore.block_row_count TO 1000; +SET cstore.compression TO 'pglz'; +CREATE TABLE test_options_1(a int, b int) USING columnar; +INSERT INTO test_options_1 SELECT i, floor(i/1000) FROM generate_series(1, 10000) i; +CREATE TABLE test_options_2(a int, b int) USING columnar; +INSERT INTO test_options_2 SELECT i, floor(i/1000) FROM generate_series(1, 10000) i; +SELECT alter_columnar_table_set('test_options_2', block_row_count => 2000); + alter_columnar_table_set +--------------------------------------------------------------------- + +(1 row) + +SELECT alter_columnar_table_set('test_options_2', stripe_row_count => 6000); + alter_columnar_table_set +--------------------------------------------------------------------- + +(1 row) + +SELECT alter_columnar_table_set('test_options_2', compression => 'none'); + alter_columnar_table_set +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO test_options_2 SELECT i, floor(i/2000) FROM generate_series(1, 10000) i; diff --git a/src/test/regress/expected/upgrade_columnar_before_0.out b/src/test/regress/expected/upgrade_columnar_before_0.out new file mode 100644 index 000000000..9d92533f9 --- /dev/null +++ b/src/test/regress/expected/upgrade_columnar_before_0.out @@ -0,0 +1,6 @@ +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int > 11 AS server_version_above_eleven +\gset +\if :server_version_above_eleven +\else +\q diff --git a/src/test/regress/sql/upgrade_columnar_after.sql b/src/test/regress/sql/upgrade_columnar_after.sql new file mode 100644 index 000000000..646ad768c --- /dev/null +++ b/src/test/regress/sql/upgrade_columnar_after.sql @@ -0,0 +1,41 @@ +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int > 12 AS server_version_above_eleven +\gset +\if :server_version_above_eleven +\else +\q +\endif + +SET search_path TO upgrade_columnar, public; + +-- test we retained data +SELECT * FROM test_retains_data ORDER BY a; + +SELECT count(*) FROM test_retains_data; + +SELECT a,c FROM test_retains_data ORDER BY a; +SELECT b,d FROM test_retains_data ORDER BY a; + +SELECT * FROM test_retains_data ORDER BY a; + +-- test we retained data with a once truncated table +SELECT * FROM test_truncated ORDER BY a; + +-- test we retained data with a once vacuum fulled table +SELECT * FROM test_vacuum_full ORDER BY a; + +-- test we retained data with a once alter typed table +SELECT * FROM test_alter_type ORDER BY a; + +-- test we retained data with a once refreshed materialized view +SELECT * FROM matview ORDER BY a; + +-- test we retained options +SELECT * FROM cstore.options WHERE regclass = 'test_options_1'::regclass; +VACUUM VERBOSE test_options_1; +SELECT count(*), sum(a), sum(b) FROM test_options_1; + +SELECT * FROM cstore.options WHERE regclass = 'test_options_2'::regclass; +VACUUM VERBOSE test_options_2; +SELECT count(*), sum(a), sum(b) FROM test_options_2; + diff --git a/src/test/regress/sql/upgrade_columnar_before.sql b/src/test/regress/sql/upgrade_columnar_before.sql new file mode 100644 index 000000000..0999d7708 --- /dev/null +++ b/src/test/regress/sql/upgrade_columnar_before.sql @@ -0,0 +1,111 @@ +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int > 11 AS server_version_above_eleven +\gset +\if :server_version_above_eleven +\else +\q +\endif + +CREATE SCHEMA upgrade_columnar; +SET search_path TO upgrade_columnar, public; + +CREATE TYPE compfoo AS (f1 int, f2 text); + +CREATE TABLE test_retains_data (a int, b text, c compfoo, d int[]) USING columnar; + +INSERT INTO test_retains_data VALUES + (1, 'abc', (1, '4'), ARRAY[1,2,3,4]), + (2, 'pi', (3, '192'), ARRAY[3,1,4,1,5]), + (3, 'earth', (4, '22'), ARRAY[1,2,7,5,6]); + +-- +-- Verify that after upgrade we can read data for tables whose +-- relfilenode has changed before upgrade. +-- + +-- truncate +CREATE TABLE test_truncated (a int) USING columnar; +INSERT INTO test_truncated SELECT * FROM generate_series(1, 10); +SELECT count(*) FROM test_truncated; + +SELECT relfilenode AS relfilenode_pre_truncate +FROM pg_class WHERE oid = 'test_truncated'::regclass::oid \gset + +TRUNCATE test_truncated; + +SELECT relfilenode AS relfilenode_post_truncate +FROM pg_class WHERE oid = 'test_truncated'::regclass::oid \gset + +SELECT :relfilenode_post_truncate <> :relfilenode_pre_truncate AS relfilenode_changed; + +INSERT INTO test_truncated SELECT * FROM generate_series(11, 13); +SELECT count(*) FROM test_truncated; + +-- vacuum full +CREATE TABLE test_vacuum_full (a int) USING columnar; +INSERT INTO test_vacuum_full SELECT * FROM generate_series(1, 10); +SELECT count(*) FROM test_vacuum_full; + +SELECT relfilenode AS relfilenode_pre_vacuum_full +FROM pg_class WHERE oid = 'test_vacuum_full'::regclass::oid \gset + +VACUUM FULL test_vacuum_full; + +SELECT relfilenode AS relfilenode_post_vacuum_full +FROM pg_class WHERE oid = 'test_vacuum_full'::regclass::oid \gset + +SELECT :relfilenode_post_vacuum_full <> :relfilenode_pre_vacuum_full AS relfilenode_changed; + +INSERT INTO test_vacuum_full SELECT * FROM generate_series(11, 13); +SELECT count(*) FROM test_vacuum_full; + +-- alter column type +CREATE TABLE test_alter_type (a int) USING columnar; +INSERT INTO test_alter_type SELECT * FROM generate_series(1, 10); +SELECT count(*) FROM test_alter_type; + +SELECT relfilenode AS relfilenode_pre_alter +FROM pg_class WHERE oid = 'test_alter_type'::regclass::oid \gset + +ALTER TABLE test_alter_type ALTER COLUMN a TYPE text; + +SELECT relfilenode AS relfilenode_post_alter +FROM pg_class WHERE oid = 'test_alter_type'::regclass::oid \gset + +SELECT :relfilenode_pre_alter <> :relfilenode_post_alter AS relfilenode_changed; + +INSERT INTO test_alter_type SELECT * FROM generate_series(11, 13); +SELECT count(*) FROM test_alter_type; + +-- materialized view +CREATE MATERIALIZED VIEW matview(a, b) USING columnar AS +SELECT floor(a/3), array_agg(b) FROM test_retains_data GROUP BY 1; + +SELECT relfilenode AS relfilenode_pre_refresh +FROM pg_class WHERE oid = 'matview'::regclass::oid \gset + +REFRESH MATERIALIZED VIEW matview; + +SELECT relfilenode AS relfilenode_post_refresh +FROM pg_class WHERE oid = 'matview'::regclass::oid \gset + +SELECT :relfilenode_pre_alter <> :relfilenode_post_alter AS relfilenode_changed; + +-- +-- Test that we retain options +-- + +SET cstore.stripe_row_count TO 5000; +SET cstore.block_row_count TO 1000; +SET cstore.compression TO 'pglz'; + +CREATE TABLE test_options_1(a int, b int) USING columnar; +INSERT INTO test_options_1 SELECT i, floor(i/1000) FROM generate_series(1, 10000) i; + +CREATE TABLE test_options_2(a int, b int) USING columnar; +INSERT INTO test_options_2 SELECT i, floor(i/1000) FROM generate_series(1, 10000) i; +SELECT alter_columnar_table_set('test_options_2', block_row_count => 2000); +SELECT alter_columnar_table_set('test_options_2', stripe_row_count => 6000); +SELECT alter_columnar_table_set('test_options_2', compression => 'none'); +INSERT INTO test_options_2 SELECT i, floor(i/2000) FROM generate_series(1, 10000) i; +