external
/
citus
mirror of https://github.com/citusdata/citus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Columnar: Make compression level configurable

pull/4399/head
Hadi Moshayedi 2020-12-08 16:34:50 -08:00
parent 4501310b4c
commit 4668fe51a6
28 changed files with 526 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/backend/columnar/cstore.c
View File

@ -31,6 +31,7 @@
int cstore_compression = DEFAULT_COMPRESSION_TYPE;
int cstore_stripe_row_count = DEFAULT_STRIPE_ROW_COUNT;
int cstore_chunk_row_count = DEFAULT_CHUNK_ROW_COUNT;
int columnar_compression_level = 3;
static const struct config_enum_entry cstore_compression_options[] =
{
@ -60,6 +61,19 @@ cstore_init()
NULL,
NULL);
DefineCustomIntVariable("columnar.compression_level",
"Compression level to be used with zstd.",
NULL,
&columnar_compression_level,
3,
COMPRESSION_LEVEL_MIN,
COMPRESSION_LEVEL_MAX,
PGC_USERSET,
0,
NULL,
NULL,
NULL);
DefineCustomIntVariable("columnar.stripe_row_count",
"Maximum number of tuples per stripe.",
NULL,

7
src/backend/columnar/cstore_compression.c
View File

@ -53,8 +53,10 @@ typedef struct CStoreCompressHeader
* outputBuffer is valid only if the function returns true.
*/
bool
CompressBuffer(StringInfo inputBuffer, StringInfo outputBuffer,
CompressionType compressionType)
CompressBuffer(StringInfo inputBuffer,
StringInfo outputBuffer,
CompressionType compressionType,
int compressionLevel)
{
switch (compressionType)
{
@ -89,7 +91,6 @@ CompressBuffer(StringInfo inputBuffer, StringInfo outputBuffer,
case COMPRESSION_ZSTD:
{
int maximumLength = ZSTD_compressBound(inputBuffer->len);
int compressionLevel = 3;
resetStringInfo(outputBuffer);
enlargeStringInfo(outputBuffer, maximumLength);

19
src/backend/columnar/cstore_metadata_tables.c
View File

@ -102,11 +102,12 @@ static bool WriteColumnarOptions(Oid regclass, ColumnarOptions *options, bool ov
PG_FUNCTION_INFO_V1(columnar_relation_storageid);
/* constants for columnar.options */
#define Natts_cstore_options 4
#define Natts_cstore_options 5
#define Anum_cstore_options_regclass 1
#define Anum_cstore_options_chunk_row_count 2
#define Anum_cstore_options_stripe_row_count 3
#define Anum_cstore_options_compression 4
#define Anum_cstore_options_compression_level 4
#define Anum_cstore_options_compression 5
/* ----------------
* columnar.options definition.
@ -117,6 +118,7 @@ typedef struct FormData_cstore_options
Oid regclass;
int32 chunk_row_count;
int32 stripe_row_count;
int32 compressionLevel;
NameData compression;
#ifdef CATALOG_VARLEN /* variable-length fields start here */
@ -137,7 +139,7 @@ typedef FormData_cstore_options *Form_cstore_options;
#define Anum_cstore_stripes_row_count 8
/* constants for cstore_skipnodes */
#define Natts_cstore_skipnodes 13
#define Natts_cstore_skipnodes 14
#define Anum_cstore_skipnodes_storageid 1
#define Anum_cstore_skipnodes_stripe 2
#define Anum_cstore_skipnodes_attr 3
@ -150,7 +152,8 @@ typedef FormData_cstore_options *Form_cstore_options;
#define Anum_cstore_skipnodes_exists_stream_offset 10
#define Anum_cstore_skipnodes_exists_stream_length 11
#define Anum_cstore_skipnodes_value_compression_type 12
#define Anum_cstore_skipnodes_value_decompressed_size 13
#define Anum_cstore_skipnodes_value_compression_level 13
#define Anum_cstore_skipnodes_value_decompressed_size 14
/*
@ -173,6 +176,7 @@ InitColumnarOptions(Oid regclass)
.chunkRowCount = cstore_chunk_row_count,
.stripeRowCount = cstore_stripe_row_count,
.compressionType = cstore_compression,
.compressionLevel = columnar_compression_level
};
WriteColumnarOptions(regclass, &defaultOptions, false);
@ -215,6 +219,7 @@ WriteColumnarOptions(Oid regclass, ColumnarOptions *options, bool overwrite)
ObjectIdGetDatum(regclass),
Int32GetDatum(options->chunkRowCount),
Int32GetDatum(options->stripeRowCount),
Int32GetDatum(options->compressionLevel),
0, /* to be filled below */
};
@ -246,6 +251,7 @@ WriteColumnarOptions(Oid regclass, ColumnarOptions *options, bool overwrite)
bool update[Natts_cstore_options] = { 0 };
update[Anum_cstore_options_chunk_row_count - 1] = true;
update[Anum_cstore_options_stripe_row_count - 1] = true;
update[Anum_cstore_options_compression_level - 1] = true;
update[Anum_cstore_options_compression - 1] = true;
HeapTuple tuple = heap_modify_tuple(heapTuple, tupleDescriptor,
@ -363,6 +369,7 @@ ReadColumnarOptions(Oid regclass, ColumnarOptions *options)
options->chunkRowCount = tupOptions->chunk_row_count;
options->stripeRowCount = tupOptions->stripe_row_count;
options->compressionLevel = tupOptions->compressionLevel;
options->compressionType = ParseCompressionType(NameStr(tupOptions->compression));
}
else
@ -371,6 +378,7 @@ ReadColumnarOptions(Oid regclass, ColumnarOptions *options)
options->compressionType = cstore_compression;
options->stripeRowCount = cstore_stripe_row_count;
options->chunkRowCount = cstore_chunk_row_count;
options->compressionLevel = columnar_compression_level;
}
systable_endscan_ordered(scanDescriptor);
@ -418,6 +426,7 @@ SaveStripeSkipList(RelFileNode relfilenode, uint64 stripe, StripeSkipList *strip
Int64GetDatum(skipNode->existsChunkOffset),
Int64GetDatum(skipNode->existsLength),
Int32GetDatum(skipNode->valueCompressionType),
Int32GetDatum(skipNode->valueCompressionLevel),
Int64GetDatum(skipNode->decompressedValueSize)
};
@ -524,6 +533,8 @@ ReadStripeSkipList(RelFileNode relfilenode, uint64 stripe, TupleDesc tupleDescri
DatumGetInt64(datumArray[Anum_cstore_skipnodes_exists_stream_length - 1]);
skipNode->valueCompressionType =
DatumGetInt32(datumArray[Anum_cstore_skipnodes_value_compression_type - 1]);
skipNode->valueCompressionLevel =
DatumGetInt32(datumArray[Anum_cstore_skipnodes_value_compression_level - 1]);
skipNode->decompressedValueSize =
DatumGetInt64(datumArray[Anum_cstore_skipnodes_value_decompressed_size - 1]);

31
src/backend/columnar/cstore_tableam.c
View File

@ -632,9 +632,7 @@ cstore_relation_copy_for_cluster(Relation OldHeap, Relation NewHeap,
ReadColumnarOptions(OldHeap->rd_id, &cstoreOptions);
TableWriteState *writeState = CStoreBeginWrite(NewHeap->rd_node,
cstoreOptions.compressionType,
cstoreOptions.stripeRowCount,
cstoreOptions.chunkRowCount,
cstoreOptions,
targetDesc);
TableReadState *readState = CStoreBeginRead(OldHeap, sourceDesc,
@ -1343,10 +1341,12 @@ CitusCreateAlterColumnarTableSet(char *qualifiedRelationName,
"SELECT alter_columnar_table_set(%s, "
"chunk_row_count => %d, "
"stripe_row_count => %lu, "
"compression_level => %d, "
"compression => %s);",
quote_literal_cstr(qualifiedRelationName),
options->chunkRowCount,
options->stripeRowCount,
options->compressionLevel,
quote_literal_cstr(CompressionTypeStr(options->compressionType)));
return buf.data;
@ -1524,6 +1524,23 @@ alter_columnar_table_set(PG_FUNCTION_ARGS)
CompressionTypeStr(options.compressionType))));
}
/* compression_level => not null */
if (!PG_ARGISNULL(4))
{
options.compressionLevel = PG_GETARG_INT32(4);
if (options.compressionLevel < COMPRESSION_LEVEL_MIN ||
options.compressionLevel > COMPRESSION_LEVEL_MAX)
{
ereport(ERROR, (errmsg("compression level out of range"),
errhint("compression level must be between %d and %d",
COMPRESSION_LEVEL_MIN,
COMPRESSION_LEVEL_MAX)));
}
ereport(DEBUG1, (errmsg("updating compression level to %d",
options.compressionLevel)));
}
if (EnableDDLPropagation && IsCitusTable(relationId))
{
/* when a columnar table is distributed update all settings on the shards */
@ -1608,6 +1625,14 @@ alter_columnar_table_reset(PG_FUNCTION_ARGS)
CompressionTypeStr(options.compressionType))));
}
/* compression_level => true */
if (!PG_ARGISNULL(4) && PG_GETARG_BOOL(4))
{
options.compressionLevel = columnar_compression_level;
ereport(DEBUG1, (errmsg("reseting compression level to %d",
columnar_compression_level)));
}
if (EnableDDLPropagation && IsCitusTable(relationId))
{
/* when a columnar table is distributed update all settings on the shards */

27
src/backend/columnar/cstore_writer.c
View File

@ -61,8 +61,7 @@ static StringInfo CopyStringInfo(StringInfo sourceString);
*/
TableWriteState *
CStoreBeginWrite(RelFileNode relfilenode,
CompressionType compressionType,
uint64 stripeMaxRowCount, uint32 chunkRowCount,
ColumnarOptions options,
TupleDesc tupleDescriptor)
{
/* get comparison function pointers for each of the columns */
@ -98,13 +97,11 @@ CStoreBeginWrite(RelFileNode relfilenode,
memset(columnMaskArray, true, columnCount);
ChunkData *chunkData = CreateEmptyChunkData(columnCount, columnMaskArray,
chunkRowCount);
options.chunkRowCount);
TableWriteState *writeState = palloc0(sizeof(TableWriteState));
writeState->relfilenode = relfilenode;
writeState->compressionType = compressionType;
writeState->stripeMaxRowCount = stripeMaxRowCount;
writeState->chunkRowCount = chunkRowCount;
writeState->options = options;
writeState->tupleDescriptor = CreateTupleDescCopy(tupleDescriptor);
writeState->comparisonFunctionArray = comparisonFunctionArray;
writeState->stripeBuffers = NULL;
@ -135,15 +132,16 @@ CStoreWriteRow(TableWriteState *writeState, Datum *columnValues, bool *columnNul
StripeBuffers *stripeBuffers = writeState->stripeBuffers;
StripeSkipList *stripeSkipList = writeState->stripeSkipList;
uint32 columnCount = writeState->tupleDescriptor->natts;
const uint32 chunkRowCount = writeState->chunkRowCount;
ColumnarOptions *options = &writeState->options;
const uint32 chunkRowCount = options->chunkRowCount;
ChunkData *chunkData = writeState->chunkData;
MemoryContext oldContext = MemoryContextSwitchTo(writeState->stripeWriteContext);
if (stripeBuffers == NULL)
{
stripeBuffers = CreateEmptyStripeBuffers(writeState->stripeMaxRowCount,
stripeBuffers = CreateEmptyStripeBuffers(options->stripeRowCount,
chunkRowCount, columnCount);
stripeSkipList = CreateEmptyStripeSkipList(writeState->stripeMaxRowCount,
stripeSkipList = CreateEmptyStripeSkipList(options->stripeRowCount,
chunkRowCount, columnCount);
writeState->stripeBuffers = stripeBuffers;
writeState->stripeSkipList = stripeSkipList;
@ -206,7 +204,7 @@ CStoreWriteRow(TableWriteState *writeState, Datum *columnValues, bool *columnNul
}
stripeBuffers->rowCount++;
if (stripeBuffers->rowCount >= writeState->stripeMaxRowCount)
if (stripeBuffers->rowCount >= options->stripeRowCount)
{
CStoreFlushPendingWrites(writeState);
}
@ -413,7 +411,7 @@ FlushStripe(TableWriteState *writeState)
TupleDesc tupleDescriptor = writeState->tupleDescriptor;
uint32 columnCount = tupleDescriptor->natts;
uint32 chunkCount = stripeSkipList->chunkCount;
uint32 chunkRowCount = writeState->chunkRowCount;
uint32 chunkRowCount = writeState->options.chunkRowCount;
uint32 lastChunkIndex = stripeBuffers->rowCount / chunkRowCount;
uint32 lastChunkRowCount = stripeBuffers->rowCount % chunkRowCount;
uint64 stripeSize = 0;
@ -463,6 +461,7 @@ FlushStripe(TableWriteState *writeState)
chunkSkipNode->valueChunkOffset = stripeSize;
chunkSkipNode->valueLength = valueBufferSize;
chunkSkipNode->valueCompressionType = valueCompressionType;
chunkSkipNode->valueCompressionLevel = writeState->options.compressionLevel;
chunkSkipNode->decompressedValueSize = chunkBuffers->decompressedValueSize;
stripeSize += valueBufferSize;
@ -606,7 +605,8 @@ SerializeChunkData(TableWriteState *writeState, uint32 chunkIndex, uint32 rowCou
uint32 columnIndex = 0;
StripeBuffers *stripeBuffers = writeState->stripeBuffers;
ChunkData *chunkData = writeState->chunkData;
CompressionType requestedCompressionType = writeState->compressionType;
CompressionType requestedCompressionType = writeState->options.compressionType;
int compressionLevel = writeState->options.compressionLevel;
const uint32 columnCount = stripeBuffers->columnCount;
StringInfo compressionBuffer = writeState->compressionBuffer;
@ -643,7 +643,8 @@ SerializeChunkData(TableWriteState *writeState, uint32 chunkIndex, uint32 rowCou
* with compressed data and store compression type.
*/
bool compressed = CompressBuffer(serializedValueBuffer, compressionBuffer,
requestedCompressionType);
requestedCompressionType,
compressionLevel);
if (compressed)
{
serializedValueBuffer = compressionBuffer;

2
src/backend/columnar/sql/columnar--9.5-1--10.0-1.sql
View File

@ -9,6 +9,7 @@ CREATE TABLE options (
regclass regclass NOT NULL PRIMARY KEY,
chunk_row_count int NOT NULL,
stripe_row_count int NOT NULL,
compression_level int NOT NULL,
compression name NOT NULL
) WITH (user_catalog_table = true);
@ -41,6 +42,7 @@ CREATE TABLE columnar_skipnodes (
exists_stream_offset bigint NOT NULL,
exists_stream_length bigint NOT NULL,
value_compression_type int NOT NULL,
value_compression_level int NOT NULL,
value_decompressed_length bigint NOT NULL,
PRIMARY KEY (storageid, stripe, attr, chunk),
FOREIGN KEY (storageid, stripe) REFERENCES columnar_stripes(storageid, stripe) ON DELETE CASCADE

6
src/backend/columnar/sql/downgrades/columnar--10.0-1--9.5-1.sql
View File

@ -11,13 +11,15 @@ IF substring(current_Setting('server_version'), '\d+')::int >= 12 THEN
table_name regclass,
chunk_row_count bool,
stripe_row_count bool,
compression bool);
compression bool,
compression_level bool);
DROP FUNCTION pg_catalog.alter_columnar_table_set(
table_name regclass,
chunk_row_count int,
stripe_row_count int,
compression name);
compression name,
compression_level int);
DROP ACCESS METHOD columnar;

6
src/backend/columnar/sql/udfs/alter_columnar_table_reset/10.0-1.sql
View File

@ -2,7 +2,8 @@ CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_reset(
table_name regclass,
chunk_row_count bool DEFAULT false,
stripe_row_count bool DEFAULT false,
compression bool DEFAULT false)
compression bool DEFAULT false,
compression_level bool DEFAULT false)
RETURNS void
LANGUAGE C
AS 'MODULE_PATHNAME', 'alter_columnar_table_reset';
@ -11,5 +12,6 @@ COMMENT ON FUNCTION pg_catalog.alter_columnar_table_reset(
table_name regclass,
chunk_row_count bool,
stripe_row_count bool,
compression bool)
compression bool,
compression_level bool)
IS 'reset on or more options on a cstore table to the system defaults';

6
src/backend/columnar/sql/udfs/alter_columnar_table_reset/latest.sql
View File

@ -2,7 +2,8 @@ CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_reset(
table_name regclass,
chunk_row_count bool DEFAULT false,
stripe_row_count bool DEFAULT false,
compression bool DEFAULT false)
compression bool DEFAULT false,
compression_level bool DEFAULT false)
RETURNS void
LANGUAGE C
AS 'MODULE_PATHNAME', 'alter_columnar_table_reset';
@ -11,5 +12,6 @@ COMMENT ON FUNCTION pg_catalog.alter_columnar_table_reset(
table_name regclass,
chunk_row_count bool,
stripe_row_count bool,
compression bool)
compression bool,
compression_level bool)
IS 'reset on or more options on a cstore table to the system defaults';

6
src/backend/columnar/sql/udfs/alter_columnar_table_set/10.0-1.sql
View File

@ -2,7 +2,8 @@ CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_set(
table_name regclass,
chunk_row_count int DEFAULT NULL,
stripe_row_count int DEFAULT NULL,
compression name DEFAULT null)
compression name DEFAULT null,
compression_level int DEFAULT NULL)
RETURNS void
LANGUAGE C
AS 'MODULE_PATHNAME', 'alter_columnar_table_set';
@ -11,5 +12,6 @@ COMMENT ON FUNCTION pg_catalog.alter_columnar_table_set(
table_name regclass,
chunk_row_count int,
stripe_row_count int,
compression name)
compression name,
compression_level int)
IS 'set one or more options on a cstore table, when set to NULL no change is made';

6
src/backend/columnar/sql/udfs/alter_columnar_table_set/latest.sql
View File

@ -2,7 +2,8 @@ CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_set(
table_name regclass,
chunk_row_count int DEFAULT NULL,
stripe_row_count int DEFAULT NULL,
compression name DEFAULT null)
compression name DEFAULT null,
compression_level int DEFAULT NULL)
RETURNS void
LANGUAGE C
AS 'MODULE_PATHNAME', 'alter_columnar_table_set';
@ -11,5 +12,6 @@ COMMENT ON FUNCTION pg_catalog.alter_columnar_table_set(
table_name regclass,
chunk_row_count int,
stripe_row_count int,
compression name)
compression name,
compression_level int)
IS 'set one or more options on a cstore table, when set to NULL no change is made';

6
src/backend/columnar/sql/udfs/columnar_ensure_objects_exist/10.0-1.sql
View File

@ -31,12 +31,14 @@ IF NOT EXISTS (SELECT 1 FROM pg_am WHERE amname = 'columnar') THEN
table_name regclass,
chunk_row_count int,
stripe_row_count int,
compression name);
compression name,
compression_level int);
ALTER EXTENSION citus ADD FUNCTION pg_catalog.alter_columnar_table_reset(
table_name regclass,
chunk_row_count bool,
stripe_row_count bool,
compression bool);
compression bool,
compression_level bool);
END IF;
END IF;

6
src/backend/columnar/sql/udfs/columnar_ensure_objects_exist/latest.sql
View File

@ -31,12 +31,14 @@ IF NOT EXISTS (SELECT 1 FROM pg_am WHERE amname = 'columnar') THEN
table_name regclass,
chunk_row_count int,
stripe_row_count int,
compression name);
compression name,
compression_level int);
ALTER EXTENSION citus ADD FUNCTION pg_catalog.alter_columnar_table_reset(
table_name regclass,
chunk_row_count bool,
stripe_row_count bool,
compression bool);
compression bool,
compression_level bool);
END IF;
END IF;

4
src/backend/columnar/write_state_management.c
View File

@ -183,9 +183,7 @@ cstore_init_write_state(Relation relation, TupleDesc tupdesc,
SubXidWriteState *stackEntry = palloc0(sizeof(SubXidWriteState));
stackEntry->writeState = CStoreBeginWrite(relation->rd_node,
cstoreOptions.compressionType,
cstoreOptions.stripeRowCount,
cstoreOptions.chunkRowCount,
cstoreOptions,
tupdesc);
stackEntry->subXid = currentSubXid;
stackEntry->next = hashEntry->writeStateStack;

19
src/include/columnar/cstore.h
View File

@ -34,6 +34,8 @@
#define STRIPE_ROW_COUNT_MAXIMUM 10000000
#define CHUNK_ROW_COUNT_MINIMUM 1000
#define CHUNK_ROW_COUNT_MAXIMUM 100000
#define COMPRESSION_LEVEL_MIN 1
#define COMPRESSION_LEVEL_MAX 19
/* String representations of compression types */
#define COMPRESSION_STRING_NONE "none"
@ -73,6 +75,7 @@ typedef struct ColumnarOptions
uint64 stripeRowCount;
uint32 chunkRowCount;
CompressionType compressionType;
int compressionLevel;
} ColumnarOptions;
@ -125,6 +128,7 @@ typedef struct ColumnChunkSkipNode
uint64 decompressedValueSize;
CompressionType valueCompressionType;
int valueCompressionLevel;
} ColumnChunkSkipNode;
@ -228,7 +232,6 @@ typedef struct TableReadState
/* TableWriteState represents state of a cstore file write operation. */
typedef struct TableWriteState
{
CompressionType compressionType;
TupleDesc tupleDescriptor;
FmgrInfo **comparisonFunctionArray;
RelFileNode relfilenode;
@ -237,8 +240,7 @@ typedef struct TableWriteState
MemoryContext perTupleContext;
StripeBuffers *stripeBuffers;
StripeSkipList *stripeSkipList;
uint32 stripeMaxRowCount;
uint32 chunkRowCount;
ColumnarOptions options;
ChunkData *chunkData;
/*
@ -253,6 +255,7 @@ typedef struct TableWriteState
extern int cstore_compression;
extern int cstore_stripe_row_count;
extern int cstore_chunk_row_count;
extern int columnar_compression_level;
extern void cstore_init(void);
@ -260,9 +263,7 @@ extern CompressionType ParseCompressionType(const char *compressionTypeString);
/* Function declarations for writing to a cstore file */
extern TableWriteState * CStoreBeginWrite(RelFileNode relfilenode,
CompressionType compressionType,
uint64 stripeMaxRowCount,
uint32 chunkRowCount,
ColumnarOptions options,
TupleDesc tupleDescriptor);
extern void CStoreWriteRow(TableWriteState *state, Datum *columnValues,
bool *columnNulls);
@ -287,8 +288,10 @@ extern ChunkData * CreateEmptyChunkData(uint32 columnCount, bool *columnMask,
uint32 chunkRowCount);
extern void FreeChunkData(ChunkData *chunkData);
extern uint64 CStoreTableRowCount(Relation relation);
extern bool CompressBuffer(StringInfo inputBuffer, StringInfo outputBuffer,
CompressionType compressionType);
extern bool CompressBuffer(StringInfo inputBuffer,
StringInfo outputBuffer,
CompressionType compressionType,
int compressionLevel);
extern StringInfo DecompressBuffer(StringInfo buffer, CompressionType compressionType,
uint64 decompressedSize);
extern const char * CompressionTypeStr(CompressionType type);

4
src/test/regress/expected/am_empty.out
View File

@ -24,9 +24,9 @@ SELECT alter_columnar_table_set('t_compressed', chunk_row_count => 100);
(1 row)
SELECT * FROM columnar.options WHERE regclass = 't_compressed'::regclass;
regclass | chunk_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression_level | compression
---------------------------------------------------------------------
t_compressed | 100 | 100 | pglz
t_compressed | 100 | 100 | 3 | pglz
(1 row)
-- select

12
src/test/regress/expected/am_matview.out
View File

@ -25,9 +25,9 @@ SELECT * FROM t_view a ORDER BY a;
-- show columnar options for materialized view
SELECT * FROM columnar.options
WHERE regclass = 't_view'::regclass;
regclass | chunk_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression_level | compression
---------------------------------------------------------------------
t_view | 10000 | 150000 | none
t_view | 10000 | 150000 | 3 | none
(1 row)
-- show we can set options on a materialized view
@ -39,18 +39,18 @@ SELECT alter_columnar_table_set('t_view', compression => 'pglz');
SELECT * FROM columnar.options
WHERE regclass = 't_view'::regclass;
regclass | chunk_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression_level | compression
---------------------------------------------------------------------
t_view | 10000 | 150000 | pglz
t_view | 10000 | 150000 | 3 | pglz
(1 row)
REFRESH MATERIALIZED VIEW t_view;
-- verify options have not been changed
SELECT * FROM columnar.options
WHERE regclass = 't_view'::regclass;
regclass | chunk_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression_level | compression
---------------------------------------------------------------------
t_view | 10000 | 150000 | pglz
t_view | 10000 | 150000 | 3 | pglz
(1 row)
SELECT * FROM t_view a ORDER BY a;

137
src/test/regress/expected/am_tableoptions.out
View File

@ -5,9 +5,9 @@ INSERT INTO table_options SELECT generate_series(1,100);
-- show table_options settings
SELECT * FROM columnar.options
WHERE regclass = 'table_options'::regclass;
regclass | chunk_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression_level | compression
---------------------------------------------------------------------
table_options | 10000 | 150000 | none
table_options | 10000 | 150000 | 3 | none
(1 row)
-- test changing the compression
@ -20,9 +20,24 @@ SELECT alter_columnar_table_set('table_options', compression => 'pglz');
-- show table_options settings
SELECT * FROM columnar.options
WHERE regclass = 'table_options'::regclass;
regclass | chunk_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression_level | compression
---------------------------------------------------------------------
table_options | 10000 | 150000 | pglz
table_options | 10000 | 150000 | 3 | pglz
(1 row)
-- test changing the compression level
SELECT alter_columnar_table_set('table_options', compression_level => 5);
alter_columnar_table_set
---------------------------------------------------------------------
(1 row)
-- show table_options settings
SELECT * FROM columnar.options
WHERE regclass = 'table_options'::regclass;
regclass | chunk_row_count | stripe_row_count | compression_level | compression
---------------------------------------------------------------------
table_options | 10000 | 150000 | 5 | pglz
(1 row)
-- test changing the chunk_row_count
@ -35,9 +50,9 @@ SELECT alter_columnar_table_set('table_options', chunk_row_count => 10);
-- show table_options settings
SELECT * FROM columnar.options
WHERE regclass = 'table_options'::regclass;
regclass | chunk_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression_level | compression
---------------------------------------------------------------------
table_options | 10 | 150000 | pglz
table_options | 10 | 150000 | 5 | pglz
(1 row)
-- test changing the chunk_row_count
@ -50,9 +65,9 @@ SELECT alter_columnar_table_set('table_options', stripe_row_count => 100);
-- show table_options settings
SELECT * FROM columnar.options
WHERE regclass = 'table_options'::regclass;
regclass | chunk_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression_level | compression
---------------------------------------------------------------------
table_options | 10 | 100 | pglz
table_options | 10 | 100 | 5 | pglz
(1 row)
-- VACUUM FULL creates a new table, make sure it copies settings from the table you are vacuuming
@ -60,13 +75,13 @@ VACUUM FULL table_options;
-- show table_options settings
SELECT * FROM columnar.options
WHERE regclass = 'table_options'::regclass;
regclass | chunk_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression_level | compression
---------------------------------------------------------------------
table_options | 10 | 100 | pglz
table_options | 10 | 100 | 5 | pglz
(1 row)
-- set all settings at the same time
SELECT alter_columnar_table_set('table_options', stripe_row_count => 1000, chunk_row_count => 100, compression => 'none');
SELECT alter_columnar_table_set('table_options', stripe_row_count => 1000, chunk_row_count => 100, compression => 'none', compression_level => 7);
alter_columnar_table_set
---------------------------------------------------------------------
@ -75,9 +90,9 @@ SELECT alter_columnar_table_set('table_options', stripe_row_count => 1000, chunk
-- show table_options settings
SELECT * FROM columnar.options
WHERE regclass = 'table_options'::regclass;
regclass | chunk_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression_level | compression
---------------------------------------------------------------------
table_options | 100 | 1000 | none
table_options | 100 | 1000 | 7 | none
(1 row)
-- make sure table options are not changed when VACUUM a table
@ -85,9 +100,9 @@ VACUUM table_options;
-- show table_options settings
SELECT * FROM columnar.options
WHERE regclass = 'table_options'::regclass;
regclass | chunk_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression_level | compression
---------------------------------------------------------------------
table_options | 100 | 1000 | none
table_options | 100 | 1000 | 7 | none
(1 row)
-- make sure table options are not changed when VACUUM FULL a table
@ -95,9 +110,9 @@ VACUUM FULL table_options;
-- show table_options settings
SELECT * FROM columnar.options
WHERE regclass = 'table_options'::regclass;
regclass | chunk_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression_level | compression
---------------------------------------------------------------------
table_options | 100 | 1000 | none
table_options | 100 | 1000 | 7 | none
(1 row)
-- make sure table options are not changed when truncating a table
@ -105,31 +120,32 @@ TRUNCATE table_options;
-- show table_options settings
SELECT * FROM columnar.options
WHERE regclass = 'table_options'::regclass;
regclass | chunk_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression_level | compression
---------------------------------------------------------------------
table_options | 100 | 1000 | none
table_options | 100 | 1000 | 7 | none
(1 row)
ALTER TABLE table_options ALTER COLUMN a TYPE bigint;
-- show table_options settings
SELECT * FROM columnar.options
WHERE regclass = 'table_options'::regclass;
regclass | chunk_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression_level | compression
---------------------------------------------------------------------
table_options | 100 | 1000 | none
table_options | 100 | 1000 | 7 | none
(1 row)
-- reset settings one by one to the version of the GUC's
SET columnar.chunk_row_count TO 1000;
SET columnar.stripe_row_count TO 10000;
SET columnar.compression TO 'pglz';
SET columnar.compression_level TO 11;
-- verify setting the GUC's didn't change the settings
-- show table_options settings
SELECT * FROM columnar.options
WHERE regclass = 'table_options'::regclass;
regclass | chunk_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression_level | compression
---------------------------------------------------------------------
table_options | 100 | 1000 | none
table_options | 100 | 1000 | 7 | none
(1 row)
SELECT alter_columnar_table_reset('table_options', chunk_row_count => true);
@ -141,9 +157,9 @@ SELECT alter_columnar_table_reset('table_options', chunk_row_count => true);
-- show table_options settings
SELECT * FROM columnar.options
WHERE regclass = 'table_options'::regclass;
regclass | chunk_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression_level | compression
---------------------------------------------------------------------
table_options | 1000 | 1000 | none
table_options | 1000 | 1000 | 7 | none
(1 row)
SELECT alter_columnar_table_reset('table_options', stripe_row_count => true);
@ -155,9 +171,9 @@ SELECT alter_columnar_table_reset('table_options', stripe_row_count => true);
-- show table_options settings
SELECT * FROM columnar.options
WHERE regclass = 'table_options'::regclass;
regclass | chunk_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression_level | compression
---------------------------------------------------------------------
table_options | 1000 | 10000 | none
table_options | 1000 | 10000 | 7 | none
(1 row)
SELECT alter_columnar_table_reset('table_options', compression => true);
@ -169,28 +185,12 @@ SELECT alter_columnar_table_reset('table_options', compression => true);
-- show table_options settings
SELECT * FROM columnar.options
WHERE regclass = 'table_options'::regclass;
regclass | chunk_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression_level | compression
---------------------------------------------------------------------
table_options | 1000 | 10000 | pglz
table_options | 1000 | 10000 | 7 | pglz
(1 row)
-- verify resetting all settings at once work
SET columnar.chunk_row_count TO 10000;
SET columnar.stripe_row_count TO 100000;
SET columnar.compression TO 'none';
-- show table_options settings
SELECT * FROM columnar.options
WHERE regclass = 'table_options'::regclass;
regclass | chunk_row_count | stripe_row_count | compression
---------------------------------------------------------------------
table_options | 1000 | 10000 | pglz
(1 row)
SELECT alter_columnar_table_reset(
'table_options',
chunk_row_count => true,
stripe_row_count => true,
compression => true);
SELECT alter_columnar_table_reset('table_options', compression_level => true);
alter_columnar_table_reset
---------------------------------------------------------------------
@ -199,9 +199,41 @@ SELECT alter_columnar_table_reset(
-- show table_options settings
SELECT * FROM columnar.options
WHERE regclass = 'table_options'::regclass;
regclass | chunk_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression_level | compression
---------------------------------------------------------------------
table_options | 10000 | 100000 | none
table_options | 1000 | 10000 | 11 | pglz
(1 row)
-- verify resetting all settings at once work
SET columnar.chunk_row_count TO 10000;
SET columnar.stripe_row_count TO 100000;
SET columnar.compression TO 'none';
SET columnar.compression_level TO 13;
-- show table_options settings
SELECT * FROM columnar.options
WHERE regclass = 'table_options'::regclass;
regclass | chunk_row_count | stripe_row_count | compression_level | compression
---------------------------------------------------------------------
table_options | 1000 | 10000 | 11 | pglz
(1 row)
SELECT alter_columnar_table_reset(
'table_options',
chunk_row_count => true,
stripe_row_count => true,
compression => true,
compression_level => true);
alter_columnar_table_reset
---------------------------------------------------------------------
(1 row)
-- show table_options settings
SELECT * FROM columnar.options
WHERE regclass = 'table_options'::regclass;
regclass | chunk_row_count | stripe_row_count | compression_level | compression
---------------------------------------------------------------------
table_options | 10000 | 100000 | 13 | none
(1 row)
-- verify edge cases
@ -214,11 +246,18 @@ ERROR: table not_a_columnar_table is not a columnar table
-- verify you can't use a compression that is not known
SELECT alter_columnar_table_set('table_options', compression => 'foobar');
ERROR: unknown compression type for cstore table: foobar
-- verify cannot set out of range compression levels
SELECT alter_columnar_table_set('table_options', compression_level => 0);
ERROR: compression level out of range
HINT: compression level must be between 1 and 19
SELECT alter_columnar_table_set('table_options', compression_level => 20);
ERROR: compression level out of range
HINT: compression level must be between 1 and 19
-- verify options are removed when table is dropped
DROP TABLE table_options;
-- we expect no entries in çstore.options for anything not found int pg_class
SELECT * FROM columnar.options o WHERE o.regclass NOT IN (SELECT oid FROM pg_class);
regclass | chunk_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression_level | compression
---------------------------------------------------------------------
(0 rows)

20
src/test/regress/expected/am_zstd.out
View File

@ -24,7 +24,7 @@ SELECT count(*) FROM test_zstd;
VACUUM VERBOSE test_zstd;
INFO: statistics for "test_zstd":
storage id: xxxxx
total file size: 40960, total data size: 14947
total file size: 40960, total data size: 14945
compression rate: 21.91x
total row count: 20001, stripe count: 2, average rows per stripe: 10000
chunk count: 9, containing data for dropped columns: 0, zstd compressed: 9
@ -39,6 +39,24 @@ SELECT DISTINCT * FROM test_zstd ORDER BY a, b, c LIMIT 5;
0 | 12 | 4
(5 rows)
-- change compression level
-- for this particular usecase, higher compression levels
-- don't improve compression ratio
SELECT alter_columnar_table_set('test_zstd', compression_level => 19);
alter_columnar_table_set
---------------------------------------------------------------------
(1 row)
VACUUM FULL test_zstd;
VACUUM VERBOSE test_zstd;
INFO: statistics for "test_zstd":
storage id: xxxxx
total file size: 32768, total data size: 15201
compression rate: 21.55x
total row count: 20001, stripe count: 1, average rows per stripe: 20001
chunk count: 9, containing data for dropped columns: 0, zstd compressed: 9
-- compare compression rate to pglz
SET columnar.compression TO 'pglz';
CREATE TABLE test_pglz (LIKE test_zstd) USING columnar;

208
src/test/regress/expected/columnar_citus_integration.out
View File

@ -63,6 +63,57 @@ $cmd$);
(localhost,57638,20090003,t,none)
(4 rows)
-- setting: compression_level
-- get baseline for setting
SELECT run_command_on_placements('table_option',$cmd$
SELECT compression_level FROM columnar.options WHERE regclass = '%s'::regclass;
$cmd$);
run_command_on_placements
---------------------------------------------------------------------
(localhost,57637,20090000,t,3)
(localhost,57638,20090001,t,3)
(localhost,57637,20090002,t,3)
(localhost,57638,20090003,t,3)
(4 rows)
-- change setting
SELECT alter_columnar_table_set('table_option', compression_level => 13);
alter_columnar_table_set
---------------------------------------------------------------------
(1 row)
-- verify setting
SELECT run_command_on_placements('table_option',$cmd$
SELECT compression_level FROM columnar.options WHERE regclass = '%s'::regclass;
$cmd$);
run_command_on_placements
---------------------------------------------------------------------
(localhost,57637,20090000,t,13)
(localhost,57638,20090001,t,13)
(localhost,57637,20090002,t,13)
(localhost,57638,20090003,t,13)
(4 rows)
-- reset setting
SELECT alter_columnar_table_reset('table_option', compression_level => true);
alter_columnar_table_reset
---------------------------------------------------------------------
(1 row)
-- verify setting
SELECT run_command_on_placements('table_option',$cmd$
SELECT compression_level FROM columnar.options WHERE regclass = '%s'::regclass;
$cmd$);
run_command_on_placements
---------------------------------------------------------------------
(localhost,57637,20090000,t,3)
(localhost,57638,20090001,t,3)
(localhost,57637,20090002,t,3)
(localhost,57638,20090003,t,3)
(4 rows)
-- setting: chunk_row_count
-- get baseline for setting
SELECT run_command_on_placements('table_option',$cmd$
@ -170,7 +221,8 @@ CREATE TABLE table_option_2 (a int, b text) USING columnar;
SELECT alter_columnar_table_set('table_option_2',
chunk_row_count => 100,
stripe_row_count => 1000,
compression => 'pglz');
compression => 'pglz',
compression_level => 15);
alter_columnar_table_set
---------------------------------------------------------------------
@ -184,14 +236,14 @@ SELECT create_distributed_table('table_option_2', 'a');
-- verify settings on placements
SELECT run_command_on_placements('table_option_2',$cmd$
SELECT ROW(chunk_row_count, stripe_row_count, compression) FROM columnar.options WHERE regclass = '%s'::regclass;
SELECT ROW(chunk_row_count, stripe_row_count, compression, compression_level) FROM columnar.options WHERE regclass = '%s'::regclass;
$cmd$);
run_command_on_placements
run_command_on_placements
---------------------------------------------------------------------
(localhost,57637,20090004,t,"(100,1000,pglz)")
(localhost,57638,20090005,t,"(100,1000,pglz)")
(localhost,57637,20090006,t,"(100,1000,pglz)")
(localhost,57638,20090007,t,"(100,1000,pglz)")
(localhost,57637,20090004,t,"(100,1000,pglz,15)")
(localhost,57638,20090005,t,"(100,1000,pglz,15)")
(localhost,57637,20090006,t,"(100,1000,pglz,15)")
(localhost,57638,20090007,t,"(100,1000,pglz,15)")
(4 rows)
DROP TABLE table_option, table_option_2;
@ -268,6 +320,69 @@ $cmd$);
(localhost,57638,20090011,t,none)
(8 rows)
-- setting: compression_level
-- get baseline for setting
SELECT run_command_on_placements('table_option',$cmd$
SELECT compression_level FROM columnar.options WHERE regclass = '%s'::regclass;
$cmd$);
run_command_on_placements
---------------------------------------------------------------------
(localhost,57637,20090008,t,3)
(localhost,57638,20090008,t,3)
(localhost,57637,20090009,t,3)
(localhost,57638,20090009,t,3)
(localhost,57637,20090010,t,3)
(localhost,57638,20090010,t,3)
(localhost,57637,20090011,t,3)
(localhost,57638,20090011,t,3)
(8 rows)
-- change setting
SELECT alter_columnar_table_set('table_option', compression_level => 17);
alter_columnar_table_set
---------------------------------------------------------------------
(1 row)
-- verify setting
SELECT run_command_on_placements('table_option',$cmd$
SELECT compression_level FROM columnar.options WHERE regclass = '%s'::regclass;
$cmd$);
run_command_on_placements
---------------------------------------------------------------------
(localhost,57637,20090008,t,17)
(localhost,57638,20090008,t,17)
(localhost,57637,20090009,t,17)
(localhost,57638,20090009,t,17)
(localhost,57637,20090010,t,17)
(localhost,57638,20090010,t,17)
(localhost,57637,20090011,t,17)
(localhost,57638,20090011,t,17)
(8 rows)
-- reset setting
SELECT alter_columnar_table_reset('table_option', compression_level => true);
alter_columnar_table_reset
---------------------------------------------------------------------
(1 row)
-- verify setting
SELECT run_command_on_placements('table_option',$cmd$
SELECT compression_level FROM columnar.options WHERE regclass = '%s'::regclass;
$cmd$);
run_command_on_placements
---------------------------------------------------------------------
(localhost,57637,20090008,t,3)
(localhost,57638,20090008,t,3)
(localhost,57637,20090009,t,3)
(localhost,57638,20090009,t,3)
(localhost,57637,20090010,t,3)
(localhost,57638,20090010,t,3)
(localhost,57637,20090011,t,3)
(localhost,57638,20090011,t,3)
(8 rows)
-- setting: chunk_row_count
-- get baseline for setting
SELECT run_command_on_placements('table_option',$cmd$
@ -399,7 +514,8 @@ CREATE TABLE table_option_2 (a int, b text) USING columnar;
SELECT alter_columnar_table_set('table_option_2',
chunk_row_count => 100,
stripe_row_count => 1000,
compression => 'pglz');
compression => 'pglz',
compression_level => 19);
alter_columnar_table_set
---------------------------------------------------------------------
@ -413,18 +529,18 @@ SELECT create_distributed_table('table_option_2', 'a');
-- verify settings on placements
SELECT run_command_on_placements('table_option_2',$cmd$
SELECT ROW(chunk_row_count, stripe_row_count, compression) FROM columnar.options WHERE regclass = '%s'::regclass;
SELECT ROW(chunk_row_count, stripe_row_count, compression, compression_level) FROM columnar.options WHERE regclass = '%s'::regclass;
$cmd$);
run_command_on_placements
run_command_on_placements
---------------------------------------------------------------------
(localhost,57637,20090012,t,"(100,1000,pglz)")
(localhost,57638,20090012,t,"(100,1000,pglz)")
(localhost,57637,20090013,t,"(100,1000,pglz)")
(localhost,57638,20090013,t,"(100,1000,pglz)")
(localhost,57637,20090014,t,"(100,1000,pglz)")
(localhost,57638,20090014,t,"(100,1000,pglz)")
(localhost,57637,20090015,t,"(100,1000,pglz)")
(localhost,57638,20090015,t,"(100,1000,pglz)")
(localhost,57637,20090012,t,"(100,1000,pglz,19)")
(localhost,57638,20090012,t,"(100,1000,pglz,19)")
(localhost,57637,20090013,t,"(100,1000,pglz,19)")
(localhost,57638,20090013,t,"(100,1000,pglz,19)")
(localhost,57637,20090014,t,"(100,1000,pglz,19)")
(localhost,57638,20090014,t,"(100,1000,pglz,19)")
(localhost,57637,20090015,t,"(100,1000,pglz,19)")
(localhost,57638,20090015,t,"(100,1000,pglz,19)")
(8 rows)
DROP TABLE table_option, table_option_2;
@ -481,6 +597,51 @@ $cmd$);
(localhost,57638,20090016,t,none)
(2 rows)
-- setting: compression_level
-- get baseline for setting
SELECT run_command_on_placements('table_option_reference',$cmd$
SELECT compression_level FROM columnar.options WHERE regclass = '%s'::regclass;
$cmd$);
run_command_on_placements
---------------------------------------------------------------------
(localhost,57637,20090016,t,3)
(localhost,57638,20090016,t,3)
(2 rows)
-- change setting
SELECT alter_columnar_table_set('table_option_reference', compression_level => 11);
alter_columnar_table_set
---------------------------------------------------------------------
(1 row)
-- verify setting
SELECT run_command_on_placements('table_option_reference',$cmd$
SELECT compression_level FROM columnar.options WHERE regclass = '%s'::regclass;
$cmd$);
run_command_on_placements
---------------------------------------------------------------------
(localhost,57637,20090016,t,11)
(localhost,57638,20090016,t,11)
(2 rows)
-- reset setting
SELECT alter_columnar_table_reset('table_option_reference', compression_level => true);
alter_columnar_table_reset
---------------------------------------------------------------------
(1 row)
-- verify setting
SELECT run_command_on_placements('table_option_reference',$cmd$
SELECT compression_level FROM columnar.options WHERE regclass = '%s'::regclass;
$cmd$);
run_command_on_placements
---------------------------------------------------------------------
(localhost,57637,20090016,t,3)
(localhost,57638,20090016,t,3)
(2 rows)
-- setting: chunk_row_count
-- get baseline for setting
SELECT run_command_on_placements('table_option_reference',$cmd$
@ -576,7 +737,8 @@ CREATE TABLE table_option_reference_2 (a int, b text) USING columnar;
SELECT alter_columnar_table_set('table_option_reference_2',
chunk_row_count => 100,
stripe_row_count => 1000,
compression => 'pglz');
compression => 'pglz',
compression_level => 9);
alter_columnar_table_set
---------------------------------------------------------------------
@ -590,12 +752,12 @@ SELECT create_reference_table('table_option_reference_2');
-- verify settings on placements
SELECT run_command_on_placements('table_option_reference_2',$cmd$
SELECT ROW(chunk_row_count, stripe_row_count, compression) FROM columnar.options WHERE regclass = '%s'::regclass;
SELECT ROW(chunk_row_count, stripe_row_count, compression, compression_level) FROM columnar.options WHERE regclass = '%s'::regclass;
$cmd$);
run_command_on_placements
run_command_on_placements
---------------------------------------------------------------------
(localhost,57637,20090017,t,"(100,1000,pglz)")
(localhost,57638,20090017,t,"(100,1000,pglz)")
(localhost,57637,20090017,t,"(100,1000,pglz,9)")
(localhost,57638,20090017,t,"(100,1000,pglz,9)")
(2 rows)
DROP TABLE table_option_reference, table_option_reference_2;

4
src/test/regress/expected/multi_extension.out
View File

@ -479,8 +479,8 @@ SELECT * FROM print_extension_changes();
previous_object | current_object
---------------------------------------------------------------------
| access method columnar
| function alter_columnar_table_reset(regclass,boolean,boolean,boolean)
| function alter_columnar_table_set(regclass,integer,integer,name)
| function alter_columnar_table_reset(regclass,boolean,boolean,boolean,boolean)
| function alter_columnar_table_set(regclass,integer,integer,name,integer)
| function citus_internal.columnar_ensure_objects_exist()
| function columnar.columnar_handler(internal)
| schema columnar

8
src/test/regress/expected/upgrade_columnar_after.out
View File

@ -102,9 +102,9 @@ SELECT * FROM matview ORDER BY a;
-- test we retained options
SELECT * FROM columnar.options WHERE regclass = 'test_options_1'::regclass;
regclass | chunk_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression_level | compression
---------------------------------------------------------------------
test_options_1 | 1000 | 5000 | pglz
test_options_1 | 1000 | 5000 | 3 | pglz
(1 row)
VACUUM VERBOSE test_options_1;
@ -122,9 +122,9 @@ SELECT count(*), sum(a), sum(b) FROM test_options_1;
(1 row)
SELECT * FROM columnar.options WHERE regclass = 'test_options_2'::regclass;
regclass | chunk_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression_level | compression
---------------------------------------------------------------------
test_options_2 | 2000 | 6000 | none
test_options_2 | 2000 | 6000 | 13 | none
(1 row)
VACUUM VERBOSE test_options_2;

10
src/test/regress/expected/upgrade_columnar_before.out
View File

@ -77,7 +77,7 @@ INSERT INTO test_alter_type SELECT * FROM generate_series(1, 10);
SELECT count(*) FROM test_alter_type;
count
---------------------------------------------------------------------
10
10
(1 row)
SELECT relfilenode AS relfilenode_pre_alter
@ -95,7 +95,7 @@ INSERT INTO test_alter_type SELECT * FROM generate_series(11, 13);
SELECT count(*) FROM test_alter_type;
count
---------------------------------------------------------------------
13
13
(1 row)
-- materialized view
@ -140,4 +140,10 @@ SELECT alter_columnar_table_set('test_options_2', compression => 'none');
(1 row)
SELECT alter_columnar_table_set('test_options_2', compression_level => 13);
alter_columnar_table_set
---------------------------------------------------------------------
(1 row)
INSERT INTO test_options_2 SELECT i, floor(i/2000) FROM generate_series(1, 10000) i;

4
src/test/regress/expected/upgrade_list_citus_objects.out
View File

@ -18,8 +18,8 @@ ORDER BY 1;
---------------------------------------------------------------------
access method columnar
event trigger citus_cascade_to_partition
function alter_columnar_table_reset(regclass,boolean,boolean,boolean)
function alter_columnar_table_set(regclass,integer,integer,name)
function alter_columnar_table_reset(regclass,boolean,boolean,boolean,boolean)
function alter_columnar_table_set(regclass,integer,integer,name,integer)
function alter_role_if_exists(text,text)
function any_value(anyelement)
function any_value_agg(anyelement,anyelement)

24
src/test/regress/sql/am_tableoptions.sql
View File

@ -15,6 +15,13 @@ SELECT alter_columnar_table_set('table_options', compression => 'pglz');
SELECT * FROM columnar.options
WHERE regclass = 'table_options'::regclass;
-- test changing the compression level
SELECT alter_columnar_table_set('table_options', compression_level => 5);
-- show table_options settings
SELECT * FROM columnar.options
WHERE regclass = 'table_options'::regclass;
-- test changing the chunk_row_count
SELECT alter_columnar_table_set('table_options', chunk_row_count => 10);
@ -37,7 +44,7 @@ SELECT * FROM columnar.options
WHERE regclass = 'table_options'::regclass;
-- set all settings at the same time
SELECT alter_columnar_table_set('table_options', stripe_row_count => 1000, chunk_row_count => 100, compression => 'none');
SELECT alter_columnar_table_set('table_options', stripe_row_count => 1000, chunk_row_count => 100, compression => 'none', compression_level => 7);
-- show table_options settings
SELECT * FROM columnar.options
@ -70,6 +77,7 @@ WHERE regclass = 'table_options'::regclass;
SET columnar.chunk_row_count TO 1000;
SET columnar.stripe_row_count TO 10000;
SET columnar.compression TO 'pglz';
SET columnar.compression_level TO 11;
-- verify setting the GUC's didn't change the settings
-- show table_options settings
@ -93,10 +101,17 @@ SELECT alter_columnar_table_reset('table_options', compression => true);
SELECT * FROM columnar.options
WHERE regclass = 'table_options'::regclass;
SELECT alter_columnar_table_reset('table_options', compression_level => true);
-- show table_options settings
SELECT * FROM columnar.options
WHERE regclass = 'table_options'::regclass;
-- verify resetting all settings at once work
SET columnar.chunk_row_count TO 10000;
SET columnar.stripe_row_count TO 100000;
SET columnar.compression TO 'none';
SET columnar.compression_level TO 13;
-- show table_options settings
SELECT * FROM columnar.options
@ -106,7 +121,8 @@ SELECT alter_columnar_table_reset(
'table_options',
chunk_row_count => true,
stripe_row_count => true,
compression => true);
compression => true,
compression_level => true);
-- show table_options settings
SELECT * FROM columnar.options
@ -121,6 +137,10 @@ SELECT alter_columnar_table_reset('not_a_columnar_table', compression => true);
-- verify you can't use a compression that is not known
SELECT alter_columnar_table_set('table_options', compression => 'foobar');
-- verify cannot set out of range compression levels
SELECT alter_columnar_table_set('table_options', compression_level => 0);
SELECT alter_columnar_table_set('table_options', compression_level => 20);
-- verify options are removed when table is dropped
DROP TABLE table_options;
-- we expect no entries in çstore.options for anything not found int pg_class

7
src/test/regress/sql/am_zstd.sql
View File

@ -20,6 +20,13 @@ VACUUM VERBOSE test_zstd;
SELECT DISTINCT * FROM test_zstd ORDER BY a, b, c LIMIT 5;
-- change compression level
-- for this particular usecase, higher compression levels
-- don't improve compression ratio
SELECT alter_columnar_table_set('test_zstd', compression_level => 19);
VACUUM FULL test_zstd;
VACUUM VERBOSE test_zstd;
-- compare compression rate to pglz
SET columnar.compression TO 'pglz';
CREATE TABLE test_pglz (LIKE test_zstd) USING columnar;

69
src/test/regress/sql/columnar_citus_integration.sql
View File

@ -28,6 +28,24 @@ SELECT run_command_on_placements('table_option',$cmd$
SELECT compression FROM columnar.options WHERE regclass = '%s'::regclass;
$cmd$);
-- setting: compression_level
-- get baseline for setting
SELECT run_command_on_placements('table_option',$cmd$
SELECT compression_level FROM columnar.options WHERE regclass = '%s'::regclass;
$cmd$);
-- change setting
SELECT alter_columnar_table_set('table_option', compression_level => 13);
-- verify setting
SELECT run_command_on_placements('table_option',$cmd$
SELECT compression_level FROM columnar.options WHERE regclass = '%s'::regclass;
$cmd$);
-- reset setting
SELECT alter_columnar_table_reset('table_option', compression_level => true);
-- verify setting
SELECT run_command_on_placements('table_option',$cmd$
SELECT compression_level FROM columnar.options WHERE regclass = '%s'::regclass;
$cmd$);
-- setting: chunk_row_count
-- get baseline for setting
SELECT run_command_on_placements('table_option',$cmd$
@ -69,12 +87,13 @@ CREATE TABLE table_option_2 (a int, b text) USING columnar;
SELECT alter_columnar_table_set('table_option_2',
chunk_row_count => 100,
stripe_row_count => 1000,
compression => 'pglz');
compression => 'pglz',
compression_level => 15);
SELECT create_distributed_table('table_option_2', 'a');
-- verify settings on placements
SELECT run_command_on_placements('table_option_2',$cmd$
SELECT ROW(chunk_row_count, stripe_row_count, compression) FROM columnar.options WHERE regclass = '%s'::regclass;
SELECT ROW(chunk_row_count, stripe_row_count, compression, compression_level) FROM columnar.options WHERE regclass = '%s'::regclass;
$cmd$);
DROP TABLE table_option, table_option_2;
@ -104,6 +123,24 @@ SELECT run_command_on_placements('table_option',$cmd$
SELECT compression FROM columnar.options WHERE regclass = '%s'::regclass;
$cmd$);
-- setting: compression_level
-- get baseline for setting
SELECT run_command_on_placements('table_option',$cmd$
SELECT compression_level FROM columnar.options WHERE regclass = '%s'::regclass;
$cmd$);
-- change setting
SELECT alter_columnar_table_set('table_option', compression_level => 17);
-- verify setting
SELECT run_command_on_placements('table_option',$cmd$
SELECT compression_level FROM columnar.options WHERE regclass = '%s'::regclass;
$cmd$);
-- reset setting
SELECT alter_columnar_table_reset('table_option', compression_level => true);
-- verify setting
SELECT run_command_on_placements('table_option',$cmd$
SELECT compression_level FROM columnar.options WHERE regclass = '%s'::regclass;
$cmd$);
-- setting: chunk_row_count
-- get baseline for setting
SELECT run_command_on_placements('table_option',$cmd$
@ -145,12 +182,13 @@ CREATE TABLE table_option_2 (a int, b text) USING columnar;
SELECT alter_columnar_table_set('table_option_2',
chunk_row_count => 100,
stripe_row_count => 1000,
compression => 'pglz');
compression => 'pglz',
compression_level => 19);
SELECT create_distributed_table('table_option_2', 'a');
-- verify settings on placements
SELECT run_command_on_placements('table_option_2',$cmd$
SELECT ROW(chunk_row_count, stripe_row_count, compression) FROM columnar.options WHERE regclass = '%s'::regclass;
SELECT ROW(chunk_row_count, stripe_row_count, compression, compression_level) FROM columnar.options WHERE regclass = '%s'::regclass;
$cmd$);
DROP TABLE table_option, table_option_2;
@ -177,6 +215,24 @@ SELECT run_command_on_placements('table_option_reference',$cmd$
SELECT compression FROM columnar.options WHERE regclass = '%s'::regclass;
$cmd$);
-- setting: compression_level
-- get baseline for setting
SELECT run_command_on_placements('table_option_reference',$cmd$
SELECT compression_level FROM columnar.options WHERE regclass = '%s'::regclass;
$cmd$);
-- change setting
SELECT alter_columnar_table_set('table_option_reference', compression_level => 11);
-- verify setting
SELECT run_command_on_placements('table_option_reference',$cmd$
SELECT compression_level FROM columnar.options WHERE regclass = '%s'::regclass;
$cmd$);
-- reset setting
SELECT alter_columnar_table_reset('table_option_reference', compression_level => true);
-- verify setting
SELECT run_command_on_placements('table_option_reference',$cmd$
SELECT compression_level FROM columnar.options WHERE regclass = '%s'::regclass;
$cmd$);
-- setting: chunk_row_count
-- get baseline for setting
SELECT run_command_on_placements('table_option_reference',$cmd$
@ -218,12 +274,13 @@ CREATE TABLE table_option_reference_2 (a int, b text) USING columnar;
SELECT alter_columnar_table_set('table_option_reference_2',
chunk_row_count => 100,
stripe_row_count => 1000,
compression => 'pglz');
compression => 'pglz',
compression_level => 9);
SELECT create_reference_table('table_option_reference_2');
-- verify settings on placements
SELECT run_command_on_placements('table_option_reference_2',$cmd$
SELECT ROW(chunk_row_count, stripe_row_count, compression) FROM columnar.options WHERE regclass = '%s'::regclass;
SELECT ROW(chunk_row_count, stripe_row_count, compression, compression_level) FROM columnar.options WHERE regclass = '%s'::regclass;
$cmd$);
DROP TABLE table_option_reference, table_option_reference_2;

1
src/test/regress/sql/upgrade_columnar_before.sql
View File

@ -107,5 +107,6 @@ INSERT INTO test_options_2 SELECT i, floor(i/1000) FROM generate_series(1, 10000
SELECT alter_columnar_table_set('test_options_2', chunk_row_count => 2000);
SELECT alter_columnar_table_set('test_options_2', stripe_row_count => 6000);
SELECT alter_columnar_table_set('test_options_2', compression => 'none');
SELECT alter_columnar_table_set('test_options_2', compression_level => 13);
INSERT INTO test_options_2 SELECT i, floor(i/2000) FROM generate_series(1, 10000) i;