diff --git a/src/backend/columnar/cstore.c b/src/backend/columnar/cstore.c index 8cef3e00a..5c4c7e120 100644 --- a/src/backend/columnar/cstore.c +++ b/src/backend/columnar/cstore.c @@ -31,6 +31,7 @@ int cstore_compression = DEFAULT_COMPRESSION_TYPE; int cstore_stripe_row_count = DEFAULT_STRIPE_ROW_COUNT; int cstore_chunk_row_count = DEFAULT_CHUNK_ROW_COUNT; +int columnar_compression_level = 3; static const struct config_enum_entry cstore_compression_options[] = { @@ -60,6 +61,19 @@ cstore_init() NULL, NULL); + DefineCustomIntVariable("columnar.compression_level", + "Compression level to be used with zstd.", + NULL, + &columnar_compression_level, + 3, + COMPRESSION_LEVEL_MIN, + COMPRESSION_LEVEL_MAX, + PGC_USERSET, + 0, + NULL, + NULL, + NULL); + DefineCustomIntVariable("columnar.stripe_row_count", "Maximum number of tuples per stripe.", NULL, diff --git a/src/backend/columnar/cstore_compression.c b/src/backend/columnar/cstore_compression.c index a9740a6c2..65478d286 100644 --- a/src/backend/columnar/cstore_compression.c +++ b/src/backend/columnar/cstore_compression.c @@ -53,8 +53,10 @@ typedef struct CStoreCompressHeader * outputBuffer is valid only if the function returns true. */ bool -CompressBuffer(StringInfo inputBuffer, StringInfo outputBuffer, - CompressionType compressionType) +CompressBuffer(StringInfo inputBuffer, + StringInfo outputBuffer, + CompressionType compressionType, + int compressionLevel) { switch (compressionType) { @@ -89,7 +91,6 @@ CompressBuffer(StringInfo inputBuffer, StringInfo outputBuffer, case COMPRESSION_ZSTD: { int maximumLength = ZSTD_compressBound(inputBuffer->len); - int compressionLevel = 3; resetStringInfo(outputBuffer); enlargeStringInfo(outputBuffer, maximumLength); diff --git a/src/backend/columnar/cstore_metadata_tables.c b/src/backend/columnar/cstore_metadata_tables.c index d2cdf66fe..9f561f38c 100644 --- a/src/backend/columnar/cstore_metadata_tables.c +++ b/src/backend/columnar/cstore_metadata_tables.c @@ -102,11 +102,12 @@ static bool WriteColumnarOptions(Oid regclass, ColumnarOptions *options, bool ov PG_FUNCTION_INFO_V1(columnar_relation_storageid); /* constants for columnar.options */ -#define Natts_cstore_options 4 +#define Natts_cstore_options 5 #define Anum_cstore_options_regclass 1 #define Anum_cstore_options_chunk_row_count 2 #define Anum_cstore_options_stripe_row_count 3 -#define Anum_cstore_options_compression 4 +#define Anum_cstore_options_compression_level 4 +#define Anum_cstore_options_compression 5 /* ---------------- * columnar.options definition. @@ -117,6 +118,7 @@ typedef struct FormData_cstore_options Oid regclass; int32 chunk_row_count; int32 stripe_row_count; + int32 compressionLevel; NameData compression; #ifdef CATALOG_VARLEN /* variable-length fields start here */ @@ -137,7 +139,7 @@ typedef FormData_cstore_options *Form_cstore_options; #define Anum_cstore_stripes_row_count 8 /* constants for cstore_skipnodes */ -#define Natts_cstore_skipnodes 13 +#define Natts_cstore_skipnodes 14 #define Anum_cstore_skipnodes_storageid 1 #define Anum_cstore_skipnodes_stripe 2 #define Anum_cstore_skipnodes_attr 3 @@ -150,7 +152,8 @@ typedef FormData_cstore_options *Form_cstore_options; #define Anum_cstore_skipnodes_exists_stream_offset 10 #define Anum_cstore_skipnodes_exists_stream_length 11 #define Anum_cstore_skipnodes_value_compression_type 12 -#define Anum_cstore_skipnodes_value_decompressed_size 13 +#define Anum_cstore_skipnodes_value_compression_level 13 +#define Anum_cstore_skipnodes_value_decompressed_size 14 /* @@ -173,6 +176,7 @@ InitColumnarOptions(Oid regclass) .chunkRowCount = cstore_chunk_row_count, .stripeRowCount = cstore_stripe_row_count, .compressionType = cstore_compression, + .compressionLevel = columnar_compression_level }; WriteColumnarOptions(regclass, &defaultOptions, false); @@ -215,6 +219,7 @@ WriteColumnarOptions(Oid regclass, ColumnarOptions *options, bool overwrite) ObjectIdGetDatum(regclass), Int32GetDatum(options->chunkRowCount), Int32GetDatum(options->stripeRowCount), + Int32GetDatum(options->compressionLevel), 0, /* to be filled below */ }; @@ -246,6 +251,7 @@ WriteColumnarOptions(Oid regclass, ColumnarOptions *options, bool overwrite) bool update[Natts_cstore_options] = { 0 }; update[Anum_cstore_options_chunk_row_count - 1] = true; update[Anum_cstore_options_stripe_row_count - 1] = true; + update[Anum_cstore_options_compression_level - 1] = true; update[Anum_cstore_options_compression - 1] = true; HeapTuple tuple = heap_modify_tuple(heapTuple, tupleDescriptor, @@ -363,6 +369,7 @@ ReadColumnarOptions(Oid regclass, ColumnarOptions *options) options->chunkRowCount = tupOptions->chunk_row_count; options->stripeRowCount = tupOptions->stripe_row_count; + options->compressionLevel = tupOptions->compressionLevel; options->compressionType = ParseCompressionType(NameStr(tupOptions->compression)); } else @@ -371,6 +378,7 @@ ReadColumnarOptions(Oid regclass, ColumnarOptions *options) options->compressionType = cstore_compression; options->stripeRowCount = cstore_stripe_row_count; options->chunkRowCount = cstore_chunk_row_count; + options->compressionLevel = columnar_compression_level; } systable_endscan_ordered(scanDescriptor); @@ -418,6 +426,7 @@ SaveStripeSkipList(RelFileNode relfilenode, uint64 stripe, StripeSkipList *strip Int64GetDatum(skipNode->existsChunkOffset), Int64GetDatum(skipNode->existsLength), Int32GetDatum(skipNode->valueCompressionType), + Int32GetDatum(skipNode->valueCompressionLevel), Int64GetDatum(skipNode->decompressedValueSize) }; @@ -524,6 +533,8 @@ ReadStripeSkipList(RelFileNode relfilenode, uint64 stripe, TupleDesc tupleDescri DatumGetInt64(datumArray[Anum_cstore_skipnodes_exists_stream_length - 1]); skipNode->valueCompressionType = DatumGetInt32(datumArray[Anum_cstore_skipnodes_value_compression_type - 1]); + skipNode->valueCompressionLevel = + DatumGetInt32(datumArray[Anum_cstore_skipnodes_value_compression_level - 1]); skipNode->decompressedValueSize = DatumGetInt64(datumArray[Anum_cstore_skipnodes_value_decompressed_size - 1]); diff --git a/src/backend/columnar/cstore_tableam.c b/src/backend/columnar/cstore_tableam.c index 35d426740..f802330d3 100644 --- a/src/backend/columnar/cstore_tableam.c +++ b/src/backend/columnar/cstore_tableam.c @@ -632,9 +632,7 @@ cstore_relation_copy_for_cluster(Relation OldHeap, Relation NewHeap, ReadColumnarOptions(OldHeap->rd_id, &cstoreOptions); TableWriteState *writeState = CStoreBeginWrite(NewHeap->rd_node, - cstoreOptions.compressionType, - cstoreOptions.stripeRowCount, - cstoreOptions.chunkRowCount, + cstoreOptions, targetDesc); TableReadState *readState = CStoreBeginRead(OldHeap, sourceDesc, @@ -1343,10 +1341,12 @@ CitusCreateAlterColumnarTableSet(char *qualifiedRelationName, "SELECT alter_columnar_table_set(%s, " "chunk_row_count => %d, " "stripe_row_count => %lu, " + "compression_level => %d, " "compression => %s);", quote_literal_cstr(qualifiedRelationName), options->chunkRowCount, options->stripeRowCount, + options->compressionLevel, quote_literal_cstr(CompressionTypeStr(options->compressionType))); return buf.data; @@ -1524,6 +1524,23 @@ alter_columnar_table_set(PG_FUNCTION_ARGS) CompressionTypeStr(options.compressionType)))); } + /* compression_level => not null */ + if (!PG_ARGISNULL(4)) + { + options.compressionLevel = PG_GETARG_INT32(4); + if (options.compressionLevel < COMPRESSION_LEVEL_MIN || + options.compressionLevel > COMPRESSION_LEVEL_MAX) + { + ereport(ERROR, (errmsg("compression level out of range"), + errhint("compression level must be between %d and %d", + COMPRESSION_LEVEL_MIN, + COMPRESSION_LEVEL_MAX))); + } + + ereport(DEBUG1, (errmsg("updating compression level to %d", + options.compressionLevel))); + } + if (EnableDDLPropagation && IsCitusTable(relationId)) { /* when a columnar table is distributed update all settings on the shards */ @@ -1608,6 +1625,14 @@ alter_columnar_table_reset(PG_FUNCTION_ARGS) CompressionTypeStr(options.compressionType)))); } + /* compression_level => true */ + if (!PG_ARGISNULL(4) && PG_GETARG_BOOL(4)) + { + options.compressionLevel = columnar_compression_level; + ereport(DEBUG1, (errmsg("reseting compression level to %d", + columnar_compression_level))); + } + if (EnableDDLPropagation && IsCitusTable(relationId)) { /* when a columnar table is distributed update all settings on the shards */ diff --git a/src/backend/columnar/cstore_writer.c b/src/backend/columnar/cstore_writer.c index eefdb5f93..548b3cfc5 100644 --- a/src/backend/columnar/cstore_writer.c +++ b/src/backend/columnar/cstore_writer.c @@ -61,8 +61,7 @@ static StringInfo CopyStringInfo(StringInfo sourceString); */ TableWriteState * CStoreBeginWrite(RelFileNode relfilenode, - CompressionType compressionType, - uint64 stripeMaxRowCount, uint32 chunkRowCount, + ColumnarOptions options, TupleDesc tupleDescriptor) { /* get comparison function pointers for each of the columns */ @@ -98,13 +97,11 @@ CStoreBeginWrite(RelFileNode relfilenode, memset(columnMaskArray, true, columnCount); ChunkData *chunkData = CreateEmptyChunkData(columnCount, columnMaskArray, - chunkRowCount); + options.chunkRowCount); TableWriteState *writeState = palloc0(sizeof(TableWriteState)); writeState->relfilenode = relfilenode; - writeState->compressionType = compressionType; - writeState->stripeMaxRowCount = stripeMaxRowCount; - writeState->chunkRowCount = chunkRowCount; + writeState->options = options; writeState->tupleDescriptor = CreateTupleDescCopy(tupleDescriptor); writeState->comparisonFunctionArray = comparisonFunctionArray; writeState->stripeBuffers = NULL; @@ -135,15 +132,16 @@ CStoreWriteRow(TableWriteState *writeState, Datum *columnValues, bool *columnNul StripeBuffers *stripeBuffers = writeState->stripeBuffers; StripeSkipList *stripeSkipList = writeState->stripeSkipList; uint32 columnCount = writeState->tupleDescriptor->natts; - const uint32 chunkRowCount = writeState->chunkRowCount; + ColumnarOptions *options = &writeState->options; + const uint32 chunkRowCount = options->chunkRowCount; ChunkData *chunkData = writeState->chunkData; MemoryContext oldContext = MemoryContextSwitchTo(writeState->stripeWriteContext); if (stripeBuffers == NULL) { - stripeBuffers = CreateEmptyStripeBuffers(writeState->stripeMaxRowCount, + stripeBuffers = CreateEmptyStripeBuffers(options->stripeRowCount, chunkRowCount, columnCount); - stripeSkipList = CreateEmptyStripeSkipList(writeState->stripeMaxRowCount, + stripeSkipList = CreateEmptyStripeSkipList(options->stripeRowCount, chunkRowCount, columnCount); writeState->stripeBuffers = stripeBuffers; writeState->stripeSkipList = stripeSkipList; @@ -206,7 +204,7 @@ CStoreWriteRow(TableWriteState *writeState, Datum *columnValues, bool *columnNul } stripeBuffers->rowCount++; - if (stripeBuffers->rowCount >= writeState->stripeMaxRowCount) + if (stripeBuffers->rowCount >= options->stripeRowCount) { CStoreFlushPendingWrites(writeState); } @@ -413,7 +411,7 @@ FlushStripe(TableWriteState *writeState) TupleDesc tupleDescriptor = writeState->tupleDescriptor; uint32 columnCount = tupleDescriptor->natts; uint32 chunkCount = stripeSkipList->chunkCount; - uint32 chunkRowCount = writeState->chunkRowCount; + uint32 chunkRowCount = writeState->options.chunkRowCount; uint32 lastChunkIndex = stripeBuffers->rowCount / chunkRowCount; uint32 lastChunkRowCount = stripeBuffers->rowCount % chunkRowCount; uint64 stripeSize = 0; @@ -463,6 +461,7 @@ FlushStripe(TableWriteState *writeState) chunkSkipNode->valueChunkOffset = stripeSize; chunkSkipNode->valueLength = valueBufferSize; chunkSkipNode->valueCompressionType = valueCompressionType; + chunkSkipNode->valueCompressionLevel = writeState->options.compressionLevel; chunkSkipNode->decompressedValueSize = chunkBuffers->decompressedValueSize; stripeSize += valueBufferSize; @@ -606,7 +605,8 @@ SerializeChunkData(TableWriteState *writeState, uint32 chunkIndex, uint32 rowCou uint32 columnIndex = 0; StripeBuffers *stripeBuffers = writeState->stripeBuffers; ChunkData *chunkData = writeState->chunkData; - CompressionType requestedCompressionType = writeState->compressionType; + CompressionType requestedCompressionType = writeState->options.compressionType; + int compressionLevel = writeState->options.compressionLevel; const uint32 columnCount = stripeBuffers->columnCount; StringInfo compressionBuffer = writeState->compressionBuffer; @@ -643,7 +643,8 @@ SerializeChunkData(TableWriteState *writeState, uint32 chunkIndex, uint32 rowCou * with compressed data and store compression type. */ bool compressed = CompressBuffer(serializedValueBuffer, compressionBuffer, - requestedCompressionType); + requestedCompressionType, + compressionLevel); if (compressed) { serializedValueBuffer = compressionBuffer; diff --git a/src/backend/columnar/sql/columnar--9.5-1--10.0-1.sql b/src/backend/columnar/sql/columnar--9.5-1--10.0-1.sql index 005da0395..c8c6feded 100644 --- a/src/backend/columnar/sql/columnar--9.5-1--10.0-1.sql +++ b/src/backend/columnar/sql/columnar--9.5-1--10.0-1.sql @@ -9,6 +9,7 @@ CREATE TABLE options ( regclass regclass NOT NULL PRIMARY KEY, chunk_row_count int NOT NULL, stripe_row_count int NOT NULL, + compression_level int NOT NULL, compression name NOT NULL ) WITH (user_catalog_table = true); @@ -41,6 +42,7 @@ CREATE TABLE columnar_skipnodes ( exists_stream_offset bigint NOT NULL, exists_stream_length bigint NOT NULL, value_compression_type int NOT NULL, + value_compression_level int NOT NULL, value_decompressed_length bigint NOT NULL, PRIMARY KEY (storageid, stripe, attr, chunk), FOREIGN KEY (storageid, stripe) REFERENCES columnar_stripes(storageid, stripe) ON DELETE CASCADE diff --git a/src/backend/columnar/sql/downgrades/columnar--10.0-1--9.5-1.sql b/src/backend/columnar/sql/downgrades/columnar--10.0-1--9.5-1.sql index 1a045de23..610e72178 100644 --- a/src/backend/columnar/sql/downgrades/columnar--10.0-1--9.5-1.sql +++ b/src/backend/columnar/sql/downgrades/columnar--10.0-1--9.5-1.sql @@ -11,13 +11,15 @@ IF substring(current_Setting('server_version'), '\d+')::int >= 12 THEN table_name regclass, chunk_row_count bool, stripe_row_count bool, - compression bool); + compression bool, + compression_level bool); DROP FUNCTION pg_catalog.alter_columnar_table_set( table_name regclass, chunk_row_count int, stripe_row_count int, - compression name); + compression name, + compression_level int); DROP ACCESS METHOD columnar; diff --git a/src/backend/columnar/sql/udfs/alter_columnar_table_reset/10.0-1.sql b/src/backend/columnar/sql/udfs/alter_columnar_table_reset/10.0-1.sql index 1fb527ec1..3390818a3 100644 --- a/src/backend/columnar/sql/udfs/alter_columnar_table_reset/10.0-1.sql +++ b/src/backend/columnar/sql/udfs/alter_columnar_table_reset/10.0-1.sql @@ -2,7 +2,8 @@ CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_reset( table_name regclass, chunk_row_count bool DEFAULT false, stripe_row_count bool DEFAULT false, - compression bool DEFAULT false) + compression bool DEFAULT false, + compression_level bool DEFAULT false) RETURNS void LANGUAGE C AS 'MODULE_PATHNAME', 'alter_columnar_table_reset'; @@ -11,5 +12,6 @@ COMMENT ON FUNCTION pg_catalog.alter_columnar_table_reset( table_name regclass, chunk_row_count bool, stripe_row_count bool, - compression bool) + compression bool, + compression_level bool) IS 'reset on or more options on a cstore table to the system defaults'; diff --git a/src/backend/columnar/sql/udfs/alter_columnar_table_reset/latest.sql b/src/backend/columnar/sql/udfs/alter_columnar_table_reset/latest.sql index 1fb527ec1..3390818a3 100644 --- a/src/backend/columnar/sql/udfs/alter_columnar_table_reset/latest.sql +++ b/src/backend/columnar/sql/udfs/alter_columnar_table_reset/latest.sql @@ -2,7 +2,8 @@ CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_reset( table_name regclass, chunk_row_count bool DEFAULT false, stripe_row_count bool DEFAULT false, - compression bool DEFAULT false) + compression bool DEFAULT false, + compression_level bool DEFAULT false) RETURNS void LANGUAGE C AS 'MODULE_PATHNAME', 'alter_columnar_table_reset'; @@ -11,5 +12,6 @@ COMMENT ON FUNCTION pg_catalog.alter_columnar_table_reset( table_name regclass, chunk_row_count bool, stripe_row_count bool, - compression bool) + compression bool, + compression_level bool) IS 'reset on or more options on a cstore table to the system defaults'; diff --git a/src/backend/columnar/sql/udfs/alter_columnar_table_set/10.0-1.sql b/src/backend/columnar/sql/udfs/alter_columnar_table_set/10.0-1.sql index 7d7cc2e62..bb22bfa63 100644 --- a/src/backend/columnar/sql/udfs/alter_columnar_table_set/10.0-1.sql +++ b/src/backend/columnar/sql/udfs/alter_columnar_table_set/10.0-1.sql @@ -2,7 +2,8 @@ CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_set( table_name regclass, chunk_row_count int DEFAULT NULL, stripe_row_count int DEFAULT NULL, - compression name DEFAULT null) + compression name DEFAULT null, + compression_level int DEFAULT NULL) RETURNS void LANGUAGE C AS 'MODULE_PATHNAME', 'alter_columnar_table_set'; @@ -11,5 +12,6 @@ COMMENT ON FUNCTION pg_catalog.alter_columnar_table_set( table_name regclass, chunk_row_count int, stripe_row_count int, - compression name) + compression name, + compression_level int) IS 'set one or more options on a cstore table, when set to NULL no change is made'; diff --git a/src/backend/columnar/sql/udfs/alter_columnar_table_set/latest.sql b/src/backend/columnar/sql/udfs/alter_columnar_table_set/latest.sql index 7d7cc2e62..bb22bfa63 100644 --- a/src/backend/columnar/sql/udfs/alter_columnar_table_set/latest.sql +++ b/src/backend/columnar/sql/udfs/alter_columnar_table_set/latest.sql @@ -2,7 +2,8 @@ CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_set( table_name regclass, chunk_row_count int DEFAULT NULL, stripe_row_count int DEFAULT NULL, - compression name DEFAULT null) + compression name DEFAULT null, + compression_level int DEFAULT NULL) RETURNS void LANGUAGE C AS 'MODULE_PATHNAME', 'alter_columnar_table_set'; @@ -11,5 +12,6 @@ COMMENT ON FUNCTION pg_catalog.alter_columnar_table_set( table_name regclass, chunk_row_count int, stripe_row_count int, - compression name) + compression name, + compression_level int) IS 'set one or more options on a cstore table, when set to NULL no change is made'; diff --git a/src/backend/columnar/sql/udfs/columnar_ensure_objects_exist/10.0-1.sql b/src/backend/columnar/sql/udfs/columnar_ensure_objects_exist/10.0-1.sql index 877c3a069..7d88914e2 100644 --- a/src/backend/columnar/sql/udfs/columnar_ensure_objects_exist/10.0-1.sql +++ b/src/backend/columnar/sql/udfs/columnar_ensure_objects_exist/10.0-1.sql @@ -31,12 +31,14 @@ IF NOT EXISTS (SELECT 1 FROM pg_am WHERE amname = 'columnar') THEN table_name regclass, chunk_row_count int, stripe_row_count int, - compression name); + compression name, + compression_level int); ALTER EXTENSION citus ADD FUNCTION pg_catalog.alter_columnar_table_reset( table_name regclass, chunk_row_count bool, stripe_row_count bool, - compression bool); + compression bool, + compression_level bool); END IF; END IF; diff --git a/src/backend/columnar/sql/udfs/columnar_ensure_objects_exist/latest.sql b/src/backend/columnar/sql/udfs/columnar_ensure_objects_exist/latest.sql index 877c3a069..7d88914e2 100644 --- a/src/backend/columnar/sql/udfs/columnar_ensure_objects_exist/latest.sql +++ b/src/backend/columnar/sql/udfs/columnar_ensure_objects_exist/latest.sql @@ -31,12 +31,14 @@ IF NOT EXISTS (SELECT 1 FROM pg_am WHERE amname = 'columnar') THEN table_name regclass, chunk_row_count int, stripe_row_count int, - compression name); + compression name, + compression_level int); ALTER EXTENSION citus ADD FUNCTION pg_catalog.alter_columnar_table_reset( table_name regclass, chunk_row_count bool, stripe_row_count bool, - compression bool); + compression bool, + compression_level bool); END IF; END IF; diff --git a/src/backend/columnar/write_state_management.c b/src/backend/columnar/write_state_management.c index e04ca38f1..192a4f060 100644 --- a/src/backend/columnar/write_state_management.c +++ b/src/backend/columnar/write_state_management.c @@ -183,9 +183,7 @@ cstore_init_write_state(Relation relation, TupleDesc tupdesc, SubXidWriteState *stackEntry = palloc0(sizeof(SubXidWriteState)); stackEntry->writeState = CStoreBeginWrite(relation->rd_node, - cstoreOptions.compressionType, - cstoreOptions.stripeRowCount, - cstoreOptions.chunkRowCount, + cstoreOptions, tupdesc); stackEntry->subXid = currentSubXid; stackEntry->next = hashEntry->writeStateStack; diff --git a/src/include/columnar/cstore.h b/src/include/columnar/cstore.h index a1a010f34..d1ee08cb4 100644 --- a/src/include/columnar/cstore.h +++ b/src/include/columnar/cstore.h @@ -34,6 +34,8 @@ #define STRIPE_ROW_COUNT_MAXIMUM 10000000 #define CHUNK_ROW_COUNT_MINIMUM 1000 #define CHUNK_ROW_COUNT_MAXIMUM 100000 +#define COMPRESSION_LEVEL_MIN 1 +#define COMPRESSION_LEVEL_MAX 19 /* String representations of compression types */ #define COMPRESSION_STRING_NONE "none" @@ -73,6 +75,7 @@ typedef struct ColumnarOptions uint64 stripeRowCount; uint32 chunkRowCount; CompressionType compressionType; + int compressionLevel; } ColumnarOptions; @@ -125,6 +128,7 @@ typedef struct ColumnChunkSkipNode uint64 decompressedValueSize; CompressionType valueCompressionType; + int valueCompressionLevel; } ColumnChunkSkipNode; @@ -228,7 +232,6 @@ typedef struct TableReadState /* TableWriteState represents state of a cstore file write operation. */ typedef struct TableWriteState { - CompressionType compressionType; TupleDesc tupleDescriptor; FmgrInfo **comparisonFunctionArray; RelFileNode relfilenode; @@ -237,8 +240,7 @@ typedef struct TableWriteState MemoryContext perTupleContext; StripeBuffers *stripeBuffers; StripeSkipList *stripeSkipList; - uint32 stripeMaxRowCount; - uint32 chunkRowCount; + ColumnarOptions options; ChunkData *chunkData; /* @@ -253,6 +255,7 @@ typedef struct TableWriteState extern int cstore_compression; extern int cstore_stripe_row_count; extern int cstore_chunk_row_count; +extern int columnar_compression_level; extern void cstore_init(void); @@ -260,9 +263,7 @@ extern CompressionType ParseCompressionType(const char *compressionTypeString); /* Function declarations for writing to a cstore file */ extern TableWriteState * CStoreBeginWrite(RelFileNode relfilenode, - CompressionType compressionType, - uint64 stripeMaxRowCount, - uint32 chunkRowCount, + ColumnarOptions options, TupleDesc tupleDescriptor); extern void CStoreWriteRow(TableWriteState *state, Datum *columnValues, bool *columnNulls); @@ -287,8 +288,10 @@ extern ChunkData * CreateEmptyChunkData(uint32 columnCount, bool *columnMask, uint32 chunkRowCount); extern void FreeChunkData(ChunkData *chunkData); extern uint64 CStoreTableRowCount(Relation relation); -extern bool CompressBuffer(StringInfo inputBuffer, StringInfo outputBuffer, - CompressionType compressionType); +extern bool CompressBuffer(StringInfo inputBuffer, + StringInfo outputBuffer, + CompressionType compressionType, + int compressionLevel); extern StringInfo DecompressBuffer(StringInfo buffer, CompressionType compressionType, uint64 decompressedSize); extern const char * CompressionTypeStr(CompressionType type); diff --git a/src/test/regress/expected/am_empty.out b/src/test/regress/expected/am_empty.out index 77a363286..2473cb8ce 100644 --- a/src/test/regress/expected/am_empty.out +++ b/src/test/regress/expected/am_empty.out @@ -24,9 +24,9 @@ SELECT alter_columnar_table_set('t_compressed', chunk_row_count => 100); (1 row) SELECT * FROM columnar.options WHERE regclass = 't_compressed'::regclass; - regclass | chunk_row_count | stripe_row_count | compression + regclass | chunk_row_count | stripe_row_count | compression_level | compression --------------------------------------------------------------------- - t_compressed | 100 | 100 | pglz + t_compressed | 100 | 100 | 3 | pglz (1 row) -- select diff --git a/src/test/regress/expected/am_matview.out b/src/test/regress/expected/am_matview.out index c89be4fcd..dab62957a 100644 --- a/src/test/regress/expected/am_matview.out +++ b/src/test/regress/expected/am_matview.out @@ -25,9 +25,9 @@ SELECT * FROM t_view a ORDER BY a; -- show columnar options for materialized view SELECT * FROM columnar.options WHERE regclass = 't_view'::regclass; - regclass | chunk_row_count | stripe_row_count | compression + regclass | chunk_row_count | stripe_row_count | compression_level | compression --------------------------------------------------------------------- - t_view | 10000 | 150000 | none + t_view | 10000 | 150000 | 3 | none (1 row) -- show we can set options on a materialized view @@ -39,18 +39,18 @@ SELECT alter_columnar_table_set('t_view', compression => 'pglz'); SELECT * FROM columnar.options WHERE regclass = 't_view'::regclass; - regclass | chunk_row_count | stripe_row_count | compression + regclass | chunk_row_count | stripe_row_count | compression_level | compression --------------------------------------------------------------------- - t_view | 10000 | 150000 | pglz + t_view | 10000 | 150000 | 3 | pglz (1 row) REFRESH MATERIALIZED VIEW t_view; -- verify options have not been changed SELECT * FROM columnar.options WHERE regclass = 't_view'::regclass; - regclass | chunk_row_count | stripe_row_count | compression + regclass | chunk_row_count | stripe_row_count | compression_level | compression --------------------------------------------------------------------- - t_view | 10000 | 150000 | pglz + t_view | 10000 | 150000 | 3 | pglz (1 row) SELECT * FROM t_view a ORDER BY a; diff --git a/src/test/regress/expected/am_tableoptions.out b/src/test/regress/expected/am_tableoptions.out index 70278f8b4..3bb1fbc8b 100644 --- a/src/test/regress/expected/am_tableoptions.out +++ b/src/test/regress/expected/am_tableoptions.out @@ -5,9 +5,9 @@ INSERT INTO table_options SELECT generate_series(1,100); -- show table_options settings SELECT * FROM columnar.options WHERE regclass = 'table_options'::regclass; - regclass | chunk_row_count | stripe_row_count | compression + regclass | chunk_row_count | stripe_row_count | compression_level | compression --------------------------------------------------------------------- - table_options | 10000 | 150000 | none + table_options | 10000 | 150000 | 3 | none (1 row) -- test changing the compression @@ -20,9 +20,24 @@ SELECT alter_columnar_table_set('table_options', compression => 'pglz'); -- show table_options settings SELECT * FROM columnar.options WHERE regclass = 'table_options'::regclass; - regclass | chunk_row_count | stripe_row_count | compression + regclass | chunk_row_count | stripe_row_count | compression_level | compression --------------------------------------------------------------------- - table_options | 10000 | 150000 | pglz + table_options | 10000 | 150000 | 3 | pglz +(1 row) + +-- test changing the compression level +SELECT alter_columnar_table_set('table_options', compression_level => 5); + alter_columnar_table_set +--------------------------------------------------------------------- + +(1 row) + +-- show table_options settings +SELECT * FROM columnar.options +WHERE regclass = 'table_options'::regclass; + regclass | chunk_row_count | stripe_row_count | compression_level | compression +--------------------------------------------------------------------- + table_options | 10000 | 150000 | 5 | pglz (1 row) -- test changing the chunk_row_count @@ -35,9 +50,9 @@ SELECT alter_columnar_table_set('table_options', chunk_row_count => 10); -- show table_options settings SELECT * FROM columnar.options WHERE regclass = 'table_options'::regclass; - regclass | chunk_row_count | stripe_row_count | compression + regclass | chunk_row_count | stripe_row_count | compression_level | compression --------------------------------------------------------------------- - table_options | 10 | 150000 | pglz + table_options | 10 | 150000 | 5 | pglz (1 row) -- test changing the chunk_row_count @@ -50,9 +65,9 @@ SELECT alter_columnar_table_set('table_options', stripe_row_count => 100); -- show table_options settings SELECT * FROM columnar.options WHERE regclass = 'table_options'::regclass; - regclass | chunk_row_count | stripe_row_count | compression + regclass | chunk_row_count | stripe_row_count | compression_level | compression --------------------------------------------------------------------- - table_options | 10 | 100 | pglz + table_options | 10 | 100 | 5 | pglz (1 row) -- VACUUM FULL creates a new table, make sure it copies settings from the table you are vacuuming @@ -60,13 +75,13 @@ VACUUM FULL table_options; -- show table_options settings SELECT * FROM columnar.options WHERE regclass = 'table_options'::regclass; - regclass | chunk_row_count | stripe_row_count | compression + regclass | chunk_row_count | stripe_row_count | compression_level | compression --------------------------------------------------------------------- - table_options | 10 | 100 | pglz + table_options | 10 | 100 | 5 | pglz (1 row) -- set all settings at the same time -SELECT alter_columnar_table_set('table_options', stripe_row_count => 1000, chunk_row_count => 100, compression => 'none'); +SELECT alter_columnar_table_set('table_options', stripe_row_count => 1000, chunk_row_count => 100, compression => 'none', compression_level => 7); alter_columnar_table_set --------------------------------------------------------------------- @@ -75,9 +90,9 @@ SELECT alter_columnar_table_set('table_options', stripe_row_count => 1000, chunk -- show table_options settings SELECT * FROM columnar.options WHERE regclass = 'table_options'::regclass; - regclass | chunk_row_count | stripe_row_count | compression + regclass | chunk_row_count | stripe_row_count | compression_level | compression --------------------------------------------------------------------- - table_options | 100 | 1000 | none + table_options | 100 | 1000 | 7 | none (1 row) -- make sure table options are not changed when VACUUM a table @@ -85,9 +100,9 @@ VACUUM table_options; -- show table_options settings SELECT * FROM columnar.options WHERE regclass = 'table_options'::regclass; - regclass | chunk_row_count | stripe_row_count | compression + regclass | chunk_row_count | stripe_row_count | compression_level | compression --------------------------------------------------------------------- - table_options | 100 | 1000 | none + table_options | 100 | 1000 | 7 | none (1 row) -- make sure table options are not changed when VACUUM FULL a table @@ -95,9 +110,9 @@ VACUUM FULL table_options; -- show table_options settings SELECT * FROM columnar.options WHERE regclass = 'table_options'::regclass; - regclass | chunk_row_count | stripe_row_count | compression + regclass | chunk_row_count | stripe_row_count | compression_level | compression --------------------------------------------------------------------- - table_options | 100 | 1000 | none + table_options | 100 | 1000 | 7 | none (1 row) -- make sure table options are not changed when truncating a table @@ -105,31 +120,32 @@ TRUNCATE table_options; -- show table_options settings SELECT * FROM columnar.options WHERE regclass = 'table_options'::regclass; - regclass | chunk_row_count | stripe_row_count | compression + regclass | chunk_row_count | stripe_row_count | compression_level | compression --------------------------------------------------------------------- - table_options | 100 | 1000 | none + table_options | 100 | 1000 | 7 | none (1 row) ALTER TABLE table_options ALTER COLUMN a TYPE bigint; -- show table_options settings SELECT * FROM columnar.options WHERE regclass = 'table_options'::regclass; - regclass | chunk_row_count | stripe_row_count | compression + regclass | chunk_row_count | stripe_row_count | compression_level | compression --------------------------------------------------------------------- - table_options | 100 | 1000 | none + table_options | 100 | 1000 | 7 | none (1 row) -- reset settings one by one to the version of the GUC's SET columnar.chunk_row_count TO 1000; SET columnar.stripe_row_count TO 10000; SET columnar.compression TO 'pglz'; +SET columnar.compression_level TO 11; -- verify setting the GUC's didn't change the settings -- show table_options settings SELECT * FROM columnar.options WHERE regclass = 'table_options'::regclass; - regclass | chunk_row_count | stripe_row_count | compression + regclass | chunk_row_count | stripe_row_count | compression_level | compression --------------------------------------------------------------------- - table_options | 100 | 1000 | none + table_options | 100 | 1000 | 7 | none (1 row) SELECT alter_columnar_table_reset('table_options', chunk_row_count => true); @@ -141,9 +157,9 @@ SELECT alter_columnar_table_reset('table_options', chunk_row_count => true); -- show table_options settings SELECT * FROM columnar.options WHERE regclass = 'table_options'::regclass; - regclass | chunk_row_count | stripe_row_count | compression + regclass | chunk_row_count | stripe_row_count | compression_level | compression --------------------------------------------------------------------- - table_options | 1000 | 1000 | none + table_options | 1000 | 1000 | 7 | none (1 row) SELECT alter_columnar_table_reset('table_options', stripe_row_count => true); @@ -155,9 +171,9 @@ SELECT alter_columnar_table_reset('table_options', stripe_row_count => true); -- show table_options settings SELECT * FROM columnar.options WHERE regclass = 'table_options'::regclass; - regclass | chunk_row_count | stripe_row_count | compression + regclass | chunk_row_count | stripe_row_count | compression_level | compression --------------------------------------------------------------------- - table_options | 1000 | 10000 | none + table_options | 1000 | 10000 | 7 | none (1 row) SELECT alter_columnar_table_reset('table_options', compression => true); @@ -169,28 +185,12 @@ SELECT alter_columnar_table_reset('table_options', compression => true); -- show table_options settings SELECT * FROM columnar.options WHERE regclass = 'table_options'::regclass; - regclass | chunk_row_count | stripe_row_count | compression + regclass | chunk_row_count | stripe_row_count | compression_level | compression --------------------------------------------------------------------- - table_options | 1000 | 10000 | pglz + table_options | 1000 | 10000 | 7 | pglz (1 row) --- verify resetting all settings at once work -SET columnar.chunk_row_count TO 10000; -SET columnar.stripe_row_count TO 100000; -SET columnar.compression TO 'none'; --- show table_options settings -SELECT * FROM columnar.options -WHERE regclass = 'table_options'::regclass; - regclass | chunk_row_count | stripe_row_count | compression ---------------------------------------------------------------------- - table_options | 1000 | 10000 | pglz -(1 row) - -SELECT alter_columnar_table_reset( - 'table_options', - chunk_row_count => true, - stripe_row_count => true, - compression => true); +SELECT alter_columnar_table_reset('table_options', compression_level => true); alter_columnar_table_reset --------------------------------------------------------------------- @@ -199,9 +199,41 @@ SELECT alter_columnar_table_reset( -- show table_options settings SELECT * FROM columnar.options WHERE regclass = 'table_options'::regclass; - regclass | chunk_row_count | stripe_row_count | compression + regclass | chunk_row_count | stripe_row_count | compression_level | compression --------------------------------------------------------------------- - table_options | 10000 | 100000 | none + table_options | 1000 | 10000 | 11 | pglz +(1 row) + +-- verify resetting all settings at once work +SET columnar.chunk_row_count TO 10000; +SET columnar.stripe_row_count TO 100000; +SET columnar.compression TO 'none'; +SET columnar.compression_level TO 13; +-- show table_options settings +SELECT * FROM columnar.options +WHERE regclass = 'table_options'::regclass; + regclass | chunk_row_count | stripe_row_count | compression_level | compression +--------------------------------------------------------------------- + table_options | 1000 | 10000 | 11 | pglz +(1 row) + +SELECT alter_columnar_table_reset( + 'table_options', + chunk_row_count => true, + stripe_row_count => true, + compression => true, + compression_level => true); + alter_columnar_table_reset +--------------------------------------------------------------------- + +(1 row) + +-- show table_options settings +SELECT * FROM columnar.options +WHERE regclass = 'table_options'::regclass; + regclass | chunk_row_count | stripe_row_count | compression_level | compression +--------------------------------------------------------------------- + table_options | 10000 | 100000 | 13 | none (1 row) -- verify edge cases @@ -214,11 +246,18 @@ ERROR: table not_a_columnar_table is not a columnar table -- verify you can't use a compression that is not known SELECT alter_columnar_table_set('table_options', compression => 'foobar'); ERROR: unknown compression type for cstore table: foobar +-- verify cannot set out of range compression levels +SELECT alter_columnar_table_set('table_options', compression_level => 0); +ERROR: compression level out of range +HINT: compression level must be between 1 and 19 +SELECT alter_columnar_table_set('table_options', compression_level => 20); +ERROR: compression level out of range +HINT: compression level must be between 1 and 19 -- verify options are removed when table is dropped DROP TABLE table_options; -- we expect no entries in çstore.options for anything not found int pg_class SELECT * FROM columnar.options o WHERE o.regclass NOT IN (SELECT oid FROM pg_class); - regclass | chunk_row_count | stripe_row_count | compression + regclass | chunk_row_count | stripe_row_count | compression_level | compression --------------------------------------------------------------------- (0 rows) diff --git a/src/test/regress/expected/am_zstd.out b/src/test/regress/expected/am_zstd.out index ac315cdfa..e7dbe0934 100644 --- a/src/test/regress/expected/am_zstd.out +++ b/src/test/regress/expected/am_zstd.out @@ -24,7 +24,7 @@ SELECT count(*) FROM test_zstd; VACUUM VERBOSE test_zstd; INFO: statistics for "test_zstd": storage id: xxxxx -total file size: 40960, total data size: 14947 +total file size: 40960, total data size: 14945 compression rate: 21.91x total row count: 20001, stripe count: 2, average rows per stripe: 10000 chunk count: 9, containing data for dropped columns: 0, zstd compressed: 9 @@ -39,6 +39,24 @@ SELECT DISTINCT * FROM test_zstd ORDER BY a, b, c LIMIT 5; 0 | 12 | 4 (5 rows) +-- change compression level +-- for this particular usecase, higher compression levels +-- don't improve compression ratio +SELECT alter_columnar_table_set('test_zstd', compression_level => 19); + alter_columnar_table_set +--------------------------------------------------------------------- + +(1 row) + +VACUUM FULL test_zstd; +VACUUM VERBOSE test_zstd; +INFO: statistics for "test_zstd": +storage id: xxxxx +total file size: 32768, total data size: 15201 +compression rate: 21.55x +total row count: 20001, stripe count: 1, average rows per stripe: 20001 +chunk count: 9, containing data for dropped columns: 0, zstd compressed: 9 + -- compare compression rate to pglz SET columnar.compression TO 'pglz'; CREATE TABLE test_pglz (LIKE test_zstd) USING columnar; diff --git a/src/test/regress/expected/columnar_citus_integration.out b/src/test/regress/expected/columnar_citus_integration.out index fff1cd47e..578fc50ee 100644 --- a/src/test/regress/expected/columnar_citus_integration.out +++ b/src/test/regress/expected/columnar_citus_integration.out @@ -63,6 +63,57 @@ $cmd$); (localhost,57638,20090003,t,none) (4 rows) +-- setting: compression_level +-- get baseline for setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT compression_level FROM columnar.options WHERE regclass = '%s'::regclass; +$cmd$); + run_command_on_placements +--------------------------------------------------------------------- + (localhost,57637,20090000,t,3) + (localhost,57638,20090001,t,3) + (localhost,57637,20090002,t,3) + (localhost,57638,20090003,t,3) +(4 rows) + +-- change setting +SELECT alter_columnar_table_set('table_option', compression_level => 13); + alter_columnar_table_set +--------------------------------------------------------------------- + +(1 row) + +-- verify setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT compression_level FROM columnar.options WHERE regclass = '%s'::regclass; +$cmd$); + run_command_on_placements +--------------------------------------------------------------------- + (localhost,57637,20090000,t,13) + (localhost,57638,20090001,t,13) + (localhost,57637,20090002,t,13) + (localhost,57638,20090003,t,13) +(4 rows) + +-- reset setting +SELECT alter_columnar_table_reset('table_option', compression_level => true); + alter_columnar_table_reset +--------------------------------------------------------------------- + +(1 row) + +-- verify setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT compression_level FROM columnar.options WHERE regclass = '%s'::regclass; +$cmd$); + run_command_on_placements +--------------------------------------------------------------------- + (localhost,57637,20090000,t,3) + (localhost,57638,20090001,t,3) + (localhost,57637,20090002,t,3) + (localhost,57638,20090003,t,3) +(4 rows) + -- setting: chunk_row_count -- get baseline for setting SELECT run_command_on_placements('table_option',$cmd$ @@ -170,7 +221,8 @@ CREATE TABLE table_option_2 (a int, b text) USING columnar; SELECT alter_columnar_table_set('table_option_2', chunk_row_count => 100, stripe_row_count => 1000, - compression => 'pglz'); + compression => 'pglz', + compression_level => 15); alter_columnar_table_set --------------------------------------------------------------------- @@ -184,14 +236,14 @@ SELECT create_distributed_table('table_option_2', 'a'); -- verify settings on placements SELECT run_command_on_placements('table_option_2',$cmd$ - SELECT ROW(chunk_row_count, stripe_row_count, compression) FROM columnar.options WHERE regclass = '%s'::regclass; + SELECT ROW(chunk_row_count, stripe_row_count, compression, compression_level) FROM columnar.options WHERE regclass = '%s'::regclass; $cmd$); - run_command_on_placements + run_command_on_placements --------------------------------------------------------------------- - (localhost,57637,20090004,t,"(100,1000,pglz)") - (localhost,57638,20090005,t,"(100,1000,pglz)") - (localhost,57637,20090006,t,"(100,1000,pglz)") - (localhost,57638,20090007,t,"(100,1000,pglz)") + (localhost,57637,20090004,t,"(100,1000,pglz,15)") + (localhost,57638,20090005,t,"(100,1000,pglz,15)") + (localhost,57637,20090006,t,"(100,1000,pglz,15)") + (localhost,57638,20090007,t,"(100,1000,pglz,15)") (4 rows) DROP TABLE table_option, table_option_2; @@ -268,6 +320,69 @@ $cmd$); (localhost,57638,20090011,t,none) (8 rows) +-- setting: compression_level +-- get baseline for setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT compression_level FROM columnar.options WHERE regclass = '%s'::regclass; +$cmd$); + run_command_on_placements +--------------------------------------------------------------------- + (localhost,57637,20090008,t,3) + (localhost,57638,20090008,t,3) + (localhost,57637,20090009,t,3) + (localhost,57638,20090009,t,3) + (localhost,57637,20090010,t,3) + (localhost,57638,20090010,t,3) + (localhost,57637,20090011,t,3) + (localhost,57638,20090011,t,3) +(8 rows) + +-- change setting +SELECT alter_columnar_table_set('table_option', compression_level => 17); + alter_columnar_table_set +--------------------------------------------------------------------- + +(1 row) + +-- verify setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT compression_level FROM columnar.options WHERE regclass = '%s'::regclass; +$cmd$); + run_command_on_placements +--------------------------------------------------------------------- + (localhost,57637,20090008,t,17) + (localhost,57638,20090008,t,17) + (localhost,57637,20090009,t,17) + (localhost,57638,20090009,t,17) + (localhost,57637,20090010,t,17) + (localhost,57638,20090010,t,17) + (localhost,57637,20090011,t,17) + (localhost,57638,20090011,t,17) +(8 rows) + +-- reset setting +SELECT alter_columnar_table_reset('table_option', compression_level => true); + alter_columnar_table_reset +--------------------------------------------------------------------- + +(1 row) + +-- verify setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT compression_level FROM columnar.options WHERE regclass = '%s'::regclass; +$cmd$); + run_command_on_placements +--------------------------------------------------------------------- + (localhost,57637,20090008,t,3) + (localhost,57638,20090008,t,3) + (localhost,57637,20090009,t,3) + (localhost,57638,20090009,t,3) + (localhost,57637,20090010,t,3) + (localhost,57638,20090010,t,3) + (localhost,57637,20090011,t,3) + (localhost,57638,20090011,t,3) +(8 rows) + -- setting: chunk_row_count -- get baseline for setting SELECT run_command_on_placements('table_option',$cmd$ @@ -399,7 +514,8 @@ CREATE TABLE table_option_2 (a int, b text) USING columnar; SELECT alter_columnar_table_set('table_option_2', chunk_row_count => 100, stripe_row_count => 1000, - compression => 'pglz'); + compression => 'pglz', + compression_level => 19); alter_columnar_table_set --------------------------------------------------------------------- @@ -413,18 +529,18 @@ SELECT create_distributed_table('table_option_2', 'a'); -- verify settings on placements SELECT run_command_on_placements('table_option_2',$cmd$ - SELECT ROW(chunk_row_count, stripe_row_count, compression) FROM columnar.options WHERE regclass = '%s'::regclass; + SELECT ROW(chunk_row_count, stripe_row_count, compression, compression_level) FROM columnar.options WHERE regclass = '%s'::regclass; $cmd$); - run_command_on_placements + run_command_on_placements --------------------------------------------------------------------- - (localhost,57637,20090012,t,"(100,1000,pglz)") - (localhost,57638,20090012,t,"(100,1000,pglz)") - (localhost,57637,20090013,t,"(100,1000,pglz)") - (localhost,57638,20090013,t,"(100,1000,pglz)") - (localhost,57637,20090014,t,"(100,1000,pglz)") - (localhost,57638,20090014,t,"(100,1000,pglz)") - (localhost,57637,20090015,t,"(100,1000,pglz)") - (localhost,57638,20090015,t,"(100,1000,pglz)") + (localhost,57637,20090012,t,"(100,1000,pglz,19)") + (localhost,57638,20090012,t,"(100,1000,pglz,19)") + (localhost,57637,20090013,t,"(100,1000,pglz,19)") + (localhost,57638,20090013,t,"(100,1000,pglz,19)") + (localhost,57637,20090014,t,"(100,1000,pglz,19)") + (localhost,57638,20090014,t,"(100,1000,pglz,19)") + (localhost,57637,20090015,t,"(100,1000,pglz,19)") + (localhost,57638,20090015,t,"(100,1000,pglz,19)") (8 rows) DROP TABLE table_option, table_option_2; @@ -481,6 +597,51 @@ $cmd$); (localhost,57638,20090016,t,none) (2 rows) +-- setting: compression_level +-- get baseline for setting +SELECT run_command_on_placements('table_option_reference',$cmd$ + SELECT compression_level FROM columnar.options WHERE regclass = '%s'::regclass; +$cmd$); + run_command_on_placements +--------------------------------------------------------------------- + (localhost,57637,20090016,t,3) + (localhost,57638,20090016,t,3) +(2 rows) + +-- change setting +SELECT alter_columnar_table_set('table_option_reference', compression_level => 11); + alter_columnar_table_set +--------------------------------------------------------------------- + +(1 row) + +-- verify setting +SELECT run_command_on_placements('table_option_reference',$cmd$ + SELECT compression_level FROM columnar.options WHERE regclass = '%s'::regclass; +$cmd$); + run_command_on_placements +--------------------------------------------------------------------- + (localhost,57637,20090016,t,11) + (localhost,57638,20090016,t,11) +(2 rows) + +-- reset setting +SELECT alter_columnar_table_reset('table_option_reference', compression_level => true); + alter_columnar_table_reset +--------------------------------------------------------------------- + +(1 row) + +-- verify setting +SELECT run_command_on_placements('table_option_reference',$cmd$ + SELECT compression_level FROM columnar.options WHERE regclass = '%s'::regclass; +$cmd$); + run_command_on_placements +--------------------------------------------------------------------- + (localhost,57637,20090016,t,3) + (localhost,57638,20090016,t,3) +(2 rows) + -- setting: chunk_row_count -- get baseline for setting SELECT run_command_on_placements('table_option_reference',$cmd$ @@ -576,7 +737,8 @@ CREATE TABLE table_option_reference_2 (a int, b text) USING columnar; SELECT alter_columnar_table_set('table_option_reference_2', chunk_row_count => 100, stripe_row_count => 1000, - compression => 'pglz'); + compression => 'pglz', + compression_level => 9); alter_columnar_table_set --------------------------------------------------------------------- @@ -590,12 +752,12 @@ SELECT create_reference_table('table_option_reference_2'); -- verify settings on placements SELECT run_command_on_placements('table_option_reference_2',$cmd$ - SELECT ROW(chunk_row_count, stripe_row_count, compression) FROM columnar.options WHERE regclass = '%s'::regclass; + SELECT ROW(chunk_row_count, stripe_row_count, compression, compression_level) FROM columnar.options WHERE regclass = '%s'::regclass; $cmd$); - run_command_on_placements + run_command_on_placements --------------------------------------------------------------------- - (localhost,57637,20090017,t,"(100,1000,pglz)") - (localhost,57638,20090017,t,"(100,1000,pglz)") + (localhost,57637,20090017,t,"(100,1000,pglz,9)") + (localhost,57638,20090017,t,"(100,1000,pglz,9)") (2 rows) DROP TABLE table_option_reference, table_option_reference_2; diff --git a/src/test/regress/expected/multi_extension.out b/src/test/regress/expected/multi_extension.out index 8ed63aeab..fc58f7bd3 100644 --- a/src/test/regress/expected/multi_extension.out +++ b/src/test/regress/expected/multi_extension.out @@ -479,8 +479,8 @@ SELECT * FROM print_extension_changes(); previous_object | current_object --------------------------------------------------------------------- | access method columnar - | function alter_columnar_table_reset(regclass,boolean,boolean,boolean) - | function alter_columnar_table_set(regclass,integer,integer,name) + | function alter_columnar_table_reset(regclass,boolean,boolean,boolean,boolean) + | function alter_columnar_table_set(regclass,integer,integer,name,integer) | function citus_internal.columnar_ensure_objects_exist() | function columnar.columnar_handler(internal) | schema columnar diff --git a/src/test/regress/expected/upgrade_columnar_after.out b/src/test/regress/expected/upgrade_columnar_after.out index 93b710f4d..189d1282f 100644 --- a/src/test/regress/expected/upgrade_columnar_after.out +++ b/src/test/regress/expected/upgrade_columnar_after.out @@ -102,9 +102,9 @@ SELECT * FROM matview ORDER BY a; -- test we retained options SELECT * FROM columnar.options WHERE regclass = 'test_options_1'::regclass; - regclass | chunk_row_count | stripe_row_count | compression + regclass | chunk_row_count | stripe_row_count | compression_level | compression --------------------------------------------------------------------- - test_options_1 | 1000 | 5000 | pglz + test_options_1 | 1000 | 5000 | 3 | pglz (1 row) VACUUM VERBOSE test_options_1; @@ -122,9 +122,9 @@ SELECT count(*), sum(a), sum(b) FROM test_options_1; (1 row) SELECT * FROM columnar.options WHERE regclass = 'test_options_2'::regclass; - regclass | chunk_row_count | stripe_row_count | compression + regclass | chunk_row_count | stripe_row_count | compression_level | compression --------------------------------------------------------------------- - test_options_2 | 2000 | 6000 | none + test_options_2 | 2000 | 6000 | 13 | none (1 row) VACUUM VERBOSE test_options_2; diff --git a/src/test/regress/expected/upgrade_columnar_before.out b/src/test/regress/expected/upgrade_columnar_before.out index 807971150..a4ae51d12 100644 --- a/src/test/regress/expected/upgrade_columnar_before.out +++ b/src/test/regress/expected/upgrade_columnar_before.out @@ -77,7 +77,7 @@ INSERT INTO test_alter_type SELECT * FROM generate_series(1, 10); SELECT count(*) FROM test_alter_type; count --------------------------------------------------------------------- - 10 + 10 (1 row) SELECT relfilenode AS relfilenode_pre_alter @@ -95,7 +95,7 @@ INSERT INTO test_alter_type SELECT * FROM generate_series(11, 13); SELECT count(*) FROM test_alter_type; count --------------------------------------------------------------------- - 13 + 13 (1 row) -- materialized view @@ -140,4 +140,10 @@ SELECT alter_columnar_table_set('test_options_2', compression => 'none'); (1 row) +SELECT alter_columnar_table_set('test_options_2', compression_level => 13); + alter_columnar_table_set +--------------------------------------------------------------------- + +(1 row) + INSERT INTO test_options_2 SELECT i, floor(i/2000) FROM generate_series(1, 10000) i; diff --git a/src/test/regress/expected/upgrade_list_citus_objects.out b/src/test/regress/expected/upgrade_list_citus_objects.out index 4141333e2..6824a774c 100644 --- a/src/test/regress/expected/upgrade_list_citus_objects.out +++ b/src/test/regress/expected/upgrade_list_citus_objects.out @@ -18,8 +18,8 @@ ORDER BY 1; --------------------------------------------------------------------- access method columnar event trigger citus_cascade_to_partition - function alter_columnar_table_reset(regclass,boolean,boolean,boolean) - function alter_columnar_table_set(regclass,integer,integer,name) + function alter_columnar_table_reset(regclass,boolean,boolean,boolean,boolean) + function alter_columnar_table_set(regclass,integer,integer,name,integer) function alter_role_if_exists(text,text) function any_value(anyelement) function any_value_agg(anyelement,anyelement) diff --git a/src/test/regress/sql/am_tableoptions.sql b/src/test/regress/sql/am_tableoptions.sql index fa66869a7..dd133ac0a 100644 --- a/src/test/regress/sql/am_tableoptions.sql +++ b/src/test/regress/sql/am_tableoptions.sql @@ -15,6 +15,13 @@ SELECT alter_columnar_table_set('table_options', compression => 'pglz'); SELECT * FROM columnar.options WHERE regclass = 'table_options'::regclass; +-- test changing the compression level +SELECT alter_columnar_table_set('table_options', compression_level => 5); + +-- show table_options settings +SELECT * FROM columnar.options +WHERE regclass = 'table_options'::regclass; + -- test changing the chunk_row_count SELECT alter_columnar_table_set('table_options', chunk_row_count => 10); @@ -37,7 +44,7 @@ SELECT * FROM columnar.options WHERE regclass = 'table_options'::regclass; -- set all settings at the same time -SELECT alter_columnar_table_set('table_options', stripe_row_count => 1000, chunk_row_count => 100, compression => 'none'); +SELECT alter_columnar_table_set('table_options', stripe_row_count => 1000, chunk_row_count => 100, compression => 'none', compression_level => 7); -- show table_options settings SELECT * FROM columnar.options @@ -70,6 +77,7 @@ WHERE regclass = 'table_options'::regclass; SET columnar.chunk_row_count TO 1000; SET columnar.stripe_row_count TO 10000; SET columnar.compression TO 'pglz'; +SET columnar.compression_level TO 11; -- verify setting the GUC's didn't change the settings -- show table_options settings @@ -93,10 +101,17 @@ SELECT alter_columnar_table_reset('table_options', compression => true); SELECT * FROM columnar.options WHERE regclass = 'table_options'::regclass; +SELECT alter_columnar_table_reset('table_options', compression_level => true); + +-- show table_options settings +SELECT * FROM columnar.options +WHERE regclass = 'table_options'::regclass; + -- verify resetting all settings at once work SET columnar.chunk_row_count TO 10000; SET columnar.stripe_row_count TO 100000; SET columnar.compression TO 'none'; +SET columnar.compression_level TO 13; -- show table_options settings SELECT * FROM columnar.options @@ -106,7 +121,8 @@ SELECT alter_columnar_table_reset( 'table_options', chunk_row_count => true, stripe_row_count => true, - compression => true); + compression => true, + compression_level => true); -- show table_options settings SELECT * FROM columnar.options @@ -121,6 +137,10 @@ SELECT alter_columnar_table_reset('not_a_columnar_table', compression => true); -- verify you can't use a compression that is not known SELECT alter_columnar_table_set('table_options', compression => 'foobar'); +-- verify cannot set out of range compression levels +SELECT alter_columnar_table_set('table_options', compression_level => 0); +SELECT alter_columnar_table_set('table_options', compression_level => 20); + -- verify options are removed when table is dropped DROP TABLE table_options; -- we expect no entries in çstore.options for anything not found int pg_class diff --git a/src/test/regress/sql/am_zstd.sql b/src/test/regress/sql/am_zstd.sql index 8e924709a..1ea7fc31b 100644 --- a/src/test/regress/sql/am_zstd.sql +++ b/src/test/regress/sql/am_zstd.sql @@ -20,6 +20,13 @@ VACUUM VERBOSE test_zstd; SELECT DISTINCT * FROM test_zstd ORDER BY a, b, c LIMIT 5; +-- change compression level +-- for this particular usecase, higher compression levels +-- don't improve compression ratio +SELECT alter_columnar_table_set('test_zstd', compression_level => 19); +VACUUM FULL test_zstd; +VACUUM VERBOSE test_zstd; + -- compare compression rate to pglz SET columnar.compression TO 'pglz'; CREATE TABLE test_pglz (LIKE test_zstd) USING columnar; diff --git a/src/test/regress/sql/columnar_citus_integration.sql b/src/test/regress/sql/columnar_citus_integration.sql index 857e08cf6..8fbceee1e 100644 --- a/src/test/regress/sql/columnar_citus_integration.sql +++ b/src/test/regress/sql/columnar_citus_integration.sql @@ -28,6 +28,24 @@ SELECT run_command_on_placements('table_option',$cmd$ SELECT compression FROM columnar.options WHERE regclass = '%s'::regclass; $cmd$); +-- setting: compression_level +-- get baseline for setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT compression_level FROM columnar.options WHERE regclass = '%s'::regclass; +$cmd$); +-- change setting +SELECT alter_columnar_table_set('table_option', compression_level => 13); +-- verify setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT compression_level FROM columnar.options WHERE regclass = '%s'::regclass; +$cmd$); +-- reset setting +SELECT alter_columnar_table_reset('table_option', compression_level => true); +-- verify setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT compression_level FROM columnar.options WHERE regclass = '%s'::regclass; +$cmd$); + -- setting: chunk_row_count -- get baseline for setting SELECT run_command_on_placements('table_option',$cmd$ @@ -69,12 +87,13 @@ CREATE TABLE table_option_2 (a int, b text) USING columnar; SELECT alter_columnar_table_set('table_option_2', chunk_row_count => 100, stripe_row_count => 1000, - compression => 'pglz'); + compression => 'pglz', + compression_level => 15); SELECT create_distributed_table('table_option_2', 'a'); -- verify settings on placements SELECT run_command_on_placements('table_option_2',$cmd$ - SELECT ROW(chunk_row_count, stripe_row_count, compression) FROM columnar.options WHERE regclass = '%s'::regclass; + SELECT ROW(chunk_row_count, stripe_row_count, compression, compression_level) FROM columnar.options WHERE regclass = '%s'::regclass; $cmd$); DROP TABLE table_option, table_option_2; @@ -104,6 +123,24 @@ SELECT run_command_on_placements('table_option',$cmd$ SELECT compression FROM columnar.options WHERE regclass = '%s'::regclass; $cmd$); +-- setting: compression_level +-- get baseline for setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT compression_level FROM columnar.options WHERE regclass = '%s'::regclass; +$cmd$); +-- change setting +SELECT alter_columnar_table_set('table_option', compression_level => 17); +-- verify setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT compression_level FROM columnar.options WHERE regclass = '%s'::regclass; +$cmd$); +-- reset setting +SELECT alter_columnar_table_reset('table_option', compression_level => true); +-- verify setting +SELECT run_command_on_placements('table_option',$cmd$ + SELECT compression_level FROM columnar.options WHERE regclass = '%s'::regclass; +$cmd$); + -- setting: chunk_row_count -- get baseline for setting SELECT run_command_on_placements('table_option',$cmd$ @@ -145,12 +182,13 @@ CREATE TABLE table_option_2 (a int, b text) USING columnar; SELECT alter_columnar_table_set('table_option_2', chunk_row_count => 100, stripe_row_count => 1000, - compression => 'pglz'); + compression => 'pglz', + compression_level => 19); SELECT create_distributed_table('table_option_2', 'a'); -- verify settings on placements SELECT run_command_on_placements('table_option_2',$cmd$ - SELECT ROW(chunk_row_count, stripe_row_count, compression) FROM columnar.options WHERE regclass = '%s'::regclass; + SELECT ROW(chunk_row_count, stripe_row_count, compression, compression_level) FROM columnar.options WHERE regclass = '%s'::regclass; $cmd$); DROP TABLE table_option, table_option_2; @@ -177,6 +215,24 @@ SELECT run_command_on_placements('table_option_reference',$cmd$ SELECT compression FROM columnar.options WHERE regclass = '%s'::regclass; $cmd$); +-- setting: compression_level +-- get baseline for setting +SELECT run_command_on_placements('table_option_reference',$cmd$ + SELECT compression_level FROM columnar.options WHERE regclass = '%s'::regclass; +$cmd$); +-- change setting +SELECT alter_columnar_table_set('table_option_reference', compression_level => 11); +-- verify setting +SELECT run_command_on_placements('table_option_reference',$cmd$ + SELECT compression_level FROM columnar.options WHERE regclass = '%s'::regclass; +$cmd$); +-- reset setting +SELECT alter_columnar_table_reset('table_option_reference', compression_level => true); +-- verify setting +SELECT run_command_on_placements('table_option_reference',$cmd$ + SELECT compression_level FROM columnar.options WHERE regclass = '%s'::regclass; +$cmd$); + -- setting: chunk_row_count -- get baseline for setting SELECT run_command_on_placements('table_option_reference',$cmd$ @@ -218,12 +274,13 @@ CREATE TABLE table_option_reference_2 (a int, b text) USING columnar; SELECT alter_columnar_table_set('table_option_reference_2', chunk_row_count => 100, stripe_row_count => 1000, - compression => 'pglz'); + compression => 'pglz', + compression_level => 9); SELECT create_reference_table('table_option_reference_2'); -- verify settings on placements SELECT run_command_on_placements('table_option_reference_2',$cmd$ - SELECT ROW(chunk_row_count, stripe_row_count, compression) FROM columnar.options WHERE regclass = '%s'::regclass; + SELECT ROW(chunk_row_count, stripe_row_count, compression, compression_level) FROM columnar.options WHERE regclass = '%s'::regclass; $cmd$); DROP TABLE table_option_reference, table_option_reference_2; diff --git a/src/test/regress/sql/upgrade_columnar_before.sql b/src/test/regress/sql/upgrade_columnar_before.sql index ac28a6e48..b7edbac06 100644 --- a/src/test/regress/sql/upgrade_columnar_before.sql +++ b/src/test/regress/sql/upgrade_columnar_before.sql @@ -107,5 +107,6 @@ INSERT INTO test_options_2 SELECT i, floor(i/1000) FROM generate_series(1, 10000 SELECT alter_columnar_table_set('test_options_2', chunk_row_count => 2000); SELECT alter_columnar_table_set('test_options_2', stripe_row_count => 6000); SELECT alter_columnar_table_set('test_options_2', compression => 'none'); +SELECT alter_columnar_table_set('test_options_2', compression_level => 13); INSERT INTO test_options_2 SELECT i, floor(i/2000) FROM generate_series(1, 10000) i;