external
/
citus
mirror of https://github.com/citusdata/citus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4379 from citusdata/cstore_chunk 

Columnar: rename block to chunk
pull/4364/head
Hadi Moshayedi 2020-12-03 09:01:10 -08:00 committed by GitHub
parent 317cf44a56 4a9aebaa7b
commit c23bdb129d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
34 changed files with 582 additions and 582 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/backend/columnar/cstore.c
View File

@ -25,11 +25,11 @@
/* Default values for option parameters */ /* Default values for option parameters */
#define DEFAULT_COMPRESSION_TYPE COMPRESSION_NONE #define DEFAULT_COMPRESSION_TYPE COMPRESSION_NONE
#define DEFAULT_STRIPE_ROW_COUNT 150000 #define DEFAULT_STRIPE_ROW_COUNT 150000
#define DEFAULT_BLOCK_ROW_COUNT 10000 #define DEFAULT_CHUNK_ROW_COUNT 10000
int cstore_compression = DEFAULT_COMPRESSION_TYPE; int cstore_compression = DEFAULT_COMPRESSION_TYPE;
int cstore_stripe_row_count = DEFAULT_STRIPE_ROW_COUNT; int cstore_stripe_row_count = DEFAULT_STRIPE_ROW_COUNT;
int cstore_block_row_count = DEFAULT_BLOCK_ROW_COUNT; int cstore_chunk_row_count = DEFAULT_CHUNK_ROW_COUNT;
static const struct config_enum_entry cstore_compression_options[] = static const struct config_enum_entry cstore_compression_options[] =
{ {
@ -66,13 +66,13 @@ cstore_init()
NULL, NULL,
NULL); NULL);
DefineCustomIntVariable("cstore.block_row_count", DefineCustomIntVariable("cstore.chunk_row_count",
"Maximum number of rows per block.", "Maximum number of rows per chunk.",
NULL, NULL,
&cstore_block_row_count, &cstore_chunk_row_count,
DEFAULT_BLOCK_ROW_COUNT, DEFAULT_CHUNK_ROW_COUNT,
BLOCK_ROW_COUNT_MINIMUM, CHUNK_ROW_COUNT_MINIMUM,
BLOCK_ROW_COUNT_MAXIMUM, CHUNK_ROW_COUNT_MAXIMUM,
PGC_USERSET, PGC_USERSET,
0, 0,
NULL, NULL,

76
src/backend/columnar/cstore_metadata_tables.c
View File

@ -104,7 +104,7 @@ PG_FUNCTION_INFO_V1(columnar_relation_storageid);
/* constants for cstore.options */ /* constants for cstore.options */
#define Natts_cstore_options 4 #define Natts_cstore_options 4
#define Anum_cstore_options_regclass 1 #define Anum_cstore_options_regclass 1
#define Anum_cstore_options_block_row_count 2 #define Anum_cstore_options_chunk_row_count 2
#define Anum_cstore_options_stripe_row_count 3 #define Anum_cstore_options_stripe_row_count 3
#define Anum_cstore_options_compression 4 #define Anum_cstore_options_compression 4
@ -115,7 +115,7 @@ PG_FUNCTION_INFO_V1(columnar_relation_storageid);
typedef struct FormData_cstore_options typedef struct FormData_cstore_options
{ {
Oid regclass; Oid regclass;
int32 block_row_count; int32 chunk_row_count;
int32 stripe_row_count; int32 stripe_row_count;
NameData compression; NameData compression;
@ -132,8 +132,8 @@ typedef FormData_cstore_options *Form_cstore_options;
#define Anum_cstore_stripes_file_offset 3 #define Anum_cstore_stripes_file_offset 3
#define Anum_cstore_stripes_data_length 4 #define Anum_cstore_stripes_data_length 4
#define Anum_cstore_stripes_column_count 5 #define Anum_cstore_stripes_column_count 5
#define Anum_cstore_stripes_block_count 6 #define Anum_cstore_stripes_chunk_count 6
#define Anum_cstore_stripes_block_row_count 7 #define Anum_cstore_stripes_chunk_row_count 7
#define Anum_cstore_stripes_row_count 8 #define Anum_cstore_stripes_row_count 8
/* constants for cstore_skipnodes */ /* constants for cstore_skipnodes */
@ -141,7 +141,7 @@ typedef FormData_cstore_options *Form_cstore_options;
#define Anum_cstore_skipnodes_storageid 1 #define Anum_cstore_skipnodes_storageid 1
#define Anum_cstore_skipnodes_stripe 2 #define Anum_cstore_skipnodes_stripe 2
#define Anum_cstore_skipnodes_attr 3 #define Anum_cstore_skipnodes_attr 3
#define Anum_cstore_skipnodes_block 4 #define Anum_cstore_skipnodes_chunk 4
#define Anum_cstore_skipnodes_row_count 5 #define Anum_cstore_skipnodes_row_count 5
#define Anum_cstore_skipnodes_minimum_value 6 #define Anum_cstore_skipnodes_minimum_value 6
#define Anum_cstore_skipnodes_maximum_value 7 #define Anum_cstore_skipnodes_maximum_value 7
@ -169,7 +169,7 @@ InitColumnarOptions(Oid regclass)
} }
ColumnarOptions defaultOptions = { ColumnarOptions defaultOptions = {
.blockRowCount = cstore_block_row_count, .chunkRowCount = cstore_chunk_row_count,
.stripeRowCount = cstore_stripe_row_count, .stripeRowCount = cstore_stripe_row_count,
.compressionType = cstore_compression, .compressionType = cstore_compression,
}; };
@ -212,7 +212,7 @@ WriteColumnarOptions(Oid regclass, ColumnarOptions *options, bool overwrite)
bool nulls[Natts_cstore_options] = { 0 }; bool nulls[Natts_cstore_options] = { 0 };
Datum values[Natts_cstore_options] = { Datum values[Natts_cstore_options] = {
ObjectIdGetDatum(regclass), ObjectIdGetDatum(regclass),
Int32GetDatum(options->blockRowCount), Int32GetDatum(options->chunkRowCount),
Int32GetDatum(options->stripeRowCount), Int32GetDatum(options->stripeRowCount),
0, /* to be filled below */ 0, /* to be filled below */
}; };
@ -243,7 +243,7 @@ WriteColumnarOptions(Oid regclass, ColumnarOptions *options, bool overwrite)
/* TODO check if the options are actually different, skip if not changed */ /* TODO check if the options are actually different, skip if not changed */
/* update existing record */ /* update existing record */
bool update[Natts_cstore_options] = { 0 }; bool update[Natts_cstore_options] = { 0 };
update[Anum_cstore_options_block_row_count - 1] = true; update[Anum_cstore_options_chunk_row_count - 1] = true;
update[Anum_cstore_options_stripe_row_count - 1] = true; update[Anum_cstore_options_stripe_row_count - 1] = true;
update[Anum_cstore_options_compression - 1] = true; update[Anum_cstore_options_compression - 1] = true;
@ -360,7 +360,7 @@ ReadColumnarOptions(Oid regclass, ColumnarOptions *options)
{ {
Form_cstore_options tupOptions = (Form_cstore_options) GETSTRUCT(heapTuple); Form_cstore_options tupOptions = (Form_cstore_options) GETSTRUCT(heapTuple);
options->blockRowCount = tupOptions->block_row_count; options->chunkRowCount = tupOptions->chunk_row_count;
options->stripeRowCount = tupOptions->stripe_row_count; options->stripeRowCount = tupOptions->stripe_row_count;
options->compressionType = ParseCompressionType(NameStr(tupOptions->compression)); options->compressionType = ParseCompressionType(NameStr(tupOptions->compression));
} }
@ -369,7 +369,7 @@ ReadColumnarOptions(Oid regclass, ColumnarOptions *options)
/* populate options with system defaults */ /* populate options with system defaults */
options->compressionType = cstore_compression; options->compressionType = cstore_compression;
options->stripeRowCount = cstore_stripe_row_count; options->stripeRowCount = cstore_stripe_row_count;
options->blockRowCount = cstore_block_row_count; options->chunkRowCount = cstore_chunk_row_count;
} }
systable_endscan_ordered(scanDescriptor); systable_endscan_ordered(scanDescriptor);
@ -389,7 +389,7 @@ SaveStripeSkipList(RelFileNode relfilenode, uint64 stripe, StripeSkipList *strip
TupleDesc tupleDescriptor) TupleDesc tupleDescriptor)
{ {
uint32 columnIndex = 0; uint32 columnIndex = 0;
uint32 blockIndex = 0; uint32 chunkIndex = 0;
uint32 columnCount = stripeSkipList->columnCount; uint32 columnCount = stripeSkipList->columnCount;
ColumnarMetapage *metapage = ReadMetapage(relfilenode, false); ColumnarMetapage *metapage = ReadMetapage(relfilenode, false);
@ -399,22 +399,22 @@ SaveStripeSkipList(RelFileNode relfilenode, uint64 stripe, StripeSkipList *strip
for (columnIndex = 0; columnIndex < columnCount; columnIndex++) for (columnIndex = 0; columnIndex < columnCount; columnIndex++)
{ {
for (blockIndex = 0; blockIndex < stripeSkipList->blockCount; blockIndex++) for (chunkIndex = 0; chunkIndex < stripeSkipList->chunkCount; chunkIndex++)
{ {
ColumnBlockSkipNode *skipNode = ColumnChunkSkipNode *skipNode =
&stripeSkipList->blockSkipNodeArray[columnIndex][blockIndex]; &stripeSkipList->chunkSkipNodeArray[columnIndex][chunkIndex];
Datum values[Natts_cstore_skipnodes] = { Datum values[Natts_cstore_skipnodes] = {
UInt64GetDatum(metapage->storageId), UInt64GetDatum(metapage->storageId),
Int64GetDatum(stripe), Int64GetDatum(stripe),
Int32GetDatum(columnIndex + 1), Int32GetDatum(columnIndex + 1),
Int32GetDatum(blockIndex), Int32GetDatum(chunkIndex),
Int64GetDatum(skipNode->rowCount), Int64GetDatum(skipNode->rowCount),
0, /* to be filled below */ 0, /* to be filled below */
0, /* to be filled below */ 0, /* to be filled below */
Int64GetDatum(skipNode->valueBlockOffset), Int64GetDatum(skipNode->valueChunkOffset),
Int64GetDatum(skipNode->valueLength), Int64GetDatum(skipNode->valueLength),
Int64GetDatum(skipNode->existsBlockOffset), Int64GetDatum(skipNode->existsChunkOffset),
Int64GetDatum(skipNode->existsLength), Int64GetDatum(skipNode->existsLength),
Int32GetDatum(skipNode->valueCompressionType) Int32GetDatum(skipNode->valueCompressionType)
}; };
@ -452,7 +452,7 @@ SaveStripeSkipList(RelFileNode relfilenode, uint64 stripe, StripeSkipList *strip
*/ */
StripeSkipList * StripeSkipList *
ReadStripeSkipList(RelFileNode relfilenode, uint64 stripe, TupleDesc tupleDescriptor, ReadStripeSkipList(RelFileNode relfilenode, uint64 stripe, TupleDesc tupleDescriptor,
uint32 blockCount) uint32 chunkCount)
{ {
int32 columnIndex = 0; int32 columnIndex = 0;
HeapTuple heapTuple = NULL; HeapTuple heapTuple = NULL;
@ -474,13 +474,13 @@ ReadStripeSkipList(RelFileNode relfilenode, uint64 stripe, TupleDesc tupleDescri
2, scanKey); 2, scanKey);
StripeSkipList *skipList = palloc0(sizeof(StripeSkipList)); StripeSkipList *skipList = palloc0(sizeof(StripeSkipList));
skipList->blockCount = blockCount; skipList->chunkCount = chunkCount;
skipList->columnCount = columnCount; skipList->columnCount = columnCount;
skipList->blockSkipNodeArray = palloc0(columnCount * sizeof(ColumnBlockSkipNode *)); skipList->chunkSkipNodeArray = palloc0(columnCount * sizeof(ColumnChunkSkipNode *));
for (columnIndex = 0; columnIndex < columnCount; columnIndex++) for (columnIndex = 0; columnIndex < columnCount; columnIndex++)
{ {
skipList->blockSkipNodeArray[columnIndex] = skipList->chunkSkipNodeArray[columnIndex] =
palloc0(blockCount * sizeof(ColumnBlockSkipNode)); palloc0(chunkCount * sizeof(ColumnChunkSkipNode));
} }
while (HeapTupleIsValid(heapTuple = systable_getnext(scanDescriptor))) while (HeapTupleIsValid(heapTuple = systable_getnext(scanDescriptor)))
@ -492,7 +492,7 @@ ReadStripeSkipList(RelFileNode relfilenode, uint64 stripe, TupleDesc tupleDescri
isNullArray); isNullArray);
int32 attr = DatumGetInt32(datumArray[Anum_cstore_skipnodes_attr - 1]); int32 attr = DatumGetInt32(datumArray[Anum_cstore_skipnodes_attr - 1]);
int32 blockIndex = DatumGetInt32(datumArray[Anum_cstore_skipnodes_block - 1]); int32 chunkIndex = DatumGetInt32(datumArray[Anum_cstore_skipnodes_chunk - 1]);
if (attr <= 0 || attr > columnCount) if (attr <= 0 || attr > columnCount)
{ {
@ -500,23 +500,23 @@ ReadStripeSkipList(RelFileNode relfilenode, uint64 stripe, TupleDesc tupleDescri
errdetail("Attribute number out of range: %d", attr))); errdetail("Attribute number out of range: %d", attr)));
} }
if (blockIndex < 0 || blockIndex >= blockCount) if (chunkIndex < 0 || chunkIndex >= chunkCount)
{ {
ereport(ERROR, (errmsg("invalid stripe skipnode entry"), ereport(ERROR, (errmsg("invalid stripe skipnode entry"),
errdetail("Block number out of range: %d", blockIndex))); errdetail("Chunk number out of range: %d", chunkIndex)));
} }
columnIndex = attr - 1; columnIndex = attr - 1;
ColumnBlockSkipNode *skipNode = ColumnChunkSkipNode *skipNode =
&skipList->blockSkipNodeArray[columnIndex][blockIndex]; &skipList->chunkSkipNodeArray[columnIndex][chunkIndex];
skipNode->rowCount = DatumGetInt64(datumArray[Anum_cstore_skipnodes_row_count - skipNode->rowCount = DatumGetInt64(datumArray[Anum_cstore_skipnodes_row_count -
1]); 1]);
skipNode->valueBlockOffset = skipNode->valueChunkOffset =
DatumGetInt64(datumArray[Anum_cstore_skipnodes_value_stream_offset - 1]); DatumGetInt64(datumArray[Anum_cstore_skipnodes_value_stream_offset - 1]);
skipNode->valueLength = skipNode->valueLength =
DatumGetInt64(datumArray[Anum_cstore_skipnodes_value_stream_length - 1]); DatumGetInt64(datumArray[Anum_cstore_skipnodes_value_stream_length - 1]);
skipNode->existsBlockOffset = skipNode->existsChunkOffset =
DatumGetInt64(datumArray[Anum_cstore_skipnodes_exists_stream_offset - 1]); DatumGetInt64(datumArray[Anum_cstore_skipnodes_exists_stream_offset - 1]);
skipNode->existsLength = skipNode->existsLength =
DatumGetInt64(datumArray[Anum_cstore_skipnodes_exists_stream_length - 1]); DatumGetInt64(datumArray[Anum_cstore_skipnodes_exists_stream_length - 1]);
@ -565,8 +565,8 @@ InsertStripeMetadataRow(uint64 storageId, StripeMetadata *stripe)
Int64GetDatum(stripe->fileOffset), Int64GetDatum(stripe->fileOffset),
Int64GetDatum(stripe->dataLength), Int64GetDatum(stripe->dataLength),
Int32GetDatum(stripe->columnCount), Int32GetDatum(stripe->columnCount),
Int32GetDatum(stripe->blockCount), Int32GetDatum(stripe->chunkCount),
Int32GetDatum(stripe->blockRowCount), Int32GetDatum(stripe->chunkRowCount),
Int64GetDatum(stripe->rowCount) Int64GetDatum(stripe->rowCount)
}; };
@ -670,7 +670,7 @@ GetHighestUsedAddressAndId(uint64 storageId,
StripeMetadata StripeMetadata
ReserveStripe(Relation rel, uint64 sizeBytes, ReserveStripe(Relation rel, uint64 sizeBytes,
uint64 rowCount, uint64 columnCount, uint64 rowCount, uint64 columnCount,
uint64 blockCount, uint64 blockRowCount) uint64 chunkCount, uint64 chunkRowCount)
{ {
StripeMetadata stripe = { 0 }; StripeMetadata stripe = { 0 };
uint64 currLogicalHigh = 0; uint64 currLogicalHigh = 0;
@ -720,8 +720,8 @@ ReserveStripe(Relation rel, uint64 sizeBytes,
stripe.fileOffset = resLogicalStart; stripe.fileOffset = resLogicalStart;
stripe.dataLength = sizeBytes; stripe.dataLength = sizeBytes;
stripe.blockCount = blockCount; stripe.chunkCount = chunkCount;
stripe.blockRowCount = blockRowCount; stripe.chunkRowCount = chunkRowCount;
stripe.columnCount = columnCount; stripe.columnCount = columnCount;
stripe.rowCount = rowCount; stripe.rowCount = rowCount;
stripe.id = highestId + 1; stripe.id = highestId + 1;
@ -773,10 +773,10 @@ ReadDataFileStripeList(uint64 storageId, Snapshot snapshot)
datumArray[Anum_cstore_stripes_data_length - 1]); datumArray[Anum_cstore_stripes_data_length - 1]);
stripeMetadata->columnCount = DatumGetInt32( stripeMetadata->columnCount = DatumGetInt32(
datumArray[Anum_cstore_stripes_column_count - 1]); datumArray[Anum_cstore_stripes_column_count - 1]);
stripeMetadata->blockCount = DatumGetInt32( stripeMetadata->chunkCount = DatumGetInt32(
datumArray[Anum_cstore_stripes_block_count - 1]); datumArray[Anum_cstore_stripes_chunk_count - 1]);
stripeMetadata->blockRowCount = DatumGetInt32( stripeMetadata->chunkRowCount = DatumGetInt32(
datumArray[Anum_cstore_stripes_block_row_count - 1]); datumArray[Anum_cstore_stripes_chunk_row_count - 1]);
stripeMetadata->rowCount = DatumGetInt64( stripeMetadata->rowCount = DatumGetInt64(
datumArray[Anum_cstore_stripes_row_count - 1]); datumArray[Anum_cstore_stripes_row_count - 1]);

344
src/backend/columnar/cstore_reader.c
View File

@ -4,7 +4,7 @@
* *
* This file contains function definitions for reading cstore files. This * This file contains function definitions for reading cstore files. This
* includes the logic for reading file level metadata, reading row stripes, * includes the logic for reading file level metadata, reading row stripes,
* and skipping unrelated row blocks and columns. * and skipping unrelated row chunks and columns.
* *
* Copyright (c) 2016, Citus Data, Inc. * Copyright (c) 2016, Citus Data, Inc.
* *
@ -46,23 +46,23 @@ static StripeBuffers * LoadFilteredStripeBuffers(Relation relation,
List *projectedColumnList, List *projectedColumnList,
List *whereClauseList); List *whereClauseList);
static void ReadStripeNextRow(StripeBuffers *stripeBuffers, List *projectedColumnList, static void ReadStripeNextRow(StripeBuffers *stripeBuffers, List *projectedColumnList,
uint64 blockIndex, uint64 blockRowIndex, uint64 chunkIndex, uint64 chunkRowIndex,
BlockData *blockData, Datum *columnValues, ChunkData *chunkData, Datum *columnValues,
bool *columnNulls); bool *columnNulls);
static ColumnBuffers * LoadColumnBuffers(Relation relation, static ColumnBuffers * LoadColumnBuffers(Relation relation,
ColumnBlockSkipNode *blockSkipNodeArray, ColumnChunkSkipNode *chunkSkipNodeArray,
uint32 blockCount, uint64 stripeOffset, uint32 chunkCount, uint64 stripeOffset,
Form_pg_attribute attributeForm); Form_pg_attribute attributeForm);
static bool * SelectedBlockMask(StripeSkipList *stripeSkipList, static bool * SelectedChunkMask(StripeSkipList *stripeSkipList,
List *projectedColumnList, List *whereClauseList); List *projectedColumnList, List *whereClauseList);
static List * BuildRestrictInfoList(List *whereClauseList); static List * BuildRestrictInfoList(List *whereClauseList);
static Node * BuildBaseConstraint(Var *variable); static Node * BuildBaseConstraint(Var *variable);
static OpExpr * MakeOpExpression(Var *variable, int16 strategyNumber); static OpExpr * MakeOpExpression(Var *variable, int16 strategyNumber);
static Oid GetOperatorByType(Oid typeId, Oid accessMethodId, int16 strategyNumber); static Oid GetOperatorByType(Oid typeId, Oid accessMethodId, int16 strategyNumber);
static void UpdateConstraint(Node *baseConstraint, Datum minValue, Datum maxValue); static void UpdateConstraint(Node *baseConstraint, Datum minValue, Datum maxValue);
static StripeSkipList * SelectedBlockSkipList(StripeSkipList *stripeSkipList, static StripeSkipList * SelectedChunkSkipList(StripeSkipList *stripeSkipList,
bool *projectedColumnMask, bool *projectedColumnMask,
bool *selectedBlockMask); bool *selectedChunkMask);
static uint32 StripeSkipListRowCount(StripeSkipList *stripeSkipList); static uint32 StripeSkipListRowCount(StripeSkipList *stripeSkipList);
static bool * ProjectedColumnMask(uint32 columnCount, List *projectedColumnList); static bool * ProjectedColumnMask(uint32 columnCount, List *projectedColumnList);
static void DeserializeBoolArray(StringInfo boolArrayBuffer, bool *boolArray, static void DeserializeBoolArray(StringInfo boolArrayBuffer, bool *boolArray,
@ -71,7 +71,7 @@ static void DeserializeDatumArray(StringInfo datumBuffer, bool *existsArray,
uint32 datumCount, bool datumTypeByValue, uint32 datumCount, bool datumTypeByValue,
int datumTypeLength, char datumTypeAlign, int datumTypeLength, char datumTypeAlign,
Datum *datumArray); Datum *datumArray);
static BlockData * DeserializeBlockData(StripeBuffers *stripeBuffers, uint64 blockIndex, static ChunkData * DeserializeChunkData(StripeBuffers *stripeBuffers, uint64 chunkIndex,
uint32 rowCount, TupleDesc tupleDescriptor, uint32 rowCount, TupleDesc tupleDescriptor,
List *projectedColumnList); List *projectedColumnList);
static Datum ColumnDefaultValue(TupleConstr *tupleConstraints, static Datum ColumnDefaultValue(TupleConstr *tupleConstraints,
@ -106,8 +106,8 @@ CStoreBeginRead(Relation relation, TupleDesc tupleDescriptor,
readState->stripeReadRowCount = 0; readState->stripeReadRowCount = 0;
readState->tupleDescriptor = tupleDescriptor; readState->tupleDescriptor = tupleDescriptor;
readState->stripeReadContext = stripeReadContext; readState->stripeReadContext = stripeReadContext;
readState->blockData = NULL; readState->chunkData = NULL;
readState->deserializedBlockIndex = -1; readState->deserializedChunkIndex = -1;
return readState; return readState;
} }
@ -126,7 +126,7 @@ CStoreReadNextRow(TableReadState *readState, Datum *columnValues, bool *columnNu
/* /*
* If no stripes are loaded, load the next non-empty stripe. Note that when * If no stripes are loaded, load the next non-empty stripe. Note that when
* loading stripes, we skip over blocks whose contents can be filtered with * loading stripes, we skip over chunks whose contents can be filtered with
* the query's restriction qualifiers. So, even when a stripe is physically * the query's restriction qualifiers. So, even when a stripe is physically
* not empty, we may end up loading it as an empty stripe. * not empty, we may end up loading it as an empty stripe.
*/ */
@ -143,7 +143,7 @@ CStoreReadNextRow(TableReadState *readState, Datum *columnValues, bool *columnNu
oldContext = MemoryContextSwitchTo(readState->stripeReadContext); oldContext = MemoryContextSwitchTo(readState->stripeReadContext);
MemoryContextReset(readState->stripeReadContext); MemoryContextReset(readState->stripeReadContext);
readState->blockData = NULL; readState->chunkData = NULL;
stripeMetadata = list_nth(stripeMetadataList, readState->readStripeCount); stripeMetadata = list_nth(stripeMetadataList, readState->readStripeCount);
StripeBuffers *stripeBuffers = LoadFilteredStripeBuffers(readState->relation, StripeBuffers *stripeBuffers = LoadFilteredStripeBuffers(readState->relation,
@ -163,44 +163,44 @@ CStoreReadNextRow(TableReadState *readState, Datum *columnValues, bool *columnNu
{ {
readState->stripeBuffers = stripeBuffers; readState->stripeBuffers = stripeBuffers;
readState->stripeReadRowCount = 0; readState->stripeReadRowCount = 0;
readState->deserializedBlockIndex = -1; readState->deserializedChunkIndex = -1;
break; break;
} }
} }
uint32 blockIndex = readState->stripeReadRowCount / stripeMetadata->blockRowCount; uint32 chunkIndex = readState->stripeReadRowCount / stripeMetadata->chunkRowCount;
uint32 blockRowIndex = readState->stripeReadRowCount % stripeMetadata->blockRowCount; uint32 chunkRowIndex = readState->stripeReadRowCount % stripeMetadata->chunkRowCount;
if (blockIndex != readState->deserializedBlockIndex) if (chunkIndex != readState->deserializedChunkIndex)
{ {
uint32 blockRowCount = 0; uint32 chunkRowCount = 0;
uint32 stripeRowCount = stripeMetadata->rowCount; uint32 stripeRowCount = stripeMetadata->rowCount;
uint32 lastBlockIndex = stripeRowCount / stripeMetadata->blockRowCount; uint32 lastChunkIndex = stripeRowCount / stripeMetadata->chunkRowCount;
if (blockIndex == lastBlockIndex) if (chunkIndex == lastChunkIndex)
{ {
blockRowCount = stripeRowCount % stripeMetadata->blockRowCount; chunkRowCount = stripeRowCount % stripeMetadata->chunkRowCount;
} }
else else
{ {
blockRowCount = stripeMetadata->blockRowCount; chunkRowCount = stripeMetadata->chunkRowCount;
} }
oldContext = MemoryContextSwitchTo(readState->stripeReadContext); oldContext = MemoryContextSwitchTo(readState->stripeReadContext);
FreeBlockData(readState->blockData); FreeChunkData(readState->chunkData);
readState->blockData = readState->chunkData =
DeserializeBlockData(readState->stripeBuffers, blockIndex, DeserializeChunkData(readState->stripeBuffers, chunkIndex,
blockRowCount, readState->tupleDescriptor, chunkRowCount, readState->tupleDescriptor,
readState->projectedColumnList); readState->projectedColumnList);
MemoryContextSwitchTo(oldContext); MemoryContextSwitchTo(oldContext);
readState->deserializedBlockIndex = blockIndex; readState->deserializedChunkIndex = chunkIndex;
} }
ReadStripeNextRow(readState->stripeBuffers, readState->projectedColumnList, ReadStripeNextRow(readState->stripeBuffers, readState->projectedColumnList,
blockIndex, blockRowIndex, readState->blockData, chunkIndex, chunkRowIndex, readState->chunkData,
columnValues, columnNulls); columnValues, columnNulls);
/* /*
@ -241,68 +241,68 @@ CStoreEndRead(TableReadState *readState)
/* /*
* CreateEmptyBlockDataArray creates data buffers to keep deserialized exist and * CreateEmptyChunkDataArray creates data buffers to keep deserialized exist and
* value arrays for requested columns in columnMask. * value arrays for requested columns in columnMask.
*/ */
BlockData * ChunkData *
CreateEmptyBlockData(uint32 columnCount, bool *columnMask, uint32 blockRowCount) CreateEmptyChunkData(uint32 columnCount, bool *columnMask, uint32 chunkRowCount)
{ {
uint32 columnIndex = 0; uint32 columnIndex = 0;
BlockData *blockData = palloc0(sizeof(BlockData)); ChunkData *chunkData = palloc0(sizeof(ChunkData));
blockData->existsArray = palloc0(columnCount * sizeof(bool *)); chunkData->existsArray = palloc0(columnCount * sizeof(bool *));
blockData->valueArray = palloc0(columnCount * sizeof(Datum *)); chunkData->valueArray = palloc0(columnCount * sizeof(Datum *));
blockData->valueBufferArray = palloc0(columnCount * sizeof(StringInfo)); chunkData->valueBufferArray = palloc0(columnCount * sizeof(StringInfo));
blockData->columnCount = columnCount; chunkData->columnCount = columnCount;
blockData->rowCount = blockRowCount; chunkData->rowCount = chunkRowCount;
/* allocate block memory for deserialized data */ /* allocate chunk memory for deserialized data */
for (columnIndex = 0; columnIndex < columnCount; columnIndex++) for (columnIndex = 0; columnIndex < columnCount; columnIndex++)
{ {
if (columnMask[columnIndex]) if (columnMask[columnIndex])
{ {
blockData->existsArray[columnIndex] = palloc0(blockRowCount * sizeof(bool)); chunkData->existsArray[columnIndex] = palloc0(chunkRowCount * sizeof(bool));
blockData->valueArray[columnIndex] = palloc0(blockRowCount * sizeof(Datum)); chunkData->valueArray[columnIndex] = palloc0(chunkRowCount * sizeof(Datum));
blockData->valueBufferArray[columnIndex] = NULL; chunkData->valueBufferArray[columnIndex] = NULL;
} }
} }
return blockData; return chunkData;
} }
/* /*
* FreeBlockData deallocates data buffers to keep deserialized exist and * FreeChunkData deallocates data buffers to keep deserialized exist and
* value arrays for requested columns in columnMask. * value arrays for requested columns in columnMask.
* ColumnBlockData->serializedValueBuffer lives in memory read/write context * ColumnChunkData->serializedValueBuffer lives in memory read/write context
* so it is deallocated automatically when the context is deleted. * so it is deallocated automatically when the context is deleted.
*/ */
void void
FreeBlockData(BlockData *blockData) FreeChunkData(ChunkData *chunkData)
{ {
uint32 columnIndex = 0; uint32 columnIndex = 0;
if (blockData == NULL) if (chunkData == NULL)
{ {
return; return;
} }
for (columnIndex = 0; columnIndex < blockData->columnCount; columnIndex++) for (columnIndex = 0; columnIndex < chunkData->columnCount; columnIndex++)
{ {
if (blockData->existsArray[columnIndex] != NULL) if (chunkData->existsArray[columnIndex] != NULL)
{ {
pfree(blockData->existsArray[columnIndex]); pfree(chunkData->existsArray[columnIndex]);
} }
if (blockData->valueArray[columnIndex] != NULL) if (chunkData->valueArray[columnIndex] != NULL)
{ {
pfree(blockData->valueArray[columnIndex]); pfree(chunkData->valueArray[columnIndex]);
} }
} }
pfree(blockData->existsArray); pfree(chunkData->existsArray);
pfree(blockData->valueArray); pfree(chunkData->valueArray);
pfree(blockData); pfree(chunkData);
} }
@ -326,7 +326,7 @@ CStoreTableRowCount(Relation relation)
/* /*
* LoadFilteredStripeBuffers reads serialized stripe data from the given file. * LoadFilteredStripeBuffers reads serialized stripe data from the given file.
* The function skips over blocks whose rows are refuted by restriction qualifiers, * The function skips over chunks whose rows are refuted by restriction qualifiers,
* and only loads columns that are projected in the query. * and only loads columns that are projected in the query.
*/ */
static StripeBuffers * static StripeBuffers *
@ -342,14 +342,14 @@ LoadFilteredStripeBuffers(Relation relation, StripeMetadata *stripeMetadata,
StripeSkipList *stripeSkipList = ReadStripeSkipList(relation->rd_node, StripeSkipList *stripeSkipList = ReadStripeSkipList(relation->rd_node,
stripeMetadata->id, stripeMetadata->id,
tupleDescriptor, tupleDescriptor,
stripeMetadata->blockCount); stripeMetadata->chunkCount);
bool *selectedBlockMask = SelectedBlockMask(stripeSkipList, projectedColumnList, bool *selectedChunkMask = SelectedChunkMask(stripeSkipList, projectedColumnList,
whereClauseList); whereClauseList);
StripeSkipList *selectedBlockSkipList = StripeSkipList *selectedChunkSkipList =
SelectedBlockSkipList(stripeSkipList, projectedColumnMask, SelectedChunkSkipList(stripeSkipList, projectedColumnMask,
selectedBlockMask); selectedChunkMask);
/* load column data for projected columns */ /* load column data for projected columns */
ColumnBuffers **columnBuffersArray = palloc0(columnCount * sizeof(ColumnBuffers *)); ColumnBuffers **columnBuffersArray = palloc0(columnCount * sizeof(ColumnBuffers *));
@ -358,13 +358,13 @@ LoadFilteredStripeBuffers(Relation relation, StripeMetadata *stripeMetadata,
{ {
if (projectedColumnMask[columnIndex]) if (projectedColumnMask[columnIndex])
{ {
ColumnBlockSkipNode *blockSkipNode = ColumnChunkSkipNode *chunkSkipNode =
selectedBlockSkipList->blockSkipNodeArray[columnIndex]; selectedChunkSkipList->chunkSkipNodeArray[columnIndex];
Form_pg_attribute attributeForm = TupleDescAttr(tupleDescriptor, columnIndex); Form_pg_attribute attributeForm = TupleDescAttr(tupleDescriptor, columnIndex);
uint32 blockCount = selectedBlockSkipList->blockCount; uint32 chunkCount = selectedChunkSkipList->chunkCount;
ColumnBuffers *columnBuffers = LoadColumnBuffers(relation, blockSkipNode, ColumnBuffers *columnBuffers = LoadColumnBuffers(relation, chunkSkipNode,
blockCount, chunkCount,
stripeMetadata->fileOffset, stripeMetadata->fileOffset,
attributeForm); attributeForm);
@ -374,7 +374,7 @@ LoadFilteredStripeBuffers(Relation relation, StripeMetadata *stripeMetadata,
StripeBuffers *stripeBuffers = palloc0(sizeof(StripeBuffers)); StripeBuffers *stripeBuffers = palloc0(sizeof(StripeBuffers));
stripeBuffers->columnCount = columnCount; stripeBuffers->columnCount = columnCount;
stripeBuffers->rowCount = StripeSkipListRowCount(selectedBlockSkipList); stripeBuffers->rowCount = StripeSkipListRowCount(selectedChunkSkipList);
stripeBuffers->columnBuffersArray = columnBuffersArray; stripeBuffers->columnBuffersArray = columnBuffersArray;
return stripeBuffers; return stripeBuffers;
@ -388,8 +388,8 @@ LoadFilteredStripeBuffers(Relation relation, StripeMetadata *stripeMetadata,
*/ */
static void static void
ReadStripeNextRow(StripeBuffers *stripeBuffers, List *projectedColumnList, ReadStripeNextRow(StripeBuffers *stripeBuffers, List *projectedColumnList,
uint64 blockIndex, uint64 blockRowIndex, uint64 chunkIndex, uint64 chunkRowIndex,
BlockData *blockData, Datum *columnValues, ChunkData *chunkData, Datum *columnValues,
bool *columnNulls) bool *columnNulls)
{ {
ListCell *projectedColumnCell = NULL; ListCell *projectedColumnCell = NULL;
@ -402,9 +402,9 @@ ReadStripeNextRow(StripeBuffers *stripeBuffers, List *projectedColumnList,
Var *projectedColumn = lfirst(projectedColumnCell); Var *projectedColumn = lfirst(projectedColumnCell);
uint32 columnIndex = projectedColumn->varattno - 1; uint32 columnIndex = projectedColumn->varattno - 1;
if (blockData->existsArray[columnIndex][blockRowIndex]) if (chunkData->existsArray[columnIndex][chunkRowIndex])
{ {
columnValues[columnIndex] = blockData->valueArray[columnIndex][blockRowIndex]; columnValues[columnIndex] = chunkData->valueArray[columnIndex][chunkRowIndex];
columnNulls[columnIndex] = false; columnNulls[columnIndex] = false;
} }
} }
@ -413,73 +413,73 @@ ReadStripeNextRow(StripeBuffers *stripeBuffers, List *projectedColumnList,
/* /*
* LoadColumnBuffers reads serialized column data from the given file. These * LoadColumnBuffers reads serialized column data from the given file. These
* column data are laid out as sequential blocks in the file; and block positions * column data are laid out as sequential chunks in the file; and chunk positions
* and lengths are retrieved from the column block skip node array. * and lengths are retrieved from the column chunk skip node array.
*/ */
static ColumnBuffers * static ColumnBuffers *
LoadColumnBuffers(Relation relation, ColumnBlockSkipNode *blockSkipNodeArray, LoadColumnBuffers(Relation relation, ColumnChunkSkipNode *chunkSkipNodeArray,
uint32 blockCount, uint64 stripeOffset, uint32 chunkCount, uint64 stripeOffset,
Form_pg_attribute attributeForm) Form_pg_attribute attributeForm)
{ {
uint32 blockIndex = 0; uint32 chunkIndex = 0;
ColumnBlockBuffers **blockBuffersArray = ColumnChunkBuffers **chunkBuffersArray =
palloc0(blockCount * sizeof(ColumnBlockBuffers *)); palloc0(chunkCount * sizeof(ColumnChunkBuffers *));
for (blockIndex = 0; blockIndex < blockCount; blockIndex++) for (chunkIndex = 0; chunkIndex < chunkCount; chunkIndex++)
{ {
blockBuffersArray[blockIndex] = palloc0(sizeof(ColumnBlockBuffers)); chunkBuffersArray[chunkIndex] = palloc0(sizeof(ColumnChunkBuffers));
} }
/* /*
* We first read the "exists" blocks. We don't read "values" array here, * We first read the "exists" chunks. We don't read "values" array here,
* because "exists" blocks are stored sequentially on disk, and we want to * because "exists" chunks are stored sequentially on disk, and we want to
* minimize disk seeks. * minimize disk seeks.
*/ */
for (blockIndex = 0; blockIndex < blockCount; blockIndex++) for (chunkIndex = 0; chunkIndex < chunkCount; chunkIndex++)
{ {
ColumnBlockSkipNode *blockSkipNode = &blockSkipNodeArray[blockIndex]; ColumnChunkSkipNode *chunkSkipNode = &chunkSkipNodeArray[chunkIndex];
uint64 existsOffset = stripeOffset + blockSkipNode->existsBlockOffset; uint64 existsOffset = stripeOffset + chunkSkipNode->existsChunkOffset;
StringInfo rawExistsBuffer = ReadFromSmgr(relation, existsOffset, StringInfo rawExistsBuffer = ReadFromSmgr(relation, existsOffset,
blockSkipNode->existsLength); chunkSkipNode->existsLength);
blockBuffersArray[blockIndex]->existsBuffer = rawExistsBuffer; chunkBuffersArray[chunkIndex]->existsBuffer = rawExistsBuffer;
} }
/* then read "values" blocks, which are also stored sequentially on disk */ /* then read "values" chunks, which are also stored sequentially on disk */
for (blockIndex = 0; blockIndex < blockCount; blockIndex++) for (chunkIndex = 0; chunkIndex < chunkCount; chunkIndex++)
{ {
ColumnBlockSkipNode *blockSkipNode = &blockSkipNodeArray[blockIndex]; ColumnChunkSkipNode *chunkSkipNode = &chunkSkipNodeArray[chunkIndex];
CompressionType compressionType = blockSkipNode->valueCompressionType; CompressionType compressionType = chunkSkipNode->valueCompressionType;
uint64 valueOffset = stripeOffset + blockSkipNode->valueBlockOffset; uint64 valueOffset = stripeOffset + chunkSkipNode->valueChunkOffset;
StringInfo rawValueBuffer = ReadFromSmgr(relation, valueOffset, StringInfo rawValueBuffer = ReadFromSmgr(relation, valueOffset,
blockSkipNode->valueLength); chunkSkipNode->valueLength);
blockBuffersArray[blockIndex]->valueBuffer = rawValueBuffer; chunkBuffersArray[chunkIndex]->valueBuffer = rawValueBuffer;
blockBuffersArray[blockIndex]->valueCompressionType = compressionType; chunkBuffersArray[chunkIndex]->valueCompressionType = compressionType;
} }
ColumnBuffers *columnBuffers = palloc0(sizeof(ColumnBuffers)); ColumnBuffers *columnBuffers = palloc0(sizeof(ColumnBuffers));
columnBuffers->blockBuffersArray = blockBuffersArray; columnBuffers->chunkBuffersArray = chunkBuffersArray;
return columnBuffers; return columnBuffers;
} }
/* /*
* SelectedBlockMask walks over each column's blocks and checks if a block can * SelectedChunkMask walks over each column's chunks and checks if a chunk can
* be filtered without reading its data. The filtering happens when all rows in * be filtered without reading its data. The filtering happens when all rows in
* the block can be refuted by the given qualifier conditions. * the chunk can be refuted by the given qualifier conditions.
*/ */
static bool * static bool *
SelectedBlockMask(StripeSkipList *stripeSkipList, List *projectedColumnList, SelectedChunkMask(StripeSkipList *stripeSkipList, List *projectedColumnList,
List *whereClauseList) List *whereClauseList)
{ {
ListCell *columnCell = NULL; ListCell *columnCell = NULL;
uint32 blockIndex = 0; uint32 chunkIndex = 0;
List *restrictInfoList = BuildRestrictInfoList(whereClauseList); List *restrictInfoList = BuildRestrictInfoList(whereClauseList);
bool *selectedBlockMask = palloc0(stripeSkipList->blockCount * sizeof(bool)); bool *selectedChunkMask = palloc0(stripeSkipList->chunkCount * sizeof(bool));
memset(selectedBlockMask, true, stripeSkipList->blockCount * sizeof(bool)); memset(selectedChunkMask, true, stripeSkipList->chunkCount * sizeof(bool));
foreach(columnCell, projectedColumnList) foreach(columnCell, projectedColumnList)
{ {
@ -496,24 +496,24 @@ SelectedBlockMask(StripeSkipList *stripeSkipList, List *projectedColumnList,
} }
Node *baseConstraint = BuildBaseConstraint(column); Node *baseConstraint = BuildBaseConstraint(column);
for (blockIndex = 0; blockIndex < stripeSkipList->blockCount; blockIndex++) for (chunkIndex = 0; chunkIndex < stripeSkipList->chunkCount; chunkIndex++)
{ {
bool predicateRefuted = false; bool predicateRefuted = false;
ColumnBlockSkipNode *blockSkipNodeArray = ColumnChunkSkipNode *chunkSkipNodeArray =
stripeSkipList->blockSkipNodeArray[columnIndex]; stripeSkipList->chunkSkipNodeArray[columnIndex];
ColumnBlockSkipNode *blockSkipNode = &blockSkipNodeArray[blockIndex]; ColumnChunkSkipNode *chunkSkipNode = &chunkSkipNodeArray[chunkIndex];
/* /*
* A column block with comparable data type can miss min/max values * A column chunk with comparable data type can miss min/max values
* if all values in the block are NULL. * if all values in the chunk are NULL.
*/ */
if (!blockSkipNode->hasMinMax) if (!chunkSkipNode->hasMinMax)
{ {
continue; continue;
} }
UpdateConstraint(baseConstraint, blockSkipNode->minimumValue, UpdateConstraint(baseConstraint, chunkSkipNode->minimumValue,
blockSkipNode->maximumValue); chunkSkipNode->maximumValue);
List *constraintList = list_make1(baseConstraint); List *constraintList = list_make1(baseConstraint);
#if (PG_VERSION_NUM >= 100000) #if (PG_VERSION_NUM >= 100000)
@ -524,12 +524,12 @@ SelectedBlockMask(StripeSkipList *stripeSkipList, List *projectedColumnList,
#endif #endif
if (predicateRefuted) if (predicateRefuted)
{ {
selectedBlockMask[blockIndex] = false; selectedChunkMask[chunkIndex] = false;
} }
} }
} }
return selectedBlockMask; return selectedChunkMask;
} }
@ -596,7 +596,7 @@ BuildRestrictInfoList(List *whereClauseList)
/* /*
* BuildBaseConstraint builds and returns a base constraint. This constraint * BuildBaseConstraint builds and returns a base constraint. This constraint
* implements an expression in the form of (var <= max && var >= min), where * implements an expression in the form of (var <= max && var >= min), where
* min and max values represent a block's min and max values. These block * min and max values represent a chunk's min and max values. These chunk
* values are filled in after the constraint is built. This function is based * values are filled in after the constraint is built. This function is based
* on a similar function from CitusDB's shard pruning logic. * on a similar function from CitusDB's shard pruning logic.
*/ */
@ -700,83 +700,83 @@ UpdateConstraint(Node *baseConstraint, Datum minValue, Datum maxValue)
/* /*
* SelectedBlockSkipList constructs a new StripeSkipList in which the * SelectedChunkSkipList constructs a new StripeSkipList in which the
* non-selected blocks are removed from the given stripeSkipList. * non-selected chunks are removed from the given stripeSkipList.
*/ */
static StripeSkipList * static StripeSkipList *
SelectedBlockSkipList(StripeSkipList *stripeSkipList, bool *projectedColumnMask, SelectedChunkSkipList(StripeSkipList *stripeSkipList, bool *projectedColumnMask,
bool *selectedBlockMask) bool *selectedChunkMask)
{ {
uint32 selectedBlockCount = 0; uint32 selectedChunkCount = 0;
uint32 blockIndex = 0; uint32 chunkIndex = 0;
uint32 columnIndex = 0; uint32 columnIndex = 0;
uint32 columnCount = stripeSkipList->columnCount; uint32 columnCount = stripeSkipList->columnCount;
for (blockIndex = 0; blockIndex < stripeSkipList->blockCount; blockIndex++) for (chunkIndex = 0; chunkIndex < stripeSkipList->chunkCount; chunkIndex++)
{ {
if (selectedBlockMask[blockIndex]) if (selectedChunkMask[chunkIndex])
{ {
selectedBlockCount++; selectedChunkCount++;
} }
} }
ColumnBlockSkipNode **selectedBlockSkipNodeArray = palloc0(columnCount * ColumnChunkSkipNode **selectedChunkSkipNodeArray = palloc0(columnCount *
sizeof(ColumnBlockSkipNode sizeof(ColumnChunkSkipNode
*)); *));
for (columnIndex = 0; columnIndex < columnCount; columnIndex++) for (columnIndex = 0; columnIndex < columnCount; columnIndex++)
{ {
uint32 selectedBlockIndex = 0; uint32 selectedChunkIndex = 0;
bool firstColumn = columnIndex == 0; bool firstColumn = columnIndex == 0;
/* first column's block skip node is always read */ /* first column's chunk skip node is always read */
if (!projectedColumnMask[columnIndex] && !firstColumn) if (!projectedColumnMask[columnIndex] && !firstColumn)
{ {
selectedBlockSkipNodeArray[columnIndex] = NULL; selectedChunkSkipNodeArray[columnIndex] = NULL;
continue; continue;
} }
Assert(stripeSkipList->blockSkipNodeArray[columnIndex] != NULL); Assert(stripeSkipList->chunkSkipNodeArray[columnIndex] != NULL);
selectedBlockSkipNodeArray[columnIndex] = palloc0(selectedBlockCount * selectedChunkSkipNodeArray[columnIndex] = palloc0(selectedChunkCount *
sizeof(ColumnBlockSkipNode)); sizeof(ColumnChunkSkipNode));
for (blockIndex = 0; blockIndex < stripeSkipList->blockCount; blockIndex++) for (chunkIndex = 0; chunkIndex < stripeSkipList->chunkCount; chunkIndex++)
{ {
if (selectedBlockMask[blockIndex]) if (selectedChunkMask[chunkIndex])
{ {
selectedBlockSkipNodeArray[columnIndex][selectedBlockIndex] = selectedChunkSkipNodeArray[columnIndex][selectedChunkIndex] =
stripeSkipList->blockSkipNodeArray[columnIndex][blockIndex]; stripeSkipList->chunkSkipNodeArray[columnIndex][chunkIndex];
selectedBlockIndex++; selectedChunkIndex++;
} }
} }
} }
StripeSkipList *SelectedBlockSkipList = palloc0(sizeof(StripeSkipList)); StripeSkipList *SelectedChunkSkipList = palloc0(sizeof(StripeSkipList));
SelectedBlockSkipList->blockSkipNodeArray = selectedBlockSkipNodeArray; SelectedChunkSkipList->chunkSkipNodeArray = selectedChunkSkipNodeArray;
SelectedBlockSkipList->blockCount = selectedBlockCount; SelectedChunkSkipList->chunkCount = selectedChunkCount;
SelectedBlockSkipList->columnCount = stripeSkipList->columnCount; SelectedChunkSkipList->columnCount = stripeSkipList->columnCount;
return SelectedBlockSkipList; return SelectedChunkSkipList;
} }
/* /*
* StripeSkipListRowCount counts the number of rows in the given stripeSkipList. * StripeSkipListRowCount counts the number of rows in the given stripeSkipList.
* To do this, the function finds the first column, and sums up row counts across * To do this, the function finds the first column, and sums up row counts across
* all blocks for that column. * all chunks for that column.
*/ */
static uint32 static uint32
StripeSkipListRowCount(StripeSkipList *stripeSkipList) StripeSkipListRowCount(StripeSkipList *stripeSkipList)
{ {
uint32 stripeSkipListRowCount = 0; uint32 stripeSkipListRowCount = 0;
uint32 blockIndex = 0; uint32 chunkIndex = 0;
ColumnBlockSkipNode *firstColumnSkipNodeArray = ColumnChunkSkipNode *firstColumnSkipNodeArray =
stripeSkipList->blockSkipNodeArray[0]; stripeSkipList->chunkSkipNodeArray[0];
for (blockIndex = 0; blockIndex < stripeSkipList->blockCount; blockIndex++) for (chunkIndex = 0; chunkIndex < stripeSkipList->chunkCount; chunkIndex++)
{ {
uint32 blockRowCount = firstColumnSkipNodeArray[blockIndex].rowCount; uint32 chunkRowCount = firstColumnSkipNodeArray[chunkIndex].rowCount;
stripeSkipListRowCount += blockRowCount; stripeSkipListRowCount += chunkRowCount;
} }
return stripeSkipListRowCount; return stripeSkipListRowCount;
@ -878,21 +878,21 @@ DeserializeDatumArray(StringInfo datumBuffer, bool *existsArray, uint32 datumCou
/* /*
* DeserializeBlockData deserializes requested data block for all columns and * DeserializeChunkData deserializes requested data chunk for all columns and
* stores in blockDataArray. It uncompresses serialized data if necessary. The * stores in chunkDataArray. It uncompresses serialized data if necessary. The
* function also deallocates data buffers used for previous block, and compressed * function also deallocates data buffers used for previous chunk, and compressed
* data buffers for the current block which will not be needed again. If a column * data buffers for the current chunk which will not be needed again. If a column
* data is not present serialized buffer, then default value (or null) is used * data is not present serialized buffer, then default value (or null) is used
* to fill value array. * to fill value array.
*/ */
static BlockData * static ChunkData *
DeserializeBlockData(StripeBuffers *stripeBuffers, uint64 blockIndex, DeserializeChunkData(StripeBuffers *stripeBuffers, uint64 chunkIndex,
uint32 rowCount, TupleDesc tupleDescriptor, uint32 rowCount, TupleDesc tupleDescriptor,
List *projectedColumnList) List *projectedColumnList)
{ {
int columnIndex = 0; int columnIndex = 0;
bool *columnMask = ProjectedColumnMask(tupleDescriptor->natts, projectedColumnList); bool *columnMask = ProjectedColumnMask(tupleDescriptor->natts, projectedColumnList);
BlockData *blockData = CreateEmptyBlockData(tupleDescriptor->natts, columnMask, ChunkData *chunkData = CreateEmptyChunkData(tupleDescriptor->natts, columnMask,
rowCount); rowCount);
for (columnIndex = 0; columnIndex < stripeBuffers->columnCount; columnIndex++) for (columnIndex = 0; columnIndex < stripeBuffers->columnCount; columnIndex++)
@ -908,30 +908,30 @@ DeserializeBlockData(StripeBuffers *stripeBuffers, uint64 blockIndex,
if (columnBuffers != NULL) if (columnBuffers != NULL)
{ {
ColumnBlockBuffers *blockBuffers = ColumnChunkBuffers *chunkBuffers =
columnBuffers->blockBuffersArray[blockIndex]; columnBuffers->chunkBuffersArray[chunkIndex];
/* decompress and deserialize current block's data */ /* decompress and deserialize current chunk's data */
StringInfo valueBuffer = DecompressBuffer(blockBuffers->valueBuffer, StringInfo valueBuffer = DecompressBuffer(chunkBuffers->valueBuffer,
blockBuffers->valueCompressionType); chunkBuffers->valueCompressionType);
if (blockBuffers->valueCompressionType != COMPRESSION_NONE) if (chunkBuffers->valueCompressionType != COMPRESSION_NONE)
{ {
/* compressed data is not needed anymore */ /* compressed data is not needed anymore */
pfree(blockBuffers->valueBuffer->data); pfree(chunkBuffers->valueBuffer->data);
pfree(blockBuffers->valueBuffer); pfree(chunkBuffers->valueBuffer);
} }
DeserializeBoolArray(blockBuffers->existsBuffer, DeserializeBoolArray(chunkBuffers->existsBuffer,
blockData->existsArray[columnIndex], chunkData->existsArray[columnIndex],
rowCount); rowCount);
DeserializeDatumArray(valueBuffer, blockData->existsArray[columnIndex], DeserializeDatumArray(valueBuffer, chunkData->existsArray[columnIndex],
rowCount, attributeForm->attbyval, rowCount, attributeForm->attbyval,
attributeForm->attlen, attributeForm->attalign, attributeForm->attlen, attributeForm->attalign,
blockData->valueArray[columnIndex]); chunkData->valueArray[columnIndex]);
/* store current block's data buffer to be freed at next block read */ /* store current chunk's data buffer to be freed at next chunk read */
blockData->valueBufferArray[columnIndex] = valueBuffer; chunkData->valueBufferArray[columnIndex] = valueBuffer;
} }
else if (columnAdded) else if (columnAdded)
{ {
@ -948,19 +948,19 @@ DeserializeBlockData(StripeBuffers *stripeBuffers, uint64 blockIndex,
for (rowIndex = 0; rowIndex < rowCount; rowIndex++) for (rowIndex = 0; rowIndex < rowCount; rowIndex++)
{ {
blockData->existsArray[columnIndex][rowIndex] = true; chunkData->existsArray[columnIndex][rowIndex] = true;
blockData->valueArray[columnIndex][rowIndex] = defaultValue; chunkData->valueArray[columnIndex][rowIndex] = defaultValue;
} }
} }
else else
{ {
memset(blockData->existsArray[columnIndex], false, memset(chunkData->existsArray[columnIndex], false,
rowCount * sizeof(bool)); rowCount * sizeof(bool));
} }
} }
} }
return blockData; return chunkData;
} }

46
src/backend/columnar/cstore_tableam.c
View File

@ -629,7 +629,7 @@ cstore_relation_copy_for_cluster(Relation OldHeap, Relation NewHeap,
TableWriteState *writeState = CStoreBeginWrite(NewHeap->rd_node, TableWriteState *writeState = CStoreBeginWrite(NewHeap->rd_node,
cstoreOptions.compressionType, cstoreOptions.compressionType,
cstoreOptions.stripeRowCount, cstoreOptions.stripeRowCount,
cstoreOptions.blockRowCount, cstoreOptions.chunkRowCount,
targetDesc); targetDesc);
TableReadState *readState = CStoreBeginRead(OldHeap, sourceDesc, TableReadState *readState = CStoreBeginRead(OldHeap, sourceDesc,
@ -691,9 +691,9 @@ LogRelationStats(Relation rel, int elevel)
int compressionStats[COMPRESSION_COUNT] = { 0 }; int compressionStats[COMPRESSION_COUNT] = { 0 };
uint64 totalStripeLength = 0; uint64 totalStripeLength = 0;
uint64 tupleCount = 0; uint64 tupleCount = 0;
uint64 blockCount = 0; uint64 chunkCount = 0;
TupleDesc tupdesc = RelationGetDescr(rel); TupleDesc tupdesc = RelationGetDescr(rel);
uint64 droppedBlocksWithData = 0; uint64 droppedChunksWithData = 0;
List *stripeList = StripesForRelfilenode(relfilenode); List *stripeList = StripesForRelfilenode(relfilenode);
int stripeCount = list_length(stripeList); int stripeCount = list_length(stripeList);
@ -703,24 +703,24 @@ LogRelationStats(Relation rel, int elevel)
StripeMetadata *stripe = lfirst(stripeMetadataCell); StripeMetadata *stripe = lfirst(stripeMetadataCell);
StripeSkipList *skiplist = ReadStripeSkipList(relfilenode, stripe->id, StripeSkipList *skiplist = ReadStripeSkipList(relfilenode, stripe->id,
RelationGetDescr(rel), RelationGetDescr(rel),
stripe->blockCount); stripe->chunkCount);
for (uint32 column = 0; column < skiplist->columnCount; column++) for (uint32 column = 0; column < skiplist->columnCount; column++)
{ {
bool attrDropped = tupdesc->attrs[column].attisdropped; bool attrDropped = tupdesc->attrs[column].attisdropped;
for (uint32 block = 0; block < skiplist->blockCount; block++) for (uint32 chunk = 0; chunk < skiplist->chunkCount; chunk++)
{ {
ColumnBlockSkipNode *skipnode = ColumnChunkSkipNode *skipnode =
&skiplist->blockSkipNodeArray[column][block]; &skiplist->chunkSkipNodeArray[column][chunk];
/* ignore zero length blocks for dropped attributes */ /* ignore zero length chunks for dropped attributes */
if (skipnode->valueLength > 0) if (skipnode->valueLength > 0)
{ {
compressionStats[skipnode->valueCompressionType]++; compressionStats[skipnode->valueCompressionType]++;
blockCount++; chunkCount++;
if (attrDropped) if (attrDropped)
{ {
droppedBlocksWithData++; droppedChunksWithData++;
} }
} }
} }
@ -746,9 +746,9 @@ LogRelationStats(Relation rel, int elevel)
tupleCount, stripeCount, tupleCount, stripeCount,
stripeCount ? tupleCount / stripeCount : 0); stripeCount ? tupleCount / stripeCount : 0);
appendStringInfo(infoBuf, appendStringInfo(infoBuf,
"block count: %ld" "chunk count: %ld"
", containing data for dropped columns: %ld", ", containing data for dropped columns: %ld",
blockCount, droppedBlocksWithData); chunkCount, droppedChunksWithData);
for (int compressionType = 0; compressionType < COMPRESSION_COUNT; compressionType++) for (int compressionType = 0; compressionType < COMPRESSION_COUNT; compressionType++)
{ {
appendStringInfo(infoBuf, appendStringInfo(infoBuf,
@ -914,7 +914,7 @@ cstore_scan_analyze_next_tuple(TableScanDesc scan, TransactionId OldestXmin,
* based access methods where it chooses random pages, and then reads * based access methods where it chooses random pages, and then reads
* tuples from those pages. * tuples from those pages.
* *
* We could do something like that here by choosing sample stripes or blocks, * We could do something like that here by choosing sample stripes or chunks,
* but getting that correct might need quite some work. Since cstore_fdw's * but getting that correct might need quite some work. Since cstore_fdw's
* ANALYZE scanned all rows, as a starter we do the same here and scan all * ANALYZE scanned all rows, as a starter we do the same here and scan all
* rows. * rows.
@ -1307,11 +1307,11 @@ CitusCreateAlterColumnarTableSet(char *qualifiedRelationName,
appendStringInfo(&buf, appendStringInfo(&buf,
"SELECT alter_columnar_table_set(%s, " "SELECT alter_columnar_table_set(%s, "
"block_row_count => %d, " "chunk_row_count => %d, "
"stripe_row_count => %lu, " "stripe_row_count => %lu, "
"compression => %s);", "compression => %s);",
quote_literal_cstr(qualifiedRelationName), quote_literal_cstr(qualifiedRelationName),
options->blockRowCount, options->chunkRowCount,
options->stripeRowCount, options->stripeRowCount,
quote_literal_cstr(CompressionTypeStr(options->compressionType))); quote_literal_cstr(CompressionTypeStr(options->compressionType)));
@ -1427,7 +1427,7 @@ ColumnarGetTableOptionsDDL(Oid relationId)
* sql syntax: * sql syntax:
* pg_catalog.alter_columnar_table_set( * pg_catalog.alter_columnar_table_set(
* table_name regclass, * table_name regclass,
* block_row_count int DEFAULT NULL, * chunk_row_count int DEFAULT NULL,
* stripe_row_count int DEFAULT NULL, * stripe_row_count int DEFAULT NULL,
* compression name DEFAULT null) * compression name DEFAULT null)
* *
@ -1459,12 +1459,12 @@ alter_columnar_table_set(PG_FUNCTION_ARGS)
ereport(ERROR, (errmsg("unable to read current options for table"))); ereport(ERROR, (errmsg("unable to read current options for table")));
} }
/* block_row_count => not null */ /* chunk_row_count => not null */
if (!PG_ARGISNULL(1)) if (!PG_ARGISNULL(1))
{ {
options.blockRowCount = PG_GETARG_INT32(1); options.chunkRowCount = PG_GETARG_INT32(1);
ereport(DEBUG1, ereport(DEBUG1,
(errmsg("updating block row count to %d", options.blockRowCount))); (errmsg("updating chunk row count to %d", options.chunkRowCount)));
} }
/* stripe_row_count => not null */ /* stripe_row_count => not null */
@ -1520,7 +1520,7 @@ alter_columnar_table_set(PG_FUNCTION_ARGS)
* pg_catalog.alter_columnar_table_re * pg_catalog.alter_columnar_table_re
* teset( * teset(
* table_name regclass, * table_name regclass,
* block_row_count bool DEFAULT FALSE, * chunk_row_count bool DEFAULT FALSE,
* stripe_row_count bool DEFAULT FALSE, * stripe_row_count bool DEFAULT FALSE,
* compression bool DEFAULT FALSE) * compression bool DEFAULT FALSE)
* *
@ -1549,12 +1549,12 @@ alter_columnar_table_reset(PG_FUNCTION_ARGS)
ereport(ERROR, (errmsg("unable to read current options for table"))); ereport(ERROR, (errmsg("unable to read current options for table")));
} }
/* block_row_count => true */ /* chunk_row_count => true */
if (!PG_ARGISNULL(1) && PG_GETARG_BOOL(1)) if (!PG_ARGISNULL(1) && PG_GETARG_BOOL(1))
{ {
options.blockRowCount = cstore_block_row_count; options.chunkRowCount = cstore_chunk_row_count;
ereport(DEBUG1, ereport(DEBUG1,
(errmsg("resetting block row count to %d", options.blockRowCount))); (errmsg("resetting chunk row count to %d", options.chunkRowCount)));
} }
/* stripe_row_count => true */ /* stripe_row_count => true */

230
src/backend/columnar/cstore_writer.c
View File

@ -4,7 +4,7 @@
* *
* This file contains function definitions for writing cstore files. This * This file contains function definitions for writing cstore files. This
* includes the logic for writing file level metadata, writing row stripes, * includes the logic for writing file level metadata, writing row stripes,
* and calculating block skip nodes. * and calculating chunk skip nodes.
* *
* Copyright (c) 2016, Citus Data, Inc. * Copyright (c) 2016, Citus Data, Inc.
* *
@ -33,19 +33,19 @@
#include "columnar/cstore_version_compat.h" #include "columnar/cstore_version_compat.h"
static StripeBuffers * CreateEmptyStripeBuffers(uint32 stripeMaxRowCount, static StripeBuffers * CreateEmptyStripeBuffers(uint32 stripeMaxRowCount,
uint32 blockRowCount, uint32 chunkRowCount,
uint32 columnCount); uint32 columnCount);
static StripeSkipList * CreateEmptyStripeSkipList(uint32 stripeMaxRowCount, static StripeSkipList * CreateEmptyStripeSkipList(uint32 stripeMaxRowCount,
uint32 blockRowCount, uint32 chunkRowCount,
uint32 columnCount); uint32 columnCount);
static void FlushStripe(TableWriteState *writeState); static void FlushStripe(TableWriteState *writeState);
static StringInfo SerializeBoolArray(bool *boolArray, uint32 boolArrayLength); static StringInfo SerializeBoolArray(bool *boolArray, uint32 boolArrayLength);
static void SerializeSingleDatum(StringInfo datumBuffer, Datum datum, static void SerializeSingleDatum(StringInfo datumBuffer, Datum datum,
bool datumTypeByValue, int datumTypeLength, bool datumTypeByValue, int datumTypeLength,
char datumTypeAlign); char datumTypeAlign);
static void SerializeBlockData(TableWriteState *writeState, uint32 blockIndex, static void SerializeChunkData(TableWriteState *writeState, uint32 chunkIndex,
uint32 rowCount); uint32 rowCount);
static void UpdateBlockSkipNodeMinMax(ColumnBlockSkipNode *blockSkipNode, static void UpdateChunkSkipNodeMinMax(ColumnChunkSkipNode *chunkSkipNode,
Datum columnValue, bool columnTypeByValue, Datum columnValue, bool columnTypeByValue,
int columnTypeLength, Oid columnCollation, int columnTypeLength, Oid columnCollation,
FmgrInfo *comparisonFunction); FmgrInfo *comparisonFunction);
@ -62,7 +62,7 @@ static StringInfo CopyStringInfo(StringInfo sourceString);
TableWriteState * TableWriteState *
CStoreBeginWrite(RelFileNode relfilenode, CStoreBeginWrite(RelFileNode relfilenode,
CompressionType compressionType, CompressionType compressionType,
uint64 stripeMaxRowCount, uint32 blockRowCount, uint64 stripeMaxRowCount, uint32 chunkRowCount,
TupleDesc tupleDescriptor) TupleDesc tupleDescriptor)
{ {
/* get comparison function pointers for each of the columns */ /* get comparison function pointers for each of the columns */
@ -97,20 +97,20 @@ CStoreBeginWrite(RelFileNode relfilenode,
bool *columnMaskArray = palloc(columnCount * sizeof(bool)); bool *columnMaskArray = palloc(columnCount * sizeof(bool));
memset(columnMaskArray, true, columnCount); memset(columnMaskArray, true, columnCount);
BlockData *blockData = CreateEmptyBlockData(columnCount, columnMaskArray, ChunkData *chunkData = CreateEmptyChunkData(columnCount, columnMaskArray,
blockRowCount); chunkRowCount);
TableWriteState *writeState = palloc0(sizeof(TableWriteState)); TableWriteState *writeState = palloc0(sizeof(TableWriteState));
writeState->relfilenode = relfilenode; writeState->relfilenode = relfilenode;
writeState->compressionType = compressionType; writeState->compressionType = compressionType;
writeState->stripeMaxRowCount = stripeMaxRowCount; writeState->stripeMaxRowCount = stripeMaxRowCount;
writeState->blockRowCount = blockRowCount; writeState->chunkRowCount = chunkRowCount;
writeState->tupleDescriptor = CreateTupleDescCopy(tupleDescriptor); writeState->tupleDescriptor = CreateTupleDescCopy(tupleDescriptor);
writeState->comparisonFunctionArray = comparisonFunctionArray; writeState->comparisonFunctionArray = comparisonFunctionArray;
writeState->stripeBuffers = NULL; writeState->stripeBuffers = NULL;
writeState->stripeSkipList = NULL; writeState->stripeSkipList = NULL;
writeState->stripeWriteContext = stripeWriteContext; writeState->stripeWriteContext = stripeWriteContext;
writeState->blockData = blockData; writeState->chunkData = chunkData;
writeState->compressionBuffer = NULL; writeState->compressionBuffer = NULL;
writeState->perTupleContext = AllocSetContextCreate(CurrentMemoryContext, writeState->perTupleContext = AllocSetContextCreate(CurrentMemoryContext,
"CStore per tuple context", "CStore per tuple context",
@ -124,8 +124,8 @@ CStoreBeginWrite(RelFileNode relfilenode,
* CStoreWriteRow adds a row to the cstore file. If the stripe is not initialized, * CStoreWriteRow adds a row to the cstore file. If the stripe is not initialized,
* we create structures to hold stripe data and skip list. Then, we serialize and * we create structures to hold stripe data and skip list. Then, we serialize and
* append data to serialized value buffer for each of the columns and update * append data to serialized value buffer for each of the columns and update
* corresponding skip nodes. Then, whole block data is compressed at every * corresponding skip nodes. Then, whole chunk data is compressed at every
* rowBlockCount insertion. Then, if row count exceeds stripeMaxRowCount, we flush * rowChunkCount insertion. Then, if row count exceeds stripeMaxRowCount, we flush
* the stripe, and add its metadata to the table footer. * the stripe, and add its metadata to the table footer.
*/ */
void void
@ -135,16 +135,16 @@ CStoreWriteRow(TableWriteState *writeState, Datum *columnValues, bool *columnNul
StripeBuffers *stripeBuffers = writeState->stripeBuffers; StripeBuffers *stripeBuffers = writeState->stripeBuffers;
StripeSkipList *stripeSkipList = writeState->stripeSkipList; StripeSkipList *stripeSkipList = writeState->stripeSkipList;
uint32 columnCount = writeState->tupleDescriptor->natts; uint32 columnCount = writeState->tupleDescriptor->natts;
const uint32 blockRowCount = writeState->blockRowCount; const uint32 chunkRowCount = writeState->chunkRowCount;
BlockData *blockData = writeState->blockData; ChunkData *chunkData = writeState->chunkData;
MemoryContext oldContext = MemoryContextSwitchTo(writeState->stripeWriteContext); MemoryContext oldContext = MemoryContextSwitchTo(writeState->stripeWriteContext);
if (stripeBuffers == NULL) if (stripeBuffers == NULL)
{ {
stripeBuffers = CreateEmptyStripeBuffers(writeState->stripeMaxRowCount, stripeBuffers = CreateEmptyStripeBuffers(writeState->stripeMaxRowCount,
blockRowCount, columnCount); chunkRowCount, columnCount);
stripeSkipList = CreateEmptyStripeSkipList(writeState->stripeMaxRowCount, stripeSkipList = CreateEmptyStripeSkipList(writeState->stripeMaxRowCount,
blockRowCount, columnCount); chunkRowCount, columnCount);
writeState->stripeBuffers = stripeBuffers; writeState->stripeBuffers = stripeBuffers;
writeState->stripeSkipList = stripeSkipList; writeState->stripeSkipList = stripeSkipList;
writeState->compressionBuffer = makeStringInfo(); writeState->compressionBuffer = makeStringInfo();
@ -155,22 +155,22 @@ CStoreWriteRow(TableWriteState *writeState, Datum *columnValues, bool *columnNul
*/ */
for (columnIndex = 0; columnIndex < columnCount; columnIndex++) for (columnIndex = 0; columnIndex < columnCount; columnIndex++)
{ {
blockData->valueBufferArray[columnIndex] = makeStringInfo(); chunkData->valueBufferArray[columnIndex] = makeStringInfo();
} }
} }
uint32 blockIndex = stripeBuffers->rowCount / blockRowCount; uint32 chunkIndex = stripeBuffers->rowCount / chunkRowCount;
uint32 blockRowIndex = stripeBuffers->rowCount % blockRowCount; uint32 chunkRowIndex = stripeBuffers->rowCount % chunkRowCount;
for (columnIndex = 0; columnIndex < columnCount; columnIndex++) for (columnIndex = 0; columnIndex < columnCount; columnIndex++)
{ {
ColumnBlockSkipNode **blockSkipNodeArray = stripeSkipList->blockSkipNodeArray; ColumnChunkSkipNode **chunkSkipNodeArray = stripeSkipList->chunkSkipNodeArray;
ColumnBlockSkipNode *blockSkipNode = ColumnChunkSkipNode *chunkSkipNode =
&blockSkipNodeArray[columnIndex][blockIndex]; &chunkSkipNodeArray[columnIndex][chunkIndex];
if (columnNulls[columnIndex]) if (columnNulls[columnIndex])
{ {
blockData->existsArray[columnIndex][blockRowIndex] = false; chunkData->existsArray[columnIndex][chunkRowIndex] = false;
} }
else else
{ {
@ -183,26 +183,26 @@ CStoreWriteRow(TableWriteState *writeState, Datum *columnValues, bool *columnNul
Oid columnCollation = attributeForm->attcollation; Oid columnCollation = attributeForm->attcollation;
char columnTypeAlign = attributeForm->attalign; char columnTypeAlign = attributeForm->attalign;
blockData->existsArray[columnIndex][blockRowIndex] = true; chunkData->existsArray[columnIndex][chunkRowIndex] = true;
SerializeSingleDatum(blockData->valueBufferArray[columnIndex], SerializeSingleDatum(chunkData->valueBufferArray[columnIndex],
columnValues[columnIndex], columnTypeByValue, columnValues[columnIndex], columnTypeByValue,
columnTypeLength, columnTypeAlign); columnTypeLength, columnTypeAlign);
UpdateBlockSkipNodeMinMax(blockSkipNode, columnValues[columnIndex], UpdateChunkSkipNodeMinMax(chunkSkipNode, columnValues[columnIndex],
columnTypeByValue, columnTypeLength, columnTypeByValue, columnTypeLength,
columnCollation, comparisonFunction); columnCollation, comparisonFunction);
} }
blockSkipNode->rowCount++; chunkSkipNode->rowCount++;
} }
stripeSkipList->blockCount = blockIndex + 1; stripeSkipList->chunkCount = chunkIndex + 1;
/* last row of the block is inserted serialize the block */ /* last row of the chunk is inserted serialize the chunk */
if (blockRowIndex == blockRowCount - 1) if (chunkRowIndex == chunkRowCount - 1)
{ {
SerializeBlockData(writeState, blockIndex, blockRowCount); SerializeChunkData(writeState, chunkIndex, chunkRowCount);
} }
stripeBuffers->rowCount++; stripeBuffers->rowCount++;
@ -228,7 +228,7 @@ CStoreEndWrite(TableWriteState *writeState)
MemoryContextDelete(writeState->stripeWriteContext); MemoryContextDelete(writeState->stripeWriteContext);
pfree(writeState->comparisonFunctionArray); pfree(writeState->comparisonFunctionArray);
FreeBlockData(writeState->blockData); FreeChunkData(writeState->chunkData);
pfree(writeState); pfree(writeState);
} }
@ -258,29 +258,29 @@ CStoreFlushPendingWrites(TableWriteState *writeState)
* column count. * column count.
*/ */
static StripeBuffers * static StripeBuffers *
CreateEmptyStripeBuffers(uint32 stripeMaxRowCount, uint32 blockRowCount, CreateEmptyStripeBuffers(uint32 stripeMaxRowCount, uint32 chunkRowCount,
uint32 columnCount) uint32 columnCount)
{ {
uint32 columnIndex = 0; uint32 columnIndex = 0;
uint32 maxBlockCount = (stripeMaxRowCount / blockRowCount) + 1; uint32 maxChunkCount = (stripeMaxRowCount / chunkRowCount) + 1;
ColumnBuffers **columnBuffersArray = palloc0(columnCount * sizeof(ColumnBuffers *)); ColumnBuffers **columnBuffersArray = palloc0(columnCount * sizeof(ColumnBuffers *));
for (columnIndex = 0; columnIndex < columnCount; columnIndex++) for (columnIndex = 0; columnIndex < columnCount; columnIndex++)
{ {
uint32 blockIndex = 0; uint32 chunkIndex = 0;
ColumnBlockBuffers **blockBuffersArray = ColumnChunkBuffers **chunkBuffersArray =
palloc0(maxBlockCount * sizeof(ColumnBlockBuffers *)); palloc0(maxChunkCount * sizeof(ColumnChunkBuffers *));
for (blockIndex = 0; blockIndex < maxBlockCount; blockIndex++) for (chunkIndex = 0; chunkIndex < maxChunkCount; chunkIndex++)
{ {
blockBuffersArray[blockIndex] = palloc0(sizeof(ColumnBlockBuffers)); chunkBuffersArray[chunkIndex] = palloc0(sizeof(ColumnChunkBuffers));
blockBuffersArray[blockIndex]->existsBuffer = NULL; chunkBuffersArray[chunkIndex]->existsBuffer = NULL;
blockBuffersArray[blockIndex]->valueBuffer = NULL; chunkBuffersArray[chunkIndex]->valueBuffer = NULL;
blockBuffersArray[blockIndex]->valueCompressionType = COMPRESSION_NONE; chunkBuffersArray[chunkIndex]->valueCompressionType = COMPRESSION_NONE;
} }
columnBuffersArray[columnIndex] = palloc0(sizeof(ColumnBuffers)); columnBuffersArray[columnIndex] = palloc0(sizeof(ColumnBuffers));
columnBuffersArray[columnIndex]->blockBuffersArray = blockBuffersArray; columnBuffersArray[columnIndex]->chunkBuffersArray = chunkBuffersArray;
} }
StripeBuffers *stripeBuffers = palloc0(sizeof(StripeBuffers)); StripeBuffers *stripeBuffers = palloc0(sizeof(StripeBuffers));
@ -294,28 +294,28 @@ CreateEmptyStripeBuffers(uint32 stripeMaxRowCount, uint32 blockRowCount,
/* /*
* CreateEmptyStripeSkipList allocates an empty StripeSkipList structure with * CreateEmptyStripeSkipList allocates an empty StripeSkipList structure with
* the given column count. This structure has enough blocks to hold statistics * the given column count. This structure has enough chunks to hold statistics
* for stripeMaxRowCount rows. * for stripeMaxRowCount rows.
*/ */
static StripeSkipList * static StripeSkipList *
CreateEmptyStripeSkipList(uint32 stripeMaxRowCount, uint32 blockRowCount, CreateEmptyStripeSkipList(uint32 stripeMaxRowCount, uint32 chunkRowCount,
uint32 columnCount) uint32 columnCount)
{ {
uint32 columnIndex = 0; uint32 columnIndex = 0;
uint32 maxBlockCount = (stripeMaxRowCount / blockRowCount) + 1; uint32 maxChunkCount = (stripeMaxRowCount / chunkRowCount) + 1;
ColumnBlockSkipNode **blockSkipNodeArray = ColumnChunkSkipNode **chunkSkipNodeArray =
palloc0(columnCount * sizeof(ColumnBlockSkipNode *)); palloc0(columnCount * sizeof(ColumnChunkSkipNode *));
for (columnIndex = 0; columnIndex < columnCount; columnIndex++) for (columnIndex = 0; columnIndex < columnCount; columnIndex++)
{ {
blockSkipNodeArray[columnIndex] = chunkSkipNodeArray[columnIndex] =
palloc0(maxBlockCount * sizeof(ColumnBlockSkipNode)); palloc0(maxChunkCount * sizeof(ColumnChunkSkipNode));
} }
StripeSkipList *stripeSkipList = palloc0(sizeof(StripeSkipList)); StripeSkipList *stripeSkipList = palloc0(sizeof(StripeSkipList));
stripeSkipList->columnCount = columnCount; stripeSkipList->columnCount = columnCount;
stripeSkipList->blockCount = 0; stripeSkipList->chunkCount = 0;
stripeSkipList->blockSkipNodeArray = blockSkipNodeArray; stripeSkipList->chunkSkipNodeArray = chunkSkipNodeArray;
return stripeSkipList; return stripeSkipList;
} }
@ -332,9 +332,9 @@ WriteToSmgr(Relation rel, uint64 logicalOffset, char *data, uint32 dataLength)
SmgrAddr addr = logical_to_smgr(logicalOffset); SmgrAddr addr = logical_to_smgr(logicalOffset);
RelationOpenSmgr(rel); RelationOpenSmgr(rel);
BlockNumber nblocks = smgrnblocks(rel->rd_smgr, MAIN_FORKNUM); BlockNumber nblocks PG_USED_FOR_ASSERTS_ONLY =
smgrnblocks(rel->rd_smgr, MAIN_FORKNUM);
Assert(addr.blockno < nblocks); Assert(addr.blockno < nblocks);
(void) nblocks; /* keep compiler quiet */
RelationCloseSmgr(rel); RelationCloseSmgr(rel);
buffer = ReadBuffer(rel, addr.blockno); buffer = ReadBuffer(rel, addr.blockno);
@ -397,7 +397,7 @@ WriteToSmgr(Relation rel, uint64 logicalOffset, char *data, uint32 dataLength)
/* /*
* FlushStripe flushes current stripe data into the file. The function first ensures * FlushStripe flushes current stripe data into the file. The function first ensures
* the last data block for each column is properly serialized and compressed. Then, * the last data chunk for each column is properly serialized and compressed. Then,
* the function creates the skip list and footer buffers. Finally, the function * the function creates the skip list and footer buffers. Finally, the function
* flushes the skip list, data, and footer buffers to the file. * flushes the skip list, data, and footer buffers to the file.
*/ */
@ -406,16 +406,16 @@ FlushStripe(TableWriteState *writeState)
{ {
StripeMetadata stripeMetadata = { 0 }; StripeMetadata stripeMetadata = { 0 };
uint32 columnIndex = 0; uint32 columnIndex = 0;
uint32 blockIndex = 0; uint32 chunkIndex = 0;
StripeBuffers *stripeBuffers = writeState->stripeBuffers; StripeBuffers *stripeBuffers = writeState->stripeBuffers;
StripeSkipList *stripeSkipList = writeState->stripeSkipList; StripeSkipList *stripeSkipList = writeState->stripeSkipList;
ColumnBlockSkipNode **columnSkipNodeArray = stripeSkipList->blockSkipNodeArray; ColumnChunkSkipNode **columnSkipNodeArray = stripeSkipList->chunkSkipNodeArray;
TupleDesc tupleDescriptor = writeState->tupleDescriptor; TupleDesc tupleDescriptor = writeState->tupleDescriptor;
uint32 columnCount = tupleDescriptor->natts; uint32 columnCount = tupleDescriptor->natts;
uint32 blockCount = stripeSkipList->blockCount; uint32 chunkCount = stripeSkipList->chunkCount;
uint32 blockRowCount = writeState->blockRowCount; uint32 chunkRowCount = writeState->chunkRowCount;
uint32 lastBlockIndex = stripeBuffers->rowCount / blockRowCount; uint32 lastChunkIndex = stripeBuffers->rowCount / chunkRowCount;
uint32 lastBlockRowCount = stripeBuffers->rowCount % blockRowCount; uint32 lastChunkRowCount = stripeBuffers->rowCount % chunkRowCount;
uint64 stripeSize = 0; uint64 stripeSize = 0;
uint64 stripeRowCount = 0; uint64 stripeRowCount = 0;
@ -426,64 +426,64 @@ FlushStripe(TableWriteState *writeState)
Relation relation = relation_open(relationId, NoLock); Relation relation = relation_open(relationId, NoLock);
/* /*
* check if the last block needs serialization , the last block was not serialized * check if the last chunk needs serialization , the last chunk was not serialized
* if it was not full yet, e.g. (rowCount > 0) * if it was not full yet, e.g. (rowCount > 0)
*/ */
if (lastBlockRowCount > 0) if (lastChunkRowCount > 0)
{ {
SerializeBlockData(writeState, lastBlockIndex, lastBlockRowCount); SerializeChunkData(writeState, lastChunkIndex, lastChunkRowCount);
} }
/* update buffer sizes in stripe skip list */ /* update buffer sizes in stripe skip list */
for (columnIndex = 0; columnIndex < columnCount; columnIndex++) for (columnIndex = 0; columnIndex < columnCount; columnIndex++)
{ {
ColumnBlockSkipNode *blockSkipNodeArray = columnSkipNodeArray[columnIndex]; ColumnChunkSkipNode *chunkSkipNodeArray = columnSkipNodeArray[columnIndex];
ColumnBuffers *columnBuffers = stripeBuffers->columnBuffersArray[columnIndex]; ColumnBuffers *columnBuffers = stripeBuffers->columnBuffersArray[columnIndex];
for (blockIndex = 0; blockIndex < blockCount; blockIndex++) for (chunkIndex = 0; chunkIndex < chunkCount; chunkIndex++)
{ {
ColumnBlockBuffers *blockBuffers = ColumnChunkBuffers *chunkBuffers =
columnBuffers->blockBuffersArray[blockIndex]; columnBuffers->chunkBuffersArray[chunkIndex];
uint64 existsBufferSize = blockBuffers->existsBuffer->len; uint64 existsBufferSize = chunkBuffers->existsBuffer->len;
ColumnBlockSkipNode *blockSkipNode = &blockSkipNodeArray[blockIndex]; ColumnChunkSkipNode *chunkSkipNode = &chunkSkipNodeArray[chunkIndex];
blockSkipNode->existsBlockOffset = stripeSize; chunkSkipNode->existsChunkOffset = stripeSize;
blockSkipNode->existsLength = existsBufferSize; chunkSkipNode->existsLength = existsBufferSize;
stripeSize += existsBufferSize; stripeSize += existsBufferSize;
} }
for (blockIndex = 0; blockIndex < blockCount; blockIndex++) for (chunkIndex = 0; chunkIndex < chunkCount; chunkIndex++)
{ {
ColumnBlockBuffers *blockBuffers = ColumnChunkBuffers *chunkBuffers =
columnBuffers->blockBuffersArray[blockIndex]; columnBuffers->chunkBuffersArray[chunkIndex];
uint64 valueBufferSize = blockBuffers->valueBuffer->len; uint64 valueBufferSize = chunkBuffers->valueBuffer->len;
CompressionType valueCompressionType = blockBuffers->valueCompressionType; CompressionType valueCompressionType = chunkBuffers->valueCompressionType;
ColumnBlockSkipNode *blockSkipNode = &blockSkipNodeArray[blockIndex]; ColumnChunkSkipNode *chunkSkipNode = &chunkSkipNodeArray[chunkIndex];
blockSkipNode->valueBlockOffset = stripeSize; chunkSkipNode->valueChunkOffset = stripeSize;
blockSkipNode->valueLength = valueBufferSize; chunkSkipNode->valueLength = valueBufferSize;
blockSkipNode->valueCompressionType = valueCompressionType; chunkSkipNode->valueCompressionType = valueCompressionType;
stripeSize += valueBufferSize; stripeSize += valueBufferSize;
} }
} }
for (blockIndex = 0; blockIndex < blockCount; blockIndex++) for (chunkIndex = 0; chunkIndex < chunkCount; chunkIndex++)
{ {
stripeRowCount += stripeRowCount +=
stripeSkipList->blockSkipNodeArray[0][blockIndex].rowCount; stripeSkipList->chunkSkipNodeArray[0][chunkIndex].rowCount;
} }
stripeMetadata = ReserveStripe(relation, stripeSize, stripeMetadata = ReserveStripe(relation, stripeSize,
stripeRowCount, columnCount, blockCount, stripeRowCount, columnCount, chunkCount,
blockRowCount); chunkRowCount);
uint64 currentFileOffset = stripeMetadata.fileOffset; uint64 currentFileOffset = stripeMetadata.fileOffset;
/* /*
* Each stripe has only one section: * Each stripe has only one section:
* Data section, in which we store data for each column continuously. * Data section, in which we store data for each column continuously.
* We store data for each for each column in blocks. For each block, we * We store data for each for each column in chunks. For each chunk, we
* store two buffers: "exists" buffer, and "value" buffer. "exists" buffer * store two buffers: "exists" buffer, and "value" buffer. "exists" buffer
* tells which values are not NULL. "value" buffer contains values for * tells which values are not NULL. "value" buffer contains values for
* present values. For each column, we first store all "exists" buffers, * present values. For each column, we first store all "exists" buffers,
@ -495,22 +495,22 @@ FlushStripe(TableWriteState *writeState)
{ {
ColumnBuffers *columnBuffers = stripeBuffers->columnBuffersArray[columnIndex]; ColumnBuffers *columnBuffers = stripeBuffers->columnBuffersArray[columnIndex];
for (blockIndex = 0; blockIndex < stripeSkipList->blockCount; blockIndex++) for (chunkIndex = 0; chunkIndex < stripeSkipList->chunkCount; chunkIndex++)
{ {
ColumnBlockBuffers *blockBuffers = ColumnChunkBuffers *chunkBuffers =
columnBuffers->blockBuffersArray[blockIndex]; columnBuffers->chunkBuffersArray[chunkIndex];
StringInfo existsBuffer = blockBuffers->existsBuffer; StringInfo existsBuffer = chunkBuffers->existsBuffer;
WriteToSmgr(relation, currentFileOffset, WriteToSmgr(relation, currentFileOffset,
existsBuffer->data, existsBuffer->len); existsBuffer->data, existsBuffer->len);
currentFileOffset += existsBuffer->len; currentFileOffset += existsBuffer->len;
} }
for (blockIndex = 0; blockIndex < stripeSkipList->blockCount; blockIndex++) for (chunkIndex = 0; chunkIndex < stripeSkipList->chunkCount; chunkIndex++)
{ {
ColumnBlockBuffers *blockBuffers = ColumnChunkBuffers *chunkBuffers =
columnBuffers->blockBuffersArray[blockIndex]; columnBuffers->chunkBuffersArray[chunkIndex];
StringInfo valueBuffer = blockBuffers->valueBuffer; StringInfo valueBuffer = chunkBuffers->valueBuffer;
WriteToSmgr(relation, currentFileOffset, WriteToSmgr(relation, currentFileOffset,
valueBuffer->data, valueBuffer->len); valueBuffer->data, valueBuffer->len);
@ -596,15 +596,15 @@ SerializeSingleDatum(StringInfo datumBuffer, Datum datum, bool datumTypeByValue,
/* /*
* SerializeBlockData serializes and compresses block data at given block index with given * SerializeChunkData serializes and compresses chunk data at given chunk index with given
* compression type for every column. * compression type for every column.
*/ */
static void static void
SerializeBlockData(TableWriteState *writeState, uint32 blockIndex, uint32 rowCount) SerializeChunkData(TableWriteState *writeState, uint32 chunkIndex, uint32 rowCount)
{ {
uint32 columnIndex = 0; uint32 columnIndex = 0;
StripeBuffers *stripeBuffers = writeState->stripeBuffers; StripeBuffers *stripeBuffers = writeState->stripeBuffers;
BlockData *blockData = writeState->blockData; ChunkData *chunkData = writeState->chunkData;
CompressionType requestedCompressionType = writeState->compressionType; CompressionType requestedCompressionType = writeState->compressionType;
const uint32 columnCount = stripeBuffers->columnCount; const uint32 columnCount = stripeBuffers->columnCount;
StringInfo compressionBuffer = writeState->compressionBuffer; StringInfo compressionBuffer = writeState->compressionBuffer;
@ -613,10 +613,10 @@ SerializeBlockData(TableWriteState *writeState, uint32 blockIndex, uint32 rowCou
for (columnIndex = 0; columnIndex < columnCount; columnIndex++) for (columnIndex = 0; columnIndex < columnCount; columnIndex++)
{ {
ColumnBuffers *columnBuffers = stripeBuffers->columnBuffersArray[columnIndex]; ColumnBuffers *columnBuffers = stripeBuffers->columnBuffersArray[columnIndex];
ColumnBlockBuffers *blockBuffers = columnBuffers->blockBuffersArray[blockIndex]; ColumnChunkBuffers *chunkBuffers = columnBuffers->chunkBuffersArray[chunkIndex];
blockBuffers->existsBuffer = chunkBuffers->existsBuffer =
SerializeBoolArray(blockData->existsArray[columnIndex], rowCount); SerializeBoolArray(chunkData->existsArray[columnIndex], rowCount);
} }
/* /*
@ -626,10 +626,10 @@ SerializeBlockData(TableWriteState *writeState, uint32 blockIndex, uint32 rowCou
for (columnIndex = 0; columnIndex < columnCount; columnIndex++) for (columnIndex = 0; columnIndex < columnCount; columnIndex++)
{ {
ColumnBuffers *columnBuffers = stripeBuffers->columnBuffersArray[columnIndex]; ColumnBuffers *columnBuffers = stripeBuffers->columnBuffersArray[columnIndex];
ColumnBlockBuffers *blockBuffers = columnBuffers->blockBuffersArray[blockIndex]; ColumnChunkBuffers *chunkBuffers = columnBuffers->chunkBuffersArray[chunkIndex];
CompressionType actualCompressionType = COMPRESSION_NONE; CompressionType actualCompressionType = COMPRESSION_NONE;
StringInfo serializedValueBuffer = blockData->valueBufferArray[columnIndex]; StringInfo serializedValueBuffer = chunkData->valueBufferArray[columnIndex];
/* the only other supported compression type is pg_lz for now */ /* the only other supported compression type is pg_lz for now */
Assert(requestedCompressionType == COMPRESSION_NONE || Assert(requestedCompressionType == COMPRESSION_NONE ||
@ -648,29 +648,29 @@ SerializeBlockData(TableWriteState *writeState, uint32 blockIndex, uint32 rowCou
} }
/* store (compressed) value buffer */ /* store (compressed) value buffer */
blockBuffers->valueCompressionType = actualCompressionType; chunkBuffers->valueCompressionType = actualCompressionType;
blockBuffers->valueBuffer = CopyStringInfo(serializedValueBuffer); chunkBuffers->valueBuffer = CopyStringInfo(serializedValueBuffer);
/* valueBuffer needs to be reset for next block's data */ /* valueBuffer needs to be reset for next chunk's data */
resetStringInfo(blockData->valueBufferArray[columnIndex]); resetStringInfo(chunkData->valueBufferArray[columnIndex]);
} }
} }
/* /*
* UpdateBlockSkipNodeMinMax takes the given column value, and checks if this * UpdateChunkSkipNodeMinMax takes the given column value, and checks if this
* value falls outside the range of minimum/maximum values of the given column * value falls outside the range of minimum/maximum values of the given column
* block skip node. If it does, the function updates the column block skip node * chunk skip node. If it does, the function updates the column chunk skip node
* accordingly. * accordingly.
*/ */
static void static void
UpdateBlockSkipNodeMinMax(ColumnBlockSkipNode *blockSkipNode, Datum columnValue, UpdateChunkSkipNodeMinMax(ColumnChunkSkipNode *chunkSkipNode, Datum columnValue,
bool columnTypeByValue, int columnTypeLength, bool columnTypeByValue, int columnTypeLength,
Oid columnCollation, FmgrInfo *comparisonFunction) Oid columnCollation, FmgrInfo *comparisonFunction)
{ {
bool hasMinMax = blockSkipNode->hasMinMax; bool hasMinMax = chunkSkipNode->hasMinMax;
Datum previousMinimum = blockSkipNode->minimumValue; Datum previousMinimum = chunkSkipNode->minimumValue;
Datum previousMaximum = blockSkipNode->maximumValue; Datum previousMaximum = chunkSkipNode->maximumValue;
Datum currentMinimum = 0; Datum currentMinimum = 0;
Datum currentMaximum = 0; Datum currentMaximum = 0;
@ -715,9 +715,9 @@ UpdateBlockSkipNodeMinMax(ColumnBlockSkipNode *blockSkipNode, Datum columnValue,
} }
} }
blockSkipNode->hasMinMax = true; chunkSkipNode->hasMinMax = true;
blockSkipNode->minimumValue = currentMinimum; chunkSkipNode->minimumValue = currentMinimum;
blockSkipNode->maximumValue = currentMaximum; chunkSkipNode->maximumValue = currentMaximum;
} }

12
src/backend/columnar/sql/columnar--9.5-1--10.0-1.sql
View File

@ -7,7 +7,7 @@ CREATE SEQUENCE storageid_seq MINVALUE 10000000000 NO CYCLE;
CREATE TABLE options ( CREATE TABLE options (
regclass regclass NOT NULL PRIMARY KEY, regclass regclass NOT NULL PRIMARY KEY,
block_row_count int NOT NULL, chunk_row_count int NOT NULL,
stripe_row_count int NOT NULL, stripe_row_count int NOT NULL,
compression name NOT NULL compression name NOT NULL
) WITH (user_catalog_table = true); ) WITH (user_catalog_table = true);
@ -20,8 +20,8 @@ CREATE TABLE cstore_stripes (
file_offset bigint NOT NULL, file_offset bigint NOT NULL,
data_length bigint NOT NULL, data_length bigint NOT NULL,
column_count int NOT NULL, column_count int NOT NULL,
block_count int NOT NULL, chunk_count int NOT NULL,
block_row_count int NOT NULL, chunk_row_count int NOT NULL,
row_count bigint NOT NULL, row_count bigint NOT NULL,
PRIMARY KEY (storageid, stripe) PRIMARY KEY (storageid, stripe)
) WITH (user_catalog_table = true); ) WITH (user_catalog_table = true);
@ -32,7 +32,7 @@ CREATE TABLE cstore_skipnodes (
storageid bigint NOT NULL, storageid bigint NOT NULL,
stripe bigint NOT NULL, stripe bigint NOT NULL,
attr int NOT NULL, attr int NOT NULL,
block int NOT NULL, chunk int NOT NULL,
row_count bigint NOT NULL, row_count bigint NOT NULL,
minimum_value bytea, minimum_value bytea,
maximum_value bytea, maximum_value bytea,
@ -41,11 +41,11 @@ CREATE TABLE cstore_skipnodes (
exists_stream_offset bigint NOT NULL, exists_stream_offset bigint NOT NULL,
exists_stream_length bigint NOT NULL, exists_stream_length bigint NOT NULL,
value_compression_type int NOT NULL, value_compression_type int NOT NULL,
PRIMARY KEY (storageid, stripe, attr, block), PRIMARY KEY (storageid, stripe, attr, chunk),
FOREIGN KEY (storageid, stripe) REFERENCES cstore_stripes(storageid, stripe) ON DELETE CASCADE FOREIGN KEY (storageid, stripe) REFERENCES cstore_stripes(storageid, stripe) ON DELETE CASCADE
) WITH (user_catalog_table = true); ) WITH (user_catalog_table = true);
COMMENT ON TABLE cstore_skipnodes IS 'CStore per block metadata'; COMMENT ON TABLE cstore_skipnodes IS 'CStore per chunk metadata';
DO $proc$ DO $proc$
BEGIN BEGIN

4
src/backend/columnar/sql/downgrades/columnar--10.0-1--9.5-1.sql
View File

@ -9,13 +9,13 @@ IF substring(current_Setting('server_version'), '\d+')::int >= 12 THEN
EXECUTE $$ EXECUTE $$
DROP FUNCTION pg_catalog.alter_columnar_table_reset( DROP FUNCTION pg_catalog.alter_columnar_table_reset(
table_name regclass, table_name regclass,
block_row_count bool, chunk_row_count bool,
stripe_row_count bool, stripe_row_count bool,
compression bool); compression bool);
DROP FUNCTION pg_catalog.alter_columnar_table_set( DROP FUNCTION pg_catalog.alter_columnar_table_set(
table_name regclass, table_name regclass,
block_row_count int, chunk_row_count int,
stripe_row_count int, stripe_row_count int,
compression name); compression name);

4
src/backend/columnar/sql/udfs/alter_columnar_table_reset/10.0-1.sql
View File

@ -1,6 +1,6 @@
CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_reset( CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_reset(
table_name regclass, table_name regclass,
block_row_count bool DEFAULT false, chunk_row_count bool DEFAULT false,
stripe_row_count bool DEFAULT false, stripe_row_count bool DEFAULT false,
compression bool DEFAULT false) compression bool DEFAULT false)
RETURNS void RETURNS void
@ -9,7 +9,7 @@ AS 'MODULE_PATHNAME', 'alter_columnar_table_reset';
COMMENT ON FUNCTION pg_catalog.alter_columnar_table_reset( COMMENT ON FUNCTION pg_catalog.alter_columnar_table_reset(
table_name regclass, table_name regclass,
block_row_count bool, chunk_row_count bool,
stripe_row_count bool, stripe_row_count bool,
compression bool) compression bool)
IS 'reset on or more options on a cstore table to the system defaults'; IS 'reset on or more options on a cstore table to the system defaults';

4
src/backend/columnar/sql/udfs/alter_columnar_table_reset/latest.sql
View File

@ -1,6 +1,6 @@
CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_reset( CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_reset(
table_name regclass, table_name regclass,
block_row_count bool DEFAULT false, chunk_row_count bool DEFAULT false,
stripe_row_count bool DEFAULT false, stripe_row_count bool DEFAULT false,
compression bool DEFAULT false) compression bool DEFAULT false)
RETURNS void RETURNS void
@ -9,7 +9,7 @@ AS 'MODULE_PATHNAME', 'alter_columnar_table_reset';
COMMENT ON FUNCTION pg_catalog.alter_columnar_table_reset( COMMENT ON FUNCTION pg_catalog.alter_columnar_table_reset(
table_name regclass, table_name regclass,
block_row_count bool, chunk_row_count bool,
stripe_row_count bool, stripe_row_count bool,
compression bool) compression bool)
IS 'reset on or more options on a cstore table to the system defaults'; IS 'reset on or more options on a cstore table to the system defaults';

4
src/backend/columnar/sql/udfs/alter_columnar_table_set/10.0-1.sql
View File

@ -1,6 +1,6 @@
CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_set( CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_set(
table_name regclass, table_name regclass,
block_row_count int DEFAULT NULL, chunk_row_count int DEFAULT NULL,
stripe_row_count int DEFAULT NULL, stripe_row_count int DEFAULT NULL,
compression name DEFAULT null) compression name DEFAULT null)
RETURNS void RETURNS void
@ -9,7 +9,7 @@ AS 'MODULE_PATHNAME', 'alter_columnar_table_set';
COMMENT ON FUNCTION pg_catalog.alter_columnar_table_set( COMMENT ON FUNCTION pg_catalog.alter_columnar_table_set(
table_name regclass, table_name regclass,
block_row_count int, chunk_row_count int,
stripe_row_count int, stripe_row_count int,
compression name) compression name)
IS 'set one or more options on a cstore table, when set to NULL no change is made'; IS 'set one or more options on a cstore table, when set to NULL no change is made';

4
src/backend/columnar/sql/udfs/alter_columnar_table_set/latest.sql
View File

@ -1,6 +1,6 @@
CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_set( CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_set(
table_name regclass, table_name regclass,
block_row_count int DEFAULT NULL, chunk_row_count int DEFAULT NULL,
stripe_row_count int DEFAULT NULL, stripe_row_count int DEFAULT NULL,
compression name DEFAULT null) compression name DEFAULT null)
RETURNS void RETURNS void
@ -9,7 +9,7 @@ AS 'MODULE_PATHNAME', 'alter_columnar_table_set';
COMMENT ON FUNCTION pg_catalog.alter_columnar_table_set( COMMENT ON FUNCTION pg_catalog.alter_columnar_table_set(
table_name regclass, table_name regclass,
block_row_count int, chunk_row_count int,
stripe_row_count int, stripe_row_count int,
compression name) compression name)
IS 'set one or more options on a cstore table, when set to NULL no change is made'; IS 'set one or more options on a cstore table, when set to NULL no change is made';

4
src/backend/columnar/sql/udfs/cstore_ensure_objects_exist/10.0-1.sql
View File

@ -29,12 +29,12 @@ IF NOT EXISTS (SELECT 1 FROM pg_am WHERE amname = 'columnar') THEN
ALTER EXTENSION citus ADD ACCESS METHOD columnar; ALTER EXTENSION citus ADD ACCESS METHOD columnar;
ALTER EXTENSION citus ADD FUNCTION pg_catalog.alter_columnar_table_set( ALTER EXTENSION citus ADD FUNCTION pg_catalog.alter_columnar_table_set(
table_name regclass, table_name regclass,
block_row_count int, chunk_row_count int,
stripe_row_count int, stripe_row_count int,
compression name); compression name);
ALTER EXTENSION citus ADD FUNCTION pg_catalog.alter_columnar_table_reset( ALTER EXTENSION citus ADD FUNCTION pg_catalog.alter_columnar_table_reset(
table_name regclass, table_name regclass,
block_row_count bool, chunk_row_count bool,
stripe_row_count bool, stripe_row_count bool,
compression bool); compression bool);

4
src/backend/columnar/sql/udfs/cstore_ensure_objects_exist/latest.sql
View File

@ -29,12 +29,12 @@ IF NOT EXISTS (SELECT 1 FROM pg_am WHERE amname = 'columnar') THEN
ALTER EXTENSION citus ADD ACCESS METHOD columnar; ALTER EXTENSION citus ADD ACCESS METHOD columnar;
ALTER EXTENSION citus ADD FUNCTION pg_catalog.alter_columnar_table_set( ALTER EXTENSION citus ADD FUNCTION pg_catalog.alter_columnar_table_set(
table_name regclass, table_name regclass,
block_row_count int, chunk_row_count int,
stripe_row_count int, stripe_row_count int,
compression name); compression name);
ALTER EXTENSION citus ADD FUNCTION pg_catalog.alter_columnar_table_reset( ALTER EXTENSION citus ADD FUNCTION pg_catalog.alter_columnar_table_reset(
table_name regclass, table_name regclass,
block_row_count bool, chunk_row_count bool,
stripe_row_count bool, stripe_row_count bool,
compression bool); compression bool);

2
src/backend/columnar/write_state_management.c
View File

@ -185,7 +185,7 @@ cstore_init_write_state(Relation relation, TupleDesc tupdesc,
stackEntry->writeState = CStoreBeginWrite(relation->rd_node, stackEntry->writeState = CStoreBeginWrite(relation->rd_node,
cstoreOptions.compressionType, cstoreOptions.compressionType,
cstoreOptions.stripeRowCount, cstoreOptions.stripeRowCount,
cstoreOptions.blockRowCount, cstoreOptions.chunkRowCount,
tupdesc); tupdesc);
stackEntry->subXid = currentSubXid; stackEntry->subXid = currentSubXid;
stackEntry->next = hashEntry->writeStateStack; stackEntry->next = hashEntry->writeStateStack;

80
src/include/columnar/cstore.h
View File

@ -27,13 +27,13 @@
/* Defines for valid option names */ /* Defines for valid option names */
#define OPTION_NAME_COMPRESSION_TYPE "compression" #define OPTION_NAME_COMPRESSION_TYPE "compression"
#define OPTION_NAME_STRIPE_ROW_COUNT "stripe_row_count" #define OPTION_NAME_STRIPE_ROW_COUNT "stripe_row_count"
#define OPTION_NAME_BLOCK_ROW_COUNT "block_row_count" #define OPTION_NAME_CHUNK_ROW_COUNT "chunk_row_count"
/* Limits for option parameters */ /* Limits for option parameters */
#define STRIPE_ROW_COUNT_MINIMUM 1000 #define STRIPE_ROW_COUNT_MINIMUM 1000
#define STRIPE_ROW_COUNT_MAXIMUM 10000000 #define STRIPE_ROW_COUNT_MAXIMUM 10000000
#define BLOCK_ROW_COUNT_MINIMUM 1000 #define CHUNK_ROW_COUNT_MINIMUM 1000
#define BLOCK_ROW_COUNT_MAXIMUM 100000 #define CHUNK_ROW_COUNT_MAXIMUM 100000
/* String representations of compression types */ /* String representations of compression types */
#define COMPRESSION_STRING_NONE "none" #define COMPRESSION_STRING_NONE "none"
@ -70,7 +70,7 @@ typedef enum
typedef struct ColumnarOptions typedef struct ColumnarOptions
{ {
uint64 stripeRowCount; uint64 stripeRowCount;
uint32 blockRowCount; uint32 chunkRowCount;
CompressionType compressionType; CompressionType compressionType;
} ColumnarOptions; } ColumnarOptions;
@ -84,8 +84,8 @@ typedef struct StripeMetadata
uint64 fileOffset; uint64 fileOffset;
uint64 dataLength; uint64 dataLength;
uint32 columnCount; uint32 columnCount;
uint32 blockCount; uint32 chunkCount;
uint32 blockRowCount; uint32 chunkRowCount;
uint64 rowCount; uint64 rowCount;
uint64 id; uint64 id;
} StripeMetadata; } StripeMetadata;
@ -98,10 +98,10 @@ typedef struct DataFileMetadata
} DataFileMetadata; } DataFileMetadata;
/* ColumnBlockSkipNode contains statistics for a ColumnBlockData. */ /* ColumnChunkSkipNode contains statistics for a ColumnChunkData. */
typedef struct ColumnBlockSkipNode typedef struct ColumnChunkSkipNode
{ {
/* statistics about values of a column block */ /* statistics about values of a column chunk */
bool hasMinMax; bool hasMinMax;
Datum minimumValue; Datum minimumValue;
Datum maximumValue; Datum maximumValue;
@ -109,39 +109,39 @@ typedef struct ColumnBlockSkipNode
/* /*
* Offsets and sizes of value and exists streams in the column data. * Offsets and sizes of value and exists streams in the column data.
* These enable us to skip reading suppressed row blocks, and start reading * These enable us to skip reading suppressed row chunks, and start reading
* a block without reading previous blocks. * a chunk without reading previous chunks.
*/ */
uint64 valueBlockOffset; uint64 valueChunkOffset;
uint64 valueLength; uint64 valueLength;
uint64 existsBlockOffset; uint64 existsChunkOffset;
uint64 existsLength; uint64 existsLength;
CompressionType valueCompressionType; CompressionType valueCompressionType;
} ColumnBlockSkipNode; } ColumnChunkSkipNode;
/* /*
* StripeSkipList can be used for skipping row blocks. It contains a column block * StripeSkipList can be used for skipping row chunks. It contains a column chunk
* skip node for each block of each column. blockSkipNodeArray[column][block] * skip node for each chunk of each column. chunkSkipNodeArray[column][chunk]
* is the entry for the specified column block. * is the entry for the specified column chunk.
*/ */
typedef struct StripeSkipList typedef struct StripeSkipList
{ {
ColumnBlockSkipNode **blockSkipNodeArray; ColumnChunkSkipNode **chunkSkipNodeArray;
uint32 columnCount; uint32 columnCount;
uint32 blockCount; uint32 chunkCount;
} StripeSkipList; } StripeSkipList;
/* /*
* BlockData represents a block of data for multiple columns. valueArray stores * ChunkData represents a chunk of data for multiple columns. valueArray stores
* the values of data, and existsArray stores whether a value is present. * the values of data, and existsArray stores whether a value is present.
* valueBuffer is used to store (uncompressed) serialized values * valueBuffer is used to store (uncompressed) serialized values
* referenced by Datum's in valueArray. It is only used for by-reference Datum's. * referenced by Datum's in valueArray. It is only used for by-reference Datum's.
* There is a one-to-one correspondence between valueArray and existsArray. * There is a one-to-one correspondence between valueArray and existsArray.
*/ */
typedef struct BlockData typedef struct ChunkData
{ {
uint32 rowCount; uint32 rowCount;
uint32 columnCount; uint32 columnCount;
@ -155,31 +155,31 @@ typedef struct BlockData
/* valueBuffer keeps actual data for type-by-reference datums from valueArray. */ /* valueBuffer keeps actual data for type-by-reference datums from valueArray. */
StringInfo *valueBufferArray; StringInfo *valueBufferArray;
} BlockData; } ChunkData;
/* /*
* ColumnBlockBuffers represents a block of serialized data in a column. * ColumnChunkBuffers represents a chunk of serialized data in a column.
* valueBuffer stores the serialized values of data, and existsBuffer stores * valueBuffer stores the serialized values of data, and existsBuffer stores
* serialized value of presence information. valueCompressionType contains * serialized value of presence information. valueCompressionType contains
* compression type if valueBuffer is compressed. Finally rowCount has * compression type if valueBuffer is compressed. Finally rowCount has
* the number of rows in this block. * the number of rows in this chunk.
*/ */
typedef struct ColumnBlockBuffers typedef struct ColumnChunkBuffers
{ {
StringInfo existsBuffer; StringInfo existsBuffer;
StringInfo valueBuffer; StringInfo valueBuffer;
CompressionType valueCompressionType; CompressionType valueCompressionType;
} ColumnBlockBuffers; } ColumnChunkBuffers;
/* /*
* ColumnBuffers represents data buffers for a column in a row stripe. Each * ColumnBuffers represents data buffers for a column in a row stripe. Each
* column is made of multiple column blocks. * column is made of multiple column chunks.
*/ */
typedef struct ColumnBuffers typedef struct ColumnBuffers
{ {
ColumnBlockBuffers **blockBuffersArray; ColumnChunkBuffers **chunkBuffersArray;
} ColumnBuffers; } ColumnBuffers;
@ -203,7 +203,7 @@ typedef struct TableReadState
/* /*
* List of Var pointers for columns in the query. We use this both for * List of Var pointers for columns in the query. We use this both for
* getting vector of projected columns, and also when we want to build * getting vector of projected columns, and also when we want to build
* base constraint to find selected row blocks. * base constraint to find selected row chunks.
*/ */
List *projectedColumnList; List *projectedColumnList;
@ -212,8 +212,8 @@ typedef struct TableReadState
StripeBuffers *stripeBuffers; StripeBuffers *stripeBuffers;
uint32 readStripeCount; uint32 readStripeCount;
uint64 stripeReadRowCount; uint64 stripeReadRowCount;
BlockData *blockData; ChunkData *chunkData;
int32 deserializedBlockIndex; int32 deserializedChunkIndex;
} TableReadState; } TableReadState;
@ -230,8 +230,8 @@ typedef struct TableWriteState
StripeBuffers *stripeBuffers; StripeBuffers *stripeBuffers;
StripeSkipList *stripeSkipList; StripeSkipList *stripeSkipList;
uint32 stripeMaxRowCount; uint32 stripeMaxRowCount;
uint32 blockRowCount; uint32 chunkRowCount;
BlockData *blockData; ChunkData *chunkData;
/* /*
* compressionBuffer buffer is used as temporary storage during * compressionBuffer buffer is used as temporary storage during
@ -244,7 +244,7 @@ typedef struct TableWriteState
extern int cstore_compression; extern int cstore_compression;
extern int cstore_stripe_row_count; extern int cstore_stripe_row_count;
extern int cstore_block_row_count; extern int cstore_chunk_row_count;
extern void cstore_init(void); extern void cstore_init(void);
@ -254,7 +254,7 @@ extern CompressionType ParseCompressionType(const char *compressionTypeString);
extern TableWriteState * CStoreBeginWrite(RelFileNode relfilenode, extern TableWriteState * CStoreBeginWrite(RelFileNode relfilenode,
CompressionType compressionType, CompressionType compressionType,
uint64 stripeMaxRowCount, uint64 stripeMaxRowCount,
uint32 blockRowCount, uint32 chunkRowCount,
TupleDesc tupleDescriptor); TupleDesc tupleDescriptor);
extern void CStoreWriteRow(TableWriteState *state, Datum *columnValues, extern void CStoreWriteRow(TableWriteState *state, Datum *columnValues,
bool *columnNulls); bool *columnNulls);
@ -275,9 +275,9 @@ extern void CStoreEndRead(TableReadState *state);
/* Function declarations for common functions */ /* Function declarations for common functions */
extern FmgrInfo * GetFunctionInfoOrNull(Oid typeId, Oid accessMethodId, extern FmgrInfo * GetFunctionInfoOrNull(Oid typeId, Oid accessMethodId,
int16 procedureId); int16 procedureId);
extern BlockData * CreateEmptyBlockData(uint32 columnCount, bool *columnMask, extern ChunkData * CreateEmptyChunkData(uint32 columnCount, bool *columnMask,
uint32 blockRowCount); uint32 chunkRowCount);
extern void FreeBlockData(BlockData *blockData); extern void FreeChunkData(ChunkData *chunkData);
extern uint64 CStoreTableRowCount(Relation relation); extern uint64 CStoreTableRowCount(Relation relation);
extern bool CompressBuffer(StringInfo inputBuffer, StringInfo outputBuffer, extern bool CompressBuffer(StringInfo inputBuffer, StringInfo outputBuffer,
CompressionType compressionType); CompressionType compressionType);
@ -300,13 +300,13 @@ extern List * StripesForRelfilenode(RelFileNode relfilenode);
extern uint64 GetHighestUsedAddress(RelFileNode relfilenode); extern uint64 GetHighestUsedAddress(RelFileNode relfilenode);
extern StripeMetadata ReserveStripe(Relation rel, uint64 size, extern StripeMetadata ReserveStripe(Relation rel, uint64 size,
uint64 rowCount, uint64 columnCount, uint64 rowCount, uint64 columnCount,
uint64 blockCount, uint64 blockRowCount); uint64 chunkCount, uint64 chunkRowCount);
extern void SaveStripeSkipList(RelFileNode relfilenode, uint64 stripe, extern void SaveStripeSkipList(RelFileNode relfilenode, uint64 stripe,
StripeSkipList *stripeSkipList, StripeSkipList *stripeSkipList,
TupleDesc tupleDescriptor); TupleDesc tupleDescriptor);
extern StripeSkipList * ReadStripeSkipList(RelFileNode relfilenode, uint64 stripe, extern StripeSkipList * ReadStripeSkipList(RelFileNode relfilenode, uint64 stripe,
TupleDesc tupleDescriptor, TupleDesc tupleDescriptor,
uint32 blockCount); uint32 chunkCount);
extern Datum columnar_relation_storageid(PG_FUNCTION_ARGS); extern Datum columnar_relation_storageid(PG_FUNCTION_ARGS);

2
src/test/regress/columnar_am_schedule
View File

@ -17,7 +17,7 @@ test: am_rollback
test: am_truncate test: am_truncate
test: am_vacuum test: am_vacuum
test: am_clean test: am_clean
test: am_block_filtering test: am_chunk_filtering
test: am_join test: am_join
test: am_trigger test: am_trigger
test: am_tableoptions test: am_tableoptions

0
src/test/regress/data/block_filtering.csv → src/test/regress/data/chunk_filtering.csv
View File

2
src/test/regress/expected/.gitignore vendored
View File

@ -21,7 +21,7 @@
/multi_behavioral_analytics_create_table.out /multi_behavioral_analytics_create_table.out
/multi_insert_select_behavioral_analytics_create_table.out /multi_insert_select_behavioral_analytics_create_table.out
/hyperscale_tutorial.out /hyperscale_tutorial.out
/am_block_filtering.out /am_chunk_filtering.out
/am_copyto.out /am_copyto.out
/am_data_types.out /am_data_types.out
/am_load.out /am_load.out

8
src/test/regress/expected/am_empty.out
View File

@ -17,14 +17,14 @@ SELECT alter_columnar_table_set('t_compressed', stripe_row_count => 100);
(1 row) (1 row)
SELECT alter_columnar_table_set('t_compressed', block_row_count => 100); SELECT alter_columnar_table_set('t_compressed', chunk_row_count => 100);
alter_columnar_table_set alter_columnar_table_set
--------------------------------------------------------------------- ---------------------------------------------------------------------
(1 row) (1 row)
SELECT * FROM cstore.options WHERE regclass = 't_compressed'::regclass; SELECT * FROM cstore.options WHERE regclass = 't_compressed'::regclass;
regclass | block_row_count | stripe_row_count | compression regclass | chunk_row_count | stripe_row_count | compression
--------------------------------------------------------------------- ---------------------------------------------------------------------
t_compressed | 100 | 100 | pglz t_compressed | 100 | 100 | pglz
(1 row) (1 row)
@ -71,14 +71,14 @@ INFO: statistics for "t_compressed":
storage id: -1 storage id: -1
total file size: 0, total data size: 0 total file size: 0, total data size: 0
total row count: 0, stripe count: 0, average rows per stripe: 0 total row count: 0, stripe count: 0, average rows per stripe: 0
block count: 0, containing data for dropped columns: 0, none compressed: 0, pglz compressed: 0 chunk count: 0, containing data for dropped columns: 0, none compressed: 0, pglz compressed: 0
vacuum verbose t_uncompressed; vacuum verbose t_uncompressed;
INFO: statistics for "t_uncompressed": INFO: statistics for "t_uncompressed":
storage id: -1 storage id: -1
total file size: 0, total data size: 0 total file size: 0, total data size: 0
total row count: 0, stripe count: 0, average rows per stripe: 0 total row count: 0, stripe count: 0, average rows per stripe: 0
block count: 0, containing data for dropped columns: 0, none compressed: 0, pglz compressed: 0 chunk count: 0, containing data for dropped columns: 0, none compressed: 0, pglz compressed: 0
-- vacuum full -- vacuum full
vacuum full t_compressed; vacuum full t_compressed;

6
src/test/regress/expected/am_matview.out
View File

@ -25,7 +25,7 @@ SELECT * FROM t_view a ORDER BY a;
-- show columnar options for materialized view -- show columnar options for materialized view
SELECT * FROM cstore.options SELECT * FROM cstore.options
WHERE regclass = 't_view'::regclass; WHERE regclass = 't_view'::regclass;
regclass | block_row_count | stripe_row_count | compression regclass | chunk_row_count | stripe_row_count | compression
--------------------------------------------------------------------- ---------------------------------------------------------------------
t_view | 10000 | 150000 | none t_view | 10000 | 150000 | none
(1 row) (1 row)
@ -39,7 +39,7 @@ SELECT alter_columnar_table_set('t_view', compression => 'pglz');
SELECT * FROM cstore.options SELECT * FROM cstore.options
WHERE regclass = 't_view'::regclass; WHERE regclass = 't_view'::regclass;
regclass | block_row_count | stripe_row_count | compression regclass | chunk_row_count | stripe_row_count | compression
--------------------------------------------------------------------- ---------------------------------------------------------------------
t_view | 10000 | 150000 | pglz t_view | 10000 | 150000 | pglz
(1 row) (1 row)
@ -48,7 +48,7 @@ REFRESH MATERIALIZED VIEW t_view;
-- verify options have not been changed -- verify options have not been changed
SELECT * FROM cstore.options SELECT * FROM cstore.options
WHERE regclass = 't_view'::regclass; WHERE regclass = 't_view'::regclass;
regclass | block_row_count | stripe_row_count | compression regclass | chunk_row_count | stripe_row_count | compression
--------------------------------------------------------------------- ---------------------------------------------------------------------
t_view | 10000 | 150000 | pglz t_view | 10000 | 150000 | pglz
(1 row) (1 row)

50
src/test/regress/expected/am_tableoptions.out
View File

@ -5,7 +5,7 @@ INSERT INTO table_options SELECT generate_series(1,100);
-- show table_options settings -- show table_options settings
SELECT * FROM cstore.options SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass; WHERE regclass = 'table_options'::regclass;
regclass | block_row_count | stripe_row_count | compression regclass | chunk_row_count | stripe_row_count | compression
--------------------------------------------------------------------- ---------------------------------------------------------------------
table_options | 10000 | 150000 | none table_options | 10000 | 150000 | none
(1 row) (1 row)
@ -20,13 +20,13 @@ SELECT alter_columnar_table_set('table_options', compression => 'pglz');
-- show table_options settings -- show table_options settings
SELECT * FROM cstore.options SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass; WHERE regclass = 'table_options'::regclass;
regclass | block_row_count | stripe_row_count | compression regclass | chunk_row_count | stripe_row_count | compression
--------------------------------------------------------------------- ---------------------------------------------------------------------
table_options | 10000 | 150000 | pglz table_options | 10000 | 150000 | pglz
(1 row) (1 row)
-- test changing the block_row_count -- test changing the chunk_row_count
SELECT alter_columnar_table_set('table_options', block_row_count => 10); SELECT alter_columnar_table_set('table_options', chunk_row_count => 10);
alter_columnar_table_set alter_columnar_table_set
--------------------------------------------------------------------- ---------------------------------------------------------------------
@ -35,12 +35,12 @@ SELECT alter_columnar_table_set('table_options', block_row_count => 10);
-- show table_options settings -- show table_options settings
SELECT * FROM cstore.options SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass; WHERE regclass = 'table_options'::regclass;
regclass | block_row_count | stripe_row_count | compression regclass | chunk_row_count | stripe_row_count | compression
--------------------------------------------------------------------- ---------------------------------------------------------------------
table_options | 10 | 150000 | pglz table_options | 10 | 150000 | pglz
(1 row) (1 row)
-- test changing the block_row_count -- test changing the chunk_row_count
SELECT alter_columnar_table_set('table_options', stripe_row_count => 100); SELECT alter_columnar_table_set('table_options', stripe_row_count => 100);
alter_columnar_table_set alter_columnar_table_set
--------------------------------------------------------------------- ---------------------------------------------------------------------
@ -50,7 +50,7 @@ SELECT alter_columnar_table_set('table_options', stripe_row_count => 100);
-- show table_options settings -- show table_options settings
SELECT * FROM cstore.options SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass; WHERE regclass = 'table_options'::regclass;
regclass | block_row_count | stripe_row_count | compression regclass | chunk_row_count | stripe_row_count | compression
--------------------------------------------------------------------- ---------------------------------------------------------------------
table_options | 10 | 100 | pglz table_options | 10 | 100 | pglz
(1 row) (1 row)
@ -60,13 +60,13 @@ VACUUM FULL table_options;
-- show table_options settings -- show table_options settings
SELECT * FROM cstore.options SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass; WHERE regclass = 'table_options'::regclass;
regclass | block_row_count | stripe_row_count | compression regclass | chunk_row_count | stripe_row_count | compression
--------------------------------------------------------------------- ---------------------------------------------------------------------
table_options | 10 | 100 | pglz table_options | 10 | 100 | pglz
(1 row) (1 row)
-- set all settings at the same time -- set all settings at the same time
SELECT alter_columnar_table_set('table_options', stripe_row_count => 1000, block_row_count => 100, compression => 'none'); SELECT alter_columnar_table_set('table_options', stripe_row_count => 1000, chunk_row_count => 100, compression => 'none');
alter_columnar_table_set alter_columnar_table_set
--------------------------------------------------------------------- ---------------------------------------------------------------------
@ -75,7 +75,7 @@ SELECT alter_columnar_table_set('table_options', stripe_row_count => 1000, block
-- show table_options settings -- show table_options settings
SELECT * FROM cstore.options SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass; WHERE regclass = 'table_options'::regclass;
regclass | block_row_count | stripe_row_count | compression regclass | chunk_row_count | stripe_row_count | compression
--------------------------------------------------------------------- ---------------------------------------------------------------------
table_options | 100 | 1000 | none table_options | 100 | 1000 | none
(1 row) (1 row)
@ -85,7 +85,7 @@ VACUUM table_options;
-- show table_options settings -- show table_options settings
SELECT * FROM cstore.options SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass; WHERE regclass = 'table_options'::regclass;
regclass | block_row_count | stripe_row_count | compression regclass | chunk_row_count | stripe_row_count | compression
--------------------------------------------------------------------- ---------------------------------------------------------------------
table_options | 100 | 1000 | none table_options | 100 | 1000 | none
(1 row) (1 row)
@ -95,7 +95,7 @@ VACUUM FULL table_options;
-- show table_options settings -- show table_options settings
SELECT * FROM cstore.options SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass; WHERE regclass = 'table_options'::regclass;
regclass | block_row_count | stripe_row_count | compression regclass | chunk_row_count | stripe_row_count | compression
--------------------------------------------------------------------- ---------------------------------------------------------------------
table_options | 100 | 1000 | none table_options | 100 | 1000 | none
(1 row) (1 row)
@ -105,7 +105,7 @@ TRUNCATE table_options;
-- show table_options settings -- show table_options settings
SELECT * FROM cstore.options SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass; WHERE regclass = 'table_options'::regclass;
regclass | block_row_count | stripe_row_count | compression regclass | chunk_row_count | stripe_row_count | compression
--------------------------------------------------------------------- ---------------------------------------------------------------------
table_options | 100 | 1000 | none table_options | 100 | 1000 | none
(1 row) (1 row)
@ -114,25 +114,25 @@ ALTER TABLE table_options ALTER COLUMN a TYPE bigint;
-- show table_options settings -- show table_options settings
SELECT * FROM cstore.options SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass; WHERE regclass = 'table_options'::regclass;
regclass | block_row_count | stripe_row_count | compression regclass | chunk_row_count | stripe_row_count | compression
--------------------------------------------------------------------- ---------------------------------------------------------------------
table_options | 100 | 1000 | none table_options | 100 | 1000 | none
(1 row) (1 row)
-- reset settings one by one to the version of the GUC's -- reset settings one by one to the version of the GUC's
SET cstore.block_row_count TO 1000; SET cstore.chunk_row_count TO 1000;
SET cstore.stripe_row_count TO 10000; SET cstore.stripe_row_count TO 10000;
SET cstore.compression TO 'pglz'; SET cstore.compression TO 'pglz';
-- verify setting the GUC's didn't change the settings -- verify setting the GUC's didn't change the settings
-- show table_options settings -- show table_options settings
SELECT * FROM cstore.options SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass; WHERE regclass = 'table_options'::regclass;
regclass | block_row_count | stripe_row_count | compression regclass | chunk_row_count | stripe_row_count | compression
--------------------------------------------------------------------- ---------------------------------------------------------------------
table_options | 100 | 1000 | none table_options | 100 | 1000 | none
(1 row) (1 row)
SELECT alter_columnar_table_reset('table_options', block_row_count => true); SELECT alter_columnar_table_reset('table_options', chunk_row_count => true);
alter_columnar_table_reset alter_columnar_table_reset
--------------------------------------------------------------------- ---------------------------------------------------------------------
@ -141,7 +141,7 @@ SELECT alter_columnar_table_reset('table_options', block_row_count => true);
-- show table_options settings -- show table_options settings
SELECT * FROM cstore.options SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass; WHERE regclass = 'table_options'::regclass;
regclass | block_row_count | stripe_row_count | compression regclass | chunk_row_count | stripe_row_count | compression
--------------------------------------------------------------------- ---------------------------------------------------------------------
table_options | 1000 | 1000 | none table_options | 1000 | 1000 | none
(1 row) (1 row)
@ -155,7 +155,7 @@ SELECT alter_columnar_table_reset('table_options', stripe_row_count => true);
-- show table_options settings -- show table_options settings
SELECT * FROM cstore.options SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass; WHERE regclass = 'table_options'::regclass;
regclass | block_row_count | stripe_row_count | compression regclass | chunk_row_count | stripe_row_count | compression
--------------------------------------------------------------------- ---------------------------------------------------------------------
table_options | 1000 | 10000 | none table_options | 1000 | 10000 | none
(1 row) (1 row)
@ -169,26 +169,26 @@ SELECT alter_columnar_table_reset('table_options', compression => true);
-- show table_options settings -- show table_options settings
SELECT * FROM cstore.options SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass; WHERE regclass = 'table_options'::regclass;
regclass | block_row_count | stripe_row_count | compression regclass | chunk_row_count | stripe_row_count | compression
--------------------------------------------------------------------- ---------------------------------------------------------------------
table_options | 1000 | 10000 | pglz table_options | 1000 | 10000 | pglz
(1 row) (1 row)
-- verify resetting all settings at once work -- verify resetting all settings at once work
SET cstore.block_row_count TO 10000; SET cstore.chunk_row_count TO 10000;
SET cstore.stripe_row_count TO 100000; SET cstore.stripe_row_count TO 100000;
SET cstore.compression TO 'none'; SET cstore.compression TO 'none';
-- show table_options settings -- show table_options settings
SELECT * FROM cstore.options SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass; WHERE regclass = 'table_options'::regclass;
regclass | block_row_count | stripe_row_count | compression regclass | chunk_row_count | stripe_row_count | compression
--------------------------------------------------------------------- ---------------------------------------------------------------------
table_options | 1000 | 10000 | pglz table_options | 1000 | 10000 | pglz
(1 row) (1 row)
SELECT alter_columnar_table_reset( SELECT alter_columnar_table_reset(
'table_options', 'table_options',
block_row_count => true, chunk_row_count => true,
stripe_row_count => true, stripe_row_count => true,
compression => true); compression => true);
alter_columnar_table_reset alter_columnar_table_reset
@ -199,7 +199,7 @@ SELECT alter_columnar_table_reset(
-- show table_options settings -- show table_options settings
SELECT * FROM cstore.options SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass; WHERE regclass = 'table_options'::regclass;
regclass | block_row_count | stripe_row_count | compression regclass | chunk_row_count | stripe_row_count | compression
--------------------------------------------------------------------- ---------------------------------------------------------------------
table_options | 10000 | 100000 | none table_options | 10000 | 100000 | none
(1 row) (1 row)
@ -218,7 +218,7 @@ ERROR: unknown compression type for cstore table: foobar
DROP TABLE table_options; DROP TABLE table_options;
-- we expect no entries in çstore.options for anything not found int pg_class -- we expect no entries in çstore.options for anything not found int pg_class
SELECT * FROM cstore.options o WHERE o.regclass NOT IN (SELECT oid FROM pg_class); SELECT * FROM cstore.options o WHERE o.regclass NOT IN (SELECT oid FROM pg_class);
regclass | block_row_count | stripe_row_count | compression regclass | chunk_row_count | stripe_row_count | compression
--------------------------------------------------------------------- ---------------------------------------------------------------------
(0 rows) (0 rows)

24
src/test/regress/expected/am_vacuum.out
View File

@ -73,10 +73,10 @@ SELECT count(*) FROM t_stripes;
-- VACUUM FULL doesn't reclaim dropped columns, but converts them to NULLs -- VACUUM FULL doesn't reclaim dropped columns, but converts them to NULLs
ALTER TABLE t DROP COLUMN a; ALTER TABLE t DROP COLUMN a;
SELECT stripe, attr, block, minimum_value IS NULL, maximum_value IS NULL SELECT stripe, attr, chunk, minimum_value IS NULL, maximum_value IS NULL
FROM cstore.cstore_skipnodes a, pg_class b FROM cstore.cstore_skipnodes a, pg_class b
WHERE a.storageid = columnar_relation_storageid(b.oid) AND b.relname='t' ORDER BY 1, 2, 3; WHERE a.storageid = columnar_relation_storageid(b.oid) AND b.relname='t' ORDER BY 1, 2, 3;
stripe | attr | block | ?column? | ?column? stripe | attr | chunk | ?column? | ?column?
--------------------------------------------------------------------- ---------------------------------------------------------------------
1 | 1 | 0 | f | f 1 | 1 | 0 | f | f
1 | 2 | 0 | f | f 1 | 2 | 0 | f | f
@ -87,10 +87,10 @@ WHERE a.storageid = columnar_relation_storageid(b.oid) AND b.relname='t' ORDER B
(6 rows) (6 rows)
VACUUM FULL t; VACUUM FULL t;
SELECT stripe, attr, block, minimum_value IS NULL, maximum_value IS NULL SELECT stripe, attr, chunk, minimum_value IS NULL, maximum_value IS NULL
FROM cstore.cstore_skipnodes a, pg_class b FROM cstore.cstore_skipnodes a, pg_class b
WHERE a.storageid = columnar_relation_storageid(b.oid) AND b.relname='t' ORDER BY 1, 2, 3; WHERE a.storageid = columnar_relation_storageid(b.oid) AND b.relname='t' ORDER BY 1, 2, 3;
stripe | attr | block | ?column? | ?column? stripe | attr | chunk | ?column? | ?column?
--------------------------------------------------------------------- ---------------------------------------------------------------------
1 | 1 | 0 | t | t 1 | 1 | 0 | t | t
1 | 2 | 0 | f | f 1 | 2 | 0 | f | f
@ -151,7 +151,7 @@ INFO: statistics for "t":
storage id: xxxxx storage id: xxxxx
total file size: 122880, total data size: 10754 total file size: 122880, total data size: 10754
total row count: 2530, stripe count: 3, average rows per stripe: 843 total row count: 2530, stripe count: 3, average rows per stripe: 843
block count: 3, containing data for dropped columns: 0, none compressed: 3, pglz compressed: 0 chunk count: 3, containing data for dropped columns: 0, none compressed: 3, pglz compressed: 0
INFO: "t": truncated 15 to 5 pages INFO: "t": truncated 15 to 5 pages
SELECT pg_size_pretty(pg_relation_size('t')); SELECT pg_size_pretty(pg_relation_size('t'));
@ -170,7 +170,7 @@ SELECT count(*) FROM t;
-- then vacuum to print stats -- then vacuum to print stats
BEGIN; BEGIN;
SELECT alter_columnar_table_set('t', SELECT alter_columnar_table_set('t',
block_row_count => 1000, chunk_row_count => 1000,
stripe_row_count => 2000, stripe_row_count => 2000,
compression => 'pglz'); compression => 'pglz');
alter_columnar_table_set alter_columnar_table_set
@ -198,7 +198,7 @@ INFO: statistics for "t":
storage id: xxxxx storage id: xxxxx
total file size: 57344, total data size: 18808 total file size: 57344, total data size: 18808
total row count: 5530, stripe count: 5, average rows per stripe: 1106 total row count: 5530, stripe count: 5, average rows per stripe: 1106
block count: 7, containing data for dropped columns: 0, none compressed: 5, pglz compressed: 2 chunk count: 7, containing data for dropped columns: 0, none compressed: 5, pglz compressed: 2
SELECT count(*) FROM t; SELECT count(*) FROM t;
count count
@ -206,7 +206,7 @@ SELECT count(*) FROM t;
5530 5530
(1 row) (1 row)
-- check that we report blocks with data for dropped columns -- check that we report chunks with data for dropped columns
ALTER TABLE t ADD COLUMN c int; ALTER TABLE t ADD COLUMN c int;
INSERT INTO t SELECT 1, i / 5 FROM generate_series(1, 1500) i; INSERT INTO t SELECT 1, i / 5 FROM generate_series(1, 1500) i;
ALTER TABLE t DROP COLUMN c; ALTER TABLE t DROP COLUMN c;
@ -215,10 +215,10 @@ INFO: statistics for "t":
storage id: xxxxx storage id: xxxxx
total file size: 73728, total data size: 31372 total file size: 73728, total data size: 31372
total row count: 7030, stripe count: 6, average rows per stripe: 1171 total row count: 7030, stripe count: 6, average rows per stripe: 1171
block count: 11, containing data for dropped columns: 2, none compressed: 9, pglz compressed: 2 chunk count: 11, containing data for dropped columns: 2, none compressed: 9, pglz compressed: 2
-- vacuum full should remove blocks for dropped columns -- vacuum full should remove chunks for dropped columns
-- note that, a block will be stored in non-compressed for if compression -- note that, a chunk will be stored in non-compressed for if compression
-- doesn't reduce its size. -- doesn't reduce its size.
SELECT alter_columnar_table_set('t', compression => 'pglz'); SELECT alter_columnar_table_set('t', compression => 'pglz');
alter_columnar_table_set alter_columnar_table_set
@ -232,7 +232,7 @@ INFO: statistics for "t":
storage id: xxxxx storage id: xxxxx
total file size: 57344, total data size: 15728 total file size: 57344, total data size: 15728
total row count: 7030, stripe count: 4, average rows per stripe: 1757 total row count: 7030, stripe count: 4, average rows per stripe: 1757
block count: 8, containing data for dropped columns: 0, none compressed: 2, pglz compressed: 6 chunk count: 8, containing data for dropped columns: 0, none compressed: 2, pglz compressed: 6
DROP TABLE t; DROP TABLE t;
DROP VIEW t_stripes; DROP VIEW t_stripes;

2
src/test/regress/expected/am_vacuum_vs_insert.out
View File

@ -14,7 +14,7 @@ s2: INFO: statistics for "test_vacuum_vs_insert":
storage id: xxxxx storage id: xxxxx
total file size: 24576, total data size: 26 total file size: 24576, total data size: 26
total row count: 3, stripe count: 1, average rows per stripe: 3 total row count: 3, stripe count: 1, average rows per stripe: 3
block count: 2, containing data for dropped columns: 0, none compressed: 2, pglz compressed: 0 chunk count: 2, containing data for dropped columns: 0, none compressed: 2, pglz compressed: 0
s2: INFO: "test_vacuum_vs_insert": stopping truncate due to conflicting lock request s2: INFO: "test_vacuum_vs_insert": stopping truncate due to conflicting lock request
step s2-vacuum: step s2-vacuum:

48
src/test/regress/expected/columnar_citus_integration.out
View File

@ -63,10 +63,10 @@ $cmd$);
(localhost,57638,20090003,t,none) (localhost,57638,20090003,t,none)
(4 rows) (4 rows)
-- setting: block_row_count -- setting: chunk_row_count
-- get baseline for setting -- get baseline for setting
SELECT run_command_on_placements('table_option',$cmd$ SELECT run_command_on_placements('table_option',$cmd$
SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass; SELECT chunk_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$); $cmd$);
run_command_on_placements run_command_on_placements
--------------------------------------------------------------------- ---------------------------------------------------------------------
@ -77,7 +77,7 @@ $cmd$);
(4 rows) (4 rows)
-- change setting -- change setting
SELECT alter_columnar_table_set('table_option', block_row_count => 100); SELECT alter_columnar_table_set('table_option', chunk_row_count => 100);
alter_columnar_table_set alter_columnar_table_set
--------------------------------------------------------------------- ---------------------------------------------------------------------
@ -85,7 +85,7 @@ SELECT alter_columnar_table_set('table_option', block_row_count => 100);
-- verify setting -- verify setting
SELECT run_command_on_placements('table_option',$cmd$ SELECT run_command_on_placements('table_option',$cmd$
SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass; SELECT chunk_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$); $cmd$);
run_command_on_placements run_command_on_placements
--------------------------------------------------------------------- ---------------------------------------------------------------------
@ -96,7 +96,7 @@ $cmd$);
(4 rows) (4 rows)
-- reset setting -- reset setting
SELECT alter_columnar_table_reset('table_option', block_row_count => true); SELECT alter_columnar_table_reset('table_option', chunk_row_count => true);
alter_columnar_table_reset alter_columnar_table_reset
--------------------------------------------------------------------- ---------------------------------------------------------------------
@ -104,7 +104,7 @@ SELECT alter_columnar_table_reset('table_option', block_row_count => true);
-- verify setting -- verify setting
SELECT run_command_on_placements('table_option',$cmd$ SELECT run_command_on_placements('table_option',$cmd$
SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass; SELECT chunk_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$); $cmd$);
run_command_on_placements run_command_on_placements
--------------------------------------------------------------------- ---------------------------------------------------------------------
@ -168,7 +168,7 @@ $cmd$);
-- verify settings are propagated when creating a table -- verify settings are propagated when creating a table
CREATE TABLE table_option_2 (a int, b text) USING columnar; CREATE TABLE table_option_2 (a int, b text) USING columnar;
SELECT alter_columnar_table_set('table_option_2', SELECT alter_columnar_table_set('table_option_2',
block_row_count => 100, chunk_row_count => 100,
stripe_row_count => 1000, stripe_row_count => 1000,
compression => 'pglz'); compression => 'pglz');
alter_columnar_table_set alter_columnar_table_set
@ -184,7 +184,7 @@ SELECT create_distributed_table('table_option_2', 'a');
-- verify settings on placements -- verify settings on placements
SELECT run_command_on_placements('table_option_2',$cmd$ SELECT run_command_on_placements('table_option_2',$cmd$
SELECT ROW(block_row_count, stripe_row_count, compression) FROM cstore.options WHERE regclass = '%s'::regclass; SELECT ROW(chunk_row_count, stripe_row_count, compression) FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$); $cmd$);
run_command_on_placements run_command_on_placements
--------------------------------------------------------------------- ---------------------------------------------------------------------
@ -268,10 +268,10 @@ $cmd$);
(localhost,57638,20090011,t,none) (localhost,57638,20090011,t,none)
(8 rows) (8 rows)
-- setting: block_row_count -- setting: chunk_row_count
-- get baseline for setting -- get baseline for setting
SELECT run_command_on_placements('table_option',$cmd$ SELECT run_command_on_placements('table_option',$cmd$
SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass; SELECT chunk_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$); $cmd$);
run_command_on_placements run_command_on_placements
--------------------------------------------------------------------- ---------------------------------------------------------------------
@ -286,7 +286,7 @@ $cmd$);
(8 rows) (8 rows)
-- change setting -- change setting
SELECT alter_columnar_table_set('table_option', block_row_count => 100); SELECT alter_columnar_table_set('table_option', chunk_row_count => 100);
alter_columnar_table_set alter_columnar_table_set
--------------------------------------------------------------------- ---------------------------------------------------------------------
@ -294,7 +294,7 @@ SELECT alter_columnar_table_set('table_option', block_row_count => 100);
-- verify setting -- verify setting
SELECT run_command_on_placements('table_option',$cmd$ SELECT run_command_on_placements('table_option',$cmd$
SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass; SELECT chunk_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$); $cmd$);
run_command_on_placements run_command_on_placements
--------------------------------------------------------------------- ---------------------------------------------------------------------
@ -309,7 +309,7 @@ $cmd$);
(8 rows) (8 rows)
-- reset setting -- reset setting
SELECT alter_columnar_table_reset('table_option', block_row_count => true); SELECT alter_columnar_table_reset('table_option', chunk_row_count => true);
alter_columnar_table_reset alter_columnar_table_reset
--------------------------------------------------------------------- ---------------------------------------------------------------------
@ -317,7 +317,7 @@ SELECT alter_columnar_table_reset('table_option', block_row_count => true);
-- verify setting -- verify setting
SELECT run_command_on_placements('table_option',$cmd$ SELECT run_command_on_placements('table_option',$cmd$
SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass; SELECT chunk_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$); $cmd$);
run_command_on_placements run_command_on_placements
--------------------------------------------------------------------- ---------------------------------------------------------------------
@ -397,7 +397,7 @@ $cmd$);
-- verify settings are propagated when creating a table -- verify settings are propagated when creating a table
CREATE TABLE table_option_2 (a int, b text) USING columnar; CREATE TABLE table_option_2 (a int, b text) USING columnar;
SELECT alter_columnar_table_set('table_option_2', SELECT alter_columnar_table_set('table_option_2',
block_row_count => 100, chunk_row_count => 100,
stripe_row_count => 1000, stripe_row_count => 1000,
compression => 'pglz'); compression => 'pglz');
alter_columnar_table_set alter_columnar_table_set
@ -413,7 +413,7 @@ SELECT create_distributed_table('table_option_2', 'a');
-- verify settings on placements -- verify settings on placements
SELECT run_command_on_placements('table_option_2',$cmd$ SELECT run_command_on_placements('table_option_2',$cmd$
SELECT ROW(block_row_count, stripe_row_count, compression) FROM cstore.options WHERE regclass = '%s'::regclass; SELECT ROW(chunk_row_count, stripe_row_count, compression) FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$); $cmd$);
run_command_on_placements run_command_on_placements
--------------------------------------------------------------------- ---------------------------------------------------------------------
@ -481,10 +481,10 @@ $cmd$);
(localhost,57638,20090016,t,none) (localhost,57638,20090016,t,none)
(2 rows) (2 rows)
-- setting: block_row_count -- setting: chunk_row_count
-- get baseline for setting -- get baseline for setting
SELECT run_command_on_placements('table_option_reference',$cmd$ SELECT run_command_on_placements('table_option_reference',$cmd$
SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass; SELECT chunk_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$); $cmd$);
run_command_on_placements run_command_on_placements
--------------------------------------------------------------------- ---------------------------------------------------------------------
@ -493,7 +493,7 @@ $cmd$);
(2 rows) (2 rows)
-- change setting -- change setting
SELECT alter_columnar_table_set('table_option_reference', block_row_count => 100); SELECT alter_columnar_table_set('table_option_reference', chunk_row_count => 100);
alter_columnar_table_set alter_columnar_table_set
--------------------------------------------------------------------- ---------------------------------------------------------------------
@ -501,7 +501,7 @@ SELECT alter_columnar_table_set('table_option_reference', block_row_count => 100
-- verify setting -- verify setting
SELECT run_command_on_placements('table_option_reference',$cmd$ SELECT run_command_on_placements('table_option_reference',$cmd$
SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass; SELECT chunk_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$); $cmd$);
run_command_on_placements run_command_on_placements
--------------------------------------------------------------------- ---------------------------------------------------------------------
@ -510,7 +510,7 @@ $cmd$);
(2 rows) (2 rows)
-- reset setting -- reset setting
SELECT alter_columnar_table_reset('table_option_reference', block_row_count => true); SELECT alter_columnar_table_reset('table_option_reference', chunk_row_count => true);
alter_columnar_table_reset alter_columnar_table_reset
--------------------------------------------------------------------- ---------------------------------------------------------------------
@ -518,7 +518,7 @@ SELECT alter_columnar_table_reset('table_option_reference', block_row_count => t
-- verify setting -- verify setting
SELECT run_command_on_placements('table_option_reference',$cmd$ SELECT run_command_on_placements('table_option_reference',$cmd$
SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass; SELECT chunk_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$); $cmd$);
run_command_on_placements run_command_on_placements
--------------------------------------------------------------------- ---------------------------------------------------------------------
@ -574,7 +574,7 @@ $cmd$);
-- verify settings are propagated when creating a table -- verify settings are propagated when creating a table
CREATE TABLE table_option_reference_2 (a int, b text) USING columnar; CREATE TABLE table_option_reference_2 (a int, b text) USING columnar;
SELECT alter_columnar_table_set('table_option_reference_2', SELECT alter_columnar_table_set('table_option_reference_2',
block_row_count => 100, chunk_row_count => 100,
stripe_row_count => 1000, stripe_row_count => 1000,
compression => 'pglz'); compression => 'pglz');
alter_columnar_table_set alter_columnar_table_set
@ -590,7 +590,7 @@ SELECT create_reference_table('table_option_reference_2');
-- verify settings on placements -- verify settings on placements
SELECT run_command_on_placements('table_option_reference_2',$cmd$ SELECT run_command_on_placements('table_option_reference_2',$cmd$
SELECT ROW(block_row_count, stripe_row_count, compression) FROM cstore.options WHERE regclass = '%s'::regclass; SELECT ROW(chunk_row_count, stripe_row_count, compression) FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$); $cmd$);
run_command_on_placements run_command_on_placements
--------------------------------------------------------------------- ---------------------------------------------------------------------

8
src/test/regress/expected/upgrade_columnar_after.out
View File

@ -102,7 +102,7 @@ SELECT * FROM matview ORDER BY a;
-- test we retained options -- test we retained options
SELECT * FROM cstore.options WHERE regclass = 'test_options_1'::regclass; SELECT * FROM cstore.options WHERE regclass = 'test_options_1'::regclass;
regclass | block_row_count | stripe_row_count | compression regclass | chunk_row_count | stripe_row_count | compression
--------------------------------------------------------------------- ---------------------------------------------------------------------
test_options_1 | 1000 | 5000 | pglz test_options_1 | 1000 | 5000 | pglz
(1 row) (1 row)
@ -112,7 +112,7 @@ INFO: statistics for "test_options_1":
storage id: xxxxx storage id: xxxxx
total file size: 65536, total data size: 43136 total file size: 65536, total data size: 43136
total row count: 10000, stripe count: 2, average rows per stripe: 5000 total row count: 10000, stripe count: 2, average rows per stripe: 5000
block count: 20, containing data for dropped columns: 0, none compressed: 10, pglz compressed: 10 chunk count: 20, containing data for dropped columns: 0, none compressed: 10, pglz compressed: 10
SELECT count(*), sum(a), sum(b) FROM test_options_1; SELECT count(*), sum(a), sum(b) FROM test_options_1;
count | sum | sum count | sum | sum
@ -121,7 +121,7 @@ SELECT count(*), sum(a), sum(b) FROM test_options_1;
(1 row) (1 row)
SELECT * FROM cstore.options WHERE regclass = 'test_options_2'::regclass; SELECT * FROM cstore.options WHERE regclass = 'test_options_2'::regclass;
regclass | block_row_count | stripe_row_count | compression regclass | chunk_row_count | stripe_row_count | compression
--------------------------------------------------------------------- ---------------------------------------------------------------------
test_options_2 | 2000 | 6000 | none test_options_2 | 2000 | 6000 | none
(1 row) (1 row)
@ -131,7 +131,7 @@ INFO: statistics for "test_options_2":
storage id: xxxxx storage id: xxxxx
total file size: 163840, total data size: 125636 total file size: 163840, total data size: 125636
total row count: 20000, stripe count: 4, average rows per stripe: 5000 total row count: 20000, stripe count: 4, average rows per stripe: 5000
block count: 30, containing data for dropped columns: 0, none compressed: 20, pglz compressed: 10 chunk count: 30, containing data for dropped columns: 0, none compressed: 20, pglz compressed: 10
SELECT count(*), sum(a), sum(b) FROM test_options_2; SELECT count(*), sum(a), sum(b) FROM test_options_2;
count | sum | sum count | sum | sum

4
src/test/regress/expected/upgrade_columnar_before.out
View File

@ -116,13 +116,13 @@ SELECT :relfilenode_pre_alter <> :relfilenode_post_alter AS relfilenode_changed;
-- Test that we retain options -- Test that we retain options
-- --
SET cstore.stripe_row_count TO 5000; SET cstore.stripe_row_count TO 5000;
SET cstore.block_row_count TO 1000; SET cstore.chunk_row_count TO 1000;
SET cstore.compression TO 'pglz'; SET cstore.compression TO 'pglz';
CREATE TABLE test_options_1(a int, b int) USING columnar; CREATE TABLE test_options_1(a int, b int) USING columnar;
INSERT INTO test_options_1 SELECT i, floor(i/1000) FROM generate_series(1, 10000) i; INSERT INTO test_options_1 SELECT i, floor(i/1000) FROM generate_series(1, 10000) i;
CREATE TABLE test_options_2(a int, b int) USING columnar; CREATE TABLE test_options_2(a int, b int) USING columnar;
INSERT INTO test_options_2 SELECT i, floor(i/1000) FROM generate_series(1, 10000) i; INSERT INTO test_options_2 SELECT i, floor(i/1000) FROM generate_series(1, 10000) i;
SELECT alter_columnar_table_set('test_options_2', block_row_count => 2000); SELECT alter_columnar_table_set('test_options_2', chunk_row_count => 2000);
alter_columnar_table_set alter_columnar_table_set
--------------------------------------------------------------------- ---------------------------------------------------------------------

46
src/test/regress/input/am_block_filtering.source → src/test/regress/input/am_chunk_filtering.source
View File

@ -1,11 +1,11 @@
-- --
-- Test block filtering in cstore_fdw using min/max values in stripe skip lists. -- Test chunk filtering in cstore_fdw using min/max values in stripe skip lists.
-- --
-- --
-- filtered_row_count returns number of rows filtered by the WHERE clause. -- filtered_row_count returns number of rows filtered by the WHERE clause.
-- If blocks get filtered by cstore_fdw, less rows are passed to WHERE -- If chunks get filtered by cstore_fdw, less rows are passed to WHERE
-- clause, so this function should return a lower number. -- clause, so this function should return a lower number.
-- --
CREATE OR REPLACE FUNCTION filtered_row_count (query text) RETURNS bigint AS CREATE OR REPLACE FUNCTION filtered_row_count (query text) RETURNS bigint AS
@ -28,46 +28,46 @@ $$ LANGUAGE PLPGSQL;
-- Create and load data -- Create and load data
-- block_row_count '1000', stripe_row_count '2000' -- chunk_row_count '1000', stripe_row_count '2000'
set cstore.stripe_row_count = 2000; set cstore.stripe_row_count = 2000;
set cstore.block_row_count = 1000; set cstore.chunk_row_count = 1000;
CREATE TABLE test_block_filtering (a int) CREATE TABLE test_chunk_filtering (a int)
USING columnar; USING columnar;
COPY test_block_filtering FROM '@abs_srcdir@/data/block_filtering.csv' WITH CSV; COPY test_chunk_filtering FROM '@abs_srcdir@/data/chunk_filtering.csv' WITH CSV;
-- Verify that filtered_row_count is less than 1000 for the following queries -- Verify that filtered_row_count is less than 1000 for the following queries
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering'); SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering');
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 200'); SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a < 200');
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a > 200'); SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a > 200');
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 9900'); SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a < 9900');
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a > 9900'); SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a > 9900');
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 0'); SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a < 0');
-- Verify that filtered_row_count is less than 2000 for the following queries -- Verify that filtered_row_count is less than 2000 for the following queries
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 1 AND 10'); SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a BETWEEN 1 AND 10');
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 990 AND 2010'); SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a BETWEEN 990 AND 2010');
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN -10 AND 0'); SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a BETWEEN -10 AND 0');
-- Load data for second time and verify that filtered_row_count is exactly twice as before -- Load data for second time and verify that filtered_row_count is exactly twice as before
COPY test_block_filtering FROM '@abs_srcdir@/data/block_filtering.csv' WITH CSV; COPY test_chunk_filtering FROM '@abs_srcdir@/data/chunk_filtering.csv' WITH CSV;
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 200'); SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a < 200');
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 0'); SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a < 0');
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 990 AND 2010'); SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a BETWEEN 990 AND 2010');
set cstore.stripe_row_count to default; set cstore.stripe_row_count to default;
set cstore.block_row_count to default; set cstore.chunk_row_count to default;
-- Verify that we are fine with collations which use a different alphabet order -- Verify that we are fine with collations which use a different alphabet order
CREATE TABLE collation_block_filtering_test(A text collate "da_DK") CREATE TABLE collation_chunk_filtering_test(A text collate "da_DK")
USING columnar; USING columnar;
COPY collation_block_filtering_test FROM STDIN; COPY collation_chunk_filtering_test FROM STDIN;
A A
Å Å
B B
\. \.
SELECT * FROM collation_block_filtering_test WHERE A > 'B'; SELECT * FROM collation_chunk_filtering_test WHERE A > 'B';

46
src/test/regress/output/am_block_filtering.source → src/test/regress/output/am_chunk_filtering.source
View File

@ -1,9 +1,9 @@
-- --
-- Test block filtering in cstore_fdw using min/max values in stripe skip lists. -- Test chunk filtering in cstore_fdw using min/max values in stripe skip lists.
-- --
-- --
-- filtered_row_count returns number of rows filtered by the WHERE clause. -- filtered_row_count returns number of rows filtered by the WHERE clause.
-- If blocks get filtered by cstore_fdw, less rows are passed to WHERE -- If chunks get filtered by cstore_fdw, less rows are passed to WHERE
-- clause, so this function should return a lower number. -- clause, so this function should return a lower number.
-- --
CREATE OR REPLACE FUNCTION filtered_row_count (query text) RETURNS bigint AS CREATE OR REPLACE FUNCTION filtered_row_count (query text) RETURNS bigint AS
@ -24,95 +24,95 @@ $$
END; END;
$$ LANGUAGE PLPGSQL; $$ LANGUAGE PLPGSQL;
-- Create and load data -- Create and load data
-- block_row_count '1000', stripe_row_count '2000' -- chunk_row_count '1000', stripe_row_count '2000'
set cstore.stripe_row_count = 2000; set cstore.stripe_row_count = 2000;
set cstore.block_row_count = 1000; set cstore.chunk_row_count = 1000;
CREATE TABLE test_block_filtering (a int) CREATE TABLE test_chunk_filtering (a int)
USING columnar; USING columnar;
COPY test_block_filtering FROM '@abs_srcdir@/data/block_filtering.csv' WITH CSV; COPY test_chunk_filtering FROM '@abs_srcdir@/data/chunk_filtering.csv' WITH CSV;
-- Verify that filtered_row_count is less than 1000 for the following queries -- Verify that filtered_row_count is less than 1000 for the following queries
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering'); SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering');
filtered_row_count filtered_row_count
-------------------- --------------------
0 0
(1 row) (1 row)
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 200'); SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a < 200');
filtered_row_count filtered_row_count
-------------------- --------------------
801 801
(1 row) (1 row)
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a > 200'); SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a > 200');
filtered_row_count filtered_row_count
-------------------- --------------------
200 200
(1 row) (1 row)
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 9900'); SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a < 9900');
filtered_row_count filtered_row_count
-------------------- --------------------
101 101
(1 row) (1 row)
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a > 9900'); SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a > 9900');
filtered_row_count filtered_row_count
-------------------- --------------------
900 900
(1 row) (1 row)
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 0'); SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a < 0');
filtered_row_count filtered_row_count
-------------------- --------------------
0 0
(1 row) (1 row)
-- Verify that filtered_row_count is less than 2000 for the following queries -- Verify that filtered_row_count is less than 2000 for the following queries
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 1 AND 10'); SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a BETWEEN 1 AND 10');
filtered_row_count filtered_row_count
-------------------- --------------------
990 990
(1 row) (1 row)
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 990 AND 2010'); SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a BETWEEN 990 AND 2010');
filtered_row_count filtered_row_count
-------------------- --------------------
1979 1979
(1 row) (1 row)
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN -10 AND 0'); SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a BETWEEN -10 AND 0');
filtered_row_count filtered_row_count
-------------------- --------------------
0 0
(1 row) (1 row)
-- Load data for second time and verify that filtered_row_count is exactly twice as before -- Load data for second time and verify that filtered_row_count is exactly twice as before
COPY test_block_filtering FROM '@abs_srcdir@/data/block_filtering.csv' WITH CSV; COPY test_chunk_filtering FROM '@abs_srcdir@/data/chunk_filtering.csv' WITH CSV;
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 200'); SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a < 200');
filtered_row_count filtered_row_count
-------------------- --------------------
1602 1602
(1 row) (1 row)
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 0'); SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a < 0');
filtered_row_count filtered_row_count
-------------------- --------------------
0 0
(1 row) (1 row)
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 990 AND 2010'); SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a BETWEEN 990 AND 2010');
filtered_row_count filtered_row_count
-------------------- --------------------
3958 3958
(1 row) (1 row)
set cstore.stripe_row_count to default; set cstore.stripe_row_count to default;
set cstore.block_row_count to default; set cstore.chunk_row_count to default;
-- Verify that we are fine with collations which use a different alphabet order -- Verify that we are fine with collations which use a different alphabet order
CREATE TABLE collation_block_filtering_test(A text collate "da_DK") CREATE TABLE collation_chunk_filtering_test(A text collate "da_DK")
USING columnar; USING columnar;
COPY collation_block_filtering_test FROM STDIN; COPY collation_chunk_filtering_test FROM STDIN;
SELECT * FROM collation_block_filtering_test WHERE A > 'B'; SELECT * FROM collation_chunk_filtering_test WHERE A > 'B';
a a
--- ---
Å Å

2
src/test/regress/sql/.gitignore vendored
View File

@ -20,7 +20,7 @@
/multi_behavioral_analytics_create_table.sql /multi_behavioral_analytics_create_table.sql
/multi_insert_select_behavioral_analytics_create_table.sql /multi_insert_select_behavioral_analytics_create_table.sql
/hyperscale_tutorial.sql /hyperscale_tutorial.sql
/am_block_filtering.sql /am_chunk_filtering.sql
/am_copyto.sql /am_copyto.sql
/am_data_types.sql /am_data_types.sql
/am_load.sql /am_load.sql

2
src/test/regress/sql/am_empty.sql
View File

@ -9,7 +9,7 @@ create table t_compressed(a int) using columnar;
-- set options -- set options
SELECT alter_columnar_table_set('t_compressed', compression => 'pglz'); SELECT alter_columnar_table_set('t_compressed', compression => 'pglz');
SELECT alter_columnar_table_set('t_compressed', stripe_row_count => 100); SELECT alter_columnar_table_set('t_compressed', stripe_row_count => 100);
SELECT alter_columnar_table_set('t_compressed', block_row_count => 100); SELECT alter_columnar_table_set('t_compressed', chunk_row_count => 100);
SELECT * FROM cstore.options WHERE regclass = 't_compressed'::regclass; SELECT * FROM cstore.options WHERE regclass = 't_compressed'::regclass;

16
src/test/regress/sql/am_tableoptions.sql
View File

@ -15,14 +15,14 @@ SELECT alter_columnar_table_set('table_options', compression => 'pglz');
SELECT * FROM cstore.options SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass; WHERE regclass = 'table_options'::regclass;
-- test changing the block_row_count -- test changing the chunk_row_count
SELECT alter_columnar_table_set('table_options', block_row_count => 10); SELECT alter_columnar_table_set('table_options', chunk_row_count => 10);
-- show table_options settings -- show table_options settings
SELECT * FROM cstore.options SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass; WHERE regclass = 'table_options'::regclass;
-- test changing the block_row_count -- test changing the chunk_row_count
SELECT alter_columnar_table_set('table_options', stripe_row_count => 100); SELECT alter_columnar_table_set('table_options', stripe_row_count => 100);
-- show table_options settings -- show table_options settings
@ -37,7 +37,7 @@ SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass; WHERE regclass = 'table_options'::regclass;
-- set all settings at the same time -- set all settings at the same time
SELECT alter_columnar_table_set('table_options', stripe_row_count => 1000, block_row_count => 100, compression => 'none'); SELECT alter_columnar_table_set('table_options', stripe_row_count => 1000, chunk_row_count => 100, compression => 'none');
-- show table_options settings -- show table_options settings
SELECT * FROM cstore.options SELECT * FROM cstore.options
@ -67,7 +67,7 @@ SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass; WHERE regclass = 'table_options'::regclass;
-- reset settings one by one to the version of the GUC's -- reset settings one by one to the version of the GUC's
SET cstore.block_row_count TO 1000; SET cstore.chunk_row_count TO 1000;
SET cstore.stripe_row_count TO 10000; SET cstore.stripe_row_count TO 10000;
SET cstore.compression TO 'pglz'; SET cstore.compression TO 'pglz';
@ -76,7 +76,7 @@ SET cstore.compression TO 'pglz';
SELECT * FROM cstore.options SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass; WHERE regclass = 'table_options'::regclass;
SELECT alter_columnar_table_reset('table_options', block_row_count => true); SELECT alter_columnar_table_reset('table_options', chunk_row_count => true);
-- show table_options settings -- show table_options settings
SELECT * FROM cstore.options SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass; WHERE regclass = 'table_options'::regclass;
@ -94,7 +94,7 @@ SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass; WHERE regclass = 'table_options'::regclass;
-- verify resetting all settings at once work -- verify resetting all settings at once work
SET cstore.block_row_count TO 10000; SET cstore.chunk_row_count TO 10000;
SET cstore.stripe_row_count TO 100000; SET cstore.stripe_row_count TO 100000;
SET cstore.compression TO 'none'; SET cstore.compression TO 'none';
@ -104,7 +104,7 @@ WHERE regclass = 'table_options'::regclass;
SELECT alter_columnar_table_reset( SELECT alter_columnar_table_reset(
'table_options', 'table_options',
block_row_count => true, chunk_row_count => true,
stripe_row_count => true, stripe_row_count => true,
compression => true); compression => true);

12
src/test/regress/sql/am_vacuum.sql
View File

@ -36,13 +36,13 @@ SELECT count(*) FROM t_stripes;
-- VACUUM FULL doesn't reclaim dropped columns, but converts them to NULLs -- VACUUM FULL doesn't reclaim dropped columns, but converts them to NULLs
ALTER TABLE t DROP COLUMN a; ALTER TABLE t DROP COLUMN a;
SELECT stripe, attr, block, minimum_value IS NULL, maximum_value IS NULL SELECT stripe, attr, chunk, minimum_value IS NULL, maximum_value IS NULL
FROM cstore.cstore_skipnodes a, pg_class b FROM cstore.cstore_skipnodes a, pg_class b
WHERE a.storageid = columnar_relation_storageid(b.oid) AND b.relname='t' ORDER BY 1, 2, 3; WHERE a.storageid = columnar_relation_storageid(b.oid) AND b.relname='t' ORDER BY 1, 2, 3;
VACUUM FULL t; VACUUM FULL t;
SELECT stripe, attr, block, minimum_value IS NULL, maximum_value IS NULL SELECT stripe, attr, chunk, minimum_value IS NULL, maximum_value IS NULL
FROM cstore.cstore_skipnodes a, pg_class b FROM cstore.cstore_skipnodes a, pg_class b
WHERE a.storageid = columnar_relation_storageid(b.oid) AND b.relname='t' ORDER BY 1, 2, 3; WHERE a.storageid = columnar_relation_storageid(b.oid) AND b.relname='t' ORDER BY 1, 2, 3;
@ -74,7 +74,7 @@ SELECT count(*) FROM t;
BEGIN; BEGIN;
SELECT alter_columnar_table_set('t', SELECT alter_columnar_table_set('t',
block_row_count => 1000, chunk_row_count => 1000,
stripe_row_count => 2000, stripe_row_count => 2000,
compression => 'pglz'); compression => 'pglz');
SAVEPOINT s1; SAVEPOINT s1;
@ -92,15 +92,15 @@ VACUUM VERBOSE t;
SELECT count(*) FROM t; SELECT count(*) FROM t;
-- check that we report blocks with data for dropped columns -- check that we report chunks with data for dropped columns
ALTER TABLE t ADD COLUMN c int; ALTER TABLE t ADD COLUMN c int;
INSERT INTO t SELECT 1, i / 5 FROM generate_series(1, 1500) i; INSERT INTO t SELECT 1, i / 5 FROM generate_series(1, 1500) i;
ALTER TABLE t DROP COLUMN c; ALTER TABLE t DROP COLUMN c;
VACUUM VERBOSE t; VACUUM VERBOSE t;
-- vacuum full should remove blocks for dropped columns -- vacuum full should remove chunks for dropped columns
-- note that, a block will be stored in non-compressed for if compression -- note that, a chunk will be stored in non-compressed for if compression
-- doesn't reduce its size. -- doesn't reduce its size.
SELECT alter_columnar_table_set('t', compression => 'pglz'); SELECT alter_columnar_table_set('t', compression => 'pglz');
VACUUM FULL t; VACUUM FULL t;

48
src/test/regress/sql/columnar_citus_integration.sql
View File

@ -28,22 +28,22 @@ SELECT run_command_on_placements('table_option',$cmd$
SELECT compression FROM cstore.options WHERE regclass = '%s'::regclass; SELECT compression FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$); $cmd$);
-- setting: block_row_count -- setting: chunk_row_count
-- get baseline for setting -- get baseline for setting
SELECT run_command_on_placements('table_option',$cmd$ SELECT run_command_on_placements('table_option',$cmd$
SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass; SELECT chunk_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$); $cmd$);
-- change setting -- change setting
SELECT alter_columnar_table_set('table_option', block_row_count => 100); SELECT alter_columnar_table_set('table_option', chunk_row_count => 100);
-- verify setting -- verify setting
SELECT run_command_on_placements('table_option',$cmd$ SELECT run_command_on_placements('table_option',$cmd$
SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass; SELECT chunk_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$); $cmd$);
-- reset setting -- reset setting
SELECT alter_columnar_table_reset('table_option', block_row_count => true); SELECT alter_columnar_table_reset('table_option', chunk_row_count => true);
-- verify setting -- verify setting
SELECT run_command_on_placements('table_option',$cmd$ SELECT run_command_on_placements('table_option',$cmd$
SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass; SELECT chunk_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$); $cmd$);
-- setting: stripe_row_count -- setting: stripe_row_count
@ -67,14 +67,14 @@ $cmd$);
-- verify settings are propagated when creating a table -- verify settings are propagated when creating a table
CREATE TABLE table_option_2 (a int, b text) USING columnar; CREATE TABLE table_option_2 (a int, b text) USING columnar;
SELECT alter_columnar_table_set('table_option_2', SELECT alter_columnar_table_set('table_option_2',
block_row_count => 100, chunk_row_count => 100,
stripe_row_count => 1000, stripe_row_count => 1000,
compression => 'pglz'); compression => 'pglz');
SELECT create_distributed_table('table_option_2', 'a'); SELECT create_distributed_table('table_option_2', 'a');
-- verify settings on placements -- verify settings on placements
SELECT run_command_on_placements('table_option_2',$cmd$ SELECT run_command_on_placements('table_option_2',$cmd$
SELECT ROW(block_row_count, stripe_row_count, compression) FROM cstore.options WHERE regclass = '%s'::regclass; SELECT ROW(chunk_row_count, stripe_row_count, compression) FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$); $cmd$);
DROP TABLE table_option, table_option_2; DROP TABLE table_option, table_option_2;
@ -104,22 +104,22 @@ SELECT run_command_on_placements('table_option',$cmd$
SELECT compression FROM cstore.options WHERE regclass = '%s'::regclass; SELECT compression FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$); $cmd$);
-- setting: block_row_count -- setting: chunk_row_count
-- get baseline for setting -- get baseline for setting
SELECT run_command_on_placements('table_option',$cmd$ SELECT run_command_on_placements('table_option',$cmd$
SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass; SELECT chunk_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$); $cmd$);
-- change setting -- change setting
SELECT alter_columnar_table_set('table_option', block_row_count => 100); SELECT alter_columnar_table_set('table_option', chunk_row_count => 100);
-- verify setting -- verify setting
SELECT run_command_on_placements('table_option',$cmd$ SELECT run_command_on_placements('table_option',$cmd$
SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass; SELECT chunk_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$); $cmd$);
-- reset setting -- reset setting
SELECT alter_columnar_table_reset('table_option', block_row_count => true); SELECT alter_columnar_table_reset('table_option', chunk_row_count => true);
-- verify setting -- verify setting
SELECT run_command_on_placements('table_option',$cmd$ SELECT run_command_on_placements('table_option',$cmd$
SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass; SELECT chunk_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$); $cmd$);
-- setting: stripe_row_count -- setting: stripe_row_count
@ -143,14 +143,14 @@ $cmd$);
-- verify settings are propagated when creating a table -- verify settings are propagated when creating a table
CREATE TABLE table_option_2 (a int, b text) USING columnar; CREATE TABLE table_option_2 (a int, b text) USING columnar;
SELECT alter_columnar_table_set('table_option_2', SELECT alter_columnar_table_set('table_option_2',
block_row_count => 100, chunk_row_count => 100,
stripe_row_count => 1000, stripe_row_count => 1000,
compression => 'pglz'); compression => 'pglz');
SELECT create_distributed_table('table_option_2', 'a'); SELECT create_distributed_table('table_option_2', 'a');
-- verify settings on placements -- verify settings on placements
SELECT run_command_on_placements('table_option_2',$cmd$ SELECT run_command_on_placements('table_option_2',$cmd$
SELECT ROW(block_row_count, stripe_row_count, compression) FROM cstore.options WHERE regclass = '%s'::regclass; SELECT ROW(chunk_row_count, stripe_row_count, compression) FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$); $cmd$);
DROP TABLE table_option, table_option_2; DROP TABLE table_option, table_option_2;
@ -177,22 +177,22 @@ SELECT run_command_on_placements('table_option_reference',$cmd$
SELECT compression FROM cstore.options WHERE regclass = '%s'::regclass; SELECT compression FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$); $cmd$);
-- setting: block_row_count -- setting: chunk_row_count
-- get baseline for setting -- get baseline for setting
SELECT run_command_on_placements('table_option_reference',$cmd$ SELECT run_command_on_placements('table_option_reference',$cmd$
SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass; SELECT chunk_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$); $cmd$);
-- change setting -- change setting
SELECT alter_columnar_table_set('table_option_reference', block_row_count => 100); SELECT alter_columnar_table_set('table_option_reference', chunk_row_count => 100);
-- verify setting -- verify setting
SELECT run_command_on_placements('table_option_reference',$cmd$ SELECT run_command_on_placements('table_option_reference',$cmd$
SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass; SELECT chunk_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$); $cmd$);
-- reset setting -- reset setting
SELECT alter_columnar_table_reset('table_option_reference', block_row_count => true); SELECT alter_columnar_table_reset('table_option_reference', chunk_row_count => true);
-- verify setting -- verify setting
SELECT run_command_on_placements('table_option_reference',$cmd$ SELECT run_command_on_placements('table_option_reference',$cmd$
SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass; SELECT chunk_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$); $cmd$);
-- setting: stripe_row_count -- setting: stripe_row_count
@ -216,14 +216,14 @@ $cmd$);
-- verify settings are propagated when creating a table -- verify settings are propagated when creating a table
CREATE TABLE table_option_reference_2 (a int, b text) USING columnar; CREATE TABLE table_option_reference_2 (a int, b text) USING columnar;
SELECT alter_columnar_table_set('table_option_reference_2', SELECT alter_columnar_table_set('table_option_reference_2',
block_row_count => 100, chunk_row_count => 100,
stripe_row_count => 1000, stripe_row_count => 1000,
compression => 'pglz'); compression => 'pglz');
SELECT create_reference_table('table_option_reference_2'); SELECT create_reference_table('table_option_reference_2');
-- verify settings on placements -- verify settings on placements
SELECT run_command_on_placements('table_option_reference_2',$cmd$ SELECT run_command_on_placements('table_option_reference_2',$cmd$
SELECT ROW(block_row_count, stripe_row_count, compression) FROM cstore.options WHERE regclass = '%s'::regclass; SELECT ROW(chunk_row_count, stripe_row_count, compression) FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$); $cmd$);
DROP TABLE table_option_reference, table_option_reference_2; DROP TABLE table_option_reference, table_option_reference_2;

4
src/test/regress/sql/upgrade_columnar_before.sql
View File

@ -96,7 +96,7 @@ SELECT :relfilenode_pre_alter <> :relfilenode_post_alter AS relfilenode_changed;
-- --
SET cstore.stripe_row_count TO 5000; SET cstore.stripe_row_count TO 5000;
SET cstore.block_row_count TO 1000; SET cstore.chunk_row_count TO 1000;
SET cstore.compression TO 'pglz'; SET cstore.compression TO 'pglz';
CREATE TABLE test_options_1(a int, b int) USING columnar; CREATE TABLE test_options_1(a int, b int) USING columnar;
@ -104,7 +104,7 @@ INSERT INTO test_options_1 SELECT i, floor(i/1000) FROM generate_series(1, 10000
CREATE TABLE test_options_2(a int, b int) USING columnar; CREATE TABLE test_options_2(a int, b int) USING columnar;
INSERT INTO test_options_2 SELECT i, floor(i/1000) FROM generate_series(1, 10000) i; INSERT INTO test_options_2 SELECT i, floor(i/1000) FROM generate_series(1, 10000) i;
SELECT alter_columnar_table_set('test_options_2', block_row_count => 2000); SELECT alter_columnar_table_set('test_options_2', chunk_row_count => 2000);
SELECT alter_columnar_table_set('test_options_2', stripe_row_count => 6000); SELECT alter_columnar_table_set('test_options_2', stripe_row_count => 6000);
SELECT alter_columnar_table_set('test_options_2', compression => 'none'); SELECT alter_columnar_table_set('test_options_2', compression => 'none');
INSERT INTO test_options_2 SELECT i, floor(i/2000) FROM generate_series(1, 10000) i; INSERT INTO test_options_2 SELECT i, floor(i/2000) FROM generate_series(1, 10000) i;