external
/
citus
mirror of https://github.com/citusdata/citus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4379 from citusdata/cstore_chunk 

Columnar: rename block to chunk
pull/4364/head
Hadi Moshayedi 2020-12-03 09:01:10 -08:00 committed by GitHub
parent 317cf44a56 4a9aebaa7b
commit c23bdb129d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
34 changed files with 582 additions and 582 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/backend/columnar/cstore.c
View File

@ -25,11 +25,11 @@
/* Default values for option parameters */
#define DEFAULT_COMPRESSION_TYPE COMPRESSION_NONE
#define DEFAULT_STRIPE_ROW_COUNT 150000
#define DEFAULT_BLOCK_ROW_COUNT 10000
#define DEFAULT_CHUNK_ROW_COUNT 10000
int cstore_compression = DEFAULT_COMPRESSION_TYPE;
int cstore_stripe_row_count = DEFAULT_STRIPE_ROW_COUNT;
int cstore_block_row_count = DEFAULT_BLOCK_ROW_COUNT;
int cstore_chunk_row_count = DEFAULT_CHUNK_ROW_COUNT;
static const struct config_enum_entry cstore_compression_options[] =
{
@ -66,13 +66,13 @@ cstore_init()
NULL,
NULL);
DefineCustomIntVariable("cstore.block_row_count",
"Maximum number of rows per block.",
DefineCustomIntVariable("cstore.chunk_row_count",
"Maximum number of rows per chunk.",
NULL,
&cstore_block_row_count,
DEFAULT_BLOCK_ROW_COUNT,
BLOCK_ROW_COUNT_MINIMUM,
BLOCK_ROW_COUNT_MAXIMUM,
&cstore_chunk_row_count,
DEFAULT_CHUNK_ROW_COUNT,
CHUNK_ROW_COUNT_MINIMUM,
CHUNK_ROW_COUNT_MAXIMUM,
PGC_USERSET,
0,
NULL,

76
src/backend/columnar/cstore_metadata_tables.c
View File

@ -104,7 +104,7 @@ PG_FUNCTION_INFO_V1(columnar_relation_storageid);
/* constants for cstore.options */
#define Natts_cstore_options 4
#define Anum_cstore_options_regclass 1
#define Anum_cstore_options_block_row_count 2
#define Anum_cstore_options_chunk_row_count 2
#define Anum_cstore_options_stripe_row_count 3
#define Anum_cstore_options_compression 4
@ -115,7 +115,7 @@ PG_FUNCTION_INFO_V1(columnar_relation_storageid);
typedef struct FormData_cstore_options
{
Oid regclass;
int32 block_row_count;
int32 chunk_row_count;
int32 stripe_row_count;
NameData compression;
@ -132,8 +132,8 @@ typedef FormData_cstore_options *Form_cstore_options;
#define Anum_cstore_stripes_file_offset 3
#define Anum_cstore_stripes_data_length 4
#define Anum_cstore_stripes_column_count 5
#define Anum_cstore_stripes_block_count 6
#define Anum_cstore_stripes_block_row_count 7
#define Anum_cstore_stripes_chunk_count 6
#define Anum_cstore_stripes_chunk_row_count 7
#define Anum_cstore_stripes_row_count 8
/* constants for cstore_skipnodes */
@ -141,7 +141,7 @@ typedef FormData_cstore_options *Form_cstore_options;
#define Anum_cstore_skipnodes_storageid 1
#define Anum_cstore_skipnodes_stripe 2
#define Anum_cstore_skipnodes_attr 3
#define Anum_cstore_skipnodes_block 4
#define Anum_cstore_skipnodes_chunk 4
#define Anum_cstore_skipnodes_row_count 5
#define Anum_cstore_skipnodes_minimum_value 6
#define Anum_cstore_skipnodes_maximum_value 7
@ -169,7 +169,7 @@ InitColumnarOptions(Oid regclass)
}
ColumnarOptions defaultOptions = {
.blockRowCount = cstore_block_row_count,
.chunkRowCount = cstore_chunk_row_count,
.stripeRowCount = cstore_stripe_row_count,
.compressionType = cstore_compression,
};
@ -212,7 +212,7 @@ WriteColumnarOptions(Oid regclass, ColumnarOptions *options, bool overwrite)
bool nulls[Natts_cstore_options] = { 0 };
Datum values[Natts_cstore_options] = {
ObjectIdGetDatum(regclass),
Int32GetDatum(options->blockRowCount),
Int32GetDatum(options->chunkRowCount),
Int32GetDatum(options->stripeRowCount),
0, /* to be filled below */
};
@ -243,7 +243,7 @@ WriteColumnarOptions(Oid regclass, ColumnarOptions *options, bool overwrite)
/* TODO check if the options are actually different, skip if not changed */
/* update existing record */
bool update[Natts_cstore_options] = { 0 };
update[Anum_cstore_options_block_row_count - 1] = true;
update[Anum_cstore_options_chunk_row_count - 1] = true;
update[Anum_cstore_options_stripe_row_count - 1] = true;
update[Anum_cstore_options_compression - 1] = true;
@ -360,7 +360,7 @@ ReadColumnarOptions(Oid regclass, ColumnarOptions *options)
{
Form_cstore_options tupOptions = (Form_cstore_options) GETSTRUCT(heapTuple);
options->blockRowCount = tupOptions->block_row_count;
options->chunkRowCount = tupOptions->chunk_row_count;
options->stripeRowCount = tupOptions->stripe_row_count;
options->compressionType = ParseCompressionType(NameStr(tupOptions->compression));
}
@ -369,7 +369,7 @@ ReadColumnarOptions(Oid regclass, ColumnarOptions *options)
/* populate options with system defaults */
options->compressionType = cstore_compression;
options->stripeRowCount = cstore_stripe_row_count;
options->blockRowCount = cstore_block_row_count;
options->chunkRowCount = cstore_chunk_row_count;
}
systable_endscan_ordered(scanDescriptor);
@ -389,7 +389,7 @@ SaveStripeSkipList(RelFileNode relfilenode, uint64 stripe, StripeSkipList *strip
TupleDesc tupleDescriptor)
{
uint32 columnIndex = 0;
uint32 blockIndex = 0;
uint32 chunkIndex = 0;
uint32 columnCount = stripeSkipList->columnCount;
ColumnarMetapage *metapage = ReadMetapage(relfilenode, false);
@ -399,22 +399,22 @@ SaveStripeSkipList(RelFileNode relfilenode, uint64 stripe, StripeSkipList *strip
for (columnIndex = 0; columnIndex < columnCount; columnIndex++)
{
for (blockIndex = 0; blockIndex < stripeSkipList->blockCount; blockIndex++)
for (chunkIndex = 0; chunkIndex < stripeSkipList->chunkCount; chunkIndex++)
{
ColumnBlockSkipNode *skipNode =
&stripeSkipList->blockSkipNodeArray[columnIndex][blockIndex];
ColumnChunkSkipNode *skipNode =
&stripeSkipList->chunkSkipNodeArray[columnIndex][chunkIndex];
Datum values[Natts_cstore_skipnodes] = {
UInt64GetDatum(metapage->storageId),
Int64GetDatum(stripe),
Int32GetDatum(columnIndex + 1),
Int32GetDatum(blockIndex),
Int32GetDatum(chunkIndex),
Int64GetDatum(skipNode->rowCount),
0, /* to be filled below */
0, /* to be filled below */
Int64GetDatum(skipNode->valueBlockOffset),
Int64GetDatum(skipNode->valueChunkOffset),
Int64GetDatum(skipNode->valueLength),
Int64GetDatum(skipNode->existsBlockOffset),
Int64GetDatum(skipNode->existsChunkOffset),
Int64GetDatum(skipNode->existsLength),
Int32GetDatum(skipNode->valueCompressionType)
};
@ -452,7 +452,7 @@ SaveStripeSkipList(RelFileNode relfilenode, uint64 stripe, StripeSkipList *strip
*/
StripeSkipList *
ReadStripeSkipList(RelFileNode relfilenode, uint64 stripe, TupleDesc tupleDescriptor,
uint32 blockCount)
uint32 chunkCount)
{
int32 columnIndex = 0;
HeapTuple heapTuple = NULL;
@ -474,13 +474,13 @@ ReadStripeSkipList(RelFileNode relfilenode, uint64 stripe, TupleDesc tupleDescri
2, scanKey);
StripeSkipList *skipList = palloc0(sizeof(StripeSkipList));
skipList->blockCount = blockCount;
skipList->chunkCount = chunkCount;
skipList->columnCount = columnCount;
skipList->blockSkipNodeArray = palloc0(columnCount * sizeof(ColumnBlockSkipNode *));
skipList->chunkSkipNodeArray = palloc0(columnCount * sizeof(ColumnChunkSkipNode *));
for (columnIndex = 0; columnIndex < columnCount; columnIndex++)
{
skipList->blockSkipNodeArray[columnIndex] =
palloc0(blockCount * sizeof(ColumnBlockSkipNode));
skipList->chunkSkipNodeArray[columnIndex] =
palloc0(chunkCount * sizeof(ColumnChunkSkipNode));
}
while (HeapTupleIsValid(heapTuple = systable_getnext(scanDescriptor)))
@ -492,7 +492,7 @@ ReadStripeSkipList(RelFileNode relfilenode, uint64 stripe, TupleDesc tupleDescri
isNullArray);
int32 attr = DatumGetInt32(datumArray[Anum_cstore_skipnodes_attr - 1]);
int32 blockIndex = DatumGetInt32(datumArray[Anum_cstore_skipnodes_block - 1]);
int32 chunkIndex = DatumGetInt32(datumArray[Anum_cstore_skipnodes_chunk - 1]);
if (attr <= 0 || attr > columnCount)
{
@ -500,23 +500,23 @@ ReadStripeSkipList(RelFileNode relfilenode, uint64 stripe, TupleDesc tupleDescri
errdetail("Attribute number out of range: %d", attr)));
}
if (blockIndex < 0 || blockIndex >= blockCount)
if (chunkIndex < 0 || chunkIndex >= chunkCount)
{
ereport(ERROR, (errmsg("invalid stripe skipnode entry"),
errdetail("Block number out of range: %d", blockIndex)));
errdetail("Chunk number out of range: %d", chunkIndex)));
}
columnIndex = attr - 1;
ColumnBlockSkipNode *skipNode =
&skipList->blockSkipNodeArray[columnIndex][blockIndex];
ColumnChunkSkipNode *skipNode =
&skipList->chunkSkipNodeArray[columnIndex][chunkIndex];
skipNode->rowCount = DatumGetInt64(datumArray[Anum_cstore_skipnodes_row_count -
1]);
skipNode->valueBlockOffset =
skipNode->valueChunkOffset =
DatumGetInt64(datumArray[Anum_cstore_skipnodes_value_stream_offset - 1]);
skipNode->valueLength =
DatumGetInt64(datumArray[Anum_cstore_skipnodes_value_stream_length - 1]);
skipNode->existsBlockOffset =
skipNode->existsChunkOffset =
DatumGetInt64(datumArray[Anum_cstore_skipnodes_exists_stream_offset - 1]);
skipNode->existsLength =
DatumGetInt64(datumArray[Anum_cstore_skipnodes_exists_stream_length - 1]);
@ -565,8 +565,8 @@ InsertStripeMetadataRow(uint64 storageId, StripeMetadata *stripe)
Int64GetDatum(stripe->fileOffset),
Int64GetDatum(stripe->dataLength),
Int32GetDatum(stripe->columnCount),
Int32GetDatum(stripe->blockCount),
Int32GetDatum(stripe->blockRowCount),
Int32GetDatum(stripe->chunkCount),
Int32GetDatum(stripe->chunkRowCount),
Int64GetDatum(stripe->rowCount)
};
@ -670,7 +670,7 @@ GetHighestUsedAddressAndId(uint64 storageId,
StripeMetadata
ReserveStripe(Relation rel, uint64 sizeBytes,
uint64 rowCount, uint64 columnCount,
uint64 blockCount, uint64 blockRowCount)
uint64 chunkCount, uint64 chunkRowCount)
{
StripeMetadata stripe = { 0 };
uint64 currLogicalHigh = 0;
@ -720,8 +720,8 @@ ReserveStripe(Relation rel, uint64 sizeBytes,
stripe.fileOffset = resLogicalStart;
stripe.dataLength = sizeBytes;
stripe.blockCount = blockCount;
stripe.blockRowCount = blockRowCount;
stripe.chunkCount = chunkCount;
stripe.chunkRowCount = chunkRowCount;
stripe.columnCount = columnCount;
stripe.rowCount = rowCount;
stripe.id = highestId + 1;
@ -773,10 +773,10 @@ ReadDataFileStripeList(uint64 storageId, Snapshot snapshot)
datumArray[Anum_cstore_stripes_data_length - 1]);
stripeMetadata->columnCount = DatumGetInt32(
datumArray[Anum_cstore_stripes_column_count - 1]);
stripeMetadata->blockCount = DatumGetInt32(
datumArray[Anum_cstore_stripes_block_count - 1]);
stripeMetadata->blockRowCount = DatumGetInt32(
datumArray[Anum_cstore_stripes_block_row_count - 1]);
stripeMetadata->chunkCount = DatumGetInt32(
datumArray[Anum_cstore_stripes_chunk_count - 1]);
stripeMetadata->chunkRowCount = DatumGetInt32(
datumArray[Anum_cstore_stripes_chunk_row_count - 1]);
stripeMetadata->rowCount = DatumGetInt64(
datumArray[Anum_cstore_stripes_row_count - 1]);

344
src/backend/columnar/cstore_reader.c
View File

@ -4,7 +4,7 @@
*
* This file contains function definitions for reading cstore files. This
* includes the logic for reading file level metadata, reading row stripes,
* and skipping unrelated row blocks and columns.
* and skipping unrelated row chunks and columns.
*
* Copyright (c) 2016, Citus Data, Inc.
*
@ -46,23 +46,23 @@ static StripeBuffers * LoadFilteredStripeBuffers(Relation relation,
List *projectedColumnList,
List *whereClauseList);
static void ReadStripeNextRow(StripeBuffers *stripeBuffers, List *projectedColumnList,
uint64 blockIndex, uint64 blockRowIndex,
BlockData *blockData, Datum *columnValues,
uint64 chunkIndex, uint64 chunkRowIndex,
ChunkData *chunkData, Datum *columnValues,
bool *columnNulls);
static ColumnBuffers * LoadColumnBuffers(Relation relation,
ColumnBlockSkipNode *blockSkipNodeArray,
uint32 blockCount, uint64 stripeOffset,
ColumnChunkSkipNode *chunkSkipNodeArray,
uint32 chunkCount, uint64 stripeOffset,
Form_pg_attribute attributeForm);
static bool * SelectedBlockMask(StripeSkipList *stripeSkipList,
static bool * SelectedChunkMask(StripeSkipList *stripeSkipList,
List *projectedColumnList, List *whereClauseList);
static List * BuildRestrictInfoList(List *whereClauseList);
static Node * BuildBaseConstraint(Var *variable);
static OpExpr * MakeOpExpression(Var *variable, int16 strategyNumber);
static Oid GetOperatorByType(Oid typeId, Oid accessMethodId, int16 strategyNumber);
static void UpdateConstraint(Node *baseConstraint, Datum minValue, Datum maxValue);
static StripeSkipList * SelectedBlockSkipList(StripeSkipList *stripeSkipList,
static StripeSkipList * SelectedChunkSkipList(StripeSkipList *stripeSkipList,
bool *projectedColumnMask,
bool *selectedBlockMask);
bool *selectedChunkMask);
static uint32 StripeSkipListRowCount(StripeSkipList *stripeSkipList);
static bool * ProjectedColumnMask(uint32 columnCount, List *projectedColumnList);
static void DeserializeBoolArray(StringInfo boolArrayBuffer, bool *boolArray,
@ -71,7 +71,7 @@ static void DeserializeDatumArray(StringInfo datumBuffer, bool *existsArray,
uint32 datumCount, bool datumTypeByValue,
int datumTypeLength, char datumTypeAlign,
Datum *datumArray);
static BlockData * DeserializeBlockData(StripeBuffers *stripeBuffers, uint64 blockIndex,
static ChunkData * DeserializeChunkData(StripeBuffers *stripeBuffers, uint64 chunkIndex,
uint32 rowCount, TupleDesc tupleDescriptor,
List *projectedColumnList);
static Datum ColumnDefaultValue(TupleConstr *tupleConstraints,
@ -106,8 +106,8 @@ CStoreBeginRead(Relation relation, TupleDesc tupleDescriptor,
readState->stripeReadRowCount = 0;
readState->tupleDescriptor = tupleDescriptor;
readState->stripeReadContext = stripeReadContext;
readState->blockData = NULL;
readState->deserializedBlockIndex = -1;
readState->chunkData = NULL;
readState->deserializedChunkIndex = -1;
return readState;
}
@ -126,7 +126,7 @@ CStoreReadNextRow(TableReadState *readState, Datum *columnValues, bool *columnNu
/*
* If no stripes are loaded, load the next non-empty stripe. Note that when
* loading stripes, we skip over blocks whose contents can be filtered with
* loading stripes, we skip over chunks whose contents can be filtered with
* the query's restriction qualifiers. So, even when a stripe is physically
* not empty, we may end up loading it as an empty stripe.
*/
@ -143,7 +143,7 @@ CStoreReadNextRow(TableReadState *readState, Datum *columnValues, bool *columnNu
oldContext = MemoryContextSwitchTo(readState->stripeReadContext);
MemoryContextReset(readState->stripeReadContext);
readState->blockData = NULL;
readState->chunkData = NULL;
stripeMetadata = list_nth(stripeMetadataList, readState->readStripeCount);
StripeBuffers *stripeBuffers = LoadFilteredStripeBuffers(readState->relation,
@ -163,44 +163,44 @@ CStoreReadNextRow(TableReadState *readState, Datum *columnValues, bool *columnNu
{
readState->stripeBuffers = stripeBuffers;
readState->stripeReadRowCount = 0;
readState->deserializedBlockIndex = -1;
readState->deserializedChunkIndex = -1;
break;
}
}
uint32 blockIndex = readState->stripeReadRowCount / stripeMetadata->blockRowCount;
uint32 blockRowIndex = readState->stripeReadRowCount % stripeMetadata->blockRowCount;
uint32 chunkIndex = readState->stripeReadRowCount / stripeMetadata->chunkRowCount;
uint32 chunkRowIndex = readState->stripeReadRowCount % stripeMetadata->chunkRowCount;
if (blockIndex != readState->deserializedBlockIndex)
if (chunkIndex != readState->deserializedChunkIndex)
{
uint32 blockRowCount = 0;
uint32 chunkRowCount = 0;
uint32 stripeRowCount = stripeMetadata->rowCount;
uint32 lastBlockIndex = stripeRowCount / stripeMetadata->blockRowCount;
if (blockIndex == lastBlockIndex)
uint32 lastChunkIndex = stripeRowCount / stripeMetadata->chunkRowCount;
if (chunkIndex == lastChunkIndex)
{
blockRowCount = stripeRowCount % stripeMetadata->blockRowCount;
chunkRowCount = stripeRowCount % stripeMetadata->chunkRowCount;
}
else
{
blockRowCount = stripeMetadata->blockRowCount;
chunkRowCount = stripeMetadata->chunkRowCount;
}
oldContext = MemoryContextSwitchTo(readState->stripeReadContext);
FreeBlockData(readState->blockData);
readState->blockData =
DeserializeBlockData(readState->stripeBuffers, blockIndex,
blockRowCount, readState->tupleDescriptor,
FreeChunkData(readState->chunkData);
readState->chunkData =
DeserializeChunkData(readState->stripeBuffers, chunkIndex,
chunkRowCount, readState->tupleDescriptor,
readState->projectedColumnList);
MemoryContextSwitchTo(oldContext);
readState->deserializedBlockIndex = blockIndex;
readState->deserializedChunkIndex = chunkIndex;
}
ReadStripeNextRow(readState->stripeBuffers, readState->projectedColumnList,
blockIndex, blockRowIndex, readState->blockData,
chunkIndex, chunkRowIndex, readState->chunkData,
columnValues, columnNulls);
/*
@ -241,68 +241,68 @@ CStoreEndRead(TableReadState *readState)
/*
* CreateEmptyBlockDataArray creates data buffers to keep deserialized exist and
* CreateEmptyChunkDataArray creates data buffers to keep deserialized exist and
* value arrays for requested columns in columnMask.
*/
BlockData *
CreateEmptyBlockData(uint32 columnCount, bool *columnMask, uint32 blockRowCount)
ChunkData *
CreateEmptyChunkData(uint32 columnCount, bool *columnMask, uint32 chunkRowCount)
{
uint32 columnIndex = 0;
BlockData *blockData = palloc0(sizeof(BlockData));
blockData->existsArray = palloc0(columnCount * sizeof(bool *));
blockData->valueArray = palloc0(columnCount * sizeof(Datum *));
blockData->valueBufferArray = palloc0(columnCount * sizeof(StringInfo));
blockData->columnCount = columnCount;
blockData->rowCount = blockRowCount;
ChunkData *chunkData = palloc0(sizeof(ChunkData));
chunkData->existsArray = palloc0(columnCount * sizeof(bool *));
chunkData->valueArray = palloc0(columnCount * sizeof(Datum *));
chunkData->valueBufferArray = palloc0(columnCount * sizeof(StringInfo));
chunkData->columnCount = columnCount;
chunkData->rowCount = chunkRowCount;
/* allocate block memory for deserialized data */
/* allocate chunk memory for deserialized data */
for (columnIndex = 0; columnIndex < columnCount; columnIndex++)
{
if (columnMask[columnIndex])
{
blockData->existsArray[columnIndex] = palloc0(blockRowCount * sizeof(bool));
blockData->valueArray[columnIndex] = palloc0(blockRowCount * sizeof(Datum));
blockData->valueBufferArray[columnIndex] = NULL;
chunkData->existsArray[columnIndex] = palloc0(chunkRowCount * sizeof(bool));
chunkData->valueArray[columnIndex] = palloc0(chunkRowCount * sizeof(Datum));
chunkData->valueBufferArray[columnIndex] = NULL;
}
}
return blockData;
return chunkData;
}
/*
* FreeBlockData deallocates data buffers to keep deserialized exist and
* FreeChunkData deallocates data buffers to keep deserialized exist and
* value arrays for requested columns in columnMask.
* ColumnBlockData->serializedValueBuffer lives in memory read/write context
* ColumnChunkData->serializedValueBuffer lives in memory read/write context
* so it is deallocated automatically when the context is deleted.
*/
void
FreeBlockData(BlockData *blockData)
FreeChunkData(ChunkData *chunkData)
{
uint32 columnIndex = 0;
if (blockData == NULL)
if (chunkData == NULL)
{
return;
}
for (columnIndex = 0; columnIndex < blockData->columnCount; columnIndex++)
for (columnIndex = 0; columnIndex < chunkData->columnCount; columnIndex++)
{
if (blockData->existsArray[columnIndex] != NULL)
if (chunkData->existsArray[columnIndex] != NULL)
{
pfree(blockData->existsArray[columnIndex]);
pfree(chunkData->existsArray[columnIndex]);
}
if (blockData->valueArray[columnIndex] != NULL)
if (chunkData->valueArray[columnIndex] != NULL)
{
pfree(blockData->valueArray[columnIndex]);
pfree(chunkData->valueArray[columnIndex]);
}
}
pfree(blockData->existsArray);
pfree(blockData->valueArray);
pfree(blockData);
pfree(chunkData->existsArray);
pfree(chunkData->valueArray);
pfree(chunkData);
}
@ -326,7 +326,7 @@ CStoreTableRowCount(Relation relation)
/*
* LoadFilteredStripeBuffers reads serialized stripe data from the given file.
* The function skips over blocks whose rows are refuted by restriction qualifiers,
* The function skips over chunks whose rows are refuted by restriction qualifiers,
* and only loads columns that are projected in the query.
*/
static StripeBuffers *
@ -342,14 +342,14 @@ LoadFilteredStripeBuffers(Relation relation, StripeMetadata *stripeMetadata,
StripeSkipList *stripeSkipList = ReadStripeSkipList(relation->rd_node,
stripeMetadata->id,
tupleDescriptor,
stripeMetadata->blockCount);
stripeMetadata->chunkCount);
bool *selectedBlockMask = SelectedBlockMask(stripeSkipList, projectedColumnList,
bool *selectedChunkMask = SelectedChunkMask(stripeSkipList, projectedColumnList,
whereClauseList);
StripeSkipList *selectedBlockSkipList =
SelectedBlockSkipList(stripeSkipList, projectedColumnMask,
selectedBlockMask);
StripeSkipList *selectedChunkSkipList =
SelectedChunkSkipList(stripeSkipList, projectedColumnMask,
selectedChunkMask);
/* load column data for projected columns */
ColumnBuffers **columnBuffersArray = palloc0(columnCount * sizeof(ColumnBuffers *));
@ -358,13 +358,13 @@ LoadFilteredStripeBuffers(Relation relation, StripeMetadata *stripeMetadata,
{
if (projectedColumnMask[columnIndex])
{
ColumnBlockSkipNode *blockSkipNode =
selectedBlockSkipList->blockSkipNodeArray[columnIndex];
ColumnChunkSkipNode *chunkSkipNode =
selectedChunkSkipList->chunkSkipNodeArray[columnIndex];
Form_pg_attribute attributeForm = TupleDescAttr(tupleDescriptor, columnIndex);
uint32 blockCount = selectedBlockSkipList->blockCount;
uint32 chunkCount = selectedChunkSkipList->chunkCount;
ColumnBuffers *columnBuffers = LoadColumnBuffers(relation, blockSkipNode,
blockCount,
ColumnBuffers *columnBuffers = LoadColumnBuffers(relation, chunkSkipNode,
chunkCount,
stripeMetadata->fileOffset,
attributeForm);
@ -374,7 +374,7 @@ LoadFilteredStripeBuffers(Relation relation, StripeMetadata *stripeMetadata,
StripeBuffers *stripeBuffers = palloc0(sizeof(StripeBuffers));
stripeBuffers->columnCount = columnCount;
stripeBuffers->rowCount = StripeSkipListRowCount(selectedBlockSkipList);
stripeBuffers->rowCount = StripeSkipListRowCount(selectedChunkSkipList);
stripeBuffers->columnBuffersArray = columnBuffersArray;
return stripeBuffers;
@ -388,8 +388,8 @@ LoadFilteredStripeBuffers(Relation relation, StripeMetadata *stripeMetadata,
*/
static void
ReadStripeNextRow(StripeBuffers *stripeBuffers, List *projectedColumnList,
uint64 blockIndex, uint64 blockRowIndex,
BlockData *blockData, Datum *columnValues,
uint64 chunkIndex, uint64 chunkRowIndex,
ChunkData *chunkData, Datum *columnValues,
bool *columnNulls)
{
ListCell *projectedColumnCell = NULL;
@ -402,9 +402,9 @@ ReadStripeNextRow(StripeBuffers *stripeBuffers, List *projectedColumnList,
Var *projectedColumn = lfirst(projectedColumnCell);
uint32 columnIndex = projectedColumn->varattno - 1;
if (blockData->existsArray[columnIndex][blockRowIndex])
if (chunkData->existsArray[columnIndex][chunkRowIndex])
{
columnValues[columnIndex] = blockData->valueArray[columnIndex][blockRowIndex];
columnValues[columnIndex] = chunkData->valueArray[columnIndex][chunkRowIndex];
columnNulls[columnIndex] = false;
}
}
@ -413,73 +413,73 @@ ReadStripeNextRow(StripeBuffers *stripeBuffers, List *projectedColumnList,
/*
* LoadColumnBuffers reads serialized column data from the given file. These
* column data are laid out as sequential blocks in the file; and block positions
* and lengths are retrieved from the column block skip node array.
* column data are laid out as sequential chunks in the file; and chunk positions
* and lengths are retrieved from the column chunk skip node array.
*/
static ColumnBuffers *
LoadColumnBuffers(Relation relation, ColumnBlockSkipNode *blockSkipNodeArray,
uint32 blockCount, uint64 stripeOffset,
LoadColumnBuffers(Relation relation, ColumnChunkSkipNode *chunkSkipNodeArray,
uint32 chunkCount, uint64 stripeOffset,
Form_pg_attribute attributeForm)
{
uint32 blockIndex = 0;
ColumnBlockBuffers **blockBuffersArray =
palloc0(blockCount * sizeof(ColumnBlockBuffers *));
uint32 chunkIndex = 0;
ColumnChunkBuffers **chunkBuffersArray =
palloc0(chunkCount * sizeof(ColumnChunkBuffers *));
for (blockIndex = 0; blockIndex < blockCount; blockIndex++)
for (chunkIndex = 0; chunkIndex < chunkCount; chunkIndex++)
{
blockBuffersArray[blockIndex] = palloc0(sizeof(ColumnBlockBuffers));
chunkBuffersArray[chunkIndex] = palloc0(sizeof(ColumnChunkBuffers));
}
/*
* We first read the "exists" blocks. We don't read "values" array here,
* because "exists" blocks are stored sequentially on disk, and we want to
* We first read the "exists" chunks. We don't read "values" array here,
* because "exists" chunks are stored sequentially on disk, and we want to
* minimize disk seeks.
*/
for (blockIndex = 0; blockIndex < blockCount; blockIndex++)
for (chunkIndex = 0; chunkIndex < chunkCount; chunkIndex++)
{
ColumnBlockSkipNode *blockSkipNode = &blockSkipNodeArray[blockIndex];
uint64 existsOffset = stripeOffset + blockSkipNode->existsBlockOffset;
ColumnChunkSkipNode *chunkSkipNode = &chunkSkipNodeArray[chunkIndex];
uint64 existsOffset = stripeOffset + chunkSkipNode->existsChunkOffset;
StringInfo rawExistsBuffer = ReadFromSmgr(relation, existsOffset,
blockSkipNode->existsLength);
chunkSkipNode->existsLength);
blockBuffersArray[blockIndex]->existsBuffer = rawExistsBuffer;
chunkBuffersArray[chunkIndex]->existsBuffer = rawExistsBuffer;
}
/* then read "values" blocks, which are also stored sequentially on disk */
for (blockIndex = 0; blockIndex < blockCount; blockIndex++)
/* then read "values" chunks, which are also stored sequentially on disk */
for (chunkIndex = 0; chunkIndex < chunkCount; chunkIndex++)
{
ColumnBlockSkipNode *blockSkipNode = &blockSkipNodeArray[blockIndex];
CompressionType compressionType = blockSkipNode->valueCompressionType;
uint64 valueOffset = stripeOffset + blockSkipNode->valueBlockOffset;
ColumnChunkSkipNode *chunkSkipNode = &chunkSkipNodeArray[chunkIndex];
CompressionType compressionType = chunkSkipNode->valueCompressionType;
uint64 valueOffset = stripeOffset + chunkSkipNode->valueChunkOffset;
StringInfo rawValueBuffer = ReadFromSmgr(relation, valueOffset,
blockSkipNode->valueLength);
chunkSkipNode->valueLength);
blockBuffersArray[blockIndex]->valueBuffer = rawValueBuffer;
blockBuffersArray[blockIndex]->valueCompressionType = compressionType;
chunkBuffersArray[chunkIndex]->valueBuffer = rawValueBuffer;
chunkBuffersArray[chunkIndex]->valueCompressionType = compressionType;
}
ColumnBuffers *columnBuffers = palloc0(sizeof(ColumnBuffers));
columnBuffers->blockBuffersArray = blockBuffersArray;
columnBuffers->chunkBuffersArray = chunkBuffersArray;
return columnBuffers;
}
/*
* SelectedBlockMask walks over each column's blocks and checks if a block can
* SelectedChunkMask walks over each column's chunks and checks if a chunk can
* be filtered without reading its data. The filtering happens when all rows in
* the block can be refuted by the given qualifier conditions.
* the chunk can be refuted by the given qualifier conditions.
*/
static bool *
SelectedBlockMask(StripeSkipList *stripeSkipList, List *projectedColumnList,
SelectedChunkMask(StripeSkipList *stripeSkipList, List *projectedColumnList,
List *whereClauseList)
{
ListCell *columnCell = NULL;
uint32 blockIndex = 0;
uint32 chunkIndex = 0;
List *restrictInfoList = BuildRestrictInfoList(whereClauseList);
bool *selectedBlockMask = palloc0(stripeSkipList->blockCount * sizeof(bool));
memset(selectedBlockMask, true, stripeSkipList->blockCount * sizeof(bool));
bool *selectedChunkMask = palloc0(stripeSkipList->chunkCount * sizeof(bool));
memset(selectedChunkMask, true, stripeSkipList->chunkCount * sizeof(bool));
foreach(columnCell, projectedColumnList)
{
@ -496,24 +496,24 @@ SelectedBlockMask(StripeSkipList *stripeSkipList, List *projectedColumnList,
}
Node *baseConstraint = BuildBaseConstraint(column);
for (blockIndex = 0; blockIndex < stripeSkipList->blockCount; blockIndex++)
for (chunkIndex = 0; chunkIndex < stripeSkipList->chunkCount; chunkIndex++)
{
bool predicateRefuted = false;
ColumnBlockSkipNode *blockSkipNodeArray =
stripeSkipList->blockSkipNodeArray[columnIndex];
ColumnBlockSkipNode *blockSkipNode = &blockSkipNodeArray[blockIndex];
ColumnChunkSkipNode *chunkSkipNodeArray =
stripeSkipList->chunkSkipNodeArray[columnIndex];
ColumnChunkSkipNode *chunkSkipNode = &chunkSkipNodeArray[chunkIndex];
/*
* A column block with comparable data type can miss min/max values
* if all values in the block are NULL.
* A column chunk with comparable data type can miss min/max values
* if all values in the chunk are NULL.
*/
if (!blockSkipNode->hasMinMax)
if (!chunkSkipNode->hasMinMax)
{
continue;
}
UpdateConstraint(baseConstraint, blockSkipNode->minimumValue,
blockSkipNode->maximumValue);
UpdateConstraint(baseConstraint, chunkSkipNode->minimumValue,
chunkSkipNode->maximumValue);
List *constraintList = list_make1(baseConstraint);
#if (PG_VERSION_NUM >= 100000)
@ -524,12 +524,12 @@ SelectedBlockMask(StripeSkipList *stripeSkipList, List *projectedColumnList,
#endif
if (predicateRefuted)
{
selectedBlockMask[blockIndex] = false;
selectedChunkMask[chunkIndex] = false;
}
}
}
return selectedBlockMask;
return selectedChunkMask;
}
@ -596,7 +596,7 @@ BuildRestrictInfoList(List *whereClauseList)
/*
* BuildBaseConstraint builds and returns a base constraint. This constraint
* implements an expression in the form of (var <= max && var >= min), where
* min and max values represent a block's min and max values. These block
* min and max values represent a chunk's min and max values. These chunk
* values are filled in after the constraint is built. This function is based
* on a similar function from CitusDB's shard pruning logic.
*/
@ -700,83 +700,83 @@ UpdateConstraint(Node *baseConstraint, Datum minValue, Datum maxValue)
/*
* SelectedBlockSkipList constructs a new StripeSkipList in which the
* non-selected blocks are removed from the given stripeSkipList.
* SelectedChunkSkipList constructs a new StripeSkipList in which the
* non-selected chunks are removed from the given stripeSkipList.
*/
static StripeSkipList *
SelectedBlockSkipList(StripeSkipList *stripeSkipList, bool *projectedColumnMask,
bool *selectedBlockMask)
SelectedChunkSkipList(StripeSkipList *stripeSkipList, bool *projectedColumnMask,
bool *selectedChunkMask)
{
uint32 selectedBlockCount = 0;
uint32 blockIndex = 0;
uint32 selectedChunkCount = 0;
uint32 chunkIndex = 0;
uint32 columnIndex = 0;
uint32 columnCount = stripeSkipList->columnCount;
for (blockIndex = 0; blockIndex < stripeSkipList->blockCount; blockIndex++)
for (chunkIndex = 0; chunkIndex < stripeSkipList->chunkCount; chunkIndex++)
{
if (selectedBlockMask[blockIndex])
if (selectedChunkMask[chunkIndex])
{
selectedBlockCount++;
selectedChunkCount++;
}
}
ColumnBlockSkipNode **selectedBlockSkipNodeArray = palloc0(columnCount *
sizeof(ColumnBlockSkipNode
ColumnChunkSkipNode **selectedChunkSkipNodeArray = palloc0(columnCount *
sizeof(ColumnChunkSkipNode
*));
for (columnIndex = 0; columnIndex < columnCount; columnIndex++)
{
uint32 selectedBlockIndex = 0;
uint32 selectedChunkIndex = 0;
bool firstColumn = columnIndex == 0;
/* first column's block skip node is always read */
/* first column's chunk skip node is always read */
if (!projectedColumnMask[columnIndex] && !firstColumn)
{
selectedBlockSkipNodeArray[columnIndex] = NULL;
selectedChunkSkipNodeArray[columnIndex] = NULL;
continue;
}
Assert(stripeSkipList->blockSkipNodeArray[columnIndex] != NULL);
Assert(stripeSkipList->chunkSkipNodeArray[columnIndex] != NULL);
selectedBlockSkipNodeArray[columnIndex] = palloc0(selectedBlockCount *
sizeof(ColumnBlockSkipNode));
selectedChunkSkipNodeArray[columnIndex] = palloc0(selectedChunkCount *
sizeof(ColumnChunkSkipNode));
for (blockIndex = 0; blockIndex < stripeSkipList->blockCount; blockIndex++)
for (chunkIndex = 0; chunkIndex < stripeSkipList->chunkCount; chunkIndex++)
{
if (selectedBlockMask[blockIndex])
if (selectedChunkMask[chunkIndex])
{
selectedBlockSkipNodeArray[columnIndex][selectedBlockIndex] =
stripeSkipList->blockSkipNodeArray[columnIndex][blockIndex];
selectedBlockIndex++;
selectedChunkSkipNodeArray[columnIndex][selectedChunkIndex] =
stripeSkipList->chunkSkipNodeArray[columnIndex][chunkIndex];
selectedChunkIndex++;
}
}
}
StripeSkipList *SelectedBlockSkipList = palloc0(sizeof(StripeSkipList));
SelectedBlockSkipList->blockSkipNodeArray = selectedBlockSkipNodeArray;
SelectedBlockSkipList->blockCount = selectedBlockCount;
SelectedBlockSkipList->columnCount = stripeSkipList->columnCount;
StripeSkipList *SelectedChunkSkipList = palloc0(sizeof(StripeSkipList));
SelectedChunkSkipList->chunkSkipNodeArray = selectedChunkSkipNodeArray;
SelectedChunkSkipList->chunkCount = selectedChunkCount;
SelectedChunkSkipList->columnCount = stripeSkipList->columnCount;
return SelectedBlockSkipList;
return SelectedChunkSkipList;
}
/*
* StripeSkipListRowCount counts the number of rows in the given stripeSkipList.
* To do this, the function finds the first column, and sums up row counts across
* all blocks for that column.
* all chunks for that column.
*/
static uint32
StripeSkipListRowCount(StripeSkipList *stripeSkipList)
{
uint32 stripeSkipListRowCount = 0;
uint32 blockIndex = 0;
ColumnBlockSkipNode *firstColumnSkipNodeArray =
stripeSkipList->blockSkipNodeArray[0];
uint32 chunkIndex = 0;
ColumnChunkSkipNode *firstColumnSkipNodeArray =
stripeSkipList->chunkSkipNodeArray[0];
for (blockIndex = 0; blockIndex < stripeSkipList->blockCount; blockIndex++)
for (chunkIndex = 0; chunkIndex < stripeSkipList->chunkCount; chunkIndex++)
{
uint32 blockRowCount = firstColumnSkipNodeArray[blockIndex].rowCount;
stripeSkipListRowCount += blockRowCount;
uint32 chunkRowCount = firstColumnSkipNodeArray[chunkIndex].rowCount;
stripeSkipListRowCount += chunkRowCount;
}
return stripeSkipListRowCount;
@ -878,21 +878,21 @@ DeserializeDatumArray(StringInfo datumBuffer, bool *existsArray, uint32 datumCou
/*
* DeserializeBlockData deserializes requested data block for all columns and
* stores in blockDataArray. It uncompresses serialized data if necessary. The
* function also deallocates data buffers used for previous block, and compressed
* data buffers for the current block which will not be needed again. If a column
* DeserializeChunkData deserializes requested data chunk for all columns and
* stores in chunkDataArray. It uncompresses serialized data if necessary. The
* function also deallocates data buffers used for previous chunk, and compressed
* data buffers for the current chunk which will not be needed again. If a column
* data is not present serialized buffer, then default value (or null) is used
* to fill value array.
*/
static BlockData *
DeserializeBlockData(StripeBuffers *stripeBuffers, uint64 blockIndex,
static ChunkData *
DeserializeChunkData(StripeBuffers *stripeBuffers, uint64 chunkIndex,
uint32 rowCount, TupleDesc tupleDescriptor,
List *projectedColumnList)
{
int columnIndex = 0;
bool *columnMask = ProjectedColumnMask(tupleDescriptor->natts, projectedColumnList);
BlockData *blockData = CreateEmptyBlockData(tupleDescriptor->natts, columnMask,
ChunkData *chunkData = CreateEmptyChunkData(tupleDescriptor->natts, columnMask,
rowCount);
for (columnIndex = 0; columnIndex < stripeBuffers->columnCount; columnIndex++)
@ -908,30 +908,30 @@ DeserializeBlockData(StripeBuffers *stripeBuffers, uint64 blockIndex,
if (columnBuffers != NULL)
{
ColumnBlockBuffers *blockBuffers =
columnBuffers->blockBuffersArray[blockIndex];
ColumnChunkBuffers *chunkBuffers =
columnBuffers->chunkBuffersArray[chunkIndex];
/* decompress and deserialize current block's data */
StringInfo valueBuffer = DecompressBuffer(blockBuffers->valueBuffer,
blockBuffers->valueCompressionType);
/* decompress and deserialize current chunk's data */
StringInfo valueBuffer = DecompressBuffer(chunkBuffers->valueBuffer,
chunkBuffers->valueCompressionType);
if (blockBuffers->valueCompressionType != COMPRESSION_NONE)
if (chunkBuffers->valueCompressionType != COMPRESSION_NONE)
{
/* compressed data is not needed anymore */
pfree(blockBuffers->valueBuffer->data);
pfree(blockBuffers->valueBuffer);
pfree(chunkBuffers->valueBuffer->data);
pfree(chunkBuffers->valueBuffer);
}
DeserializeBoolArray(blockBuffers->existsBuffer,
blockData->existsArray[columnIndex],
DeserializeBoolArray(chunkBuffers->existsBuffer,
chunkData->existsArray[columnIndex],
rowCount);
DeserializeDatumArray(valueBuffer, blockData->existsArray[columnIndex],
DeserializeDatumArray(valueBuffer, chunkData->existsArray[columnIndex],
rowCount, attributeForm->attbyval,
attributeForm->attlen, attributeForm->attalign,
blockData->valueArray[columnIndex]);
chunkData->valueArray[columnIndex]);
/* store current block's data buffer to be freed at next block read */
blockData->valueBufferArray[columnIndex] = valueBuffer;
/* store current chunk's data buffer to be freed at next chunk read */
chunkData->valueBufferArray[columnIndex] = valueBuffer;
}
else if (columnAdded)
{
@ -948,19 +948,19 @@ DeserializeBlockData(StripeBuffers *stripeBuffers, uint64 blockIndex,
for (rowIndex = 0; rowIndex < rowCount; rowIndex++)
{
blockData->existsArray[columnIndex][rowIndex] = true;
blockData->valueArray[columnIndex][rowIndex] = defaultValue;
chunkData->existsArray[columnIndex][rowIndex] = true;
chunkData->valueArray[columnIndex][rowIndex] = defaultValue;
}
}
else
{
memset(blockData->existsArray[columnIndex], false,
memset(chunkData->existsArray[columnIndex], false,
rowCount * sizeof(bool));
}
}
}
return blockData;
return chunkData;
}

46
src/backend/columnar/cstore_tableam.c
View File

@ -629,7 +629,7 @@ cstore_relation_copy_for_cluster(Relation OldHeap, Relation NewHeap,
TableWriteState *writeState = CStoreBeginWrite(NewHeap->rd_node,
cstoreOptions.compressionType,
cstoreOptions.stripeRowCount,
cstoreOptions.blockRowCount,
cstoreOptions.chunkRowCount,
targetDesc);
TableReadState *readState = CStoreBeginRead(OldHeap, sourceDesc,
@ -691,9 +691,9 @@ LogRelationStats(Relation rel, int elevel)
int compressionStats[COMPRESSION_COUNT] = { 0 };
uint64 totalStripeLength = 0;
uint64 tupleCount = 0;
uint64 blockCount = 0;
uint64 chunkCount = 0;
TupleDesc tupdesc = RelationGetDescr(rel);
uint64 droppedBlocksWithData = 0;
uint64 droppedChunksWithData = 0;
List *stripeList = StripesForRelfilenode(relfilenode);
int stripeCount = list_length(stripeList);
@ -703,24 +703,24 @@ LogRelationStats(Relation rel, int elevel)
StripeMetadata *stripe = lfirst(stripeMetadataCell);
StripeSkipList *skiplist = ReadStripeSkipList(relfilenode, stripe->id,
RelationGetDescr(rel),
stripe->blockCount);
stripe->chunkCount);
for (uint32 column = 0; column < skiplist->columnCount; column++)
{
bool attrDropped = tupdesc->attrs[column].attisdropped;
for (uint32 block = 0; block < skiplist->blockCount; block++)
for (uint32 chunk = 0; chunk < skiplist->chunkCount; chunk++)
{
ColumnBlockSkipNode *skipnode =
&skiplist->blockSkipNodeArray[column][block];
ColumnChunkSkipNode *skipnode =
&skiplist->chunkSkipNodeArray[column][chunk];
/* ignore zero length blocks for dropped attributes */
/* ignore zero length chunks for dropped attributes */
if (skipnode->valueLength > 0)
{
compressionStats[skipnode->valueCompressionType]++;
blockCount++;
chunkCount++;
if (attrDropped)
{
droppedBlocksWithData++;
droppedChunksWithData++;
}
}
}
@ -746,9 +746,9 @@ LogRelationStats(Relation rel, int elevel)
tupleCount, stripeCount,
stripeCount ? tupleCount / stripeCount : 0);
appendStringInfo(infoBuf,
"block count: %ld"
"chunk count: %ld"
", containing data for dropped columns: %ld",
blockCount, droppedBlocksWithData);
chunkCount, droppedChunksWithData);
for (int compressionType = 0; compressionType < COMPRESSION_COUNT; compressionType++)
{
appendStringInfo(infoBuf,
@ -914,7 +914,7 @@ cstore_scan_analyze_next_tuple(TableScanDesc scan, TransactionId OldestXmin,
* based access methods where it chooses random pages, and then reads
* tuples from those pages.
*
* We could do something like that here by choosing sample stripes or blocks,
* We could do something like that here by choosing sample stripes or chunks,
* but getting that correct might need quite some work. Since cstore_fdw's
* ANALYZE scanned all rows, as a starter we do the same here and scan all
* rows.
@ -1307,11 +1307,11 @@ CitusCreateAlterColumnarTableSet(char *qualifiedRelationName,
appendStringInfo(&buf,
"SELECT alter_columnar_table_set(%s, "
"block_row_count => %d, "
"chunk_row_count => %d, "
"stripe_row_count => %lu, "
"compression => %s);",
quote_literal_cstr(qualifiedRelationName),
options->blockRowCount,
options->chunkRowCount,
options->stripeRowCount,
quote_literal_cstr(CompressionTypeStr(options->compressionType)));
@ -1427,7 +1427,7 @@ ColumnarGetTableOptionsDDL(Oid relationId)
* sql syntax:
* pg_catalog.alter_columnar_table_set(
* table_name regclass,
* block_row_count int DEFAULT NULL,
* chunk_row_count int DEFAULT NULL,
* stripe_row_count int DEFAULT NULL,
* compression name DEFAULT null)
*
@ -1459,12 +1459,12 @@ alter_columnar_table_set(PG_FUNCTION_ARGS)
ereport(ERROR, (errmsg("unable to read current options for table")));
}
/* block_row_count => not null */
/* chunk_row_count => not null */
if (!PG_ARGISNULL(1))
{
options.blockRowCount = PG_GETARG_INT32(1);
options.chunkRowCount = PG_GETARG_INT32(1);
ereport(DEBUG1,
(errmsg("updating block row count to %d", options.blockRowCount)));
(errmsg("updating chunk row count to %d", options.chunkRowCount)));
}
/* stripe_row_count => not null */
@ -1520,7 +1520,7 @@ alter_columnar_table_set(PG_FUNCTION_ARGS)
* pg_catalog.alter_columnar_table_re
* teset(
* table_name regclass,
* block_row_count bool DEFAULT FALSE,
* chunk_row_count bool DEFAULT FALSE,
* stripe_row_count bool DEFAULT FALSE,
* compression bool DEFAULT FALSE)
*
@ -1549,12 +1549,12 @@ alter_columnar_table_reset(PG_FUNCTION_ARGS)
ereport(ERROR, (errmsg("unable to read current options for table")));
}
/* block_row_count => true */
/* chunk_row_count => true */
if (!PG_ARGISNULL(1) && PG_GETARG_BOOL(1))
{
options.blockRowCount = cstore_block_row_count;
options.chunkRowCount = cstore_chunk_row_count;
ereport(DEBUG1,
(errmsg("resetting block row count to %d", options.blockRowCount)));
(errmsg("resetting chunk row count to %d", options.chunkRowCount)));
}
/* stripe_row_count => true */

230
src/backend/columnar/cstore_writer.c
View File

@ -4,7 +4,7 @@
*
* This file contains function definitions for writing cstore files. This
* includes the logic for writing file level metadata, writing row stripes,
* and calculating block skip nodes.
* and calculating chunk skip nodes.
*
* Copyright (c) 2016, Citus Data, Inc.
*
@ -33,19 +33,19 @@
#include "columnar/cstore_version_compat.h"
static StripeBuffers * CreateEmptyStripeBuffers(uint32 stripeMaxRowCount,
uint32 blockRowCount,
uint32 chunkRowCount,
uint32 columnCount);
static StripeSkipList * CreateEmptyStripeSkipList(uint32 stripeMaxRowCount,
uint32 blockRowCount,
uint32 chunkRowCount,
uint32 columnCount);
static void FlushStripe(TableWriteState *writeState);
static StringInfo SerializeBoolArray(bool *boolArray, uint32 boolArrayLength);
static void SerializeSingleDatum(StringInfo datumBuffer, Datum datum,
bool datumTypeByValue, int datumTypeLength,
char datumTypeAlign);
static void SerializeBlockData(TableWriteState *writeState, uint32 blockIndex,
static void SerializeChunkData(TableWriteState *writeState, uint32 chunkIndex,
uint32 rowCount);
static void UpdateBlockSkipNodeMinMax(ColumnBlockSkipNode *blockSkipNode,
static void UpdateChunkSkipNodeMinMax(ColumnChunkSkipNode *chunkSkipNode,
Datum columnValue, bool columnTypeByValue,
int columnTypeLength, Oid columnCollation,
FmgrInfo *comparisonFunction);
@ -62,7 +62,7 @@ static StringInfo CopyStringInfo(StringInfo sourceString);
TableWriteState *
CStoreBeginWrite(RelFileNode relfilenode,
CompressionType compressionType,
uint64 stripeMaxRowCount, uint32 blockRowCount,
uint64 stripeMaxRowCount, uint32 chunkRowCount,
TupleDesc tupleDescriptor)
{
/* get comparison function pointers for each of the columns */
@ -97,20 +97,20 @@ CStoreBeginWrite(RelFileNode relfilenode,
bool *columnMaskArray = palloc(columnCount * sizeof(bool));
memset(columnMaskArray, true, columnCount);
BlockData *blockData = CreateEmptyBlockData(columnCount, columnMaskArray,
blockRowCount);
ChunkData *chunkData = CreateEmptyChunkData(columnCount, columnMaskArray,
chunkRowCount);
TableWriteState *writeState = palloc0(sizeof(TableWriteState));
writeState->relfilenode = relfilenode;
writeState->compressionType = compressionType;
writeState->stripeMaxRowCount = stripeMaxRowCount;
writeState->blockRowCount = blockRowCount;
writeState->chunkRowCount = chunkRowCount;
writeState->tupleDescriptor = CreateTupleDescCopy(tupleDescriptor);
writeState->comparisonFunctionArray = comparisonFunctionArray;
writeState->stripeBuffers = NULL;
writeState->stripeSkipList = NULL;
writeState->stripeWriteContext = stripeWriteContext;
writeState->blockData = blockData;
writeState->chunkData = chunkData;
writeState->compressionBuffer = NULL;
writeState->perTupleContext = AllocSetContextCreate(CurrentMemoryContext,
"CStore per tuple context",
@ -124,8 +124,8 @@ CStoreBeginWrite(RelFileNode relfilenode,
* CStoreWriteRow adds a row to the cstore file. If the stripe is not initialized,
* we create structures to hold stripe data and skip list. Then, we serialize and
* append data to serialized value buffer for each of the columns and update
* corresponding skip nodes. Then, whole block data is compressed at every
* rowBlockCount insertion. Then, if row count exceeds stripeMaxRowCount, we flush
* corresponding skip nodes. Then, whole chunk data is compressed at every
* rowChunkCount insertion. Then, if row count exceeds stripeMaxRowCount, we flush
* the stripe, and add its metadata to the table footer.
*/
void
@ -135,16 +135,16 @@ CStoreWriteRow(TableWriteState *writeState, Datum *columnValues, bool *columnNul
StripeBuffers *stripeBuffers = writeState->stripeBuffers;
StripeSkipList *stripeSkipList = writeState->stripeSkipList;
uint32 columnCount = writeState->tupleDescriptor->natts;
const uint32 blockRowCount = writeState->blockRowCount;
BlockData *blockData = writeState->blockData;
const uint32 chunkRowCount = writeState->chunkRowCount;
ChunkData *chunkData = writeState->chunkData;
MemoryContext oldContext = MemoryContextSwitchTo(writeState->stripeWriteContext);
if (stripeBuffers == NULL)
{
stripeBuffers = CreateEmptyStripeBuffers(writeState->stripeMaxRowCount,
blockRowCount, columnCount);
chunkRowCount, columnCount);
stripeSkipList = CreateEmptyStripeSkipList(writeState->stripeMaxRowCount,
blockRowCount, columnCount);
chunkRowCount, columnCount);
writeState->stripeBuffers = stripeBuffers;
writeState->stripeSkipList = stripeSkipList;
writeState->compressionBuffer = makeStringInfo();
@ -155,22 +155,22 @@ CStoreWriteRow(TableWriteState *writeState, Datum *columnValues, bool *columnNul
*/
for (columnIndex = 0; columnIndex < columnCount; columnIndex++)
{
blockData->valueBufferArray[columnIndex] = makeStringInfo();
chunkData->valueBufferArray[columnIndex] = makeStringInfo();
}
}
uint32 blockIndex = stripeBuffers->rowCount / blockRowCount;
uint32 blockRowIndex = stripeBuffers->rowCount % blockRowCount;
uint32 chunkIndex = stripeBuffers->rowCount / chunkRowCount;
uint32 chunkRowIndex = stripeBuffers->rowCount % chunkRowCount;
for (columnIndex = 0; columnIndex < columnCount; columnIndex++)
{
ColumnBlockSkipNode **blockSkipNodeArray = stripeSkipList->blockSkipNodeArray;
ColumnBlockSkipNode *blockSkipNode =
&blockSkipNodeArray[columnIndex][blockIndex];
ColumnChunkSkipNode **chunkSkipNodeArray = stripeSkipList->chunkSkipNodeArray;
ColumnChunkSkipNode *chunkSkipNode =
&chunkSkipNodeArray[columnIndex][chunkIndex];
if (columnNulls[columnIndex])
{
blockData->existsArray[columnIndex][blockRowIndex] = false;
chunkData->existsArray[columnIndex][chunkRowIndex] = false;
}
else
{
@ -183,26 +183,26 @@ CStoreWriteRow(TableWriteState *writeState, Datum *columnValues, bool *columnNul
Oid columnCollation = attributeForm->attcollation;
char columnTypeAlign = attributeForm->attalign;
blockData->existsArray[columnIndex][blockRowIndex] = true;
chunkData->existsArray[columnIndex][chunkRowIndex] = true;
SerializeSingleDatum(blockData->valueBufferArray[columnIndex],
SerializeSingleDatum(chunkData->valueBufferArray[columnIndex],
columnValues[columnIndex], columnTypeByValue,
columnTypeLength, columnTypeAlign);
UpdateBlockSkipNodeMinMax(blockSkipNode, columnValues[columnIndex],
UpdateChunkSkipNodeMinMax(chunkSkipNode, columnValues[columnIndex],
columnTypeByValue, columnTypeLength,
columnCollation, comparisonFunction);
}
blockSkipNode->rowCount++;
chunkSkipNode->rowCount++;
}
stripeSkipList->blockCount = blockIndex + 1;
stripeSkipList->chunkCount = chunkIndex + 1;
/* last row of the block is inserted serialize the block */
if (blockRowIndex == blockRowCount - 1)
/* last row of the chunk is inserted serialize the chunk */
if (chunkRowIndex == chunkRowCount - 1)
{
SerializeBlockData(writeState, blockIndex, blockRowCount);
SerializeChunkData(writeState, chunkIndex, chunkRowCount);
}
stripeBuffers->rowCount++;
@ -228,7 +228,7 @@ CStoreEndWrite(TableWriteState *writeState)
MemoryContextDelete(writeState->stripeWriteContext);
pfree(writeState->comparisonFunctionArray);
FreeBlockData(writeState->blockData);
FreeChunkData(writeState->chunkData);
pfree(writeState);
}
@ -258,29 +258,29 @@ CStoreFlushPendingWrites(TableWriteState *writeState)
* column count.
*/
static StripeBuffers *
CreateEmptyStripeBuffers(uint32 stripeMaxRowCount, uint32 blockRowCount,
CreateEmptyStripeBuffers(uint32 stripeMaxRowCount, uint32 chunkRowCount,
uint32 columnCount)
{
uint32 columnIndex = 0;
uint32 maxBlockCount = (stripeMaxRowCount / blockRowCount) + 1;
uint32 maxChunkCount = (stripeMaxRowCount / chunkRowCount) + 1;
ColumnBuffers **columnBuffersArray = palloc0(columnCount * sizeof(ColumnBuffers *));
for (columnIndex = 0; columnIndex < columnCount; columnIndex++)
{
uint32 blockIndex = 0;
ColumnBlockBuffers **blockBuffersArray =
palloc0(maxBlockCount * sizeof(ColumnBlockBuffers *));
uint32 chunkIndex = 0;
ColumnChunkBuffers **chunkBuffersArray =
palloc0(maxChunkCount * sizeof(ColumnChunkBuffers *));
for (blockIndex = 0; blockIndex < maxBlockCount; blockIndex++)
for (chunkIndex = 0; chunkIndex < maxChunkCount; chunkIndex++)
{
blockBuffersArray[blockIndex] = palloc0(sizeof(ColumnBlockBuffers));
blockBuffersArray[blockIndex]->existsBuffer = NULL;
blockBuffersArray[blockIndex]->valueBuffer = NULL;
blockBuffersArray[blockIndex]->valueCompressionType = COMPRESSION_NONE;
chunkBuffersArray[chunkIndex] = palloc0(sizeof(ColumnChunkBuffers));
chunkBuffersArray[chunkIndex]->existsBuffer = NULL;
chunkBuffersArray[chunkIndex]->valueBuffer = NULL;
chunkBuffersArray[chunkIndex]->valueCompressionType = COMPRESSION_NONE;
}
columnBuffersArray[columnIndex] = palloc0(sizeof(ColumnBuffers));
columnBuffersArray[columnIndex]->blockBuffersArray = blockBuffersArray;
columnBuffersArray[columnIndex]->chunkBuffersArray = chunkBuffersArray;
}
StripeBuffers *stripeBuffers = palloc0(sizeof(StripeBuffers));
@ -294,28 +294,28 @@ CreateEmptyStripeBuffers(uint32 stripeMaxRowCount, uint32 blockRowCount,
/*
* CreateEmptyStripeSkipList allocates an empty StripeSkipList structure with
* the given column count. This structure has enough blocks to hold statistics
* the given column count. This structure has enough chunks to hold statistics
* for stripeMaxRowCount rows.
*/
static StripeSkipList *
CreateEmptyStripeSkipList(uint32 stripeMaxRowCount, uint32 blockRowCount,
CreateEmptyStripeSkipList(uint32 stripeMaxRowCount, uint32 chunkRowCount,
uint32 columnCount)
{
uint32 columnIndex = 0;
uint32 maxBlockCount = (stripeMaxRowCount / blockRowCount) + 1;
uint32 maxChunkCount = (stripeMaxRowCount / chunkRowCount) + 1;
ColumnBlockSkipNode **blockSkipNodeArray =
palloc0(columnCount * sizeof(ColumnBlockSkipNode *));
ColumnChunkSkipNode **chunkSkipNodeArray =
palloc0(columnCount * sizeof(ColumnChunkSkipNode *));
for (columnIndex = 0; columnIndex < columnCount; columnIndex++)
{
blockSkipNodeArray[columnIndex] =
palloc0(maxBlockCount * sizeof(ColumnBlockSkipNode));
chunkSkipNodeArray[columnIndex] =
palloc0(maxChunkCount * sizeof(ColumnChunkSkipNode));
}
StripeSkipList *stripeSkipList = palloc0(sizeof(StripeSkipList));
stripeSkipList->columnCount = columnCount;
stripeSkipList->blockCount = 0;
stripeSkipList->blockSkipNodeArray = blockSkipNodeArray;
stripeSkipList->chunkCount = 0;
stripeSkipList->chunkSkipNodeArray = chunkSkipNodeArray;
return stripeSkipList;
}
@ -332,9 +332,9 @@ WriteToSmgr(Relation rel, uint64 logicalOffset, char *data, uint32 dataLength)
SmgrAddr addr = logical_to_smgr(logicalOffset);
RelationOpenSmgr(rel);
BlockNumber nblocks = smgrnblocks(rel->rd_smgr, MAIN_FORKNUM);
BlockNumber nblocks PG_USED_FOR_ASSERTS_ONLY =
smgrnblocks(rel->rd_smgr, MAIN_FORKNUM);
Assert(addr.blockno < nblocks);
(void) nblocks; /* keep compiler quiet */
RelationCloseSmgr(rel);
buffer = ReadBuffer(rel, addr.blockno);
@ -397,7 +397,7 @@ WriteToSmgr(Relation rel, uint64 logicalOffset, char *data, uint32 dataLength)
/*
* FlushStripe flushes current stripe data into the file. The function first ensures
* the last data block for each column is properly serialized and compressed. Then,
* the last data chunk for each column is properly serialized and compressed. Then,
* the function creates the skip list and footer buffers. Finally, the function
* flushes the skip list, data, and footer buffers to the file.
*/
@ -406,16 +406,16 @@ FlushStripe(TableWriteState *writeState)
{
StripeMetadata stripeMetadata = { 0 };
uint32 columnIndex = 0;
uint32 blockIndex = 0;
uint32 chunkIndex = 0;
StripeBuffers *stripeBuffers = writeState->stripeBuffers;
StripeSkipList *stripeSkipList = writeState->stripeSkipList;
ColumnBlockSkipNode **columnSkipNodeArray = stripeSkipList->blockSkipNodeArray;
ColumnChunkSkipNode **columnSkipNodeArray = stripeSkipList->chunkSkipNodeArray;
TupleDesc tupleDescriptor = writeState->tupleDescriptor;
uint32 columnCount = tupleDescriptor->natts;
uint32 blockCount = stripeSkipList->blockCount;
uint32 blockRowCount = writeState->blockRowCount;
uint32 lastBlockIndex = stripeBuffers->rowCount / blockRowCount;
uint32 lastBlockRowCount = stripeBuffers->rowCount % blockRowCount;
uint32 chunkCount = stripeSkipList->chunkCount;
uint32 chunkRowCount = writeState->chunkRowCount;
uint32 lastChunkIndex = stripeBuffers->rowCount / chunkRowCount;
uint32 lastChunkRowCount = stripeBuffers->rowCount % chunkRowCount;
uint64 stripeSize = 0;
uint64 stripeRowCount = 0;
@ -426,64 +426,64 @@ FlushStripe(TableWriteState *writeState)
Relation relation = relation_open(relationId, NoLock);
/*
* check if the last block needs serialization , the last block was not serialized
* check if the last chunk needs serialization , the last chunk was not serialized
* if it was not full yet, e.g. (rowCount > 0)
*/
if (lastBlockRowCount > 0)
if (lastChunkRowCount > 0)
{
SerializeBlockData(writeState, lastBlockIndex, lastBlockRowCount);
SerializeChunkData(writeState, lastChunkIndex, lastChunkRowCount);
}
/* update buffer sizes in stripe skip list */
for (columnIndex = 0; columnIndex < columnCount; columnIndex++)
{
ColumnBlockSkipNode *blockSkipNodeArray = columnSkipNodeArray[columnIndex];
ColumnChunkSkipNode *chunkSkipNodeArray = columnSkipNodeArray[columnIndex];
ColumnBuffers *columnBuffers = stripeBuffers->columnBuffersArray[columnIndex];
for (blockIndex = 0; blockIndex < blockCount; blockIndex++)
for (chunkIndex = 0; chunkIndex < chunkCount; chunkIndex++)
{
ColumnBlockBuffers *blockBuffers =
columnBuffers->blockBuffersArray[blockIndex];
uint64 existsBufferSize = blockBuffers->existsBuffer->len;
ColumnBlockSkipNode *blockSkipNode = &blockSkipNodeArray[blockIndex];
ColumnChunkBuffers *chunkBuffers =
columnBuffers->chunkBuffersArray[chunkIndex];
uint64 existsBufferSize = chunkBuffers->existsBuffer->len;
ColumnChunkSkipNode *chunkSkipNode = &chunkSkipNodeArray[chunkIndex];
blockSkipNode->existsBlockOffset = stripeSize;
blockSkipNode->existsLength = existsBufferSize;
chunkSkipNode->existsChunkOffset = stripeSize;
chunkSkipNode->existsLength = existsBufferSize;
stripeSize += existsBufferSize;
}
for (blockIndex = 0; blockIndex < blockCount; blockIndex++)
for (chunkIndex = 0; chunkIndex < chunkCount; chunkIndex++)
{
ColumnBlockBuffers *blockBuffers =
columnBuffers->blockBuffersArray[blockIndex];
uint64 valueBufferSize = blockBuffers->valueBuffer->len;
CompressionType valueCompressionType = blockBuffers->valueCompressionType;
ColumnBlockSkipNode *blockSkipNode = &blockSkipNodeArray[blockIndex];
ColumnChunkBuffers *chunkBuffers =
columnBuffers->chunkBuffersArray[chunkIndex];
uint64 valueBufferSize = chunkBuffers->valueBuffer->len;
CompressionType valueCompressionType = chunkBuffers->valueCompressionType;
ColumnChunkSkipNode *chunkSkipNode = &chunkSkipNodeArray[chunkIndex];
blockSkipNode->valueBlockOffset = stripeSize;
blockSkipNode->valueLength = valueBufferSize;
blockSkipNode->valueCompressionType = valueCompressionType;
chunkSkipNode->valueChunkOffset = stripeSize;
chunkSkipNode->valueLength = valueBufferSize;
chunkSkipNode->valueCompressionType = valueCompressionType;
stripeSize += valueBufferSize;
}
}
for (blockIndex = 0; blockIndex < blockCount; blockIndex++)
for (chunkIndex = 0; chunkIndex < chunkCount; chunkIndex++)
{
stripeRowCount +=
stripeSkipList->blockSkipNodeArray[0][blockIndex].rowCount;
stripeSkipList->chunkSkipNodeArray[0][chunkIndex].rowCount;
}
stripeMetadata = ReserveStripe(relation, stripeSize,
stripeRowCount, columnCount, blockCount,
blockRowCount);
stripeRowCount, columnCount, chunkCount,
chunkRowCount);
uint64 currentFileOffset = stripeMetadata.fileOffset;
/*
* Each stripe has only one section:
* Data section, in which we store data for each column continuously.
* We store data for each for each column in blocks. For each block, we
* We store data for each for each column in chunks. For each chunk, we
* store two buffers: "exists" buffer, and "value" buffer. "exists" buffer
* tells which values are not NULL. "value" buffer contains values for
* present values. For each column, we first store all "exists" buffers,
@ -495,22 +495,22 @@ FlushStripe(TableWriteState *writeState)
{
ColumnBuffers *columnBuffers = stripeBuffers->columnBuffersArray[columnIndex];
for (blockIndex = 0; blockIndex < stripeSkipList->blockCount; blockIndex++)
for (chunkIndex = 0; chunkIndex < stripeSkipList->chunkCount; chunkIndex++)
{
ColumnBlockBuffers *blockBuffers =
columnBuffers->blockBuffersArray[blockIndex];
StringInfo existsBuffer = blockBuffers->existsBuffer;
ColumnChunkBuffers *chunkBuffers =
columnBuffers->chunkBuffersArray[chunkIndex];
StringInfo existsBuffer = chunkBuffers->existsBuffer;
WriteToSmgr(relation, currentFileOffset,
existsBuffer->data, existsBuffer->len);
currentFileOffset += existsBuffer->len;
}
for (blockIndex = 0; blockIndex < stripeSkipList->blockCount; blockIndex++)
for (chunkIndex = 0; chunkIndex < stripeSkipList->chunkCount; chunkIndex++)
{
ColumnBlockBuffers *blockBuffers =
columnBuffers->blockBuffersArray[blockIndex];
StringInfo valueBuffer = blockBuffers->valueBuffer;
ColumnChunkBuffers *chunkBuffers =
columnBuffers->chunkBuffersArray[chunkIndex];
StringInfo valueBuffer = chunkBuffers->valueBuffer;
WriteToSmgr(relation, currentFileOffset,
valueBuffer->data, valueBuffer->len);
@ -596,15 +596,15 @@ SerializeSingleDatum(StringInfo datumBuffer, Datum datum, bool datumTypeByValue,
/*
* SerializeBlockData serializes and compresses block data at given block index with given
* SerializeChunkData serializes and compresses chunk data at given chunk index with given
* compression type for every column.
*/
static void
SerializeBlockData(TableWriteState *writeState, uint32 blockIndex, uint32 rowCount)
SerializeChunkData(TableWriteState *writeState, uint32 chunkIndex, uint32 rowCount)
{
uint32 columnIndex = 0;
StripeBuffers *stripeBuffers = writeState->stripeBuffers;
BlockData *blockData = writeState->blockData;
ChunkData *chunkData = writeState->chunkData;
CompressionType requestedCompressionType = writeState->compressionType;
const uint32 columnCount = stripeBuffers->columnCount;
StringInfo compressionBuffer = writeState->compressionBuffer;
@ -613,10 +613,10 @@ SerializeBlockData(TableWriteState *writeState, uint32 blockIndex, uint32 rowCou
for (columnIndex = 0; columnIndex < columnCount; columnIndex++)
{
ColumnBuffers *columnBuffers = stripeBuffers->columnBuffersArray[columnIndex];
ColumnBlockBuffers *blockBuffers = columnBuffers->blockBuffersArray[blockIndex];
ColumnChunkBuffers *chunkBuffers = columnBuffers->chunkBuffersArray[chunkIndex];
blockBuffers->existsBuffer =
SerializeBoolArray(blockData->existsArray[columnIndex], rowCount);
chunkBuffers->existsBuffer =
SerializeBoolArray(chunkData->existsArray[columnIndex], rowCount);
}
/*
@ -626,10 +626,10 @@ SerializeBlockData(TableWriteState *writeState, uint32 blockIndex, uint32 rowCou
for (columnIndex = 0; columnIndex < columnCount; columnIndex++)
{
ColumnBuffers *columnBuffers = stripeBuffers->columnBuffersArray[columnIndex];
ColumnBlockBuffers *blockBuffers = columnBuffers->blockBuffersArray[blockIndex];
ColumnChunkBuffers *chunkBuffers = columnBuffers->chunkBuffersArray[chunkIndex];
CompressionType actualCompressionType = COMPRESSION_NONE;
StringInfo serializedValueBuffer = blockData->valueBufferArray[columnIndex];
StringInfo serializedValueBuffer = chunkData->valueBufferArray[columnIndex];
/* the only other supported compression type is pg_lz for now */
Assert(requestedCompressionType == COMPRESSION_NONE ||
@ -648,29 +648,29 @@ SerializeBlockData(TableWriteState *writeState, uint32 blockIndex, uint32 rowCou
}
/* store (compressed) value buffer */
blockBuffers->valueCompressionType = actualCompressionType;
blockBuffers->valueBuffer = CopyStringInfo(serializedValueBuffer);
chunkBuffers->valueCompressionType = actualCompressionType;
chunkBuffers->valueBuffer = CopyStringInfo(serializedValueBuffer);
/* valueBuffer needs to be reset for next block's data */
resetStringInfo(blockData->valueBufferArray[columnIndex]);
/* valueBuffer needs to be reset for next chunk's data */
resetStringInfo(chunkData->valueBufferArray[columnIndex]);
}
}
/*
* UpdateBlockSkipNodeMinMax takes the given column value, and checks if this
* UpdateChunkSkipNodeMinMax takes the given column value, and checks if this
* value falls outside the range of minimum/maximum values of the given column
* block skip node. If it does, the function updates the column block skip node
* chunk skip node. If it does, the function updates the column chunk skip node
* accordingly.
*/
static void
UpdateBlockSkipNodeMinMax(ColumnBlockSkipNode *blockSkipNode, Datum columnValue,
UpdateChunkSkipNodeMinMax(ColumnChunkSkipNode *chunkSkipNode, Datum columnValue,
bool columnTypeByValue, int columnTypeLength,
Oid columnCollation, FmgrInfo *comparisonFunction)
{
bool hasMinMax = blockSkipNode->hasMinMax;
Datum previousMinimum = blockSkipNode->minimumValue;
Datum previousMaximum = blockSkipNode->maximumValue;
bool hasMinMax = chunkSkipNode->hasMinMax;
Datum previousMinimum = chunkSkipNode->minimumValue;
Datum previousMaximum = chunkSkipNode->maximumValue;
Datum currentMinimum = 0;
Datum currentMaximum = 0;
@ -715,9 +715,9 @@ UpdateBlockSkipNodeMinMax(ColumnBlockSkipNode *blockSkipNode, Datum columnValue,
}
}
blockSkipNode->hasMinMax = true;
blockSkipNode->minimumValue = currentMinimum;
blockSkipNode->maximumValue = currentMaximum;
chunkSkipNode->hasMinMax = true;
chunkSkipNode->minimumValue = currentMinimum;
chunkSkipNode->maximumValue = currentMaximum;
}

12
src/backend/columnar/sql/columnar--9.5-1--10.0-1.sql
View File

@ -7,7 +7,7 @@ CREATE SEQUENCE storageid_seq MINVALUE 10000000000 NO CYCLE;
CREATE TABLE options (
regclass regclass NOT NULL PRIMARY KEY,
block_row_count int NOT NULL,
chunk_row_count int NOT NULL,
stripe_row_count int NOT NULL,
compression name NOT NULL
) WITH (user_catalog_table = true);
@ -20,8 +20,8 @@ CREATE TABLE cstore_stripes (
file_offset bigint NOT NULL,
data_length bigint NOT NULL,
column_count int NOT NULL,
block_count int NOT NULL,
block_row_count int NOT NULL,
chunk_count int NOT NULL,
chunk_row_count int NOT NULL,
row_count bigint NOT NULL,
PRIMARY KEY (storageid, stripe)
) WITH (user_catalog_table = true);
@ -32,7 +32,7 @@ CREATE TABLE cstore_skipnodes (
storageid bigint NOT NULL,
stripe bigint NOT NULL,
attr int NOT NULL,
block int NOT NULL,
chunk int NOT NULL,
row_count bigint NOT NULL,
minimum_value bytea,
maximum_value bytea,
@ -41,11 +41,11 @@ CREATE TABLE cstore_skipnodes (
exists_stream_offset bigint NOT NULL,
exists_stream_length bigint NOT NULL,
value_compression_type int NOT NULL,
PRIMARY KEY (storageid, stripe, attr, block),
PRIMARY KEY (storageid, stripe, attr, chunk),
FOREIGN KEY (storageid, stripe) REFERENCES cstore_stripes(storageid, stripe) ON DELETE CASCADE
) WITH (user_catalog_table = true);
COMMENT ON TABLE cstore_skipnodes IS 'CStore per block metadata';
COMMENT ON TABLE cstore_skipnodes IS 'CStore per chunk metadata';
DO $proc$
BEGIN

4
src/backend/columnar/sql/downgrades/columnar--10.0-1--9.5-1.sql
View File

@ -9,13 +9,13 @@ IF substring(current_Setting('server_version'), '\d+')::int >= 12 THEN
EXECUTE $$
DROP FUNCTION pg_catalog.alter_columnar_table_reset(
table_name regclass,
block_row_count bool,
chunk_row_count bool,
stripe_row_count bool,
compression bool);
DROP FUNCTION pg_catalog.alter_columnar_table_set(
table_name regclass,
block_row_count int,
chunk_row_count int,
stripe_row_count int,
compression name);

4
src/backend/columnar/sql/udfs/alter_columnar_table_reset/10.0-1.sql
View File

@ -1,6 +1,6 @@
CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_reset(
table_name regclass,
block_row_count bool DEFAULT false,
chunk_row_count bool DEFAULT false,
stripe_row_count bool DEFAULT false,
compression bool DEFAULT false)
RETURNS void
@ -9,7 +9,7 @@ AS 'MODULE_PATHNAME', 'alter_columnar_table_reset';
COMMENT ON FUNCTION pg_catalog.alter_columnar_table_reset(
table_name regclass,
block_row_count bool,
chunk_row_count bool,
stripe_row_count bool,
compression bool)
IS 'reset on or more options on a cstore table to the system defaults';

4
src/backend/columnar/sql/udfs/alter_columnar_table_reset/latest.sql
View File

@ -1,6 +1,6 @@
CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_reset(
table_name regclass,
block_row_count bool DEFAULT false,
chunk_row_count bool DEFAULT false,
stripe_row_count bool DEFAULT false,
compression bool DEFAULT false)
RETURNS void
@ -9,7 +9,7 @@ AS 'MODULE_PATHNAME', 'alter_columnar_table_reset';
COMMENT ON FUNCTION pg_catalog.alter_columnar_table_reset(
table_name regclass,
block_row_count bool,
chunk_row_count bool,
stripe_row_count bool,
compression bool)
IS 'reset on or more options on a cstore table to the system defaults';

4
src/backend/columnar/sql/udfs/alter_columnar_table_set/10.0-1.sql
View File

@ -1,6 +1,6 @@
CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_set(
table_name regclass,
block_row_count int DEFAULT NULL,
chunk_row_count int DEFAULT NULL,
stripe_row_count int DEFAULT NULL,
compression name DEFAULT null)
RETURNS void
@ -9,7 +9,7 @@ AS 'MODULE_PATHNAME', 'alter_columnar_table_set';
COMMENT ON FUNCTION pg_catalog.alter_columnar_table_set(
table_name regclass,
block_row_count int,
chunk_row_count int,
stripe_row_count int,
compression name)
IS 'set one or more options on a cstore table, when set to NULL no change is made';

4
src/backend/columnar/sql/udfs/alter_columnar_table_set/latest.sql
View File

@ -1,6 +1,6 @@
CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_set(
table_name regclass,
block_row_count int DEFAULT NULL,
chunk_row_count int DEFAULT NULL,
stripe_row_count int DEFAULT NULL,
compression name DEFAULT null)
RETURNS void
@ -9,7 +9,7 @@ AS 'MODULE_PATHNAME', 'alter_columnar_table_set';
COMMENT ON FUNCTION pg_catalog.alter_columnar_table_set(
table_name regclass,
block_row_count int,
chunk_row_count int,
stripe_row_count int,
compression name)
IS 'set one or more options on a cstore table, when set to NULL no change is made';

4
src/backend/columnar/sql/udfs/cstore_ensure_objects_exist/10.0-1.sql
View File

@ -29,12 +29,12 @@ IF NOT EXISTS (SELECT 1 FROM pg_am WHERE amname = 'columnar') THEN
ALTER EXTENSION citus ADD ACCESS METHOD columnar;
ALTER EXTENSION citus ADD FUNCTION pg_catalog.alter_columnar_table_set(
table_name regclass,
block_row_count int,
chunk_row_count int,
stripe_row_count int,
compression name);
ALTER EXTENSION citus ADD FUNCTION pg_catalog.alter_columnar_table_reset(
table_name regclass,
block_row_count bool,
chunk_row_count bool,
stripe_row_count bool,
compression bool);

4
src/backend/columnar/sql/udfs/cstore_ensure_objects_exist/latest.sql
View File

@ -29,12 +29,12 @@ IF NOT EXISTS (SELECT 1 FROM pg_am WHERE amname = 'columnar') THEN
ALTER EXTENSION citus ADD ACCESS METHOD columnar;
ALTER EXTENSION citus ADD FUNCTION pg_catalog.alter_columnar_table_set(
table_name regclass,
block_row_count int,
chunk_row_count int,
stripe_row_count int,
compression name);
ALTER EXTENSION citus ADD FUNCTION pg_catalog.alter_columnar_table_reset(
table_name regclass,
block_row_count bool,
chunk_row_count bool,
stripe_row_count bool,
compression bool);

2
src/backend/columnar/write_state_management.c
View File

@ -185,7 +185,7 @@ cstore_init_write_state(Relation relation, TupleDesc tupdesc,
stackEntry->writeState = CStoreBeginWrite(relation->rd_node,
cstoreOptions.compressionType,
cstoreOptions.stripeRowCount,
cstoreOptions.blockRowCount,
cstoreOptions.chunkRowCount,
tupdesc);
stackEntry->subXid = currentSubXid;
stackEntry->next = hashEntry->writeStateStack;

80
src/include/columnar/cstore.h
View File

@ -27,13 +27,13 @@
/* Defines for valid option names */
#define OPTION_NAME_COMPRESSION_TYPE "compression"
#define OPTION_NAME_STRIPE_ROW_COUNT "stripe_row_count"
#define OPTION_NAME_BLOCK_ROW_COUNT "block_row_count"
#define OPTION_NAME_CHUNK_ROW_COUNT "chunk_row_count"
/* Limits for option parameters */
#define STRIPE_ROW_COUNT_MINIMUM 1000
#define STRIPE_ROW_COUNT_MAXIMUM 10000000
#define BLOCK_ROW_COUNT_MINIMUM 1000
#define BLOCK_ROW_COUNT_MAXIMUM 100000
#define CHUNK_ROW_COUNT_MINIMUM 1000
#define CHUNK_ROW_COUNT_MAXIMUM 100000
/* String representations of compression types */
#define COMPRESSION_STRING_NONE "none"
@ -70,7 +70,7 @@ typedef enum
typedef struct ColumnarOptions
{
uint64 stripeRowCount;
uint32 blockRowCount;
uint32 chunkRowCount;
CompressionType compressionType;
} ColumnarOptions;
@ -84,8 +84,8 @@ typedef struct StripeMetadata
uint64 fileOffset;
uint64 dataLength;
uint32 columnCount;
uint32 blockCount;
uint32 blockRowCount;
uint32 chunkCount;
uint32 chunkRowCount;
uint64 rowCount;
uint64 id;
} StripeMetadata;
@ -98,10 +98,10 @@ typedef struct DataFileMetadata
} DataFileMetadata;
/* ColumnBlockSkipNode contains statistics for a ColumnBlockData. */
typedef struct ColumnBlockSkipNode
/* ColumnChunkSkipNode contains statistics for a ColumnChunkData. */
typedef struct ColumnChunkSkipNode
{
/* statistics about values of a column block */
/* statistics about values of a column chunk */
bool hasMinMax;
Datum minimumValue;
Datum maximumValue;
@ -109,39 +109,39 @@ typedef struct ColumnBlockSkipNode
/*
* Offsets and sizes of value and exists streams in the column data.
* These enable us to skip reading suppressed row blocks, and start reading
* a block without reading previous blocks.
* These enable us to skip reading suppressed row chunks, and start reading
* a chunk without reading previous chunks.
*/
uint64 valueBlockOffset;
uint64 valueChunkOffset;
uint64 valueLength;
uint64 existsBlockOffset;
uint64 existsChunkOffset;
uint64 existsLength;
CompressionType valueCompressionType;
} ColumnBlockSkipNode;
} ColumnChunkSkipNode;
/*
* StripeSkipList can be used for skipping row blocks. It contains a column block
* skip node for each block of each column. blockSkipNodeArray[column][block]
* is the entry for the specified column block.
* StripeSkipList can be used for skipping row chunks. It contains a column chunk
* skip node for each chunk of each column. chunkSkipNodeArray[column][chunk]
* is the entry for the specified column chunk.
*/
typedef struct StripeSkipList
{
ColumnBlockSkipNode **blockSkipNodeArray;
ColumnChunkSkipNode **chunkSkipNodeArray;
uint32 columnCount;
uint32 blockCount;
uint32 chunkCount;
} StripeSkipList;
/*
* BlockData represents a block of data for multiple columns. valueArray stores
* ChunkData represents a chunk of data for multiple columns. valueArray stores
* the values of data, and existsArray stores whether a value is present.
* valueBuffer is used to store (uncompressed) serialized values
* referenced by Datum's in valueArray. It is only used for by-reference Datum's.
* There is a one-to-one correspondence between valueArray and existsArray.
*/
typedef struct BlockData
typedef struct ChunkData
{
uint32 rowCount;
uint32 columnCount;
@ -155,31 +155,31 @@ typedef struct BlockData
/* valueBuffer keeps actual data for type-by-reference datums from valueArray. */
StringInfo *valueBufferArray;
} BlockData;
} ChunkData;
/*
* ColumnBlockBuffers represents a block of serialized data in a column.
* ColumnChunkBuffers represents a chunk of serialized data in a column.
* valueBuffer stores the serialized values of data, and existsBuffer stores
* serialized value of presence information. valueCompressionType contains
* compression type if valueBuffer is compressed. Finally rowCount has
* the number of rows in this block.
* the number of rows in this chunk.
*/
typedef struct ColumnBlockBuffers
typedef struct ColumnChunkBuffers
{
StringInfo existsBuffer;
StringInfo valueBuffer;
CompressionType valueCompressionType;
} ColumnBlockBuffers;
} ColumnChunkBuffers;
/*
* ColumnBuffers represents data buffers for a column in a row stripe. Each
* column is made of multiple column blocks.
* column is made of multiple column chunks.
*/
typedef struct ColumnBuffers
{
ColumnBlockBuffers **blockBuffersArray;
ColumnChunkBuffers **chunkBuffersArray;
} ColumnBuffers;
@ -203,7 +203,7 @@ typedef struct TableReadState
/*
* List of Var pointers for columns in the query. We use this both for
* getting vector of projected columns, and also when we want to build
* base constraint to find selected row blocks.
* base constraint to find selected row chunks.
*/
List *projectedColumnList;
@ -212,8 +212,8 @@ typedef struct TableReadState
StripeBuffers *stripeBuffers;
uint32 readStripeCount;
uint64 stripeReadRowCount;
BlockData *blockData;
int32 deserializedBlockIndex;
ChunkData *chunkData;
int32 deserializedChunkIndex;
} TableReadState;
@ -230,8 +230,8 @@ typedef struct TableWriteState
StripeBuffers *stripeBuffers;
StripeSkipList *stripeSkipList;
uint32 stripeMaxRowCount;
uint32 blockRowCount;
BlockData *blockData;
uint32 chunkRowCount;
ChunkData *chunkData;
/*
* compressionBuffer buffer is used as temporary storage during
@ -244,7 +244,7 @@ typedef struct TableWriteState
extern int cstore_compression;
extern int cstore_stripe_row_count;
extern int cstore_block_row_count;
extern int cstore_chunk_row_count;
extern void cstore_init(void);
@ -254,7 +254,7 @@ extern CompressionType ParseCompressionType(const char *compressionTypeString);
extern TableWriteState * CStoreBeginWrite(RelFileNode relfilenode,
CompressionType compressionType,
uint64 stripeMaxRowCount,
uint32 blockRowCount,
uint32 chunkRowCount,
TupleDesc tupleDescriptor);
extern void CStoreWriteRow(TableWriteState *state, Datum *columnValues,
bool *columnNulls);
@ -275,9 +275,9 @@ extern void CStoreEndRead(TableReadState *state);
/* Function declarations for common functions */
extern FmgrInfo * GetFunctionInfoOrNull(Oid typeId, Oid accessMethodId,
int16 procedureId);
extern BlockData * CreateEmptyBlockData(uint32 columnCount, bool *columnMask,
uint32 blockRowCount);
extern void FreeBlockData(BlockData *blockData);
extern ChunkData * CreateEmptyChunkData(uint32 columnCount, bool *columnMask,
uint32 chunkRowCount);
extern void FreeChunkData(ChunkData *chunkData);
extern uint64 CStoreTableRowCount(Relation relation);
extern bool CompressBuffer(StringInfo inputBuffer, StringInfo outputBuffer,
CompressionType compressionType);
@ -300,13 +300,13 @@ extern List * StripesForRelfilenode(RelFileNode relfilenode);
extern uint64 GetHighestUsedAddress(RelFileNode relfilenode);
extern StripeMetadata ReserveStripe(Relation rel, uint64 size,
uint64 rowCount, uint64 columnCount,
uint64 blockCount, uint64 blockRowCount);
uint64 chunkCount, uint64 chunkRowCount);
extern void SaveStripeSkipList(RelFileNode relfilenode, uint64 stripe,
StripeSkipList *stripeSkipList,
TupleDesc tupleDescriptor);
extern StripeSkipList * ReadStripeSkipList(RelFileNode relfilenode, uint64 stripe,
TupleDesc tupleDescriptor,
uint32 blockCount);
uint32 chunkCount);
extern Datum columnar_relation_storageid(PG_FUNCTION_ARGS);

2
src/test/regress/columnar_am_schedule
View File

@ -17,7 +17,7 @@ test: am_rollback
test: am_truncate
test: am_vacuum
test: am_clean
test: am_block_filtering
test: am_chunk_filtering
test: am_join
test: am_trigger
test: am_tableoptions

0
src/test/regress/data/block_filtering.csv → src/test/regress/data/chunk_filtering.csv
View File

2
src/test/regress/expected/.gitignore vendored
View File

@ -21,7 +21,7 @@
/multi_behavioral_analytics_create_table.out
/multi_insert_select_behavioral_analytics_create_table.out
/hyperscale_tutorial.out
/am_block_filtering.out
/am_chunk_filtering.out
/am_copyto.out
/am_data_types.out
/am_load.out

8
src/test/regress/expected/am_empty.out
View File

@ -17,14 +17,14 @@ SELECT alter_columnar_table_set('t_compressed', stripe_row_count => 100);
(1 row)
SELECT alter_columnar_table_set('t_compressed', block_row_count => 100);
SELECT alter_columnar_table_set('t_compressed', chunk_row_count => 100);
alter_columnar_table_set
---------------------------------------------------------------------
(1 row)
SELECT * FROM cstore.options WHERE regclass = 't_compressed'::regclass;
regclass | block_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression
---------------------------------------------------------------------
t_compressed | 100 | 100 | pglz
(1 row)
@ -71,14 +71,14 @@ INFO: statistics for "t_compressed":
storage id: -1
total file size: 0, total data size: 0
total row count: 0, stripe count: 0, average rows per stripe: 0
block count: 0, containing data for dropped columns: 0, none compressed: 0, pglz compressed: 0
chunk count: 0, containing data for dropped columns: 0, none compressed: 0, pglz compressed: 0
vacuum verbose t_uncompressed;
INFO: statistics for "t_uncompressed":
storage id: -1
total file size: 0, total data size: 0
total row count: 0, stripe count: 0, average rows per stripe: 0
block count: 0, containing data for dropped columns: 0, none compressed: 0, pglz compressed: 0
chunk count: 0, containing data for dropped columns: 0, none compressed: 0, pglz compressed: 0
-- vacuum full
vacuum full t_compressed;

6
src/test/regress/expected/am_matview.out
View File

@ -25,7 +25,7 @@ SELECT * FROM t_view a ORDER BY a;
-- show columnar options for materialized view
SELECT * FROM cstore.options
WHERE regclass = 't_view'::regclass;
regclass | block_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression
---------------------------------------------------------------------
t_view | 10000 | 150000 | none
(1 row)
@ -39,7 +39,7 @@ SELECT alter_columnar_table_set('t_view', compression => 'pglz');
SELECT * FROM cstore.options
WHERE regclass = 't_view'::regclass;
regclass | block_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression
---------------------------------------------------------------------
t_view | 10000 | 150000 | pglz
(1 row)
@ -48,7 +48,7 @@ REFRESH MATERIALIZED VIEW t_view;
-- verify options have not been changed
SELECT * FROM cstore.options
WHERE regclass = 't_view'::regclass;
regclass | block_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression
---------------------------------------------------------------------
t_view | 10000 | 150000 | pglz
(1 row)

50
src/test/regress/expected/am_tableoptions.out
View File

@ -5,7 +5,7 @@ INSERT INTO table_options SELECT generate_series(1,100);
-- show table_options settings
SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass;
regclass | block_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression
---------------------------------------------------------------------
table_options | 10000 | 150000 | none
(1 row)
@ -20,13 +20,13 @@ SELECT alter_columnar_table_set('table_options', compression => 'pglz');
-- show table_options settings
SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass;
regclass | block_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression
---------------------------------------------------------------------
table_options | 10000 | 150000 | pglz
(1 row)
-- test changing the block_row_count
SELECT alter_columnar_table_set('table_options', block_row_count => 10);
-- test changing the chunk_row_count
SELECT alter_columnar_table_set('table_options', chunk_row_count => 10);
alter_columnar_table_set
---------------------------------------------------------------------
@ -35,12 +35,12 @@ SELECT alter_columnar_table_set('table_options', block_row_count => 10);
-- show table_options settings
SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass;
regclass | block_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression
---------------------------------------------------------------------
table_options | 10 | 150000 | pglz
(1 row)
-- test changing the block_row_count
-- test changing the chunk_row_count
SELECT alter_columnar_table_set('table_options', stripe_row_count => 100);
alter_columnar_table_set
---------------------------------------------------------------------
@ -50,7 +50,7 @@ SELECT alter_columnar_table_set('table_options', stripe_row_count => 100);
-- show table_options settings
SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass;
regclass | block_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression
---------------------------------------------------------------------
table_options | 10 | 100 | pglz
(1 row)
@ -60,13 +60,13 @@ VACUUM FULL table_options;
-- show table_options settings
SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass;
regclass | block_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression
---------------------------------------------------------------------
table_options | 10 | 100 | pglz
(1 row)
-- set all settings at the same time
SELECT alter_columnar_table_set('table_options', stripe_row_count => 1000, block_row_count => 100, compression => 'none');
SELECT alter_columnar_table_set('table_options', stripe_row_count => 1000, chunk_row_count => 100, compression => 'none');
alter_columnar_table_set
---------------------------------------------------------------------
@ -75,7 +75,7 @@ SELECT alter_columnar_table_set('table_options', stripe_row_count => 1000, block
-- show table_options settings
SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass;
regclass | block_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression
---------------------------------------------------------------------
table_options | 100 | 1000 | none
(1 row)
@ -85,7 +85,7 @@ VACUUM table_options;
-- show table_options settings
SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass;
regclass | block_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression
---------------------------------------------------------------------
table_options | 100 | 1000 | none
(1 row)
@ -95,7 +95,7 @@ VACUUM FULL table_options;
-- show table_options settings
SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass;
regclass | block_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression
---------------------------------------------------------------------
table_options | 100 | 1000 | none
(1 row)
@ -105,7 +105,7 @@ TRUNCATE table_options;
-- show table_options settings
SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass;
regclass | block_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression
---------------------------------------------------------------------
table_options | 100 | 1000 | none
(1 row)
@ -114,25 +114,25 @@ ALTER TABLE table_options ALTER COLUMN a TYPE bigint;
-- show table_options settings
SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass;
regclass | block_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression
---------------------------------------------------------------------
table_options | 100 | 1000 | none
(1 row)
-- reset settings one by one to the version of the GUC's
SET cstore.block_row_count TO 1000;
SET cstore.chunk_row_count TO 1000;
SET cstore.stripe_row_count TO 10000;
SET cstore.compression TO 'pglz';
-- verify setting the GUC's didn't change the settings
-- show table_options settings
SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass;
regclass | block_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression
---------------------------------------------------------------------
table_options | 100 | 1000 | none
(1 row)
SELECT alter_columnar_table_reset('table_options', block_row_count => true);
SELECT alter_columnar_table_reset('table_options', chunk_row_count => true);
alter_columnar_table_reset
---------------------------------------------------------------------
@ -141,7 +141,7 @@ SELECT alter_columnar_table_reset('table_options', block_row_count => true);
-- show table_options settings
SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass;
regclass | block_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression
---------------------------------------------------------------------
table_options | 1000 | 1000 | none
(1 row)
@ -155,7 +155,7 @@ SELECT alter_columnar_table_reset('table_options', stripe_row_count => true);
-- show table_options settings
SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass;
regclass | block_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression
---------------------------------------------------------------------
table_options | 1000 | 10000 | none
(1 row)
@ -169,26 +169,26 @@ SELECT alter_columnar_table_reset('table_options', compression => true);
-- show table_options settings
SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass;
regclass | block_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression
---------------------------------------------------------------------
table_options | 1000 | 10000 | pglz
(1 row)
-- verify resetting all settings at once work
SET cstore.block_row_count TO 10000;
SET cstore.chunk_row_count TO 10000;
SET cstore.stripe_row_count TO 100000;
SET cstore.compression TO 'none';
-- show table_options settings
SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass;
regclass | block_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression
---------------------------------------------------------------------
table_options | 1000 | 10000 | pglz
(1 row)
SELECT alter_columnar_table_reset(
'table_options',
block_row_count => true,
chunk_row_count => true,
stripe_row_count => true,
compression => true);
alter_columnar_table_reset
@ -199,7 +199,7 @@ SELECT alter_columnar_table_reset(
-- show table_options settings
SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass;
regclass | block_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression
---------------------------------------------------------------------
table_options | 10000 | 100000 | none
(1 row)
@ -218,7 +218,7 @@ ERROR: unknown compression type for cstore table: foobar
DROP TABLE table_options;
-- we expect no entries in çstore.options for anything not found int pg_class
SELECT * FROM cstore.options o WHERE o.regclass NOT IN (SELECT oid FROM pg_class);
regclass | block_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression
---------------------------------------------------------------------
(0 rows)

24
src/test/regress/expected/am_vacuum.out
View File

@ -73,10 +73,10 @@ SELECT count(*) FROM t_stripes;
-- VACUUM FULL doesn't reclaim dropped columns, but converts them to NULLs
ALTER TABLE t DROP COLUMN a;
SELECT stripe, attr, block, minimum_value IS NULL, maximum_value IS NULL
SELECT stripe, attr, chunk, minimum_value IS NULL, maximum_value IS NULL
FROM cstore.cstore_skipnodes a, pg_class b
WHERE a.storageid = columnar_relation_storageid(b.oid) AND b.relname='t' ORDER BY 1, 2, 3;
stripe | attr | block | ?column? | ?column?
stripe | attr | chunk | ?column? | ?column?
---------------------------------------------------------------------
1 | 1 | 0 | f | f
1 | 2 | 0 | f | f
@ -87,10 +87,10 @@ WHERE a.storageid = columnar_relation_storageid(b.oid) AND b.relname='t' ORDER B
(6 rows)
VACUUM FULL t;
SELECT stripe, attr, block, minimum_value IS NULL, maximum_value IS NULL
SELECT stripe, attr, chunk, minimum_value IS NULL, maximum_value IS NULL
FROM cstore.cstore_skipnodes a, pg_class b
WHERE a.storageid = columnar_relation_storageid(b.oid) AND b.relname='t' ORDER BY 1, 2, 3;
stripe | attr | block | ?column? | ?column?
stripe | attr | chunk | ?column? | ?column?
---------------------------------------------------------------------
1 | 1 | 0 | t | t
1 | 2 | 0 | f | f
@ -151,7 +151,7 @@ INFO: statistics for "t":
storage id: xxxxx
total file size: 122880, total data size: 10754
total row count: 2530, stripe count: 3, average rows per stripe: 843
block count: 3, containing data for dropped columns: 0, none compressed: 3, pglz compressed: 0
chunk count: 3, containing data for dropped columns: 0, none compressed: 3, pglz compressed: 0
INFO: "t": truncated 15 to 5 pages
SELECT pg_size_pretty(pg_relation_size('t'));
@ -170,7 +170,7 @@ SELECT count(*) FROM t;
-- then vacuum to print stats
BEGIN;
SELECT alter_columnar_table_set('t',
block_row_count => 1000,
chunk_row_count => 1000,
stripe_row_count => 2000,
compression => 'pglz');
alter_columnar_table_set
@ -198,7 +198,7 @@ INFO: statistics for "t":
storage id: xxxxx
total file size: 57344, total data size: 18808
total row count: 5530, stripe count: 5, average rows per stripe: 1106
block count: 7, containing data for dropped columns: 0, none compressed: 5, pglz compressed: 2
chunk count: 7, containing data for dropped columns: 0, none compressed: 5, pglz compressed: 2
SELECT count(*) FROM t;
count
@ -206,7 +206,7 @@ SELECT count(*) FROM t;
5530
(1 row)
-- check that we report blocks with data for dropped columns
-- check that we report chunks with data for dropped columns
ALTER TABLE t ADD COLUMN c int;
INSERT INTO t SELECT 1, i / 5 FROM generate_series(1, 1500) i;
ALTER TABLE t DROP COLUMN c;
@ -215,10 +215,10 @@ INFO: statistics for "t":
storage id: xxxxx
total file size: 73728, total data size: 31372
total row count: 7030, stripe count: 6, average rows per stripe: 1171
block count: 11, containing data for dropped columns: 2, none compressed: 9, pglz compressed: 2
chunk count: 11, containing data for dropped columns: 2, none compressed: 9, pglz compressed: 2
-- vacuum full should remove blocks for dropped columns
-- note that, a block will be stored in non-compressed for if compression
-- vacuum full should remove chunks for dropped columns
-- note that, a chunk will be stored in non-compressed for if compression
-- doesn't reduce its size.
SELECT alter_columnar_table_set('t', compression => 'pglz');
alter_columnar_table_set
@ -232,7 +232,7 @@ INFO: statistics for "t":
storage id: xxxxx
total file size: 57344, total data size: 15728
total row count: 7030, stripe count: 4, average rows per stripe: 1757
block count: 8, containing data for dropped columns: 0, none compressed: 2, pglz compressed: 6
chunk count: 8, containing data for dropped columns: 0, none compressed: 2, pglz compressed: 6
DROP TABLE t;
DROP VIEW t_stripes;

2
src/test/regress/expected/am_vacuum_vs_insert.out
View File

@ -14,7 +14,7 @@ s2: INFO: statistics for "test_vacuum_vs_insert":
storage id: xxxxx
total file size: 24576, total data size: 26
total row count: 3, stripe count: 1, average rows per stripe: 3
block count: 2, containing data for dropped columns: 0, none compressed: 2, pglz compressed: 0
chunk count: 2, containing data for dropped columns: 0, none compressed: 2, pglz compressed: 0
s2: INFO: "test_vacuum_vs_insert": stopping truncate due to conflicting lock request
step s2-vacuum:

48
src/test/regress/expected/columnar_citus_integration.out
View File

@ -63,10 +63,10 @@ $cmd$);
(localhost,57638,20090003,t,none)
(4 rows)
-- setting: block_row_count
-- setting: chunk_row_count
-- get baseline for setting
SELECT run_command_on_placements('table_option',$cmd$
SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
SELECT chunk_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$);
run_command_on_placements
---------------------------------------------------------------------
@ -77,7 +77,7 @@ $cmd$);
(4 rows)
-- change setting
SELECT alter_columnar_table_set('table_option', block_row_count => 100);
SELECT alter_columnar_table_set('table_option', chunk_row_count => 100);
alter_columnar_table_set
---------------------------------------------------------------------
@ -85,7 +85,7 @@ SELECT alter_columnar_table_set('table_option', block_row_count => 100);
-- verify setting
SELECT run_command_on_placements('table_option',$cmd$
SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
SELECT chunk_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$);
run_command_on_placements
---------------------------------------------------------------------
@ -96,7 +96,7 @@ $cmd$);
(4 rows)
-- reset setting
SELECT alter_columnar_table_reset('table_option', block_row_count => true);
SELECT alter_columnar_table_reset('table_option', chunk_row_count => true);
alter_columnar_table_reset
---------------------------------------------------------------------
@ -104,7 +104,7 @@ SELECT alter_columnar_table_reset('table_option', block_row_count => true);
-- verify setting
SELECT run_command_on_placements('table_option',$cmd$
SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
SELECT chunk_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$);
run_command_on_placements
---------------------------------------------------------------------
@ -168,7 +168,7 @@ $cmd$);
-- verify settings are propagated when creating a table
CREATE TABLE table_option_2 (a int, b text) USING columnar;
SELECT alter_columnar_table_set('table_option_2',
block_row_count => 100,
chunk_row_count => 100,
stripe_row_count => 1000,
compression => 'pglz');
alter_columnar_table_set
@ -184,7 +184,7 @@ SELECT create_distributed_table('table_option_2', 'a');
-- verify settings on placements
SELECT run_command_on_placements('table_option_2',$cmd$
SELECT ROW(block_row_count, stripe_row_count, compression) FROM cstore.options WHERE regclass = '%s'::regclass;
SELECT ROW(chunk_row_count, stripe_row_count, compression) FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$);
run_command_on_placements
---------------------------------------------------------------------
@ -268,10 +268,10 @@ $cmd$);
(localhost,57638,20090011,t,none)
(8 rows)
-- setting: block_row_count
-- setting: chunk_row_count
-- get baseline for setting
SELECT run_command_on_placements('table_option',$cmd$
SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
SELECT chunk_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$);
run_command_on_placements
---------------------------------------------------------------------
@ -286,7 +286,7 @@ $cmd$);
(8 rows)
-- change setting
SELECT alter_columnar_table_set('table_option', block_row_count => 100);
SELECT alter_columnar_table_set('table_option', chunk_row_count => 100);
alter_columnar_table_set
---------------------------------------------------------------------
@ -294,7 +294,7 @@ SELECT alter_columnar_table_set('table_option', block_row_count => 100);
-- verify setting
SELECT run_command_on_placements('table_option',$cmd$
SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
SELECT chunk_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$);
run_command_on_placements
---------------------------------------------------------------------
@ -309,7 +309,7 @@ $cmd$);
(8 rows)
-- reset setting
SELECT alter_columnar_table_reset('table_option', block_row_count => true);
SELECT alter_columnar_table_reset('table_option', chunk_row_count => true);
alter_columnar_table_reset
---------------------------------------------------------------------
@ -317,7 +317,7 @@ SELECT alter_columnar_table_reset('table_option', block_row_count => true);
-- verify setting
SELECT run_command_on_placements('table_option',$cmd$
SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
SELECT chunk_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$);
run_command_on_placements
---------------------------------------------------------------------
@ -397,7 +397,7 @@ $cmd$);
-- verify settings are propagated when creating a table
CREATE TABLE table_option_2 (a int, b text) USING columnar;
SELECT alter_columnar_table_set('table_option_2',
block_row_count => 100,
chunk_row_count => 100,
stripe_row_count => 1000,
compression => 'pglz');
alter_columnar_table_set
@ -413,7 +413,7 @@ SELECT create_distributed_table('table_option_2', 'a');
-- verify settings on placements
SELECT run_command_on_placements('table_option_2',$cmd$
SELECT ROW(block_row_count, stripe_row_count, compression) FROM cstore.options WHERE regclass = '%s'::regclass;
SELECT ROW(chunk_row_count, stripe_row_count, compression) FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$);
run_command_on_placements
---------------------------------------------------------------------
@ -481,10 +481,10 @@ $cmd$);
(localhost,57638,20090016,t,none)
(2 rows)
-- setting: block_row_count
-- setting: chunk_row_count
-- get baseline for setting
SELECT run_command_on_placements('table_option_reference',$cmd$
SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
SELECT chunk_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$);
run_command_on_placements
---------------------------------------------------------------------
@ -493,7 +493,7 @@ $cmd$);
(2 rows)
-- change setting
SELECT alter_columnar_table_set('table_option_reference', block_row_count => 100);
SELECT alter_columnar_table_set('table_option_reference', chunk_row_count => 100);
alter_columnar_table_set
---------------------------------------------------------------------
@ -501,7 +501,7 @@ SELECT alter_columnar_table_set('table_option_reference', block_row_count => 100
-- verify setting
SELECT run_command_on_placements('table_option_reference',$cmd$
SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
SELECT chunk_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$);
run_command_on_placements
---------------------------------------------------------------------
@ -510,7 +510,7 @@ $cmd$);
(2 rows)
-- reset setting
SELECT alter_columnar_table_reset('table_option_reference', block_row_count => true);
SELECT alter_columnar_table_reset('table_option_reference', chunk_row_count => true);
alter_columnar_table_reset
---------------------------------------------------------------------
@ -518,7 +518,7 @@ SELECT alter_columnar_table_reset('table_option_reference', block_row_count => t
-- verify setting
SELECT run_command_on_placements('table_option_reference',$cmd$
SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
SELECT chunk_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$);
run_command_on_placements
---------------------------------------------------------------------
@ -574,7 +574,7 @@ $cmd$);
-- verify settings are propagated when creating a table
CREATE TABLE table_option_reference_2 (a int, b text) USING columnar;
SELECT alter_columnar_table_set('table_option_reference_2',
block_row_count => 100,
chunk_row_count => 100,
stripe_row_count => 1000,
compression => 'pglz');
alter_columnar_table_set
@ -590,7 +590,7 @@ SELECT create_reference_table('table_option_reference_2');
-- verify settings on placements
SELECT run_command_on_placements('table_option_reference_2',$cmd$
SELECT ROW(block_row_count, stripe_row_count, compression) FROM cstore.options WHERE regclass = '%s'::regclass;
SELECT ROW(chunk_row_count, stripe_row_count, compression) FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$);
run_command_on_placements
---------------------------------------------------------------------

8
src/test/regress/expected/upgrade_columnar_after.out
View File

@ -102,7 +102,7 @@ SELECT * FROM matview ORDER BY a;
-- test we retained options
SELECT * FROM cstore.options WHERE regclass = 'test_options_1'::regclass;
regclass | block_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression
---------------------------------------------------------------------
test_options_1 | 1000 | 5000 | pglz
(1 row)
@ -112,7 +112,7 @@ INFO: statistics for "test_options_1":
storage id: xxxxx
total file size: 65536, total data size: 43136
total row count: 10000, stripe count: 2, average rows per stripe: 5000
block count: 20, containing data for dropped columns: 0, none compressed: 10, pglz compressed: 10
chunk count: 20, containing data for dropped columns: 0, none compressed: 10, pglz compressed: 10
SELECT count(*), sum(a), sum(b) FROM test_options_1;
count | sum | sum
@ -121,7 +121,7 @@ SELECT count(*), sum(a), sum(b) FROM test_options_1;
(1 row)
SELECT * FROM cstore.options WHERE regclass = 'test_options_2'::regclass;
regclass | block_row_count | stripe_row_count | compression
regclass | chunk_row_count | stripe_row_count | compression
---------------------------------------------------------------------
test_options_2 | 2000 | 6000 | none
(1 row)
@ -131,7 +131,7 @@ INFO: statistics for "test_options_2":
storage id: xxxxx
total file size: 163840, total data size: 125636
total row count: 20000, stripe count: 4, average rows per stripe: 5000
block count: 30, containing data for dropped columns: 0, none compressed: 20, pglz compressed: 10
chunk count: 30, containing data for dropped columns: 0, none compressed: 20, pglz compressed: 10
SELECT count(*), sum(a), sum(b) FROM test_options_2;
count | sum | sum

4
src/test/regress/expected/upgrade_columnar_before.out
View File

@ -116,13 +116,13 @@ SELECT :relfilenode_pre_alter <> :relfilenode_post_alter AS relfilenode_changed;
-- Test that we retain options
--
SET cstore.stripe_row_count TO 5000;
SET cstore.block_row_count TO 1000;
SET cstore.chunk_row_count TO 1000;
SET cstore.compression TO 'pglz';
CREATE TABLE test_options_1(a int, b int) USING columnar;
INSERT INTO test_options_1 SELECT i, floor(i/1000) FROM generate_series(1, 10000) i;
CREATE TABLE test_options_2(a int, b int) USING columnar;
INSERT INTO test_options_2 SELECT i, floor(i/1000) FROM generate_series(1, 10000) i;
SELECT alter_columnar_table_set('test_options_2', block_row_count => 2000);
SELECT alter_columnar_table_set('test_options_2', chunk_row_count => 2000);
alter_columnar_table_set
---------------------------------------------------------------------

46
src/test/regress/input/am_block_filtering.source → src/test/regress/input/am_chunk_filtering.source
View File

@ -1,11 +1,11 @@
--
-- Test block filtering in cstore_fdw using min/max values in stripe skip lists.
-- Test chunk filtering in cstore_fdw using min/max values in stripe skip lists.
--
--
-- filtered_row_count returns number of rows filtered by the WHERE clause.
-- If blocks get filtered by cstore_fdw, less rows are passed to WHERE
-- If chunks get filtered by cstore_fdw, less rows are passed to WHERE
-- clause, so this function should return a lower number.
--
CREATE OR REPLACE FUNCTION filtered_row_count (query text) RETURNS bigint AS
@ -28,46 +28,46 @@ $$ LANGUAGE PLPGSQL;
-- Create and load data
-- block_row_count '1000', stripe_row_count '2000'
-- chunk_row_count '1000', stripe_row_count '2000'
set cstore.stripe_row_count = 2000;
set cstore.block_row_count = 1000;
CREATE TABLE test_block_filtering (a int)
set cstore.chunk_row_count = 1000;
CREATE TABLE test_chunk_filtering (a int)
USING columnar;
COPY test_block_filtering FROM '@abs_srcdir@/data/block_filtering.csv' WITH CSV;
COPY test_chunk_filtering FROM '@abs_srcdir@/data/chunk_filtering.csv' WITH CSV;
-- Verify that filtered_row_count is less than 1000 for the following queries
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering');
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 200');
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a > 200');
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 9900');
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a > 9900');
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 0');
SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering');
SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a < 200');
SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a > 200');
SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a < 9900');
SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a > 9900');
SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a < 0');
-- Verify that filtered_row_count is less than 2000 for the following queries
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 1 AND 10');
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 990 AND 2010');
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN -10 AND 0');
SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a BETWEEN 1 AND 10');
SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a BETWEEN 990 AND 2010');
SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a BETWEEN -10 AND 0');
-- Load data for second time and verify that filtered_row_count is exactly twice as before
COPY test_block_filtering FROM '@abs_srcdir@/data/block_filtering.csv' WITH CSV;
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 200');
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 0');
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 990 AND 2010');
COPY test_chunk_filtering FROM '@abs_srcdir@/data/chunk_filtering.csv' WITH CSV;
SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a < 200');
SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a < 0');
SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a BETWEEN 990 AND 2010');
set cstore.stripe_row_count to default;
set cstore.block_row_count to default;
set cstore.chunk_row_count to default;
-- Verify that we are fine with collations which use a different alphabet order
CREATE TABLE collation_block_filtering_test(A text collate "da_DK")
CREATE TABLE collation_chunk_filtering_test(A text collate "da_DK")
USING columnar;
COPY collation_block_filtering_test FROM STDIN;
COPY collation_chunk_filtering_test FROM STDIN;
A
Å
B
\.
SELECT * FROM collation_block_filtering_test WHERE A > 'B';
SELECT * FROM collation_chunk_filtering_test WHERE A > 'B';

46
src/test/regress/output/am_block_filtering.source → src/test/regress/output/am_chunk_filtering.source
View File

@ -1,9 +1,9 @@
--
-- Test block filtering in cstore_fdw using min/max values in stripe skip lists.
-- Test chunk filtering in cstore_fdw using min/max values in stripe skip lists.
--
--
-- filtered_row_count returns number of rows filtered by the WHERE clause.
-- If blocks get filtered by cstore_fdw, less rows are passed to WHERE
-- If chunks get filtered by cstore_fdw, less rows are passed to WHERE
-- clause, so this function should return a lower number.
--
CREATE OR REPLACE FUNCTION filtered_row_count (query text) RETURNS bigint AS
@ -24,95 +24,95 @@ $$
END;
$$ LANGUAGE PLPGSQL;
-- Create and load data
-- block_row_count '1000', stripe_row_count '2000'
-- chunk_row_count '1000', stripe_row_count '2000'
set cstore.stripe_row_count = 2000;
set cstore.block_row_count = 1000;
CREATE TABLE test_block_filtering (a int)
set cstore.chunk_row_count = 1000;
CREATE TABLE test_chunk_filtering (a int)
USING columnar;
COPY test_block_filtering FROM '@abs_srcdir@/data/block_filtering.csv' WITH CSV;
COPY test_chunk_filtering FROM '@abs_srcdir@/data/chunk_filtering.csv' WITH CSV;
-- Verify that filtered_row_count is less than 1000 for the following queries
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering');
SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering');
filtered_row_count
--------------------
0
(1 row)
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 200');
SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a < 200');
filtered_row_count
--------------------
801
(1 row)
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a > 200');
SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a > 200');
filtered_row_count
--------------------
200
(1 row)
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 9900');
SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a < 9900');
filtered_row_count
--------------------
101
(1 row)
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a > 9900');
SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a > 9900');
filtered_row_count
--------------------
900
(1 row)
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 0');
SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a < 0');
filtered_row_count
--------------------
0
(1 row)
-- Verify that filtered_row_count is less than 2000 for the following queries
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 1 AND 10');
SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a BETWEEN 1 AND 10');
filtered_row_count
--------------------
990
(1 row)
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 990 AND 2010');
SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a BETWEEN 990 AND 2010');
filtered_row_count
--------------------
1979
(1 row)
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN -10 AND 0');
SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a BETWEEN -10 AND 0');
filtered_row_count
--------------------
0
(1 row)
-- Load data for second time and verify that filtered_row_count is exactly twice as before
COPY test_block_filtering FROM '@abs_srcdir@/data/block_filtering.csv' WITH CSV;
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 200');
COPY test_chunk_filtering FROM '@abs_srcdir@/data/chunk_filtering.csv' WITH CSV;
SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a < 200');
filtered_row_count
--------------------
1602
(1 row)
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 0');
SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a < 0');
filtered_row_count
--------------------
0
(1 row)
SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 990 AND 2010');
SELECT filtered_row_count('SELECT count(*) FROM test_chunk_filtering WHERE a BETWEEN 990 AND 2010');
filtered_row_count
--------------------
3958
(1 row)
set cstore.stripe_row_count to default;
set cstore.block_row_count to default;
set cstore.chunk_row_count to default;
-- Verify that we are fine with collations which use a different alphabet order
CREATE TABLE collation_block_filtering_test(A text collate "da_DK")
CREATE TABLE collation_chunk_filtering_test(A text collate "da_DK")
USING columnar;
COPY collation_block_filtering_test FROM STDIN;
SELECT * FROM collation_block_filtering_test WHERE A > 'B';
COPY collation_chunk_filtering_test FROM STDIN;
SELECT * FROM collation_chunk_filtering_test WHERE A > 'B';
a
---
Å

2
src/test/regress/sql/.gitignore vendored
View File

@ -20,7 +20,7 @@
/multi_behavioral_analytics_create_table.sql
/multi_insert_select_behavioral_analytics_create_table.sql
/hyperscale_tutorial.sql
/am_block_filtering.sql
/am_chunk_filtering.sql
/am_copyto.sql
/am_data_types.sql
/am_load.sql

2
src/test/regress/sql/am_empty.sql
View File

@ -9,7 +9,7 @@ create table t_compressed(a int) using columnar;
-- set options
SELECT alter_columnar_table_set('t_compressed', compression => 'pglz');
SELECT alter_columnar_table_set('t_compressed', stripe_row_count => 100);
SELECT alter_columnar_table_set('t_compressed', block_row_count => 100);
SELECT alter_columnar_table_set('t_compressed', chunk_row_count => 100);
SELECT * FROM cstore.options WHERE regclass = 't_compressed'::regclass;

16
src/test/regress/sql/am_tableoptions.sql
View File

@ -15,14 +15,14 @@ SELECT alter_columnar_table_set('table_options', compression => 'pglz');
SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass;
-- test changing the block_row_count
SELECT alter_columnar_table_set('table_options', block_row_count => 10);
-- test changing the chunk_row_count
SELECT alter_columnar_table_set('table_options', chunk_row_count => 10);
-- show table_options settings
SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass;
-- test changing the block_row_count
-- test changing the chunk_row_count
SELECT alter_columnar_table_set('table_options', stripe_row_count => 100);
-- show table_options settings
@ -37,7 +37,7 @@ SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass;
-- set all settings at the same time
SELECT alter_columnar_table_set('table_options', stripe_row_count => 1000, block_row_count => 100, compression => 'none');
SELECT alter_columnar_table_set('table_options', stripe_row_count => 1000, chunk_row_count => 100, compression => 'none');
-- show table_options settings
SELECT * FROM cstore.options
@ -67,7 +67,7 @@ SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass;
-- reset settings one by one to the version of the GUC's
SET cstore.block_row_count TO 1000;
SET cstore.chunk_row_count TO 1000;
SET cstore.stripe_row_count TO 10000;
SET cstore.compression TO 'pglz';
@ -76,7 +76,7 @@ SET cstore.compression TO 'pglz';
SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass;
SELECT alter_columnar_table_reset('table_options', block_row_count => true);
SELECT alter_columnar_table_reset('table_options', chunk_row_count => true);
-- show table_options settings
SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass;
@ -94,7 +94,7 @@ SELECT * FROM cstore.options
WHERE regclass = 'table_options'::regclass;
-- verify resetting all settings at once work
SET cstore.block_row_count TO 10000;
SET cstore.chunk_row_count TO 10000;
SET cstore.stripe_row_count TO 100000;
SET cstore.compression TO 'none';
@ -104,7 +104,7 @@ WHERE regclass = 'table_options'::regclass;
SELECT alter_columnar_table_reset(
'table_options',
block_row_count => true,
chunk_row_count => true,
stripe_row_count => true,
compression => true);

12
src/test/regress/sql/am_vacuum.sql
View File

@ -36,13 +36,13 @@ SELECT count(*) FROM t_stripes;
-- VACUUM FULL doesn't reclaim dropped columns, but converts them to NULLs
ALTER TABLE t DROP COLUMN a;
SELECT stripe, attr, block, minimum_value IS NULL, maximum_value IS NULL
SELECT stripe, attr, chunk, minimum_value IS NULL, maximum_value IS NULL
FROM cstore.cstore_skipnodes a, pg_class b
WHERE a.storageid = columnar_relation_storageid(b.oid) AND b.relname='t' ORDER BY 1, 2, 3;
VACUUM FULL t;
SELECT stripe, attr, block, minimum_value IS NULL, maximum_value IS NULL
SELECT stripe, attr, chunk, minimum_value IS NULL, maximum_value IS NULL
FROM cstore.cstore_skipnodes a, pg_class b
WHERE a.storageid = columnar_relation_storageid(b.oid) AND b.relname='t' ORDER BY 1, 2, 3;
@ -74,7 +74,7 @@ SELECT count(*) FROM t;
BEGIN;
SELECT alter_columnar_table_set('t',
block_row_count => 1000,
chunk_row_count => 1000,
stripe_row_count => 2000,
compression => 'pglz');
SAVEPOINT s1;
@ -92,15 +92,15 @@ VACUUM VERBOSE t;
SELECT count(*) FROM t;
-- check that we report blocks with data for dropped columns
-- check that we report chunks with data for dropped columns
ALTER TABLE t ADD COLUMN c int;
INSERT INTO t SELECT 1, i / 5 FROM generate_series(1, 1500) i;
ALTER TABLE t DROP COLUMN c;
VACUUM VERBOSE t;
-- vacuum full should remove blocks for dropped columns
-- note that, a block will be stored in non-compressed for if compression
-- vacuum full should remove chunks for dropped columns
-- note that, a chunk will be stored in non-compressed for if compression
-- doesn't reduce its size.
SELECT alter_columnar_table_set('t', compression => 'pglz');
VACUUM FULL t;

48
src/test/regress/sql/columnar_citus_integration.sql
View File

@ -28,22 +28,22 @@ SELECT run_command_on_placements('table_option',$cmd$
SELECT compression FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$);
-- setting: block_row_count
-- setting: chunk_row_count
-- get baseline for setting
SELECT run_command_on_placements('table_option',$cmd$
SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
SELECT chunk_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$);
-- change setting
SELECT alter_columnar_table_set('table_option', block_row_count => 100);
SELECT alter_columnar_table_set('table_option', chunk_row_count => 100);
-- verify setting
SELECT run_command_on_placements('table_option',$cmd$
SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
SELECT chunk_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$);
-- reset setting
SELECT alter_columnar_table_reset('table_option', block_row_count => true);
SELECT alter_columnar_table_reset('table_option', chunk_row_count => true);
-- verify setting
SELECT run_command_on_placements('table_option',$cmd$
SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
SELECT chunk_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$);
-- setting: stripe_row_count
@ -67,14 +67,14 @@ $cmd$);
-- verify settings are propagated when creating a table
CREATE TABLE table_option_2 (a int, b text) USING columnar;
SELECT alter_columnar_table_set('table_option_2',
block_row_count => 100,
chunk_row_count => 100,
stripe_row_count => 1000,
compression => 'pglz');
SELECT create_distributed_table('table_option_2', 'a');
-- verify settings on placements
SELECT run_command_on_placements('table_option_2',$cmd$
SELECT ROW(block_row_count, stripe_row_count, compression) FROM cstore.options WHERE regclass = '%s'::regclass;
SELECT ROW(chunk_row_count, stripe_row_count, compression) FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$);
DROP TABLE table_option, table_option_2;
@ -104,22 +104,22 @@ SELECT run_command_on_placements('table_option',$cmd$
SELECT compression FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$);
-- setting: block_row_count
-- setting: chunk_row_count
-- get baseline for setting
SELECT run_command_on_placements('table_option',$cmd$
SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
SELECT chunk_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$);
-- change setting
SELECT alter_columnar_table_set('table_option', block_row_count => 100);
SELECT alter_columnar_table_set('table_option', chunk_row_count => 100);
-- verify setting
SELECT run_command_on_placements('table_option',$cmd$
SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
SELECT chunk_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$);
-- reset setting
SELECT alter_columnar_table_reset('table_option', block_row_count => true);
SELECT alter_columnar_table_reset('table_option', chunk_row_count => true);
-- verify setting
SELECT run_command_on_placements('table_option',$cmd$
SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
SELECT chunk_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$);
-- setting: stripe_row_count
@ -143,14 +143,14 @@ $cmd$);
-- verify settings are propagated when creating a table
CREATE TABLE table_option_2 (a int, b text) USING columnar;
SELECT alter_columnar_table_set('table_option_2',
block_row_count => 100,
chunk_row_count => 100,
stripe_row_count => 1000,
compression => 'pglz');
SELECT create_distributed_table('table_option_2', 'a');
-- verify settings on placements
SELECT run_command_on_placements('table_option_2',$cmd$
SELECT ROW(block_row_count, stripe_row_count, compression) FROM cstore.options WHERE regclass = '%s'::regclass;
SELECT ROW(chunk_row_count, stripe_row_count, compression) FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$);
DROP TABLE table_option, table_option_2;
@ -177,22 +177,22 @@ SELECT run_command_on_placements('table_option_reference',$cmd$
SELECT compression FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$);
-- setting: block_row_count
-- setting: chunk_row_count
-- get baseline for setting
SELECT run_command_on_placements('table_option_reference',$cmd$
SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
SELECT chunk_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$);
-- change setting
SELECT alter_columnar_table_set('table_option_reference', block_row_count => 100);
SELECT alter_columnar_table_set('table_option_reference', chunk_row_count => 100);
-- verify setting
SELECT run_command_on_placements('table_option_reference',$cmd$
SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
SELECT chunk_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$);
-- reset setting
SELECT alter_columnar_table_reset('table_option_reference', block_row_count => true);
SELECT alter_columnar_table_reset('table_option_reference', chunk_row_count => true);
-- verify setting
SELECT run_command_on_placements('table_option_reference',$cmd$
SELECT block_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
SELECT chunk_row_count FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$);
-- setting: stripe_row_count
@ -216,14 +216,14 @@ $cmd$);
-- verify settings are propagated when creating a table
CREATE TABLE table_option_reference_2 (a int, b text) USING columnar;
SELECT alter_columnar_table_set('table_option_reference_2',
block_row_count => 100,
chunk_row_count => 100,
stripe_row_count => 1000,
compression => 'pglz');
SELECT create_reference_table('table_option_reference_2');
-- verify settings on placements
SELECT run_command_on_placements('table_option_reference_2',$cmd$
SELECT ROW(block_row_count, stripe_row_count, compression) FROM cstore.options WHERE regclass = '%s'::regclass;
SELECT ROW(chunk_row_count, stripe_row_count, compression) FROM cstore.options WHERE regclass = '%s'::regclass;
$cmd$);
DROP TABLE table_option_reference, table_option_reference_2;

4
src/test/regress/sql/upgrade_columnar_before.sql
View File

@ -96,7 +96,7 @@ SELECT :relfilenode_pre_alter <> :relfilenode_post_alter AS relfilenode_changed;
--
SET cstore.stripe_row_count TO 5000;
SET cstore.block_row_count TO 1000;
SET cstore.chunk_row_count TO 1000;
SET cstore.compression TO 'pglz';
CREATE TABLE test_options_1(a int, b int) USING columnar;
@ -104,7 +104,7 @@ INSERT INTO test_options_1 SELECT i, floor(i/1000) FROM generate_series(1, 10000
CREATE TABLE test_options_2(a int, b int) USING columnar;
INSERT INTO test_options_2 SELECT i, floor(i/1000) FROM generate_series(1, 10000) i;
SELECT alter_columnar_table_set('test_options_2', block_row_count => 2000);
SELECT alter_columnar_table_set('test_options_2', chunk_row_count => 2000);
SELECT alter_columnar_table_set('test_options_2', stripe_row_count => 6000);
SELECT alter_columnar_table_set('test_options_2', compression => 'none');
INSERT INTO test_options_2 SELECT i, floor(i/2000) FROM generate_series(1, 10000) i;