external
/
citus
mirror of https://github.com/citusdata/citus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

More meaningful columnar metadata table names

pull/4552/head
Hadi Moshayedi 2021-01-21 10:07:50 -08:00
parent 9051bf485f
commit ff38996645
19 changed files with 228 additions and 230 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

302
src/backend/columnar/cstore_metadata_tables.c
View File

@ -79,12 +79,12 @@ static void GetHighestUsedAddressAndId(uint64 storageId,
uint64 *highestUsedAddress,
uint64 *highestUsedId);
static List * ReadDataFileStripeList(uint64 storageId, Snapshot snapshot);
static Oid ColumnarStripesRelationId(void);
static Oid ColumnarStripesIndexRelationId(void);
static Oid ColumnarStripeRelationId(void);
static Oid ColumnarStripeIndexRelationId(void);
static Oid ColumnarOptionsRelationId(void);
static Oid ColumnarOptionsIndexRegclass(void);
static Oid ColumnarSkipNodesRelationId(void);
static Oid ColumnarSkipNodesIndexRelationId(void);
static Oid ColumnarChunkRelationId(void);
static Oid ColumnarChunkIndexRelationId(void);
static Oid ColumnarNamespaceId(void);
static ModifyState * StartModifyRelation(Relation rel);
static void InsertTupleAndEnforceConstraints(ModifyState *state, Datum *values,
@ -127,33 +127,33 @@ typedef struct FormData_columnar_options
typedef FormData_columnar_options *Form_columnar_options;
/* constants for columnar_stripe */
#define Natts_columnar_stripes 8
#define Anum_columnar_stripes_storageid 1
#define Anum_columnar_stripes_stripe 2
#define Anum_columnar_stripes_file_offset 3
#define Anum_columnar_stripes_data_length 4
#define Anum_columnar_stripes_column_count 5
#define Anum_columnar_stripes_chunk_count 6
#define Anum_columnar_stripes_chunk_row_count 7
#define Anum_columnar_stripes_row_count 8
/* constants for columnar.stripe */
#define Natts_columnar_stripe 8
#define Anum_columnar_stripe_storageid 1
#define Anum_columnar_stripe_stripe 2
#define Anum_columnar_stripe_file_offset 3
#define Anum_columnar_stripe_data_length 4
#define Anum_columnar_stripe_column_count 5
#define Anum_columnar_stripe_chunk_count 6
#define Anum_columnar_stripe_chunk_row_count 7
#define Anum_columnar_stripe_row_count 8
/* constants for columnar_skipnodes */
#define Natts_columnar_skipnodes 14
#define Anum_columnar_skipnodes_storageid 1
#define Anum_columnar_skipnodes_stripe 2
#define Anum_columnar_skipnodes_attr 3
#define Anum_columnar_skipnodes_chunk 4
#define Anum_columnar_skipnodes_row_count 5
#define Anum_columnar_skipnodes_minimum_value 6
#define Anum_columnar_skipnodes_maximum_value 7
#define Anum_columnar_skipnodes_value_stream_offset 8
#define Anum_columnar_skipnodes_value_stream_length 9
#define Anum_columnar_skipnodes_exists_stream_offset 10
#define Anum_columnar_skipnodes_exists_stream_length 11
#define Anum_columnar_skipnodes_value_compression_type 12
#define Anum_columnar_skipnodes_value_compression_level 13
#define Anum_columnar_skipnodes_value_decompressed_size 14
/* constants for columnar.chunk */
#define Natts_columnar_chunk 14
#define Anum_columnar_chunk_storageid 1
#define Anum_columnar_chunk_stripe 2
#define Anum_columnar_chunk_attr 3
#define Anum_columnar_chunk_chunk 4
#define Anum_columnar_chunk_row_count 5
#define Anum_columnar_chunk_minimum_value 6
#define Anum_columnar_chunk_maximum_value 7
#define Anum_columnar_chunk_value_stream_offset 8
#define Anum_columnar_chunk_value_stream_length 9
#define Anum_columnar_chunk_exists_stream_offset 10
#define Anum_columnar_chunk_exists_stream_length 11
#define Anum_columnar_chunk_value_compression_type 12
#define Anum_columnar_chunk_value_compression_level 13
#define Anum_columnar_chunk_value_decompressed_size 14
/*
@ -393,61 +393,61 @@ ReadColumnarOptions(Oid regclass, ColumnarOptions *options)
/*
* SaveStripeSkipList saves StripeSkipList for a given stripe as rows
* of columnar_skipnodes.
* SaveStripeSkipList saves chunkList for a given stripe as rows
* of columnar.chunk.
*/
void
SaveStripeSkipList(RelFileNode relfilenode, uint64 stripe, StripeSkipList *stripeSkipList,
SaveStripeSkipList(RelFileNode relfilenode, uint64 stripe, StripeSkipList *chunkList,
TupleDesc tupleDescriptor)
{
uint32 columnIndex = 0;
uint32 chunkIndex = 0;
uint32 columnCount = stripeSkipList->columnCount;
uint32 columnCount = chunkList->columnCount;
ColumnarMetapage *metapage = ReadMetapage(relfilenode, false);
Oid cstoreSkipNodesOid = ColumnarSkipNodesRelationId();
Relation cstoreSkipNodes = table_open(cstoreSkipNodesOid, RowExclusiveLock);
ModifyState *modifyState = StartModifyRelation(cstoreSkipNodes);
Oid columnarChunkOid = ColumnarChunkRelationId();
Relation columnarChunk = table_open(columnarChunkOid, RowExclusiveLock);
ModifyState *modifyState = StartModifyRelation(columnarChunk);
for (columnIndex = 0; columnIndex < columnCount; columnIndex++)
{
for (chunkIndex = 0; chunkIndex < stripeSkipList->chunkCount; chunkIndex++)
for (chunkIndex = 0; chunkIndex < chunkList->chunkCount; chunkIndex++)
{
ColumnChunkSkipNode *skipNode =
&stripeSkipList->chunkSkipNodeArray[columnIndex][chunkIndex];
ColumnChunkSkipNode *chunk =
&chunkList->chunkSkipNodeArray[columnIndex][chunkIndex];
Datum values[Natts_columnar_skipnodes] = {
Datum values[Natts_columnar_chunk] = {
UInt64GetDatum(metapage->storageId),
Int64GetDatum(stripe),
Int32GetDatum(columnIndex + 1),
Int32GetDatum(chunkIndex),
Int64GetDatum(skipNode->rowCount),
Int64GetDatum(chunk->rowCount),
0, /* to be filled below */
0, /* to be filled below */
Int64GetDatum(skipNode->valueChunkOffset),
Int64GetDatum(skipNode->valueLength),
Int64GetDatum(skipNode->existsChunkOffset),
Int64GetDatum(skipNode->existsLength),
Int32GetDatum(skipNode->valueCompressionType),
Int32GetDatum(skipNode->valueCompressionLevel),
Int64GetDatum(skipNode->decompressedValueSize)
Int64GetDatum(chunk->valueChunkOffset),
Int64GetDatum(chunk->valueLength),
Int64GetDatum(chunk->existsChunkOffset),
Int64GetDatum(chunk->existsLength),
Int32GetDatum(chunk->valueCompressionType),
Int32GetDatum(chunk->valueCompressionLevel),
Int64GetDatum(chunk->decompressedValueSize)
};
bool nulls[Natts_columnar_skipnodes] = { false };
bool nulls[Natts_columnar_chunk] = { false };
if (skipNode->hasMinMax)
if (chunk->hasMinMax)
{
values[Anum_columnar_skipnodes_minimum_value - 1] =
PointerGetDatum(DatumToBytea(skipNode->minimumValue,
values[Anum_columnar_chunk_minimum_value - 1] =
PointerGetDatum(DatumToBytea(chunk->minimumValue,
&tupleDescriptor->attrs[columnIndex]));
values[Anum_columnar_skipnodes_maximum_value - 1] =
PointerGetDatum(DatumToBytea(skipNode->maximumValue,
values[Anum_columnar_chunk_maximum_value - 1] =
PointerGetDatum(DatumToBytea(chunk->maximumValue,
&tupleDescriptor->attrs[columnIndex]));
}
else
{
nulls[Anum_columnar_skipnodes_minimum_value - 1] = true;
nulls[Anum_columnar_skipnodes_maximum_value - 1] = true;
nulls[Anum_columnar_chunk_minimum_value - 1] = true;
nulls[Anum_columnar_chunk_maximum_value - 1] = true;
}
InsertTupleAndEnforceConstraints(modifyState, values, nulls);
@ -455,14 +455,14 @@ SaveStripeSkipList(RelFileNode relfilenode, uint64 stripe, StripeSkipList *strip
}
FinishModifyRelation(modifyState);
table_close(cstoreSkipNodes, NoLock);
table_close(columnarChunk, NoLock);
CommandCounterIncrement();
}
/*
* ReadStripeSkipList fetches StripeSkipList for a given stripe.
* ReadStripeSkipList fetches chunk metadata for a given stripe.
*/
StripeSkipList *
ReadStripeSkipList(RelFileNode relfilenode, uint64 stripe, TupleDesc tupleDescriptor,
@ -475,111 +475,109 @@ ReadStripeSkipList(RelFileNode relfilenode, uint64 stripe, TupleDesc tupleDescri
ColumnarMetapage *metapage = ReadMetapage(relfilenode, false);
Oid cstoreSkipNodesOid = ColumnarSkipNodesRelationId();
Relation cstoreSkipNodes = table_open(cstoreSkipNodesOid, AccessShareLock);
Relation index = index_open(ColumnarSkipNodesIndexRelationId(), AccessShareLock);
Oid columnarChunkOid = ColumnarChunkRelationId();
Relation columnarChunk = table_open(columnarChunkOid, AccessShareLock);
Relation index = index_open(ColumnarChunkIndexRelationId(), AccessShareLock);
ScanKeyInit(&scanKey[0], Anum_columnar_skipnodes_storageid,
ScanKeyInit(&scanKey[0], Anum_columnar_chunk_storageid,
BTEqualStrategyNumber, F_OIDEQ, UInt64GetDatum(metapage->storageId));
ScanKeyInit(&scanKey[1], Anum_columnar_skipnodes_stripe,
ScanKeyInit(&scanKey[1], Anum_columnar_chunk_stripe,
BTEqualStrategyNumber, F_OIDEQ, Int32GetDatum(stripe));
SysScanDesc scanDescriptor = systable_beginscan_ordered(cstoreSkipNodes, index, NULL,
SysScanDesc scanDescriptor = systable_beginscan_ordered(columnarChunk, index, NULL,
2, scanKey);
StripeSkipList *skipList = palloc0(sizeof(StripeSkipList));
skipList->chunkCount = chunkCount;
skipList->columnCount = columnCount;
skipList->chunkSkipNodeArray = palloc0(columnCount * sizeof(ColumnChunkSkipNode *));
StripeSkipList *chunkList = palloc0(sizeof(StripeSkipList));
chunkList->chunkCount = chunkCount;
chunkList->columnCount = columnCount;
chunkList->chunkSkipNodeArray = palloc0(columnCount * sizeof(ColumnChunkSkipNode *));
for (columnIndex = 0; columnIndex < columnCount; columnIndex++)
{
skipList->chunkSkipNodeArray[columnIndex] =
chunkList->chunkSkipNodeArray[columnIndex] =
palloc0(chunkCount * sizeof(ColumnChunkSkipNode));
}
while (HeapTupleIsValid(heapTuple = systable_getnext(scanDescriptor)))
{
Datum datumArray[Natts_columnar_skipnodes];
bool isNullArray[Natts_columnar_skipnodes];
Datum datumArray[Natts_columnar_chunk];
bool isNullArray[Natts_columnar_chunk];
heap_deform_tuple(heapTuple, RelationGetDescr(cstoreSkipNodes), datumArray,
heap_deform_tuple(heapTuple, RelationGetDescr(columnarChunk), datumArray,
isNullArray);
int32 attr = DatumGetInt32(datumArray[Anum_columnar_skipnodes_attr - 1]);
int32 chunkIndex = DatumGetInt32(datumArray[Anum_columnar_skipnodes_chunk - 1]);
int32 attr = DatumGetInt32(datumArray[Anum_columnar_chunk_attr - 1]);
int32 chunkIndex = DatumGetInt32(datumArray[Anum_columnar_chunk_chunk - 1]);
if (attr <= 0 || attr > columnCount)
{
ereport(ERROR, (errmsg("invalid stripe skipnode entry"),
ereport(ERROR, (errmsg("invalid columnar chunk entry"),
errdetail("Attribute number out of range: %d", attr)));
}
if (chunkIndex < 0 || chunkIndex >= chunkCount)
{
ereport(ERROR, (errmsg("invalid stripe skipnode entry"),
ereport(ERROR, (errmsg("invalid columnar chunk entry"),
errdetail("Chunk number out of range: %d", chunkIndex)));
}
columnIndex = attr - 1;
ColumnChunkSkipNode *skipNode =
&skipList->chunkSkipNodeArray[columnIndex][chunkIndex];
skipNode->rowCount = DatumGetInt64(datumArray[Anum_columnar_skipnodes_row_count -
1]);
skipNode->valueChunkOffset =
DatumGetInt64(datumArray[Anum_columnar_skipnodes_value_stream_offset - 1]);
skipNode->valueLength =
DatumGetInt64(datumArray[Anum_columnar_skipnodes_value_stream_length - 1]);
skipNode->existsChunkOffset =
DatumGetInt64(datumArray[Anum_columnar_skipnodes_exists_stream_offset - 1]);
skipNode->existsLength =
DatumGetInt64(datumArray[Anum_columnar_skipnodes_exists_stream_length - 1]);
skipNode->valueCompressionType =
DatumGetInt32(datumArray[Anum_columnar_skipnodes_value_compression_type - 1]);
skipNode->valueCompressionLevel =
DatumGetInt32(datumArray[Anum_columnar_skipnodes_value_compression_level -
1]);
skipNode->decompressedValueSize =
DatumGetInt64(datumArray[Anum_columnar_skipnodes_value_decompressed_size -
ColumnChunkSkipNode *chunk =
&chunkList->chunkSkipNodeArray[columnIndex][chunkIndex];
chunk->rowCount = DatumGetInt64(datumArray[Anum_columnar_chunk_row_count -
1]);
chunk->valueChunkOffset =
DatumGetInt64(datumArray[Anum_columnar_chunk_value_stream_offset - 1]);
chunk->valueLength =
DatumGetInt64(datumArray[Anum_columnar_chunk_value_stream_length - 1]);
chunk->existsChunkOffset =
DatumGetInt64(datumArray[Anum_columnar_chunk_exists_stream_offset - 1]);
chunk->existsLength =
DatumGetInt64(datumArray[Anum_columnar_chunk_exists_stream_length - 1]);
chunk->valueCompressionType =
DatumGetInt32(datumArray[Anum_columnar_chunk_value_compression_type - 1]);
chunk->valueCompressionLevel =
DatumGetInt32(datumArray[Anum_columnar_chunk_value_compression_level - 1]);
chunk->decompressedValueSize =
DatumGetInt64(datumArray[Anum_columnar_chunk_value_decompressed_size - 1]);
if (isNullArray[Anum_columnar_skipnodes_minimum_value - 1] ||
isNullArray[Anum_columnar_skipnodes_maximum_value - 1])
if (isNullArray[Anum_columnar_chunk_minimum_value - 1] ||
isNullArray[Anum_columnar_chunk_maximum_value - 1])
{
skipNode->hasMinMax = false;
chunk->hasMinMax = false;
}
else
{
bytea *minValue = DatumGetByteaP(
datumArray[Anum_columnar_skipnodes_minimum_value - 1]);
datumArray[Anum_columnar_chunk_minimum_value - 1]);
bytea *maxValue = DatumGetByteaP(
datumArray[Anum_columnar_skipnodes_maximum_value - 1]);
datumArray[Anum_columnar_chunk_maximum_value - 1]);
skipNode->minimumValue =
chunk->minimumValue =
ByteaToDatum(minValue, &tupleDescriptor->attrs[columnIndex]);
skipNode->maximumValue =
chunk->maximumValue =
ByteaToDatum(maxValue, &tupleDescriptor->attrs[columnIndex]);
skipNode->hasMinMax = true;
chunk->hasMinMax = true;
}
}
systable_endscan_ordered(scanDescriptor);
index_close(index, NoLock);
table_close(cstoreSkipNodes, NoLock);
table_close(columnarChunk, NoLock);
return skipList;
return chunkList;
}
/*
* InsertStripeMetadataRow adds a row to columnar_stripes.
* InsertStripeMetadataRow adds a row to columnar.stripe.
*/
static void
InsertStripeMetadataRow(uint64 storageId, StripeMetadata *stripe)
{
bool nulls[Natts_columnar_stripes] = { 0 };
Datum values[Natts_columnar_stripes] = {
bool nulls[Natts_columnar_stripe] = { 0 };
Datum values[Natts_columnar_stripe] = {
UInt64GetDatum(storageId),
Int64GetDatum(stripe->id),
Int64GetDatum(stripe->fileOffset),
@ -590,10 +588,10 @@ InsertStripeMetadataRow(uint64 storageId, StripeMetadata *stripe)
Int64GetDatum(stripe->rowCount)
};
Oid cstoreStripesOid = ColumnarStripesRelationId();
Relation cstoreStripes = table_open(cstoreStripesOid, RowExclusiveLock);
Oid columnarStripesOid = ColumnarStripeRelationId();
Relation columnarStripes = table_open(columnarStripesOid, RowExclusiveLock);
ModifyState *modifyState = StartModifyRelation(cstoreStripes);
ModifyState *modifyState = StartModifyRelation(columnarStripes);
InsertTupleAndEnforceConstraints(modifyState, values, nulls);
@ -601,7 +599,7 @@ InsertStripeMetadataRow(uint64 storageId, StripeMetadata *stripe)
CommandCounterIncrement();
table_close(cstoreStripes, NoLock);
table_close(columnarStripes, NoLock);
}
@ -684,7 +682,7 @@ GetHighestUsedAddressAndId(uint64 storageId,
/*
* ReserveStripe reserves and stripe of given size for the given relation,
* and inserts it into columnar_stripes. It is guaranteed that concurrent
* and inserts it into columnar.stripe. It is guaranteed that concurrent
* writes won't overwrite the returned stripe.
*/
StripeMetadata
@ -765,54 +763,54 @@ ReadDataFileStripeList(uint64 storageId, Snapshot snapshot)
ScanKeyData scanKey[1];
HeapTuple heapTuple;
ScanKeyInit(&scanKey[0], Anum_columnar_stripes_storageid,
ScanKeyInit(&scanKey[0], Anum_columnar_stripe_storageid,
BTEqualStrategyNumber, F_OIDEQ, Int32GetDatum(storageId));
Oid cstoreStripesOid = ColumnarStripesRelationId();
Oid columnarStripesOid = ColumnarStripeRelationId();
Relation cstoreStripes = table_open(cstoreStripesOid, AccessShareLock);
Relation index = index_open(ColumnarStripesIndexRelationId(), AccessShareLock);
TupleDesc tupleDescriptor = RelationGetDescr(cstoreStripes);
Relation columnarStripes = table_open(columnarStripesOid, AccessShareLock);
Relation index = index_open(ColumnarStripeIndexRelationId(), AccessShareLock);
TupleDesc tupleDescriptor = RelationGetDescr(columnarStripes);
SysScanDesc scanDescriptor = systable_beginscan_ordered(cstoreStripes, index,
SysScanDesc scanDescriptor = systable_beginscan_ordered(columnarStripes, index,
snapshot, 1,
scanKey);
while (HeapTupleIsValid(heapTuple = systable_getnext(scanDescriptor)))
{
Datum datumArray[Natts_columnar_stripes];
bool isNullArray[Natts_columnar_stripes];
Datum datumArray[Natts_columnar_stripe];
bool isNullArray[Natts_columnar_stripe];
heap_deform_tuple(heapTuple, tupleDescriptor, datumArray, isNullArray);
StripeMetadata *stripeMetadata = palloc0(sizeof(StripeMetadata));
stripeMetadata->id = DatumGetInt64(datumArray[Anum_columnar_stripes_stripe - 1]);
stripeMetadata->id = DatumGetInt64(datumArray[Anum_columnar_stripe_stripe - 1]);
stripeMetadata->fileOffset = DatumGetInt64(
datumArray[Anum_columnar_stripes_file_offset - 1]);
datumArray[Anum_columnar_stripe_file_offset - 1]);
stripeMetadata->dataLength = DatumGetInt64(
datumArray[Anum_columnar_stripes_data_length - 1]);
datumArray[Anum_columnar_stripe_data_length - 1]);
stripeMetadata->columnCount = DatumGetInt32(
datumArray[Anum_columnar_stripes_column_count - 1]);
datumArray[Anum_columnar_stripe_column_count - 1]);
stripeMetadata->chunkCount = DatumGetInt32(
datumArray[Anum_columnar_stripes_chunk_count - 1]);
datumArray[Anum_columnar_stripe_chunk_count - 1]);
stripeMetadata->chunkRowCount = DatumGetInt32(
datumArray[Anum_columnar_stripes_chunk_row_count - 1]);
datumArray[Anum_columnar_stripe_chunk_row_count - 1]);
stripeMetadata->rowCount = DatumGetInt64(
datumArray[Anum_columnar_stripes_row_count - 1]);
datumArray[Anum_columnar_stripe_row_count - 1]);
stripeMetadataList = lappend(stripeMetadataList, stripeMetadata);
}
systable_endscan_ordered(scanDescriptor);
index_close(index, NoLock);
table_close(cstoreStripes, NoLock);
table_close(columnarStripes, NoLock);
return stripeMetadataList;
}
/*
* DeleteMetadataRows removes the rows with given relfilenode from columnar_stripes.
* DeleteMetadataRows removes the rows with given relfilenode from columnar.stripe.
*/
void
DeleteMetadataRows(RelFileNode relfilenode)
@ -838,23 +836,23 @@ DeleteMetadataRows(RelFileNode relfilenode)
return;
}
ScanKeyInit(&scanKey[0], Anum_columnar_stripes_storageid,
ScanKeyInit(&scanKey[0], Anum_columnar_stripe_storageid,
BTEqualStrategyNumber, F_INT8EQ, UInt64GetDatum(metapage->storageId));
Oid cstoreStripesOid = ColumnarStripesRelationId();
Relation cstoreStripes = try_relation_open(cstoreStripesOid, AccessShareLock);
if (cstoreStripes == NULL)
Oid columnarStripesOid = ColumnarStripeRelationId();
Relation columnarStripes = try_relation_open(columnarStripesOid, AccessShareLock);
if (columnarStripes == NULL)
{
/* extension has been dropped */
return;
}
Relation index = index_open(ColumnarStripesIndexRelationId(), AccessShareLock);
Relation index = index_open(ColumnarStripeIndexRelationId(), AccessShareLock);
SysScanDesc scanDescriptor = systable_beginscan_ordered(cstoreStripes, index, NULL,
SysScanDesc scanDescriptor = systable_beginscan_ordered(columnarStripes, index, NULL,
1, scanKey);
ModifyState *modifyState = StartModifyRelation(cstoreStripes);
ModifyState *modifyState = StartModifyRelation(columnarStripes);
HeapTuple heapTuple = systable_getnext(scanDescriptor);
while (HeapTupleIsValid(heapTuple))
@ -867,7 +865,7 @@ DeleteMetadataRows(RelFileNode relfilenode)
systable_endscan_ordered(scanDescriptor);
index_close(index, NoLock);
table_close(cstoreStripes, NoLock);
table_close(columnarStripes, NoLock);
}
@ -1051,24 +1049,24 @@ ByteaToDatum(bytea *bytes, Form_pg_attribute attrForm)
/*
* ColumnarStripesRelationId returns relation id of columnar_stripes.
* ColumnarStripeRelationId returns relation id of columnar.stripe.
* TODO: should we cache this similar to citus?
*/
static Oid
ColumnarStripesRelationId(void)
ColumnarStripeRelationId(void)
{
return get_relname_relid("columnar_stripes", ColumnarNamespaceId());
return get_relname_relid("stripe", ColumnarNamespaceId());
}
/*
* ColumnarStripesIndexRelationId returns relation id of columnar_stripes_idx.
* ColumnarStripeIndexRelationId returns relation id of columnar.stripe_pkey.
* TODO: should we cache this similar to citus?
*/
static Oid
ColumnarStripesIndexRelationId(void)
ColumnarStripeIndexRelationId(void)
{
return get_relname_relid("columnar_stripes_pkey", ColumnarNamespaceId());
return get_relname_relid("stripe_pkey", ColumnarNamespaceId());
}
@ -1093,29 +1091,29 @@ ColumnarOptionsIndexRegclass(void)
/*
* ColumnarSkipNodesRelationId returns relation id of columnar_skipnodes.
* ColumnarChunkRelationId returns relation id of columnar.chunk.
* TODO: should we cache this similar to citus?
*/
static Oid
ColumnarSkipNodesRelationId(void)
ColumnarChunkRelationId(void)
{
return get_relname_relid("columnar_skipnodes", ColumnarNamespaceId());
return get_relname_relid("chunk", ColumnarNamespaceId());
}
/*
* ColumnarSkipNodesIndexRelationId returns relation id of columnar_skipnodes_pkey.
* ColumnarChunkIndexRelationId returns relation id of columnar.chunk_pkey.
* TODO: should we cache this similar to citus?
*/
static Oid
ColumnarSkipNodesIndexRelationId(void)
ColumnarChunkIndexRelationId(void)
{
return get_relname_relid("columnar_skipnodes_pkey", ColumnarNamespaceId());
return get_relname_relid("chunk_pkey", ColumnarNamespaceId());
}
/*
* ColumnarNamespaceId returns namespace id of the schema we store cstore
* ColumnarNamespaceId returns namespace id of the schema we store columnar
* related tables.
*/
static Oid

22
src/backend/columnar/sql/columnar--9.5-1--10.0-1.sql
View File

@ -15,25 +15,25 @@ CREATE TABLE options (
COMMENT ON TABLE options IS 'columnar table specific options, maintained by alter_columnar_table_set';
CREATE TABLE columnar_stripes (
CREATE TABLE stripe (
storageid bigint NOT NULL,
stripe bigint NOT NULL,
stripeid bigint NOT NULL,
file_offset bigint NOT NULL,
data_length bigint NOT NULL,
column_count int NOT NULL,
chunk_count int NOT NULL,
chunk_row_count int NOT NULL,
row_count bigint NOT NULL,
PRIMARY KEY (storageid, stripe)
PRIMARY KEY (storageid, stripeid)
) WITH (user_catalog_table = true);
COMMENT ON TABLE columnar_stripes IS 'Columnar per stripe metadata';
COMMENT ON TABLE stripe IS 'Columnar per stripe metadata';
CREATE TABLE columnar_skipnodes (
CREATE TABLE chunk (
storageid bigint NOT NULL,
stripe bigint NOT NULL,
attr int NOT NULL,
chunk int NOT NULL,
stripeid bigint NOT NULL,
attnum int NOT NULL,
chunkid int NOT NULL,
row_count bigint NOT NULL,
minimum_value bytea,
maximum_value bytea,
@ -44,11 +44,11 @@ CREATE TABLE columnar_skipnodes (
value_compression_type int NOT NULL,
value_compression_level int NOT NULL,
value_decompressed_length bigint NOT NULL,
PRIMARY KEY (storageid, stripe, attr, chunk),
FOREIGN KEY (storageid, stripe) REFERENCES columnar_stripes(storageid, stripe) ON DELETE CASCADE
PRIMARY KEY (storageid, stripeid, attnum, chunkid),
FOREIGN KEY (storageid, stripeid) REFERENCES stripe(storageid, stripeid) ON DELETE CASCADE
) WITH (user_catalog_table = true);
COMMENT ON TABLE columnar_skipnodes IS 'Columnar per chunk metadata';
COMMENT ON TABLE chunk IS 'Columnar per chunk metadata';
DO $proc$
BEGIN

4
src/backend/columnar/sql/downgrades/columnar--10.0-1--9.5-1.sql
View File

@ -29,8 +29,8 @@ IF substring(current_Setting('server_version'), '\d+')::int >= 12 THEN
END IF;
END$proc$;
DROP TABLE columnar_skipnodes;
DROP TABLE columnar_stripes;
DROP TABLE chunk;
DROP TABLE stripe;
DROP TABLE options;
DROP SEQUENCE storageid_seq;

8
src/test/regress/expected/am_drop.out
View File

@ -12,12 +12,12 @@
-- 'postgres' directory is excluded from comparison to have the same result.
-- store postgres database oid
SELECT oid postgres_oid FROM pg_database WHERE datname = 'postgres' \gset
SELECT count(distinct storageid) AS columnar_stripes_before_drop FROM columnar.columnar_stripes \gset
SELECT count(distinct storageid) AS columnar_stripes_before_drop FROM columnar.stripe \gset
-- DROP columnar tables
DROP TABLE contestant;
DROP TABLE contestant_compressed;
-- make sure DROP deletes metadata
SELECT :columnar_stripes_before_drop - count(distinct storageid) FROM columnar.columnar_stripes;
SELECT :columnar_stripes_before_drop - count(distinct storageid) FROM columnar.stripe;
?column?
---------------------------------------------------------------------
2
@ -27,10 +27,10 @@ SELECT :columnar_stripes_before_drop - count(distinct storageid) FROM columnar.c
CREATE SCHEMA test_schema;
CREATE TABLE test_schema.test_table(data int) USING columnar;
INSERT INTO test_schema.test_table VALUES (1);
SELECT count(*) AS columnar_stripes_before_drop FROM columnar.columnar_stripes \gset
SELECT count(*) AS columnar_stripes_before_drop FROM columnar.stripe \gset
DROP SCHEMA test_schema CASCADE;
NOTICE: drop cascades to table test_schema.test_table
SELECT :columnar_stripes_before_drop - count(distinct storageid) FROM columnar.columnar_stripes;
SELECT :columnar_stripes_before_drop - count(distinct storageid) FROM columnar.stripe;
?column?
---------------------------------------------------------------------
1

8
src/test/regress/expected/am_matview.out
View File

@ -68,13 +68,13 @@ SELECT * FROM t_view a ORDER BY a;
-- verify that we have created metadata entries for the materialized view
SELECT columnar_relation_storageid(oid) AS storageid
FROM pg_class WHERE relname='t_view' \gset
SELECT count(*) FROM columnar.columnar_stripes WHERE storageid=:storageid;
SELECT count(*) FROM columnar.stripe WHERE storageid=:storageid;
count
---------------------------------------------------------------------
1
(1 row)
SELECT count(*) FROM columnar.columnar_skipnodes WHERE storageid=:storageid;
SELECT count(*) FROM columnar.chunk WHERE storageid=:storageid;
count
---------------------------------------------------------------------
3
@ -83,13 +83,13 @@ SELECT count(*) FROM columnar.columnar_skipnodes WHERE storageid=:storageid;
DROP TABLE t CASCADE;
NOTICE: drop cascades to materialized view t_view
-- dropping must remove metadata
SELECT count(*) FROM columnar.columnar_stripes WHERE storageid=:storageid;
SELECT count(*) FROM columnar.stripe WHERE storageid=:storageid;
count
---------------------------------------------------------------------
0
(1 row)
SELECT count(*) FROM columnar.columnar_skipnodes WHERE storageid=:storageid;
SELECT count(*) FROM columnar.chunk WHERE storageid=:storageid;
count
---------------------------------------------------------------------
0

2
src/test/regress/expected/am_recursive.out
View File

@ -11,7 +11,7 @@ $$ LANGUAGE SQL;
INSERT INTO t2 SELECT i, f(i) FROM generate_series(1, 5) i;
-- there are no subtransactions, so above statement should batch
-- INSERTs inside the UDF and create on stripe per table.
SELECT relname, count(*) FROM columnar.columnar_stripes a, pg_class b
SELECT relname, count(*) FROM columnar.stripe a, pg_class b
WHERE columnar_relation_storageid(b.oid)=a.storageid AND relname IN ('t1', 't2')
GROUP BY relname
ORDER BY relname;

2
src/test/regress/expected/am_rollback.out
View File

@ -3,7 +3,7 @@
--
CREATE TABLE t(a int, b int) USING columnar;
CREATE VIEW t_stripes AS
SELECT * FROM columnar.columnar_stripes a, pg_class b
SELECT * FROM columnar.stripe a, pg_class b
WHERE a.storageid = columnar_relation_storageid(b.oid) AND b.relname = 't';
BEGIN;
INSERT INTO t SELECT i, i+1 FROM generate_series(1, 10) i;

6
src/test/regress/expected/am_truncate.out
View File

@ -15,7 +15,7 @@ CREATE TABLE columnar_truncate_test_second (a int, b int) USING columnar;
-- COMPRESSED
CREATE TABLE columnar_truncate_test_compressed (a int, b int) USING columnar;
CREATE TABLE columnar_truncate_test_regular (a int, b int);
SELECT count(distinct storageid) AS columnar_data_files_before_truncate FROM columnar.columnar_stripes \gset
SELECT count(distinct storageid) AS columnar_data_files_before_truncate FROM columnar.stripe \gset
INSERT INTO columnar_truncate_test select a, a from generate_series(1, 10) a;
set columnar.compression = 'pglz';
INSERT INTO columnar_truncate_test_compressed select a, a from generate_series(1, 10) a;
@ -147,7 +147,7 @@ SELECT * from columnar_truncate_test;
(0 rows)
-- make sure TRUNATE deletes metadata for old relfilenode
SELECT :columnar_data_files_before_truncate - count(distinct storageid) FROM columnar.columnar_stripes;
SELECT :columnar_data_files_before_truncate - count(distinct storageid) FROM columnar.stripe;
?column?
---------------------------------------------------------------------
0
@ -161,7 +161,7 @@ TRUNCATE columnar_same_transaction_truncate;
INSERT INTO columnar_same_transaction_truncate SELECT * FROM generate_series(20, 23);
COMMIT;
-- should output "1" for the newly created relation
SELECT count(distinct storageid) - :columnar_data_files_before_truncate FROM columnar.columnar_stripes;
SELECT count(distinct storageid) - :columnar_data_files_before_truncate FROM columnar.stripe;
?column?
---------------------------------------------------------------------
1

20
src/test/regress/expected/am_vacuum.out
View File

@ -1,8 +1,8 @@
SET columnar.compression TO 'none';
SELECT count(distinct storageid) AS columnar_table_count FROM columnar.columnar_stripes \gset
SELECT count(distinct storageid) AS columnar_table_count FROM columnar.stripe \gset
CREATE TABLE t(a int, b int) USING columnar;
CREATE VIEW t_stripes AS
SELECT * FROM columnar.columnar_stripes a, pg_class b
SELECT * FROM columnar.stripe a, pg_class b
WHERE a.storageid = columnar_relation_storageid(b.oid) AND b.relname='t';
SELECT count(*) FROM t_stripes;
count
@ -74,10 +74,10 @@ SELECT count(*) FROM t_stripes;
-- VACUUM FULL doesn't reclaim dropped columns, but converts them to NULLs
ALTER TABLE t DROP COLUMN a;
SELECT stripe, attr, chunk, minimum_value IS NULL, maximum_value IS NULL
FROM columnar.columnar_skipnodes a, pg_class b
SELECT stripeid, attnum, chunkid, minimum_value IS NULL, maximum_value IS NULL
FROM columnar.chunk a, pg_class b
WHERE a.storageid = columnar_relation_storageid(b.oid) AND b.relname='t' ORDER BY 1, 2, 3;
stripe | attr | chunk | ?column? | ?column?
stripeid | attnum | chunkid | ?column? | ?column?
---------------------------------------------------------------------
1 | 1 | 0 | f | f
1 | 2 | 0 | f | f
@ -88,10 +88,10 @@ WHERE a.storageid = columnar_relation_storageid(b.oid) AND b.relname='t' ORDER B
(6 rows)
VACUUM FULL t;
SELECT stripe, attr, chunk, minimum_value IS NULL, maximum_value IS NULL
FROM columnar.columnar_skipnodes a, pg_class b
SELECT stripeid, attnum, chunkid, minimum_value IS NULL, maximum_value IS NULL
FROM columnar.chunk a, pg_class b
WHERE a.storageid = columnar_relation_storageid(b.oid) AND b.relname='t' ORDER BY 1, 2, 3;
stripe | attr | chunk | ?column? | ?column?
stripeid | attnum | chunkid | ?column? | ?column?
---------------------------------------------------------------------
1 | 1 | 0 | t | t
1 | 2 | 0 | f | f
@ -102,7 +102,7 @@ WHERE a.storageid = columnar_relation_storageid(b.oid) AND b.relname='t' ORDER B
(6 rows)
-- Make sure we cleaned-up the transient table metadata after VACUUM FULL commands
SELECT count(distinct storageid) - :columnar_table_count FROM columnar.columnar_stripes;
SELECT count(distinct storageid) - :columnar_table_count FROM columnar.stripe;
?column?
---------------------------------------------------------------------
1
@ -242,7 +242,7 @@ chunk count: 8, containing data for dropped columns: 0, none compressed: 2, pglz
DROP TABLE t;
DROP VIEW t_stripes;
-- Make sure we cleaned the metadata for t too
SELECT count(distinct storageid) - :columnar_table_count FROM columnar.columnar_stripes;
SELECT count(distinct storageid) - :columnar_table_count FROM columnar.stripe;
?column?
---------------------------------------------------------------------
0

4
src/test/regress/expected/multi_extension.out
View File

@ -499,9 +499,9 @@ SELECT * FROM print_extension_changes();
| function worker_change_sequence_dependency(regclass,regclass,regclass)
| schema columnar
| sequence columnar.storageid_seq
| table columnar.columnar_skipnodes
| table columnar.columnar_stripes
| table columnar.chunk
| table columnar.options
| table columnar.stripe
| view citus_shards
| view citus_tables
| view time_partitions

4
src/test/regress/expected/multi_extension_0.out
View File

@ -495,9 +495,9 @@ SELECT * FROM print_extension_changes();
| function worker_change_sequence_dependency(regclass,regclass,regclass)
| schema columnar
| sequence columnar.storageid_seq
| table columnar.columnar_skipnodes
| table columnar.columnar_stripes
| table columnar.chunk
| table columnar.options
| table columnar.stripe
| view citus_shards
| view citus_tables
| view time_partitions

4
src/test/regress/expected/upgrade_list_citus_objects.out
View File

@ -209,9 +209,9 @@ ORDER BY 1;
sequence pg_dist_placement_placementid_seq
sequence pg_dist_shardid_seq
table citus.pg_dist_object
table columnar.columnar_skipnodes
table columnar.columnar_stripes
table columnar.chunk
table columnar.options
table columnar.stripe
table pg_dist_authinfo
table pg_dist_colocation
table pg_dist_local_group

4
src/test/regress/expected/upgrade_list_citus_objects_0.out
View File

@ -205,9 +205,9 @@ ORDER BY 1;
sequence pg_dist_placement_placementid_seq
sequence pg_dist_shardid_seq
table citus.pg_dist_object
table columnar.columnar_skipnodes
table columnar.columnar_stripes
table columnar.chunk
table columnar.options
table columnar.stripe
table pg_dist_authinfo
table pg_dist_colocation
table pg_dist_local_group

8
src/test/regress/sql/am_drop.sql
View File

@ -15,23 +15,23 @@
-- store postgres database oid
SELECT oid postgres_oid FROM pg_database WHERE datname = 'postgres' \gset
SELECT count(distinct storageid) AS columnar_stripes_before_drop FROM columnar.columnar_stripes \gset
SELECT count(distinct storageid) AS columnar_stripes_before_drop FROM columnar.stripe \gset
-- DROP columnar tables
DROP TABLE contestant;
DROP TABLE contestant_compressed;
-- make sure DROP deletes metadata
SELECT :columnar_stripes_before_drop - count(distinct storageid) FROM columnar.columnar_stripes;
SELECT :columnar_stripes_before_drop - count(distinct storageid) FROM columnar.stripe;
-- Create a columnar table under a schema and drop it.
CREATE SCHEMA test_schema;
CREATE TABLE test_schema.test_table(data int) USING columnar;
INSERT INTO test_schema.test_table VALUES (1);
SELECT count(*) AS columnar_stripes_before_drop FROM columnar.columnar_stripes \gset
SELECT count(*) AS columnar_stripes_before_drop FROM columnar.stripe \gset
DROP SCHEMA test_schema CASCADE;
SELECT :columnar_stripes_before_drop - count(distinct storageid) FROM columnar.columnar_stripes;
SELECT :columnar_stripes_before_drop - count(distinct storageid) FROM columnar.stripe;
SELECT current_database() datname \gset

8
src/test/regress/sql/am_matview.sql
View File

@ -38,11 +38,11 @@ SELECT * FROM t_view a ORDER BY a;
SELECT columnar_relation_storageid(oid) AS storageid
FROM pg_class WHERE relname='t_view' \gset
SELECT count(*) FROM columnar.columnar_stripes WHERE storageid=:storageid;
SELECT count(*) FROM columnar.columnar_skipnodes WHERE storageid=:storageid;
SELECT count(*) FROM columnar.stripe WHERE storageid=:storageid;
SELECT count(*) FROM columnar.chunk WHERE storageid=:storageid;
DROP TABLE t CASCADE;
-- dropping must remove metadata
SELECT count(*) FROM columnar.columnar_stripes WHERE storageid=:storageid;
SELECT count(*) FROM columnar.columnar_skipnodes WHERE storageid=:storageid;
SELECT count(*) FROM columnar.stripe WHERE storageid=:storageid;
SELECT count(*) FROM columnar.chunk WHERE storageid=:storageid;

2
src/test/regress/sql/am_recursive.sql
View File

@ -15,7 +15,7 @@ INSERT INTO t2 SELECT i, f(i) FROM generate_series(1, 5) i;
-- there are no subtransactions, so above statement should batch
-- INSERTs inside the UDF and create on stripe per table.
SELECT relname, count(*) FROM columnar.columnar_stripes a, pg_class b
SELECT relname, count(*) FROM columnar.stripe a, pg_class b
WHERE columnar_relation_storageid(b.oid)=a.storageid AND relname IN ('t1', 't2')
GROUP BY relname
ORDER BY relname;

2
src/test/regress/sql/am_rollback.sql
View File

@ -5,7 +5,7 @@
CREATE TABLE t(a int, b int) USING columnar;
CREATE VIEW t_stripes AS
SELECT * FROM columnar.columnar_stripes a, pg_class b
SELECT * FROM columnar.stripe a, pg_class b
WHERE a.storageid = columnar_relation_storageid(b.oid) AND b.relname = 't';
BEGIN;

6
src/test/regress/sql/am_truncate.sql
View File

@ -13,7 +13,7 @@ CREATE TABLE columnar_truncate_test_second (a int, b int) USING columnar;
CREATE TABLE columnar_truncate_test_compressed (a int, b int) USING columnar;
CREATE TABLE columnar_truncate_test_regular (a int, b int);
SELECT count(distinct storageid) AS columnar_data_files_before_truncate FROM columnar.columnar_stripes \gset
SELECT count(distinct storageid) AS columnar_data_files_before_truncate FROM columnar.stripe \gset
INSERT INTO columnar_truncate_test select a, a from generate_series(1, 10) a;
@ -63,7 +63,7 @@ TRUNCATE TABLE columnar_truncate_test;
SELECT * from columnar_truncate_test;
-- make sure TRUNATE deletes metadata for old relfilenode
SELECT :columnar_data_files_before_truncate - count(distinct storageid) FROM columnar.columnar_stripes;
SELECT :columnar_data_files_before_truncate - count(distinct storageid) FROM columnar.stripe;
-- test if truncation in the same transaction that created the table works properly
BEGIN;
@ -74,7 +74,7 @@ INSERT INTO columnar_same_transaction_truncate SELECT * FROM generate_series(20,
COMMIT;
-- should output "1" for the newly created relation
SELECT count(distinct storageid) - :columnar_data_files_before_truncate FROM columnar.columnar_stripes;
SELECT count(distinct storageid) - :columnar_data_files_before_truncate FROM columnar.stripe;
SELECT * FROM columnar_same_transaction_truncate;
DROP TABLE columnar_same_transaction_truncate;

16
src/test/regress/sql/am_vacuum.sql
View File

@ -1,11 +1,11 @@
SET columnar.compression TO 'none';
SELECT count(distinct storageid) AS columnar_table_count FROM columnar.columnar_stripes \gset
SELECT count(distinct storageid) AS columnar_table_count FROM columnar.stripe \gset
CREATE TABLE t(a int, b int) USING columnar;
CREATE VIEW t_stripes AS
SELECT * FROM columnar.columnar_stripes a, pg_class b
SELECT * FROM columnar.stripe a, pg_class b
WHERE a.storageid = columnar_relation_storageid(b.oid) AND b.relname='t';
SELECT count(*) FROM t_stripes;
@ -38,18 +38,18 @@ SELECT count(*) FROM t_stripes;
-- VACUUM FULL doesn't reclaim dropped columns, but converts them to NULLs
ALTER TABLE t DROP COLUMN a;
SELECT stripe, attr, chunk, minimum_value IS NULL, maximum_value IS NULL
FROM columnar.columnar_skipnodes a, pg_class b
SELECT stripeid, attnum, chunkid, minimum_value IS NULL, maximum_value IS NULL
FROM columnar.chunk a, pg_class b
WHERE a.storageid = columnar_relation_storageid(b.oid) AND b.relname='t' ORDER BY 1, 2, 3;
VACUUM FULL t;
SELECT stripe, attr, chunk, minimum_value IS NULL, maximum_value IS NULL
FROM columnar.columnar_skipnodes a, pg_class b
SELECT stripeid, attnum, chunkid, minimum_value IS NULL, maximum_value IS NULL
FROM columnar.chunk a, pg_class b
WHERE a.storageid = columnar_relation_storageid(b.oid) AND b.relname='t' ORDER BY 1, 2, 3;
-- Make sure we cleaned-up the transient table metadata after VACUUM FULL commands
SELECT count(distinct storageid) - :columnar_table_count FROM columnar.columnar_stripes;
SELECT count(distinct storageid) - :columnar_table_count FROM columnar.stripe;
-- do this in a transaction so concurrent autovacuum doesn't interfere with results
BEGIN;
@ -112,7 +112,7 @@ DROP TABLE t;
DROP VIEW t_stripes;
-- Make sure we cleaned the metadata for t too
SELECT count(distinct storageid) - :columnar_table_count FROM columnar.columnar_stripes;
SELECT count(distinct storageid) - :columnar_table_count FROM columnar.stripe;
-- A table with high compression ratio
SET columnar.compression TO 'pglz';