external
/
citus
mirror of https://github.com/citusdata/citus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Rename cstore_tables to cstore_data_files

merge-cstore-pykello
Hadi Moshayedi 2020-10-01 21:23:06 -07:00
parent a87c15a1e1
commit a70b0c362e
12 changed files with 125 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
cstore.c
View File

@ -99,14 +99,3 @@ ParseCompressionType(const char *compressionTypeString)
return compressionType; return compressionType;
} }
/*
* InitializeCStoreTableFile initializes metadata for the given relation
* file node.
*/
void
InitializeCStoreTableFile(Oid relNode, CStoreOptions *cstoreOptions)
{
InitCStoreTableMetadata(relNode, cstoreOptions->blockRowCount);
}

18
cstore.h
View File

@ -87,12 +87,12 @@ typedef struct StripeMetadata
} StripeMetadata; } StripeMetadata;
/* TableMetadata represents the metadata of a cstore file. */ /* DataFileMetadata represents the metadata of a cstore file. */
typedef struct TableMetadata typedef struct DataFileMetadata
{ {
List *stripeMetadataList; List *stripeMetadataList;
uint64 blockRowCount; uint64 blockRowCount;
} TableMetadata; } DataFileMetadata;
/* ColumnBlockSkipNode contains statistics for a ColumnBlockData. */ /* ColumnBlockSkipNode contains statistics for a ColumnBlockData. */
@ -192,7 +192,7 @@ typedef struct StripeBuffers
/* TableReadState represents state of a cstore file read operation. */ /* TableReadState represents state of a cstore file read operation. */
typedef struct TableReadState typedef struct TableReadState
{ {
TableMetadata *tableMetadata; DataFileMetadata *datafileMetadata;
StripeMetadata *currentStripeMetadata; StripeMetadata *currentStripeMetadata;
TupleDesc tupleDescriptor; TupleDesc tupleDescriptor;
Relation relation; Relation relation;
@ -217,7 +217,7 @@ typedef struct TableReadState
/* TableWriteState represents state of a cstore file write operation. */ /* TableWriteState represents state of a cstore file write operation. */
typedef struct TableWriteState typedef struct TableWriteState
{ {
TableMetadata *tableMetadata; DataFileMetadata *datafileMetadata;
CompressionType compressionType; CompressionType compressionType;
TupleDesc tupleDescriptor; TupleDesc tupleDescriptor;
FmgrInfo **comparisonFunctionArray; FmgrInfo **comparisonFunctionArray;
@ -248,7 +248,6 @@ extern int cstore_block_row_count;
extern void cstore_init(void); extern void cstore_init(void);
extern CompressionType ParseCompressionType(const char *compressionTypeString); extern CompressionType ParseCompressionType(const char *compressionTypeString);
extern void InitializeCStoreTableFile(Oid relNode, CStoreOptions *cstoreOptions);
/* Function declarations for writing to a cstore file */ /* Function declarations for writing to a cstore file */
extern TableWriteState * CStoreBeginWrite(Relation relation, extern TableWriteState * CStoreBeginWrite(Relation relation,
@ -281,11 +280,10 @@ extern bool CompressBuffer(StringInfo inputBuffer, StringInfo outputBuffer,
extern StringInfo DecompressBuffer(StringInfo buffer, CompressionType compressionType); extern StringInfo DecompressBuffer(StringInfo buffer, CompressionType compressionType);
/* cstore_metadata_tables.c */ /* cstore_metadata_tables.c */
extern bool IsCStoreStorage(Oid relfilenode); extern void DeleteDataFileMetadataRowIfExists(Oid relfilenode);
extern void DeleteTableMetadataRowIfExists(Oid relfilenode); extern void InitCStoreDataFileMetadata(Oid relfilenode, int blockRowCount);
extern void InitCStoreTableMetadata(Oid relfilenode, int blockRowCount);
extern void InsertStripeMetadataRow(Oid relfilenode, StripeMetadata *stripe); extern void InsertStripeMetadataRow(Oid relfilenode, StripeMetadata *stripe);
extern TableMetadata * ReadTableMetadata(Oid relfilenode); extern DataFileMetadata * ReadDataFileMetadata(Oid relfilenode);
extern void SaveStripeSkipList(Oid relfilenode, uint64 stripe, extern void SaveStripeSkipList(Oid relfilenode, uint64 stripe,
StripeSkipList *stripeSkipList, StripeSkipList *stripeSkipList,
TupleDesc tupleDescriptor); TupleDesc tupleDescriptor);

10
cstore_fdw--1.7.sql
View File

@ -31,7 +31,7 @@ RETURNS bigint
AS 'MODULE_PATHNAME' AS 'MODULE_PATHNAME'
LANGUAGE C STRICT; LANGUAGE C STRICT;
CREATE TABLE cstore_tables ( CREATE TABLE cstore_data_files (
relfilenode oid NOT NULL, relfilenode oid NOT NULL,
block_row_count int NOT NULL, block_row_count int NOT NULL,
version_major bigint NOT NULL, version_major bigint NOT NULL,
@ -39,7 +39,7 @@ CREATE TABLE cstore_tables (
PRIMARY KEY (relfilenode) PRIMARY KEY (relfilenode)
) WITH (user_catalog_table = true); ) WITH (user_catalog_table = true);
COMMENT ON TABLE cstore_tables IS 'CStore table wide metadata'; COMMENT ON TABLE cstore_data_files IS 'CStore data file wide metadata';
CREATE TABLE cstore_stripes ( CREATE TABLE cstore_stripes (
relfilenode oid NOT NULL, relfilenode oid NOT NULL,
@ -51,10 +51,10 @@ CREATE TABLE cstore_stripes (
block_row_count int NOT NULL, block_row_count int NOT NULL,
row_count bigint NOT NULL, row_count bigint NOT NULL,
PRIMARY KEY (relfilenode, stripe), PRIMARY KEY (relfilenode, stripe),
FOREIGN KEY (relfilenode) REFERENCES cstore_tables(relfilenode) ON DELETE CASCADE INITIALLY DEFERRED FOREIGN KEY (relfilenode) REFERENCES cstore_data_files(relfilenode) ON DELETE CASCADE INITIALLY DEFERRED
) WITH (user_catalog_table = true); ) WITH (user_catalog_table = true);
COMMENT ON TABLE cstore_tables IS 'CStore per stripe metadata'; COMMENT ON TABLE cstore_stripes IS 'CStore per stripe metadata';
CREATE TABLE cstore_skipnodes ( CREATE TABLE cstore_skipnodes (
relfilenode oid NOT NULL, relfilenode oid NOT NULL,
@ -73,4 +73,4 @@ CREATE TABLE cstore_skipnodes (
FOREIGN KEY (relfilenode, stripe) REFERENCES cstore_stripes(relfilenode, stripe) ON DELETE CASCADE INITIALLY DEFERRED FOREIGN KEY (relfilenode, stripe) REFERENCES cstore_stripes(relfilenode, stripe) ON DELETE CASCADE INITIALLY DEFERRED
) WITH (user_catalog_table = true); ) WITH (user_catalog_table = true);
COMMENT ON TABLE cstore_tables IS 'CStore per block metadata'; COMMENT ON TABLE cstore_skipnodes IS 'CStore per block metadata';

19
cstore_fdw.c
View File

@ -267,16 +267,8 @@ cstore_ddl_event_end_trigger(PG_FUNCTION_ARGS)
Oid relationId = RangeVarGetRelid(createStatement->base.relation, Oid relationId = RangeVarGetRelid(createStatement->base.relation,
AccessShareLock, false); AccessShareLock, false);
Relation relation = cstore_fdw_open(relationId, AccessExclusiveLock); Relation relation = cstore_fdw_open(relationId, AccessExclusiveLock);
CStoreOptions *options = CStoreGetOptions(relationId);
/* InitCStoreDataFileMetadata(relation->rd_node.relNode, options->blockRowCount);
* Make sure database directory exists before creating a table.
* This is necessary when a foreign server is created inside
* a template database and a new database is created out of it.
* We have no chance to hook into server creation to create data
* directory for it during database creation time.
*/
InitializeCStoreTableFile(relation->rd_node.relNode,
CStoreGetOptions(relationId));
heap_close(relation, AccessExclusiveLock); heap_close(relation, AccessExclusiveLock);
} }
} }
@ -369,6 +361,7 @@ CStoreProcessUtility(Node * parseTree, const char * queryString,
CALL_PREVIOUS_UTILITY(parseTree, queryString, context, paramListInfo, CALL_PREVIOUS_UTILITY(parseTree, queryString, context, paramListInfo,
destReceiver, completionTag); destReceiver, completionTag);
} }
/* handle other utility statements */ /* handle other utility statements */
else else
{ {
@ -782,12 +775,12 @@ TruncateCStoreTables(List *cstoreRelationList)
{ {
Relation relation = (Relation) lfirst(relationCell); Relation relation = (Relation) lfirst(relationCell);
Oid relationId = relation->rd_id; Oid relationId = relation->rd_id;
CStoreOptions *options = CStoreGetOptions(relationId);
Assert(IsCStoreFdwTable(relationId)); Assert(IsCStoreFdwTable(relationId));
FdwNewRelFileNode(relation); FdwNewRelFileNode(relation);
InitializeCStoreTableFile(relation->rd_node.relNode, InitCStoreDataFileMetadata(relation->rd_node.relNode, options->blockRowCount);
CStoreGetOptions(relationId));
} }
} }
@ -2225,7 +2218,7 @@ CStoreFdwObjectAccessHook(ObjectAccessType access, Oid classId, Oid objectId,
Relation rel = cstore_fdw_open(objectId, AccessExclusiveLock); Relation rel = cstore_fdw_open(objectId, AccessExclusiveLock);
RelationOpenSmgr(rel); RelationOpenSmgr(rel);
RelationDropStorage(rel); RelationDropStorage(rel);
DeleteTableMetadataRowIfExists(rel->rd_node.relNode); DeleteDataFileMetadataRowIfExists(rel->rd_node.relNode);
/* keep the lock since we did physical changes to the relation */ /* keep the lock since we did physical changes to the relation */
relation_close(rel, NoLock); relation_close(rel, NoLock);

126
cstore_metadata_tables.c
View File

@ -45,12 +45,12 @@ typedef struct
static Oid CStoreStripesRelationId(void); static Oid CStoreStripesRelationId(void);
static Oid CStoreStripesIndexRelationId(void); static Oid CStoreStripesIndexRelationId(void);
static Oid CStoreTablesRelationId(void); static Oid CStoreDataFilesRelationId(void);
static Oid CStoreTablesIndexRelationId(void); static Oid CStoreDataFilesIndexRelationId(void);
static Oid CStoreSkipNodesRelationId(void); static Oid CStoreSkipNodesRelationId(void);
static Oid CStoreSkipNodesIndexRelationId(void); static Oid CStoreSkipNodesIndexRelationId(void);
static Oid CStoreNamespaceId(void); static Oid CStoreNamespaceId(void);
static bool ReadCStoreTables(Oid relfilenode, uint64 *blockRowCount); static bool ReadCStoreDataFiles(Oid relfilenode, uint64 *blockRowCount);
static ModifyState * StartModifyRelation(Relation rel); static ModifyState * StartModifyRelation(Relation rel);
static void InsertTupleAndEnforceConstraints(ModifyState *state, Datum *values, static void InsertTupleAndEnforceConstraints(ModifyState *state, Datum *values,
bool *nulls); bool *nulls);
@ -61,11 +61,11 @@ static bytea * DatumToBytea(Datum value, Form_pg_attribute attrForm);
static Datum ByteaToDatum(bytea *bytes, Form_pg_attribute attrForm); static Datum ByteaToDatum(bytea *bytes, Form_pg_attribute attrForm);
/* constants for cstore_table */ /* constants for cstore_table */
#define Natts_cstore_tables 4 #define Natts_cstore_data_files 4
#define Anum_cstore_tables_relfilenode 1 #define Anum_cstore_data_files_relfilenode 1
#define Anum_cstore_tables_block_row_count 2 #define Anum_cstore_data_files_block_row_count 2
#define Anum_cstore_tables_version_major 3 #define Anum_cstore_data_files_version_major 3
#define Anum_cstore_tables_version_minor 4 #define Anum_cstore_data_files_version_minor 4
/* constants for cstore_stripe */ /* constants for cstore_stripe */
#define Natts_cstore_stripes 8 #define Natts_cstore_stripes 8
@ -95,35 +95,36 @@ static Datum ByteaToDatum(bytea *bytes, Form_pg_attribute attrForm);
/* /*
* InitCStoreTableMetadata adds a record for the given relation in cstore_table. * InitCStoreDataFileMetadata adds a record for the given relfilenode
* in cstore_data_files.
*/ */
void void
InitCStoreTableMetadata(Oid relfilenode, int blockRowCount) InitCStoreDataFileMetadata(Oid relfilenode, int blockRowCount)
{ {
Oid cstoreTablesOid = InvalidOid; Oid cstoreDataFilesOid = InvalidOid;
Relation cstoreTables = NULL; Relation cstoreDataFiles = NULL;
ModifyState *modifyState = NULL; ModifyState *modifyState = NULL;
bool nulls[Natts_cstore_tables] = { 0 }; bool nulls[Natts_cstore_data_files] = { 0 };
Datum values[Natts_cstore_tables] = { Datum values[Natts_cstore_data_files] = {
ObjectIdGetDatum(relfilenode), ObjectIdGetDatum(relfilenode),
Int32GetDatum(blockRowCount), Int32GetDatum(blockRowCount),
Int32GetDatum(CSTORE_VERSION_MAJOR), Int32GetDatum(CSTORE_VERSION_MAJOR),
Int32GetDatum(CSTORE_VERSION_MINOR) Int32GetDatum(CSTORE_VERSION_MINOR)
}; };
DeleteTableMetadataRowIfExists(relfilenode); DeleteDataFileMetadataRowIfExists(relfilenode);
cstoreTablesOid = CStoreTablesRelationId(); cstoreDataFilesOid = CStoreDataFilesRelationId();
cstoreTables = heap_open(cstoreTablesOid, RowExclusiveLock); cstoreDataFiles = heap_open(cstoreDataFilesOid, RowExclusiveLock);
modifyState = StartModifyRelation(cstoreTables); modifyState = StartModifyRelation(cstoreDataFiles);
InsertTupleAndEnforceConstraints(modifyState, values, nulls); InsertTupleAndEnforceConstraints(modifyState, values, nulls);
FinishModifyRelation(modifyState); FinishModifyRelation(modifyState);
CommandCounterIncrement(); CommandCounterIncrement();
heap_close(cstoreTables, NoLock); heap_close(cstoreDataFiles, NoLock);
} }
@ -338,11 +339,11 @@ InsertStripeMetadataRow(Oid relfilenode, StripeMetadata *stripe)
/* /*
* ReadTableMetadata constructs TableMetadata for a given relfilenode by reading * ReadDataFileMetadata constructs DataFileMetadata for a given relfilenode by reading
* from cstore_tables and cstore_stripes. * from cstore_data_files and cstore_stripes.
*/ */
TableMetadata * DataFileMetadata *
ReadTableMetadata(Oid relfilenode) ReadDataFileMetadata(Oid relfilenode)
{ {
Oid cstoreStripesOid = InvalidOid; Oid cstoreStripesOid = InvalidOid;
Relation cstoreStripes = NULL; Relation cstoreStripes = NULL;
@ -353,8 +354,8 @@ ReadTableMetadata(Oid relfilenode)
HeapTuple heapTuple; HeapTuple heapTuple;
bool found = false; bool found = false;
TableMetadata *tableMetadata = palloc0(sizeof(TableMetadata)); DataFileMetadata *datafileMetadata = palloc0(sizeof(DataFileMetadata));
found = ReadCStoreTables(relfilenode, &tableMetadata->blockRowCount); found = ReadCStoreDataFiles(relfilenode, &datafileMetadata->blockRowCount);
if (!found) if (!found)
{ {
ereport(ERROR, (errmsg("Relfilenode %d doesn't belong to a cstore table.", ereport(ERROR, (errmsg("Relfilenode %d doesn't belong to a cstore table.",
@ -394,7 +395,8 @@ ReadTableMetadata(Oid relfilenode)
stripeMetadata->rowCount = DatumGetInt64( stripeMetadata->rowCount = DatumGetInt64(
datumArray[Anum_cstore_stripes_row_count - 1]); datumArray[Anum_cstore_stripes_row_count - 1]);
tableMetadata->stripeMetadataList = lappend(tableMetadata->stripeMetadataList, datafileMetadata->stripeMetadataList = lappend(
datafileMetadata->stripeMetadataList,
stripeMetadata); stripeMetadata);
} }
@ -402,32 +404,32 @@ ReadTableMetadata(Oid relfilenode)
index_close(index, NoLock); index_close(index, NoLock);
heap_close(cstoreStripes, NoLock); heap_close(cstoreStripes, NoLock);
return tableMetadata; return datafileMetadata;
} }
/* /*
* ReadCStoreTables reads corresponding record from cstore_tables. Returns false if * ReadCStoreDataFiles reads corresponding record from cstore_data_files. Returns
* table was not found in cstore_tables. * false if table was not found in cstore_data_files.
*/ */
static bool static bool
ReadCStoreTables(Oid relfilenode, uint64 *blockRowCount) ReadCStoreDataFiles(Oid relfilenode, uint64 *blockRowCount)
{ {
bool found = false; bool found = false;
Oid cstoreTablesOid = InvalidOid; Oid cstoreDataFilesOid = InvalidOid;
Relation cstoreTables = NULL; Relation cstoreDataFiles = NULL;
Relation index = NULL; Relation index = NULL;
TupleDesc tupleDescriptor = NULL; TupleDesc tupleDescriptor = NULL;
ScanKeyData scanKey[1]; ScanKeyData scanKey[1];
SysScanDesc scanDescriptor = NULL; SysScanDesc scanDescriptor = NULL;
HeapTuple heapTuple = NULL; HeapTuple heapTuple = NULL;
ScanKeyInit(&scanKey[0], Anum_cstore_tables_relfilenode, ScanKeyInit(&scanKey[0], Anum_cstore_data_files_relfilenode,
BTEqualStrategyNumber, F_OIDEQ, Int32GetDatum(relfilenode)); BTEqualStrategyNumber, F_OIDEQ, Int32GetDatum(relfilenode));
cstoreTablesOid = CStoreTablesRelationId(); cstoreDataFilesOid = CStoreDataFilesRelationId();
cstoreTables = try_relation_open(cstoreTablesOid, AccessShareLock); cstoreDataFiles = try_relation_open(cstoreDataFilesOid, AccessShareLock);
if (cstoreTables == NULL) if (cstoreDataFiles == NULL)
{ {
/* /*
* Extension has been dropped. This can be called while * Extension has been dropped. This can be called while
@ -436,77 +438,77 @@ ReadCStoreTables(Oid relfilenode, uint64 *blockRowCount)
return false; return false;
} }
index = try_relation_open(CStoreTablesIndexRelationId(), AccessShareLock); index = try_relation_open(CStoreDataFilesIndexRelationId(), AccessShareLock);
if (index == NULL) if (index == NULL)
{ {
heap_close(cstoreTables, NoLock); heap_close(cstoreDataFiles, NoLock);
/* extension has been dropped */ /* extension has been dropped */
return false; return false;
} }
tupleDescriptor = RelationGetDescr(cstoreTables); tupleDescriptor = RelationGetDescr(cstoreDataFiles);
scanDescriptor = systable_beginscan_ordered(cstoreTables, index, NULL, 1, scanKey); scanDescriptor = systable_beginscan_ordered(cstoreDataFiles, index, NULL, 1, scanKey);
heapTuple = systable_getnext(scanDescriptor); heapTuple = systable_getnext(scanDescriptor);
if (HeapTupleIsValid(heapTuple)) if (HeapTupleIsValid(heapTuple))
{ {
Datum datumArray[Natts_cstore_tables]; Datum datumArray[Natts_cstore_data_files];
bool isNullArray[Natts_cstore_tables]; bool isNullArray[Natts_cstore_data_files];
heap_deform_tuple(heapTuple, tupleDescriptor, datumArray, isNullArray); heap_deform_tuple(heapTuple, tupleDescriptor, datumArray, isNullArray);
*blockRowCount = DatumGetInt32(datumArray[Anum_cstore_tables_block_row_count - *blockRowCount = DatumGetInt32(datumArray[Anum_cstore_data_files_block_row_count -
1]); 1]);
found = true; found = true;
} }
systable_endscan_ordered(scanDescriptor); systable_endscan_ordered(scanDescriptor);
index_close(index, NoLock); index_close(index, NoLock);
heap_close(cstoreTables, NoLock); heap_close(cstoreDataFiles, NoLock);
return found; return found;
} }
/* /*
* DeleteTableMetadataRowIfExists removes the row with given relfilenode from cstore_stripes. * DeleteDataFileMetadataRowIfExists removes the row with given relfilenode from cstore_stripes.
*/ */
void void
DeleteTableMetadataRowIfExists(Oid relfilenode) DeleteDataFileMetadataRowIfExists(Oid relfilenode)
{ {
Oid cstoreTablesOid = InvalidOid; Oid cstoreDataFilesOid = InvalidOid;
Relation cstoreTables = NULL; Relation cstoreDataFiles = NULL;
Relation index = NULL; Relation index = NULL;
ScanKeyData scanKey[1]; ScanKeyData scanKey[1];
SysScanDesc scanDescriptor = NULL; SysScanDesc scanDescriptor = NULL;
HeapTuple heapTuple = NULL; HeapTuple heapTuple = NULL;
ScanKeyInit(&scanKey[0], Anum_cstore_tables_relfilenode, ScanKeyInit(&scanKey[0], Anum_cstore_data_files_relfilenode,
BTEqualStrategyNumber, F_OIDEQ, Int32GetDatum(relfilenode)); BTEqualStrategyNumber, F_OIDEQ, Int32GetDatum(relfilenode));
cstoreTablesOid = CStoreTablesRelationId(); cstoreDataFilesOid = CStoreDataFilesRelationId();
cstoreTables = try_relation_open(cstoreTablesOid, AccessShareLock); cstoreDataFiles = try_relation_open(cstoreDataFilesOid, AccessShareLock);
if (cstoreTables == NULL) if (cstoreDataFiles == NULL)
{ {
/* extension has been dropped */ /* extension has been dropped */
return; return;
} }
index = index_open(CStoreTablesIndexRelationId(), AccessShareLock); index = index_open(CStoreDataFilesIndexRelationId(), AccessShareLock);
scanDescriptor = systable_beginscan_ordered(cstoreTables, index, NULL, 1, scanKey); scanDescriptor = systable_beginscan_ordered(cstoreDataFiles, index, NULL, 1, scanKey);
heapTuple = systable_getnext(scanDescriptor); heapTuple = systable_getnext(scanDescriptor);
if (HeapTupleIsValid(heapTuple)) if (HeapTupleIsValid(heapTuple))
{ {
ModifyState *modifyState = StartModifyRelation(cstoreTables); ModifyState *modifyState = StartModifyRelation(cstoreDataFiles);
DeleteTupleAndEnforceConstraints(modifyState, heapTuple); DeleteTupleAndEnforceConstraints(modifyState, heapTuple);
FinishModifyRelation(modifyState); FinishModifyRelation(modifyState);
} }
systable_endscan_ordered(scanDescriptor); systable_endscan_ordered(scanDescriptor);
index_close(index, NoLock); index_close(index, NoLock);
heap_close(cstoreTables, NoLock); heap_close(cstoreDataFiles, NoLock);
} }
@ -711,24 +713,24 @@ CStoreStripesIndexRelationId(void)
/* /*
* CStoreTablesRelationId returns relation id of cstore_tables. * CStoreDataFilesRelationId returns relation id of cstore_data_files.
* TODO: should we cache this similar to citus? * TODO: should we cache this similar to citus?
*/ */
static Oid static Oid
CStoreTablesRelationId(void) CStoreDataFilesRelationId(void)
{ {
return get_relname_relid("cstore_tables", CStoreNamespaceId()); return get_relname_relid("cstore_data_files", CStoreNamespaceId());
} }
/* /*
* CStoreTablesIndexRelationId returns relation id of cstore_tables_idx. * CStoreDataFilesIndexRelationId returns relation id of cstore_data_files_pkey.
* TODO: should we cache this similar to citus? * TODO: should we cache this similar to citus?
*/ */
static Oid static Oid
CStoreTablesIndexRelationId(void) CStoreDataFilesIndexRelationId(void)
{ {
return get_relname_relid("cstore_tables_pkey", CStoreNamespaceId()); return get_relname_relid("cstore_data_files_pkey", CStoreNamespaceId());
} }

18
cstore_reader.c
View File

@ -84,11 +84,11 @@ CStoreBeginRead(Relation relation, TupleDesc tupleDescriptor,
List *projectedColumnList, List *whereClauseList) List *projectedColumnList, List *whereClauseList)
{ {
TableReadState *readState = NULL; TableReadState *readState = NULL;
TableMetadata *tableMetadata = NULL; DataFileMetadata *datafileMetadata = NULL;
MemoryContext stripeReadContext = NULL; MemoryContext stripeReadContext = NULL;
Oid relNode = relation->rd_node.relNode; Oid relNode = relation->rd_node.relNode;
tableMetadata = ReadTableMetadata(relNode); datafileMetadata = ReadDataFileMetadata(relNode);
/* /*
* We allocate all stripe specific data in the stripeReadContext, and reset * We allocate all stripe specific data in the stripeReadContext, and reset
@ -101,7 +101,7 @@ CStoreBeginRead(Relation relation, TupleDesc tupleDescriptor,
readState = palloc0(sizeof(TableReadState)); readState = palloc0(sizeof(TableReadState));
readState->relation = relation; readState->relation = relation;
readState->tableMetadata = tableMetadata; readState->datafileMetadata = datafileMetadata;
readState->projectedColumnList = projectedColumnList; readState->projectedColumnList = projectedColumnList;
readState->whereClauseList = whereClauseList; readState->whereClauseList = whereClauseList;
readState->stripeBuffers = NULL; readState->stripeBuffers = NULL;
@ -139,7 +139,7 @@ CStoreReadNextRow(TableReadState *readState, Datum *columnValues, bool *columnNu
{ {
StripeBuffers *stripeBuffers = NULL; StripeBuffers *stripeBuffers = NULL;
StripeMetadata *stripeMetadata = NULL; StripeMetadata *stripeMetadata = NULL;
List *stripeMetadataList = readState->tableMetadata->stripeMetadataList; List *stripeMetadataList = readState->datafileMetadata->stripeMetadataList;
uint32 stripeCount = list_length(stripeMetadataList); uint32 stripeCount = list_length(stripeMetadataList);
/* if we have read all stripes, return false */ /* if we have read all stripes, return false */
@ -229,8 +229,8 @@ void
CStoreEndRead(TableReadState *readState) CStoreEndRead(TableReadState *readState)
{ {
MemoryContextDelete(readState->stripeReadContext); MemoryContextDelete(readState->stripeReadContext);
list_free_deep(readState->tableMetadata->stripeMetadataList); list_free_deep(readState->datafileMetadata->stripeMetadataList);
pfree(readState->tableMetadata); pfree(readState->datafileMetadata);
pfree(readState); pfree(readState);
} }
@ -305,13 +305,13 @@ FreeBlockData(BlockData *blockData)
uint64 uint64
CStoreTableRowCount(Relation relation) CStoreTableRowCount(Relation relation)
{ {
TableMetadata *tableMetadata = NULL; DataFileMetadata *datafileMetadata = NULL;
ListCell *stripeMetadataCell = NULL; ListCell *stripeMetadataCell = NULL;
uint64 totalRowCount = 0; uint64 totalRowCount = 0;
tableMetadata = ReadTableMetadata(relation->rd_node.relNode); datafileMetadata = ReadDataFileMetadata(relation->rd_node.relNode);
foreach(stripeMetadataCell, tableMetadata->stripeMetadataList) foreach(stripeMetadataCell, datafileMetadata->stripeMetadataList)
{ {
StripeMetadata *stripeMetadata = (StripeMetadata *) lfirst(stripeMetadataCell); StripeMetadata *stripeMetadata = (StripeMetadata *) lfirst(stripeMetadataCell);
totalRowCount += stripeMetadata->rowCount; totalRowCount += stripeMetadata->rowCount;

5
cstore_tableam.c
View File

@ -443,12 +443,13 @@ cstore_relation_set_new_filenode(Relation rel,
MultiXactId *minmulti) MultiXactId *minmulti)
{ {
SMgrRelation srel; SMgrRelation srel;
CStoreOptions *options = CStoreTableAMGetOptions();
Assert(persistence == RELPERSISTENCE_PERMANENT); Assert(persistence == RELPERSISTENCE_PERMANENT);
*freezeXid = RecentXmin; *freezeXid = RecentXmin;
*minmulti = GetOldestMultiXactId(); *minmulti = GetOldestMultiXactId();
srel = RelationCreateStorage(*newrnode, persistence); srel = RelationCreateStorage(*newrnode, persistence);
InitializeCStoreTableFile(newrnode->relNode, CStoreTableAMGetOptions()); InitCStoreDataFileMetadata(newrnode->relNode, options->blockRowCount);
smgrclose(srel); smgrclose(srel);
} }
@ -686,7 +687,7 @@ CStoreTableAMObjectAccessHook(ObjectAccessType access, Oid classId, Oid objectId
* tableam tables storage is managed by postgres. * tableam tables storage is managed by postgres.
*/ */
Relation rel = table_open(objectId, AccessExclusiveLock); Relation rel = table_open(objectId, AccessExclusiveLock);
DeleteTableMetadataRowIfExists(rel->rd_node.relNode); DeleteDataFileMetadataRowIfExists(rel->rd_node.relNode);
/* keep the lock since we did physical changes to the relation */ /* keep the lock since we did physical changes to the relation */
table_close(rel, NoLock); table_close(rel, NoLock);

24
cstore_writer.c
View File

@ -45,7 +45,7 @@ static void UpdateBlockSkipNodeMinMax(ColumnBlockSkipNode *blockSkipNode,
int columnTypeLength, Oid columnCollation, int columnTypeLength, Oid columnCollation,
FmgrInfo *comparisonFunction); FmgrInfo *comparisonFunction);
static Datum DatumCopy(Datum datum, bool datumTypeByValue, int datumTypeLength); static Datum DatumCopy(Datum datum, bool datumTypeByValue, int datumTypeLength);
static void AppendStripeMetadata(TableMetadata *tableMetadata, static void AppendStripeMetadata(DataFileMetadata *datafileMetadata,
StripeMetadata stripeMetadata); StripeMetadata stripeMetadata);
static StringInfo CopyStringInfo(StringInfo sourceString); static StringInfo CopyStringInfo(StringInfo sourceString);
@ -64,7 +64,7 @@ CStoreBeginWrite(Relation relation,
TupleDesc tupleDescriptor) TupleDesc tupleDescriptor)
{ {
TableWriteState *writeState = NULL; TableWriteState *writeState = NULL;
TableMetadata *tableMetadata = NULL; DataFileMetadata *datafileMetadata = NULL;
FmgrInfo **comparisonFunctionArray = NULL; FmgrInfo **comparisonFunctionArray = NULL;
MemoryContext stripeWriteContext = NULL; MemoryContext stripeWriteContext = NULL;
uint64 currentFileOffset = 0; uint64 currentFileOffset = 0;
@ -75,18 +75,18 @@ CStoreBeginWrite(Relation relation,
uint64 currentStripeId = 0; uint64 currentStripeId = 0;
Oid relNode = relation->rd_node.relNode; Oid relNode = relation->rd_node.relNode;
tableMetadata = ReadTableMetadata(relNode); datafileMetadata = ReadDataFileMetadata(relNode);
/* /*
* If stripeMetadataList is not empty, jump to the position right after * If stripeMetadataList is not empty, jump to the position right after
* the last position. * the last position.
*/ */
if (tableMetadata->stripeMetadataList != NIL) if (datafileMetadata->stripeMetadataList != NIL)
{ {
StripeMetadata *lastStripe = NULL; StripeMetadata *lastStripe = NULL;
uint64 lastStripeSize = 0; uint64 lastStripeSize = 0;
lastStripe = llast(tableMetadata->stripeMetadataList); lastStripe = llast(datafileMetadata->stripeMetadataList);
lastStripeSize += lastStripe->dataLength; lastStripeSize += lastStripe->dataLength;
currentFileOffset = lastStripe->fileOffset + lastStripeSize; currentFileOffset = lastStripe->fileOffset + lastStripeSize;
@ -129,7 +129,7 @@ CStoreBeginWrite(Relation relation,
writeState = palloc0(sizeof(TableWriteState)); writeState = palloc0(sizeof(TableWriteState));
writeState->relation = relation; writeState->relation = relation;
writeState->tableMetadata = tableMetadata; writeState->datafileMetadata = datafileMetadata;
writeState->compressionType = compressionType; writeState->compressionType = compressionType;
writeState->stripeMaxRowCount = stripeMaxRowCount; writeState->stripeMaxRowCount = stripeMaxRowCount;
writeState->blockRowCount = blockRowCount; writeState->blockRowCount = blockRowCount;
@ -164,7 +164,7 @@ CStoreWriteRow(TableWriteState *writeState, Datum *columnValues, bool *columnNul
StripeBuffers *stripeBuffers = writeState->stripeBuffers; StripeBuffers *stripeBuffers = writeState->stripeBuffers;
StripeSkipList *stripeSkipList = writeState->stripeSkipList; StripeSkipList *stripeSkipList = writeState->stripeSkipList;
uint32 columnCount = writeState->tupleDescriptor->natts; uint32 columnCount = writeState->tupleDescriptor->natts;
TableMetadata *tableMetadata = writeState->tableMetadata; DataFileMetadata *datafileMetadata = writeState->datafileMetadata;
const uint32 blockRowCount = writeState->blockRowCount; const uint32 blockRowCount = writeState->blockRowCount;
BlockData *blockData = writeState->blockData; BlockData *blockData = writeState->blockData;
MemoryContext oldContext = MemoryContextSwitchTo(writeState->stripeWriteContext); MemoryContext oldContext = MemoryContextSwitchTo(writeState->stripeWriteContext);
@ -254,7 +254,7 @@ CStoreWriteRow(TableWriteState *writeState, Datum *columnValues, bool *columnNul
MemoryContextSwitchTo(oldContext); MemoryContextSwitchTo(oldContext);
InsertStripeMetadataRow(writeState->relation->rd_node.relNode, InsertStripeMetadataRow(writeState->relation->rd_node.relNode,
&stripeMetadata); &stripeMetadata);
AppendStripeMetadata(tableMetadata, stripeMetadata); AppendStripeMetadata(datafileMetadata, stripeMetadata);
} }
else else
{ {
@ -284,11 +284,11 @@ CStoreEndWrite(TableWriteState *writeState)
MemoryContextSwitchTo(oldContext); MemoryContextSwitchTo(oldContext);
InsertStripeMetadataRow(writeState->relation->rd_node.relNode, InsertStripeMetadataRow(writeState->relation->rd_node.relNode,
&stripeMetadata); &stripeMetadata);
AppendStripeMetadata(writeState->tableMetadata, stripeMetadata); AppendStripeMetadata(writeState->datafileMetadata, stripeMetadata);
} }
MemoryContextDelete(writeState->stripeWriteContext); MemoryContextDelete(writeState->stripeWriteContext);
list_free_deep(writeState->tableMetadata->stripeMetadataList); list_free_deep(writeState->datafileMetadata->stripeMetadataList);
pfree(writeState->comparisonFunctionArray); pfree(writeState->comparisonFunctionArray);
FreeBlockData(writeState->blockData); FreeBlockData(writeState->blockData);
pfree(writeState); pfree(writeState);
@ -791,12 +791,12 @@ DatumCopy(Datum datum, bool datumTypeByValue, int datumTypeLength)
* table footer's stripeMetadataList. * table footer's stripeMetadataList.
*/ */
static void static void
AppendStripeMetadata(TableMetadata *tableMetadata, StripeMetadata stripeMetadata) AppendStripeMetadata(DataFileMetadata *datafileMetadata, StripeMetadata stripeMetadata)
{ {
StripeMetadata *stripeMetadataCopy = palloc0(sizeof(StripeMetadata)); StripeMetadata *stripeMetadataCopy = palloc0(sizeof(StripeMetadata));
memcpy(stripeMetadataCopy, &stripeMetadata, sizeof(StripeMetadata)); memcpy(stripeMetadataCopy, &stripeMetadata, sizeof(StripeMetadata));
tableMetadata->stripeMetadataList = lappend(tableMetadata->stripeMetadataList, datafileMetadata->stripeMetadataList = lappend(datafileMetadata->stripeMetadataList,
stripeMetadataCopy); stripeMetadataCopy);
} }

8
expected/am_drop.out
View File

@ -12,12 +12,12 @@
-- 'postgres' directory is excluded from comparison to have the same result. -- 'postgres' directory is excluded from comparison to have the same result.
-- store postgres database oid -- store postgres database oid
SELECT oid postgres_oid FROM pg_database WHERE datname = 'postgres' \gset SELECT oid postgres_oid FROM pg_database WHERE datname = 'postgres' \gset
SELECT count(*) AS cstore_tables_before_drop FROM cstore.cstore_tables \gset SELECT count(*) AS cstore_data_files_before_drop FROM cstore.cstore_data_files \gset
-- DROP cstore_fdw tables -- DROP cstore_fdw tables
DROP TABLE contestant; DROP TABLE contestant;
DROP TABLE contestant_compressed; DROP TABLE contestant_compressed;
-- make sure DROP deletes metadata -- make sure DROP deletes metadata
SELECT :cstore_tables_before_drop - count(*) FROM cstore.cstore_tables; SELECT :cstore_data_files_before_drop - count(*) FROM cstore.cstore_data_files;
?column? ?column?
---------- ----------
2 2
@ -26,10 +26,10 @@ SELECT :cstore_tables_before_drop - count(*) FROM cstore.cstore_tables;
-- Create a cstore_fdw table under a schema and drop it. -- Create a cstore_fdw table under a schema and drop it.
CREATE SCHEMA test_schema; CREATE SCHEMA test_schema;
CREATE TABLE test_schema.test_table(data int) USING cstore_tableam; CREATE TABLE test_schema.test_table(data int) USING cstore_tableam;
SELECT count(*) AS cstore_tables_before_drop FROM cstore.cstore_tables \gset SELECT count(*) AS cstore_data_files_before_drop FROM cstore.cstore_data_files \gset
DROP SCHEMA test_schema CASCADE; DROP SCHEMA test_schema CASCADE;
NOTICE: drop cascades to table test_schema.test_table NOTICE: drop cascades to table test_schema.test_table
SELECT :cstore_tables_before_drop - count(*) FROM cstore.cstore_tables; SELECT :cstore_data_files_before_drop - count(*) FROM cstore.cstore_data_files;
?column? ?column?
---------- ----------
1 1

8
expected/fdw_drop.out
View File

@ -12,12 +12,12 @@
-- 'postgres' directory is excluded from comparison to have the same result. -- 'postgres' directory is excluded from comparison to have the same result.
-- store postgres database oid -- store postgres database oid
SELECT oid postgres_oid FROM pg_database WHERE datname = 'postgres' \gset SELECT oid postgres_oid FROM pg_database WHERE datname = 'postgres' \gset
SELECT count(*) AS cstore_tables_before_drop FROM cstore.cstore_tables \gset SELECT count(*) AS cstore_data_files_before_drop FROM cstore.cstore_data_files \gset
-- DROP cstore_fdw tables -- DROP cstore_fdw tables
DROP FOREIGN TABLE contestant; DROP FOREIGN TABLE contestant;
DROP FOREIGN TABLE contestant_compressed; DROP FOREIGN TABLE contestant_compressed;
-- make sure DROP deletes metadata -- make sure DROP deletes metadata
SELECT :cstore_tables_before_drop - count(*) FROM cstore.cstore_tables; SELECT :cstore_data_files_before_drop - count(*) FROM cstore.cstore_data_files;
?column? ?column?
---------- ----------
2 2
@ -26,10 +26,10 @@ SELECT :cstore_tables_before_drop - count(*) FROM cstore.cstore_tables;
-- Create a cstore_fdw table under a schema and drop it. -- Create a cstore_fdw table under a schema and drop it.
CREATE SCHEMA test_schema; CREATE SCHEMA test_schema;
CREATE FOREIGN TABLE test_schema.test_table(data int) SERVER cstore_server; CREATE FOREIGN TABLE test_schema.test_table(data int) SERVER cstore_server;
SELECT count(*) AS cstore_tables_before_drop FROM cstore.cstore_tables \gset SELECT count(*) AS cstore_data_files_before_drop FROM cstore.cstore_data_files \gset
DROP SCHEMA test_schema CASCADE; DROP SCHEMA test_schema CASCADE;
NOTICE: drop cascades to foreign table test_schema.test_table NOTICE: drop cascades to foreign table test_schema.test_table
SELECT :cstore_tables_before_drop - count(*) FROM cstore.cstore_tables; SELECT :cstore_data_files_before_drop - count(*) FROM cstore.cstore_data_files;
?column? ?column?
---------- ----------
1 1

8
sql/am_drop.sql
View File

@ -15,22 +15,22 @@
-- store postgres database oid -- store postgres database oid
SELECT oid postgres_oid FROM pg_database WHERE datname = 'postgres' \gset SELECT oid postgres_oid FROM pg_database WHERE datname = 'postgres' \gset
SELECT count(*) AS cstore_tables_before_drop FROM cstore.cstore_tables \gset SELECT count(*) AS cstore_data_files_before_drop FROM cstore.cstore_data_files \gset
-- DROP cstore_fdw tables -- DROP cstore_fdw tables
DROP TABLE contestant; DROP TABLE contestant;
DROP TABLE contestant_compressed; DROP TABLE contestant_compressed;
-- make sure DROP deletes metadata -- make sure DROP deletes metadata
SELECT :cstore_tables_before_drop - count(*) FROM cstore.cstore_tables; SELECT :cstore_data_files_before_drop - count(*) FROM cstore.cstore_data_files;
-- Create a cstore_fdw table under a schema and drop it. -- Create a cstore_fdw table under a schema and drop it.
CREATE SCHEMA test_schema; CREATE SCHEMA test_schema;
CREATE TABLE test_schema.test_table(data int) USING cstore_tableam; CREATE TABLE test_schema.test_table(data int) USING cstore_tableam;
SELECT count(*) AS cstore_tables_before_drop FROM cstore.cstore_tables \gset SELECT count(*) AS cstore_data_files_before_drop FROM cstore.cstore_data_files \gset
DROP SCHEMA test_schema CASCADE; DROP SCHEMA test_schema CASCADE;
SELECT :cstore_tables_before_drop - count(*) FROM cstore.cstore_tables; SELECT :cstore_data_files_before_drop - count(*) FROM cstore.cstore_data_files;
SELECT current_database() datname \gset SELECT current_database() datname \gset

8
sql/fdw_drop.sql
View File

@ -15,22 +15,22 @@
-- store postgres database oid -- store postgres database oid
SELECT oid postgres_oid FROM pg_database WHERE datname = 'postgres' \gset SELECT oid postgres_oid FROM pg_database WHERE datname = 'postgres' \gset
SELECT count(*) AS cstore_tables_before_drop FROM cstore.cstore_tables \gset SELECT count(*) AS cstore_data_files_before_drop FROM cstore.cstore_data_files \gset
-- DROP cstore_fdw tables -- DROP cstore_fdw tables
DROP FOREIGN TABLE contestant; DROP FOREIGN TABLE contestant;
DROP FOREIGN TABLE contestant_compressed; DROP FOREIGN TABLE contestant_compressed;
-- make sure DROP deletes metadata -- make sure DROP deletes metadata
SELECT :cstore_tables_before_drop - count(*) FROM cstore.cstore_tables; SELECT :cstore_data_files_before_drop - count(*) FROM cstore.cstore_data_files;
-- Create a cstore_fdw table under a schema and drop it. -- Create a cstore_fdw table under a schema and drop it.
CREATE SCHEMA test_schema; CREATE SCHEMA test_schema;
CREATE FOREIGN TABLE test_schema.test_table(data int) SERVER cstore_server; CREATE FOREIGN TABLE test_schema.test_table(data int) SERVER cstore_server;
SELECT count(*) AS cstore_tables_before_drop FROM cstore.cstore_tables \gset SELECT count(*) AS cstore_data_files_before_drop FROM cstore.cstore_data_files \gset
DROP SCHEMA test_schema CASCADE; DROP SCHEMA test_schema CASCADE;
SELECT :cstore_tables_before_drop - count(*) FROM cstore.cstore_tables; SELECT :cstore_data_files_before_drop - count(*) FROM cstore.cstore_data_files;
SELECT current_database() datname \gset SELECT current_database() datname \gset