external
/
citus
mirror of https://github.com/citusdata/citus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #17 from citusdata/truncate_cleanup 

Implement nontransactional TRUNATE + resource clean-up on TRUNCATE
merge-cstore-pykello
Hadi Moshayedi 2020-10-05 11:41:09 -07:00 committed by GitHub
parent 2e47bf5172 62fc59202c
commit 434275d46b
10 changed files with 149 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cstore.h
View File

@ -283,7 +283,7 @@ extern StringInfo DecompressBuffer(StringInfo buffer, CompressionType compressio
extern void DeleteDataFileMetadataRowIfExists(Oid relfilenode);
extern void InitCStoreDataFileMetadata(Oid relfilenode, int blockRowCount);
extern void InsertStripeMetadataRow(Oid relfilenode, StripeMetadata *stripe);
extern DataFileMetadata * ReadDataFileMetadata(Oid relfilenode);
extern DataFileMetadata * ReadDataFileMetadata(Oid relfilenode, bool missingOk);
extern void SaveStripeSkipList(Oid relfilenode, uint64 stripe,
StripeSkipList *stripeSkipList,
TupleDesc tupleDescriptor);

1
cstore_fdw.c
View File

@ -825,6 +825,7 @@ FdwNewRelFileNode(Relation relation)
if (OidIsValid(relation->rd_rel->relfilenode))
{
RelationDropStorage(relation);
DeleteDataFileMetadataRowIfExists(relation->rd_rel->relfilenode);
}
if (OidIsValid(relation->rd_rel->reltablespace))

13
cstore_metadata_tables.c
View File

@ -343,7 +343,7 @@ InsertStripeMetadataRow(Oid relfilenode, StripeMetadata *stripe)
* from cstore_data_files and cstore_stripes.
*/
DataFileMetadata *
ReadDataFileMetadata(Oid relfilenode)
ReadDataFileMetadata(Oid relfilenode, bool missingOk)
{
Oid cstoreStripesOid = InvalidOid;
Relation cstoreStripes = NULL;
@ -358,8 +358,15 @@ ReadDataFileMetadata(Oid relfilenode)
found = ReadCStoreDataFiles(relfilenode, &datafileMetadata->blockRowCount);
if (!found)
{
ereport(ERROR, (errmsg("Relfilenode %d doesn't belong to a cstore table.",
relfilenode)));
if (!missingOk)
{
ereport(ERROR, (errmsg("Relfilenode %d doesn't belong to a cstore table.",
relfilenode)));
}
else
{
return NULL;
}
}
ScanKeyInit(&scanKey[0], Anum_cstore_stripes_relfilenode,

4
cstore_reader.c
View File

@ -88,7 +88,7 @@ CStoreBeginRead(Relation relation, TupleDesc tupleDescriptor,
MemoryContext stripeReadContext = NULL;
Oid relNode = relation->rd_node.relNode;
datafileMetadata = ReadDataFileMetadata(relNode);
datafileMetadata = ReadDataFileMetadata(relNode, false);
/*
* We allocate all stripe specific data in the stripeReadContext, and reset
@ -309,7 +309,7 @@ CStoreTableRowCount(Relation relation)
ListCell *stripeMetadataCell = NULL;
uint64 totalRowCount = 0;
datafileMetadata = ReadDataFileMetadata(relation->rd_node.relNode);
datafileMetadata = ReadDataFileMetadata(relation->rd_node.relNode, false);
foreach(stripeMetadataCell, datafileMetadata->stripeMetadataList)
{

35
cstore_tableam.c
View File

@ -443,13 +443,29 @@ cstore_relation_set_new_filenode(Relation rel,
MultiXactId *minmulti)
{
SMgrRelation srel;
CStoreOptions *options = CStoreTableAMGetOptions();
DataFileMetadata *metadata = ReadDataFileMetadata(rel->rd_node.relNode, true);
uint64 blockRowCount = 0;
if (metadata != NULL)
{
/* existing table (e.g. TRUNCATE), use existing blockRowCount */
blockRowCount = metadata->blockRowCount;
}
else
{
/* new table, use options */
CStoreOptions *options = CStoreTableAMGetOptions();
blockRowCount = options->blockRowCount;
}
/* delete old relfilenode metadata */
DeleteDataFileMetadataRowIfExists(rel->rd_node.relNode);
Assert(persistence == RELPERSISTENCE_PERMANENT);
*freezeXid = RecentXmin;
*minmulti = GetOldestMultiXactId();
srel = RelationCreateStorage(*newrnode, persistence);
InitCStoreDataFileMetadata(newrnode->relNode, options->blockRowCount);
InitCStoreDataFileMetadata(newrnode->relNode, blockRowCount);
smgrclose(srel);
}
@ -457,7 +473,20 @@ cstore_relation_set_new_filenode(Relation rel,
static void
cstore_relation_nontransactional_truncate(Relation rel)
{
elog(ERROR, "cstore_relation_nontransactional_truncate not implemented");
DataFileMetadata *metadata = ReadDataFileMetadata(rel->rd_node.relNode, false);
/*
* No need to set new relfilenode, since the table was created in this
* transaction and no other transaction can see this relation yet. We
* can just truncate the relation.
*
* This is similar to what is done in heapam_relation_nontransactional_truncate.
*/
RelationTruncate(rel, 0);
/* Delete old relfilenode metadata and recreate it */
DeleteDataFileMetadataRowIfExists(rel->rd_node.relNode);
InitCStoreDataFileMetadata(rel->rd_node.relNode, metadata->blockRowCount);
}

2
cstore_writer.c
View File

@ -75,7 +75,7 @@ CStoreBeginWrite(Relation relation,
uint64 currentStripeId = 0;
Oid relNode = relation->rd_node.relNode;
datafileMetadata = ReadDataFileMetadata(relNode);
datafileMetadata = ReadDataFileMetadata(relNode, false);
/*
* If stripeMetadataList is not empty, jump to the position right after

32
expected/am_truncate.out
View File

@ -15,6 +15,7 @@ CREATE TABLE cstore_truncate_test_second (a int, b int) USING cstore_tableam;
-- COMPRESSED
CREATE TABLE cstore_truncate_test_compressed (a int, b int) USING cstore_tableam;
CREATE TABLE cstore_truncate_test_regular (a int, b int);
SELECT count(*) AS cstore_data_files_before_truncate FROM cstore.cstore_data_files \gset
INSERT INTO cstore_truncate_test select a, a from generate_series(1, 10) a;
set cstore.compression = 'pglz';
INSERT INTO cstore_truncate_test_compressed select a, a from generate_series(1, 10) a;
@ -145,6 +146,37 @@ SELECT * from cstore_truncate_test;
---+---
(0 rows)
-- make sure TRUNATE deletes metadata for old relfilenode
SELECT :cstore_data_files_before_truncate - count(*) FROM cstore.cstore_data_files;
?column?
----------
0
(1 row)
-- test if truncation in the same transaction that created the table works properly
BEGIN;
CREATE TABLE cstore_same_transaction_truncate(a int) USING cstore_tableam;
INSERT INTO cstore_same_transaction_truncate SELECT * FROM generate_series(1, 100);
TRUNCATE cstore_same_transaction_truncate;
INSERT INTO cstore_same_transaction_truncate SELECT * FROM generate_series(20, 23);
COMMIT;
-- should output "1" for the newly created relation
SELECT count(*) - :cstore_data_files_before_truncate FROM cstore.cstore_data_files;
?column?
----------
1
(1 row)
SELECT * FROM cstore_same_transaction_truncate;
a
----
20
21
22
23
(4 rows)
DROP TABLE cstore_same_transaction_truncate;
-- test if a cached truncate from a pl/pgsql function works
CREATE FUNCTION cstore_truncate_test_regular_func() RETURNS void AS $$
BEGIN

32
expected/fdw_truncate.out
View File

@ -14,6 +14,7 @@ CREATE FOREIGN TABLE cstore_truncate_test (a int, b int) SERVER cstore_server;
CREATE FOREIGN TABLE cstore_truncate_test_second (a int, b int) SERVER cstore_server;
CREATE FOREIGN TABLE cstore_truncate_test_compressed (a int, b int) SERVER cstore_server OPTIONS (compression 'pglz');
CREATE TABLE cstore_truncate_test_regular (a int, b int);
SELECT count(*) AS cstore_data_files_before_truncate FROM cstore.cstore_data_files \gset
INSERT INTO cstore_truncate_test select a, a from generate_series(1, 10) a;
INSERT INTO cstore_truncate_test_compressed select a, a from generate_series(1, 10) a;
INSERT INTO cstore_truncate_test_compressed select a, a from generate_series(1, 10) a;
@ -142,6 +143,37 @@ SELECT * from cstore_truncate_test;
---+---
(0 rows)
-- make sure TRUNATE deletes metadata for old relfilenode
SELECT :cstore_data_files_before_truncate - count(*) FROM cstore.cstore_data_files;
?column?
----------
0
(1 row)
-- test if truncation in the same transaction that created the table works properly
BEGIN;
CREATE FOREIGN TABLE cstore_same_transaction_truncate(a int) SERVER cstore_server;
INSERT INTO cstore_same_transaction_truncate SELECT * FROM generate_series(1, 100);
TRUNCATE cstore_same_transaction_truncate;
INSERT INTO cstore_same_transaction_truncate SELECT * FROM generate_series(20, 23);
COMMIT;
-- should output "1" for the newly created relation
SELECT count(*) - :cstore_data_files_before_truncate FROM cstore.cstore_data_files;
?column?
----------
1
(1 row)
SELECT * FROM cstore_same_transaction_truncate;
a
----
20
21
22
23
(4 rows)
DROP FOREIGN TABLE cstore_same_transaction_truncate;
-- test if a cached truncate from a pl/pgsql function works
CREATE FUNCTION cstore_truncate_test_regular_func() RETURNS void AS $$
BEGIN

19
sql/am_truncate.sql
View File

@ -13,6 +13,8 @@ CREATE TABLE cstore_truncate_test_second (a int, b int) USING cstore_tableam;
CREATE TABLE cstore_truncate_test_compressed (a int, b int) USING cstore_tableam;
CREATE TABLE cstore_truncate_test_regular (a int, b int);
SELECT count(*) AS cstore_data_files_before_truncate FROM cstore.cstore_data_files \gset
INSERT INTO cstore_truncate_test select a, a from generate_series(1, 10) a;
set cstore.compression = 'pglz';
@ -60,6 +62,23 @@ SELECT * from cstore_truncate_test_regular;
TRUNCATE TABLE cstore_truncate_test;
SELECT * from cstore_truncate_test;
-- make sure TRUNATE deletes metadata for old relfilenode
SELECT :cstore_data_files_before_truncate - count(*) FROM cstore.cstore_data_files;
-- test if truncation in the same transaction that created the table works properly
BEGIN;
CREATE TABLE cstore_same_transaction_truncate(a int) USING cstore_tableam;
INSERT INTO cstore_same_transaction_truncate SELECT * FROM generate_series(1, 100);
TRUNCATE cstore_same_transaction_truncate;
INSERT INTO cstore_same_transaction_truncate SELECT * FROM generate_series(20, 23);
COMMIT;
-- should output "1" for the newly created relation
SELECT count(*) - :cstore_data_files_before_truncate FROM cstore.cstore_data_files;
SELECT * FROM cstore_same_transaction_truncate;
DROP TABLE cstore_same_transaction_truncate;
-- test if a cached truncate from a pl/pgsql function works
CREATE FUNCTION cstore_truncate_test_regular_func() RETURNS void AS $$
BEGIN

19
sql/fdw_truncate.sql
View File

@ -12,6 +12,8 @@ CREATE FOREIGN TABLE cstore_truncate_test_second (a int, b int) SERVER cstore_se
CREATE FOREIGN TABLE cstore_truncate_test_compressed (a int, b int) SERVER cstore_server OPTIONS (compression 'pglz');
CREATE TABLE cstore_truncate_test_regular (a int, b int);
SELECT count(*) AS cstore_data_files_before_truncate FROM cstore.cstore_data_files \gset
INSERT INTO cstore_truncate_test select a, a from generate_series(1, 10) a;
INSERT INTO cstore_truncate_test_compressed select a, a from generate_series(1, 10) a;
@ -57,6 +59,23 @@ SELECT * from cstore_truncate_test_regular;
TRUNCATE TABLE cstore_truncate_test;
SELECT * from cstore_truncate_test;
-- make sure TRUNATE deletes metadata for old relfilenode
SELECT :cstore_data_files_before_truncate - count(*) FROM cstore.cstore_data_files;
-- test if truncation in the same transaction that created the table works properly
BEGIN;
CREATE FOREIGN TABLE cstore_same_transaction_truncate(a int) SERVER cstore_server;
INSERT INTO cstore_same_transaction_truncate SELECT * FROM generate_series(1, 100);
TRUNCATE cstore_same_transaction_truncate;
INSERT INTO cstore_same_transaction_truncate SELECT * FROM generate_series(20, 23);
COMMIT;
-- should output "1" for the newly created relation
SELECT count(*) - :cstore_data_files_before_truncate FROM cstore.cstore_data_files;
SELECT * FROM cstore_same_transaction_truncate;
DROP FOREIGN TABLE cstore_same_transaction_truncate;
-- test if a cached truncate from a pl/pgsql function works
CREATE FUNCTION cstore_truncate_test_regular_func() RETURNS void AS $$
BEGIN