From 06f1c9697584a1ef7dccdc21f6b5364d5671b5c3 Mon Sep 17 00:00:00 2001 From: Jeff Davis Date: Fri, 18 Sep 2020 11:16:35 -0700 Subject: [PATCH] almost works --- Makefile | 3 ++- cstore_tableam.c | 29 ++++++++++++++++++++--------- expected/am_block_filtering.out | 14 +++++++++----- expected/am_create.out | 1 + expected/am_load.out | 2 ++ expected/am_truncate.out | 8 ++++++++ input/am_block_filtering.source | 5 +++++ input/am_create.source | 1 + input/am_load.source | 2 ++ output/am_block_filtering.source | 14 +++++++++----- output/am_create.source | 1 + output/am_load.source | 2 ++ sql/am_block_filtering.sql | 5 +++++ sql/am_create.sql | 1 + sql/am_load.sql | 2 ++ sql/am_truncate.sql | 10 ++++++++-- 16 files changed, 78 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index ea5a858bf..033e8d4a6 100644 --- a/Makefile +++ b/Makefile @@ -53,7 +53,8 @@ ifeq ($(USE_TABLEAM),yes) PG_CFLAGS += -DUSE_TABLEAM OBJS += cstore_tableam.o REGRESS += am_create am_load am_query am_data_types am_functions \ - am_drop am_insert am_copyto am_alter am_truncate am_clean + am_block_filtering am_drop am_insert am_copyto am_alter \ + am_truncate am_clean endif ifeq ($(enable_coverage),yes) diff --git a/cstore_tableam.c b/cstore_tableam.c index 95630c3e1..57ec2fa94 100644 --- a/cstore_tableam.c +++ b/cstore_tableam.c @@ -56,6 +56,16 @@ CStoreTableAMGetOptions(void) return cstoreOptions; } +static MemoryContext +CStoreMemoryContext(void) +{ + if (CStoreContext == NULL) + { + CStoreContext = AllocSetContextCreate(TopMemoryContext, "cstore context", + ALLOCSET_DEFAULT_SIZES); + } + return CStoreContext; +} static void cstore_init_write_state(Relation relation) @@ -73,22 +83,13 @@ cstore_init_write_state(Relation relation) { CStoreOptions *cstoreOptions = CStoreTableAMGetOptions(); TupleDesc tupdesc = RelationGetDescr(relation); - MemoryContext oldContext; - - if (CStoreContext == NULL) - { - CStoreContext = AllocSetContextCreate(TopMemoryContext, "cstore context", - ALLOCSET_DEFAULT_SIZES); - } elog(LOG, "initializing write state for relation %d", relation->rd_id); - oldContext = MemoryContextSwitchTo(CStoreContext); CStoreWriteState = CStoreBeginWrite(relation->rd_id, cstoreOptions->compressionType, cstoreOptions->stripeRowCount, cstoreOptions->blockRowCount, tupdesc); - MemoryContextSwitchTo(oldContext); CStoreWriteState->relation = relation; } @@ -127,6 +128,7 @@ cstore_beginscan(Relation relation, Snapshot snapshot, TableReadState *readState = NULL; CStoreScanDesc scan = palloc(sizeof(CStoreScanDescData)); List *columnList = NIL; + MemoryContext oldContext = MemoryContextSwitchTo(CStoreMemoryContext()); cstoreOptions = CStoreTableAMGetOptions(); @@ -162,6 +164,7 @@ cstore_beginscan(Relation relation, Snapshot snapshot, scan->cs_readState = readState; + MemoryContextSwitchTo(oldContext); return ((TableScanDesc) scan); } @@ -187,12 +190,15 @@ cstore_getnextslot(TableScanDesc sscan, ScanDirection direction, TupleTableSlot { CStoreScanDesc scan = (CStoreScanDesc) sscan; bool nextRowFound; + MemoryContext oldContext = MemoryContextSwitchTo(CStoreMemoryContext()); ExecClearTuple(slot); nextRowFound = CStoreReadNextRow(scan->cs_readState, slot->tts_values, slot->tts_isnull); + MemoryContextSwitchTo(oldContext); + if (!nextRowFound) { return false; @@ -303,6 +309,7 @@ cstore_tuple_insert(Relation relation, TupleTableSlot *slot, CommandId cid, int options, BulkInsertState bistate) { HeapTuple heapTuple; + MemoryContext oldContext = MemoryContextSwitchTo(CStoreMemoryContext()); cstore_init_write_state(relation); @@ -319,6 +326,7 @@ cstore_tuple_insert(Relation relation, TupleTableSlot *slot, CommandId cid, slot_getallattrs(slot); CStoreWriteRow(CStoreWriteState, slot->tts_values, slot->tts_isnull); + MemoryContextSwitchTo(oldContext); } @@ -343,6 +351,8 @@ static void cstore_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples, CommandId cid, int options, BulkInsertState bistate) { + MemoryContext oldContext = MemoryContextSwitchTo(CStoreMemoryContext()); + cstore_init_write_state(relation); for (int i = 0; i < ntuples; i++) @@ -363,6 +373,7 @@ cstore_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples, CStoreWriteRow(CStoreWriteState, tupleSlot->tts_values, tupleSlot->tts_isnull); } + MemoryContextSwitchTo(oldContext); } diff --git a/expected/am_block_filtering.out b/expected/am_block_filtering.out index bccfafd15..005b42e64 100644 --- a/expected/am_block_filtering.out +++ b/expected/am_block_filtering.out @@ -24,9 +24,11 @@ $$ END; $$ LANGUAGE PLPGSQL; -- Create and load data -CREATE FOREIGN TABLE test_block_filtering (a int) - SERVER cstore_server - OPTIONS(block_row_count '1000', stripe_row_count '2000'); +-- block_row_count '1000', stripe_row_count '2000' +set cstore.stripe_row_count = 2000; +set cstore.block_row_count = 1000; +CREATE TABLE test_block_filtering (a int) + USING cstore_tableam; COPY test_block_filtering FROM '/Users/jefdavi/wd/cstore2/data/block_filtering.csv' WITH CSV; -- Verify that filtered_row_count is less than 1000 for the following queries SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering'); @@ -104,9 +106,11 @@ SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BET 3958 (1 row) +set cstore.stripe_row_count to default; +set cstore.block_row_count to default; -- Verify that we are fine with collations which use a different alphabet order -CREATE FOREIGN TABLE collation_block_filtering_test(A text collate "da_DK") - SERVER cstore_server; +CREATE TABLE collation_block_filtering_test(A text collate "da_DK") + USING cstore_tableam; COPY collation_block_filtering_test FROM STDIN; SELECT * FROM collation_block_filtering_test WHERE A > 'B'; a diff --git a/expected/am_create.out b/expected/am_create.out index e62447252..47c6a6c44 100644 --- a/expected/am_create.out +++ b/expected/am_create.out @@ -6,6 +6,7 @@ CREATE TABLE contestant (handle TEXT, birthdate DATE, rating INT, percentile FLOAT, country CHAR(3), achievements TEXT[]) USING cstore_tableam; -- Create compressed table with automatically determined file path +-- COMPRESSED CREATE TABLE contestant_compressed (handle TEXT, birthdate DATE, rating INT, percentile FLOAT, country CHAR(3), achievements TEXT[]) USING cstore_tableam; diff --git a/expected/am_load.out b/expected/am_load.out index 110e444fa..02cff343a 100644 --- a/expected/am_load.out +++ b/expected/am_load.out @@ -15,10 +15,12 @@ COPY contestant FROM '/Users/jefdavi/wd/cstore2/data/contestants.1.csv' WITH CSV -- COPY into uncompressed table from program COPY contestant FROM PROGRAM 'cat /Users/jefdavi/wd/cstore2/data/contestants.2.csv' WITH CSV; -- COPY into compressed table +set cstore.compression = 'pglz'; COPY contestant_compressed FROM '/Users/jefdavi/wd/cstore2/data/contestants.1.csv' WITH CSV; -- COPY into uncompressed table from program COPY contestant_compressed FROM PROGRAM 'cat /Users/jefdavi/wd/cstore2/data/contestants.2.csv' WITH CSV; +set cstore.compression to default; -- Test column list CREATE TABLE famous_constants (id int, name text, value real) USING cstore_tableam; diff --git a/expected/am_truncate.out b/expected/am_truncate.out index 538b9ddac..99db7fe72 100644 --- a/expected/am_truncate.out +++ b/expected/am_truncate.out @@ -12,11 +12,14 @@ SELECT substring(:'server_version', '\d+')::int > 10 AS version_above_ten; -- CREATE a cstore_fdw table, fill with some data -- CREATE TABLE cstore_truncate_test (a int, b int) USING cstore_tableam; CREATE TABLE cstore_truncate_test_second (a int, b int) USING cstore_tableam; +-- COMPRESSED CREATE TABLE cstore_truncate_test_compressed (a int, b int) USING cstore_tableam; CREATE TABLE cstore_truncate_test_regular (a int, b int); INSERT INTO cstore_truncate_test select a, a from generate_series(1, 10) a; +set cstore.compression = 'pglz'; INSERT INTO cstore_truncate_test_compressed select a, a from generate_series(1, 10) a; INSERT INTO cstore_truncate_test_compressed select a, a from generate_series(1, 10) a; +set cstore.compression to default; -- query rows SELECT * FROM cstore_truncate_test; a | b @@ -168,8 +171,11 @@ DROP TABLE cstore_truncate_test_regular; DROP TABLE cstore_truncate_test_compressed; -- test truncate with schema CREATE SCHEMA truncate_schema; +-- COMPRESSED CREATE TABLE truncate_schema.truncate_tbl (id int) USING cstore_tableam; +set cstore.compression = 'pglz'; INSERT INTO truncate_schema.truncate_tbl SELECT generate_series(1, 100); +set cstore.compression to default; SELECT COUNT(*) FROM truncate_schema.truncate_tbl; count ------- @@ -183,7 +189,9 @@ SELECT COUNT(*) FROM truncate_schema.truncate_tbl; 0 (1 row) +set cstore.compression = 'pglz'; INSERT INTO truncate_schema.truncate_tbl SELECT generate_series(1, 100); +set cstore.compression to default; -- create a user that can not truncate CREATE USER truncate_user; GRANT USAGE ON SCHEMA truncate_schema TO truncate_user; diff --git a/input/am_block_filtering.source b/input/am_block_filtering.source index 7ca6862c7..f93eb1988 100644 --- a/input/am_block_filtering.source +++ b/input/am_block_filtering.source @@ -28,6 +28,9 @@ $$ LANGUAGE PLPGSQL; -- Create and load data +-- block_row_count '1000', stripe_row_count '2000' +set cstore.stripe_row_count = 2000; +set cstore.block_row_count = 1000; CREATE TABLE test_block_filtering (a int) USING cstore_tableam; @@ -55,6 +58,8 @@ SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 2 SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 0'); SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 990 AND 2010'); +set cstore.stripe_row_count to default; +set cstore.block_row_count to default; -- Verify that we are fine with collations which use a different alphabet order CREATE TABLE collation_block_filtering_test(A text collate "da_DK") diff --git a/input/am_create.source b/input/am_create.source index 8a1612f7a..6d4d5a388 100644 --- a/input/am_create.source +++ b/input/am_create.source @@ -10,6 +10,7 @@ CREATE TABLE contestant (handle TEXT, birthdate DATE, rating INT, -- Create compressed table with automatically determined file path +-- COMPRESSED CREATE TABLE contestant_compressed (handle TEXT, birthdate DATE, rating INT, percentile FLOAT, country CHAR(3), achievements TEXT[]) USING cstore_tableam; diff --git a/input/am_load.source b/input/am_load.source index c2ad581e8..d0ef9bfac 100644 --- a/input/am_load.source +++ b/input/am_load.source @@ -16,11 +16,13 @@ COPY contestant FROM '@abs_srcdir@/data/contestants.1.csv' WITH CSV; COPY contestant FROM PROGRAM 'cat @abs_srcdir@/data/contestants.2.csv' WITH CSV; -- COPY into compressed table +set cstore.compression = 'pglz'; COPY contestant_compressed FROM '@abs_srcdir@/data/contestants.1.csv' WITH CSV; -- COPY into uncompressed table from program COPY contestant_compressed FROM PROGRAM 'cat @abs_srcdir@/data/contestants.2.csv' WITH CSV; +set cstore.compression to default; -- Test column list CREATE TABLE famous_constants (id int, name text, value real) diff --git a/output/am_block_filtering.source b/output/am_block_filtering.source index 2f664a78a..45cb702b6 100644 --- a/output/am_block_filtering.source +++ b/output/am_block_filtering.source @@ -24,9 +24,11 @@ $$ END; $$ LANGUAGE PLPGSQL; -- Create and load data -CREATE FOREIGN TABLE test_block_filtering (a int) - SERVER cstore_server - OPTIONS(block_row_count '1000', stripe_row_count '2000'); +-- block_row_count '1000', stripe_row_count '2000' +set cstore.stripe_row_count = 2000; +set cstore.block_row_count = 1000; +CREATE TABLE test_block_filtering (a int) + USING cstore_tableam; COPY test_block_filtering FROM '@abs_srcdir@/data/block_filtering.csv' WITH CSV; -- Verify that filtered_row_count is less than 1000 for the following queries SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering'); @@ -104,9 +106,11 @@ SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BET 3958 (1 row) +set cstore.stripe_row_count to default; +set cstore.block_row_count to default; -- Verify that we are fine with collations which use a different alphabet order -CREATE FOREIGN TABLE collation_block_filtering_test(A text collate "da_DK") - SERVER cstore_server; +CREATE TABLE collation_block_filtering_test(A text collate "da_DK") + USING cstore_tableam; COPY collation_block_filtering_test FROM STDIN; SELECT * FROM collation_block_filtering_test WHERE A > 'B'; a diff --git a/output/am_create.source b/output/am_create.source index e62447252..47c6a6c44 100644 --- a/output/am_create.source +++ b/output/am_create.source @@ -6,6 +6,7 @@ CREATE TABLE contestant (handle TEXT, birthdate DATE, rating INT, percentile FLOAT, country CHAR(3), achievements TEXT[]) USING cstore_tableam; -- Create compressed table with automatically determined file path +-- COMPRESSED CREATE TABLE contestant_compressed (handle TEXT, birthdate DATE, rating INT, percentile FLOAT, country CHAR(3), achievements TEXT[]) USING cstore_tableam; diff --git a/output/am_load.source b/output/am_load.source index d1f41f717..5eb81a250 100644 --- a/output/am_load.source +++ b/output/am_load.source @@ -15,10 +15,12 @@ COPY contestant FROM '@abs_srcdir@/data/contestants.1.csv' WITH CSV; -- COPY into uncompressed table from program COPY contestant FROM PROGRAM 'cat @abs_srcdir@/data/contestants.2.csv' WITH CSV; -- COPY into compressed table +set cstore.compression = 'pglz'; COPY contestant_compressed FROM '@abs_srcdir@/data/contestants.1.csv' WITH CSV; -- COPY into uncompressed table from program COPY contestant_compressed FROM PROGRAM 'cat @abs_srcdir@/data/contestants.2.csv' WITH CSV; +set cstore.compression to default; -- Test column list CREATE TABLE famous_constants (id int, name text, value real) USING cstore_tableam; diff --git a/sql/am_block_filtering.sql b/sql/am_block_filtering.sql index 38c63535c..2a45716ed 100644 --- a/sql/am_block_filtering.sql +++ b/sql/am_block_filtering.sql @@ -28,6 +28,9 @@ $$ LANGUAGE PLPGSQL; -- Create and load data +-- block_row_count '1000', stripe_row_count '2000' +set cstore.stripe_row_count = 2000; +set cstore.block_row_count = 1000; CREATE TABLE test_block_filtering (a int) USING cstore_tableam; @@ -55,6 +58,8 @@ SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 2 SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 0'); SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 990 AND 2010'); +set cstore.stripe_row_count to default; +set cstore.block_row_count to default; -- Verify that we are fine with collations which use a different alphabet order CREATE TABLE collation_block_filtering_test(A text collate "da_DK") diff --git a/sql/am_create.sql b/sql/am_create.sql index 8a1612f7a..6d4d5a388 100644 --- a/sql/am_create.sql +++ b/sql/am_create.sql @@ -10,6 +10,7 @@ CREATE TABLE contestant (handle TEXT, birthdate DATE, rating INT, -- Create compressed table with automatically determined file path +-- COMPRESSED CREATE TABLE contestant_compressed (handle TEXT, birthdate DATE, rating INT, percentile FLOAT, country CHAR(3), achievements TEXT[]) USING cstore_tableam; diff --git a/sql/am_load.sql b/sql/am_load.sql index c7e9e5287..edc727b3c 100644 --- a/sql/am_load.sql +++ b/sql/am_load.sql @@ -16,11 +16,13 @@ COPY contestant FROM '/Users/jefdavi/wd/cstore2/data/contestants.1.csv' WITH CSV COPY contestant FROM PROGRAM 'cat /Users/jefdavi/wd/cstore2/data/contestants.2.csv' WITH CSV; -- COPY into compressed table +set cstore.compression = 'pglz'; COPY contestant_compressed FROM '/Users/jefdavi/wd/cstore2/data/contestants.1.csv' WITH CSV; -- COPY into uncompressed table from program COPY contestant_compressed FROM PROGRAM 'cat /Users/jefdavi/wd/cstore2/data/contestants.2.csv' WITH CSV; +set cstore.compression to default; -- Test column list CREATE TABLE famous_constants (id int, name text, value real) diff --git a/sql/am_truncate.sql b/sql/am_truncate.sql index e124a7831..3fdce1d82 100644 --- a/sql/am_truncate.sql +++ b/sql/am_truncate.sql @@ -9,13 +9,16 @@ SELECT substring(:'server_version', '\d+')::int > 10 AS version_above_ten; -- CREATE a cstore_fdw table, fill with some data -- CREATE TABLE cstore_truncate_test (a int, b int) USING cstore_tableam; CREATE TABLE cstore_truncate_test_second (a int, b int) USING cstore_tableam; +-- COMPRESSED CREATE TABLE cstore_truncate_test_compressed (a int, b int) USING cstore_tableam; CREATE TABLE cstore_truncate_test_regular (a int, b int); INSERT INTO cstore_truncate_test select a, a from generate_series(1, 10) a; +set cstore.compression = 'pglz'; INSERT INTO cstore_truncate_test_compressed select a, a from generate_series(1, 10) a; INSERT INTO cstore_truncate_test_compressed select a, a from generate_series(1, 10) a; +set cstore.compression to default; -- query rows SELECT * FROM cstore_truncate_test; @@ -76,15 +79,19 @@ DROP TABLE cstore_truncate_test_compressed; -- test truncate with schema CREATE SCHEMA truncate_schema; +-- COMPRESSED CREATE TABLE truncate_schema.truncate_tbl (id int) USING cstore_tableam; +set cstore.compression = 'pglz'; INSERT INTO truncate_schema.truncate_tbl SELECT generate_series(1, 100); +set cstore.compression to default; SELECT COUNT(*) FROM truncate_schema.truncate_tbl; TRUNCATE TABLE truncate_schema.truncate_tbl; SELECT COUNT(*) FROM truncate_schema.truncate_tbl; +set cstore.compression = 'pglz'; INSERT INTO truncate_schema.truncate_tbl SELECT generate_series(1, 100); - +set cstore.compression to default; -- create a user that can not truncate CREATE USER truncate_user; GRANT USAGE ON SCHEMA truncate_schema TO truncate_user; @@ -108,7 +115,6 @@ GRANT TRUNCATE ON TABLE truncate_schema.truncate_tbl TO truncate_user; SELECT count(*) FROM truncate_schema.truncate_tbl; TRUNCATE TABLE truncate_schema.truncate_tbl; SELECT count(*) FROM truncate_schema.truncate_tbl; - \c - :current_user -- cleanup