almost works

Makefile

@@ -53,7 +53,8 @@ ifeq ($(USE_TABLEAM),yes)
 	PG_CFLAGS += -DUSE_TABLEAM
 	OBJS += cstore_tableam.o
 	REGRESS += am_create am_load am_query am_data_types am_functions \
-	           am_drop am_insert am_copyto am_alter am_truncate am_clean
+	           am_block_filtering am_drop am_insert am_copyto am_alter \
+		   am_truncate am_clean
 endif
 
 ifeq ($(enable_coverage),yes)

cstore_tableam.c

@@ -56,6 +56,16 @@ CStoreTableAMGetOptions(void)
 	return cstoreOptions;
 }
 
+static MemoryContext
+CStoreMemoryContext(void)
+{
+	if (CStoreContext == NULL)
+	{
+		CStoreContext = AllocSetContextCreate(TopMemoryContext, "cstore context",
+											  ALLOCSET_DEFAULT_SIZES);
+	}
+	return CStoreContext;
+}
 
 static void
 cstore_init_write_state(Relation relation)
@@ -73,22 +83,13 @@ cstore_init_write_state(Relation relation)
 	{
 		CStoreOptions *cstoreOptions = CStoreTableAMGetOptions();
 		TupleDesc tupdesc = RelationGetDescr(relation);
-		MemoryContext oldContext;
-
-		if (CStoreContext == NULL)
-		{
-			CStoreContext = AllocSetContextCreate(TopMemoryContext, "cstore context",
-												  ALLOCSET_DEFAULT_SIZES);
-		}
 
 		elog(LOG, "initializing write state for relation %d", relation->rd_id);
-		oldContext = MemoryContextSwitchTo(CStoreContext);
 		CStoreWriteState = CStoreBeginWrite(relation->rd_id,
 											cstoreOptions->compressionType,
 											cstoreOptions->stripeRowCount,
 											cstoreOptions->blockRowCount,
 											tupdesc);
-		MemoryContextSwitchTo(oldContext);
 
 		CStoreWriteState->relation = relation;
 	}
@@ -127,6 +128,7 @@ cstore_beginscan(Relation relation, Snapshot snapshot,
 	TableReadState *readState = NULL;
 	CStoreScanDesc scan = palloc(sizeof(CStoreScanDescData));
 	List *columnList = NIL;
+	MemoryContext oldContext = MemoryContextSwitchTo(CStoreMemoryContext());
 
 	cstoreOptions = CStoreTableAMGetOptions();
 
@@ -162,6 +164,7 @@ cstore_beginscan(Relation relation, Snapshot snapshot,
 
 	scan->cs_readState = readState;
 
+	MemoryContextSwitchTo(oldContext);
 	return ((TableScanDesc) scan);
 }
 
@@ -187,12 +190,15 @@ cstore_getnextslot(TableScanDesc sscan, ScanDirection direction, TupleTableSlot
 {
 	CStoreScanDesc scan = (CStoreScanDesc) sscan;
 	bool nextRowFound;
+	MemoryContext oldContext = MemoryContextSwitchTo(CStoreMemoryContext());
 
 	ExecClearTuple(slot);
 
 	nextRowFound = CStoreReadNextRow(scan->cs_readState, slot->tts_values,
 									 slot->tts_isnull);
 
+	MemoryContextSwitchTo(oldContext);
+
 	if (!nextRowFound)
 	{
 		return false;
@@ -303,6 +309,7 @@ cstore_tuple_insert(Relation relation, TupleTableSlot *slot, CommandId cid,
 					int options, BulkInsertState bistate)
 {
 	HeapTuple heapTuple;
+	MemoryContext oldContext = MemoryContextSwitchTo(CStoreMemoryContext());
 
 	cstore_init_write_state(relation);
 
@@ -319,6 +326,7 @@ cstore_tuple_insert(Relation relation, TupleTableSlot *slot, CommandId cid,
 	slot_getallattrs(slot);
 
 	CStoreWriteRow(CStoreWriteState, slot->tts_values, slot->tts_isnull);
+	MemoryContextSwitchTo(oldContext);
 }
 
 
@@ -343,6 +351,8 @@ static void
 cstore_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
 					CommandId cid, int options, BulkInsertState bistate)
 {
+	MemoryContext oldContext = MemoryContextSwitchTo(CStoreMemoryContext());
+
 	cstore_init_write_state(relation);
 
 	for (int i = 0; i < ntuples; i++)
@@ -363,6 +373,7 @@ cstore_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
 
 		CStoreWriteRow(CStoreWriteState, tupleSlot->tts_values, tupleSlot->tts_isnull);
 	}
+	MemoryContextSwitchTo(oldContext);
 }
 
 

expected/am_block_filtering.out

@@ -24,9 +24,11 @@ $$
     END;
 $$ LANGUAGE PLPGSQL;
 -- Create and load data
-CREATE FOREIGN TABLE test_block_filtering (a int)
+-- block_row_count '1000', stripe_row_count '2000'
-    SERVER cstore_server
+set cstore.stripe_row_count = 2000;
-    OPTIONS(block_row_count '1000', stripe_row_count '2000');
+set cstore.block_row_count = 1000;
+CREATE TABLE test_block_filtering (a int)
+    USING cstore_tableam;
 COPY test_block_filtering FROM '/Users/jefdavi/wd/cstore2/data/block_filtering.csv' WITH CSV;
 -- Verify that filtered_row_count is less than 1000 for the following queries
 SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering');
@@ -104,9 +106,11 @@ SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BET
                3958
 (1 row)
 
+set cstore.stripe_row_count to default;
+set cstore.block_row_count to default;
 -- Verify that we are fine with collations which use a different alphabet order
-CREATE FOREIGN TABLE collation_block_filtering_test(A text collate "da_DK")
+CREATE TABLE collation_block_filtering_test(A text collate "da_DK")
-    SERVER cstore_server;
+    USING cstore_tableam;
 COPY collation_block_filtering_test FROM STDIN;
 SELECT * FROM collation_block_filtering_test WHERE A > 'B';
  a 

expected/am_create.out

@@ -6,6 +6,7 @@ CREATE TABLE contestant (handle TEXT, birthdate DATE, rating INT,
 	percentile FLOAT, country CHAR(3), achievements TEXT[])
 	USING cstore_tableam;
 -- Create compressed table with automatically determined file path
+-- COMPRESSED
 CREATE TABLE contestant_compressed (handle TEXT, birthdate DATE, rating INT,
 	percentile FLOAT, country CHAR(3), achievements TEXT[])
 	USING cstore_tableam;

expected/am_load.out

@@ -15,10 +15,12 @@ COPY contestant FROM '/Users/jefdavi/wd/cstore2/data/contestants.1.csv' WITH CSV
 -- COPY into uncompressed table from program
 COPY contestant FROM PROGRAM 'cat /Users/jefdavi/wd/cstore2/data/contestants.2.csv' WITH CSV;
 -- COPY into compressed table
+set cstore.compression = 'pglz';
 COPY contestant_compressed FROM '/Users/jefdavi/wd/cstore2/data/contestants.1.csv' WITH CSV;
 -- COPY into uncompressed table from program
 COPY contestant_compressed FROM PROGRAM 'cat /Users/jefdavi/wd/cstore2/data/contestants.2.csv'
 	WITH CSV;
+set cstore.compression to default;
 -- Test column list
 CREATE TABLE famous_constants (id int, name text, value real)
     USING cstore_tableam;

expected/am_truncate.out

@@ -12,11 +12,14 @@ SELECT substring(:'server_version', '\d+')::int > 10 AS version_above_ten;
 -- CREATE a cstore_fdw table, fill with some data --
 CREATE TABLE cstore_truncate_test (a int, b int) USING cstore_tableam;
 CREATE TABLE cstore_truncate_test_second (a int, b int) USING cstore_tableam;
+-- COMPRESSED
 CREATE TABLE cstore_truncate_test_compressed (a int, b int) USING cstore_tableam;
 CREATE TABLE cstore_truncate_test_regular (a int, b int);
 INSERT INTO cstore_truncate_test select a, a from generate_series(1, 10) a;
+set cstore.compression = 'pglz';
 INSERT INTO cstore_truncate_test_compressed select a, a from generate_series(1, 10) a;
 INSERT INTO cstore_truncate_test_compressed select a, a from generate_series(1, 10) a;
+set cstore.compression to default;
 -- query rows
 SELECT * FROM cstore_truncate_test;
  a  | b  
@@ -168,8 +171,11 @@ DROP TABLE cstore_truncate_test_regular;
 DROP TABLE cstore_truncate_test_compressed;
 -- test truncate with schema
 CREATE SCHEMA truncate_schema;
+-- COMPRESSED
 CREATE TABLE truncate_schema.truncate_tbl (id int) USING cstore_tableam;
+set cstore.compression = 'pglz';
 INSERT INTO truncate_schema.truncate_tbl SELECT generate_series(1, 100);
+set cstore.compression to default;
 SELECT COUNT(*) FROM truncate_schema.truncate_tbl;
  count 
 -------
@@ -183,7 +189,9 @@ SELECT COUNT(*) FROM truncate_schema.truncate_tbl;
      0
 (1 row)
 
+set cstore.compression = 'pglz';
 INSERT INTO truncate_schema.truncate_tbl SELECT generate_series(1, 100);
+set cstore.compression to default;
 -- create a user that can not truncate
 CREATE USER truncate_user;
 GRANT USAGE ON SCHEMA truncate_schema TO truncate_user;

input/am_block_filtering.source

@@ -28,6 +28,9 @@ $$ LANGUAGE PLPGSQL;
 
 
 -- Create and load data
+-- block_row_count '1000', stripe_row_count '2000'
+set cstore.stripe_row_count = 2000;
+set cstore.block_row_count = 1000;
 CREATE TABLE test_block_filtering (a int)
     USING cstore_tableam;
 
@@ -55,6 +58,8 @@ SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 2
 SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 0');
 SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 990 AND 2010');
 
+set cstore.stripe_row_count to default;
+set cstore.block_row_count to default;
 
 -- Verify that we are fine with collations which use a different alphabet order
 CREATE TABLE collation_block_filtering_test(A text collate "da_DK")

input/am_create.source

@@ -10,6 +10,7 @@ CREATE TABLE contestant (handle TEXT, birthdate DATE, rating INT,
 
 
 -- Create compressed table with automatically determined file path
+-- COMPRESSED
 CREATE TABLE contestant_compressed (handle TEXT, birthdate DATE, rating INT,
 	percentile FLOAT, country CHAR(3), achievements TEXT[])
 	USING cstore_tableam;

input/am_load.source

@@ -16,11 +16,13 @@ COPY contestant FROM '@abs_srcdir@/data/contestants.1.csv' WITH CSV;
 COPY contestant FROM PROGRAM 'cat @abs_srcdir@/data/contestants.2.csv' WITH CSV;
 
 -- COPY into compressed table
+set cstore.compression = 'pglz';
 COPY contestant_compressed FROM '@abs_srcdir@/data/contestants.1.csv' WITH CSV;
 
 -- COPY into uncompressed table from program
 COPY contestant_compressed FROM PROGRAM 'cat @abs_srcdir@/data/contestants.2.csv'
 	WITH CSV;
+set cstore.compression to default;
 
 -- Test column list
 CREATE TABLE famous_constants (id int, name text, value real)

output/am_block_filtering.source

@@ -24,9 +24,11 @@ $$
     END;
 $$ LANGUAGE PLPGSQL;
 -- Create and load data
-CREATE FOREIGN TABLE test_block_filtering (a int)
+-- block_row_count '1000', stripe_row_count '2000'
-    SERVER cstore_server
+set cstore.stripe_row_count = 2000;
-    OPTIONS(block_row_count '1000', stripe_row_count '2000');
+set cstore.block_row_count = 1000;
+CREATE TABLE test_block_filtering (a int)
+    USING cstore_tableam;
 COPY test_block_filtering FROM '@abs_srcdir@/data/block_filtering.csv' WITH CSV;
 -- Verify that filtered_row_count is less than 1000 for the following queries
 SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering');
@@ -104,9 +106,11 @@ SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BET
                3958
 (1 row)
 
+set cstore.stripe_row_count to default;
+set cstore.block_row_count to default;
 -- Verify that we are fine with collations which use a different alphabet order
-CREATE FOREIGN TABLE collation_block_filtering_test(A text collate "da_DK")
+CREATE TABLE collation_block_filtering_test(A text collate "da_DK")
-    SERVER cstore_server;
+    USING cstore_tableam;
 COPY collation_block_filtering_test FROM STDIN;
 SELECT * FROM collation_block_filtering_test WHERE A > 'B';
  a 

output/am_create.source

@@ -6,6 +6,7 @@ CREATE TABLE contestant (handle TEXT, birthdate DATE, rating INT,
 	percentile FLOAT, country CHAR(3), achievements TEXT[])
 	USING cstore_tableam;
 -- Create compressed table with automatically determined file path
+-- COMPRESSED
 CREATE TABLE contestant_compressed (handle TEXT, birthdate DATE, rating INT,
 	percentile FLOAT, country CHAR(3), achievements TEXT[])
 	USING cstore_tableam;

output/am_load.source

@@ -15,10 +15,12 @@ COPY contestant FROM '@abs_srcdir@/data/contestants.1.csv' WITH CSV;
 -- COPY into uncompressed table from program
 COPY contestant FROM PROGRAM 'cat @abs_srcdir@/data/contestants.2.csv' WITH CSV;
 -- COPY into compressed table
+set cstore.compression = 'pglz';
 COPY contestant_compressed FROM '@abs_srcdir@/data/contestants.1.csv' WITH CSV;
 -- COPY into uncompressed table from program
 COPY contestant_compressed FROM PROGRAM 'cat @abs_srcdir@/data/contestants.2.csv'
 	WITH CSV;
+set cstore.compression to default;
 -- Test column list
 CREATE TABLE famous_constants (id int, name text, value real)
     USING cstore_tableam;

sql/am_block_filtering.sql

@@ -28,6 +28,9 @@ $$ LANGUAGE PLPGSQL;
 
 
 -- Create and load data
+-- block_row_count '1000', stripe_row_count '2000'
+set cstore.stripe_row_count = 2000;
+set cstore.block_row_count = 1000;
 CREATE TABLE test_block_filtering (a int)
     USING cstore_tableam;
 
@@ -55,6 +58,8 @@ SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 2
 SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 0');
 SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 990 AND 2010');
 
+set cstore.stripe_row_count to default;
+set cstore.block_row_count to default;
 
 -- Verify that we are fine with collations which use a different alphabet order
 CREATE TABLE collation_block_filtering_test(A text collate "da_DK")

sql/am_create.sql

@@ -10,6 +10,7 @@ CREATE TABLE contestant (handle TEXT, birthdate DATE, rating INT,
 
 
 -- Create compressed table with automatically determined file path
+-- COMPRESSED
 CREATE TABLE contestant_compressed (handle TEXT, birthdate DATE, rating INT,
 	percentile FLOAT, country CHAR(3), achievements TEXT[])
 	USING cstore_tableam;

sql/am_load.sql

@@ -16,11 +16,13 @@ COPY contestant FROM '/Users/jefdavi/wd/cstore2/data/contestants.1.csv' WITH CSV
 COPY contestant FROM PROGRAM 'cat /Users/jefdavi/wd/cstore2/data/contestants.2.csv' WITH CSV;
 
 -- COPY into compressed table
+set cstore.compression = 'pglz';
 COPY contestant_compressed FROM '/Users/jefdavi/wd/cstore2/data/contestants.1.csv' WITH CSV;
 
 -- COPY into uncompressed table from program
 COPY contestant_compressed FROM PROGRAM 'cat /Users/jefdavi/wd/cstore2/data/contestants.2.csv'
 	WITH CSV;
+set cstore.compression to default;
 
 -- Test column list
 CREATE TABLE famous_constants (id int, name text, value real)

sql/am_truncate.sql

@@ -9,13 +9,16 @@ SELECT substring(:'server_version', '\d+')::int > 10 AS version_above_ten;
 -- CREATE a cstore_fdw table, fill with some data --
 CREATE TABLE cstore_truncate_test (a int, b int) USING cstore_tableam;
 CREATE TABLE cstore_truncate_test_second (a int, b int) USING cstore_tableam;
+-- COMPRESSED
 CREATE TABLE cstore_truncate_test_compressed (a int, b int) USING cstore_tableam;
 CREATE TABLE cstore_truncate_test_regular (a int, b int);
 
 INSERT INTO cstore_truncate_test select a, a from generate_series(1, 10) a;
 
+set cstore.compression = 'pglz';
 INSERT INTO cstore_truncate_test_compressed select a, a from generate_series(1, 10) a;
 INSERT INTO cstore_truncate_test_compressed select a, a from generate_series(1, 10) a;
+set cstore.compression to default;
 
 -- query rows
 SELECT * FROM cstore_truncate_test;
@@ -76,15 +79,19 @@ DROP TABLE cstore_truncate_test_compressed;
 
 -- test truncate with schema
 CREATE SCHEMA truncate_schema;
+-- COMPRESSED
 CREATE TABLE truncate_schema.truncate_tbl (id int) USING cstore_tableam;
+set cstore.compression = 'pglz';
 INSERT INTO truncate_schema.truncate_tbl SELECT generate_series(1, 100);
+set cstore.compression to default;
 SELECT COUNT(*) FROM truncate_schema.truncate_tbl;
 
 TRUNCATE TABLE truncate_schema.truncate_tbl;
 SELECT COUNT(*) FROM truncate_schema.truncate_tbl;
 
+set cstore.compression = 'pglz';
 INSERT INTO truncate_schema.truncate_tbl SELECT generate_series(1, 100);
-
+set cstore.compression to default;
 -- create a user that can not truncate
 CREATE USER truncate_user;
 GRANT USAGE ON SCHEMA truncate_schema TO truncate_user;
@@ -108,7 +115,6 @@ GRANT TRUNCATE ON TABLE truncate_schema.truncate_tbl TO truncate_user;
 SELECT count(*) FROM truncate_schema.truncate_tbl;
 TRUNCATE TABLE truncate_schema.truncate_tbl;
 SELECT count(*) FROM truncate_schema.truncate_tbl;
-
 \c - :current_user
 
 -- cleanup