add AM tests

Makefile

@@ -14,7 +14,9 @@ DATA = cstore_fdw--1.7.sql cstore_fdw--1.6--1.7.sql  cstore_fdw--1.5--1.6.sql cs
 	   cstore_fdw--1.3--1.4.sql cstore_fdw--1.2--1.3.sql cstore_fdw--1.1--1.2.sql \
 	   cstore_fdw--1.0--1.1.sql cstore_fdw--1.7--1.8.sql
 
-REGRESS = fdw_create fdw_load fdw_query fdw_analyze fdw_data_types fdw_functions \
+REGRESS = am_create am_load am_query am_analyze am_data_types am_functions \
+	  am_block_filtering am_drop am_insert am_copyto am_alter am_truncate \
+	  fdw_create fdw_load fdw_query fdw_analyze fdw_data_types fdw_functions \
 	  fdw_block_filtering fdw_drop fdw_insert fdw_copyto fdw_alter fdw_truncate
 EXTRA_CLEAN = cstore.pb-c.h cstore.pb-c.c data/*.cstore data/*.cstore.footer \
               sql/block_filtering.sql sql/create.sql sql/data_types.sql sql/load.sql \

expected/am_alter.out

@@ -0,0 +1,178 @@
+--
+-- Testing ALTER TABLE on cstore_fdw tables.
+--
+CREATE FOREIGN TABLE test_alter_table (a int, b int, c int) SERVER cstore_server;
+WITH sample_data AS (VALUES
+    (1, 2, 3),
+    (4, 5, 6),
+    (7, 8, 9)
+)
+INSERT INTO test_alter_table SELECT * FROM sample_data;
+-- drop a column
+ALTER FOREIGN TABLE test_alter_table DROP COLUMN a;
+-- test analyze
+ANALYZE test_alter_table;
+-- verify select queries run as expected
+SELECT * FROM test_alter_table;
+ b | c 
+---+---
+ 2 | 3
+ 5 | 6
+ 8 | 9
+(3 rows)
+
+SELECT a FROM test_alter_table;
+ERROR:  column "a" does not exist
+LINE 1: SELECT a FROM test_alter_table;
+               ^
+SELECT b FROM test_alter_table;
+ b 
+---
+ 2
+ 5
+ 8
+(3 rows)
+
+-- verify insert runs as expected
+INSERT INTO test_alter_table (SELECT 3, 5, 8);
+ERROR:  INSERT has more expressions than target columns
+LINE 1: INSERT INTO test_alter_table (SELECT 3, 5, 8);
+                                                   ^
+INSERT INTO test_alter_table (SELECT 5, 8);
+-- add a column with no defaults
+ALTER FOREIGN TABLE test_alter_table ADD COLUMN d int;
+SELECT * FROM test_alter_table;
+ b | c | d 
+---+---+---
+ 2 | 3 |  
+ 5 | 6 |  
+ 8 | 9 |  
+ 5 | 8 |  
+(4 rows)
+
+INSERT INTO test_alter_table (SELECT 3, 5, 8);
+SELECT * FROM test_alter_table;
+ b | c | d 
+---+---+---
+ 2 | 3 |  
+ 5 | 6 |  
+ 8 | 9 |  
+ 5 | 8 |  
+ 3 | 5 | 8
+(5 rows)
+
+-- add a fixed-length column with default value
+ALTER FOREIGN TABLE test_alter_table ADD COLUMN e int default 3;
+SELECT * from test_alter_table;
+ b | c | d | e 
+---+---+---+---
+ 2 | 3 |   | 3
+ 5 | 6 |   | 3
+ 8 | 9 |   | 3
+ 5 | 8 |   | 3
+ 3 | 5 | 8 | 3
+(5 rows)
+
+INSERT INTO test_alter_table (SELECT 1, 2, 4, 8);
+SELECT * from test_alter_table;
+ b | c | d | e 
+---+---+---+---
+ 2 | 3 |   | 3
+ 5 | 6 |   | 3
+ 8 | 9 |   | 3
+ 5 | 8 |   | 3
+ 3 | 5 | 8 | 3
+ 1 | 2 | 4 | 8
+(6 rows)
+
+-- add a variable-length column with default value
+ALTER FOREIGN TABLE test_alter_table ADD COLUMN f text DEFAULT 'TEXT ME';
+SELECT * from test_alter_table;
+ b | c | d | e |    f    
+---+---+---+---+---------
+ 2 | 3 |   | 3 | TEXT ME
+ 5 | 6 |   | 3 | TEXT ME
+ 8 | 9 |   | 3 | TEXT ME
+ 5 | 8 |   | 3 | TEXT ME
+ 3 | 5 | 8 | 3 | TEXT ME
+ 1 | 2 | 4 | 8 | TEXT ME
+(6 rows)
+
+INSERT INTO test_alter_table (SELECT 1, 2, 4, 8, 'ABCDEF');
+SELECT * from test_alter_table;
+ b | c | d | e |    f    
+---+---+---+---+---------
+ 2 | 3 |   | 3 | TEXT ME
+ 5 | 6 |   | 3 | TEXT ME
+ 8 | 9 |   | 3 | TEXT ME
+ 5 | 8 |   | 3 | TEXT ME
+ 3 | 5 | 8 | 3 | TEXT ME
+ 1 | 2 | 4 | 8 | TEXT ME
+ 1 | 2 | 4 | 8 | ABCDEF
+(7 rows)
+
+-- drop couple of columns
+ALTER FOREIGN TABLE test_alter_table DROP COLUMN c;
+ALTER FOREIGN TABLE test_alter_table DROP COLUMN e;
+ANALYZE test_alter_table;
+SELECT * from test_alter_table;
+ b | d |    f    
+---+---+---------
+ 2 |   | TEXT ME
+ 5 |   | TEXT ME
+ 8 |   | TEXT ME
+ 5 |   | TEXT ME
+ 3 | 8 | TEXT ME
+ 1 | 4 | TEXT ME
+ 1 | 4 | ABCDEF
+(7 rows)
+
+SELECT count(*) from test_alter_table;
+ count 
+-------
+     7
+(1 row)
+
+SELECT count(t.*) from test_alter_table t;
+ count 
+-------
+     7
+(1 row)
+
+-- unsupported default values
+ALTER FOREIGN TABLE test_alter_table ADD COLUMN g boolean DEFAULT isfinite(current_date);
+ALTER FOREIGN TABLE test_alter_table ADD COLUMN h DATE DEFAULT current_date;
+SELECT * FROM test_alter_table;
+ERROR:  unsupported default value for column "g"
+HINT:  Expression is either mutable or does not evaluate to constant value
+ALTER FOREIGN TABLE test_alter_table ALTER COLUMN g DROP DEFAULT;
+SELECT * FROM test_alter_table;
+ERROR:  unsupported default value for column "h"
+HINT:  Expression is either mutable or does not evaluate to constant value
+ALTER FOREIGN TABLE test_alter_table ALTER COLUMN h DROP DEFAULT;
+ANALYZE test_alter_table;
+SELECT * FROM test_alter_table;
+ b | d |    f    | g | h 
+---+---+---------+---+---
+ 2 |   | TEXT ME |   | 
+ 5 |   | TEXT ME |   | 
+ 8 |   | TEXT ME |   | 
+ 5 |   | TEXT ME |   | 
+ 3 | 8 | TEXT ME |   | 
+ 1 | 4 | TEXT ME |   | 
+ 1 | 4 | ABCDEF  |   | 
+(7 rows)
+
+-- unsupported type change
+ALTER FOREIGN TABLE test_alter_table ADD COLUMN i int;
+ALTER FOREIGN TABLE test_alter_table ADD COLUMN j float;
+ALTER FOREIGN TABLE test_alter_table ADD COLUMN k text;
+-- this is valid type change
+ALTER FOREIGN TABLE test_alter_table ALTER COLUMN i TYPE float;
+-- this is not valid
+ALTER FOREIGN TABLE test_alter_table ALTER COLUMN j TYPE int;
+ERROR:  Column j cannot be cast automatically to type pg_catalog.int4
+-- text / varchar conversion is valid both ways
+ALTER FOREIGN TABLE test_alter_table ALTER COLUMN k TYPE varchar(20);
+ALTER FOREIGN TABLE test_alter_table ALTER COLUMN k TYPE text;
+DROP FOREIGN TABLE test_alter_table;

expected/am_analyze.out

@@ -0,0 +1,19 @@
+--
+-- Test the ANALYZE command for cstore_fdw tables.
+--
+-- ANALYZE uncompressed table
+ANALYZE contestant;
+SELECT count(*) FROM pg_stats WHERE tablename='contestant';
+ count 
+-------
+     6
+(1 row)
+
+-- ANALYZE compressed table
+ANALYZE contestant_compressed;
+SELECT count(*) FROM pg_stats WHERE tablename='contestant_compressed';
+ count 
+-------
+     6
+(1 row)
+

expected/am_block_filtering.out

@@ -0,0 +1,116 @@
+--
+-- Test block filtering in cstore_fdw using min/max values in stripe skip lists.
+--
+--
+-- filtered_row_count returns number of rows filtered by the WHERE clause.
+-- If blocks get filtered by cstore_fdw, less rows are passed to WHERE
+-- clause, so this function should return a lower number.
+--
+CREATE OR REPLACE FUNCTION filtered_row_count (query text) RETURNS bigint AS
+$$
+    DECLARE
+        result bigint;
+        rec text;
+    BEGIN
+        result := 0;
+
+        FOR rec IN EXECUTE 'EXPLAIN ANALYZE ' || query LOOP
+            IF rec ~ '^\s+Rows Removed by Filter' then
+                result := regexp_replace(rec, '[^0-9]*', '', 'g');
+            END IF;
+        END LOOP;
+
+        RETURN result;
+    END;
+$$ LANGUAGE PLPGSQL;
+-- Create and load data
+CREATE FOREIGN TABLE test_block_filtering (a int)
+    SERVER cstore_server
+    OPTIONS(block_row_count '1000', stripe_row_count '2000');
+COPY test_block_filtering FROM '/Users/jefdavi/wd/cstore2/data/block_filtering.csv' WITH CSV;
+-- Verify that filtered_row_count is less than 1000 for the following queries
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering');
+ filtered_row_count 
+--------------------
+                  0
+(1 row)
+
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 200');
+ filtered_row_count 
+--------------------
+                801
+(1 row)
+
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a > 200');
+ filtered_row_count 
+--------------------
+                200
+(1 row)
+
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 9900');
+ filtered_row_count 
+--------------------
+                101
+(1 row)
+
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a > 9900');
+ filtered_row_count 
+--------------------
+                900
+(1 row)
+
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 0');
+ filtered_row_count 
+--------------------
+                  0
+(1 row)
+
+-- Verify that filtered_row_count is less than 2000 for the following queries
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 1 AND 10');
+ filtered_row_count 
+--------------------
+                990
+(1 row)
+
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 990 AND 2010');
+ filtered_row_count 
+--------------------
+               1979
+(1 row)
+
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN -10 AND 0');
+ filtered_row_count 
+--------------------
+                  0
+(1 row)
+
+-- Load data for second time and verify that filtered_row_count is exactly twice as before
+COPY test_block_filtering FROM '/Users/jefdavi/wd/cstore2/data/block_filtering.csv' WITH CSV;
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 200');
+ filtered_row_count 
+--------------------
+               1602
+(1 row)
+
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 0');
+ filtered_row_count 
+--------------------
+                  0
+(1 row)
+
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 990 AND 2010');
+ filtered_row_count 
+--------------------
+               3958
+(1 row)
+
+-- Verify that we are fine with collations which use a different alphabet order
+CREATE FOREIGN TABLE collation_block_filtering_test(A text collate "da_DK")
+    SERVER cstore_server;
+COPY collation_block_filtering_test FROM STDIN;
+SELECT * FROM collation_block_filtering_test WHERE A > 'B';
+ a 
+---
+ Å
+(1 row)
+

expected/am_copyto.out

@@ -0,0 +1,23 @@
+--
+-- Test copying data from cstore_fdw tables.
+--
+CREATE FOREIGN TABLE test_contestant(handle TEXT, birthdate DATE, rating INT,
+        percentile FLOAT, country CHAR(3), achievements TEXT[])
+        SERVER cstore_server;
+-- load table data from file
+COPY test_contestant FROM '/Users/jefdavi/wd/cstore2/data/contestants.1.csv' WITH CSV;
+-- export using COPY table TO ...
+COPY test_contestant TO STDOUT;
+a	01-10-1990	2090	97.1	XA 	{a}
+b	11-01-1990	2203	98.1	XA 	{a,b}
+c	11-01-1988	2907	99.4	XB 	{w,y}
+d	05-05-1985	2314	98.3	XB 	{}
+e	05-05-1995	2236	98.2	XC 	{a}
+-- export using COPY (SELECT * FROM table) TO ...
+COPY (select * from test_contestant) TO STDOUT;
+a	01-10-1990	2090	97.1	XA 	{a}
+b	11-01-1990	2203	98.1	XA 	{a,b}
+c	11-01-1988	2907	99.4	XB 	{w,y}
+d	05-05-1985	2314	98.3	XB 	{}
+e	05-05-1995	2236	98.2	XC 	{a}
+DROP FOREIGN TABLE test_contestant CASCADE;

expected/am_create.out

@@ -0,0 +1,44 @@
+--
+-- Test the CREATE statements related to cstore_fdw.
+--
+-- Install cstore_fdw
+CREATE EXTENSION cstore_fdw;
+CREATE SERVER cstore_server FOREIGN DATA WRAPPER cstore_fdw;
+-- Validator tests
+CREATE FOREIGN TABLE test_validator_invalid_option () 
+	SERVER cstore_server 
+	OPTIONS(bad_option_name '1'); -- ERROR
+ERROR:  invalid option "bad_option_name"
+HINT:  Valid options in this context are: compression, stripe_row_count, block_row_count
+CREATE FOREIGN TABLE test_validator_invalid_stripe_row_count () 
+	SERVER cstore_server
+	OPTIONS(stripe_row_count '0'); -- ERROR
+ERROR:  invalid stripe row count
+HINT:  Stripe row count must be an integer between 1000 and 10000000
+CREATE FOREIGN TABLE test_validator_invalid_block_row_count () 
+	SERVER cstore_server
+	OPTIONS(block_row_count '0'); -- ERROR
+ERROR:  invalid block row count
+HINT:  Block row count must be an integer between 1000 and 100000
+CREATE FOREIGN TABLE test_validator_invalid_compression_type () 
+	SERVER cstore_server
+	OPTIONS(compression 'invalid_compression'); -- ERROR
+ERROR:  invalid compression type
+HINT:  Valid options are: none, pglz
+-- Create uncompressed table
+CREATE FOREIGN TABLE contestant (handle TEXT, birthdate DATE, rating INT,
+	percentile FLOAT, country CHAR(3), achievements TEXT[])
+	SERVER cstore_server;
+-- Create compressed table with automatically determined file path
+CREATE FOREIGN TABLE contestant_compressed (handle TEXT, birthdate DATE, rating INT,
+	percentile FLOAT, country CHAR(3), achievements TEXT[])
+	SERVER cstore_server
+	OPTIONS(compression 'pglz');
+-- Test that querying an empty table works
+ANALYZE contestant;
+SELECT count(*) FROM contestant;
+ count 
+-------
+     0
+(1 row)
+

expected/am_data_types.out

@@ -0,0 +1,78 @@
+--
+-- Test loading and reading different data types to/from cstore_fdw foreign tables.
+--
+-- Settings to make the result deterministic
+SET datestyle = "ISO, YMD";
+SET timezone to 'GMT';
+SET intervalstyle TO 'POSTGRES_VERBOSE';
+-- Test array types
+CREATE FOREIGN TABLE test_array_types (int_array int[], bigint_array bigint[],
+	text_array text[]) SERVER cstore_server;
+COPY test_array_types FROM '/Users/jefdavi/wd/cstore2/data/array_types.csv' WITH CSV;
+SELECT * FROM test_array_types;
+        int_array         |                bigint_array                | text_array 
+--------------------------+--------------------------------------------+------------
+ {1,2,3}                  | {1,2,3}                                    | {a,b,c}
+ {}                       | {}                                         | {}
+ {-2147483648,2147483647} | {-9223372036854775808,9223372036854775807} | {""}
+(3 rows)
+
+-- Test date/time types
+CREATE FOREIGN TABLE test_datetime_types (timestamp timestamp,
+	timestamp_with_timezone timestamp with time zone, date date, time time,
+	interval interval) SERVER cstore_server;
+COPY test_datetime_types FROM '/Users/jefdavi/wd/cstore2/data/datetime_types.csv' WITH CSV;
+SELECT * FROM test_datetime_types;
+      timestamp      | timestamp_with_timezone |    date    |   time   | interval  
+---------------------+-------------------------+------------+----------+-----------
+ 2000-01-02 04:05:06 | 1999-01-08 12:05:06+00  | 2000-01-02 | 04:05:06 | @ 4 hours
+ 1970-01-01 00:00:00 | infinity                | -infinity  | 00:00:00 | @ 0
+(2 rows)
+
+-- Test enum and composite types
+CREATE TYPE enum_type AS ENUM ('a', 'b', 'c');
+CREATE TYPE composite_type AS (a int, b text);
+CREATE FOREIGN TABLE test_enum_and_composite_types (enum enum_type,
+	composite composite_type) SERVER cstore_server;
+COPY test_enum_and_composite_types FROM
+	'/Users/jefdavi/wd/cstore2/data/enum_and_composite_types.csv' WITH CSV;
+SELECT * FROM test_enum_and_composite_types;
+ enum | composite 
+------+-----------
+ a    | (2,b)
+ b    | (3,c)
+(2 rows)
+
+-- Test range types
+CREATE FOREIGN TABLE test_range_types (int4range int4range, int8range int8range,
+	numrange numrange, tsrange tsrange) SERVER cstore_server;
+COPY test_range_types FROM '/Users/jefdavi/wd/cstore2/data/range_types.csv' WITH CSV;
+SELECT * FROM test_range_types;
+ int4range | int8range | numrange |                    tsrange                    
+-----------+-----------+----------+-----------------------------------------------
+ [1,3)     | [1,3)     | [1,3)    | ["2000-01-02 00:30:00","2010-02-03 12:30:00")
+ empty     | [1,)      | (,)      | empty
+(2 rows)
+
+-- Test other types
+CREATE FOREIGN TABLE test_other_types (bool boolean, bytea bytea, money money,
+	inet inet, bitstring bit varying(5), uuid uuid, json json) SERVER cstore_server;
+COPY test_other_types FROM '/Users/jefdavi/wd/cstore2/data/other_types.csv' WITH CSV;
+SELECT * FROM test_other_types;
+ bool |   bytea    | money |    inet     | bitstring |                 uuid                 |       json       
+------+------------+-------+-------------+-----------+--------------------------------------+------------------
+ f    | \xdeadbeef | $1.00 | 192.168.1.2 | 10101     | a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11 | {"key": "value"}
+ t    | \xcdb0     | $1.50 | 127.0.0.1   |           | a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11 | []
+(2 rows)
+
+-- Test null values
+CREATE FOREIGN TABLE test_null_values (a int, b int[], c composite_type)
+	SERVER cstore_server;
+COPY test_null_values FROM '/Users/jefdavi/wd/cstore2/data/null_values.csv' WITH CSV;
+SELECT * FROM test_null_values;
+ a |   b    |  c  
+---+--------+-----
+   | {NULL} | (,)
+   |        | 
+(2 rows)
+

expected/am_drop.out

@@ -0,0 +1,40 @@
+--
+-- Tests the different DROP commands for cstore_fdw tables.
+--
+-- DROP FOREIGN TABL
+-- DROP SCHEMA
+-- DROP EXTENSION
+-- DROP DATABASE
+--
+-- Note that travis does not create
+-- cstore_fdw extension in default database (postgres). This has caused
+-- different behavior between travis tests and local tests. Thus
+-- 'postgres' directory is excluded from comparison to have the same result.
+-- store postgres database oid
+SELECT oid postgres_oid FROM pg_database WHERE datname = 'postgres' \gset
+-- DROP cstore_fdw tables
+DROP FOREIGN TABLE contestant;
+DROP FOREIGN TABLE contestant_compressed;
+-- Create a cstore_fdw table under a schema and drop it.
+CREATE SCHEMA test_schema;
+CREATE FOREIGN TABLE test_schema.test_table(data int) SERVER cstore_server;
+DROP SCHEMA test_schema CASCADE;
+NOTICE:  drop cascades to foreign table test_schema.test_table
+SELECT current_database() datname \gset
+CREATE DATABASE db_to_drop;
+\c db_to_drop
+CREATE EXTENSION cstore_fdw;
+CREATE SERVER cstore_server FOREIGN DATA WRAPPER cstore_fdw;
+SELECT oid::text databaseoid FROM pg_database WHERE datname = current_database() \gset
+CREATE FOREIGN TABLE test_table(data int) SERVER cstore_server;
+DROP EXTENSION cstore_fdw CASCADE;
+NOTICE:  drop cascades to 2 other objects
+DETAIL:  drop cascades to server cstore_server
+drop cascades to foreign table test_table
+-- test database drop
+CREATE EXTENSION cstore_fdw;
+CREATE SERVER cstore_server FOREIGN DATA WRAPPER cstore_fdw;
+SELECT oid::text databaseoid FROM pg_database WHERE datname = current_database() \gset
+CREATE FOREIGN TABLE test_table(data int) SERVER cstore_server;
+\c :datname
+DROP DATABASE db_to_drop;

expected/am_functions.out

@@ -0,0 +1,18 @@
+--
+-- Test utility functions for cstore_fdw tables.
+--
+CREATE FOREIGN TABLE empty_table (a int) SERVER cstore_server;
+CREATE FOREIGN TABLE table_with_data (a int) SERVER cstore_server;
+CREATE TABLE non_cstore_table (a int);
+COPY table_with_data FROM STDIN;
+SELECT cstore_table_size('empty_table') < cstore_table_size('table_with_data');
+ ?column? 
+----------
+ t
+(1 row)
+
+SELECT cstore_table_size('non_cstore_table');
+ERROR:  relation is not a cstore table
+DROP FOREIGN TABLE empty_table;
+DROP FOREIGN TABLE table_with_data;
+DROP TABLE non_cstore_table;

expected/am_insert.out

@@ -0,0 +1,88 @@
+--
+-- Testing insert on cstore_fdw tables.
+--
+CREATE FOREIGN TABLE test_insert_command (a int) SERVER cstore_server;
+-- test single row inserts fail
+select count(*) from test_insert_command;
+ count 
+-------
+     0
+(1 row)
+
+insert into test_insert_command values(1);
+ERROR:  operation is not supported
+select count(*) from test_insert_command;
+ count 
+-------
+     0
+(1 row)
+
+insert into test_insert_command default values;
+ERROR:  operation is not supported
+select count(*) from test_insert_command;
+ count 
+-------
+     0
+(1 row)
+
+-- test inserting from another table succeed
+CREATE TABLE test_insert_command_data (a int);
+select count(*) from test_insert_command_data;
+ count 
+-------
+     0
+(1 row)
+
+insert into test_insert_command_data values(1);
+select count(*) from test_insert_command_data;
+ count 
+-------
+     1
+(1 row)
+
+insert into test_insert_command select * from test_insert_command_data;
+select count(*) from test_insert_command;
+ count 
+-------
+     1
+(1 row)
+
+drop table test_insert_command_data;
+drop foreign table test_insert_command;
+-- test long attribute value insertion
+-- create sufficiently long text so that data is stored in toast
+CREATE TABLE test_long_text AS
+SELECT a as int_val, string_agg(random()::text, '') as text_val
+FROM generate_series(1, 10) a, generate_series(1, 1000) b
+GROUP BY a ORDER BY a;
+-- store hash values of text for later comparison
+CREATE TABLE test_long_text_hash AS
+SELECT int_val, md5(text_val) AS hash
+FROM test_long_text;
+CREATE FOREIGN TABLE test_cstore_long_text(int_val int, text_val text)
+SERVER cstore_server;
+-- store long text in cstore table
+INSERT INTO test_cstore_long_text SELECT * FROM test_long_text;
+-- drop source table to remove original text from toast
+DROP TABLE test_long_text;
+-- check if text data is still available in cstore table
+-- by comparing previously stored hash.
+SELECT a.int_val
+FROM  test_long_text_hash a, test_cstore_long_text c
+WHERE a.int_val = c.int_val AND a.hash = md5(c.text_val);
+ int_val 
+---------
+       1
+       2
+       3
+       4
+       5
+       6
+       7
+       8
+       9
+      10
+(10 rows)
+
+DROP TABLE test_long_text_hash;
+DROP FOREIGN TABLE test_cstore_long_text;

expected/am_load.out

@@ -0,0 +1,39 @@
+--
+-- Test loading data into cstore_fdw tables.
+--
+-- COPY with incorrect delimiter
+COPY contestant FROM '/Users/jefdavi/wd/cstore2/data/contestants.1.csv'
+	WITH DELIMITER '|'; -- ERROR
+ERROR:  missing data for column "birthdate"
+-- COPY with invalid program
+COPY contestant FROM PROGRAM 'invalid_program' WITH CSV; -- ERROR
+ERROR:  program "invalid_program" failed
+DETAIL:  command not found
+-- COPY into uncompressed table from file
+COPY contestant FROM '/Users/jefdavi/wd/cstore2/data/contestants.1.csv' WITH CSV;
+-- COPY into uncompressed table from program
+COPY contestant FROM PROGRAM 'cat /Users/jefdavi/wd/cstore2/data/contestants.2.csv' WITH CSV;
+-- COPY into compressed table
+COPY contestant_compressed FROM '/Users/jefdavi/wd/cstore2/data/contestants.1.csv' WITH CSV;
+-- COPY into uncompressed table from program
+COPY contestant_compressed FROM PROGRAM 'cat /Users/jefdavi/wd/cstore2/data/contestants.2.csv'
+	WITH CSV;
+-- Test column list
+CREATE FOREIGN TABLE famous_constants (id int, name text, value real)
+    SERVER cstore_server;
+COPY famous_constants (value, name, id) FROM STDIN WITH CSV;
+COPY famous_constants (name, value) FROM STDIN WITH CSV;
+SELECT * FROM famous_constants ORDER BY id, name;
+ id |      name      |   value   
+----+----------------+-----------
+  1 | pi             |     3.141
+  2 | e              |     2.718
+  3 | gamma          |     0.577
+  4 | bohr radius    | 5.291e-11
+    | avagadro       | 6.022e+23
+    | electron mass  | 9.109e-31
+    | proton mass    | 1.672e-27
+    | speed of light | 2.997e+08
+(8 rows)
+
+DROP FOREIGN TABLE famous_constants;

expected/am_query.out

@@ -0,0 +1,105 @@
+--
+-- Test querying cstore_fdw tables.
+--
+-- Settings to make the result deterministic
+SET datestyle = "ISO, YMD";
+-- Query uncompressed data
+SELECT count(*) FROM contestant;
+ count 
+-------
+     8
+(1 row)
+
+SELECT avg(rating), stddev_samp(rating) FROM contestant;
+          avg          |   stddev_samp    
+-----------------------+------------------
+ 2344.3750000000000000 | 433.746119785032
+(1 row)
+
+SELECT country, avg(rating) FROM contestant WHERE rating > 2200
+	GROUP BY country ORDER BY country;
+ country |          avg          
+---------+-----------------------
+ XA      | 2203.0000000000000000
+ XB      | 2610.5000000000000000
+ XC      | 2236.0000000000000000
+ XD      | 3090.0000000000000000
+(4 rows)
+
+SELECT * FROM contestant ORDER BY handle;
+ handle | birthdate  | rating | percentile | country | achievements 
+--------+------------+--------+------------+---------+--------------
+ a      | 1990-01-10 |   2090 |       97.1 | XA      | {a}
+ b      | 1990-11-01 |   2203 |       98.1 | XA      | {a,b}
+ c      | 1988-11-01 |   2907 |       99.4 | XB      | {w,y}
+ d      | 1985-05-05 |   2314 |       98.3 | XB      | {}
+ e      | 1995-05-05 |   2236 |       98.2 | XC      | {a}
+ f      | 1983-04-02 |   3090 |       99.6 | XD      | {a,b,c,y}
+ g      | 1991-12-13 |   1803 |       85.1 | XD      | {a,c}
+ h      | 1987-10-26 |   2112 |       95.4 | XD      | {w,a}
+(8 rows)
+
+-- Query compressed data
+SELECT count(*) FROM contestant_compressed;
+ count 
+-------
+     8
+(1 row)
+
+SELECT avg(rating), stddev_samp(rating) FROM contestant_compressed;
+          avg          |   stddev_samp    
+-----------------------+------------------
+ 2344.3750000000000000 | 433.746119785032
+(1 row)
+
+SELECT country, avg(rating) FROM contestant_compressed WHERE rating > 2200
+	GROUP BY country ORDER BY country;
+ country |          avg          
+---------+-----------------------
+ XA      | 2203.0000000000000000
+ XB      | 2610.5000000000000000
+ XC      | 2236.0000000000000000
+ XD      | 3090.0000000000000000
+(4 rows)
+
+SELECT * FROM contestant_compressed ORDER BY handle;
+ handle | birthdate  | rating | percentile | country | achievements 
+--------+------------+--------+------------+---------+--------------
+ a      | 1990-01-10 |   2090 |       97.1 | XA      | {a}
+ b      | 1990-11-01 |   2203 |       98.1 | XA      | {a,b}
+ c      | 1988-11-01 |   2907 |       99.4 | XB      | {w,y}
+ d      | 1985-05-05 |   2314 |       98.3 | XB      | {}
+ e      | 1995-05-05 |   2236 |       98.2 | XC      | {a}
+ f      | 1983-04-02 |   3090 |       99.6 | XD      | {a,b,c,y}
+ g      | 1991-12-13 |   1803 |       85.1 | XD      | {a,c}
+ h      | 1987-10-26 |   2112 |       95.4 | XD      | {w,a}
+(8 rows)
+
+-- Verify that we handle whole-row references correctly
+SELECT to_json(v) FROM contestant v ORDER BY rating LIMIT 1;
+                                                     to_json                                                      
+------------------------------------------------------------------------------------------------------------------
+ {"handle":"g","birthdate":"1991-12-13","rating":1803,"percentile":85.1,"country":"XD ","achievements":["a","c"]}
+(1 row)
+
+-- Test variables used in expressions
+CREATE FOREIGN TABLE union_first (a int, b int) SERVER cstore_server;
+CREATE FOREIGN TABLE union_second (a int, b int) SERVER cstore_server;
+INSERT INTO union_first SELECT a, a FROM generate_series(1, 5) a;
+INSERT INTO union_second SELECT a, a FROM generate_series(11, 15) a;
+(SELECT a*1, b FROM union_first) union all (SELECT a*1, b FROM union_second);
+ ?column? | b  
+----------+----
+        1 |  1
+        2 |  2
+        3 |  3
+        4 |  4
+        5 |  5
+       11 | 11
+       12 | 12
+       13 | 13
+       14 | 14
+       15 | 15
+(10 rows)
+
+DROP FOREIGN TABLE union_first, union_second;

expected/am_truncate.out

@@ -0,0 +1,231 @@
+--
+-- Test the TRUNCATE TABLE command for cstore_fdw tables.
+--
+-- print whether we're using version > 10 to make version-specific tests clear
+SHOW server_version \gset
+SELECT substring(:'server_version', '\d+')::int > 10 AS version_above_ten;
+ version_above_ten 
+-------------------
+ t
+(1 row)
+
+-- CREATE a cstore_fdw table, fill with some data --
+CREATE FOREIGN TABLE cstore_truncate_test (a int, b int) SERVER cstore_server;
+CREATE FOREIGN TABLE cstore_truncate_test_second (a int, b int) SERVER cstore_server;
+CREATE FOREIGN TABLE cstore_truncate_test_compressed (a int, b int) SERVER cstore_server OPTIONS (compression 'pglz');
+CREATE TABLE cstore_truncate_test_regular (a int, b int);
+INSERT INTO cstore_truncate_test select a, a from generate_series(1, 10) a;
+INSERT INTO cstore_truncate_test_compressed select a, a from generate_series(1, 10) a;
+INSERT INTO cstore_truncate_test_compressed select a, a from generate_series(1, 10) a;
+-- query rows
+SELECT * FROM cstore_truncate_test;
+ a  | b  
+----+----
+  1 |  1
+  2 |  2
+  3 |  3
+  4 |  4
+  5 |  5
+  6 |  6
+  7 |  7
+  8 |  8
+  9 |  9
+ 10 | 10
+(10 rows)
+
+TRUNCATE TABLE cstore_truncate_test;
+SELECT * FROM cstore_truncate_test;
+ a | b 
+---+---
+(0 rows)
+
+SELECT COUNT(*) from cstore_truncate_test;
+ count 
+-------
+     0
+(1 row)
+
+SELECT count(*) FROM cstore_truncate_test_compressed;
+ count 
+-------
+    20
+(1 row)
+
+TRUNCATE TABLE cstore_truncate_test_compressed;
+SELECT count(*) FROM cstore_truncate_test_compressed;
+ count 
+-------
+     0
+(1 row)
+
+SELECT cstore_table_size('cstore_truncate_test_compressed');
+ cstore_table_size 
+-------------------
+                 0
+(1 row)
+
+INSERT INTO cstore_truncate_test select a, a from generate_series(1, 10) a;
+INSERT INTO cstore_truncate_test_regular select a, a from generate_series(10, 20) a;
+INSERT INTO cstore_truncate_test_second select a, a from generate_series(20, 30) a;
+SELECT * from cstore_truncate_test;
+ a  | b  
+----+----
+  1 |  1
+  2 |  2
+  3 |  3
+  4 |  4
+  5 |  5
+  6 |  6
+  7 |  7
+  8 |  8
+  9 |  9
+ 10 | 10
+(10 rows)
+
+SELECT * from cstore_truncate_test_second;
+ a  | b  
+----+----
+ 20 | 20
+ 21 | 21
+ 22 | 22
+ 23 | 23
+ 24 | 24
+ 25 | 25
+ 26 | 26
+ 27 | 27
+ 28 | 28
+ 29 | 29
+ 30 | 30
+(11 rows)
+
+SELECT * from cstore_truncate_test_regular;
+ a  | b  
+----+----
+ 10 | 10
+ 11 | 11
+ 12 | 12
+ 13 | 13
+ 14 | 14
+ 15 | 15
+ 16 | 16
+ 17 | 17
+ 18 | 18
+ 19 | 19
+ 20 | 20
+(11 rows)
+
+-- make sure multi truncate works
+-- notice that the same table might be repeated
+TRUNCATE TABLE cstore_truncate_test,
+			   cstore_truncate_test_regular,
+			   cstore_truncate_test_second,
+   			   cstore_truncate_test;
+SELECT * from cstore_truncate_test;
+ a | b 
+---+---
+(0 rows)
+
+SELECT * from cstore_truncate_test_second;
+ a | b 
+---+---
+(0 rows)
+
+SELECT * from cstore_truncate_test_regular;
+ a | b 
+---+---
+(0 rows)
+
+-- test if truncate on empty table works
+TRUNCATE TABLE cstore_truncate_test;
+SELECT * from cstore_truncate_test;
+ a | b 
+---+---
+(0 rows)
+
+-- test if a cached truncate from a pl/pgsql function works
+CREATE FUNCTION cstore_truncate_test_regular_func() RETURNS void AS $$
+BEGIN
+	INSERT INTO cstore_truncate_test_regular select a, a from generate_series(1, 10) a;
+	TRUNCATE TABLE cstore_truncate_test_regular;
+END;$$
+LANGUAGE plpgsql;
+SELECT cstore_truncate_test_regular_func();
+ cstore_truncate_test_regular_func 
+-----------------------------------
+ 
+(1 row)
+
+-- the cached plans are used stating from the second call
+SELECT cstore_truncate_test_regular_func();
+ cstore_truncate_test_regular_func 
+-----------------------------------
+ 
+(1 row)
+
+DROP FUNCTION cstore_truncate_test_regular_func();
+DROP FOREIGN TABLE cstore_truncate_test, cstore_truncate_test_second;
+DROP TABLE cstore_truncate_test_regular;
+DROP FOREIGN TABLE cstore_truncate_test_compressed;
+-- test truncate with schema
+CREATE SCHEMA truncate_schema;
+CREATE FOREIGN TABLE truncate_schema.truncate_tbl (id int) SERVER cstore_server OPTIONS(compression 'pglz');
+INSERT INTO truncate_schema.truncate_tbl SELECT generate_series(1, 100);
+SELECT COUNT(*) FROM truncate_schema.truncate_tbl;
+ count 
+-------
+   100
+(1 row)
+
+TRUNCATE TABLE truncate_schema.truncate_tbl;
+SELECT COUNT(*) FROM truncate_schema.truncate_tbl;
+ count 
+-------
+     0
+(1 row)
+
+INSERT INTO truncate_schema.truncate_tbl SELECT generate_series(1, 100);
+-- create a user that can not truncate
+CREATE USER truncate_user;
+GRANT USAGE ON SCHEMA truncate_schema TO truncate_user;
+GRANT SELECT ON TABLE truncate_schema.truncate_tbl TO truncate_user;
+REVOKE TRUNCATE ON TABLE truncate_schema.truncate_tbl FROM truncate_user;
+SELECT current_user \gset
+\c - truncate_user
+-- verify truncate command fails and check number of rows
+SELECT count(*) FROM truncate_schema.truncate_tbl;
+ count 
+-------
+   100
+(1 row)
+
+TRUNCATE TABLE truncate_schema.truncate_tbl;
+ERROR:  permission denied for table truncate_tbl
+SELECT count(*) FROM truncate_schema.truncate_tbl;
+ count 
+-------
+   100
+(1 row)
+
+-- switch to super user, grant truncate to truncate_user 
+\c - :current_user
+GRANT TRUNCATE ON TABLE truncate_schema.truncate_tbl TO truncate_user;
+-- verify truncate_user can truncate now
+\c - truncate_user
+SELECT count(*) FROM truncate_schema.truncate_tbl;
+ count 
+-------
+   100
+(1 row)
+
+TRUNCATE TABLE truncate_schema.truncate_tbl;
+SELECT count(*) FROM truncate_schema.truncate_tbl;
+ count 
+-------
+     0
+(1 row)
+
+\c - :current_user
+-- cleanup
+DROP SCHEMA truncate_schema CASCADE;
+NOTICE:  drop cascades to foreign table truncate_schema.truncate_tbl
+DROP USER truncate_user;

expected/am_truncate_0.out

@@ -0,0 +1,262 @@
+--
+-- Test the TRUNCATE TABLE command for cstore_fdw tables.
+--
+-- print whether we're using version > 10 to make version-specific tests clear
+SHOW server_version \gset
+SELECT substring(:'server_version', '\d+')::int > 10 AS version_above_ten;
+ version_above_ten 
+-------------------
+ f
+(1 row)
+
+-- Check that files for the automatically managed table exist in the
+-- cstore_fdw/{databaseoid} directory.
+SELECT count(*) FROM (
+	SELECT pg_ls_dir('cstore_fdw/' || databaseoid ) FROM (
+	SELECT oid::text databaseoid FROM pg_database WHERE datname = current_database()
+	) AS q1) AS q2;
+ count 
+-------
+     0
+(1 row)
+
+-- CREATE a cstore_fdw table, fill with some data --
+CREATE FOREIGN TABLE cstore_truncate_test (a int, b int) SERVER cstore_server;
+CREATE FOREIGN TABLE cstore_truncate_test_second (a int, b int) SERVER cstore_server;
+CREATE FOREIGN TABLE cstore_truncate_test_compressed (a int, b int) SERVER cstore_server OPTIONS (compression 'pglz');
+CREATE TABLE cstore_truncate_test_regular (a int, b int);
+INSERT INTO cstore_truncate_test select a, a from generate_series(1, 10) a;
+INSERT INTO cstore_truncate_test_compressed select a, a from generate_series(1, 10) a;
+INSERT INTO cstore_truncate_test_compressed select a, a from generate_series(1, 10) a;
+-- query rows
+SELECT * FROM cstore_truncate_test;
+ a  | b  
+----+----
+  1 |  1
+  2 |  2
+  3 |  3
+  4 |  4
+  5 |  5
+  6 |  6
+  7 |  7
+  8 |  8
+  9 |  9
+ 10 | 10
+(10 rows)
+
+TRUNCATE TABLE cstore_truncate_test;
+SELECT * FROM cstore_truncate_test;
+ a | b 
+---+---
+(0 rows)
+
+SELECT COUNT(*) from cstore_truncate_test;
+ count 
+-------
+     0
+(1 row)
+
+SELECT count(*) FROM cstore_truncate_test_compressed;
+ count 
+-------
+    20
+(1 row)
+
+TRUNCATE TABLE cstore_truncate_test_compressed;
+SELECT count(*) FROM cstore_truncate_test_compressed;
+ count 
+-------
+     0
+(1 row)
+
+SELECT cstore_table_size('cstore_truncate_test_compressed');
+ cstore_table_size 
+-------------------
+                26
+(1 row)
+
+-- make sure data files still present 
+SELECT count(*) FROM (
+	SELECT pg_ls_dir('cstore_fdw/' || databaseoid ) FROM (
+	SELECT oid::text databaseoid FROM pg_database WHERE datname = current_database()
+	) AS q1) AS q2;
+ count 
+-------
+     6
+(1 row)
+
+INSERT INTO cstore_truncate_test select a, a from generate_series(1, 10) a;
+INSERT INTO cstore_truncate_test_regular select a, a from generate_series(10, 20) a;
+INSERT INTO cstore_truncate_test_second select a, a from generate_series(20, 30) a;
+SELECT * from cstore_truncate_test;
+ a  | b  
+----+----
+  1 |  1
+  2 |  2
+  3 |  3
+  4 |  4
+  5 |  5
+  6 |  6
+  7 |  7
+  8 |  8
+  9 |  9
+ 10 | 10
+(10 rows)
+
+SELECT * from cstore_truncate_test_second;
+ a  | b  
+----+----
+ 20 | 20
+ 21 | 21
+ 22 | 22
+ 23 | 23
+ 24 | 24
+ 25 | 25
+ 26 | 26
+ 27 | 27
+ 28 | 28
+ 29 | 29
+ 30 | 30
+(11 rows)
+
+SELECT * from cstore_truncate_test_regular;
+ a  | b  
+----+----
+ 10 | 10
+ 11 | 11
+ 12 | 12
+ 13 | 13
+ 14 | 14
+ 15 | 15
+ 16 | 16
+ 17 | 17
+ 18 | 18
+ 19 | 19
+ 20 | 20
+(11 rows)
+
+-- make sure multi truncate works
+-- notice that the same table might be repeated
+TRUNCATE TABLE cstore_truncate_test,
+			   cstore_truncate_test_regular,
+			   cstore_truncate_test_second,
+   			   cstore_truncate_test;
+SELECT * from cstore_truncate_test;
+ a | b 
+---+---
+(0 rows)
+
+SELECT * from cstore_truncate_test_second;
+ a | b 
+---+---
+(0 rows)
+
+SELECT * from cstore_truncate_test_regular;
+ a | b 
+---+---
+(0 rows)
+
+-- test if truncate on empty table works
+TRUNCATE TABLE cstore_truncate_test;
+SELECT * from cstore_truncate_test;
+ a | b 
+---+---
+(0 rows)
+
+-- test if a cached truncate from a pl/pgsql function works
+CREATE FUNCTION cstore_truncate_test_regular_func() RETURNS void AS $$
+BEGIN
+	INSERT INTO cstore_truncate_test_regular select a, a from generate_series(1, 10) a;
+	TRUNCATE TABLE cstore_truncate_test_regular;
+END;$$
+LANGUAGE plpgsql;
+SELECT cstore_truncate_test_regular_func();
+ cstore_truncate_test_regular_func 
+-----------------------------------
+ 
+(1 row)
+
+-- the cached plans are used stating from the second call
+SELECT cstore_truncate_test_regular_func();
+ cstore_truncate_test_regular_func 
+-----------------------------------
+ 
+(1 row)
+
+DROP FUNCTION cstore_truncate_test_regular_func();
+DROP FOREIGN TABLE cstore_truncate_test, cstore_truncate_test_second;
+DROP TABLE cstore_truncate_test_regular;
+DROP FOREIGN TABLE cstore_truncate_test_compressed;
+-- test truncate with schema
+CREATE SCHEMA truncate_schema;
+CREATE FOREIGN TABLE truncate_schema.truncate_tbl (id int) SERVER cstore_server OPTIONS(compression 'pglz');
+INSERT INTO truncate_schema.truncate_tbl SELECT generate_series(1, 100);
+SELECT COUNT(*) FROM truncate_schema.truncate_tbl;
+ count 
+-------
+   100
+(1 row)
+
+TRUNCATE TABLE truncate_schema.truncate_tbl;
+SELECT COUNT(*) FROM truncate_schema.truncate_tbl;
+ count 
+-------
+     0
+(1 row)
+
+INSERT INTO truncate_schema.truncate_tbl SELECT generate_series(1, 100);
+-- create a user that can not truncate
+CREATE USER truncate_user;
+GRANT USAGE ON SCHEMA truncate_schema TO truncate_user;
+GRANT SELECT ON TABLE truncate_schema.truncate_tbl TO truncate_user;
+REVOKE TRUNCATE ON TABLE truncate_schema.truncate_tbl FROM truncate_user;
+SELECT current_user \gset
+\c - truncate_user
+-- verify truncate command fails and check number of rows
+SELECT count(*) FROM truncate_schema.truncate_tbl;
+ count 
+-------
+   100
+(1 row)
+
+TRUNCATE TABLE truncate_schema.truncate_tbl;
+ERROR:  permission denied for relation truncate_tbl
+SELECT count(*) FROM truncate_schema.truncate_tbl;
+ count 
+-------
+   100
+(1 row)
+
+-- switch to super user, grant truncate to truncate_user 
+\c - :current_user
+GRANT TRUNCATE ON TABLE truncate_schema.truncate_tbl TO truncate_user;
+-- verify truncate_user can truncate now
+\c - truncate_user
+SELECT count(*) FROM truncate_schema.truncate_tbl;
+ count 
+-------
+   100
+(1 row)
+
+TRUNCATE TABLE truncate_schema.truncate_tbl;
+SELECT count(*) FROM truncate_schema.truncate_tbl;
+ count 
+-------
+     0
+(1 row)
+
+\c - :current_user
+-- cleanup
+DROP SCHEMA truncate_schema CASCADE;
+NOTICE:  drop cascades to foreign table truncate_schema.truncate_tbl
+DROP USER truncate_user;
+-- verify files are removed
+SELECT count(*) FROM (
+	SELECT pg_ls_dir('cstore_fdw/' || databaseoid ) FROM (
+	SELECT oid::text databaseoid FROM pg_database WHERE datname = current_database()
+	) AS q1) AS q2;
+ count 
+-------
+     0
+(1 row)
+

input/am_block_filtering.source

@@ -0,0 +1,69 @@
+--
+-- Test block filtering in cstore_fdw using min/max values in stripe skip lists.
+--
+
+
+--
+-- filtered_row_count returns number of rows filtered by the WHERE clause.
+-- If blocks get filtered by cstore_fdw, less rows are passed to WHERE
+-- clause, so this function should return a lower number.
+--
+CREATE OR REPLACE FUNCTION filtered_row_count (query text) RETURNS bigint AS
+$$
+    DECLARE
+        result bigint;
+        rec text;
+    BEGIN
+        result := 0;
+
+        FOR rec IN EXECUTE 'EXPLAIN ANALYZE ' || query LOOP
+            IF rec ~ '^\s+Rows Removed by Filter' then
+                result := regexp_replace(rec, '[^0-9]*', '', 'g');
+            END IF;
+        END LOOP;
+
+        RETURN result;
+    END;
+$$ LANGUAGE PLPGSQL;
+
+
+-- Create and load data
+CREATE FOREIGN TABLE test_block_filtering (a int)
+    SERVER cstore_server
+    OPTIONS(block_row_count '1000', stripe_row_count '2000');
+
+COPY test_block_filtering FROM '@abs_srcdir@/data/block_filtering.csv' WITH CSV;
+
+
+-- Verify that filtered_row_count is less than 1000 for the following queries
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering');
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 200');
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a > 200');
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 9900');
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a > 9900');
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 0');
+
+
+-- Verify that filtered_row_count is less than 2000 for the following queries
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 1 AND 10');
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 990 AND 2010');
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN -10 AND 0');
+
+
+-- Load data for second time and verify that filtered_row_count is exactly twice as before
+COPY test_block_filtering FROM '@abs_srcdir@/data/block_filtering.csv' WITH CSV;
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 200');
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 0');
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 990 AND 2010');
+
+
+-- Verify that we are fine with collations which use a different alphabet order
+CREATE FOREIGN TABLE collation_block_filtering_test(A text collate "da_DK")
+    SERVER cstore_server;
+COPY collation_block_filtering_test FROM STDIN;
+A
+Å
+B
+\.
+
+SELECT * FROM collation_block_filtering_test WHERE A > 'B';

input/am_copyto.source

@@ -0,0 +1,17 @@
+--
+-- Test copying data from cstore_fdw tables.
+--
+CREATE FOREIGN TABLE test_contestant(handle TEXT, birthdate DATE, rating INT,
+        percentile FLOAT, country CHAR(3), achievements TEXT[])
+        SERVER cstore_server;
+
+-- load table data from file
+COPY test_contestant FROM '@abs_srcdir@/data/contestants.1.csv' WITH CSV;
+
+-- export using COPY table TO ...
+COPY test_contestant TO STDOUT;
+
+-- export using COPY (SELECT * FROM table) TO ...
+COPY (select * from test_contestant) TO STDOUT;
+
+DROP FOREIGN TABLE test_contestant CASCADE;

input/am_create.source

@@ -0,0 +1,43 @@
+--
+-- Test the CREATE statements related to cstore_fdw.
+--
+
+
+-- Install cstore_fdw
+CREATE EXTENSION cstore_fdw;
+
+CREATE SERVER cstore_server FOREIGN DATA WRAPPER cstore_fdw;
+
+
+-- Validator tests
+CREATE FOREIGN TABLE test_validator_invalid_option () 
+	SERVER cstore_server 
+	OPTIONS(bad_option_name '1'); -- ERROR
+
+CREATE FOREIGN TABLE test_validator_invalid_stripe_row_count () 
+	SERVER cstore_server
+	OPTIONS(stripe_row_count '0'); -- ERROR
+
+CREATE FOREIGN TABLE test_validator_invalid_block_row_count () 
+	SERVER cstore_server
+	OPTIONS(block_row_count '0'); -- ERROR
+
+CREATE FOREIGN TABLE test_validator_invalid_compression_type () 
+	SERVER cstore_server
+	OPTIONS(compression 'invalid_compression'); -- ERROR
+
+-- Create uncompressed table
+CREATE FOREIGN TABLE contestant (handle TEXT, birthdate DATE, rating INT,
+	percentile FLOAT, country CHAR(3), achievements TEXT[])
+	SERVER cstore_server;
+
+
+-- Create compressed table with automatically determined file path
+CREATE FOREIGN TABLE contestant_compressed (handle TEXT, birthdate DATE, rating INT,
+	percentile FLOAT, country CHAR(3), achievements TEXT[])
+	SERVER cstore_server
+	OPTIONS(compression 'pglz');
+
+-- Test that querying an empty table works
+ANALYZE contestant;
+SELECT count(*) FROM contestant;

input/am_data_types.source

@@ -0,0 +1,68 @@
+--
+-- Test loading and reading different data types to/from cstore_fdw foreign tables.
+--
+
+
+-- Settings to make the result deterministic
+SET datestyle = "ISO, YMD";
+SET timezone to 'GMT';
+SET intervalstyle TO 'POSTGRES_VERBOSE';
+
+
+-- Test array types
+CREATE FOREIGN TABLE test_array_types (int_array int[], bigint_array bigint[],
+	text_array text[]) SERVER cstore_server;
+
+COPY test_array_types FROM '@abs_srcdir@/data/array_types.csv' WITH CSV;
+
+SELECT * FROM test_array_types;
+
+
+-- Test date/time types
+CREATE FOREIGN TABLE test_datetime_types (timestamp timestamp,
+	timestamp_with_timezone timestamp with time zone, date date, time time,
+	interval interval) SERVER cstore_server;
+
+COPY test_datetime_types FROM '@abs_srcdir@/data/datetime_types.csv' WITH CSV;
+
+SELECT * FROM test_datetime_types;
+
+
+-- Test enum and composite types
+CREATE TYPE enum_type AS ENUM ('a', 'b', 'c');
+CREATE TYPE composite_type AS (a int, b text);
+
+CREATE FOREIGN TABLE test_enum_and_composite_types (enum enum_type,
+	composite composite_type) SERVER cstore_server;
+
+COPY test_enum_and_composite_types FROM
+	'@abs_srcdir@/data/enum_and_composite_types.csv' WITH CSV;
+
+SELECT * FROM test_enum_and_composite_types;
+
+
+-- Test range types
+CREATE FOREIGN TABLE test_range_types (int4range int4range, int8range int8range,
+	numrange numrange, tsrange tsrange) SERVER cstore_server;
+
+COPY test_range_types FROM '@abs_srcdir@/data/range_types.csv' WITH CSV;
+
+SELECT * FROM test_range_types;
+
+
+-- Test other types
+CREATE FOREIGN TABLE test_other_types (bool boolean, bytea bytea, money money,
+	inet inet, bitstring bit varying(5), uuid uuid, json json) SERVER cstore_server;
+
+COPY test_other_types FROM '@abs_srcdir@/data/other_types.csv' WITH CSV;
+
+SELECT * FROM test_other_types;
+
+
+-- Test null values
+CREATE FOREIGN TABLE test_null_values (a int, b int[], c composite_type)
+	SERVER cstore_server;
+
+COPY test_null_values FROM '@abs_srcdir@/data/null_values.csv' WITH CSV;
+
+SELECT * FROM test_null_values;

input/am_load.source

@@ -0,0 +1,44 @@
+--
+-- Test loading data into cstore_fdw tables.
+--
+
+-- COPY with incorrect delimiter
+COPY contestant FROM '@abs_srcdir@/data/contestants.1.csv'
+	WITH DELIMITER '|'; -- ERROR
+
+-- COPY with invalid program
+COPY contestant FROM PROGRAM 'invalid_program' WITH CSV; -- ERROR
+
+-- COPY into uncompressed table from file
+COPY contestant FROM '@abs_srcdir@/data/contestants.1.csv' WITH CSV;
+
+-- COPY into uncompressed table from program
+COPY contestant FROM PROGRAM 'cat @abs_srcdir@/data/contestants.2.csv' WITH CSV;
+
+-- COPY into compressed table
+COPY contestant_compressed FROM '@abs_srcdir@/data/contestants.1.csv' WITH CSV;
+
+-- COPY into uncompressed table from program
+COPY contestant_compressed FROM PROGRAM 'cat @abs_srcdir@/data/contestants.2.csv'
+	WITH CSV;
+
+-- Test column list
+CREATE FOREIGN TABLE famous_constants (id int, name text, value real)
+    SERVER cstore_server;
+COPY famous_constants (value, name, id) FROM STDIN WITH CSV;
+3.141,pi,1
+2.718,e,2
+0.577,gamma,3
+5.291e-11,bohr radius,4
+\.
+
+COPY famous_constants (name, value) FROM STDIN WITH CSV;
+avagadro,6.022e23
+electron mass,9.109e-31
+proton mass,1.672e-27
+speed of light,2.997e8
+\.
+
+SELECT * FROM famous_constants ORDER BY id, name;
+
+DROP FOREIGN TABLE famous_constants;

output/am_block_filtering.source

@@ -0,0 +1,116 @@
+--
+-- Test block filtering in cstore_fdw using min/max values in stripe skip lists.
+--
+--
+-- filtered_row_count returns number of rows filtered by the WHERE clause.
+-- If blocks get filtered by cstore_fdw, less rows are passed to WHERE
+-- clause, so this function should return a lower number.
+--
+CREATE OR REPLACE FUNCTION filtered_row_count (query text) RETURNS bigint AS
+$$
+    DECLARE
+        result bigint;
+        rec text;
+    BEGIN
+        result := 0;
+
+        FOR rec IN EXECUTE 'EXPLAIN ANALYZE ' || query LOOP
+            IF rec ~ '^\s+Rows Removed by Filter' then
+                result := regexp_replace(rec, '[^0-9]*', '', 'g');
+            END IF;
+        END LOOP;
+
+        RETURN result;
+    END;
+$$ LANGUAGE PLPGSQL;
+-- Create and load data
+CREATE FOREIGN TABLE test_block_filtering (a int)
+    SERVER cstore_server
+    OPTIONS(block_row_count '1000', stripe_row_count '2000');
+COPY test_block_filtering FROM '@abs_srcdir@/data/block_filtering.csv' WITH CSV;
+-- Verify that filtered_row_count is less than 1000 for the following queries
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering');
+ filtered_row_count 
+--------------------
+                  0
+(1 row)
+
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 200');
+ filtered_row_count 
+--------------------
+                801
+(1 row)
+
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a > 200');
+ filtered_row_count 
+--------------------
+                200
+(1 row)
+
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 9900');
+ filtered_row_count 
+--------------------
+                101
+(1 row)
+
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a > 9900');
+ filtered_row_count 
+--------------------
+                900
+(1 row)
+
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 0');
+ filtered_row_count 
+--------------------
+                  0
+(1 row)
+
+-- Verify that filtered_row_count is less than 2000 for the following queries
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 1 AND 10');
+ filtered_row_count 
+--------------------
+                990
+(1 row)
+
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 990 AND 2010');
+ filtered_row_count 
+--------------------
+               1979
+(1 row)
+
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN -10 AND 0');
+ filtered_row_count 
+--------------------
+                  0
+(1 row)
+
+-- Load data for second time and verify that filtered_row_count is exactly twice as before
+COPY test_block_filtering FROM '@abs_srcdir@/data/block_filtering.csv' WITH CSV;
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 200');
+ filtered_row_count 
+--------------------
+               1602
+(1 row)
+
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 0');
+ filtered_row_count 
+--------------------
+                  0
+(1 row)
+
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 990 AND 2010');
+ filtered_row_count 
+--------------------
+               3958
+(1 row)
+
+-- Verify that we are fine with collations which use a different alphabet order
+CREATE FOREIGN TABLE collation_block_filtering_test(A text collate "da_DK")
+    SERVER cstore_server;
+COPY collation_block_filtering_test FROM STDIN;
+SELECT * FROM collation_block_filtering_test WHERE A > 'B';
+ a 
+---
+ Å
+(1 row)
+

output/am_copyto.source

@@ -0,0 +1,23 @@
+--
+-- Test copying data from cstore_fdw tables.
+--
+CREATE FOREIGN TABLE test_contestant(handle TEXT, birthdate DATE, rating INT,
+        percentile FLOAT, country CHAR(3), achievements TEXT[])
+        SERVER cstore_server;
+-- load table data from file
+COPY test_contestant FROM '@abs_srcdir@/data/contestants.1.csv' WITH CSV;
+-- export using COPY table TO ...
+COPY test_contestant TO STDOUT;
+a	01-10-1990	2090	97.1	XA 	{a}
+b	11-01-1990	2203	98.1	XA 	{a,b}
+c	11-01-1988	2907	99.4	XB 	{w,y}
+d	05-05-1985	2314	98.3	XB 	{}
+e	05-05-1995	2236	98.2	XC 	{a}
+-- export using COPY (SELECT * FROM table) TO ...
+COPY (select * from test_contestant) TO STDOUT;
+a	01-10-1990	2090	97.1	XA 	{a}
+b	11-01-1990	2203	98.1	XA 	{a,b}
+c	11-01-1988	2907	99.4	XB 	{w,y}
+d	05-05-1985	2314	98.3	XB 	{}
+e	05-05-1995	2236	98.2	XC 	{a}
+DROP FOREIGN TABLE test_contestant CASCADE;

output/am_create.source

@@ -0,0 +1,44 @@
+--
+-- Test the CREATE statements related to cstore_fdw.
+--
+-- Install cstore_fdw
+CREATE EXTENSION cstore_fdw;
+CREATE SERVER cstore_server FOREIGN DATA WRAPPER cstore_fdw;
+-- Validator tests
+CREATE FOREIGN TABLE test_validator_invalid_option () 
+	SERVER cstore_server 
+	OPTIONS(bad_option_name '1'); -- ERROR
+ERROR:  invalid option "bad_option_name"
+HINT:  Valid options in this context are: compression, stripe_row_count, block_row_count
+CREATE FOREIGN TABLE test_validator_invalid_stripe_row_count () 
+	SERVER cstore_server
+	OPTIONS(stripe_row_count '0'); -- ERROR
+ERROR:  invalid stripe row count
+HINT:  Stripe row count must be an integer between 1000 and 10000000
+CREATE FOREIGN TABLE test_validator_invalid_block_row_count () 
+	SERVER cstore_server
+	OPTIONS(block_row_count '0'); -- ERROR
+ERROR:  invalid block row count
+HINT:  Block row count must be an integer between 1000 and 100000
+CREATE FOREIGN TABLE test_validator_invalid_compression_type () 
+	SERVER cstore_server
+	OPTIONS(compression 'invalid_compression'); -- ERROR
+ERROR:  invalid compression type
+HINT:  Valid options are: none, pglz
+-- Create uncompressed table
+CREATE FOREIGN TABLE contestant (handle TEXT, birthdate DATE, rating INT,
+	percentile FLOAT, country CHAR(3), achievements TEXT[])
+	SERVER cstore_server;
+-- Create compressed table with automatically determined file path
+CREATE FOREIGN TABLE contestant_compressed (handle TEXT, birthdate DATE, rating INT,
+	percentile FLOAT, country CHAR(3), achievements TEXT[])
+	SERVER cstore_server
+	OPTIONS(compression 'pglz');
+-- Test that querying an empty table works
+ANALYZE contestant;
+SELECT count(*) FROM contestant;
+ count 
+-------
+     0
+(1 row)
+

output/am_data_types.source

@@ -0,0 +1,78 @@
+--
+-- Test loading and reading different data types to/from cstore_fdw foreign tables.
+--
+-- Settings to make the result deterministic
+SET datestyle = "ISO, YMD";
+SET timezone to 'GMT';
+SET intervalstyle TO 'POSTGRES_VERBOSE';
+-- Test array types
+CREATE FOREIGN TABLE test_array_types (int_array int[], bigint_array bigint[],
+	text_array text[]) SERVER cstore_server;
+COPY test_array_types FROM '@abs_srcdir@/data/array_types.csv' WITH CSV;
+SELECT * FROM test_array_types;
+        int_array         |                bigint_array                | text_array 
+--------------------------+--------------------------------------------+------------
+ {1,2,3}                  | {1,2,3}                                    | {a,b,c}
+ {}                       | {}                                         | {}
+ {-2147483648,2147483647} | {-9223372036854775808,9223372036854775807} | {""}
+(3 rows)
+
+-- Test date/time types
+CREATE FOREIGN TABLE test_datetime_types (timestamp timestamp,
+	timestamp_with_timezone timestamp with time zone, date date, time time,
+	interval interval) SERVER cstore_server;
+COPY test_datetime_types FROM '@abs_srcdir@/data/datetime_types.csv' WITH CSV;
+SELECT * FROM test_datetime_types;
+      timestamp      | timestamp_with_timezone |    date    |   time   | interval  
+---------------------+-------------------------+------------+----------+-----------
+ 2000-01-02 04:05:06 | 1999-01-08 12:05:06+00  | 2000-01-02 | 04:05:06 | @ 4 hours
+ 1970-01-01 00:00:00 | infinity                | -infinity  | 00:00:00 | @ 0
+(2 rows)
+
+-- Test enum and composite types
+CREATE TYPE enum_type AS ENUM ('a', 'b', 'c');
+CREATE TYPE composite_type AS (a int, b text);
+CREATE FOREIGN TABLE test_enum_and_composite_types (enum enum_type,
+	composite composite_type) SERVER cstore_server;
+COPY test_enum_and_composite_types FROM
+	'@abs_srcdir@/data/enum_and_composite_types.csv' WITH CSV;
+SELECT * FROM test_enum_and_composite_types;
+ enum | composite 
+------+-----------
+ a    | (2,b)
+ b    | (3,c)
+(2 rows)
+
+-- Test range types
+CREATE FOREIGN TABLE test_range_types (int4range int4range, int8range int8range,
+	numrange numrange, tsrange tsrange) SERVER cstore_server;
+COPY test_range_types FROM '@abs_srcdir@/data/range_types.csv' WITH CSV;
+SELECT * FROM test_range_types;
+ int4range | int8range | numrange |                    tsrange                    
+-----------+-----------+----------+-----------------------------------------------
+ [1,3)     | [1,3)     | [1,3)    | ["2000-01-02 00:30:00","2010-02-03 12:30:00")
+ empty     | [1,)      | (,)      | empty
+(2 rows)
+
+-- Test other types
+CREATE FOREIGN TABLE test_other_types (bool boolean, bytea bytea, money money,
+	inet inet, bitstring bit varying(5), uuid uuid, json json) SERVER cstore_server;
+COPY test_other_types FROM '@abs_srcdir@/data/other_types.csv' WITH CSV;
+SELECT * FROM test_other_types;
+ bool |   bytea    | money |    inet     | bitstring |                 uuid                 |       json       
+------+------------+-------+-------------+-----------+--------------------------------------+------------------
+ f    | \xdeadbeef | $1.00 | 192.168.1.2 | 10101     | a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11 | {"key": "value"}
+ t    | \xcdb0     | $1.50 | 127.0.0.1   |           | a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11 | []
+(2 rows)
+
+-- Test null values
+CREATE FOREIGN TABLE test_null_values (a int, b int[], c composite_type)
+	SERVER cstore_server;
+COPY test_null_values FROM '@abs_srcdir@/data/null_values.csv' WITH CSV;
+SELECT * FROM test_null_values;
+ a |   b    |  c  
+---+--------+-----
+   | {NULL} | (,)
+   |        | 
+(2 rows)
+

output/am_load.source

@@ -0,0 +1,39 @@
+--
+-- Test loading data into cstore_fdw tables.
+--
+-- COPY with incorrect delimiter
+COPY contestant FROM '@abs_srcdir@/data/contestants.1.csv'
+	WITH DELIMITER '|'; -- ERROR
+ERROR:  missing data for column "birthdate"
+-- COPY with invalid program
+COPY contestant FROM PROGRAM 'invalid_program' WITH CSV; -- ERROR
+ERROR:  program "invalid_program" failed
+DETAIL:  command not found
+-- COPY into uncompressed table from file
+COPY contestant FROM '@abs_srcdir@/data/contestants.1.csv' WITH CSV;
+-- COPY into uncompressed table from program
+COPY contestant FROM PROGRAM 'cat @abs_srcdir@/data/contestants.2.csv' WITH CSV;
+-- COPY into compressed table
+COPY contestant_compressed FROM '@abs_srcdir@/data/contestants.1.csv' WITH CSV;
+-- COPY into uncompressed table from program
+COPY contestant_compressed FROM PROGRAM 'cat @abs_srcdir@/data/contestants.2.csv'
+	WITH CSV;
+-- Test column list
+CREATE FOREIGN TABLE famous_constants (id int, name text, value real)
+    SERVER cstore_server;
+COPY famous_constants (value, name, id) FROM STDIN WITH CSV;
+COPY famous_constants (name, value) FROM STDIN WITH CSV;
+SELECT * FROM famous_constants ORDER BY id, name;
+ id |      name      |   value   
+----+----------------+-----------
+  1 | pi             |     3.141
+  2 | e              |     2.718
+  3 | gamma          |     0.577
+  4 | bohr radius    | 5.291e-11
+    | avagadro       | 6.022e+23
+    | electron mass  | 9.109e-31
+    | proton mass    | 1.672e-27
+    | speed of light | 2.997e+08
+(8 rows)
+
+DROP FOREIGN TABLE famous_constants;

sql/am_alter.sql

@@ -0,0 +1,85 @@
+--
+-- Testing ALTER TABLE on cstore_fdw tables.
+--
+
+CREATE FOREIGN TABLE test_alter_table (a int, b int, c int) SERVER cstore_server;
+
+WITH sample_data AS (VALUES
+    (1, 2, 3),
+    (4, 5, 6),
+    (7, 8, 9)
+)
+INSERT INTO test_alter_table SELECT * FROM sample_data;
+
+-- drop a column
+ALTER FOREIGN TABLE test_alter_table DROP COLUMN a;
+
+-- test analyze
+ANALYZE test_alter_table;
+
+-- verify select queries run as expected
+SELECT * FROM test_alter_table;
+SELECT a FROM test_alter_table;
+SELECT b FROM test_alter_table;
+
+-- verify insert runs as expected
+INSERT INTO test_alter_table (SELECT 3, 5, 8);
+INSERT INTO test_alter_table (SELECT 5, 8);
+
+
+-- add a column with no defaults
+ALTER FOREIGN TABLE test_alter_table ADD COLUMN d int;
+SELECT * FROM test_alter_table;
+INSERT INTO test_alter_table (SELECT 3, 5, 8);
+SELECT * FROM test_alter_table;
+
+
+-- add a fixed-length column with default value
+ALTER FOREIGN TABLE test_alter_table ADD COLUMN e int default 3;
+SELECT * from test_alter_table;
+INSERT INTO test_alter_table (SELECT 1, 2, 4, 8);
+SELECT * from test_alter_table;
+
+
+-- add a variable-length column with default value
+ALTER FOREIGN TABLE test_alter_table ADD COLUMN f text DEFAULT 'TEXT ME';
+SELECT * from test_alter_table;
+INSERT INTO test_alter_table (SELECT 1, 2, 4, 8, 'ABCDEF');
+SELECT * from test_alter_table;
+
+
+-- drop couple of columns
+ALTER FOREIGN TABLE test_alter_table DROP COLUMN c;
+ALTER FOREIGN TABLE test_alter_table DROP COLUMN e;
+ANALYZE test_alter_table;
+SELECT * from test_alter_table;
+SELECT count(*) from test_alter_table;
+SELECT count(t.*) from test_alter_table t;
+
+
+-- unsupported default values
+ALTER FOREIGN TABLE test_alter_table ADD COLUMN g boolean DEFAULT isfinite(current_date);
+ALTER FOREIGN TABLE test_alter_table ADD COLUMN h DATE DEFAULT current_date;
+SELECT * FROM test_alter_table;
+ALTER FOREIGN TABLE test_alter_table ALTER COLUMN g DROP DEFAULT;
+SELECT * FROM test_alter_table;
+ALTER FOREIGN TABLE test_alter_table ALTER COLUMN h DROP DEFAULT;
+ANALYZE test_alter_table;
+SELECT * FROM test_alter_table;
+
+-- unsupported type change
+ALTER FOREIGN TABLE test_alter_table ADD COLUMN i int;
+ALTER FOREIGN TABLE test_alter_table ADD COLUMN j float;
+ALTER FOREIGN TABLE test_alter_table ADD COLUMN k text;
+
+-- this is valid type change
+ALTER FOREIGN TABLE test_alter_table ALTER COLUMN i TYPE float;
+
+-- this is not valid
+ALTER FOREIGN TABLE test_alter_table ALTER COLUMN j TYPE int;
+
+-- text / varchar conversion is valid both ways
+ALTER FOREIGN TABLE test_alter_table ALTER COLUMN k TYPE varchar(20);
+ALTER FOREIGN TABLE test_alter_table ALTER COLUMN k TYPE text;
+
+DROP FOREIGN TABLE test_alter_table;

sql/am_analyze.sql

@@ -0,0 +1,11 @@
+--
+-- Test the ANALYZE command for cstore_fdw tables.
+--
+
+-- ANALYZE uncompressed table
+ANALYZE contestant;
+SELECT count(*) FROM pg_stats WHERE tablename='contestant';
+
+-- ANALYZE compressed table
+ANALYZE contestant_compressed;
+SELECT count(*) FROM pg_stats WHERE tablename='contestant_compressed';

sql/am_block_filtering.sql

@@ -0,0 +1,69 @@
+--
+-- Test block filtering in cstore_fdw using min/max values in stripe skip lists.
+--
+
+
+--
+-- filtered_row_count returns number of rows filtered by the WHERE clause.
+-- If blocks get filtered by cstore_fdw, less rows are passed to WHERE
+-- clause, so this function should return a lower number.
+--
+CREATE OR REPLACE FUNCTION filtered_row_count (query text) RETURNS bigint AS
+$$
+    DECLARE
+        result bigint;
+        rec text;
+    BEGIN
+        result := 0;
+
+        FOR rec IN EXECUTE 'EXPLAIN ANALYZE ' || query LOOP
+            IF rec ~ '^\s+Rows Removed by Filter' then
+                result := regexp_replace(rec, '[^0-9]*', '', 'g');
+            END IF;
+        END LOOP;
+
+        RETURN result;
+    END;
+$$ LANGUAGE PLPGSQL;
+
+
+-- Create and load data
+CREATE FOREIGN TABLE test_block_filtering (a int)
+    SERVER cstore_server
+    OPTIONS(block_row_count '1000', stripe_row_count '2000');
+
+COPY test_block_filtering FROM '/Users/jefdavi/wd/cstore2/data/block_filtering.csv' WITH CSV;
+
+
+-- Verify that filtered_row_count is less than 1000 for the following queries
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering');
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 200');
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a > 200');
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 9900');
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a > 9900');
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 0');
+
+
+-- Verify that filtered_row_count is less than 2000 for the following queries
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 1 AND 10');
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 990 AND 2010');
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN -10 AND 0');
+
+
+-- Load data for second time and verify that filtered_row_count is exactly twice as before
+COPY test_block_filtering FROM '/Users/jefdavi/wd/cstore2/data/block_filtering.csv' WITH CSV;
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 200');
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 0');
+SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 990 AND 2010');
+
+
+-- Verify that we are fine with collations which use a different alphabet order
+CREATE FOREIGN TABLE collation_block_filtering_test(A text collate "da_DK")
+    SERVER cstore_server;
+COPY collation_block_filtering_test FROM STDIN;
+A
+Å
+B
+\.
+
+SELECT * FROM collation_block_filtering_test WHERE A > 'B';

sql/am_copyto.sql

@@ -0,0 +1,17 @@
+--
+-- Test copying data from cstore_fdw tables.
+--
+CREATE FOREIGN TABLE test_contestant(handle TEXT, birthdate DATE, rating INT,
+        percentile FLOAT, country CHAR(3), achievements TEXT[])
+        SERVER cstore_server;
+
+-- load table data from file
+COPY test_contestant FROM '/Users/jefdavi/wd/cstore2/data/contestants.1.csv' WITH CSV;
+
+-- export using COPY table TO ...
+COPY test_contestant TO STDOUT;
+
+-- export using COPY (SELECT * FROM table) TO ...
+COPY (select * from test_contestant) TO STDOUT;
+
+DROP FOREIGN TABLE test_contestant CASCADE;

sql/am_create.sql

@@ -0,0 +1,43 @@
+--
+-- Test the CREATE statements related to cstore_fdw.
+--
+
+
+-- Install cstore_fdw
+CREATE EXTENSION cstore_fdw;
+
+CREATE SERVER cstore_server FOREIGN DATA WRAPPER cstore_fdw;
+
+
+-- Validator tests
+CREATE FOREIGN TABLE test_validator_invalid_option () 
+	SERVER cstore_server 
+	OPTIONS(bad_option_name '1'); -- ERROR
+
+CREATE FOREIGN TABLE test_validator_invalid_stripe_row_count () 
+	SERVER cstore_server
+	OPTIONS(stripe_row_count '0'); -- ERROR
+
+CREATE FOREIGN TABLE test_validator_invalid_block_row_count () 
+	SERVER cstore_server
+	OPTIONS(block_row_count '0'); -- ERROR
+
+CREATE FOREIGN TABLE test_validator_invalid_compression_type () 
+	SERVER cstore_server
+	OPTIONS(compression 'invalid_compression'); -- ERROR
+
+-- Create uncompressed table
+CREATE FOREIGN TABLE contestant (handle TEXT, birthdate DATE, rating INT,
+	percentile FLOAT, country CHAR(3), achievements TEXT[])
+	SERVER cstore_server;
+
+
+-- Create compressed table with automatically determined file path
+CREATE FOREIGN TABLE contestant_compressed (handle TEXT, birthdate DATE, rating INT,
+	percentile FLOAT, country CHAR(3), achievements TEXT[])
+	SERVER cstore_server
+	OPTIONS(compression 'pglz');
+
+-- Test that querying an empty table works
+ANALYZE contestant;
+SELECT count(*) FROM contestant;

sql/am_data_types.sql

@@ -0,0 +1,68 @@
+--
+-- Test loading and reading different data types to/from cstore_fdw foreign tables.
+--
+
+
+-- Settings to make the result deterministic
+SET datestyle = "ISO, YMD";
+SET timezone to 'GMT';
+SET intervalstyle TO 'POSTGRES_VERBOSE';
+
+
+-- Test array types
+CREATE FOREIGN TABLE test_array_types (int_array int[], bigint_array bigint[],
+	text_array text[]) SERVER cstore_server;
+
+COPY test_array_types FROM '/Users/jefdavi/wd/cstore2/data/array_types.csv' WITH CSV;
+
+SELECT * FROM test_array_types;
+
+
+-- Test date/time types
+CREATE FOREIGN TABLE test_datetime_types (timestamp timestamp,
+	timestamp_with_timezone timestamp with time zone, date date, time time,
+	interval interval) SERVER cstore_server;
+
+COPY test_datetime_types FROM '/Users/jefdavi/wd/cstore2/data/datetime_types.csv' WITH CSV;
+
+SELECT * FROM test_datetime_types;
+
+
+-- Test enum and composite types
+CREATE TYPE enum_type AS ENUM ('a', 'b', 'c');
+CREATE TYPE composite_type AS (a int, b text);
+
+CREATE FOREIGN TABLE test_enum_and_composite_types (enum enum_type,
+	composite composite_type) SERVER cstore_server;
+
+COPY test_enum_and_composite_types FROM
+	'/Users/jefdavi/wd/cstore2/data/enum_and_composite_types.csv' WITH CSV;
+
+SELECT * FROM test_enum_and_composite_types;
+
+
+-- Test range types
+CREATE FOREIGN TABLE test_range_types (int4range int4range, int8range int8range,
+	numrange numrange, tsrange tsrange) SERVER cstore_server;
+
+COPY test_range_types FROM '/Users/jefdavi/wd/cstore2/data/range_types.csv' WITH CSV;
+
+SELECT * FROM test_range_types;
+
+
+-- Test other types
+CREATE FOREIGN TABLE test_other_types (bool boolean, bytea bytea, money money,
+	inet inet, bitstring bit varying(5), uuid uuid, json json) SERVER cstore_server;
+
+COPY test_other_types FROM '/Users/jefdavi/wd/cstore2/data/other_types.csv' WITH CSV;
+
+SELECT * FROM test_other_types;
+
+
+-- Test null values
+CREATE FOREIGN TABLE test_null_values (a int, b int[], c composite_type)
+	SERVER cstore_server;
+
+COPY test_null_values FROM '/Users/jefdavi/wd/cstore2/data/null_values.csv' WITH CSV;
+
+SELECT * FROM test_null_values;

sql/am_drop.sql

@@ -0,0 +1,48 @@
+--
+-- Tests the different DROP commands for cstore_fdw tables.
+--
+-- DROP FOREIGN TABL
+-- DROP SCHEMA
+-- DROP EXTENSION
+-- DROP DATABASE
+--
+
+-- Note that travis does not create
+-- cstore_fdw extension in default database (postgres). This has caused
+-- different behavior between travis tests and local tests. Thus
+-- 'postgres' directory is excluded from comparison to have the same result.
+
+-- store postgres database oid
+SELECT oid postgres_oid FROM pg_database WHERE datname = 'postgres' \gset
+
+-- DROP cstore_fdw tables
+DROP FOREIGN TABLE contestant;
+DROP FOREIGN TABLE contestant_compressed;
+
+-- Create a cstore_fdw table under a schema and drop it.
+CREATE SCHEMA test_schema;
+CREATE FOREIGN TABLE test_schema.test_table(data int) SERVER cstore_server;
+DROP SCHEMA test_schema CASCADE;
+
+SELECT current_database() datname \gset
+
+CREATE DATABASE db_to_drop;
+\c db_to_drop
+CREATE EXTENSION cstore_fdw;
+CREATE SERVER cstore_server FOREIGN DATA WRAPPER cstore_fdw;
+SELECT oid::text databaseoid FROM pg_database WHERE datname = current_database() \gset
+
+CREATE FOREIGN TABLE test_table(data int) SERVER cstore_server;
+
+DROP EXTENSION cstore_fdw CASCADE;
+
+-- test database drop
+CREATE EXTENSION cstore_fdw;
+CREATE SERVER cstore_server FOREIGN DATA WRAPPER cstore_fdw;
+SELECT oid::text databaseoid FROM pg_database WHERE datname = current_database() \gset
+
+CREATE FOREIGN TABLE test_table(data int) SERVER cstore_server;
+
+\c :datname
+
+DROP DATABASE db_to_drop;

sql/am_functions.sql

@@ -0,0 +1,20 @@
+--
+-- Test utility functions for cstore_fdw tables.
+--
+
+CREATE FOREIGN TABLE empty_table (a int) SERVER cstore_server;
+CREATE FOREIGN TABLE table_with_data (a int) SERVER cstore_server;
+CREATE TABLE non_cstore_table (a int);
+
+COPY table_with_data FROM STDIN;
+1
+2
+3
+\.
+
+SELECT cstore_table_size('empty_table') < cstore_table_size('table_with_data');
+SELECT cstore_table_size('non_cstore_table');
+
+DROP FOREIGN TABLE empty_table;
+DROP FOREIGN TABLE table_with_data;
+DROP TABLE non_cstore_table;

sql/am_insert.sql

@@ -0,0 +1,56 @@
+--
+-- Testing insert on cstore_fdw tables.
+--
+
+CREATE FOREIGN TABLE test_insert_command (a int) SERVER cstore_server;
+
+-- test single row inserts fail
+select count(*) from test_insert_command;
+insert into test_insert_command values(1);
+select count(*) from test_insert_command;
+
+insert into test_insert_command default values;
+select count(*) from test_insert_command;
+
+-- test inserting from another table succeed
+CREATE TABLE test_insert_command_data (a int);
+
+select count(*) from test_insert_command_data;
+insert into test_insert_command_data values(1);
+select count(*) from test_insert_command_data;
+
+insert into test_insert_command select * from test_insert_command_data;
+select count(*) from test_insert_command;
+
+drop table test_insert_command_data;
+drop foreign table test_insert_command;
+
+-- test long attribute value insertion
+-- create sufficiently long text so that data is stored in toast
+CREATE TABLE test_long_text AS
+SELECT a as int_val, string_agg(random()::text, '') as text_val
+FROM generate_series(1, 10) a, generate_series(1, 1000) b
+GROUP BY a ORDER BY a;
+
+-- store hash values of text for later comparison
+CREATE TABLE test_long_text_hash AS
+SELECT int_val, md5(text_val) AS hash
+FROM test_long_text;
+
+CREATE FOREIGN TABLE test_cstore_long_text(int_val int, text_val text)
+SERVER cstore_server;
+
+-- store long text in cstore table
+INSERT INTO test_cstore_long_text SELECT * FROM test_long_text;
+
+-- drop source table to remove original text from toast
+DROP TABLE test_long_text;
+
+-- check if text data is still available in cstore table
+-- by comparing previously stored hash.
+SELECT a.int_val
+FROM  test_long_text_hash a, test_cstore_long_text c
+WHERE a.int_val = c.int_val AND a.hash = md5(c.text_val);
+
+DROP TABLE test_long_text_hash;
+DROP FOREIGN TABLE test_cstore_long_text;

sql/am_load.sql

@@ -0,0 +1,44 @@
+--
+-- Test loading data into cstore_fdw tables.
+--
+
+-- COPY with incorrect delimiter
+COPY contestant FROM '/Users/jefdavi/wd/cstore2/data/contestants.1.csv'
+	WITH DELIMITER '|'; -- ERROR
+
+-- COPY with invalid program
+COPY contestant FROM PROGRAM 'invalid_program' WITH CSV; -- ERROR
+
+-- COPY into uncompressed table from file
+COPY contestant FROM '/Users/jefdavi/wd/cstore2/data/contestants.1.csv' WITH CSV;
+
+-- COPY into uncompressed table from program
+COPY contestant FROM PROGRAM 'cat /Users/jefdavi/wd/cstore2/data/contestants.2.csv' WITH CSV;
+
+-- COPY into compressed table
+COPY contestant_compressed FROM '/Users/jefdavi/wd/cstore2/data/contestants.1.csv' WITH CSV;
+
+-- COPY into uncompressed table from program
+COPY contestant_compressed FROM PROGRAM 'cat /Users/jefdavi/wd/cstore2/data/contestants.2.csv'
+	WITH CSV;
+
+-- Test column list
+CREATE FOREIGN TABLE famous_constants (id int, name text, value real)
+    SERVER cstore_server;
+COPY famous_constants (value, name, id) FROM STDIN WITH CSV;
+3.141,pi,1
+2.718,e,2
+0.577,gamma,3
+5.291e-11,bohr radius,4
+\.
+
+COPY famous_constants (name, value) FROM STDIN WITH CSV;
+avagadro,6.022e23
+electron mass,9.109e-31
+proton mass,1.672e-27
+speed of light,2.997e8
+\.
+
+SELECT * FROM famous_constants ORDER BY id, name;
+
+DROP FOREIGN TABLE famous_constants;

sql/am_query.sql

@@ -0,0 +1,34 @@
+--
+-- Test querying cstore_fdw tables.
+--
+
+-- Settings to make the result deterministic
+SET datestyle = "ISO, YMD";
+
+-- Query uncompressed data
+SELECT count(*) FROM contestant;
+SELECT avg(rating), stddev_samp(rating) FROM contestant;
+SELECT country, avg(rating) FROM contestant WHERE rating > 2200
+	GROUP BY country ORDER BY country;
+SELECT * FROM contestant ORDER BY handle;
+
+-- Query compressed data
+SELECT count(*) FROM contestant_compressed;
+SELECT avg(rating), stddev_samp(rating) FROM contestant_compressed;
+SELECT country, avg(rating) FROM contestant_compressed WHERE rating > 2200
+	GROUP BY country ORDER BY country;
+SELECT * FROM contestant_compressed ORDER BY handle;
+
+-- Verify that we handle whole-row references correctly
+SELECT to_json(v) FROM contestant v ORDER BY rating LIMIT 1;
+
+-- Test variables used in expressions
+CREATE FOREIGN TABLE union_first (a int, b int) SERVER cstore_server;
+CREATE FOREIGN TABLE union_second (a int, b int) SERVER cstore_server;
+
+INSERT INTO union_first SELECT a, a FROM generate_series(1, 5) a;
+INSERT INTO union_second SELECT a, a FROM generate_series(11, 15) a;
+
+(SELECT a*1, b FROM union_first) union all (SELECT a*1, b FROM union_second);
+
+DROP FOREIGN TABLE union_first, union_second;

sql/am_truncate.sql

@@ -0,0 +1,116 @@
+--
+-- Test the TRUNCATE TABLE command for cstore_fdw tables.
+--
+
+-- print whether we're using version > 10 to make version-specific tests clear
+SHOW server_version \gset
+SELECT substring(:'server_version', '\d+')::int > 10 AS version_above_ten;
+
+-- CREATE a cstore_fdw table, fill with some data --
+CREATE FOREIGN TABLE cstore_truncate_test (a int, b int) SERVER cstore_server;
+CREATE FOREIGN TABLE cstore_truncate_test_second (a int, b int) SERVER cstore_server;
+CREATE FOREIGN TABLE cstore_truncate_test_compressed (a int, b int) SERVER cstore_server OPTIONS (compression 'pglz');
+CREATE TABLE cstore_truncate_test_regular (a int, b int);
+
+INSERT INTO cstore_truncate_test select a, a from generate_series(1, 10) a;
+
+INSERT INTO cstore_truncate_test_compressed select a, a from generate_series(1, 10) a;
+INSERT INTO cstore_truncate_test_compressed select a, a from generate_series(1, 10) a;
+
+-- query rows
+SELECT * FROM cstore_truncate_test;
+
+TRUNCATE TABLE cstore_truncate_test;
+
+SELECT * FROM cstore_truncate_test;
+
+SELECT COUNT(*) from cstore_truncate_test;
+
+SELECT count(*) FROM cstore_truncate_test_compressed;
+TRUNCATE TABLE cstore_truncate_test_compressed;
+SELECT count(*) FROM cstore_truncate_test_compressed;
+
+SELECT cstore_table_size('cstore_truncate_test_compressed');
+
+INSERT INTO cstore_truncate_test select a, a from generate_series(1, 10) a;
+INSERT INTO cstore_truncate_test_regular select a, a from generate_series(10, 20) a;
+INSERT INTO cstore_truncate_test_second select a, a from generate_series(20, 30) a;
+
+SELECT * from cstore_truncate_test;
+
+SELECT * from cstore_truncate_test_second;
+
+SELECT * from cstore_truncate_test_regular;
+
+-- make sure multi truncate works
+-- notice that the same table might be repeated
+TRUNCATE TABLE cstore_truncate_test,
+			   cstore_truncate_test_regular,
+			   cstore_truncate_test_second,
+   			   cstore_truncate_test;
+
+SELECT * from cstore_truncate_test;
+SELECT * from cstore_truncate_test_second;
+SELECT * from cstore_truncate_test_regular;
+
+-- test if truncate on empty table works
+TRUNCATE TABLE cstore_truncate_test;
+SELECT * from cstore_truncate_test;
+
+-- test if a cached truncate from a pl/pgsql function works
+CREATE FUNCTION cstore_truncate_test_regular_func() RETURNS void AS $$
+BEGIN
+	INSERT INTO cstore_truncate_test_regular select a, a from generate_series(1, 10) a;
+	TRUNCATE TABLE cstore_truncate_test_regular;
+END;$$
+LANGUAGE plpgsql;
+
+SELECT cstore_truncate_test_regular_func();
+-- the cached plans are used stating from the second call
+SELECT cstore_truncate_test_regular_func();
+DROP FUNCTION cstore_truncate_test_regular_func();
+
+DROP FOREIGN TABLE cstore_truncate_test, cstore_truncate_test_second;
+DROP TABLE cstore_truncate_test_regular;
+DROP FOREIGN TABLE cstore_truncate_test_compressed;
+
+-- test truncate with schema
+CREATE SCHEMA truncate_schema;
+CREATE FOREIGN TABLE truncate_schema.truncate_tbl (id int) SERVER cstore_server OPTIONS(compression 'pglz');
+INSERT INTO truncate_schema.truncate_tbl SELECT generate_series(1, 100);
+SELECT COUNT(*) FROM truncate_schema.truncate_tbl;
+
+TRUNCATE TABLE truncate_schema.truncate_tbl;
+SELECT COUNT(*) FROM truncate_schema.truncate_tbl;
+
+INSERT INTO truncate_schema.truncate_tbl SELECT generate_series(1, 100);
+
+-- create a user that can not truncate
+CREATE USER truncate_user;
+GRANT USAGE ON SCHEMA truncate_schema TO truncate_user;
+GRANT SELECT ON TABLE truncate_schema.truncate_tbl TO truncate_user;
+REVOKE TRUNCATE ON TABLE truncate_schema.truncate_tbl FROM truncate_user;
+
+SELECT current_user \gset
+
+\c - truncate_user
+-- verify truncate command fails and check number of rows
+SELECT count(*) FROM truncate_schema.truncate_tbl;
+TRUNCATE TABLE truncate_schema.truncate_tbl;
+SELECT count(*) FROM truncate_schema.truncate_tbl;
+
+-- switch to super user, grant truncate to truncate_user 
+\c - :current_user
+GRANT TRUNCATE ON TABLE truncate_schema.truncate_tbl TO truncate_user;
+
+-- verify truncate_user can truncate now
+\c - truncate_user
+SELECT count(*) FROM truncate_schema.truncate_tbl;
+TRUNCATE TABLE truncate_schema.truncate_tbl;
+SELECT count(*) FROM truncate_schema.truncate_tbl;
+
+\c - :current_user
+
+-- cleanup
+DROP SCHEMA truncate_schema CASCADE;
+DROP USER truncate_user;