diff --git a/Makefile b/Makefile index 97f4d9e64..2fc550ca9 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,9 @@ DATA = cstore_fdw--1.7.sql cstore_fdw--1.6--1.7.sql cstore_fdw--1.5--1.6.sql cs cstore_fdw--1.3--1.4.sql cstore_fdw--1.2--1.3.sql cstore_fdw--1.1--1.2.sql \ cstore_fdw--1.0--1.1.sql cstore_fdw--1.7--1.8.sql -REGRESS = fdw_create fdw_load fdw_query fdw_analyze fdw_data_types fdw_functions \ +REGRESS = am_create am_load am_query am_analyze am_data_types am_functions \ + am_block_filtering am_drop am_insert am_copyto am_alter am_truncate \ + fdw_create fdw_load fdw_query fdw_analyze fdw_data_types fdw_functions \ fdw_block_filtering fdw_drop fdw_insert fdw_copyto fdw_alter fdw_truncate EXTRA_CLEAN = cstore.pb-c.h cstore.pb-c.c data/*.cstore data/*.cstore.footer \ sql/block_filtering.sql sql/create.sql sql/data_types.sql sql/load.sql \ diff --git a/expected/am_alter.out b/expected/am_alter.out new file mode 100644 index 000000000..659e2723e --- /dev/null +++ b/expected/am_alter.out @@ -0,0 +1,178 @@ +-- +-- Testing ALTER TABLE on cstore_fdw tables. +-- +CREATE FOREIGN TABLE test_alter_table (a int, b int, c int) SERVER cstore_server; +WITH sample_data AS (VALUES + (1, 2, 3), + (4, 5, 6), + (7, 8, 9) +) +INSERT INTO test_alter_table SELECT * FROM sample_data; +-- drop a column +ALTER FOREIGN TABLE test_alter_table DROP COLUMN a; +-- test analyze +ANALYZE test_alter_table; +-- verify select queries run as expected +SELECT * FROM test_alter_table; + b | c +---+--- + 2 | 3 + 5 | 6 + 8 | 9 +(3 rows) + +SELECT a FROM test_alter_table; +ERROR: column "a" does not exist +LINE 1: SELECT a FROM test_alter_table; + ^ +SELECT b FROM test_alter_table; + b +--- + 2 + 5 + 8 +(3 rows) + +-- verify insert runs as expected +INSERT INTO test_alter_table (SELECT 3, 5, 8); +ERROR: INSERT has more expressions than target columns +LINE 1: INSERT INTO test_alter_table (SELECT 3, 5, 8); + ^ +INSERT INTO test_alter_table (SELECT 5, 8); +-- add a column with no defaults +ALTER FOREIGN TABLE test_alter_table ADD COLUMN d int; +SELECT * FROM test_alter_table; + b | c | d +---+---+--- + 2 | 3 | + 5 | 6 | + 8 | 9 | + 5 | 8 | +(4 rows) + +INSERT INTO test_alter_table (SELECT 3, 5, 8); +SELECT * FROM test_alter_table; + b | c | d +---+---+--- + 2 | 3 | + 5 | 6 | + 8 | 9 | + 5 | 8 | + 3 | 5 | 8 +(5 rows) + +-- add a fixed-length column with default value +ALTER FOREIGN TABLE test_alter_table ADD COLUMN e int default 3; +SELECT * from test_alter_table; + b | c | d | e +---+---+---+--- + 2 | 3 | | 3 + 5 | 6 | | 3 + 8 | 9 | | 3 + 5 | 8 | | 3 + 3 | 5 | 8 | 3 +(5 rows) + +INSERT INTO test_alter_table (SELECT 1, 2, 4, 8); +SELECT * from test_alter_table; + b | c | d | e +---+---+---+--- + 2 | 3 | | 3 + 5 | 6 | | 3 + 8 | 9 | | 3 + 5 | 8 | | 3 + 3 | 5 | 8 | 3 + 1 | 2 | 4 | 8 +(6 rows) + +-- add a variable-length column with default value +ALTER FOREIGN TABLE test_alter_table ADD COLUMN f text DEFAULT 'TEXT ME'; +SELECT * from test_alter_table; + b | c | d | e | f +---+---+---+---+--------- + 2 | 3 | | 3 | TEXT ME + 5 | 6 | | 3 | TEXT ME + 8 | 9 | | 3 | TEXT ME + 5 | 8 | | 3 | TEXT ME + 3 | 5 | 8 | 3 | TEXT ME + 1 | 2 | 4 | 8 | TEXT ME +(6 rows) + +INSERT INTO test_alter_table (SELECT 1, 2, 4, 8, 'ABCDEF'); +SELECT * from test_alter_table; + b | c | d | e | f +---+---+---+---+--------- + 2 | 3 | | 3 | TEXT ME + 5 | 6 | | 3 | TEXT ME + 8 | 9 | | 3 | TEXT ME + 5 | 8 | | 3 | TEXT ME + 3 | 5 | 8 | 3 | TEXT ME + 1 | 2 | 4 | 8 | TEXT ME + 1 | 2 | 4 | 8 | ABCDEF +(7 rows) + +-- drop couple of columns +ALTER FOREIGN TABLE test_alter_table DROP COLUMN c; +ALTER FOREIGN TABLE test_alter_table DROP COLUMN e; +ANALYZE test_alter_table; +SELECT * from test_alter_table; + b | d | f +---+---+--------- + 2 | | TEXT ME + 5 | | TEXT ME + 8 | | TEXT ME + 5 | | TEXT ME + 3 | 8 | TEXT ME + 1 | 4 | TEXT ME + 1 | 4 | ABCDEF +(7 rows) + +SELECT count(*) from test_alter_table; + count +------- + 7 +(1 row) + +SELECT count(t.*) from test_alter_table t; + count +------- + 7 +(1 row) + +-- unsupported default values +ALTER FOREIGN TABLE test_alter_table ADD COLUMN g boolean DEFAULT isfinite(current_date); +ALTER FOREIGN TABLE test_alter_table ADD COLUMN h DATE DEFAULT current_date; +SELECT * FROM test_alter_table; +ERROR: unsupported default value for column "g" +HINT: Expression is either mutable or does not evaluate to constant value +ALTER FOREIGN TABLE test_alter_table ALTER COLUMN g DROP DEFAULT; +SELECT * FROM test_alter_table; +ERROR: unsupported default value for column "h" +HINT: Expression is either mutable or does not evaluate to constant value +ALTER FOREIGN TABLE test_alter_table ALTER COLUMN h DROP DEFAULT; +ANALYZE test_alter_table; +SELECT * FROM test_alter_table; + b | d | f | g | h +---+---+---------+---+--- + 2 | | TEXT ME | | + 5 | | TEXT ME | | + 8 | | TEXT ME | | + 5 | | TEXT ME | | + 3 | 8 | TEXT ME | | + 1 | 4 | TEXT ME | | + 1 | 4 | ABCDEF | | +(7 rows) + +-- unsupported type change +ALTER FOREIGN TABLE test_alter_table ADD COLUMN i int; +ALTER FOREIGN TABLE test_alter_table ADD COLUMN j float; +ALTER FOREIGN TABLE test_alter_table ADD COLUMN k text; +-- this is valid type change +ALTER FOREIGN TABLE test_alter_table ALTER COLUMN i TYPE float; +-- this is not valid +ALTER FOREIGN TABLE test_alter_table ALTER COLUMN j TYPE int; +ERROR: Column j cannot be cast automatically to type pg_catalog.int4 +-- text / varchar conversion is valid both ways +ALTER FOREIGN TABLE test_alter_table ALTER COLUMN k TYPE varchar(20); +ALTER FOREIGN TABLE test_alter_table ALTER COLUMN k TYPE text; +DROP FOREIGN TABLE test_alter_table; diff --git a/expected/am_analyze.out b/expected/am_analyze.out new file mode 100644 index 000000000..f8c4d974a --- /dev/null +++ b/expected/am_analyze.out @@ -0,0 +1,19 @@ +-- +-- Test the ANALYZE command for cstore_fdw tables. +-- +-- ANALYZE uncompressed table +ANALYZE contestant; +SELECT count(*) FROM pg_stats WHERE tablename='contestant'; + count +------- + 6 +(1 row) + +-- ANALYZE compressed table +ANALYZE contestant_compressed; +SELECT count(*) FROM pg_stats WHERE tablename='contestant_compressed'; + count +------- + 6 +(1 row) + diff --git a/expected/am_block_filtering.out b/expected/am_block_filtering.out new file mode 100644 index 000000000..bccfafd15 --- /dev/null +++ b/expected/am_block_filtering.out @@ -0,0 +1,116 @@ +-- +-- Test block filtering in cstore_fdw using min/max values in stripe skip lists. +-- +-- +-- filtered_row_count returns number of rows filtered by the WHERE clause. +-- If blocks get filtered by cstore_fdw, less rows are passed to WHERE +-- clause, so this function should return a lower number. +-- +CREATE OR REPLACE FUNCTION filtered_row_count (query text) RETURNS bigint AS +$$ + DECLARE + result bigint; + rec text; + BEGIN + result := 0; + + FOR rec IN EXECUTE 'EXPLAIN ANALYZE ' || query LOOP + IF rec ~ '^\s+Rows Removed by Filter' then + result := regexp_replace(rec, '[^0-9]*', '', 'g'); + END IF; + END LOOP; + + RETURN result; + END; +$$ LANGUAGE PLPGSQL; +-- Create and load data +CREATE FOREIGN TABLE test_block_filtering (a int) + SERVER cstore_server + OPTIONS(block_row_count '1000', stripe_row_count '2000'); +COPY test_block_filtering FROM '/Users/jefdavi/wd/cstore2/data/block_filtering.csv' WITH CSV; +-- Verify that filtered_row_count is less than 1000 for the following queries +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering'); + filtered_row_count +-------------------- + 0 +(1 row) + +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 200'); + filtered_row_count +-------------------- + 801 +(1 row) + +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a > 200'); + filtered_row_count +-------------------- + 200 +(1 row) + +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 9900'); + filtered_row_count +-------------------- + 101 +(1 row) + +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a > 9900'); + filtered_row_count +-------------------- + 900 +(1 row) + +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 0'); + filtered_row_count +-------------------- + 0 +(1 row) + +-- Verify that filtered_row_count is less than 2000 for the following queries +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 1 AND 10'); + filtered_row_count +-------------------- + 990 +(1 row) + +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 990 AND 2010'); + filtered_row_count +-------------------- + 1979 +(1 row) + +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN -10 AND 0'); + filtered_row_count +-------------------- + 0 +(1 row) + +-- Load data for second time and verify that filtered_row_count is exactly twice as before +COPY test_block_filtering FROM '/Users/jefdavi/wd/cstore2/data/block_filtering.csv' WITH CSV; +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 200'); + filtered_row_count +-------------------- + 1602 +(1 row) + +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 0'); + filtered_row_count +-------------------- + 0 +(1 row) + +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 990 AND 2010'); + filtered_row_count +-------------------- + 3958 +(1 row) + +-- Verify that we are fine with collations which use a different alphabet order +CREATE FOREIGN TABLE collation_block_filtering_test(A text collate "da_DK") + SERVER cstore_server; +COPY collation_block_filtering_test FROM STDIN; +SELECT * FROM collation_block_filtering_test WHERE A > 'B'; + a +--- + Å +(1 row) + diff --git a/expected/am_copyto.out b/expected/am_copyto.out new file mode 100644 index 000000000..2b68d0ad5 --- /dev/null +++ b/expected/am_copyto.out @@ -0,0 +1,23 @@ +-- +-- Test copying data from cstore_fdw tables. +-- +CREATE FOREIGN TABLE test_contestant(handle TEXT, birthdate DATE, rating INT, + percentile FLOAT, country CHAR(3), achievements TEXT[]) + SERVER cstore_server; +-- load table data from file +COPY test_contestant FROM '/Users/jefdavi/wd/cstore2/data/contestants.1.csv' WITH CSV; +-- export using COPY table TO ... +COPY test_contestant TO STDOUT; +a 01-10-1990 2090 97.1 XA {a} +b 11-01-1990 2203 98.1 XA {a,b} +c 11-01-1988 2907 99.4 XB {w,y} +d 05-05-1985 2314 98.3 XB {} +e 05-05-1995 2236 98.2 XC {a} +-- export using COPY (SELECT * FROM table) TO ... +COPY (select * from test_contestant) TO STDOUT; +a 01-10-1990 2090 97.1 XA {a} +b 11-01-1990 2203 98.1 XA {a,b} +c 11-01-1988 2907 99.4 XB {w,y} +d 05-05-1985 2314 98.3 XB {} +e 05-05-1995 2236 98.2 XC {a} +DROP FOREIGN TABLE test_contestant CASCADE; diff --git a/expected/am_create.out b/expected/am_create.out new file mode 100644 index 000000000..961c0494d --- /dev/null +++ b/expected/am_create.out @@ -0,0 +1,44 @@ +-- +-- Test the CREATE statements related to cstore_fdw. +-- +-- Install cstore_fdw +CREATE EXTENSION cstore_fdw; +CREATE SERVER cstore_server FOREIGN DATA WRAPPER cstore_fdw; +-- Validator tests +CREATE FOREIGN TABLE test_validator_invalid_option () + SERVER cstore_server + OPTIONS(bad_option_name '1'); -- ERROR +ERROR: invalid option "bad_option_name" +HINT: Valid options in this context are: compression, stripe_row_count, block_row_count +CREATE FOREIGN TABLE test_validator_invalid_stripe_row_count () + SERVER cstore_server + OPTIONS(stripe_row_count '0'); -- ERROR +ERROR: invalid stripe row count +HINT: Stripe row count must be an integer between 1000 and 10000000 +CREATE FOREIGN TABLE test_validator_invalid_block_row_count () + SERVER cstore_server + OPTIONS(block_row_count '0'); -- ERROR +ERROR: invalid block row count +HINT: Block row count must be an integer between 1000 and 100000 +CREATE FOREIGN TABLE test_validator_invalid_compression_type () + SERVER cstore_server + OPTIONS(compression 'invalid_compression'); -- ERROR +ERROR: invalid compression type +HINT: Valid options are: none, pglz +-- Create uncompressed table +CREATE FOREIGN TABLE contestant (handle TEXT, birthdate DATE, rating INT, + percentile FLOAT, country CHAR(3), achievements TEXT[]) + SERVER cstore_server; +-- Create compressed table with automatically determined file path +CREATE FOREIGN TABLE contestant_compressed (handle TEXT, birthdate DATE, rating INT, + percentile FLOAT, country CHAR(3), achievements TEXT[]) + SERVER cstore_server + OPTIONS(compression 'pglz'); +-- Test that querying an empty table works +ANALYZE contestant; +SELECT count(*) FROM contestant; + count +------- + 0 +(1 row) + diff --git a/expected/am_data_types.out b/expected/am_data_types.out new file mode 100644 index 000000000..a27a25eb9 --- /dev/null +++ b/expected/am_data_types.out @@ -0,0 +1,78 @@ +-- +-- Test loading and reading different data types to/from cstore_fdw foreign tables. +-- +-- Settings to make the result deterministic +SET datestyle = "ISO, YMD"; +SET timezone to 'GMT'; +SET intervalstyle TO 'POSTGRES_VERBOSE'; +-- Test array types +CREATE FOREIGN TABLE test_array_types (int_array int[], bigint_array bigint[], + text_array text[]) SERVER cstore_server; +COPY test_array_types FROM '/Users/jefdavi/wd/cstore2/data/array_types.csv' WITH CSV; +SELECT * FROM test_array_types; + int_array | bigint_array | text_array +--------------------------+--------------------------------------------+------------ + {1,2,3} | {1,2,3} | {a,b,c} + {} | {} | {} + {-2147483648,2147483647} | {-9223372036854775808,9223372036854775807} | {""} +(3 rows) + +-- Test date/time types +CREATE FOREIGN TABLE test_datetime_types (timestamp timestamp, + timestamp_with_timezone timestamp with time zone, date date, time time, + interval interval) SERVER cstore_server; +COPY test_datetime_types FROM '/Users/jefdavi/wd/cstore2/data/datetime_types.csv' WITH CSV; +SELECT * FROM test_datetime_types; + timestamp | timestamp_with_timezone | date | time | interval +---------------------+-------------------------+------------+----------+----------- + 2000-01-02 04:05:06 | 1999-01-08 12:05:06+00 | 2000-01-02 | 04:05:06 | @ 4 hours + 1970-01-01 00:00:00 | infinity | -infinity | 00:00:00 | @ 0 +(2 rows) + +-- Test enum and composite types +CREATE TYPE enum_type AS ENUM ('a', 'b', 'c'); +CREATE TYPE composite_type AS (a int, b text); +CREATE FOREIGN TABLE test_enum_and_composite_types (enum enum_type, + composite composite_type) SERVER cstore_server; +COPY test_enum_and_composite_types FROM + '/Users/jefdavi/wd/cstore2/data/enum_and_composite_types.csv' WITH CSV; +SELECT * FROM test_enum_and_composite_types; + enum | composite +------+----------- + a | (2,b) + b | (3,c) +(2 rows) + +-- Test range types +CREATE FOREIGN TABLE test_range_types (int4range int4range, int8range int8range, + numrange numrange, tsrange tsrange) SERVER cstore_server; +COPY test_range_types FROM '/Users/jefdavi/wd/cstore2/data/range_types.csv' WITH CSV; +SELECT * FROM test_range_types; + int4range | int8range | numrange | tsrange +-----------+-----------+----------+----------------------------------------------- + [1,3) | [1,3) | [1,3) | ["2000-01-02 00:30:00","2010-02-03 12:30:00") + empty | [1,) | (,) | empty +(2 rows) + +-- Test other types +CREATE FOREIGN TABLE test_other_types (bool boolean, bytea bytea, money money, + inet inet, bitstring bit varying(5), uuid uuid, json json) SERVER cstore_server; +COPY test_other_types FROM '/Users/jefdavi/wd/cstore2/data/other_types.csv' WITH CSV; +SELECT * FROM test_other_types; + bool | bytea | money | inet | bitstring | uuid | json +------+------------+-------+-------------+-----------+--------------------------------------+------------------ + f | \xdeadbeef | $1.00 | 192.168.1.2 | 10101 | a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11 | {"key": "value"} + t | \xcdb0 | $1.50 | 127.0.0.1 | | a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11 | [] +(2 rows) + +-- Test null values +CREATE FOREIGN TABLE test_null_values (a int, b int[], c composite_type) + SERVER cstore_server; +COPY test_null_values FROM '/Users/jefdavi/wd/cstore2/data/null_values.csv' WITH CSV; +SELECT * FROM test_null_values; + a | b | c +---+--------+----- + | {NULL} | (,) + | | +(2 rows) + diff --git a/expected/am_drop.out b/expected/am_drop.out new file mode 100644 index 000000000..926f69337 --- /dev/null +++ b/expected/am_drop.out @@ -0,0 +1,40 @@ +-- +-- Tests the different DROP commands for cstore_fdw tables. +-- +-- DROP FOREIGN TABL +-- DROP SCHEMA +-- DROP EXTENSION +-- DROP DATABASE +-- +-- Note that travis does not create +-- cstore_fdw extension in default database (postgres). This has caused +-- different behavior between travis tests and local tests. Thus +-- 'postgres' directory is excluded from comparison to have the same result. +-- store postgres database oid +SELECT oid postgres_oid FROM pg_database WHERE datname = 'postgres' \gset +-- DROP cstore_fdw tables +DROP FOREIGN TABLE contestant; +DROP FOREIGN TABLE contestant_compressed; +-- Create a cstore_fdw table under a schema and drop it. +CREATE SCHEMA test_schema; +CREATE FOREIGN TABLE test_schema.test_table(data int) SERVER cstore_server; +DROP SCHEMA test_schema CASCADE; +NOTICE: drop cascades to foreign table test_schema.test_table +SELECT current_database() datname \gset +CREATE DATABASE db_to_drop; +\c db_to_drop +CREATE EXTENSION cstore_fdw; +CREATE SERVER cstore_server FOREIGN DATA WRAPPER cstore_fdw; +SELECT oid::text databaseoid FROM pg_database WHERE datname = current_database() \gset +CREATE FOREIGN TABLE test_table(data int) SERVER cstore_server; +DROP EXTENSION cstore_fdw CASCADE; +NOTICE: drop cascades to 2 other objects +DETAIL: drop cascades to server cstore_server +drop cascades to foreign table test_table +-- test database drop +CREATE EXTENSION cstore_fdw; +CREATE SERVER cstore_server FOREIGN DATA WRAPPER cstore_fdw; +SELECT oid::text databaseoid FROM pg_database WHERE datname = current_database() \gset +CREATE FOREIGN TABLE test_table(data int) SERVER cstore_server; +\c :datname +DROP DATABASE db_to_drop; diff --git a/expected/am_functions.out b/expected/am_functions.out new file mode 100644 index 000000000..117fc15f9 --- /dev/null +++ b/expected/am_functions.out @@ -0,0 +1,18 @@ +-- +-- Test utility functions for cstore_fdw tables. +-- +CREATE FOREIGN TABLE empty_table (a int) SERVER cstore_server; +CREATE FOREIGN TABLE table_with_data (a int) SERVER cstore_server; +CREATE TABLE non_cstore_table (a int); +COPY table_with_data FROM STDIN; +SELECT cstore_table_size('empty_table') < cstore_table_size('table_with_data'); + ?column? +---------- + t +(1 row) + +SELECT cstore_table_size('non_cstore_table'); +ERROR: relation is not a cstore table +DROP FOREIGN TABLE empty_table; +DROP FOREIGN TABLE table_with_data; +DROP TABLE non_cstore_table; diff --git a/expected/am_insert.out b/expected/am_insert.out new file mode 100644 index 000000000..49d9ed132 --- /dev/null +++ b/expected/am_insert.out @@ -0,0 +1,88 @@ +-- +-- Testing insert on cstore_fdw tables. +-- +CREATE FOREIGN TABLE test_insert_command (a int) SERVER cstore_server; +-- test single row inserts fail +select count(*) from test_insert_command; + count +------- + 0 +(1 row) + +insert into test_insert_command values(1); +ERROR: operation is not supported +select count(*) from test_insert_command; + count +------- + 0 +(1 row) + +insert into test_insert_command default values; +ERROR: operation is not supported +select count(*) from test_insert_command; + count +------- + 0 +(1 row) + +-- test inserting from another table succeed +CREATE TABLE test_insert_command_data (a int); +select count(*) from test_insert_command_data; + count +------- + 0 +(1 row) + +insert into test_insert_command_data values(1); +select count(*) from test_insert_command_data; + count +------- + 1 +(1 row) + +insert into test_insert_command select * from test_insert_command_data; +select count(*) from test_insert_command; + count +------- + 1 +(1 row) + +drop table test_insert_command_data; +drop foreign table test_insert_command; +-- test long attribute value insertion +-- create sufficiently long text so that data is stored in toast +CREATE TABLE test_long_text AS +SELECT a as int_val, string_agg(random()::text, '') as text_val +FROM generate_series(1, 10) a, generate_series(1, 1000) b +GROUP BY a ORDER BY a; +-- store hash values of text for later comparison +CREATE TABLE test_long_text_hash AS +SELECT int_val, md5(text_val) AS hash +FROM test_long_text; +CREATE FOREIGN TABLE test_cstore_long_text(int_val int, text_val text) +SERVER cstore_server; +-- store long text in cstore table +INSERT INTO test_cstore_long_text SELECT * FROM test_long_text; +-- drop source table to remove original text from toast +DROP TABLE test_long_text; +-- check if text data is still available in cstore table +-- by comparing previously stored hash. +SELECT a.int_val +FROM test_long_text_hash a, test_cstore_long_text c +WHERE a.int_val = c.int_val AND a.hash = md5(c.text_val); + int_val +--------- + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 +(10 rows) + +DROP TABLE test_long_text_hash; +DROP FOREIGN TABLE test_cstore_long_text; diff --git a/expected/am_load.out b/expected/am_load.out new file mode 100644 index 000000000..162ece55b --- /dev/null +++ b/expected/am_load.out @@ -0,0 +1,39 @@ +-- +-- Test loading data into cstore_fdw tables. +-- +-- COPY with incorrect delimiter +COPY contestant FROM '/Users/jefdavi/wd/cstore2/data/contestants.1.csv' + WITH DELIMITER '|'; -- ERROR +ERROR: missing data for column "birthdate" +-- COPY with invalid program +COPY contestant FROM PROGRAM 'invalid_program' WITH CSV; -- ERROR +ERROR: program "invalid_program" failed +DETAIL: command not found +-- COPY into uncompressed table from file +COPY contestant FROM '/Users/jefdavi/wd/cstore2/data/contestants.1.csv' WITH CSV; +-- COPY into uncompressed table from program +COPY contestant FROM PROGRAM 'cat /Users/jefdavi/wd/cstore2/data/contestants.2.csv' WITH CSV; +-- COPY into compressed table +COPY contestant_compressed FROM '/Users/jefdavi/wd/cstore2/data/contestants.1.csv' WITH CSV; +-- COPY into uncompressed table from program +COPY contestant_compressed FROM PROGRAM 'cat /Users/jefdavi/wd/cstore2/data/contestants.2.csv' + WITH CSV; +-- Test column list +CREATE FOREIGN TABLE famous_constants (id int, name text, value real) + SERVER cstore_server; +COPY famous_constants (value, name, id) FROM STDIN WITH CSV; +COPY famous_constants (name, value) FROM STDIN WITH CSV; +SELECT * FROM famous_constants ORDER BY id, name; + id | name | value +----+----------------+----------- + 1 | pi | 3.141 + 2 | e | 2.718 + 3 | gamma | 0.577 + 4 | bohr radius | 5.291e-11 + | avagadro | 6.022e+23 + | electron mass | 9.109e-31 + | proton mass | 1.672e-27 + | speed of light | 2.997e+08 +(8 rows) + +DROP FOREIGN TABLE famous_constants; diff --git a/expected/am_query.out b/expected/am_query.out new file mode 100644 index 000000000..7ac3508a4 --- /dev/null +++ b/expected/am_query.out @@ -0,0 +1,105 @@ +-- +-- Test querying cstore_fdw tables. +-- +-- Settings to make the result deterministic +SET datestyle = "ISO, YMD"; +-- Query uncompressed data +SELECT count(*) FROM contestant; + count +------- + 8 +(1 row) + +SELECT avg(rating), stddev_samp(rating) FROM contestant; + avg | stddev_samp +-----------------------+------------------ + 2344.3750000000000000 | 433.746119785032 +(1 row) + +SELECT country, avg(rating) FROM contestant WHERE rating > 2200 + GROUP BY country ORDER BY country; + country | avg +---------+----------------------- + XA | 2203.0000000000000000 + XB | 2610.5000000000000000 + XC | 2236.0000000000000000 + XD | 3090.0000000000000000 +(4 rows) + +SELECT * FROM contestant ORDER BY handle; + handle | birthdate | rating | percentile | country | achievements +--------+------------+--------+------------+---------+-------------- + a | 1990-01-10 | 2090 | 97.1 | XA | {a} + b | 1990-11-01 | 2203 | 98.1 | XA | {a,b} + c | 1988-11-01 | 2907 | 99.4 | XB | {w,y} + d | 1985-05-05 | 2314 | 98.3 | XB | {} + e | 1995-05-05 | 2236 | 98.2 | XC | {a} + f | 1983-04-02 | 3090 | 99.6 | XD | {a,b,c,y} + g | 1991-12-13 | 1803 | 85.1 | XD | {a,c} + h | 1987-10-26 | 2112 | 95.4 | XD | {w,a} +(8 rows) + +-- Query compressed data +SELECT count(*) FROM contestant_compressed; + count +------- + 8 +(1 row) + +SELECT avg(rating), stddev_samp(rating) FROM contestant_compressed; + avg | stddev_samp +-----------------------+------------------ + 2344.3750000000000000 | 433.746119785032 +(1 row) + +SELECT country, avg(rating) FROM contestant_compressed WHERE rating > 2200 + GROUP BY country ORDER BY country; + country | avg +---------+----------------------- + XA | 2203.0000000000000000 + XB | 2610.5000000000000000 + XC | 2236.0000000000000000 + XD | 3090.0000000000000000 +(4 rows) + +SELECT * FROM contestant_compressed ORDER BY handle; + handle | birthdate | rating | percentile | country | achievements +--------+------------+--------+------------+---------+-------------- + a | 1990-01-10 | 2090 | 97.1 | XA | {a} + b | 1990-11-01 | 2203 | 98.1 | XA | {a,b} + c | 1988-11-01 | 2907 | 99.4 | XB | {w,y} + d | 1985-05-05 | 2314 | 98.3 | XB | {} + e | 1995-05-05 | 2236 | 98.2 | XC | {a} + f | 1983-04-02 | 3090 | 99.6 | XD | {a,b,c,y} + g | 1991-12-13 | 1803 | 85.1 | XD | {a,c} + h | 1987-10-26 | 2112 | 95.4 | XD | {w,a} +(8 rows) + +-- Verify that we handle whole-row references correctly +SELECT to_json(v) FROM contestant v ORDER BY rating LIMIT 1; + to_json +------------------------------------------------------------------------------------------------------------------ + {"handle":"g","birthdate":"1991-12-13","rating":1803,"percentile":85.1,"country":"XD ","achievements":["a","c"]} +(1 row) + +-- Test variables used in expressions +CREATE FOREIGN TABLE union_first (a int, b int) SERVER cstore_server; +CREATE FOREIGN TABLE union_second (a int, b int) SERVER cstore_server; +INSERT INTO union_first SELECT a, a FROM generate_series(1, 5) a; +INSERT INTO union_second SELECT a, a FROM generate_series(11, 15) a; +(SELECT a*1, b FROM union_first) union all (SELECT a*1, b FROM union_second); + ?column? | b +----------+---- + 1 | 1 + 2 | 2 + 3 | 3 + 4 | 4 + 5 | 5 + 11 | 11 + 12 | 12 + 13 | 13 + 14 | 14 + 15 | 15 +(10 rows) + +DROP FOREIGN TABLE union_first, union_second; diff --git a/expected/am_truncate.out b/expected/am_truncate.out new file mode 100644 index 000000000..c92c15559 --- /dev/null +++ b/expected/am_truncate.out @@ -0,0 +1,231 @@ +-- +-- Test the TRUNCATE TABLE command for cstore_fdw tables. +-- +-- print whether we're using version > 10 to make version-specific tests clear +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int > 10 AS version_above_ten; + version_above_ten +------------------- + t +(1 row) + +-- CREATE a cstore_fdw table, fill with some data -- +CREATE FOREIGN TABLE cstore_truncate_test (a int, b int) SERVER cstore_server; +CREATE FOREIGN TABLE cstore_truncate_test_second (a int, b int) SERVER cstore_server; +CREATE FOREIGN TABLE cstore_truncate_test_compressed (a int, b int) SERVER cstore_server OPTIONS (compression 'pglz'); +CREATE TABLE cstore_truncate_test_regular (a int, b int); +INSERT INTO cstore_truncate_test select a, a from generate_series(1, 10) a; +INSERT INTO cstore_truncate_test_compressed select a, a from generate_series(1, 10) a; +INSERT INTO cstore_truncate_test_compressed select a, a from generate_series(1, 10) a; +-- query rows +SELECT * FROM cstore_truncate_test; + a | b +----+---- + 1 | 1 + 2 | 2 + 3 | 3 + 4 | 4 + 5 | 5 + 6 | 6 + 7 | 7 + 8 | 8 + 9 | 9 + 10 | 10 +(10 rows) + +TRUNCATE TABLE cstore_truncate_test; +SELECT * FROM cstore_truncate_test; + a | b +---+--- +(0 rows) + +SELECT COUNT(*) from cstore_truncate_test; + count +------- + 0 +(1 row) + +SELECT count(*) FROM cstore_truncate_test_compressed; + count +------- + 20 +(1 row) + +TRUNCATE TABLE cstore_truncate_test_compressed; +SELECT count(*) FROM cstore_truncate_test_compressed; + count +------- + 0 +(1 row) + +SELECT cstore_table_size('cstore_truncate_test_compressed'); + cstore_table_size +------------------- + 0 +(1 row) + +INSERT INTO cstore_truncate_test select a, a from generate_series(1, 10) a; +INSERT INTO cstore_truncate_test_regular select a, a from generate_series(10, 20) a; +INSERT INTO cstore_truncate_test_second select a, a from generate_series(20, 30) a; +SELECT * from cstore_truncate_test; + a | b +----+---- + 1 | 1 + 2 | 2 + 3 | 3 + 4 | 4 + 5 | 5 + 6 | 6 + 7 | 7 + 8 | 8 + 9 | 9 + 10 | 10 +(10 rows) + +SELECT * from cstore_truncate_test_second; + a | b +----+---- + 20 | 20 + 21 | 21 + 22 | 22 + 23 | 23 + 24 | 24 + 25 | 25 + 26 | 26 + 27 | 27 + 28 | 28 + 29 | 29 + 30 | 30 +(11 rows) + +SELECT * from cstore_truncate_test_regular; + a | b +----+---- + 10 | 10 + 11 | 11 + 12 | 12 + 13 | 13 + 14 | 14 + 15 | 15 + 16 | 16 + 17 | 17 + 18 | 18 + 19 | 19 + 20 | 20 +(11 rows) + +-- make sure multi truncate works +-- notice that the same table might be repeated +TRUNCATE TABLE cstore_truncate_test, + cstore_truncate_test_regular, + cstore_truncate_test_second, + cstore_truncate_test; +SELECT * from cstore_truncate_test; + a | b +---+--- +(0 rows) + +SELECT * from cstore_truncate_test_second; + a | b +---+--- +(0 rows) + +SELECT * from cstore_truncate_test_regular; + a | b +---+--- +(0 rows) + +-- test if truncate on empty table works +TRUNCATE TABLE cstore_truncate_test; +SELECT * from cstore_truncate_test; + a | b +---+--- +(0 rows) + +-- test if a cached truncate from a pl/pgsql function works +CREATE FUNCTION cstore_truncate_test_regular_func() RETURNS void AS $$ +BEGIN + INSERT INTO cstore_truncate_test_regular select a, a from generate_series(1, 10) a; + TRUNCATE TABLE cstore_truncate_test_regular; +END;$$ +LANGUAGE plpgsql; +SELECT cstore_truncate_test_regular_func(); + cstore_truncate_test_regular_func +----------------------------------- + +(1 row) + +-- the cached plans are used stating from the second call +SELECT cstore_truncate_test_regular_func(); + cstore_truncate_test_regular_func +----------------------------------- + +(1 row) + +DROP FUNCTION cstore_truncate_test_regular_func(); +DROP FOREIGN TABLE cstore_truncate_test, cstore_truncate_test_second; +DROP TABLE cstore_truncate_test_regular; +DROP FOREIGN TABLE cstore_truncate_test_compressed; +-- test truncate with schema +CREATE SCHEMA truncate_schema; +CREATE FOREIGN TABLE truncate_schema.truncate_tbl (id int) SERVER cstore_server OPTIONS(compression 'pglz'); +INSERT INTO truncate_schema.truncate_tbl SELECT generate_series(1, 100); +SELECT COUNT(*) FROM truncate_schema.truncate_tbl; + count +------- + 100 +(1 row) + +TRUNCATE TABLE truncate_schema.truncate_tbl; +SELECT COUNT(*) FROM truncate_schema.truncate_tbl; + count +------- + 0 +(1 row) + +INSERT INTO truncate_schema.truncate_tbl SELECT generate_series(1, 100); +-- create a user that can not truncate +CREATE USER truncate_user; +GRANT USAGE ON SCHEMA truncate_schema TO truncate_user; +GRANT SELECT ON TABLE truncate_schema.truncate_tbl TO truncate_user; +REVOKE TRUNCATE ON TABLE truncate_schema.truncate_tbl FROM truncate_user; +SELECT current_user \gset +\c - truncate_user +-- verify truncate command fails and check number of rows +SELECT count(*) FROM truncate_schema.truncate_tbl; + count +------- + 100 +(1 row) + +TRUNCATE TABLE truncate_schema.truncate_tbl; +ERROR: permission denied for table truncate_tbl +SELECT count(*) FROM truncate_schema.truncate_tbl; + count +------- + 100 +(1 row) + +-- switch to super user, grant truncate to truncate_user +\c - :current_user +GRANT TRUNCATE ON TABLE truncate_schema.truncate_tbl TO truncate_user; +-- verify truncate_user can truncate now +\c - truncate_user +SELECT count(*) FROM truncate_schema.truncate_tbl; + count +------- + 100 +(1 row) + +TRUNCATE TABLE truncate_schema.truncate_tbl; +SELECT count(*) FROM truncate_schema.truncate_tbl; + count +------- + 0 +(1 row) + +\c - :current_user +-- cleanup +DROP SCHEMA truncate_schema CASCADE; +NOTICE: drop cascades to foreign table truncate_schema.truncate_tbl +DROP USER truncate_user; diff --git a/expected/am_truncate_0.out b/expected/am_truncate_0.out new file mode 100644 index 000000000..c8cc4ad98 --- /dev/null +++ b/expected/am_truncate_0.out @@ -0,0 +1,262 @@ +-- +-- Test the TRUNCATE TABLE command for cstore_fdw tables. +-- +-- print whether we're using version > 10 to make version-specific tests clear +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int > 10 AS version_above_ten; + version_above_ten +------------------- + f +(1 row) + +-- Check that files for the automatically managed table exist in the +-- cstore_fdw/{databaseoid} directory. +SELECT count(*) FROM ( + SELECT pg_ls_dir('cstore_fdw/' || databaseoid ) FROM ( + SELECT oid::text databaseoid FROM pg_database WHERE datname = current_database() + ) AS q1) AS q2; + count +------- + 0 +(1 row) + +-- CREATE a cstore_fdw table, fill with some data -- +CREATE FOREIGN TABLE cstore_truncate_test (a int, b int) SERVER cstore_server; +CREATE FOREIGN TABLE cstore_truncate_test_second (a int, b int) SERVER cstore_server; +CREATE FOREIGN TABLE cstore_truncate_test_compressed (a int, b int) SERVER cstore_server OPTIONS (compression 'pglz'); +CREATE TABLE cstore_truncate_test_regular (a int, b int); +INSERT INTO cstore_truncate_test select a, a from generate_series(1, 10) a; +INSERT INTO cstore_truncate_test_compressed select a, a from generate_series(1, 10) a; +INSERT INTO cstore_truncate_test_compressed select a, a from generate_series(1, 10) a; +-- query rows +SELECT * FROM cstore_truncate_test; + a | b +----+---- + 1 | 1 + 2 | 2 + 3 | 3 + 4 | 4 + 5 | 5 + 6 | 6 + 7 | 7 + 8 | 8 + 9 | 9 + 10 | 10 +(10 rows) + +TRUNCATE TABLE cstore_truncate_test; +SELECT * FROM cstore_truncate_test; + a | b +---+--- +(0 rows) + +SELECT COUNT(*) from cstore_truncate_test; + count +------- + 0 +(1 row) + +SELECT count(*) FROM cstore_truncate_test_compressed; + count +------- + 20 +(1 row) + +TRUNCATE TABLE cstore_truncate_test_compressed; +SELECT count(*) FROM cstore_truncate_test_compressed; + count +------- + 0 +(1 row) + +SELECT cstore_table_size('cstore_truncate_test_compressed'); + cstore_table_size +------------------- + 26 +(1 row) + +-- make sure data files still present +SELECT count(*) FROM ( + SELECT pg_ls_dir('cstore_fdw/' || databaseoid ) FROM ( + SELECT oid::text databaseoid FROM pg_database WHERE datname = current_database() + ) AS q1) AS q2; + count +------- + 6 +(1 row) + +INSERT INTO cstore_truncate_test select a, a from generate_series(1, 10) a; +INSERT INTO cstore_truncate_test_regular select a, a from generate_series(10, 20) a; +INSERT INTO cstore_truncate_test_second select a, a from generate_series(20, 30) a; +SELECT * from cstore_truncate_test; + a | b +----+---- + 1 | 1 + 2 | 2 + 3 | 3 + 4 | 4 + 5 | 5 + 6 | 6 + 7 | 7 + 8 | 8 + 9 | 9 + 10 | 10 +(10 rows) + +SELECT * from cstore_truncate_test_second; + a | b +----+---- + 20 | 20 + 21 | 21 + 22 | 22 + 23 | 23 + 24 | 24 + 25 | 25 + 26 | 26 + 27 | 27 + 28 | 28 + 29 | 29 + 30 | 30 +(11 rows) + +SELECT * from cstore_truncate_test_regular; + a | b +----+---- + 10 | 10 + 11 | 11 + 12 | 12 + 13 | 13 + 14 | 14 + 15 | 15 + 16 | 16 + 17 | 17 + 18 | 18 + 19 | 19 + 20 | 20 +(11 rows) + +-- make sure multi truncate works +-- notice that the same table might be repeated +TRUNCATE TABLE cstore_truncate_test, + cstore_truncate_test_regular, + cstore_truncate_test_second, + cstore_truncate_test; +SELECT * from cstore_truncate_test; + a | b +---+--- +(0 rows) + +SELECT * from cstore_truncate_test_second; + a | b +---+--- +(0 rows) + +SELECT * from cstore_truncate_test_regular; + a | b +---+--- +(0 rows) + +-- test if truncate on empty table works +TRUNCATE TABLE cstore_truncate_test; +SELECT * from cstore_truncate_test; + a | b +---+--- +(0 rows) + +-- test if a cached truncate from a pl/pgsql function works +CREATE FUNCTION cstore_truncate_test_regular_func() RETURNS void AS $$ +BEGIN + INSERT INTO cstore_truncate_test_regular select a, a from generate_series(1, 10) a; + TRUNCATE TABLE cstore_truncate_test_regular; +END;$$ +LANGUAGE plpgsql; +SELECT cstore_truncate_test_regular_func(); + cstore_truncate_test_regular_func +----------------------------------- + +(1 row) + +-- the cached plans are used stating from the second call +SELECT cstore_truncate_test_regular_func(); + cstore_truncate_test_regular_func +----------------------------------- + +(1 row) + +DROP FUNCTION cstore_truncate_test_regular_func(); +DROP FOREIGN TABLE cstore_truncate_test, cstore_truncate_test_second; +DROP TABLE cstore_truncate_test_regular; +DROP FOREIGN TABLE cstore_truncate_test_compressed; +-- test truncate with schema +CREATE SCHEMA truncate_schema; +CREATE FOREIGN TABLE truncate_schema.truncate_tbl (id int) SERVER cstore_server OPTIONS(compression 'pglz'); +INSERT INTO truncate_schema.truncate_tbl SELECT generate_series(1, 100); +SELECT COUNT(*) FROM truncate_schema.truncate_tbl; + count +------- + 100 +(1 row) + +TRUNCATE TABLE truncate_schema.truncate_tbl; +SELECT COUNT(*) FROM truncate_schema.truncate_tbl; + count +------- + 0 +(1 row) + +INSERT INTO truncate_schema.truncate_tbl SELECT generate_series(1, 100); +-- create a user that can not truncate +CREATE USER truncate_user; +GRANT USAGE ON SCHEMA truncate_schema TO truncate_user; +GRANT SELECT ON TABLE truncate_schema.truncate_tbl TO truncate_user; +REVOKE TRUNCATE ON TABLE truncate_schema.truncate_tbl FROM truncate_user; +SELECT current_user \gset +\c - truncate_user +-- verify truncate command fails and check number of rows +SELECT count(*) FROM truncate_schema.truncate_tbl; + count +------- + 100 +(1 row) + +TRUNCATE TABLE truncate_schema.truncate_tbl; +ERROR: permission denied for relation truncate_tbl +SELECT count(*) FROM truncate_schema.truncate_tbl; + count +------- + 100 +(1 row) + +-- switch to super user, grant truncate to truncate_user +\c - :current_user +GRANT TRUNCATE ON TABLE truncate_schema.truncate_tbl TO truncate_user; +-- verify truncate_user can truncate now +\c - truncate_user +SELECT count(*) FROM truncate_schema.truncate_tbl; + count +------- + 100 +(1 row) + +TRUNCATE TABLE truncate_schema.truncate_tbl; +SELECT count(*) FROM truncate_schema.truncate_tbl; + count +------- + 0 +(1 row) + +\c - :current_user +-- cleanup +DROP SCHEMA truncate_schema CASCADE; +NOTICE: drop cascades to foreign table truncate_schema.truncate_tbl +DROP USER truncate_user; +-- verify files are removed +SELECT count(*) FROM ( + SELECT pg_ls_dir('cstore_fdw/' || databaseoid ) FROM ( + SELECT oid::text databaseoid FROM pg_database WHERE datname = current_database() + ) AS q1) AS q2; + count +------- + 0 +(1 row) + diff --git a/input/am_block_filtering.source b/input/am_block_filtering.source new file mode 100644 index 000000000..dc3170f0d --- /dev/null +++ b/input/am_block_filtering.source @@ -0,0 +1,69 @@ +-- +-- Test block filtering in cstore_fdw using min/max values in stripe skip lists. +-- + + +-- +-- filtered_row_count returns number of rows filtered by the WHERE clause. +-- If blocks get filtered by cstore_fdw, less rows are passed to WHERE +-- clause, so this function should return a lower number. +-- +CREATE OR REPLACE FUNCTION filtered_row_count (query text) RETURNS bigint AS +$$ + DECLARE + result bigint; + rec text; + BEGIN + result := 0; + + FOR rec IN EXECUTE 'EXPLAIN ANALYZE ' || query LOOP + IF rec ~ '^\s+Rows Removed by Filter' then + result := regexp_replace(rec, '[^0-9]*', '', 'g'); + END IF; + END LOOP; + + RETURN result; + END; +$$ LANGUAGE PLPGSQL; + + +-- Create and load data +CREATE FOREIGN TABLE test_block_filtering (a int) + SERVER cstore_server + OPTIONS(block_row_count '1000', stripe_row_count '2000'); + +COPY test_block_filtering FROM '@abs_srcdir@/data/block_filtering.csv' WITH CSV; + + +-- Verify that filtered_row_count is less than 1000 for the following queries +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering'); +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 200'); +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a > 200'); +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 9900'); +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a > 9900'); +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 0'); + + +-- Verify that filtered_row_count is less than 2000 for the following queries +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 1 AND 10'); +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 990 AND 2010'); +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN -10 AND 0'); + + +-- Load data for second time and verify that filtered_row_count is exactly twice as before +COPY test_block_filtering FROM '@abs_srcdir@/data/block_filtering.csv' WITH CSV; +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 200'); +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 0'); +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 990 AND 2010'); + + +-- Verify that we are fine with collations which use a different alphabet order +CREATE FOREIGN TABLE collation_block_filtering_test(A text collate "da_DK") + SERVER cstore_server; +COPY collation_block_filtering_test FROM STDIN; +A +Å +B +\. + +SELECT * FROM collation_block_filtering_test WHERE A > 'B'; diff --git a/input/am_copyto.source b/input/am_copyto.source new file mode 100644 index 000000000..a4b753a8d --- /dev/null +++ b/input/am_copyto.source @@ -0,0 +1,17 @@ +-- +-- Test copying data from cstore_fdw tables. +-- +CREATE FOREIGN TABLE test_contestant(handle TEXT, birthdate DATE, rating INT, + percentile FLOAT, country CHAR(3), achievements TEXT[]) + SERVER cstore_server; + +-- load table data from file +COPY test_contestant FROM '@abs_srcdir@/data/contestants.1.csv' WITH CSV; + +-- export using COPY table TO ... +COPY test_contestant TO STDOUT; + +-- export using COPY (SELECT * FROM table) TO ... +COPY (select * from test_contestant) TO STDOUT; + +DROP FOREIGN TABLE test_contestant CASCADE; diff --git a/input/am_create.source b/input/am_create.source new file mode 100644 index 000000000..ba52137c1 --- /dev/null +++ b/input/am_create.source @@ -0,0 +1,43 @@ +-- +-- Test the CREATE statements related to cstore_fdw. +-- + + +-- Install cstore_fdw +CREATE EXTENSION cstore_fdw; + +CREATE SERVER cstore_server FOREIGN DATA WRAPPER cstore_fdw; + + +-- Validator tests +CREATE FOREIGN TABLE test_validator_invalid_option () + SERVER cstore_server + OPTIONS(bad_option_name '1'); -- ERROR + +CREATE FOREIGN TABLE test_validator_invalid_stripe_row_count () + SERVER cstore_server + OPTIONS(stripe_row_count '0'); -- ERROR + +CREATE FOREIGN TABLE test_validator_invalid_block_row_count () + SERVER cstore_server + OPTIONS(block_row_count '0'); -- ERROR + +CREATE FOREIGN TABLE test_validator_invalid_compression_type () + SERVER cstore_server + OPTIONS(compression 'invalid_compression'); -- ERROR + +-- Create uncompressed table +CREATE FOREIGN TABLE contestant (handle TEXT, birthdate DATE, rating INT, + percentile FLOAT, country CHAR(3), achievements TEXT[]) + SERVER cstore_server; + + +-- Create compressed table with automatically determined file path +CREATE FOREIGN TABLE contestant_compressed (handle TEXT, birthdate DATE, rating INT, + percentile FLOAT, country CHAR(3), achievements TEXT[]) + SERVER cstore_server + OPTIONS(compression 'pglz'); + +-- Test that querying an empty table works +ANALYZE contestant; +SELECT count(*) FROM contestant; diff --git a/input/am_data_types.source b/input/am_data_types.source new file mode 100644 index 000000000..ec83c4d8c --- /dev/null +++ b/input/am_data_types.source @@ -0,0 +1,68 @@ +-- +-- Test loading and reading different data types to/from cstore_fdw foreign tables. +-- + + +-- Settings to make the result deterministic +SET datestyle = "ISO, YMD"; +SET timezone to 'GMT'; +SET intervalstyle TO 'POSTGRES_VERBOSE'; + + +-- Test array types +CREATE FOREIGN TABLE test_array_types (int_array int[], bigint_array bigint[], + text_array text[]) SERVER cstore_server; + +COPY test_array_types FROM '@abs_srcdir@/data/array_types.csv' WITH CSV; + +SELECT * FROM test_array_types; + + +-- Test date/time types +CREATE FOREIGN TABLE test_datetime_types (timestamp timestamp, + timestamp_with_timezone timestamp with time zone, date date, time time, + interval interval) SERVER cstore_server; + +COPY test_datetime_types FROM '@abs_srcdir@/data/datetime_types.csv' WITH CSV; + +SELECT * FROM test_datetime_types; + + +-- Test enum and composite types +CREATE TYPE enum_type AS ENUM ('a', 'b', 'c'); +CREATE TYPE composite_type AS (a int, b text); + +CREATE FOREIGN TABLE test_enum_and_composite_types (enum enum_type, + composite composite_type) SERVER cstore_server; + +COPY test_enum_and_composite_types FROM + '@abs_srcdir@/data/enum_and_composite_types.csv' WITH CSV; + +SELECT * FROM test_enum_and_composite_types; + + +-- Test range types +CREATE FOREIGN TABLE test_range_types (int4range int4range, int8range int8range, + numrange numrange, tsrange tsrange) SERVER cstore_server; + +COPY test_range_types FROM '@abs_srcdir@/data/range_types.csv' WITH CSV; + +SELECT * FROM test_range_types; + + +-- Test other types +CREATE FOREIGN TABLE test_other_types (bool boolean, bytea bytea, money money, + inet inet, bitstring bit varying(5), uuid uuid, json json) SERVER cstore_server; + +COPY test_other_types FROM '@abs_srcdir@/data/other_types.csv' WITH CSV; + +SELECT * FROM test_other_types; + + +-- Test null values +CREATE FOREIGN TABLE test_null_values (a int, b int[], c composite_type) + SERVER cstore_server; + +COPY test_null_values FROM '@abs_srcdir@/data/null_values.csv' WITH CSV; + +SELECT * FROM test_null_values; diff --git a/input/am_load.source b/input/am_load.source new file mode 100644 index 000000000..0913acde7 --- /dev/null +++ b/input/am_load.source @@ -0,0 +1,44 @@ +-- +-- Test loading data into cstore_fdw tables. +-- + +-- COPY with incorrect delimiter +COPY contestant FROM '@abs_srcdir@/data/contestants.1.csv' + WITH DELIMITER '|'; -- ERROR + +-- COPY with invalid program +COPY contestant FROM PROGRAM 'invalid_program' WITH CSV; -- ERROR + +-- COPY into uncompressed table from file +COPY contestant FROM '@abs_srcdir@/data/contestants.1.csv' WITH CSV; + +-- COPY into uncompressed table from program +COPY contestant FROM PROGRAM 'cat @abs_srcdir@/data/contestants.2.csv' WITH CSV; + +-- COPY into compressed table +COPY contestant_compressed FROM '@abs_srcdir@/data/contestants.1.csv' WITH CSV; + +-- COPY into uncompressed table from program +COPY contestant_compressed FROM PROGRAM 'cat @abs_srcdir@/data/contestants.2.csv' + WITH CSV; + +-- Test column list +CREATE FOREIGN TABLE famous_constants (id int, name text, value real) + SERVER cstore_server; +COPY famous_constants (value, name, id) FROM STDIN WITH CSV; +3.141,pi,1 +2.718,e,2 +0.577,gamma,3 +5.291e-11,bohr radius,4 +\. + +COPY famous_constants (name, value) FROM STDIN WITH CSV; +avagadro,6.022e23 +electron mass,9.109e-31 +proton mass,1.672e-27 +speed of light,2.997e8 +\. + +SELECT * FROM famous_constants ORDER BY id, name; + +DROP FOREIGN TABLE famous_constants; diff --git a/output/am_block_filtering.source b/output/am_block_filtering.source new file mode 100644 index 000000000..2f664a78a --- /dev/null +++ b/output/am_block_filtering.source @@ -0,0 +1,116 @@ +-- +-- Test block filtering in cstore_fdw using min/max values in stripe skip lists. +-- +-- +-- filtered_row_count returns number of rows filtered by the WHERE clause. +-- If blocks get filtered by cstore_fdw, less rows are passed to WHERE +-- clause, so this function should return a lower number. +-- +CREATE OR REPLACE FUNCTION filtered_row_count (query text) RETURNS bigint AS +$$ + DECLARE + result bigint; + rec text; + BEGIN + result := 0; + + FOR rec IN EXECUTE 'EXPLAIN ANALYZE ' || query LOOP + IF rec ~ '^\s+Rows Removed by Filter' then + result := regexp_replace(rec, '[^0-9]*', '', 'g'); + END IF; + END LOOP; + + RETURN result; + END; +$$ LANGUAGE PLPGSQL; +-- Create and load data +CREATE FOREIGN TABLE test_block_filtering (a int) + SERVER cstore_server + OPTIONS(block_row_count '1000', stripe_row_count '2000'); +COPY test_block_filtering FROM '@abs_srcdir@/data/block_filtering.csv' WITH CSV; +-- Verify that filtered_row_count is less than 1000 for the following queries +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering'); + filtered_row_count +-------------------- + 0 +(1 row) + +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 200'); + filtered_row_count +-------------------- + 801 +(1 row) + +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a > 200'); + filtered_row_count +-------------------- + 200 +(1 row) + +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 9900'); + filtered_row_count +-------------------- + 101 +(1 row) + +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a > 9900'); + filtered_row_count +-------------------- + 900 +(1 row) + +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 0'); + filtered_row_count +-------------------- + 0 +(1 row) + +-- Verify that filtered_row_count is less than 2000 for the following queries +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 1 AND 10'); + filtered_row_count +-------------------- + 990 +(1 row) + +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 990 AND 2010'); + filtered_row_count +-------------------- + 1979 +(1 row) + +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN -10 AND 0'); + filtered_row_count +-------------------- + 0 +(1 row) + +-- Load data for second time and verify that filtered_row_count is exactly twice as before +COPY test_block_filtering FROM '@abs_srcdir@/data/block_filtering.csv' WITH CSV; +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 200'); + filtered_row_count +-------------------- + 1602 +(1 row) + +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 0'); + filtered_row_count +-------------------- + 0 +(1 row) + +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 990 AND 2010'); + filtered_row_count +-------------------- + 3958 +(1 row) + +-- Verify that we are fine with collations which use a different alphabet order +CREATE FOREIGN TABLE collation_block_filtering_test(A text collate "da_DK") + SERVER cstore_server; +COPY collation_block_filtering_test FROM STDIN; +SELECT * FROM collation_block_filtering_test WHERE A > 'B'; + a +--- + Å +(1 row) + diff --git a/output/am_copyto.source b/output/am_copyto.source new file mode 100644 index 000000000..a8d841f18 --- /dev/null +++ b/output/am_copyto.source @@ -0,0 +1,23 @@ +-- +-- Test copying data from cstore_fdw tables. +-- +CREATE FOREIGN TABLE test_contestant(handle TEXT, birthdate DATE, rating INT, + percentile FLOAT, country CHAR(3), achievements TEXT[]) + SERVER cstore_server; +-- load table data from file +COPY test_contestant FROM '@abs_srcdir@/data/contestants.1.csv' WITH CSV; +-- export using COPY table TO ... +COPY test_contestant TO STDOUT; +a 01-10-1990 2090 97.1 XA {a} +b 11-01-1990 2203 98.1 XA {a,b} +c 11-01-1988 2907 99.4 XB {w,y} +d 05-05-1985 2314 98.3 XB {} +e 05-05-1995 2236 98.2 XC {a} +-- export using COPY (SELECT * FROM table) TO ... +COPY (select * from test_contestant) TO STDOUT; +a 01-10-1990 2090 97.1 XA {a} +b 11-01-1990 2203 98.1 XA {a,b} +c 11-01-1988 2907 99.4 XB {w,y} +d 05-05-1985 2314 98.3 XB {} +e 05-05-1995 2236 98.2 XC {a} +DROP FOREIGN TABLE test_contestant CASCADE; diff --git a/output/am_create.source b/output/am_create.source new file mode 100644 index 000000000..961c0494d --- /dev/null +++ b/output/am_create.source @@ -0,0 +1,44 @@ +-- +-- Test the CREATE statements related to cstore_fdw. +-- +-- Install cstore_fdw +CREATE EXTENSION cstore_fdw; +CREATE SERVER cstore_server FOREIGN DATA WRAPPER cstore_fdw; +-- Validator tests +CREATE FOREIGN TABLE test_validator_invalid_option () + SERVER cstore_server + OPTIONS(bad_option_name '1'); -- ERROR +ERROR: invalid option "bad_option_name" +HINT: Valid options in this context are: compression, stripe_row_count, block_row_count +CREATE FOREIGN TABLE test_validator_invalid_stripe_row_count () + SERVER cstore_server + OPTIONS(stripe_row_count '0'); -- ERROR +ERROR: invalid stripe row count +HINT: Stripe row count must be an integer between 1000 and 10000000 +CREATE FOREIGN TABLE test_validator_invalid_block_row_count () + SERVER cstore_server + OPTIONS(block_row_count '0'); -- ERROR +ERROR: invalid block row count +HINT: Block row count must be an integer between 1000 and 100000 +CREATE FOREIGN TABLE test_validator_invalid_compression_type () + SERVER cstore_server + OPTIONS(compression 'invalid_compression'); -- ERROR +ERROR: invalid compression type +HINT: Valid options are: none, pglz +-- Create uncompressed table +CREATE FOREIGN TABLE contestant (handle TEXT, birthdate DATE, rating INT, + percentile FLOAT, country CHAR(3), achievements TEXT[]) + SERVER cstore_server; +-- Create compressed table with automatically determined file path +CREATE FOREIGN TABLE contestant_compressed (handle TEXT, birthdate DATE, rating INT, + percentile FLOAT, country CHAR(3), achievements TEXT[]) + SERVER cstore_server + OPTIONS(compression 'pglz'); +-- Test that querying an empty table works +ANALYZE contestant; +SELECT count(*) FROM contestant; + count +------- + 0 +(1 row) + diff --git a/output/am_data_types.source b/output/am_data_types.source new file mode 100644 index 000000000..23fdcfa29 --- /dev/null +++ b/output/am_data_types.source @@ -0,0 +1,78 @@ +-- +-- Test loading and reading different data types to/from cstore_fdw foreign tables. +-- +-- Settings to make the result deterministic +SET datestyle = "ISO, YMD"; +SET timezone to 'GMT'; +SET intervalstyle TO 'POSTGRES_VERBOSE'; +-- Test array types +CREATE FOREIGN TABLE test_array_types (int_array int[], bigint_array bigint[], + text_array text[]) SERVER cstore_server; +COPY test_array_types FROM '@abs_srcdir@/data/array_types.csv' WITH CSV; +SELECT * FROM test_array_types; + int_array | bigint_array | text_array +--------------------------+--------------------------------------------+------------ + {1,2,3} | {1,2,3} | {a,b,c} + {} | {} | {} + {-2147483648,2147483647} | {-9223372036854775808,9223372036854775807} | {""} +(3 rows) + +-- Test date/time types +CREATE FOREIGN TABLE test_datetime_types (timestamp timestamp, + timestamp_with_timezone timestamp with time zone, date date, time time, + interval interval) SERVER cstore_server; +COPY test_datetime_types FROM '@abs_srcdir@/data/datetime_types.csv' WITH CSV; +SELECT * FROM test_datetime_types; + timestamp | timestamp_with_timezone | date | time | interval +---------------------+-------------------------+------------+----------+----------- + 2000-01-02 04:05:06 | 1999-01-08 12:05:06+00 | 2000-01-02 | 04:05:06 | @ 4 hours + 1970-01-01 00:00:00 | infinity | -infinity | 00:00:00 | @ 0 +(2 rows) + +-- Test enum and composite types +CREATE TYPE enum_type AS ENUM ('a', 'b', 'c'); +CREATE TYPE composite_type AS (a int, b text); +CREATE FOREIGN TABLE test_enum_and_composite_types (enum enum_type, + composite composite_type) SERVER cstore_server; +COPY test_enum_and_composite_types FROM + '@abs_srcdir@/data/enum_and_composite_types.csv' WITH CSV; +SELECT * FROM test_enum_and_composite_types; + enum | composite +------+----------- + a | (2,b) + b | (3,c) +(2 rows) + +-- Test range types +CREATE FOREIGN TABLE test_range_types (int4range int4range, int8range int8range, + numrange numrange, tsrange tsrange) SERVER cstore_server; +COPY test_range_types FROM '@abs_srcdir@/data/range_types.csv' WITH CSV; +SELECT * FROM test_range_types; + int4range | int8range | numrange | tsrange +-----------+-----------+----------+----------------------------------------------- + [1,3) | [1,3) | [1,3) | ["2000-01-02 00:30:00","2010-02-03 12:30:00") + empty | [1,) | (,) | empty +(2 rows) + +-- Test other types +CREATE FOREIGN TABLE test_other_types (bool boolean, bytea bytea, money money, + inet inet, bitstring bit varying(5), uuid uuid, json json) SERVER cstore_server; +COPY test_other_types FROM '@abs_srcdir@/data/other_types.csv' WITH CSV; +SELECT * FROM test_other_types; + bool | bytea | money | inet | bitstring | uuid | json +------+------------+-------+-------------+-----------+--------------------------------------+------------------ + f | \xdeadbeef | $1.00 | 192.168.1.2 | 10101 | a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11 | {"key": "value"} + t | \xcdb0 | $1.50 | 127.0.0.1 | | a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11 | [] +(2 rows) + +-- Test null values +CREATE FOREIGN TABLE test_null_values (a int, b int[], c composite_type) + SERVER cstore_server; +COPY test_null_values FROM '@abs_srcdir@/data/null_values.csv' WITH CSV; +SELECT * FROM test_null_values; + a | b | c +---+--------+----- + | {NULL} | (,) + | | +(2 rows) + diff --git a/output/am_load.source b/output/am_load.source new file mode 100644 index 000000000..c76f203eb --- /dev/null +++ b/output/am_load.source @@ -0,0 +1,39 @@ +-- +-- Test loading data into cstore_fdw tables. +-- +-- COPY with incorrect delimiter +COPY contestant FROM '@abs_srcdir@/data/contestants.1.csv' + WITH DELIMITER '|'; -- ERROR +ERROR: missing data for column "birthdate" +-- COPY with invalid program +COPY contestant FROM PROGRAM 'invalid_program' WITH CSV; -- ERROR +ERROR: program "invalid_program" failed +DETAIL: command not found +-- COPY into uncompressed table from file +COPY contestant FROM '@abs_srcdir@/data/contestants.1.csv' WITH CSV; +-- COPY into uncompressed table from program +COPY contestant FROM PROGRAM 'cat @abs_srcdir@/data/contestants.2.csv' WITH CSV; +-- COPY into compressed table +COPY contestant_compressed FROM '@abs_srcdir@/data/contestants.1.csv' WITH CSV; +-- COPY into uncompressed table from program +COPY contestant_compressed FROM PROGRAM 'cat @abs_srcdir@/data/contestants.2.csv' + WITH CSV; +-- Test column list +CREATE FOREIGN TABLE famous_constants (id int, name text, value real) + SERVER cstore_server; +COPY famous_constants (value, name, id) FROM STDIN WITH CSV; +COPY famous_constants (name, value) FROM STDIN WITH CSV; +SELECT * FROM famous_constants ORDER BY id, name; + id | name | value +----+----------------+----------- + 1 | pi | 3.141 + 2 | e | 2.718 + 3 | gamma | 0.577 + 4 | bohr radius | 5.291e-11 + | avagadro | 6.022e+23 + | electron mass | 9.109e-31 + | proton mass | 1.672e-27 + | speed of light | 2.997e+08 +(8 rows) + +DROP FOREIGN TABLE famous_constants; diff --git a/sql/am_alter.sql b/sql/am_alter.sql new file mode 100644 index 000000000..5ba3beb34 --- /dev/null +++ b/sql/am_alter.sql @@ -0,0 +1,85 @@ +-- +-- Testing ALTER TABLE on cstore_fdw tables. +-- + +CREATE FOREIGN TABLE test_alter_table (a int, b int, c int) SERVER cstore_server; + +WITH sample_data AS (VALUES + (1, 2, 3), + (4, 5, 6), + (7, 8, 9) +) +INSERT INTO test_alter_table SELECT * FROM sample_data; + +-- drop a column +ALTER FOREIGN TABLE test_alter_table DROP COLUMN a; + +-- test analyze +ANALYZE test_alter_table; + +-- verify select queries run as expected +SELECT * FROM test_alter_table; +SELECT a FROM test_alter_table; +SELECT b FROM test_alter_table; + +-- verify insert runs as expected +INSERT INTO test_alter_table (SELECT 3, 5, 8); +INSERT INTO test_alter_table (SELECT 5, 8); + + +-- add a column with no defaults +ALTER FOREIGN TABLE test_alter_table ADD COLUMN d int; +SELECT * FROM test_alter_table; +INSERT INTO test_alter_table (SELECT 3, 5, 8); +SELECT * FROM test_alter_table; + + +-- add a fixed-length column with default value +ALTER FOREIGN TABLE test_alter_table ADD COLUMN e int default 3; +SELECT * from test_alter_table; +INSERT INTO test_alter_table (SELECT 1, 2, 4, 8); +SELECT * from test_alter_table; + + +-- add a variable-length column with default value +ALTER FOREIGN TABLE test_alter_table ADD COLUMN f text DEFAULT 'TEXT ME'; +SELECT * from test_alter_table; +INSERT INTO test_alter_table (SELECT 1, 2, 4, 8, 'ABCDEF'); +SELECT * from test_alter_table; + + +-- drop couple of columns +ALTER FOREIGN TABLE test_alter_table DROP COLUMN c; +ALTER FOREIGN TABLE test_alter_table DROP COLUMN e; +ANALYZE test_alter_table; +SELECT * from test_alter_table; +SELECT count(*) from test_alter_table; +SELECT count(t.*) from test_alter_table t; + + +-- unsupported default values +ALTER FOREIGN TABLE test_alter_table ADD COLUMN g boolean DEFAULT isfinite(current_date); +ALTER FOREIGN TABLE test_alter_table ADD COLUMN h DATE DEFAULT current_date; +SELECT * FROM test_alter_table; +ALTER FOREIGN TABLE test_alter_table ALTER COLUMN g DROP DEFAULT; +SELECT * FROM test_alter_table; +ALTER FOREIGN TABLE test_alter_table ALTER COLUMN h DROP DEFAULT; +ANALYZE test_alter_table; +SELECT * FROM test_alter_table; + +-- unsupported type change +ALTER FOREIGN TABLE test_alter_table ADD COLUMN i int; +ALTER FOREIGN TABLE test_alter_table ADD COLUMN j float; +ALTER FOREIGN TABLE test_alter_table ADD COLUMN k text; + +-- this is valid type change +ALTER FOREIGN TABLE test_alter_table ALTER COLUMN i TYPE float; + +-- this is not valid +ALTER FOREIGN TABLE test_alter_table ALTER COLUMN j TYPE int; + +-- text / varchar conversion is valid both ways +ALTER FOREIGN TABLE test_alter_table ALTER COLUMN k TYPE varchar(20); +ALTER FOREIGN TABLE test_alter_table ALTER COLUMN k TYPE text; + +DROP FOREIGN TABLE test_alter_table; diff --git a/sql/am_analyze.sql b/sql/am_analyze.sql new file mode 100644 index 000000000..4476454a6 --- /dev/null +++ b/sql/am_analyze.sql @@ -0,0 +1,11 @@ +-- +-- Test the ANALYZE command for cstore_fdw tables. +-- + +-- ANALYZE uncompressed table +ANALYZE contestant; +SELECT count(*) FROM pg_stats WHERE tablename='contestant'; + +-- ANALYZE compressed table +ANALYZE contestant_compressed; +SELECT count(*) FROM pg_stats WHERE tablename='contestant_compressed'; diff --git a/sql/am_block_filtering.sql b/sql/am_block_filtering.sql new file mode 100644 index 000000000..bb90c72ca --- /dev/null +++ b/sql/am_block_filtering.sql @@ -0,0 +1,69 @@ +-- +-- Test block filtering in cstore_fdw using min/max values in stripe skip lists. +-- + + +-- +-- filtered_row_count returns number of rows filtered by the WHERE clause. +-- If blocks get filtered by cstore_fdw, less rows are passed to WHERE +-- clause, so this function should return a lower number. +-- +CREATE OR REPLACE FUNCTION filtered_row_count (query text) RETURNS bigint AS +$$ + DECLARE + result bigint; + rec text; + BEGIN + result := 0; + + FOR rec IN EXECUTE 'EXPLAIN ANALYZE ' || query LOOP + IF rec ~ '^\s+Rows Removed by Filter' then + result := regexp_replace(rec, '[^0-9]*', '', 'g'); + END IF; + END LOOP; + + RETURN result; + END; +$$ LANGUAGE PLPGSQL; + + +-- Create and load data +CREATE FOREIGN TABLE test_block_filtering (a int) + SERVER cstore_server + OPTIONS(block_row_count '1000', stripe_row_count '2000'); + +COPY test_block_filtering FROM '/Users/jefdavi/wd/cstore2/data/block_filtering.csv' WITH CSV; + + +-- Verify that filtered_row_count is less than 1000 for the following queries +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering'); +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 200'); +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a > 200'); +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 9900'); +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a > 9900'); +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 0'); + + +-- Verify that filtered_row_count is less than 2000 for the following queries +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 1 AND 10'); +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 990 AND 2010'); +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN -10 AND 0'); + + +-- Load data for second time and verify that filtered_row_count is exactly twice as before +COPY test_block_filtering FROM '/Users/jefdavi/wd/cstore2/data/block_filtering.csv' WITH CSV; +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 200'); +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a < 0'); +SELECT filtered_row_count('SELECT count(*) FROM test_block_filtering WHERE a BETWEEN 990 AND 2010'); + + +-- Verify that we are fine with collations which use a different alphabet order +CREATE FOREIGN TABLE collation_block_filtering_test(A text collate "da_DK") + SERVER cstore_server; +COPY collation_block_filtering_test FROM STDIN; +A +Å +B +\. + +SELECT * FROM collation_block_filtering_test WHERE A > 'B'; diff --git a/sql/am_copyto.sql b/sql/am_copyto.sql new file mode 100644 index 000000000..4e9e839b7 --- /dev/null +++ b/sql/am_copyto.sql @@ -0,0 +1,17 @@ +-- +-- Test copying data from cstore_fdw tables. +-- +CREATE FOREIGN TABLE test_contestant(handle TEXT, birthdate DATE, rating INT, + percentile FLOAT, country CHAR(3), achievements TEXT[]) + SERVER cstore_server; + +-- load table data from file +COPY test_contestant FROM '/Users/jefdavi/wd/cstore2/data/contestants.1.csv' WITH CSV; + +-- export using COPY table TO ... +COPY test_contestant TO STDOUT; + +-- export using COPY (SELECT * FROM table) TO ... +COPY (select * from test_contestant) TO STDOUT; + +DROP FOREIGN TABLE test_contestant CASCADE; diff --git a/sql/am_create.sql b/sql/am_create.sql new file mode 100644 index 000000000..ba52137c1 --- /dev/null +++ b/sql/am_create.sql @@ -0,0 +1,43 @@ +-- +-- Test the CREATE statements related to cstore_fdw. +-- + + +-- Install cstore_fdw +CREATE EXTENSION cstore_fdw; + +CREATE SERVER cstore_server FOREIGN DATA WRAPPER cstore_fdw; + + +-- Validator tests +CREATE FOREIGN TABLE test_validator_invalid_option () + SERVER cstore_server + OPTIONS(bad_option_name '1'); -- ERROR + +CREATE FOREIGN TABLE test_validator_invalid_stripe_row_count () + SERVER cstore_server + OPTIONS(stripe_row_count '0'); -- ERROR + +CREATE FOREIGN TABLE test_validator_invalid_block_row_count () + SERVER cstore_server + OPTIONS(block_row_count '0'); -- ERROR + +CREATE FOREIGN TABLE test_validator_invalid_compression_type () + SERVER cstore_server + OPTIONS(compression 'invalid_compression'); -- ERROR + +-- Create uncompressed table +CREATE FOREIGN TABLE contestant (handle TEXT, birthdate DATE, rating INT, + percentile FLOAT, country CHAR(3), achievements TEXT[]) + SERVER cstore_server; + + +-- Create compressed table with automatically determined file path +CREATE FOREIGN TABLE contestant_compressed (handle TEXT, birthdate DATE, rating INT, + percentile FLOAT, country CHAR(3), achievements TEXT[]) + SERVER cstore_server + OPTIONS(compression 'pglz'); + +-- Test that querying an empty table works +ANALYZE contestant; +SELECT count(*) FROM contestant; diff --git a/sql/am_data_types.sql b/sql/am_data_types.sql new file mode 100644 index 000000000..092538a57 --- /dev/null +++ b/sql/am_data_types.sql @@ -0,0 +1,68 @@ +-- +-- Test loading and reading different data types to/from cstore_fdw foreign tables. +-- + + +-- Settings to make the result deterministic +SET datestyle = "ISO, YMD"; +SET timezone to 'GMT'; +SET intervalstyle TO 'POSTGRES_VERBOSE'; + + +-- Test array types +CREATE FOREIGN TABLE test_array_types (int_array int[], bigint_array bigint[], + text_array text[]) SERVER cstore_server; + +COPY test_array_types FROM '/Users/jefdavi/wd/cstore2/data/array_types.csv' WITH CSV; + +SELECT * FROM test_array_types; + + +-- Test date/time types +CREATE FOREIGN TABLE test_datetime_types (timestamp timestamp, + timestamp_with_timezone timestamp with time zone, date date, time time, + interval interval) SERVER cstore_server; + +COPY test_datetime_types FROM '/Users/jefdavi/wd/cstore2/data/datetime_types.csv' WITH CSV; + +SELECT * FROM test_datetime_types; + + +-- Test enum and composite types +CREATE TYPE enum_type AS ENUM ('a', 'b', 'c'); +CREATE TYPE composite_type AS (a int, b text); + +CREATE FOREIGN TABLE test_enum_and_composite_types (enum enum_type, + composite composite_type) SERVER cstore_server; + +COPY test_enum_and_composite_types FROM + '/Users/jefdavi/wd/cstore2/data/enum_and_composite_types.csv' WITH CSV; + +SELECT * FROM test_enum_and_composite_types; + + +-- Test range types +CREATE FOREIGN TABLE test_range_types (int4range int4range, int8range int8range, + numrange numrange, tsrange tsrange) SERVER cstore_server; + +COPY test_range_types FROM '/Users/jefdavi/wd/cstore2/data/range_types.csv' WITH CSV; + +SELECT * FROM test_range_types; + + +-- Test other types +CREATE FOREIGN TABLE test_other_types (bool boolean, bytea bytea, money money, + inet inet, bitstring bit varying(5), uuid uuid, json json) SERVER cstore_server; + +COPY test_other_types FROM '/Users/jefdavi/wd/cstore2/data/other_types.csv' WITH CSV; + +SELECT * FROM test_other_types; + + +-- Test null values +CREATE FOREIGN TABLE test_null_values (a int, b int[], c composite_type) + SERVER cstore_server; + +COPY test_null_values FROM '/Users/jefdavi/wd/cstore2/data/null_values.csv' WITH CSV; + +SELECT * FROM test_null_values; diff --git a/sql/am_drop.sql b/sql/am_drop.sql new file mode 100644 index 000000000..c64b5c99b --- /dev/null +++ b/sql/am_drop.sql @@ -0,0 +1,48 @@ +-- +-- Tests the different DROP commands for cstore_fdw tables. +-- +-- DROP FOREIGN TABL +-- DROP SCHEMA +-- DROP EXTENSION +-- DROP DATABASE +-- + +-- Note that travis does not create +-- cstore_fdw extension in default database (postgres). This has caused +-- different behavior between travis tests and local tests. Thus +-- 'postgres' directory is excluded from comparison to have the same result. + +-- store postgres database oid +SELECT oid postgres_oid FROM pg_database WHERE datname = 'postgres' \gset + +-- DROP cstore_fdw tables +DROP FOREIGN TABLE contestant; +DROP FOREIGN TABLE contestant_compressed; + +-- Create a cstore_fdw table under a schema and drop it. +CREATE SCHEMA test_schema; +CREATE FOREIGN TABLE test_schema.test_table(data int) SERVER cstore_server; +DROP SCHEMA test_schema CASCADE; + +SELECT current_database() datname \gset + +CREATE DATABASE db_to_drop; +\c db_to_drop +CREATE EXTENSION cstore_fdw; +CREATE SERVER cstore_server FOREIGN DATA WRAPPER cstore_fdw; +SELECT oid::text databaseoid FROM pg_database WHERE datname = current_database() \gset + +CREATE FOREIGN TABLE test_table(data int) SERVER cstore_server; + +DROP EXTENSION cstore_fdw CASCADE; + +-- test database drop +CREATE EXTENSION cstore_fdw; +CREATE SERVER cstore_server FOREIGN DATA WRAPPER cstore_fdw; +SELECT oid::text databaseoid FROM pg_database WHERE datname = current_database() \gset + +CREATE FOREIGN TABLE test_table(data int) SERVER cstore_server; + +\c :datname + +DROP DATABASE db_to_drop; diff --git a/sql/am_functions.sql b/sql/am_functions.sql new file mode 100644 index 000000000..ed7e260b3 --- /dev/null +++ b/sql/am_functions.sql @@ -0,0 +1,20 @@ +-- +-- Test utility functions for cstore_fdw tables. +-- + +CREATE FOREIGN TABLE empty_table (a int) SERVER cstore_server; +CREATE FOREIGN TABLE table_with_data (a int) SERVER cstore_server; +CREATE TABLE non_cstore_table (a int); + +COPY table_with_data FROM STDIN; +1 +2 +3 +\. + +SELECT cstore_table_size('empty_table') < cstore_table_size('table_with_data'); +SELECT cstore_table_size('non_cstore_table'); + +DROP FOREIGN TABLE empty_table; +DROP FOREIGN TABLE table_with_data; +DROP TABLE non_cstore_table; diff --git a/sql/am_insert.sql b/sql/am_insert.sql new file mode 100644 index 000000000..7a6b075ce --- /dev/null +++ b/sql/am_insert.sql @@ -0,0 +1,56 @@ +-- +-- Testing insert on cstore_fdw tables. +-- + +CREATE FOREIGN TABLE test_insert_command (a int) SERVER cstore_server; + +-- test single row inserts fail +select count(*) from test_insert_command; +insert into test_insert_command values(1); +select count(*) from test_insert_command; + +insert into test_insert_command default values; +select count(*) from test_insert_command; + +-- test inserting from another table succeed +CREATE TABLE test_insert_command_data (a int); + +select count(*) from test_insert_command_data; +insert into test_insert_command_data values(1); +select count(*) from test_insert_command_data; + +insert into test_insert_command select * from test_insert_command_data; +select count(*) from test_insert_command; + +drop table test_insert_command_data; +drop foreign table test_insert_command; + +-- test long attribute value insertion +-- create sufficiently long text so that data is stored in toast +CREATE TABLE test_long_text AS +SELECT a as int_val, string_agg(random()::text, '') as text_val +FROM generate_series(1, 10) a, generate_series(1, 1000) b +GROUP BY a ORDER BY a; + +-- store hash values of text for later comparison +CREATE TABLE test_long_text_hash AS +SELECT int_val, md5(text_val) AS hash +FROM test_long_text; + +CREATE FOREIGN TABLE test_cstore_long_text(int_val int, text_val text) +SERVER cstore_server; + +-- store long text in cstore table +INSERT INTO test_cstore_long_text SELECT * FROM test_long_text; + +-- drop source table to remove original text from toast +DROP TABLE test_long_text; + +-- check if text data is still available in cstore table +-- by comparing previously stored hash. +SELECT a.int_val +FROM test_long_text_hash a, test_cstore_long_text c +WHERE a.int_val = c.int_val AND a.hash = md5(c.text_val); + +DROP TABLE test_long_text_hash; +DROP FOREIGN TABLE test_cstore_long_text; diff --git a/sql/am_load.sql b/sql/am_load.sql new file mode 100644 index 000000000..7f9238b57 --- /dev/null +++ b/sql/am_load.sql @@ -0,0 +1,44 @@ +-- +-- Test loading data into cstore_fdw tables. +-- + +-- COPY with incorrect delimiter +COPY contestant FROM '/Users/jefdavi/wd/cstore2/data/contestants.1.csv' + WITH DELIMITER '|'; -- ERROR + +-- COPY with invalid program +COPY contestant FROM PROGRAM 'invalid_program' WITH CSV; -- ERROR + +-- COPY into uncompressed table from file +COPY contestant FROM '/Users/jefdavi/wd/cstore2/data/contestants.1.csv' WITH CSV; + +-- COPY into uncompressed table from program +COPY contestant FROM PROGRAM 'cat /Users/jefdavi/wd/cstore2/data/contestants.2.csv' WITH CSV; + +-- COPY into compressed table +COPY contestant_compressed FROM '/Users/jefdavi/wd/cstore2/data/contestants.1.csv' WITH CSV; + +-- COPY into uncompressed table from program +COPY contestant_compressed FROM PROGRAM 'cat /Users/jefdavi/wd/cstore2/data/contestants.2.csv' + WITH CSV; + +-- Test column list +CREATE FOREIGN TABLE famous_constants (id int, name text, value real) + SERVER cstore_server; +COPY famous_constants (value, name, id) FROM STDIN WITH CSV; +3.141,pi,1 +2.718,e,2 +0.577,gamma,3 +5.291e-11,bohr radius,4 +\. + +COPY famous_constants (name, value) FROM STDIN WITH CSV; +avagadro,6.022e23 +electron mass,9.109e-31 +proton mass,1.672e-27 +speed of light,2.997e8 +\. + +SELECT * FROM famous_constants ORDER BY id, name; + +DROP FOREIGN TABLE famous_constants; diff --git a/sql/am_query.sql b/sql/am_query.sql new file mode 100644 index 000000000..87743e7bd --- /dev/null +++ b/sql/am_query.sql @@ -0,0 +1,34 @@ +-- +-- Test querying cstore_fdw tables. +-- + +-- Settings to make the result deterministic +SET datestyle = "ISO, YMD"; + +-- Query uncompressed data +SELECT count(*) FROM contestant; +SELECT avg(rating), stddev_samp(rating) FROM contestant; +SELECT country, avg(rating) FROM contestant WHERE rating > 2200 + GROUP BY country ORDER BY country; +SELECT * FROM contestant ORDER BY handle; + +-- Query compressed data +SELECT count(*) FROM contestant_compressed; +SELECT avg(rating), stddev_samp(rating) FROM contestant_compressed; +SELECT country, avg(rating) FROM contestant_compressed WHERE rating > 2200 + GROUP BY country ORDER BY country; +SELECT * FROM contestant_compressed ORDER BY handle; + +-- Verify that we handle whole-row references correctly +SELECT to_json(v) FROM contestant v ORDER BY rating LIMIT 1; + +-- Test variables used in expressions +CREATE FOREIGN TABLE union_first (a int, b int) SERVER cstore_server; +CREATE FOREIGN TABLE union_second (a int, b int) SERVER cstore_server; + +INSERT INTO union_first SELECT a, a FROM generate_series(1, 5) a; +INSERT INTO union_second SELECT a, a FROM generate_series(11, 15) a; + +(SELECT a*1, b FROM union_first) union all (SELECT a*1, b FROM union_second); + +DROP FOREIGN TABLE union_first, union_second; diff --git a/sql/am_truncate.sql b/sql/am_truncate.sql new file mode 100644 index 000000000..a1849045e --- /dev/null +++ b/sql/am_truncate.sql @@ -0,0 +1,116 @@ +-- +-- Test the TRUNCATE TABLE command for cstore_fdw tables. +-- + +-- print whether we're using version > 10 to make version-specific tests clear +SHOW server_version \gset +SELECT substring(:'server_version', '\d+')::int > 10 AS version_above_ten; + +-- CREATE a cstore_fdw table, fill with some data -- +CREATE FOREIGN TABLE cstore_truncate_test (a int, b int) SERVER cstore_server; +CREATE FOREIGN TABLE cstore_truncate_test_second (a int, b int) SERVER cstore_server; +CREATE FOREIGN TABLE cstore_truncate_test_compressed (a int, b int) SERVER cstore_server OPTIONS (compression 'pglz'); +CREATE TABLE cstore_truncate_test_regular (a int, b int); + +INSERT INTO cstore_truncate_test select a, a from generate_series(1, 10) a; + +INSERT INTO cstore_truncate_test_compressed select a, a from generate_series(1, 10) a; +INSERT INTO cstore_truncate_test_compressed select a, a from generate_series(1, 10) a; + +-- query rows +SELECT * FROM cstore_truncate_test; + +TRUNCATE TABLE cstore_truncate_test; + +SELECT * FROM cstore_truncate_test; + +SELECT COUNT(*) from cstore_truncate_test; + +SELECT count(*) FROM cstore_truncate_test_compressed; +TRUNCATE TABLE cstore_truncate_test_compressed; +SELECT count(*) FROM cstore_truncate_test_compressed; + +SELECT cstore_table_size('cstore_truncate_test_compressed'); + +INSERT INTO cstore_truncate_test select a, a from generate_series(1, 10) a; +INSERT INTO cstore_truncate_test_regular select a, a from generate_series(10, 20) a; +INSERT INTO cstore_truncate_test_second select a, a from generate_series(20, 30) a; + +SELECT * from cstore_truncate_test; + +SELECT * from cstore_truncate_test_second; + +SELECT * from cstore_truncate_test_regular; + +-- make sure multi truncate works +-- notice that the same table might be repeated +TRUNCATE TABLE cstore_truncate_test, + cstore_truncate_test_regular, + cstore_truncate_test_second, + cstore_truncate_test; + +SELECT * from cstore_truncate_test; +SELECT * from cstore_truncate_test_second; +SELECT * from cstore_truncate_test_regular; + +-- test if truncate on empty table works +TRUNCATE TABLE cstore_truncate_test; +SELECT * from cstore_truncate_test; + +-- test if a cached truncate from a pl/pgsql function works +CREATE FUNCTION cstore_truncate_test_regular_func() RETURNS void AS $$ +BEGIN + INSERT INTO cstore_truncate_test_regular select a, a from generate_series(1, 10) a; + TRUNCATE TABLE cstore_truncate_test_regular; +END;$$ +LANGUAGE plpgsql; + +SELECT cstore_truncate_test_regular_func(); +-- the cached plans are used stating from the second call +SELECT cstore_truncate_test_regular_func(); +DROP FUNCTION cstore_truncate_test_regular_func(); + +DROP FOREIGN TABLE cstore_truncate_test, cstore_truncate_test_second; +DROP TABLE cstore_truncate_test_regular; +DROP FOREIGN TABLE cstore_truncate_test_compressed; + +-- test truncate with schema +CREATE SCHEMA truncate_schema; +CREATE FOREIGN TABLE truncate_schema.truncate_tbl (id int) SERVER cstore_server OPTIONS(compression 'pglz'); +INSERT INTO truncate_schema.truncate_tbl SELECT generate_series(1, 100); +SELECT COUNT(*) FROM truncate_schema.truncate_tbl; + +TRUNCATE TABLE truncate_schema.truncate_tbl; +SELECT COUNT(*) FROM truncate_schema.truncate_tbl; + +INSERT INTO truncate_schema.truncate_tbl SELECT generate_series(1, 100); + +-- create a user that can not truncate +CREATE USER truncate_user; +GRANT USAGE ON SCHEMA truncate_schema TO truncate_user; +GRANT SELECT ON TABLE truncate_schema.truncate_tbl TO truncate_user; +REVOKE TRUNCATE ON TABLE truncate_schema.truncate_tbl FROM truncate_user; + +SELECT current_user \gset + +\c - truncate_user +-- verify truncate command fails and check number of rows +SELECT count(*) FROM truncate_schema.truncate_tbl; +TRUNCATE TABLE truncate_schema.truncate_tbl; +SELECT count(*) FROM truncate_schema.truncate_tbl; + +-- switch to super user, grant truncate to truncate_user +\c - :current_user +GRANT TRUNCATE ON TABLE truncate_schema.truncate_tbl TO truncate_user; + +-- verify truncate_user can truncate now +\c - truncate_user +SELECT count(*) FROM truncate_schema.truncate_tbl; +TRUNCATE TABLE truncate_schema.truncate_tbl; +SELECT count(*) FROM truncate_schema.truncate_tbl; + +\c - :current_user + +-- cleanup +DROP SCHEMA truncate_schema CASCADE; +DROP USER truncate_user;