From 96278822d9a767235c2ac63ed72609800830ee52 Mon Sep 17 00:00:00 2001 From: Onur Tirtir Date: Fri, 16 Apr 2021 18:56:21 +0300 Subject: [PATCH] Move columnar test helpers to a separate file (#4908) * Move columnar test helpers to another file * Rename column_store_memory_stats to columnar_store_memory_stats --- src/backend/columnar/columnar_debug.c | 6 +- src/test/regress/base_schedule | 2 +- src/test/regress/columnar_schedule | 2 +- src/test/regress/expected/columnar_create.out | 90 +++-------------- src/test/regress/expected/columnar_insert.out | 12 +-- src/test/regress/expected/columnar_lz4.out | 2 +- src/test/regress/expected/columnar_lz4_0.out | 2 +- .../regress/expected/columnar_matview.out | 2 +- src/test/regress/expected/columnar_memory.out | 26 ++--- .../regress/expected/columnar_recursive.out | 14 +-- .../regress/expected/columnar_rollback.out | 2 +- .../expected/columnar_test_helpers.out | 69 +++++++++++++ .../regress/expected/columnar_truncate.out | 10 +- src/test/regress/expected/columnar_vacuum.out | 16 ++-- src/test/regress/expected/columnar_zstd.out | 2 +- src/test/regress/expected/columnar_zstd_0.out | 2 +- src/test/regress/input/columnar_load.source | 2 +- src/test/regress/minimal_schedule | 2 +- src/test/regress/output/columnar_load.source | 4 +- src/test/regress/sql/columnar_create.sql | 96 +++---------------- src/test/regress/sql/columnar_insert.sql | 12 +-- src/test/regress/sql/columnar_lz4.sql | 2 +- src/test/regress/sql/columnar_matview.sql | 2 +- src/test/regress/sql/columnar_memory.sql | 28 ++---- src/test/regress/sql/columnar_recursive.sql | 14 +-- src/test/regress/sql/columnar_rollback.sql | 2 +- .../regress/sql/columnar_test_helpers.sql | 75 +++++++++++++++ src/test/regress/sql/columnar_truncate.sql | 10 +- src/test/regress/sql/columnar_vacuum.sql | 16 ++-- src/test/regress/sql/columnar_zstd.sql | 2 +- 30 files changed, 261 insertions(+), 265 deletions(-) create mode 100644 src/test/regress/expected/columnar_test_helpers.out create mode 100644 src/test/regress/sql/columnar_test_helpers.sql diff --git a/src/backend/columnar/columnar_debug.c b/src/backend/columnar/columnar_debug.c index babdce5c3..2a772e7da 100644 --- a/src/backend/columnar/columnar_debug.c +++ b/src/backend/columnar/columnar_debug.c @@ -29,15 +29,15 @@ static void MemoryContextTotals(MemoryContext context, MemoryContextCounters *counters); -PG_FUNCTION_INFO_V1(column_store_memory_stats); +PG_FUNCTION_INFO_V1(columnar_store_memory_stats); /* - * column_store_memory_stats returns a record of 3 values: size of + * columnar_store_memory_stats returns a record of 3 values: size of * TopMemoryContext, TopTransactionContext, and Write State context. */ Datum -column_store_memory_stats(PG_FUNCTION_ARGS) +columnar_store_memory_stats(PG_FUNCTION_ARGS) { const int resultColumnCount = 3; diff --git a/src/test/regress/base_schedule b/src/test/regress/base_schedule index b77672c5c..34e770b5a 100644 --- a/src/test/regress/base_schedule +++ b/src/test/regress/base_schedule @@ -2,7 +2,7 @@ # Only run few basic tests to set up a testing environment # ---------- test: multi_cluster_management -test: multi_test_helpers multi_test_helpers_superuser multi_create_fdw +test: multi_test_helpers multi_test_helpers_superuser multi_create_fdw columnar_test_helpers test: multi_test_catalog_views test: multi_create_table multi_behavioral_analytics_create_table test: multi_create_table_superuser multi_behavioral_analytics_create_table_superuser diff --git a/src/test/regress/columnar_schedule b/src/test/regress/columnar_schedule index f9ac5d822..8c25f1384 100644 --- a/src/test/regress/columnar_schedule +++ b/src/test/regress/columnar_schedule @@ -1,5 +1,5 @@ test: multi_cluster_management -test: multi_test_helpers multi_test_helpers_superuser +test: multi_test_helpers multi_test_helpers_superuser columnar_test_helpers test: multi_test_catalog_views test: columnar_create diff --git a/src/test/regress/expected/columnar_create.out b/src/test/regress/expected/columnar_create.out index 23d21ac5e..5630ee811 100644 --- a/src/test/regress/expected/columnar_create.out +++ b/src/test/regress/expected/columnar_create.out @@ -24,86 +24,26 @@ SELECT count(*) FROM contestant; -- Should fail: unlogged tables not supported CREATE UNLOGGED TABLE columnar_unlogged(i int) USING columnar; ERROR: unlogged columnar tables are not supported --- --- Utility functions to be used throughout tests --- -CREATE FUNCTION columnar_relation_storageid(relid oid) RETURNS bigint - LANGUAGE C STABLE STRICT - AS 'citus', $$columnar_relation_storageid$$; -CREATE FUNCTION compression_type_supported(type text) RETURNS boolean -AS $$ -BEGIN - EXECUTE 'SET LOCAL columnar.compression TO ' || quote_literal(type); - return true; -EXCEPTION WHEN invalid_parameter_value THEN - return false; -END; -$$ LANGUAGE plpgsql; --- are chunk groups and chunks consistent? -CREATE view chunk_group_consistency AS -WITH a as ( - SELECT storage_id, stripe_num, chunk_group_num, min(value_count) as row_count - FROM columnar.chunk - GROUP BY 1,2,3 -), b as ( - SELECT storage_id, stripe_num, chunk_group_num, max(value_count) as row_count - FROM columnar.chunk - GROUP BY 1,2,3 -), c as ( - (TABLE a EXCEPT TABLE b) UNION (TABLE b EXCEPT TABLE a) UNION - (TABLE a EXCEPT TABLE columnar.chunk_group) UNION (TABLE columnar.chunk_group EXCEPT TABLE a) -), d as ( - SELECT storage_id, stripe_num, count(*) as chunk_group_count - FROM columnar.chunk_group - GROUP BY 1,2 -), e as ( - SELECT storage_id, stripe_num, chunk_group_count - FROM columnar.stripe -), f as ( - (TABLE d EXCEPT TABLE e) UNION (TABLE e EXCEPT TABLE d) -) -SELECT (SELECT count(*) = 0 FROM c) AND - (SELECT count(*) = 0 FROM f) as consistent; -CREATE FUNCTION columnar_metadata_has_storage_id(input_storage_id bigint) RETURNS boolean -AS $$ -DECLARE - union_storage_id_count integer; -BEGIN - SELECT count(*) INTO union_storage_id_count FROM - ( - SELECT storage_id FROM columnar.stripe UNION ALL - SELECT storage_id FROM columnar.chunk UNION ALL - SELECT storage_id FROM columnar.chunk_group - ) AS union_storage_id - WHERE storage_id=input_storage_id; - - IF union_storage_id_count > 0 THEN - RETURN true; - END IF; - - RETURN false; -END; -$$ LANGUAGE plpgsql; CREATE TABLE columnar_table_1 (a int) USING columnar; INSERT INTO columnar_table_1 VALUES (1); CREATE MATERIALIZED VIEW columnar_table_1_mv USING columnar AS SELECT * FROM columnar_table_1; -SELECT columnar_relation_storageid(oid) AS columnar_table_1_mv_storage_id +SELECT columnar_test_helpers.columnar_relation_storageid(oid) AS columnar_table_1_mv_storage_id FROM pg_class WHERE relname='columnar_table_1_mv' \gset -- test columnar_relation_set_new_filenode REFRESH MATERIALIZED VIEW columnar_table_1_mv; -SELECT columnar_metadata_has_storage_id(:columnar_table_1_mv_storage_id); +SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_table_1_mv_storage_id); columnar_metadata_has_storage_id --------------------------------------------------------------------- f (1 row) -SELECT columnar_relation_storageid(oid) AS columnar_table_1_storage_id +SELECT columnar_test_helpers.columnar_relation_storageid(oid) AS columnar_table_1_storage_id FROM pg_class WHERE relname='columnar_table_1' \gset BEGIN; -- test columnar_relation_nontransactional_truncate TRUNCATE columnar_table_1; - SELECT columnar_metadata_has_storage_id(:columnar_table_1_storage_id); + SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_table_1_storage_id); columnar_metadata_has_storage_id --------------------------------------------------------------------- f @@ -111,7 +51,7 @@ BEGIN; ROLLBACK; -- since we rollback'ed above xact, should return true -SELECT columnar_metadata_has_storage_id(:columnar_table_1_storage_id); +SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_table_1_storage_id); columnar_metadata_has_storage_id --------------------------------------------------------------------- t @@ -120,7 +60,7 @@ SELECT columnar_metadata_has_storage_id(:columnar_table_1_storage_id); -- test dropping columnar table DROP TABLE columnar_table_1 CASCADE; NOTICE: drop cascades to materialized view columnar_table_1_mv -SELECT columnar_metadata_has_storage_id(:columnar_table_1_storage_id); +SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_table_1_storage_id); columnar_metadata_has_storage_id --------------------------------------------------------------------- f @@ -131,7 +71,7 @@ SELECT columnar_metadata_has_storage_id(:columnar_table_1_storage_id); CREATE TEMPORARY TABLE columnar_temp(i int) USING columnar; -- reserve some chunks and a stripe INSERT INTO columnar_temp SELECT i FROM generate_series(1,5) i; -SELECT columnar_relation_storageid(oid) AS columnar_temp_storage_id +SELECT columnar_test_helpers.columnar_relation_storageid(oid) AS columnar_temp_storage_id FROM pg_class WHERE relname='columnar_temp' \gset \c - - - :master_port -- show that temporary table itself and it's metadata is removed @@ -141,7 +81,7 @@ SELECT COUNT(*)=0 FROM pg_class WHERE relname='columnar_temp'; t (1 row) -SELECT columnar_metadata_has_storage_id(:columnar_temp_storage_id); +SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_temp_storage_id); columnar_metadata_has_storage_id --------------------------------------------------------------------- f @@ -158,12 +98,12 @@ SELECT COUNT(*) FROM columnar_temp WHERE i < 5; 4 (1 row) -SELECT columnar_relation_storageid(oid) AS columnar_temp_storage_id +SELECT columnar_test_helpers.columnar_relation_storageid(oid) AS columnar_temp_storage_id FROM pg_class WHERE relname='columnar_temp' \gset BEGIN; DROP TABLE columnar_temp; -- show that we drop stripes properly - SELECT columnar_metadata_has_storage_id(:columnar_temp_storage_id); + SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_temp_storage_id); columnar_metadata_has_storage_id --------------------------------------------------------------------- f @@ -178,7 +118,7 @@ SELECT COUNT(*)=1 FROM pg_class WHERE relname='columnar_temp'; (1 row) -- show that we preserve the stripe of the temp columanar table after rollback -SELECT columnar_metadata_has_storage_id(:columnar_temp_storage_id); +SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_temp_storage_id); columnar_metadata_has_storage_id --------------------------------------------------------------------- t @@ -190,7 +130,7 @@ BEGIN; CREATE TEMPORARY TABLE columnar_temp(i int) USING columnar ON COMMIT DROP; -- force flushing stripe INSERT INTO columnar_temp SELECT i FROM generate_series(1,150000) i; - SELECT columnar_relation_storageid(oid) AS columnar_temp_storage_id + SELECT columnar_test_helpers.columnar_relation_storageid(oid) AS columnar_temp_storage_id FROM pg_class WHERE relname='columnar_temp' \gset COMMIT; -- make sure that table & it's stripe is dropped after commiting above xact @@ -200,7 +140,7 @@ SELECT COUNT(*)=0 FROM pg_class WHERE relname='columnar_temp'; t (1 row) -SELECT columnar_metadata_has_storage_id(:columnar_temp_storage_id); +SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_temp_storage_id); columnar_metadata_has_storage_id --------------------------------------------------------------------- f @@ -210,7 +150,7 @@ BEGIN; CREATE TEMPORARY TABLE columnar_temp(i int) USING columnar ON COMMIT DELETE ROWS; -- force flushing stripe INSERT INTO columnar_temp SELECT i FROM generate_series(1,150000) i; - SELECT columnar_relation_storageid(oid) AS columnar_temp_storage_id + SELECT columnar_test_helpers.columnar_relation_storageid(oid) AS columnar_temp_storage_id FROM pg_class WHERE relname='columnar_temp' \gset COMMIT; -- make sure that table is not dropped but it's rows's are deleted after commiting above xact @@ -227,7 +167,7 @@ SELECT COUNT(*)=0 FROM columnar_temp; (1 row) -- since we deleted all the rows, we shouldn't have any stripes for table -SELECT columnar_metadata_has_storage_id(:columnar_temp_storage_id); +SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_temp_storage_id); columnar_metadata_has_storage_id --------------------------------------------------------------------- f diff --git a/src/test/regress/expected/columnar_insert.out b/src/test/regress/expected/columnar_insert.out index d7cb90e1a..24a7b6de2 100644 --- a/src/test/regress/expected/columnar_insert.out +++ b/src/test/regress/expected/columnar_insert.out @@ -45,7 +45,7 @@ select count(*) from test_insert_command; 3 (1 row) -SELECT * FROM chunk_group_consistency; +SELECT * FROM columnar_test_helpers.chunk_group_consistency; consistent --------------------------------------------------------------------- t @@ -67,7 +67,7 @@ CREATE TABLE test_columnar_long_text(int_val int, text_val text) USING columnar; -- store long text in columnar table INSERT INTO test_columnar_long_text SELECT * FROM test_long_text; -SELECT * FROM chunk_group_consistency; +SELECT * FROM columnar_test_helpers.chunk_group_consistency; consistent --------------------------------------------------------------------- t @@ -141,7 +141,7 @@ FROM test_toast_columnar; 5004 | 5004 | 5004 | 5004 (1 row) -SELECT * FROM chunk_group_consistency; +SELECT * FROM columnar_test_helpers.chunk_group_consistency; consistent --------------------------------------------------------------------- t @@ -174,7 +174,7 @@ INSERT INTO zero_col_heap SELECT * FROM zero_col_heap; INSERT INTO zero_col_heap SELECT * FROM zero_col_heap; INSERT INTO zero_col SELECT * FROM zero_col_heap; SELECT relname, stripe_num, chunk_group_count, row_count FROM columnar.stripe a, pg_class b -WHERE columnar_relation_storageid(b.oid)=a.storage_id AND relname = 'zero_col' +WHERE columnar_test_helpers.columnar_relation_storageid(b.oid)=a.storage_id AND relname = 'zero_col' ORDER BY 1,2,3,4; relname | stripe_num | chunk_group_count | row_count --------------------------------------------------------------------- @@ -186,14 +186,14 @@ ORDER BY 1,2,3,4; (5 rows) SELECT relname, stripe_num, value_count FROM columnar.chunk a, pg_class b -WHERE columnar_relation_storageid(b.oid)=a.storage_id AND relname = 'zero_col' +WHERE columnar_test_helpers.columnar_relation_storageid(b.oid)=a.storage_id AND relname = 'zero_col' ORDER BY 1,2,3; relname | stripe_num | value_count --------------------------------------------------------------------- (0 rows) SELECT relname, stripe_num, chunk_group_num, row_count FROM columnar.chunk_group a, pg_class b -WHERE columnar_relation_storageid(b.oid)=a.storage_id AND relname = 'zero_col' +WHERE columnar_test_helpers.columnar_relation_storageid(b.oid)=a.storage_id AND relname = 'zero_col' ORDER BY 1,2,3,4; relname | stripe_num | chunk_group_num | row_count --------------------------------------------------------------------- diff --git a/src/test/regress/expected/columnar_lz4.out b/src/test/regress/expected/columnar_lz4.out index 2859277eb..fbfc4da7b 100644 --- a/src/test/regress/expected/columnar_lz4.out +++ b/src/test/regress/expected/columnar_lz4.out @@ -1,4 +1,4 @@ -SELECT compression_type_supported('lz4') AS lz4_supported \gset +SELECT columnar_test_helpers.compression_type_supported('lz4') AS lz4_supported \gset \if :lz4_supported \else \q diff --git a/src/test/regress/expected/columnar_lz4_0.out b/src/test/regress/expected/columnar_lz4_0.out index 30fee238c..2bbfabce3 100644 --- a/src/test/regress/expected/columnar_lz4_0.out +++ b/src/test/regress/expected/columnar_lz4_0.out @@ -1,4 +1,4 @@ -SELECT compression_type_supported('lz4') AS lz4_supported \gset +SELECT columnar_test_helpers.compression_type_supported('lz4') AS lz4_supported \gset \if :lz4_supported \else \q diff --git a/src/test/regress/expected/columnar_matview.out b/src/test/regress/expected/columnar_matview.out index f186bda2e..495cb0548 100644 --- a/src/test/regress/expected/columnar_matview.out +++ b/src/test/regress/expected/columnar_matview.out @@ -66,7 +66,7 @@ SELECT * FROM t_view a ORDER BY a; (6 rows) -- verify that we have created metadata entries for the materialized view -SELECT columnar_relation_storageid(oid) AS storageid +SELECT columnar_test_helpers.columnar_relation_storageid(oid) AS storageid FROM pg_class WHERE relname='t_view' \gset SELECT count(*) FROM columnar.stripe WHERE storage_id=:storageid; count diff --git a/src/test/regress/expected/columnar_memory.out b/src/test/regress/expected/columnar_memory.out index e3a94f322..117b8f900 100644 --- a/src/test/regress/expected/columnar_memory.out +++ b/src/test/regress/expected/columnar_memory.out @@ -3,35 +3,25 @@ -- CREATE SCHEMA columnar_memory; SET search_path TO 'columnar_memory'; -CREATE OR REPLACE FUNCTION column_store_memory_stats() - RETURNS TABLE(TopMemoryContext BIGINT, - TopTransactionContext BIGINT, - WriteStateContext BIGINT) - LANGUAGE C STRICT VOLATILE - AS 'citus', $$column_store_memory_stats$$; -CREATE FUNCTION top_memory_context_usage() - RETURNS BIGINT AS $$ - SELECT TopMemoryContext FROM column_store_memory_stats(); - $$ LANGUAGE SQL VOLATILE; SET columnar.stripe_row_limit TO 50000; SET columnar.compression TO 'pglz'; CREATE TABLE t (a int, tag text, memusage bigint) USING columnar; -- measure memory before doing writes SELECT TopMemoryContext as top_pre, WriteStateContext write_pre -FROM column_store_memory_stats() \gset +FROM columnar_test_helpers.columnar_store_memory_stats() \gset BEGIN; SET LOCAL client_min_messages TO DEBUG1; -- measure memory just before flushing 1st stripe INSERT INTO t SELECT i, 'first batch', -- sample memusage instead of recording everyr row for speed - CASE WHEN i % 100 = 0 THEN top_memory_context_usage() ELSE 0 END + CASE WHEN i % 100 = 0 THEN columnar_test_helpers.top_memory_context_usage() ELSE 0 END FROM generate_series(1, 49999) i; SELECT TopMemoryContext as top0, TopTransactionContext xact0, WriteStateContext write0 -FROM column_store_memory_stats() \gset +FROM columnar_test_helpers.columnar_store_memory_stats() \gset -- flush 1st stripe, and measure memory just before flushing 2nd stripe INSERT INTO t SELECT i, 'second batch', 0 /* no need to record memusage per row */ @@ -40,7 +30,7 @@ DEBUG: Flushing Stripe of size 50000 SELECT TopMemoryContext as top1, TopTransactionContext xact1, WriteStateContext write1 -FROM column_store_memory_stats() \gset +FROM columnar_test_helpers.columnar_store_memory_stats() \gset -- flush 2nd stripe, and measure memory just before flushing 3rd stripe INSERT INTO t SELECT i, 'third batch', 0 /* no need to record memusage per row */ @@ -49,12 +39,12 @@ DEBUG: Flushing Stripe of size 50000 SELECT TopMemoryContext as top2, TopTransactionContext xact2, WriteStateContext write2 -FROM column_store_memory_stats() \gset +FROM columnar_test_helpers.columnar_store_memory_stats() \gset -- insert a large batch INSERT INTO t SELECT i, 'large batch', -- sample memusage instead of recording everyr row for speed - CASE WHEN i % 100 = 0 THEN top_memory_context_usage() ELSE 0 END + CASE WHEN i % 100 = 0 THEN columnar_test_helpers.top_memory_context_usage() ELSE 0 END FROM generate_series(1, 100000) i; DEBUG: Flushing Stripe of size 50000 DEBUG: Flushing Stripe of size 50000 @@ -63,7 +53,7 @@ DEBUG: Flushing Stripe of size 49999 -- measure memory after doing writes SELECT TopMemoryContext as top_post, WriteStateContext write_post -FROM column_store_memory_stats() \gset +FROM columnar_test_helpers.columnar_store_memory_stats() \gset \x SELECT (1.0 * :top2/:top1 BETWEEN 0.99 AND 1.01) AS top_growth_ok, (1.0 * :xact1/:xact0 BETWEEN 0.99 AND 1.01) AND @@ -82,7 +72,7 @@ INSERT INTO t SELECT i, 'last batch', 0 /* no need to record memusage per row */ FROM generate_series(1, 50000) i; SELECT 1.0 * TopMemoryContext / :top_post BETWEEN 0.98 AND 1.02 AS top_growth_ok -FROM column_store_memory_stats(); +FROM columnar_test_helpers.columnar_store_memory_stats(); -[ RECORD 1 ]-+-- top_growth_ok | t diff --git a/src/test/regress/expected/columnar_recursive.out b/src/test/regress/expected/columnar_recursive.out index 2e3e2a30e..14f954a5a 100644 --- a/src/test/regress/expected/columnar_recursive.out +++ b/src/test/regress/expected/columnar_recursive.out @@ -12,7 +12,7 @@ INSERT INTO t2 SELECT i, f(i) FROM generate_series(1, 5) i; -- there are no subtransactions, so above statement should batch -- INSERTs inside the UDF and create on stripe per table. SELECT relname, count(*) FROM columnar.stripe a, pg_class b -WHERE columnar_relation_storageid(b.oid)=a.storage_id AND relname IN ('t1', 't2') +WHERE columnar_test_helpers.columnar_relation_storageid(b.oid)=a.storage_id AND relname IN ('t1', 't2') GROUP BY relname ORDER BY relname; relname | count @@ -72,7 +72,7 @@ SELECT * FROM t2; 5 | 6 (5 rows) -SELECT * FROM chunk_group_consistency; +SELECT * FROM columnar_test_helpers.chunk_group_consistency; consistent --------------------------------------------------------------------- t @@ -107,7 +107,7 @@ SELECT * FROM t2; 3 | 0 (3 rows) -SELECT * FROM chunk_group_consistency; +SELECT * FROM columnar_test_helpers.chunk_group_consistency; consistent --------------------------------------------------------------------- t @@ -137,7 +137,7 @@ SELECT * FROM t1 ORDER BY a, b; 5 | 10 (10 rows) -SELECT * FROM chunk_group_consistency; +SELECT * FROM columnar_test_helpers.chunk_group_consistency; consistent --------------------------------------------------------------------- t @@ -183,7 +183,7 @@ SELECT * FROM t2 ORDER BY a, b; 5 | 5 (5 rows) -SELECT * FROM chunk_group_consistency; +SELECT * FROM columnar_test_helpers.chunk_group_consistency; consistent --------------------------------------------------------------------- t @@ -228,7 +228,7 @@ SELECT * FROM t3 ORDER BY a, b; --------------------------------------------------------------------- (0 rows) -SELECT * FROM chunk_group_consistency; +SELECT * FROM columnar_test_helpers.chunk_group_consistency; consistent --------------------------------------------------------------------- t @@ -278,7 +278,7 @@ SELECT * FROM t1 ORDER BY a, b; 23 | 46 (10 rows) -SELECT * FROM chunk_group_consistency; +SELECT * FROM columnar_test_helpers.chunk_group_consistency; consistent --------------------------------------------------------------------- t diff --git a/src/test/regress/expected/columnar_rollback.out b/src/test/regress/expected/columnar_rollback.out index 2d68ff086..91c73bf63 100644 --- a/src/test/regress/expected/columnar_rollback.out +++ b/src/test/regress/expected/columnar_rollback.out @@ -4,7 +4,7 @@ CREATE TABLE t(a int, b int) USING columnar; CREATE VIEW t_stripes AS SELECT * FROM columnar.stripe a, pg_class b -WHERE a.storage_id = columnar_relation_storageid(b.oid) AND b.relname = 't'; +WHERE a.storage_id = columnar_test_helpers.columnar_relation_storageid(b.oid) AND b.relname = 't'; BEGIN; INSERT INTO t SELECT i, i+1 FROM generate_series(1, 10) i; ROLLBACK; diff --git a/src/test/regress/expected/columnar_test_helpers.out b/src/test/regress/expected/columnar_test_helpers.out new file mode 100644 index 000000000..b28086243 --- /dev/null +++ b/src/test/regress/expected/columnar_test_helpers.out @@ -0,0 +1,69 @@ +CREATE SCHEMA columnar_test_helpers; +SET search_path TO columnar_test_helpers; +CREATE FUNCTION columnar_relation_storageid(relid oid) RETURNS bigint + LANGUAGE C STABLE STRICT + AS 'citus', $$columnar_relation_storageid$$; +CREATE FUNCTION compression_type_supported(type text) RETURNS boolean +AS $$ +BEGIN + EXECUTE 'SET LOCAL columnar.compression TO ' || quote_literal(type); + return true; +EXCEPTION WHEN invalid_parameter_value THEN + return false; +END; +$$ LANGUAGE plpgsql; +-- are chunk groups and chunks consistent? +CREATE view chunk_group_consistency AS +WITH a as ( + SELECT storage_id, stripe_num, chunk_group_num, min(value_count) as row_count + FROM columnar.chunk + GROUP BY 1,2,3 +), b as ( + SELECT storage_id, stripe_num, chunk_group_num, max(value_count) as row_count + FROM columnar.chunk + GROUP BY 1,2,3 +), c as ( + (TABLE a EXCEPT TABLE b) UNION (TABLE b EXCEPT TABLE a) UNION + (TABLE a EXCEPT TABLE columnar.chunk_group) UNION (TABLE columnar.chunk_group EXCEPT TABLE a) +), d as ( + SELECT storage_id, stripe_num, count(*) as chunk_group_count + FROM columnar.chunk_group + GROUP BY 1,2 +), e as ( + SELECT storage_id, stripe_num, chunk_group_count + FROM columnar.stripe +), f as ( + (TABLE d EXCEPT TABLE e) UNION (TABLE e EXCEPT TABLE d) +) +SELECT (SELECT count(*) = 0 FROM c) AND + (SELECT count(*) = 0 FROM f) as consistent; +CREATE FUNCTION columnar_metadata_has_storage_id(input_storage_id bigint) RETURNS boolean +AS $$ +DECLARE + union_storage_id_count integer; +BEGIN + SELECT count(*) INTO union_storage_id_count FROM + ( + SELECT storage_id FROM columnar.stripe UNION ALL + SELECT storage_id FROM columnar.chunk UNION ALL + SELECT storage_id FROM columnar.chunk_group + ) AS union_storage_id + WHERE storage_id=input_storage_id; + + IF union_storage_id_count > 0 THEN + RETURN true; + END IF; + + RETURN false; +END; +$$ LANGUAGE plpgsql; +CREATE OR REPLACE FUNCTION columnar_store_memory_stats() + RETURNS TABLE(TopMemoryContext BIGINT, + TopTransactionContext BIGINT, + WriteStateContext BIGINT) + LANGUAGE C STRICT VOLATILE + AS 'citus', $$columnar_store_memory_stats$$; +CREATE FUNCTION top_memory_context_usage() + RETURNS BIGINT AS $$ + SELECT TopMemoryContext FROM columnar_test_helpers.columnar_store_memory_stats(); + $$ LANGUAGE SQL VOLATILE; diff --git a/src/test/regress/expected/columnar_truncate.out b/src/test/regress/expected/columnar_truncate.out index 1f1d7668f..55a496644 100644 --- a/src/test/regress/expected/columnar_truncate.out +++ b/src/test/regress/expected/columnar_truncate.out @@ -37,14 +37,14 @@ SELECT * FROM columnar_truncate_test; 10 | 10 (10 rows) -SELECT * FROM chunk_group_consistency; +SELECT * FROM columnar_test_helpers.chunk_group_consistency; consistent --------------------------------------------------------------------- t (1 row) TRUNCATE TABLE columnar_truncate_test; -SELECT * FROM chunk_group_consistency; +SELECT * FROM columnar_test_helpers.chunk_group_consistency; consistent --------------------------------------------------------------------- t @@ -130,7 +130,7 @@ SELECT * from columnar_truncate_test_regular; 20 | 20 (11 rows) -SELECT * FROM chunk_group_consistency; +SELECT * FROM columnar_test_helpers.chunk_group_consistency; consistent --------------------------------------------------------------------- t @@ -142,7 +142,7 @@ TRUNCATE TABLE columnar_truncate_test, columnar_truncate_test_regular, columnar_truncate_test_second, columnar_truncate_test; -SELECT * FROM chunk_group_consistency; +SELECT * FROM columnar_test_helpers.chunk_group_consistency; consistent --------------------------------------------------------------------- t @@ -291,7 +291,7 @@ SELECT count(*) FROM truncate_schema.truncate_tbl; (1 row) \c - :current_user -SELECT * FROM chunk_group_consistency; +SELECT * FROM columnar_test_helpers.chunk_group_consistency; consistent --------------------------------------------------------------------- t diff --git a/src/test/regress/expected/columnar_vacuum.out b/src/test/regress/expected/columnar_vacuum.out index b59680dd5..949220632 100644 --- a/src/test/regress/expected/columnar_vacuum.out +++ b/src/test/regress/expected/columnar_vacuum.out @@ -3,7 +3,7 @@ SELECT count(distinct storage_id) AS columnar_table_count FROM columnar.stripe \ CREATE TABLE t(a int, b int) USING columnar; CREATE VIEW t_stripes AS SELECT * FROM columnar.stripe a, pg_class b -WHERE a.storage_id = columnar_relation_storageid(b.oid) AND b.relname='t'; +WHERE a.storage_id = columnar_test_helpers.columnar_relation_storageid(b.oid) AND b.relname='t'; SELECT count(*) FROM t_stripes; count --------------------------------------------------------------------- @@ -27,7 +27,7 @@ SELECT count(*) FROM t_stripes; -- vacuum full should merge stripes together VACUUM FULL t; -SELECT * FROM chunk_group_consistency; +SELECT * FROM columnar_test_helpers.chunk_group_consistency; consistent --------------------------------------------------------------------- t @@ -66,7 +66,7 @@ SELECT count(*) FROM t_stripes; (1 row) VACUUM FULL t; -SELECT * FROM chunk_group_consistency; +SELECT * FROM columnar_test_helpers.chunk_group_consistency; consistent --------------------------------------------------------------------- t @@ -88,7 +88,7 @@ SELECT count(*) FROM t_stripes; ALTER TABLE t DROP COLUMN a; SELECT stripe_num, attr_num, chunk_group_num, minimum_value IS NULL, maximum_value IS NULL FROM columnar.chunk a, pg_class b -WHERE a.storage_id = columnar_relation_storageid(b.oid) AND b.relname='t' ORDER BY 1, 2, 3; +WHERE a.storage_id = columnar_test_helpers.columnar_relation_storageid(b.oid) AND b.relname='t' ORDER BY 1, 2, 3; stripe_num | attr_num | chunk_group_num | ?column? | ?column? --------------------------------------------------------------------- 1 | 1 | 0 | f | f @@ -102,7 +102,7 @@ WHERE a.storage_id = columnar_relation_storageid(b.oid) AND b.relname='t' ORDER VACUUM FULL t; SELECT stripe_num, attr_num, chunk_group_num, minimum_value IS NULL, maximum_value IS NULL FROM columnar.chunk a, pg_class b -WHERE a.storage_id = columnar_relation_storageid(b.oid) AND b.relname='t' ORDER BY 1, 2, 3; +WHERE a.storage_id = columnar_test_helpers.columnar_relation_storageid(b.oid) AND b.relname='t' ORDER BY 1, 2, 3; stripe_num | attr_num | chunk_group_num | ?column? | ?column? --------------------------------------------------------------------- 1 | 1 | 0 | t | t @@ -215,7 +215,7 @@ compression rate: 1.25x total row count: 5530, stripe count: 5, average rows per stripe: 1106 chunk count: 7, containing data for dropped columns: 0, none compressed: 5, pglz compressed: 2 -SELECT * FROM chunk_group_consistency; +SELECT * FROM columnar_test_helpers.chunk_group_consistency; consistent --------------------------------------------------------------------- t @@ -257,7 +257,7 @@ compression rate: 1.96x total row count: 7030, stripe count: 4, average rows per stripe: 1757 chunk count: 8, containing data for dropped columns: 0, none compressed: 2, pglz compressed: 6 -SELECT * FROM chunk_group_consistency; +SELECT * FROM columnar_test_helpers.chunk_group_consistency; consistent --------------------------------------------------------------------- t @@ -286,7 +286,7 @@ compression rate: 33.71x total row count: 1000000, stripe count: 1, average rows per stripe: 1000000 chunk count: 30, containing data for dropped columns: 0, pglz compressed: 30 -SELECT * FROM chunk_group_consistency; +SELECT * FROM columnar_test_helpers.chunk_group_consistency; consistent --------------------------------------------------------------------- t diff --git a/src/test/regress/expected/columnar_zstd.out b/src/test/regress/expected/columnar_zstd.out index 1af9a8de2..ab245ba76 100644 --- a/src/test/regress/expected/columnar_zstd.out +++ b/src/test/regress/expected/columnar_zstd.out @@ -1,4 +1,4 @@ -SELECT compression_type_supported('zstd') AS zstd_supported \gset +SELECT columnar_test_helpers.compression_type_supported('zstd') AS zstd_supported \gset \if :zstd_supported \else \q diff --git a/src/test/regress/expected/columnar_zstd_0.out b/src/test/regress/expected/columnar_zstd_0.out index 08128a713..9d34bf7b8 100644 --- a/src/test/regress/expected/columnar_zstd_0.out +++ b/src/test/regress/expected/columnar_zstd_0.out @@ -1,4 +1,4 @@ -SELECT compression_type_supported('zstd') AS zstd_supported \gset +SELECT columnar_test_helpers.compression_type_supported('zstd') AS zstd_supported \gset \if :zstd_supported \else \q diff --git a/src/test/regress/input/columnar_load.source b/src/test/regress/input/columnar_load.source index 41653f3c4..b4b72926c 100644 --- a/src/test/regress/input/columnar_load.source +++ b/src/test/regress/input/columnar_load.source @@ -43,6 +43,6 @@ speed of light,2.997e8 SELECT * FROM famous_constants ORDER BY id, name; -SELECT * FROM chunk_group_consistency; +SELECT * FROM columnar_test_helpers.chunk_group_consistency; DROP TABLE famous_constants; diff --git a/src/test/regress/minimal_schedule b/src/test/regress/minimal_schedule index e231fd895..982059e75 100644 --- a/src/test/regress/minimal_schedule +++ b/src/test/regress/minimal_schedule @@ -1,3 +1,3 @@ test: multi_cluster_management -test: multi_test_helpers multi_test_helpers_superuser +test: multi_test_helpers multi_test_helpers_superuser columnar_test_helpers test: multi_test_catalog_views diff --git a/src/test/regress/output/columnar_load.source b/src/test/regress/output/columnar_load.source index f671b45a8..c58f0bfb0 100644 --- a/src/test/regress/output/columnar_load.source +++ b/src/test/regress/output/columnar_load.source @@ -27,7 +27,7 @@ CREATE TABLE famous_constants (id int, name text, value real) COPY famous_constants (value, name, id) FROM STDIN WITH CSV; COPY famous_constants (name, value) FROM STDIN WITH CSV; SELECT * FROM famous_constants ORDER BY id, name; - id | name | value + id | name | value ----+----------------+----------- 1 | pi | 3.141 2 | e | 2.718 @@ -39,7 +39,7 @@ SELECT * FROM famous_constants ORDER BY id, name; | speed of light | 2.997e+08 (8 rows) -SELECT * FROM chunk_group_consistency; +SELECT * FROM columnar_test_helpers.chunk_group_consistency; consistent --------------------------------------------------------------------- t diff --git a/src/test/regress/sql/columnar_create.sql b/src/test/regress/sql/columnar_create.sql index 2a3263233..41ffc1053 100644 --- a/src/test/regress/sql/columnar_create.sql +++ b/src/test/regress/sql/columnar_create.sql @@ -24,100 +24,34 @@ SELECT count(*) FROM contestant; -- Should fail: unlogged tables not supported CREATE UNLOGGED TABLE columnar_unlogged(i int) USING columnar; --- --- Utility functions to be used throughout tests --- - -CREATE FUNCTION columnar_relation_storageid(relid oid) RETURNS bigint - LANGUAGE C STABLE STRICT - AS 'citus', $$columnar_relation_storageid$$; - -CREATE FUNCTION compression_type_supported(type text) RETURNS boolean -AS $$ -BEGIN - EXECUTE 'SET LOCAL columnar.compression TO ' || quote_literal(type); - return true; -EXCEPTION WHEN invalid_parameter_value THEN - return false; -END; -$$ LANGUAGE plpgsql; - - --- are chunk groups and chunks consistent? -CREATE view chunk_group_consistency AS -WITH a as ( - SELECT storage_id, stripe_num, chunk_group_num, min(value_count) as row_count - FROM columnar.chunk - GROUP BY 1,2,3 -), b as ( - SELECT storage_id, stripe_num, chunk_group_num, max(value_count) as row_count - FROM columnar.chunk - GROUP BY 1,2,3 -), c as ( - (TABLE a EXCEPT TABLE b) UNION (TABLE b EXCEPT TABLE a) UNION - (TABLE a EXCEPT TABLE columnar.chunk_group) UNION (TABLE columnar.chunk_group EXCEPT TABLE a) -), d as ( - SELECT storage_id, stripe_num, count(*) as chunk_group_count - FROM columnar.chunk_group - GROUP BY 1,2 -), e as ( - SELECT storage_id, stripe_num, chunk_group_count - FROM columnar.stripe -), f as ( - (TABLE d EXCEPT TABLE e) UNION (TABLE e EXCEPT TABLE d) -) -SELECT (SELECT count(*) = 0 FROM c) AND - (SELECT count(*) = 0 FROM f) as consistent; - -CREATE FUNCTION columnar_metadata_has_storage_id(input_storage_id bigint) RETURNS boolean -AS $$ -DECLARE - union_storage_id_count integer; -BEGIN - SELECT count(*) INTO union_storage_id_count FROM - ( - SELECT storage_id FROM columnar.stripe UNION ALL - SELECT storage_id FROM columnar.chunk UNION ALL - SELECT storage_id FROM columnar.chunk_group - ) AS union_storage_id - WHERE storage_id=input_storage_id; - - IF union_storage_id_count > 0 THEN - RETURN true; - END IF; - - RETURN false; -END; -$$ LANGUAGE plpgsql; - CREATE TABLE columnar_table_1 (a int) USING columnar; INSERT INTO columnar_table_1 VALUES (1); CREATE MATERIALIZED VIEW columnar_table_1_mv USING columnar AS SELECT * FROM columnar_table_1; -SELECT columnar_relation_storageid(oid) AS columnar_table_1_mv_storage_id +SELECT columnar_test_helpers.columnar_relation_storageid(oid) AS columnar_table_1_mv_storage_id FROM pg_class WHERE relname='columnar_table_1_mv' \gset -- test columnar_relation_set_new_filenode REFRESH MATERIALIZED VIEW columnar_table_1_mv; -SELECT columnar_metadata_has_storage_id(:columnar_table_1_mv_storage_id); +SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_table_1_mv_storage_id); -SELECT columnar_relation_storageid(oid) AS columnar_table_1_storage_id +SELECT columnar_test_helpers.columnar_relation_storageid(oid) AS columnar_table_1_storage_id FROM pg_class WHERE relname='columnar_table_1' \gset BEGIN; -- test columnar_relation_nontransactional_truncate TRUNCATE columnar_table_1; - SELECT columnar_metadata_has_storage_id(:columnar_table_1_storage_id); + SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_table_1_storage_id); ROLLBACK; -- since we rollback'ed above xact, should return true -SELECT columnar_metadata_has_storage_id(:columnar_table_1_storage_id); +SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_table_1_storage_id); -- test dropping columnar table DROP TABLE columnar_table_1 CASCADE; -SELECT columnar_metadata_has_storage_id(:columnar_table_1_storage_id); +SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_table_1_storage_id); -- test temporary columnar tables @@ -127,14 +61,14 @@ CREATE TEMPORARY TABLE columnar_temp(i int) USING columnar; -- reserve some chunks and a stripe INSERT INTO columnar_temp SELECT i FROM generate_series(1,5) i; -SELECT columnar_relation_storageid(oid) AS columnar_temp_storage_id +SELECT columnar_test_helpers.columnar_relation_storageid(oid) AS columnar_temp_storage_id FROM pg_class WHERE relname='columnar_temp' \gset \c - - - :master_port -- show that temporary table itself and it's metadata is removed SELECT COUNT(*)=0 FROM pg_class WHERE relname='columnar_temp'; -SELECT columnar_metadata_has_storage_id(:columnar_temp_storage_id); +SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_temp_storage_id); -- connect to another session and create a temp table with same name CREATE TEMPORARY TABLE columnar_temp(i int) USING columnar; @@ -145,19 +79,19 @@ INSERT INTO columnar_temp SELECT i FROM generate_series(1,5) i; -- test basic select SELECT COUNT(*) FROM columnar_temp WHERE i < 5; -SELECT columnar_relation_storageid(oid) AS columnar_temp_storage_id +SELECT columnar_test_helpers.columnar_relation_storageid(oid) AS columnar_temp_storage_id FROM pg_class WHERE relname='columnar_temp' \gset BEGIN; DROP TABLE columnar_temp; -- show that we drop stripes properly - SELECT columnar_metadata_has_storage_id(:columnar_temp_storage_id); + SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_temp_storage_id); ROLLBACK; -- make sure that table is not dropped yet since we rollbacked above xact SELECT COUNT(*)=1 FROM pg_class WHERE relname='columnar_temp'; -- show that we preserve the stripe of the temp columanar table after rollback -SELECT columnar_metadata_has_storage_id(:columnar_temp_storage_id); +SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_temp_storage_id); -- drop it for next tests DROP TABLE columnar_temp; @@ -167,20 +101,20 @@ BEGIN; -- force flushing stripe INSERT INTO columnar_temp SELECT i FROM generate_series(1,150000) i; - SELECT columnar_relation_storageid(oid) AS columnar_temp_storage_id + SELECT columnar_test_helpers.columnar_relation_storageid(oid) AS columnar_temp_storage_id FROM pg_class WHERE relname='columnar_temp' \gset COMMIT; -- make sure that table & it's stripe is dropped after commiting above xact SELECT COUNT(*)=0 FROM pg_class WHERE relname='columnar_temp'; -SELECT columnar_metadata_has_storage_id(:columnar_temp_storage_id); +SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_temp_storage_id); BEGIN; CREATE TEMPORARY TABLE columnar_temp(i int) USING columnar ON COMMIT DELETE ROWS; -- force flushing stripe INSERT INTO columnar_temp SELECT i FROM generate_series(1,150000) i; - SELECT columnar_relation_storageid(oid) AS columnar_temp_storage_id + SELECT columnar_test_helpers.columnar_relation_storageid(oid) AS columnar_temp_storage_id FROM pg_class WHERE relname='columnar_temp' \gset COMMIT; @@ -188,4 +122,4 @@ COMMIT; SELECT COUNT(*)=1 FROM pg_class WHERE relname='columnar_temp'; SELECT COUNT(*)=0 FROM columnar_temp; -- since we deleted all the rows, we shouldn't have any stripes for table -SELECT columnar_metadata_has_storage_id(:columnar_temp_storage_id); +SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_temp_storage_id); diff --git a/src/test/regress/sql/columnar_insert.sql b/src/test/regress/sql/columnar_insert.sql index 4a5c8c8e3..c5ffe73b2 100644 --- a/src/test/regress/sql/columnar_insert.sql +++ b/src/test/regress/sql/columnar_insert.sql @@ -22,7 +22,7 @@ select count(*) from test_insert_command_data; insert into test_insert_command select * from test_insert_command_data; select count(*) from test_insert_command; -SELECT * FROM chunk_group_consistency; +SELECT * FROM columnar_test_helpers.chunk_group_consistency; drop table test_insert_command_data; drop table test_insert_command; @@ -45,7 +45,7 @@ USING columnar; -- store long text in columnar table INSERT INTO test_columnar_long_text SELECT * FROM test_long_text; -SELECT * FROM chunk_group_consistency; +SELECT * FROM columnar_test_helpers.chunk_group_consistency; -- drop source table to remove original text from toast DROP TABLE test_long_text; @@ -99,7 +99,7 @@ SELECT pg_column_size(external), pg_column_size(extended) FROM test_toast_columnar; -SELECT * FROM chunk_group_consistency; +SELECT * FROM columnar_test_helpers.chunk_group_consistency; DROP TABLE test_toast_row; DROP TABLE test_toast_columnar; @@ -129,15 +129,15 @@ INSERT INTO zero_col_heap SELECT * FROM zero_col_heap; INSERT INTO zero_col SELECT * FROM zero_col_heap; SELECT relname, stripe_num, chunk_group_count, row_count FROM columnar.stripe a, pg_class b -WHERE columnar_relation_storageid(b.oid)=a.storage_id AND relname = 'zero_col' +WHERE columnar_test_helpers.columnar_relation_storageid(b.oid)=a.storage_id AND relname = 'zero_col' ORDER BY 1,2,3,4; SELECT relname, stripe_num, value_count FROM columnar.chunk a, pg_class b -WHERE columnar_relation_storageid(b.oid)=a.storage_id AND relname = 'zero_col' +WHERE columnar_test_helpers.columnar_relation_storageid(b.oid)=a.storage_id AND relname = 'zero_col' ORDER BY 1,2,3; SELECT relname, stripe_num, chunk_group_num, row_count FROM columnar.chunk_group a, pg_class b -WHERE columnar_relation_storageid(b.oid)=a.storage_id AND relname = 'zero_col' +WHERE columnar_test_helpers.columnar_relation_storageid(b.oid)=a.storage_id AND relname = 'zero_col' ORDER BY 1,2,3,4; DROP TABLE zero_col; diff --git a/src/test/regress/sql/columnar_lz4.sql b/src/test/regress/sql/columnar_lz4.sql index a174d047f..8f9bd5edd 100644 --- a/src/test/regress/sql/columnar_lz4.sql +++ b/src/test/regress/sql/columnar_lz4.sql @@ -1,4 +1,4 @@ -SELECT compression_type_supported('lz4') AS lz4_supported \gset +SELECT columnar_test_helpers.compression_type_supported('lz4') AS lz4_supported \gset \if :lz4_supported \else \q diff --git a/src/test/regress/sql/columnar_matview.sql b/src/test/regress/sql/columnar_matview.sql index 22c59f6a3..ff526d1c0 100644 --- a/src/test/regress/sql/columnar_matview.sql +++ b/src/test/regress/sql/columnar_matview.sql @@ -35,7 +35,7 @@ WHERE regclass = 't_view'::regclass; SELECT * FROM t_view a ORDER BY a; -- verify that we have created metadata entries for the materialized view -SELECT columnar_relation_storageid(oid) AS storageid +SELECT columnar_test_helpers.columnar_relation_storageid(oid) AS storageid FROM pg_class WHERE relname='t_view' \gset SELECT count(*) FROM columnar.stripe WHERE storage_id=:storageid; diff --git a/src/test/regress/sql/columnar_memory.sql b/src/test/regress/sql/columnar_memory.sql index a5a01b773..b5d251644 100644 --- a/src/test/regress/sql/columnar_memory.sql +++ b/src/test/regress/sql/columnar_memory.sql @@ -5,18 +5,6 @@ CREATE SCHEMA columnar_memory; SET search_path TO 'columnar_memory'; -CREATE OR REPLACE FUNCTION column_store_memory_stats() - RETURNS TABLE(TopMemoryContext BIGINT, - TopTransactionContext BIGINT, - WriteStateContext BIGINT) - LANGUAGE C STRICT VOLATILE - AS 'citus', $$column_store_memory_stats$$; - -CREATE FUNCTION top_memory_context_usage() - RETURNS BIGINT AS $$ - SELECT TopMemoryContext FROM column_store_memory_stats(); - $$ LANGUAGE SQL VOLATILE; - SET columnar.stripe_row_limit TO 50000; SET columnar.compression TO 'pglz'; CREATE TABLE t (a int, tag text, memusage bigint) USING columnar; @@ -24,7 +12,7 @@ CREATE TABLE t (a int, tag text, memusage bigint) USING columnar; -- measure memory before doing writes SELECT TopMemoryContext as top_pre, WriteStateContext write_pre -FROM column_store_memory_stats() \gset +FROM columnar_test_helpers.columnar_store_memory_stats() \gset BEGIN; SET LOCAL client_min_messages TO DEBUG1; @@ -33,12 +21,12 @@ SET LOCAL client_min_messages TO DEBUG1; INSERT INTO t SELECT i, 'first batch', -- sample memusage instead of recording everyr row for speed - CASE WHEN i % 100 = 0 THEN top_memory_context_usage() ELSE 0 END + CASE WHEN i % 100 = 0 THEN columnar_test_helpers.top_memory_context_usage() ELSE 0 END FROM generate_series(1, 49999) i; SELECT TopMemoryContext as top0, TopTransactionContext xact0, WriteStateContext write0 -FROM column_store_memory_stats() \gset +FROM columnar_test_helpers.columnar_store_memory_stats() \gset -- flush 1st stripe, and measure memory just before flushing 2nd stripe INSERT INTO t @@ -47,7 +35,7 @@ INSERT INTO t SELECT TopMemoryContext as top1, TopTransactionContext xact1, WriteStateContext write1 -FROM column_store_memory_stats() \gset +FROM columnar_test_helpers.columnar_store_memory_stats() \gset -- flush 2nd stripe, and measure memory just before flushing 3rd stripe INSERT INTO t @@ -56,13 +44,13 @@ INSERT INTO t SELECT TopMemoryContext as top2, TopTransactionContext xact2, WriteStateContext write2 -FROM column_store_memory_stats() \gset +FROM columnar_test_helpers.columnar_store_memory_stats() \gset -- insert a large batch INSERT INTO t SELECT i, 'large batch', -- sample memusage instead of recording everyr row for speed - CASE WHEN i % 100 = 0 THEN top_memory_context_usage() ELSE 0 END + CASE WHEN i % 100 = 0 THEN columnar_test_helpers.top_memory_context_usage() ELSE 0 END FROM generate_series(1, 100000) i; COMMIT; @@ -70,7 +58,7 @@ COMMIT; -- measure memory after doing writes SELECT TopMemoryContext as top_post, WriteStateContext write_post -FROM column_store_memory_stats() \gset +FROM columnar_test_helpers.columnar_store_memory_stats() \gset \x SELECT (1.0 * :top2/:top1 BETWEEN 0.99 AND 1.01) AS top_growth_ok, @@ -86,7 +74,7 @@ INSERT INTO t FROM generate_series(1, 50000) i; SELECT 1.0 * TopMemoryContext / :top_post BETWEEN 0.98 AND 1.02 AS top_growth_ok -FROM column_store_memory_stats(); +FROM columnar_test_helpers.columnar_store_memory_stats(); -- before this change, max mem usage while executing inserts was 28MB and -- with this change it's less than 8MB. diff --git a/src/test/regress/sql/columnar_recursive.sql b/src/test/regress/sql/columnar_recursive.sql index b8f3438b5..ada6167f6 100644 --- a/src/test/regress/sql/columnar_recursive.sql +++ b/src/test/regress/sql/columnar_recursive.sql @@ -16,7 +16,7 @@ INSERT INTO t2 SELECT i, f(i) FROM generate_series(1, 5) i; -- there are no subtransactions, so above statement should batch -- INSERTs inside the UDF and create on stripe per table. SELECT relname, count(*) FROM columnar.stripe a, pg_class b -WHERE columnar_relation_storageid(b.oid)=a.storage_id AND relname IN ('t1', 't2') +WHERE columnar_test_helpers.columnar_relation_storageid(b.oid)=a.storage_id AND relname IN ('t1', 't2') GROUP BY relname ORDER BY relname; @@ -39,7 +39,7 @@ INSERT INTO t2 SELECT t.a, t.a+1 FROM t; SELECT * FROM t1; SELECT * FROM t2; -SELECT * FROM chunk_group_consistency; +SELECT * FROM columnar_test_helpers.chunk_group_consistency; TRUNCATE t1; TRUNCATE t2; @@ -57,7 +57,7 @@ INSERT INTO t2 SELECT i, (select count(*) from t1) FROM generate_series(1, 3) i; SELECT * FROM t1; SELECT * FROM t2; -SELECT * FROM chunk_group_consistency; +SELECT * FROM columnar_test_helpers.chunk_group_consistency; TRUNCATE t1; TRUNCATE t2; @@ -72,7 +72,7 @@ INSERT INTO t1 SELECT t.a, t.a+1 FROM t; SELECT * FROM t1 ORDER BY a, b; -SELECT * FROM chunk_group_consistency; +SELECT * FROM columnar_test_helpers.chunk_group_consistency; TRUNCATE t1; TRUNCATE t2; @@ -105,7 +105,7 @@ INSERT INTO t4 SELECT i, g2(i) FROM generate_series(1, 5) i; SELECT * FROM t2 ORDER BY a, b; -SELECT * FROM chunk_group_consistency; +SELECT * FROM columnar_test_helpers.chunk_group_consistency; TRUNCATE t1, t2, t3, t4; @@ -121,7 +121,7 @@ SELECT * FROM t3 ORDER BY a, b; ((table t1) except (table t3)) union ((table t3) except (table t1)); ((table t2) except (table t4)) union ((table t4) except (table t2)); -SELECT * FROM chunk_group_consistency; +SELECT * FROM columnar_test_helpers.chunk_group_consistency; DROP FUNCTION g(int), g2(int); TRUNCATE t1, t2, t3, t4; @@ -148,7 +148,7 @@ SELECT f(0), f(20); SELECT * FROM t1 ORDER BY a, b; -SELECT * FROM chunk_group_consistency; +SELECT * FROM columnar_test_helpers.chunk_group_consistency; DROP FUNCTION f(int); DROP TABLE t1, t2, t3, t4; diff --git a/src/test/regress/sql/columnar_rollback.sql b/src/test/regress/sql/columnar_rollback.sql index 3cf7cecf5..3022ab906 100644 --- a/src/test/regress/sql/columnar_rollback.sql +++ b/src/test/regress/sql/columnar_rollback.sql @@ -6,7 +6,7 @@ CREATE TABLE t(a int, b int) USING columnar; CREATE VIEW t_stripes AS SELECT * FROM columnar.stripe a, pg_class b -WHERE a.storage_id = columnar_relation_storageid(b.oid) AND b.relname = 't'; +WHERE a.storage_id = columnar_test_helpers.columnar_relation_storageid(b.oid) AND b.relname = 't'; BEGIN; INSERT INTO t SELECT i, i+1 FROM generate_series(1, 10) i; diff --git a/src/test/regress/sql/columnar_test_helpers.sql b/src/test/regress/sql/columnar_test_helpers.sql new file mode 100644 index 000000000..d240a98d8 --- /dev/null +++ b/src/test/regress/sql/columnar_test_helpers.sql @@ -0,0 +1,75 @@ +CREATE SCHEMA columnar_test_helpers; +SET search_path TO columnar_test_helpers; + +CREATE FUNCTION columnar_relation_storageid(relid oid) RETURNS bigint + LANGUAGE C STABLE STRICT + AS 'citus', $$columnar_relation_storageid$$; + +CREATE FUNCTION compression_type_supported(type text) RETURNS boolean +AS $$ +BEGIN + EXECUTE 'SET LOCAL columnar.compression TO ' || quote_literal(type); + return true; +EXCEPTION WHEN invalid_parameter_value THEN + return false; +END; +$$ LANGUAGE plpgsql; + +-- are chunk groups and chunks consistent? +CREATE view chunk_group_consistency AS +WITH a as ( + SELECT storage_id, stripe_num, chunk_group_num, min(value_count) as row_count + FROM columnar.chunk + GROUP BY 1,2,3 +), b as ( + SELECT storage_id, stripe_num, chunk_group_num, max(value_count) as row_count + FROM columnar.chunk + GROUP BY 1,2,3 +), c as ( + (TABLE a EXCEPT TABLE b) UNION (TABLE b EXCEPT TABLE a) UNION + (TABLE a EXCEPT TABLE columnar.chunk_group) UNION (TABLE columnar.chunk_group EXCEPT TABLE a) +), d as ( + SELECT storage_id, stripe_num, count(*) as chunk_group_count + FROM columnar.chunk_group + GROUP BY 1,2 +), e as ( + SELECT storage_id, stripe_num, chunk_group_count + FROM columnar.stripe +), f as ( + (TABLE d EXCEPT TABLE e) UNION (TABLE e EXCEPT TABLE d) +) +SELECT (SELECT count(*) = 0 FROM c) AND + (SELECT count(*) = 0 FROM f) as consistent; + +CREATE FUNCTION columnar_metadata_has_storage_id(input_storage_id bigint) RETURNS boolean +AS $$ +DECLARE + union_storage_id_count integer; +BEGIN + SELECT count(*) INTO union_storage_id_count FROM + ( + SELECT storage_id FROM columnar.stripe UNION ALL + SELECT storage_id FROM columnar.chunk UNION ALL + SELECT storage_id FROM columnar.chunk_group + ) AS union_storage_id + WHERE storage_id=input_storage_id; + + IF union_storage_id_count > 0 THEN + RETURN true; + END IF; + + RETURN false; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION columnar_store_memory_stats() + RETURNS TABLE(TopMemoryContext BIGINT, + TopTransactionContext BIGINT, + WriteStateContext BIGINT) + LANGUAGE C STRICT VOLATILE + AS 'citus', $$columnar_store_memory_stats$$; + +CREATE FUNCTION top_memory_context_usage() + RETURNS BIGINT AS $$ + SELECT TopMemoryContext FROM columnar_test_helpers.columnar_store_memory_stats(); + $$ LANGUAGE SQL VOLATILE; diff --git a/src/test/regress/sql/columnar_truncate.sql b/src/test/regress/sql/columnar_truncate.sql index c34dcf036..701b41ceb 100644 --- a/src/test/regress/sql/columnar_truncate.sql +++ b/src/test/regress/sql/columnar_truncate.sql @@ -25,11 +25,11 @@ set columnar.compression to default; -- query rows SELECT * FROM columnar_truncate_test; -SELECT * FROM chunk_group_consistency; +SELECT * FROM columnar_test_helpers.chunk_group_consistency; TRUNCATE TABLE columnar_truncate_test; -SELECT * FROM chunk_group_consistency; +SELECT * FROM columnar_test_helpers.chunk_group_consistency; SELECT * FROM columnar_truncate_test; @@ -51,7 +51,7 @@ SELECT * from columnar_truncate_test_second; SELECT * from columnar_truncate_test_regular; -SELECT * FROM chunk_group_consistency; +SELECT * FROM columnar_test_helpers.chunk_group_consistency; -- make sure multi truncate works -- notice that the same table might be repeated @@ -60,7 +60,7 @@ TRUNCATE TABLE columnar_truncate_test, columnar_truncate_test_second, columnar_truncate_test; -SELECT * FROM chunk_group_consistency; +SELECT * FROM columnar_test_helpers.chunk_group_consistency; SELECT * from columnar_truncate_test; SELECT * from columnar_truncate_test_second; @@ -144,7 +144,7 @@ TRUNCATE TABLE truncate_schema.truncate_tbl; SELECT count(*) FROM truncate_schema.truncate_tbl; \c - :current_user -SELECT * FROM chunk_group_consistency; +SELECT * FROM columnar_test_helpers.chunk_group_consistency; -- cleanup DROP SCHEMA truncate_schema CASCADE; diff --git a/src/test/regress/sql/columnar_vacuum.sql b/src/test/regress/sql/columnar_vacuum.sql index 944213a3f..29babd84e 100644 --- a/src/test/regress/sql/columnar_vacuum.sql +++ b/src/test/regress/sql/columnar_vacuum.sql @@ -6,7 +6,7 @@ CREATE TABLE t(a int, b int) USING columnar; CREATE VIEW t_stripes AS SELECT * FROM columnar.stripe a, pg_class b -WHERE a.storage_id = columnar_relation_storageid(b.oid) AND b.relname='t'; +WHERE a.storage_id = columnar_test_helpers.columnar_relation_storageid(b.oid) AND b.relname='t'; SELECT count(*) FROM t_stripes; @@ -20,7 +20,7 @@ SELECT count(*) FROM t_stripes; -- vacuum full should merge stripes together VACUUM FULL t; -SELECT * FROM chunk_group_consistency; +SELECT * FROM columnar_test_helpers.chunk_group_consistency; SELECT sum(a), sum(b) FROM t; SELECT count(*) FROM t_stripes; @@ -34,7 +34,7 @@ SELECT count(*) FROM t_stripes; VACUUM FULL t; -SELECT * FROM chunk_group_consistency; +SELECT * FROM columnar_test_helpers.chunk_group_consistency; SELECT sum(a), sum(b) FROM t; SELECT count(*) FROM t_stripes; @@ -44,13 +44,13 @@ ALTER TABLE t DROP COLUMN a; SELECT stripe_num, attr_num, chunk_group_num, minimum_value IS NULL, maximum_value IS NULL FROM columnar.chunk a, pg_class b -WHERE a.storage_id = columnar_relation_storageid(b.oid) AND b.relname='t' ORDER BY 1, 2, 3; +WHERE a.storage_id = columnar_test_helpers.columnar_relation_storageid(b.oid) AND b.relname='t' ORDER BY 1, 2, 3; VACUUM FULL t; SELECT stripe_num, attr_num, chunk_group_num, minimum_value IS NULL, maximum_value IS NULL FROM columnar.chunk a, pg_class b -WHERE a.storage_id = columnar_relation_storageid(b.oid) AND b.relname='t' ORDER BY 1, 2, 3; +WHERE a.storage_id = columnar_test_helpers.columnar_relation_storageid(b.oid) AND b.relname='t' ORDER BY 1, 2, 3; -- Make sure we cleaned-up the transient table metadata after VACUUM FULL commands SELECT count(distinct storage_id) - :columnar_table_count FROM columnar.stripe; @@ -96,7 +96,7 @@ COMMIT; VACUUM VERBOSE t; -SELECT * FROM chunk_group_consistency; +SELECT * FROM columnar_test_helpers.chunk_group_consistency; SELECT count(*) FROM t; @@ -114,7 +114,7 @@ SELECT alter_columnar_table_set('t', compression => 'pglz'); VACUUM FULL t; VACUUM VERBOSE t; -SELECT * FROM chunk_group_consistency; +SELECT * FROM columnar_test_helpers.chunk_group_consistency; DROP TABLE t; DROP VIEW t_stripes; @@ -131,6 +131,6 @@ INSERT INTO t SELECT 1, 'a', 'xyz' FROM generate_series(1, 1000000) i; VACUUM VERBOSE t; -SELECT * FROM chunk_group_consistency; +SELECT * FROM columnar_test_helpers.chunk_group_consistency; DROP TABLE t; diff --git a/src/test/regress/sql/columnar_zstd.sql b/src/test/regress/sql/columnar_zstd.sql index fba90f77b..ab8ed1b31 100644 --- a/src/test/regress/sql/columnar_zstd.sql +++ b/src/test/regress/sql/columnar_zstd.sql @@ -1,4 +1,4 @@ -SELECT compression_type_supported('zstd') AS zstd_supported \gset +SELECT columnar_test_helpers.compression_type_supported('zstd') AS zstd_supported \gset \if :zstd_supported \else \q