mirror of https://github.com/citusdata/citus.git
Move columnar test helpers to a separate file (#4908)
* Move columnar test helpers to another file * Rename column_store_memory_stats to columnar_store_memory_statsexpose/defer_marked_shards_API
parent
31d4ed41d7
commit
96278822d9
|
@ -29,15 +29,15 @@
|
|||
|
||||
static void MemoryContextTotals(MemoryContext context, MemoryContextCounters *counters);
|
||||
|
||||
PG_FUNCTION_INFO_V1(column_store_memory_stats);
|
||||
PG_FUNCTION_INFO_V1(columnar_store_memory_stats);
|
||||
|
||||
|
||||
/*
|
||||
* column_store_memory_stats returns a record of 3 values: size of
|
||||
* columnar_store_memory_stats returns a record of 3 values: size of
|
||||
* TopMemoryContext, TopTransactionContext, and Write State context.
|
||||
*/
|
||||
Datum
|
||||
column_store_memory_stats(PG_FUNCTION_ARGS)
|
||||
columnar_store_memory_stats(PG_FUNCTION_ARGS)
|
||||
{
|
||||
const int resultColumnCount = 3;
|
||||
|
||||
|
|
|
@ -2,7 +2,7 @@
|
|||
# Only run few basic tests to set up a testing environment
|
||||
# ----------
|
||||
test: multi_cluster_management
|
||||
test: multi_test_helpers multi_test_helpers_superuser multi_create_fdw
|
||||
test: multi_test_helpers multi_test_helpers_superuser multi_create_fdw columnar_test_helpers
|
||||
test: multi_test_catalog_views
|
||||
test: multi_create_table multi_behavioral_analytics_create_table
|
||||
test: multi_create_table_superuser multi_behavioral_analytics_create_table_superuser
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
test: multi_cluster_management
|
||||
test: multi_test_helpers multi_test_helpers_superuser
|
||||
test: multi_test_helpers multi_test_helpers_superuser columnar_test_helpers
|
||||
test: multi_test_catalog_views
|
||||
|
||||
test: columnar_create
|
||||
|
|
|
@ -24,86 +24,26 @@ SELECT count(*) FROM contestant;
|
|||
-- Should fail: unlogged tables not supported
|
||||
CREATE UNLOGGED TABLE columnar_unlogged(i int) USING columnar;
|
||||
ERROR: unlogged columnar tables are not supported
|
||||
--
|
||||
-- Utility functions to be used throughout tests
|
||||
--
|
||||
CREATE FUNCTION columnar_relation_storageid(relid oid) RETURNS bigint
|
||||
LANGUAGE C STABLE STRICT
|
||||
AS 'citus', $$columnar_relation_storageid$$;
|
||||
CREATE FUNCTION compression_type_supported(type text) RETURNS boolean
|
||||
AS $$
|
||||
BEGIN
|
||||
EXECUTE 'SET LOCAL columnar.compression TO ' || quote_literal(type);
|
||||
return true;
|
||||
EXCEPTION WHEN invalid_parameter_value THEN
|
||||
return false;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
-- are chunk groups and chunks consistent?
|
||||
CREATE view chunk_group_consistency AS
|
||||
WITH a as (
|
||||
SELECT storage_id, stripe_num, chunk_group_num, min(value_count) as row_count
|
||||
FROM columnar.chunk
|
||||
GROUP BY 1,2,3
|
||||
), b as (
|
||||
SELECT storage_id, stripe_num, chunk_group_num, max(value_count) as row_count
|
||||
FROM columnar.chunk
|
||||
GROUP BY 1,2,3
|
||||
), c as (
|
||||
(TABLE a EXCEPT TABLE b) UNION (TABLE b EXCEPT TABLE a) UNION
|
||||
(TABLE a EXCEPT TABLE columnar.chunk_group) UNION (TABLE columnar.chunk_group EXCEPT TABLE a)
|
||||
), d as (
|
||||
SELECT storage_id, stripe_num, count(*) as chunk_group_count
|
||||
FROM columnar.chunk_group
|
||||
GROUP BY 1,2
|
||||
), e as (
|
||||
SELECT storage_id, stripe_num, chunk_group_count
|
||||
FROM columnar.stripe
|
||||
), f as (
|
||||
(TABLE d EXCEPT TABLE e) UNION (TABLE e EXCEPT TABLE d)
|
||||
)
|
||||
SELECT (SELECT count(*) = 0 FROM c) AND
|
||||
(SELECT count(*) = 0 FROM f) as consistent;
|
||||
CREATE FUNCTION columnar_metadata_has_storage_id(input_storage_id bigint) RETURNS boolean
|
||||
AS $$
|
||||
DECLARE
|
||||
union_storage_id_count integer;
|
||||
BEGIN
|
||||
SELECT count(*) INTO union_storage_id_count FROM
|
||||
(
|
||||
SELECT storage_id FROM columnar.stripe UNION ALL
|
||||
SELECT storage_id FROM columnar.chunk UNION ALL
|
||||
SELECT storage_id FROM columnar.chunk_group
|
||||
) AS union_storage_id
|
||||
WHERE storage_id=input_storage_id;
|
||||
|
||||
IF union_storage_id_count > 0 THEN
|
||||
RETURN true;
|
||||
END IF;
|
||||
|
||||
RETURN false;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
CREATE TABLE columnar_table_1 (a int) USING columnar;
|
||||
INSERT INTO columnar_table_1 VALUES (1);
|
||||
CREATE MATERIALIZED VIEW columnar_table_1_mv USING columnar
|
||||
AS SELECT * FROM columnar_table_1;
|
||||
SELECT columnar_relation_storageid(oid) AS columnar_table_1_mv_storage_id
|
||||
SELECT columnar_test_helpers.columnar_relation_storageid(oid) AS columnar_table_1_mv_storage_id
|
||||
FROM pg_class WHERE relname='columnar_table_1_mv' \gset
|
||||
-- test columnar_relation_set_new_filenode
|
||||
REFRESH MATERIALIZED VIEW columnar_table_1_mv;
|
||||
SELECT columnar_metadata_has_storage_id(:columnar_table_1_mv_storage_id);
|
||||
SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_table_1_mv_storage_id);
|
||||
columnar_metadata_has_storage_id
|
||||
---------------------------------------------------------------------
|
||||
f
|
||||
(1 row)
|
||||
|
||||
SELECT columnar_relation_storageid(oid) AS columnar_table_1_storage_id
|
||||
SELECT columnar_test_helpers.columnar_relation_storageid(oid) AS columnar_table_1_storage_id
|
||||
FROM pg_class WHERE relname='columnar_table_1' \gset
|
||||
BEGIN;
|
||||
-- test columnar_relation_nontransactional_truncate
|
||||
TRUNCATE columnar_table_1;
|
||||
SELECT columnar_metadata_has_storage_id(:columnar_table_1_storage_id);
|
||||
SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_table_1_storage_id);
|
||||
columnar_metadata_has_storage_id
|
||||
---------------------------------------------------------------------
|
||||
f
|
||||
|
@ -111,7 +51,7 @@ BEGIN;
|
|||
|
||||
ROLLBACK;
|
||||
-- since we rollback'ed above xact, should return true
|
||||
SELECT columnar_metadata_has_storage_id(:columnar_table_1_storage_id);
|
||||
SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_table_1_storage_id);
|
||||
columnar_metadata_has_storage_id
|
||||
---------------------------------------------------------------------
|
||||
t
|
||||
|
@ -120,7 +60,7 @@ SELECT columnar_metadata_has_storage_id(:columnar_table_1_storage_id);
|
|||
-- test dropping columnar table
|
||||
DROP TABLE columnar_table_1 CASCADE;
|
||||
NOTICE: drop cascades to materialized view columnar_table_1_mv
|
||||
SELECT columnar_metadata_has_storage_id(:columnar_table_1_storage_id);
|
||||
SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_table_1_storage_id);
|
||||
columnar_metadata_has_storage_id
|
||||
---------------------------------------------------------------------
|
||||
f
|
||||
|
@ -131,7 +71,7 @@ SELECT columnar_metadata_has_storage_id(:columnar_table_1_storage_id);
|
|||
CREATE TEMPORARY TABLE columnar_temp(i int) USING columnar;
|
||||
-- reserve some chunks and a stripe
|
||||
INSERT INTO columnar_temp SELECT i FROM generate_series(1,5) i;
|
||||
SELECT columnar_relation_storageid(oid) AS columnar_temp_storage_id
|
||||
SELECT columnar_test_helpers.columnar_relation_storageid(oid) AS columnar_temp_storage_id
|
||||
FROM pg_class WHERE relname='columnar_temp' \gset
|
||||
\c - - - :master_port
|
||||
-- show that temporary table itself and it's metadata is removed
|
||||
|
@ -141,7 +81,7 @@ SELECT COUNT(*)=0 FROM pg_class WHERE relname='columnar_temp';
|
|||
t
|
||||
(1 row)
|
||||
|
||||
SELECT columnar_metadata_has_storage_id(:columnar_temp_storage_id);
|
||||
SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_temp_storage_id);
|
||||
columnar_metadata_has_storage_id
|
||||
---------------------------------------------------------------------
|
||||
f
|
||||
|
@ -158,12 +98,12 @@ SELECT COUNT(*) FROM columnar_temp WHERE i < 5;
|
|||
4
|
||||
(1 row)
|
||||
|
||||
SELECT columnar_relation_storageid(oid) AS columnar_temp_storage_id
|
||||
SELECT columnar_test_helpers.columnar_relation_storageid(oid) AS columnar_temp_storage_id
|
||||
FROM pg_class WHERE relname='columnar_temp' \gset
|
||||
BEGIN;
|
||||
DROP TABLE columnar_temp;
|
||||
-- show that we drop stripes properly
|
||||
SELECT columnar_metadata_has_storage_id(:columnar_temp_storage_id);
|
||||
SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_temp_storage_id);
|
||||
columnar_metadata_has_storage_id
|
||||
---------------------------------------------------------------------
|
||||
f
|
||||
|
@ -178,7 +118,7 @@ SELECT COUNT(*)=1 FROM pg_class WHERE relname='columnar_temp';
|
|||
(1 row)
|
||||
|
||||
-- show that we preserve the stripe of the temp columanar table after rollback
|
||||
SELECT columnar_metadata_has_storage_id(:columnar_temp_storage_id);
|
||||
SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_temp_storage_id);
|
||||
columnar_metadata_has_storage_id
|
||||
---------------------------------------------------------------------
|
||||
t
|
||||
|
@ -190,7 +130,7 @@ BEGIN;
|
|||
CREATE TEMPORARY TABLE columnar_temp(i int) USING columnar ON COMMIT DROP;
|
||||
-- force flushing stripe
|
||||
INSERT INTO columnar_temp SELECT i FROM generate_series(1,150000) i;
|
||||
SELECT columnar_relation_storageid(oid) AS columnar_temp_storage_id
|
||||
SELECT columnar_test_helpers.columnar_relation_storageid(oid) AS columnar_temp_storage_id
|
||||
FROM pg_class WHERE relname='columnar_temp' \gset
|
||||
COMMIT;
|
||||
-- make sure that table & it's stripe is dropped after commiting above xact
|
||||
|
@ -200,7 +140,7 @@ SELECT COUNT(*)=0 FROM pg_class WHERE relname='columnar_temp';
|
|||
t
|
||||
(1 row)
|
||||
|
||||
SELECT columnar_metadata_has_storage_id(:columnar_temp_storage_id);
|
||||
SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_temp_storage_id);
|
||||
columnar_metadata_has_storage_id
|
||||
---------------------------------------------------------------------
|
||||
f
|
||||
|
@ -210,7 +150,7 @@ BEGIN;
|
|||
CREATE TEMPORARY TABLE columnar_temp(i int) USING columnar ON COMMIT DELETE ROWS;
|
||||
-- force flushing stripe
|
||||
INSERT INTO columnar_temp SELECT i FROM generate_series(1,150000) i;
|
||||
SELECT columnar_relation_storageid(oid) AS columnar_temp_storage_id
|
||||
SELECT columnar_test_helpers.columnar_relation_storageid(oid) AS columnar_temp_storage_id
|
||||
FROM pg_class WHERE relname='columnar_temp' \gset
|
||||
COMMIT;
|
||||
-- make sure that table is not dropped but it's rows's are deleted after commiting above xact
|
||||
|
@ -227,7 +167,7 @@ SELECT COUNT(*)=0 FROM columnar_temp;
|
|||
(1 row)
|
||||
|
||||
-- since we deleted all the rows, we shouldn't have any stripes for table
|
||||
SELECT columnar_metadata_has_storage_id(:columnar_temp_storage_id);
|
||||
SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_temp_storage_id);
|
||||
columnar_metadata_has_storage_id
|
||||
---------------------------------------------------------------------
|
||||
f
|
||||
|
|
|
@ -45,7 +45,7 @@ select count(*) from test_insert_command;
|
|||
3
|
||||
(1 row)
|
||||
|
||||
SELECT * FROM chunk_group_consistency;
|
||||
SELECT * FROM columnar_test_helpers.chunk_group_consistency;
|
||||
consistent
|
||||
---------------------------------------------------------------------
|
||||
t
|
||||
|
@ -67,7 +67,7 @@ CREATE TABLE test_columnar_long_text(int_val int, text_val text)
|
|||
USING columnar;
|
||||
-- store long text in columnar table
|
||||
INSERT INTO test_columnar_long_text SELECT * FROM test_long_text;
|
||||
SELECT * FROM chunk_group_consistency;
|
||||
SELECT * FROM columnar_test_helpers.chunk_group_consistency;
|
||||
consistent
|
||||
---------------------------------------------------------------------
|
||||
t
|
||||
|
@ -141,7 +141,7 @@ FROM test_toast_columnar;
|
|||
5004 | 5004 | 5004 | 5004
|
||||
(1 row)
|
||||
|
||||
SELECT * FROM chunk_group_consistency;
|
||||
SELECT * FROM columnar_test_helpers.chunk_group_consistency;
|
||||
consistent
|
||||
---------------------------------------------------------------------
|
||||
t
|
||||
|
@ -174,7 +174,7 @@ INSERT INTO zero_col_heap SELECT * FROM zero_col_heap;
|
|||
INSERT INTO zero_col_heap SELECT * FROM zero_col_heap;
|
||||
INSERT INTO zero_col SELECT * FROM zero_col_heap;
|
||||
SELECT relname, stripe_num, chunk_group_count, row_count FROM columnar.stripe a, pg_class b
|
||||
WHERE columnar_relation_storageid(b.oid)=a.storage_id AND relname = 'zero_col'
|
||||
WHERE columnar_test_helpers.columnar_relation_storageid(b.oid)=a.storage_id AND relname = 'zero_col'
|
||||
ORDER BY 1,2,3,4;
|
||||
relname | stripe_num | chunk_group_count | row_count
|
||||
---------------------------------------------------------------------
|
||||
|
@ -186,14 +186,14 @@ ORDER BY 1,2,3,4;
|
|||
(5 rows)
|
||||
|
||||
SELECT relname, stripe_num, value_count FROM columnar.chunk a, pg_class b
|
||||
WHERE columnar_relation_storageid(b.oid)=a.storage_id AND relname = 'zero_col'
|
||||
WHERE columnar_test_helpers.columnar_relation_storageid(b.oid)=a.storage_id AND relname = 'zero_col'
|
||||
ORDER BY 1,2,3;
|
||||
relname | stripe_num | value_count
|
||||
---------------------------------------------------------------------
|
||||
(0 rows)
|
||||
|
||||
SELECT relname, stripe_num, chunk_group_num, row_count FROM columnar.chunk_group a, pg_class b
|
||||
WHERE columnar_relation_storageid(b.oid)=a.storage_id AND relname = 'zero_col'
|
||||
WHERE columnar_test_helpers.columnar_relation_storageid(b.oid)=a.storage_id AND relname = 'zero_col'
|
||||
ORDER BY 1,2,3,4;
|
||||
relname | stripe_num | chunk_group_num | row_count
|
||||
---------------------------------------------------------------------
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
SELECT compression_type_supported('lz4') AS lz4_supported \gset
|
||||
SELECT columnar_test_helpers.compression_type_supported('lz4') AS lz4_supported \gset
|
||||
\if :lz4_supported
|
||||
\else
|
||||
\q
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
SELECT compression_type_supported('lz4') AS lz4_supported \gset
|
||||
SELECT columnar_test_helpers.compression_type_supported('lz4') AS lz4_supported \gset
|
||||
\if :lz4_supported
|
||||
\else
|
||||
\q
|
||||
|
|
|
@ -66,7 +66,7 @@ SELECT * FROM t_view a ORDER BY a;
|
|||
(6 rows)
|
||||
|
||||
-- verify that we have created metadata entries for the materialized view
|
||||
SELECT columnar_relation_storageid(oid) AS storageid
|
||||
SELECT columnar_test_helpers.columnar_relation_storageid(oid) AS storageid
|
||||
FROM pg_class WHERE relname='t_view' \gset
|
||||
SELECT count(*) FROM columnar.stripe WHERE storage_id=:storageid;
|
||||
count
|
||||
|
|
|
@ -3,35 +3,25 @@
|
|||
--
|
||||
CREATE SCHEMA columnar_memory;
|
||||
SET search_path TO 'columnar_memory';
|
||||
CREATE OR REPLACE FUNCTION column_store_memory_stats()
|
||||
RETURNS TABLE(TopMemoryContext BIGINT,
|
||||
TopTransactionContext BIGINT,
|
||||
WriteStateContext BIGINT)
|
||||
LANGUAGE C STRICT VOLATILE
|
||||
AS 'citus', $$column_store_memory_stats$$;
|
||||
CREATE FUNCTION top_memory_context_usage()
|
||||
RETURNS BIGINT AS $$
|
||||
SELECT TopMemoryContext FROM column_store_memory_stats();
|
||||
$$ LANGUAGE SQL VOLATILE;
|
||||
SET columnar.stripe_row_limit TO 50000;
|
||||
SET columnar.compression TO 'pglz';
|
||||
CREATE TABLE t (a int, tag text, memusage bigint) USING columnar;
|
||||
-- measure memory before doing writes
|
||||
SELECT TopMemoryContext as top_pre,
|
||||
WriteStateContext write_pre
|
||||
FROM column_store_memory_stats() \gset
|
||||
FROM columnar_test_helpers.columnar_store_memory_stats() \gset
|
||||
BEGIN;
|
||||
SET LOCAL client_min_messages TO DEBUG1;
|
||||
-- measure memory just before flushing 1st stripe
|
||||
INSERT INTO t
|
||||
SELECT i, 'first batch',
|
||||
-- sample memusage instead of recording everyr row for speed
|
||||
CASE WHEN i % 100 = 0 THEN top_memory_context_usage() ELSE 0 END
|
||||
CASE WHEN i % 100 = 0 THEN columnar_test_helpers.top_memory_context_usage() ELSE 0 END
|
||||
FROM generate_series(1, 49999) i;
|
||||
SELECT TopMemoryContext as top0,
|
||||
TopTransactionContext xact0,
|
||||
WriteStateContext write0
|
||||
FROM column_store_memory_stats() \gset
|
||||
FROM columnar_test_helpers.columnar_store_memory_stats() \gset
|
||||
-- flush 1st stripe, and measure memory just before flushing 2nd stripe
|
||||
INSERT INTO t
|
||||
SELECT i, 'second batch', 0 /* no need to record memusage per row */
|
||||
|
@ -40,7 +30,7 @@ DEBUG: Flushing Stripe of size 50000
|
|||
SELECT TopMemoryContext as top1,
|
||||
TopTransactionContext xact1,
|
||||
WriteStateContext write1
|
||||
FROM column_store_memory_stats() \gset
|
||||
FROM columnar_test_helpers.columnar_store_memory_stats() \gset
|
||||
-- flush 2nd stripe, and measure memory just before flushing 3rd stripe
|
||||
INSERT INTO t
|
||||
SELECT i, 'third batch', 0 /* no need to record memusage per row */
|
||||
|
@ -49,12 +39,12 @@ DEBUG: Flushing Stripe of size 50000
|
|||
SELECT TopMemoryContext as top2,
|
||||
TopTransactionContext xact2,
|
||||
WriteStateContext write2
|
||||
FROM column_store_memory_stats() \gset
|
||||
FROM columnar_test_helpers.columnar_store_memory_stats() \gset
|
||||
-- insert a large batch
|
||||
INSERT INTO t
|
||||
SELECT i, 'large batch',
|
||||
-- sample memusage instead of recording everyr row for speed
|
||||
CASE WHEN i % 100 = 0 THEN top_memory_context_usage() ELSE 0 END
|
||||
CASE WHEN i % 100 = 0 THEN columnar_test_helpers.top_memory_context_usage() ELSE 0 END
|
||||
FROM generate_series(1, 100000) i;
|
||||
DEBUG: Flushing Stripe of size 50000
|
||||
DEBUG: Flushing Stripe of size 50000
|
||||
|
@ -63,7 +53,7 @@ DEBUG: Flushing Stripe of size 49999
|
|||
-- measure memory after doing writes
|
||||
SELECT TopMemoryContext as top_post,
|
||||
WriteStateContext write_post
|
||||
FROM column_store_memory_stats() \gset
|
||||
FROM columnar_test_helpers.columnar_store_memory_stats() \gset
|
||||
\x
|
||||
SELECT (1.0 * :top2/:top1 BETWEEN 0.99 AND 1.01) AS top_growth_ok,
|
||||
(1.0 * :xact1/:xact0 BETWEEN 0.99 AND 1.01) AND
|
||||
|
@ -82,7 +72,7 @@ INSERT INTO t
|
|||
SELECT i, 'last batch', 0 /* no need to record memusage per row */
|
||||
FROM generate_series(1, 50000) i;
|
||||
SELECT 1.0 * TopMemoryContext / :top_post BETWEEN 0.98 AND 1.02 AS top_growth_ok
|
||||
FROM column_store_memory_stats();
|
||||
FROM columnar_test_helpers.columnar_store_memory_stats();
|
||||
-[ RECORD 1 ]-+--
|
||||
top_growth_ok | t
|
||||
|
||||
|
|
|
@ -12,7 +12,7 @@ INSERT INTO t2 SELECT i, f(i) FROM generate_series(1, 5) i;
|
|||
-- there are no subtransactions, so above statement should batch
|
||||
-- INSERTs inside the UDF and create on stripe per table.
|
||||
SELECT relname, count(*) FROM columnar.stripe a, pg_class b
|
||||
WHERE columnar_relation_storageid(b.oid)=a.storage_id AND relname IN ('t1', 't2')
|
||||
WHERE columnar_test_helpers.columnar_relation_storageid(b.oid)=a.storage_id AND relname IN ('t1', 't2')
|
||||
GROUP BY relname
|
||||
ORDER BY relname;
|
||||
relname | count
|
||||
|
@ -72,7 +72,7 @@ SELECT * FROM t2;
|
|||
5 | 6
|
||||
(5 rows)
|
||||
|
||||
SELECT * FROM chunk_group_consistency;
|
||||
SELECT * FROM columnar_test_helpers.chunk_group_consistency;
|
||||
consistent
|
||||
---------------------------------------------------------------------
|
||||
t
|
||||
|
@ -107,7 +107,7 @@ SELECT * FROM t2;
|
|||
3 | 0
|
||||
(3 rows)
|
||||
|
||||
SELECT * FROM chunk_group_consistency;
|
||||
SELECT * FROM columnar_test_helpers.chunk_group_consistency;
|
||||
consistent
|
||||
---------------------------------------------------------------------
|
||||
t
|
||||
|
@ -137,7 +137,7 @@ SELECT * FROM t1 ORDER BY a, b;
|
|||
5 | 10
|
||||
(10 rows)
|
||||
|
||||
SELECT * FROM chunk_group_consistency;
|
||||
SELECT * FROM columnar_test_helpers.chunk_group_consistency;
|
||||
consistent
|
||||
---------------------------------------------------------------------
|
||||
t
|
||||
|
@ -183,7 +183,7 @@ SELECT * FROM t2 ORDER BY a, b;
|
|||
5 | 5
|
||||
(5 rows)
|
||||
|
||||
SELECT * FROM chunk_group_consistency;
|
||||
SELECT * FROM columnar_test_helpers.chunk_group_consistency;
|
||||
consistent
|
||||
---------------------------------------------------------------------
|
||||
t
|
||||
|
@ -228,7 +228,7 @@ SELECT * FROM t3 ORDER BY a, b;
|
|||
---------------------------------------------------------------------
|
||||
(0 rows)
|
||||
|
||||
SELECT * FROM chunk_group_consistency;
|
||||
SELECT * FROM columnar_test_helpers.chunk_group_consistency;
|
||||
consistent
|
||||
---------------------------------------------------------------------
|
||||
t
|
||||
|
@ -278,7 +278,7 @@ SELECT * FROM t1 ORDER BY a, b;
|
|||
23 | 46
|
||||
(10 rows)
|
||||
|
||||
SELECT * FROM chunk_group_consistency;
|
||||
SELECT * FROM columnar_test_helpers.chunk_group_consistency;
|
||||
consistent
|
||||
---------------------------------------------------------------------
|
||||
t
|
||||
|
|
|
@ -4,7 +4,7 @@
|
|||
CREATE TABLE t(a int, b int) USING columnar;
|
||||
CREATE VIEW t_stripes AS
|
||||
SELECT * FROM columnar.stripe a, pg_class b
|
||||
WHERE a.storage_id = columnar_relation_storageid(b.oid) AND b.relname = 't';
|
||||
WHERE a.storage_id = columnar_test_helpers.columnar_relation_storageid(b.oid) AND b.relname = 't';
|
||||
BEGIN;
|
||||
INSERT INTO t SELECT i, i+1 FROM generate_series(1, 10) i;
|
||||
ROLLBACK;
|
||||
|
|
|
@ -0,0 +1,69 @@
|
|||
CREATE SCHEMA columnar_test_helpers;
|
||||
SET search_path TO columnar_test_helpers;
|
||||
CREATE FUNCTION columnar_relation_storageid(relid oid) RETURNS bigint
|
||||
LANGUAGE C STABLE STRICT
|
||||
AS 'citus', $$columnar_relation_storageid$$;
|
||||
CREATE FUNCTION compression_type_supported(type text) RETURNS boolean
|
||||
AS $$
|
||||
BEGIN
|
||||
EXECUTE 'SET LOCAL columnar.compression TO ' || quote_literal(type);
|
||||
return true;
|
||||
EXCEPTION WHEN invalid_parameter_value THEN
|
||||
return false;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
-- are chunk groups and chunks consistent?
|
||||
CREATE view chunk_group_consistency AS
|
||||
WITH a as (
|
||||
SELECT storage_id, stripe_num, chunk_group_num, min(value_count) as row_count
|
||||
FROM columnar.chunk
|
||||
GROUP BY 1,2,3
|
||||
), b as (
|
||||
SELECT storage_id, stripe_num, chunk_group_num, max(value_count) as row_count
|
||||
FROM columnar.chunk
|
||||
GROUP BY 1,2,3
|
||||
), c as (
|
||||
(TABLE a EXCEPT TABLE b) UNION (TABLE b EXCEPT TABLE a) UNION
|
||||
(TABLE a EXCEPT TABLE columnar.chunk_group) UNION (TABLE columnar.chunk_group EXCEPT TABLE a)
|
||||
), d as (
|
||||
SELECT storage_id, stripe_num, count(*) as chunk_group_count
|
||||
FROM columnar.chunk_group
|
||||
GROUP BY 1,2
|
||||
), e as (
|
||||
SELECT storage_id, stripe_num, chunk_group_count
|
||||
FROM columnar.stripe
|
||||
), f as (
|
||||
(TABLE d EXCEPT TABLE e) UNION (TABLE e EXCEPT TABLE d)
|
||||
)
|
||||
SELECT (SELECT count(*) = 0 FROM c) AND
|
||||
(SELECT count(*) = 0 FROM f) as consistent;
|
||||
CREATE FUNCTION columnar_metadata_has_storage_id(input_storage_id bigint) RETURNS boolean
|
||||
AS $$
|
||||
DECLARE
|
||||
union_storage_id_count integer;
|
||||
BEGIN
|
||||
SELECT count(*) INTO union_storage_id_count FROM
|
||||
(
|
||||
SELECT storage_id FROM columnar.stripe UNION ALL
|
||||
SELECT storage_id FROM columnar.chunk UNION ALL
|
||||
SELECT storage_id FROM columnar.chunk_group
|
||||
) AS union_storage_id
|
||||
WHERE storage_id=input_storage_id;
|
||||
|
||||
IF union_storage_id_count > 0 THEN
|
||||
RETURN true;
|
||||
END IF;
|
||||
|
||||
RETURN false;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
CREATE OR REPLACE FUNCTION columnar_store_memory_stats()
|
||||
RETURNS TABLE(TopMemoryContext BIGINT,
|
||||
TopTransactionContext BIGINT,
|
||||
WriteStateContext BIGINT)
|
||||
LANGUAGE C STRICT VOLATILE
|
||||
AS 'citus', $$columnar_store_memory_stats$$;
|
||||
CREATE FUNCTION top_memory_context_usage()
|
||||
RETURNS BIGINT AS $$
|
||||
SELECT TopMemoryContext FROM columnar_test_helpers.columnar_store_memory_stats();
|
||||
$$ LANGUAGE SQL VOLATILE;
|
|
@ -37,14 +37,14 @@ SELECT * FROM columnar_truncate_test;
|
|||
10 | 10
|
||||
(10 rows)
|
||||
|
||||
SELECT * FROM chunk_group_consistency;
|
||||
SELECT * FROM columnar_test_helpers.chunk_group_consistency;
|
||||
consistent
|
||||
---------------------------------------------------------------------
|
||||
t
|
||||
(1 row)
|
||||
|
||||
TRUNCATE TABLE columnar_truncate_test;
|
||||
SELECT * FROM chunk_group_consistency;
|
||||
SELECT * FROM columnar_test_helpers.chunk_group_consistency;
|
||||
consistent
|
||||
---------------------------------------------------------------------
|
||||
t
|
||||
|
@ -130,7 +130,7 @@ SELECT * from columnar_truncate_test_regular;
|
|||
20 | 20
|
||||
(11 rows)
|
||||
|
||||
SELECT * FROM chunk_group_consistency;
|
||||
SELECT * FROM columnar_test_helpers.chunk_group_consistency;
|
||||
consistent
|
||||
---------------------------------------------------------------------
|
||||
t
|
||||
|
@ -142,7 +142,7 @@ TRUNCATE TABLE columnar_truncate_test,
|
|||
columnar_truncate_test_regular,
|
||||
columnar_truncate_test_second,
|
||||
columnar_truncate_test;
|
||||
SELECT * FROM chunk_group_consistency;
|
||||
SELECT * FROM columnar_test_helpers.chunk_group_consistency;
|
||||
consistent
|
||||
---------------------------------------------------------------------
|
||||
t
|
||||
|
@ -291,7 +291,7 @@ SELECT count(*) FROM truncate_schema.truncate_tbl;
|
|||
(1 row)
|
||||
|
||||
\c - :current_user
|
||||
SELECT * FROM chunk_group_consistency;
|
||||
SELECT * FROM columnar_test_helpers.chunk_group_consistency;
|
||||
consistent
|
||||
---------------------------------------------------------------------
|
||||
t
|
||||
|
|
|
@ -3,7 +3,7 @@ SELECT count(distinct storage_id) AS columnar_table_count FROM columnar.stripe \
|
|||
CREATE TABLE t(a int, b int) USING columnar;
|
||||
CREATE VIEW t_stripes AS
|
||||
SELECT * FROM columnar.stripe a, pg_class b
|
||||
WHERE a.storage_id = columnar_relation_storageid(b.oid) AND b.relname='t';
|
||||
WHERE a.storage_id = columnar_test_helpers.columnar_relation_storageid(b.oid) AND b.relname='t';
|
||||
SELECT count(*) FROM t_stripes;
|
||||
count
|
||||
---------------------------------------------------------------------
|
||||
|
@ -27,7 +27,7 @@ SELECT count(*) FROM t_stripes;
|
|||
|
||||
-- vacuum full should merge stripes together
|
||||
VACUUM FULL t;
|
||||
SELECT * FROM chunk_group_consistency;
|
||||
SELECT * FROM columnar_test_helpers.chunk_group_consistency;
|
||||
consistent
|
||||
---------------------------------------------------------------------
|
||||
t
|
||||
|
@ -66,7 +66,7 @@ SELECT count(*) FROM t_stripes;
|
|||
(1 row)
|
||||
|
||||
VACUUM FULL t;
|
||||
SELECT * FROM chunk_group_consistency;
|
||||
SELECT * FROM columnar_test_helpers.chunk_group_consistency;
|
||||
consistent
|
||||
---------------------------------------------------------------------
|
||||
t
|
||||
|
@ -88,7 +88,7 @@ SELECT count(*) FROM t_stripes;
|
|||
ALTER TABLE t DROP COLUMN a;
|
||||
SELECT stripe_num, attr_num, chunk_group_num, minimum_value IS NULL, maximum_value IS NULL
|
||||
FROM columnar.chunk a, pg_class b
|
||||
WHERE a.storage_id = columnar_relation_storageid(b.oid) AND b.relname='t' ORDER BY 1, 2, 3;
|
||||
WHERE a.storage_id = columnar_test_helpers.columnar_relation_storageid(b.oid) AND b.relname='t' ORDER BY 1, 2, 3;
|
||||
stripe_num | attr_num | chunk_group_num | ?column? | ?column?
|
||||
---------------------------------------------------------------------
|
||||
1 | 1 | 0 | f | f
|
||||
|
@ -102,7 +102,7 @@ WHERE a.storage_id = columnar_relation_storageid(b.oid) AND b.relname='t' ORDER
|
|||
VACUUM FULL t;
|
||||
SELECT stripe_num, attr_num, chunk_group_num, minimum_value IS NULL, maximum_value IS NULL
|
||||
FROM columnar.chunk a, pg_class b
|
||||
WHERE a.storage_id = columnar_relation_storageid(b.oid) AND b.relname='t' ORDER BY 1, 2, 3;
|
||||
WHERE a.storage_id = columnar_test_helpers.columnar_relation_storageid(b.oid) AND b.relname='t' ORDER BY 1, 2, 3;
|
||||
stripe_num | attr_num | chunk_group_num | ?column? | ?column?
|
||||
---------------------------------------------------------------------
|
||||
1 | 1 | 0 | t | t
|
||||
|
@ -215,7 +215,7 @@ compression rate: 1.25x
|
|||
total row count: 5530, stripe count: 5, average rows per stripe: 1106
|
||||
chunk count: 7, containing data for dropped columns: 0, none compressed: 5, pglz compressed: 2
|
||||
|
||||
SELECT * FROM chunk_group_consistency;
|
||||
SELECT * FROM columnar_test_helpers.chunk_group_consistency;
|
||||
consistent
|
||||
---------------------------------------------------------------------
|
||||
t
|
||||
|
@ -257,7 +257,7 @@ compression rate: 1.96x
|
|||
total row count: 7030, stripe count: 4, average rows per stripe: 1757
|
||||
chunk count: 8, containing data for dropped columns: 0, none compressed: 2, pglz compressed: 6
|
||||
|
||||
SELECT * FROM chunk_group_consistency;
|
||||
SELECT * FROM columnar_test_helpers.chunk_group_consistency;
|
||||
consistent
|
||||
---------------------------------------------------------------------
|
||||
t
|
||||
|
@ -286,7 +286,7 @@ compression rate: 33.71x
|
|||
total row count: 1000000, stripe count: 1, average rows per stripe: 1000000
|
||||
chunk count: 30, containing data for dropped columns: 0, pglz compressed: 30
|
||||
|
||||
SELECT * FROM chunk_group_consistency;
|
||||
SELECT * FROM columnar_test_helpers.chunk_group_consistency;
|
||||
consistent
|
||||
---------------------------------------------------------------------
|
||||
t
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
SELECT compression_type_supported('zstd') AS zstd_supported \gset
|
||||
SELECT columnar_test_helpers.compression_type_supported('zstd') AS zstd_supported \gset
|
||||
\if :zstd_supported
|
||||
\else
|
||||
\q
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
SELECT compression_type_supported('zstd') AS zstd_supported \gset
|
||||
SELECT columnar_test_helpers.compression_type_supported('zstd') AS zstd_supported \gset
|
||||
\if :zstd_supported
|
||||
\else
|
||||
\q
|
||||
|
|
|
@ -43,6 +43,6 @@ speed of light,2.997e8
|
|||
|
||||
SELECT * FROM famous_constants ORDER BY id, name;
|
||||
|
||||
SELECT * FROM chunk_group_consistency;
|
||||
SELECT * FROM columnar_test_helpers.chunk_group_consistency;
|
||||
|
||||
DROP TABLE famous_constants;
|
||||
|
|
|
@ -1,3 +1,3 @@
|
|||
test: multi_cluster_management
|
||||
test: multi_test_helpers multi_test_helpers_superuser
|
||||
test: multi_test_helpers multi_test_helpers_superuser columnar_test_helpers
|
||||
test: multi_test_catalog_views
|
||||
|
|
|
@ -27,7 +27,7 @@ CREATE TABLE famous_constants (id int, name text, value real)
|
|||
COPY famous_constants (value, name, id) FROM STDIN WITH CSV;
|
||||
COPY famous_constants (name, value) FROM STDIN WITH CSV;
|
||||
SELECT * FROM famous_constants ORDER BY id, name;
|
||||
id | name | value
|
||||
id | name | value
|
||||
----+----------------+-----------
|
||||
1 | pi | 3.141
|
||||
2 | e | 2.718
|
||||
|
@ -39,7 +39,7 @@ SELECT * FROM famous_constants ORDER BY id, name;
|
|||
| speed of light | 2.997e+08
|
||||
(8 rows)
|
||||
|
||||
SELECT * FROM chunk_group_consistency;
|
||||
SELECT * FROM columnar_test_helpers.chunk_group_consistency;
|
||||
consistent
|
||||
---------------------------------------------------------------------
|
||||
t
|
||||
|
|
|
@ -24,100 +24,34 @@ SELECT count(*) FROM contestant;
|
|||
-- Should fail: unlogged tables not supported
|
||||
CREATE UNLOGGED TABLE columnar_unlogged(i int) USING columnar;
|
||||
|
||||
--
|
||||
-- Utility functions to be used throughout tests
|
||||
--
|
||||
|
||||
CREATE FUNCTION columnar_relation_storageid(relid oid) RETURNS bigint
|
||||
LANGUAGE C STABLE STRICT
|
||||
AS 'citus', $$columnar_relation_storageid$$;
|
||||
|
||||
CREATE FUNCTION compression_type_supported(type text) RETURNS boolean
|
||||
AS $$
|
||||
BEGIN
|
||||
EXECUTE 'SET LOCAL columnar.compression TO ' || quote_literal(type);
|
||||
return true;
|
||||
EXCEPTION WHEN invalid_parameter_value THEN
|
||||
return false;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
|
||||
-- are chunk groups and chunks consistent?
|
||||
CREATE view chunk_group_consistency AS
|
||||
WITH a as (
|
||||
SELECT storage_id, stripe_num, chunk_group_num, min(value_count) as row_count
|
||||
FROM columnar.chunk
|
||||
GROUP BY 1,2,3
|
||||
), b as (
|
||||
SELECT storage_id, stripe_num, chunk_group_num, max(value_count) as row_count
|
||||
FROM columnar.chunk
|
||||
GROUP BY 1,2,3
|
||||
), c as (
|
||||
(TABLE a EXCEPT TABLE b) UNION (TABLE b EXCEPT TABLE a) UNION
|
||||
(TABLE a EXCEPT TABLE columnar.chunk_group) UNION (TABLE columnar.chunk_group EXCEPT TABLE a)
|
||||
), d as (
|
||||
SELECT storage_id, stripe_num, count(*) as chunk_group_count
|
||||
FROM columnar.chunk_group
|
||||
GROUP BY 1,2
|
||||
), e as (
|
||||
SELECT storage_id, stripe_num, chunk_group_count
|
||||
FROM columnar.stripe
|
||||
), f as (
|
||||
(TABLE d EXCEPT TABLE e) UNION (TABLE e EXCEPT TABLE d)
|
||||
)
|
||||
SELECT (SELECT count(*) = 0 FROM c) AND
|
||||
(SELECT count(*) = 0 FROM f) as consistent;
|
||||
|
||||
CREATE FUNCTION columnar_metadata_has_storage_id(input_storage_id bigint) RETURNS boolean
|
||||
AS $$
|
||||
DECLARE
|
||||
union_storage_id_count integer;
|
||||
BEGIN
|
||||
SELECT count(*) INTO union_storage_id_count FROM
|
||||
(
|
||||
SELECT storage_id FROM columnar.stripe UNION ALL
|
||||
SELECT storage_id FROM columnar.chunk UNION ALL
|
||||
SELECT storage_id FROM columnar.chunk_group
|
||||
) AS union_storage_id
|
||||
WHERE storage_id=input_storage_id;
|
||||
|
||||
IF union_storage_id_count > 0 THEN
|
||||
RETURN true;
|
||||
END IF;
|
||||
|
||||
RETURN false;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
CREATE TABLE columnar_table_1 (a int) USING columnar;
|
||||
INSERT INTO columnar_table_1 VALUES (1);
|
||||
|
||||
CREATE MATERIALIZED VIEW columnar_table_1_mv USING columnar
|
||||
AS SELECT * FROM columnar_table_1;
|
||||
|
||||
SELECT columnar_relation_storageid(oid) AS columnar_table_1_mv_storage_id
|
||||
SELECT columnar_test_helpers.columnar_relation_storageid(oid) AS columnar_table_1_mv_storage_id
|
||||
FROM pg_class WHERE relname='columnar_table_1_mv' \gset
|
||||
|
||||
-- test columnar_relation_set_new_filenode
|
||||
REFRESH MATERIALIZED VIEW columnar_table_1_mv;
|
||||
SELECT columnar_metadata_has_storage_id(:columnar_table_1_mv_storage_id);
|
||||
SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_table_1_mv_storage_id);
|
||||
|
||||
SELECT columnar_relation_storageid(oid) AS columnar_table_1_storage_id
|
||||
SELECT columnar_test_helpers.columnar_relation_storageid(oid) AS columnar_table_1_storage_id
|
||||
FROM pg_class WHERE relname='columnar_table_1' \gset
|
||||
|
||||
BEGIN;
|
||||
-- test columnar_relation_nontransactional_truncate
|
||||
TRUNCATE columnar_table_1;
|
||||
SELECT columnar_metadata_has_storage_id(:columnar_table_1_storage_id);
|
||||
SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_table_1_storage_id);
|
||||
ROLLBACK;
|
||||
|
||||
-- since we rollback'ed above xact, should return true
|
||||
SELECT columnar_metadata_has_storage_id(:columnar_table_1_storage_id);
|
||||
SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_table_1_storage_id);
|
||||
|
||||
-- test dropping columnar table
|
||||
DROP TABLE columnar_table_1 CASCADE;
|
||||
SELECT columnar_metadata_has_storage_id(:columnar_table_1_storage_id);
|
||||
SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_table_1_storage_id);
|
||||
|
||||
-- test temporary columnar tables
|
||||
|
||||
|
@ -127,14 +61,14 @@ CREATE TEMPORARY TABLE columnar_temp(i int) USING columnar;
|
|||
-- reserve some chunks and a stripe
|
||||
INSERT INTO columnar_temp SELECT i FROM generate_series(1,5) i;
|
||||
|
||||
SELECT columnar_relation_storageid(oid) AS columnar_temp_storage_id
|
||||
SELECT columnar_test_helpers.columnar_relation_storageid(oid) AS columnar_temp_storage_id
|
||||
FROM pg_class WHERE relname='columnar_temp' \gset
|
||||
|
||||
\c - - - :master_port
|
||||
|
||||
-- show that temporary table itself and it's metadata is removed
|
||||
SELECT COUNT(*)=0 FROM pg_class WHERE relname='columnar_temp';
|
||||
SELECT columnar_metadata_has_storage_id(:columnar_temp_storage_id);
|
||||
SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_temp_storage_id);
|
||||
|
||||
-- connect to another session and create a temp table with same name
|
||||
CREATE TEMPORARY TABLE columnar_temp(i int) USING columnar;
|
||||
|
@ -145,19 +79,19 @@ INSERT INTO columnar_temp SELECT i FROM generate_series(1,5) i;
|
|||
-- test basic select
|
||||
SELECT COUNT(*) FROM columnar_temp WHERE i < 5;
|
||||
|
||||
SELECT columnar_relation_storageid(oid) AS columnar_temp_storage_id
|
||||
SELECT columnar_test_helpers.columnar_relation_storageid(oid) AS columnar_temp_storage_id
|
||||
FROM pg_class WHERE relname='columnar_temp' \gset
|
||||
|
||||
BEGIN;
|
||||
DROP TABLE columnar_temp;
|
||||
-- show that we drop stripes properly
|
||||
SELECT columnar_metadata_has_storage_id(:columnar_temp_storage_id);
|
||||
SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_temp_storage_id);
|
||||
ROLLBACK;
|
||||
|
||||
-- make sure that table is not dropped yet since we rollbacked above xact
|
||||
SELECT COUNT(*)=1 FROM pg_class WHERE relname='columnar_temp';
|
||||
-- show that we preserve the stripe of the temp columanar table after rollback
|
||||
SELECT columnar_metadata_has_storage_id(:columnar_temp_storage_id);
|
||||
SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_temp_storage_id);
|
||||
|
||||
-- drop it for next tests
|
||||
DROP TABLE columnar_temp;
|
||||
|
@ -167,20 +101,20 @@ BEGIN;
|
|||
-- force flushing stripe
|
||||
INSERT INTO columnar_temp SELECT i FROM generate_series(1,150000) i;
|
||||
|
||||
SELECT columnar_relation_storageid(oid) AS columnar_temp_storage_id
|
||||
SELECT columnar_test_helpers.columnar_relation_storageid(oid) AS columnar_temp_storage_id
|
||||
FROM pg_class WHERE relname='columnar_temp' \gset
|
||||
COMMIT;
|
||||
|
||||
-- make sure that table & it's stripe is dropped after commiting above xact
|
||||
SELECT COUNT(*)=0 FROM pg_class WHERE relname='columnar_temp';
|
||||
SELECT columnar_metadata_has_storage_id(:columnar_temp_storage_id);
|
||||
SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_temp_storage_id);
|
||||
|
||||
BEGIN;
|
||||
CREATE TEMPORARY TABLE columnar_temp(i int) USING columnar ON COMMIT DELETE ROWS;
|
||||
-- force flushing stripe
|
||||
INSERT INTO columnar_temp SELECT i FROM generate_series(1,150000) i;
|
||||
|
||||
SELECT columnar_relation_storageid(oid) AS columnar_temp_storage_id
|
||||
SELECT columnar_test_helpers.columnar_relation_storageid(oid) AS columnar_temp_storage_id
|
||||
FROM pg_class WHERE relname='columnar_temp' \gset
|
||||
COMMIT;
|
||||
|
||||
|
@ -188,4 +122,4 @@ COMMIT;
|
|||
SELECT COUNT(*)=1 FROM pg_class WHERE relname='columnar_temp';
|
||||
SELECT COUNT(*)=0 FROM columnar_temp;
|
||||
-- since we deleted all the rows, we shouldn't have any stripes for table
|
||||
SELECT columnar_metadata_has_storage_id(:columnar_temp_storage_id);
|
||||
SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_temp_storage_id);
|
||||
|
|
|
@ -22,7 +22,7 @@ select count(*) from test_insert_command_data;
|
|||
insert into test_insert_command select * from test_insert_command_data;
|
||||
select count(*) from test_insert_command;
|
||||
|
||||
SELECT * FROM chunk_group_consistency;
|
||||
SELECT * FROM columnar_test_helpers.chunk_group_consistency;
|
||||
|
||||
drop table test_insert_command_data;
|
||||
drop table test_insert_command;
|
||||
|
@ -45,7 +45,7 @@ USING columnar;
|
|||
-- store long text in columnar table
|
||||
INSERT INTO test_columnar_long_text SELECT * FROM test_long_text;
|
||||
|
||||
SELECT * FROM chunk_group_consistency;
|
||||
SELECT * FROM columnar_test_helpers.chunk_group_consistency;
|
||||
|
||||
-- drop source table to remove original text from toast
|
||||
DROP TABLE test_long_text;
|
||||
|
@ -99,7 +99,7 @@ SELECT
|
|||
pg_column_size(external), pg_column_size(extended)
|
||||
FROM test_toast_columnar;
|
||||
|
||||
SELECT * FROM chunk_group_consistency;
|
||||
SELECT * FROM columnar_test_helpers.chunk_group_consistency;
|
||||
|
||||
DROP TABLE test_toast_row;
|
||||
DROP TABLE test_toast_columnar;
|
||||
|
@ -129,15 +129,15 @@ INSERT INTO zero_col_heap SELECT * FROM zero_col_heap;
|
|||
INSERT INTO zero_col SELECT * FROM zero_col_heap;
|
||||
|
||||
SELECT relname, stripe_num, chunk_group_count, row_count FROM columnar.stripe a, pg_class b
|
||||
WHERE columnar_relation_storageid(b.oid)=a.storage_id AND relname = 'zero_col'
|
||||
WHERE columnar_test_helpers.columnar_relation_storageid(b.oid)=a.storage_id AND relname = 'zero_col'
|
||||
ORDER BY 1,2,3,4;
|
||||
|
||||
SELECT relname, stripe_num, value_count FROM columnar.chunk a, pg_class b
|
||||
WHERE columnar_relation_storageid(b.oid)=a.storage_id AND relname = 'zero_col'
|
||||
WHERE columnar_test_helpers.columnar_relation_storageid(b.oid)=a.storage_id AND relname = 'zero_col'
|
||||
ORDER BY 1,2,3;
|
||||
|
||||
SELECT relname, stripe_num, chunk_group_num, row_count FROM columnar.chunk_group a, pg_class b
|
||||
WHERE columnar_relation_storageid(b.oid)=a.storage_id AND relname = 'zero_col'
|
||||
WHERE columnar_test_helpers.columnar_relation_storageid(b.oid)=a.storage_id AND relname = 'zero_col'
|
||||
ORDER BY 1,2,3,4;
|
||||
|
||||
DROP TABLE zero_col;
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
SELECT compression_type_supported('lz4') AS lz4_supported \gset
|
||||
SELECT columnar_test_helpers.compression_type_supported('lz4') AS lz4_supported \gset
|
||||
\if :lz4_supported
|
||||
\else
|
||||
\q
|
||||
|
|
|
@ -35,7 +35,7 @@ WHERE regclass = 't_view'::regclass;
|
|||
SELECT * FROM t_view a ORDER BY a;
|
||||
|
||||
-- verify that we have created metadata entries for the materialized view
|
||||
SELECT columnar_relation_storageid(oid) AS storageid
|
||||
SELECT columnar_test_helpers.columnar_relation_storageid(oid) AS storageid
|
||||
FROM pg_class WHERE relname='t_view' \gset
|
||||
|
||||
SELECT count(*) FROM columnar.stripe WHERE storage_id=:storageid;
|
||||
|
|
|
@ -5,18 +5,6 @@
|
|||
CREATE SCHEMA columnar_memory;
|
||||
SET search_path TO 'columnar_memory';
|
||||
|
||||
CREATE OR REPLACE FUNCTION column_store_memory_stats()
|
||||
RETURNS TABLE(TopMemoryContext BIGINT,
|
||||
TopTransactionContext BIGINT,
|
||||
WriteStateContext BIGINT)
|
||||
LANGUAGE C STRICT VOLATILE
|
||||
AS 'citus', $$column_store_memory_stats$$;
|
||||
|
||||
CREATE FUNCTION top_memory_context_usage()
|
||||
RETURNS BIGINT AS $$
|
||||
SELECT TopMemoryContext FROM column_store_memory_stats();
|
||||
$$ LANGUAGE SQL VOLATILE;
|
||||
|
||||
SET columnar.stripe_row_limit TO 50000;
|
||||
SET columnar.compression TO 'pglz';
|
||||
CREATE TABLE t (a int, tag text, memusage bigint) USING columnar;
|
||||
|
@ -24,7 +12,7 @@ CREATE TABLE t (a int, tag text, memusage bigint) USING columnar;
|
|||
-- measure memory before doing writes
|
||||
SELECT TopMemoryContext as top_pre,
|
||||
WriteStateContext write_pre
|
||||
FROM column_store_memory_stats() \gset
|
||||
FROM columnar_test_helpers.columnar_store_memory_stats() \gset
|
||||
|
||||
BEGIN;
|
||||
SET LOCAL client_min_messages TO DEBUG1;
|
||||
|
@ -33,12 +21,12 @@ SET LOCAL client_min_messages TO DEBUG1;
|
|||
INSERT INTO t
|
||||
SELECT i, 'first batch',
|
||||
-- sample memusage instead of recording everyr row for speed
|
||||
CASE WHEN i % 100 = 0 THEN top_memory_context_usage() ELSE 0 END
|
||||
CASE WHEN i % 100 = 0 THEN columnar_test_helpers.top_memory_context_usage() ELSE 0 END
|
||||
FROM generate_series(1, 49999) i;
|
||||
SELECT TopMemoryContext as top0,
|
||||
TopTransactionContext xact0,
|
||||
WriteStateContext write0
|
||||
FROM column_store_memory_stats() \gset
|
||||
FROM columnar_test_helpers.columnar_store_memory_stats() \gset
|
||||
|
||||
-- flush 1st stripe, and measure memory just before flushing 2nd stripe
|
||||
INSERT INTO t
|
||||
|
@ -47,7 +35,7 @@ INSERT INTO t
|
|||
SELECT TopMemoryContext as top1,
|
||||
TopTransactionContext xact1,
|
||||
WriteStateContext write1
|
||||
FROM column_store_memory_stats() \gset
|
||||
FROM columnar_test_helpers.columnar_store_memory_stats() \gset
|
||||
|
||||
-- flush 2nd stripe, and measure memory just before flushing 3rd stripe
|
||||
INSERT INTO t
|
||||
|
@ -56,13 +44,13 @@ INSERT INTO t
|
|||
SELECT TopMemoryContext as top2,
|
||||
TopTransactionContext xact2,
|
||||
WriteStateContext write2
|
||||
FROM column_store_memory_stats() \gset
|
||||
FROM columnar_test_helpers.columnar_store_memory_stats() \gset
|
||||
|
||||
-- insert a large batch
|
||||
INSERT INTO t
|
||||
SELECT i, 'large batch',
|
||||
-- sample memusage instead of recording everyr row for speed
|
||||
CASE WHEN i % 100 = 0 THEN top_memory_context_usage() ELSE 0 END
|
||||
CASE WHEN i % 100 = 0 THEN columnar_test_helpers.top_memory_context_usage() ELSE 0 END
|
||||
FROM generate_series(1, 100000) i;
|
||||
|
||||
COMMIT;
|
||||
|
@ -70,7 +58,7 @@ COMMIT;
|
|||
-- measure memory after doing writes
|
||||
SELECT TopMemoryContext as top_post,
|
||||
WriteStateContext write_post
|
||||
FROM column_store_memory_stats() \gset
|
||||
FROM columnar_test_helpers.columnar_store_memory_stats() \gset
|
||||
|
||||
\x
|
||||
SELECT (1.0 * :top2/:top1 BETWEEN 0.99 AND 1.01) AS top_growth_ok,
|
||||
|
@ -86,7 +74,7 @@ INSERT INTO t
|
|||
FROM generate_series(1, 50000) i;
|
||||
|
||||
SELECT 1.0 * TopMemoryContext / :top_post BETWEEN 0.98 AND 1.02 AS top_growth_ok
|
||||
FROM column_store_memory_stats();
|
||||
FROM columnar_test_helpers.columnar_store_memory_stats();
|
||||
|
||||
-- before this change, max mem usage while executing inserts was 28MB and
|
||||
-- with this change it's less than 8MB.
|
||||
|
|
|
@ -16,7 +16,7 @@ INSERT INTO t2 SELECT i, f(i) FROM generate_series(1, 5) i;
|
|||
-- there are no subtransactions, so above statement should batch
|
||||
-- INSERTs inside the UDF and create on stripe per table.
|
||||
SELECT relname, count(*) FROM columnar.stripe a, pg_class b
|
||||
WHERE columnar_relation_storageid(b.oid)=a.storage_id AND relname IN ('t1', 't2')
|
||||
WHERE columnar_test_helpers.columnar_relation_storageid(b.oid)=a.storage_id AND relname IN ('t1', 't2')
|
||||
GROUP BY relname
|
||||
ORDER BY relname;
|
||||
|
||||
|
@ -39,7 +39,7 @@ INSERT INTO t2 SELECT t.a, t.a+1 FROM t;
|
|||
SELECT * FROM t1;
|
||||
SELECT * FROM t2;
|
||||
|
||||
SELECT * FROM chunk_group_consistency;
|
||||
SELECT * FROM columnar_test_helpers.chunk_group_consistency;
|
||||
|
||||
TRUNCATE t1;
|
||||
TRUNCATE t2;
|
||||
|
@ -57,7 +57,7 @@ INSERT INTO t2 SELECT i, (select count(*) from t1) FROM generate_series(1, 3) i;
|
|||
SELECT * FROM t1;
|
||||
SELECT * FROM t2;
|
||||
|
||||
SELECT * FROM chunk_group_consistency;
|
||||
SELECT * FROM columnar_test_helpers.chunk_group_consistency;
|
||||
|
||||
TRUNCATE t1;
|
||||
TRUNCATE t2;
|
||||
|
@ -72,7 +72,7 @@ INSERT INTO t1 SELECT t.a, t.a+1 FROM t;
|
|||
|
||||
SELECT * FROM t1 ORDER BY a, b;
|
||||
|
||||
SELECT * FROM chunk_group_consistency;
|
||||
SELECT * FROM columnar_test_helpers.chunk_group_consistency;
|
||||
|
||||
TRUNCATE t1;
|
||||
TRUNCATE t2;
|
||||
|
@ -105,7 +105,7 @@ INSERT INTO t4 SELECT i, g2(i) FROM generate_series(1, 5) i;
|
|||
|
||||
SELECT * FROM t2 ORDER BY a, b;
|
||||
|
||||
SELECT * FROM chunk_group_consistency;
|
||||
SELECT * FROM columnar_test_helpers.chunk_group_consistency;
|
||||
|
||||
TRUNCATE t1, t2, t3, t4;
|
||||
|
||||
|
@ -121,7 +121,7 @@ SELECT * FROM t3 ORDER BY a, b;
|
|||
((table t1) except (table t3)) union ((table t3) except (table t1));
|
||||
((table t2) except (table t4)) union ((table t4) except (table t2));
|
||||
|
||||
SELECT * FROM chunk_group_consistency;
|
||||
SELECT * FROM columnar_test_helpers.chunk_group_consistency;
|
||||
|
||||
DROP FUNCTION g(int), g2(int);
|
||||
TRUNCATE t1, t2, t3, t4;
|
||||
|
@ -148,7 +148,7 @@ SELECT f(0), f(20);
|
|||
|
||||
SELECT * FROM t1 ORDER BY a, b;
|
||||
|
||||
SELECT * FROM chunk_group_consistency;
|
||||
SELECT * FROM columnar_test_helpers.chunk_group_consistency;
|
||||
|
||||
DROP FUNCTION f(int);
|
||||
DROP TABLE t1, t2, t3, t4;
|
||||
|
|
|
@ -6,7 +6,7 @@ CREATE TABLE t(a int, b int) USING columnar;
|
|||
|
||||
CREATE VIEW t_stripes AS
|
||||
SELECT * FROM columnar.stripe a, pg_class b
|
||||
WHERE a.storage_id = columnar_relation_storageid(b.oid) AND b.relname = 't';
|
||||
WHERE a.storage_id = columnar_test_helpers.columnar_relation_storageid(b.oid) AND b.relname = 't';
|
||||
|
||||
BEGIN;
|
||||
INSERT INTO t SELECT i, i+1 FROM generate_series(1, 10) i;
|
||||
|
|
|
@ -0,0 +1,75 @@
|
|||
CREATE SCHEMA columnar_test_helpers;
|
||||
SET search_path TO columnar_test_helpers;
|
||||
|
||||
CREATE FUNCTION columnar_relation_storageid(relid oid) RETURNS bigint
|
||||
LANGUAGE C STABLE STRICT
|
||||
AS 'citus', $$columnar_relation_storageid$$;
|
||||
|
||||
CREATE FUNCTION compression_type_supported(type text) RETURNS boolean
|
||||
AS $$
|
||||
BEGIN
|
||||
EXECUTE 'SET LOCAL columnar.compression TO ' || quote_literal(type);
|
||||
return true;
|
||||
EXCEPTION WHEN invalid_parameter_value THEN
|
||||
return false;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
-- are chunk groups and chunks consistent?
|
||||
CREATE view chunk_group_consistency AS
|
||||
WITH a as (
|
||||
SELECT storage_id, stripe_num, chunk_group_num, min(value_count) as row_count
|
||||
FROM columnar.chunk
|
||||
GROUP BY 1,2,3
|
||||
), b as (
|
||||
SELECT storage_id, stripe_num, chunk_group_num, max(value_count) as row_count
|
||||
FROM columnar.chunk
|
||||
GROUP BY 1,2,3
|
||||
), c as (
|
||||
(TABLE a EXCEPT TABLE b) UNION (TABLE b EXCEPT TABLE a) UNION
|
||||
(TABLE a EXCEPT TABLE columnar.chunk_group) UNION (TABLE columnar.chunk_group EXCEPT TABLE a)
|
||||
), d as (
|
||||
SELECT storage_id, stripe_num, count(*) as chunk_group_count
|
||||
FROM columnar.chunk_group
|
||||
GROUP BY 1,2
|
||||
), e as (
|
||||
SELECT storage_id, stripe_num, chunk_group_count
|
||||
FROM columnar.stripe
|
||||
), f as (
|
||||
(TABLE d EXCEPT TABLE e) UNION (TABLE e EXCEPT TABLE d)
|
||||
)
|
||||
SELECT (SELECT count(*) = 0 FROM c) AND
|
||||
(SELECT count(*) = 0 FROM f) as consistent;
|
||||
|
||||
CREATE FUNCTION columnar_metadata_has_storage_id(input_storage_id bigint) RETURNS boolean
|
||||
AS $$
|
||||
DECLARE
|
||||
union_storage_id_count integer;
|
||||
BEGIN
|
||||
SELECT count(*) INTO union_storage_id_count FROM
|
||||
(
|
||||
SELECT storage_id FROM columnar.stripe UNION ALL
|
||||
SELECT storage_id FROM columnar.chunk UNION ALL
|
||||
SELECT storage_id FROM columnar.chunk_group
|
||||
) AS union_storage_id
|
||||
WHERE storage_id=input_storage_id;
|
||||
|
||||
IF union_storage_id_count > 0 THEN
|
||||
RETURN true;
|
||||
END IF;
|
||||
|
||||
RETURN false;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
CREATE OR REPLACE FUNCTION columnar_store_memory_stats()
|
||||
RETURNS TABLE(TopMemoryContext BIGINT,
|
||||
TopTransactionContext BIGINT,
|
||||
WriteStateContext BIGINT)
|
||||
LANGUAGE C STRICT VOLATILE
|
||||
AS 'citus', $$columnar_store_memory_stats$$;
|
||||
|
||||
CREATE FUNCTION top_memory_context_usage()
|
||||
RETURNS BIGINT AS $$
|
||||
SELECT TopMemoryContext FROM columnar_test_helpers.columnar_store_memory_stats();
|
||||
$$ LANGUAGE SQL VOLATILE;
|
|
@ -25,11 +25,11 @@ set columnar.compression to default;
|
|||
-- query rows
|
||||
SELECT * FROM columnar_truncate_test;
|
||||
|
||||
SELECT * FROM chunk_group_consistency;
|
||||
SELECT * FROM columnar_test_helpers.chunk_group_consistency;
|
||||
|
||||
TRUNCATE TABLE columnar_truncate_test;
|
||||
|
||||
SELECT * FROM chunk_group_consistency;
|
||||
SELECT * FROM columnar_test_helpers.chunk_group_consistency;
|
||||
|
||||
SELECT * FROM columnar_truncate_test;
|
||||
|
||||
|
@ -51,7 +51,7 @@ SELECT * from columnar_truncate_test_second;
|
|||
|
||||
SELECT * from columnar_truncate_test_regular;
|
||||
|
||||
SELECT * FROM chunk_group_consistency;
|
||||
SELECT * FROM columnar_test_helpers.chunk_group_consistency;
|
||||
|
||||
-- make sure multi truncate works
|
||||
-- notice that the same table might be repeated
|
||||
|
@ -60,7 +60,7 @@ TRUNCATE TABLE columnar_truncate_test,
|
|||
columnar_truncate_test_second,
|
||||
columnar_truncate_test;
|
||||
|
||||
SELECT * FROM chunk_group_consistency;
|
||||
SELECT * FROM columnar_test_helpers.chunk_group_consistency;
|
||||
|
||||
SELECT * from columnar_truncate_test;
|
||||
SELECT * from columnar_truncate_test_second;
|
||||
|
@ -144,7 +144,7 @@ TRUNCATE TABLE truncate_schema.truncate_tbl;
|
|||
SELECT count(*) FROM truncate_schema.truncate_tbl;
|
||||
\c - :current_user
|
||||
|
||||
SELECT * FROM chunk_group_consistency;
|
||||
SELECT * FROM columnar_test_helpers.chunk_group_consistency;
|
||||
|
||||
-- cleanup
|
||||
DROP SCHEMA truncate_schema CASCADE;
|
||||
|
|
|
@ -6,7 +6,7 @@ CREATE TABLE t(a int, b int) USING columnar;
|
|||
|
||||
CREATE VIEW t_stripes AS
|
||||
SELECT * FROM columnar.stripe a, pg_class b
|
||||
WHERE a.storage_id = columnar_relation_storageid(b.oid) AND b.relname='t';
|
||||
WHERE a.storage_id = columnar_test_helpers.columnar_relation_storageid(b.oid) AND b.relname='t';
|
||||
|
||||
SELECT count(*) FROM t_stripes;
|
||||
|
||||
|
@ -20,7 +20,7 @@ SELECT count(*) FROM t_stripes;
|
|||
-- vacuum full should merge stripes together
|
||||
VACUUM FULL t;
|
||||
|
||||
SELECT * FROM chunk_group_consistency;
|
||||
SELECT * FROM columnar_test_helpers.chunk_group_consistency;
|
||||
|
||||
SELECT sum(a), sum(b) FROM t;
|
||||
SELECT count(*) FROM t_stripes;
|
||||
|
@ -34,7 +34,7 @@ SELECT count(*) FROM t_stripes;
|
|||
|
||||
VACUUM FULL t;
|
||||
|
||||
SELECT * FROM chunk_group_consistency;
|
||||
SELECT * FROM columnar_test_helpers.chunk_group_consistency;
|
||||
|
||||
SELECT sum(a), sum(b) FROM t;
|
||||
SELECT count(*) FROM t_stripes;
|
||||
|
@ -44,13 +44,13 @@ ALTER TABLE t DROP COLUMN a;
|
|||
|
||||
SELECT stripe_num, attr_num, chunk_group_num, minimum_value IS NULL, maximum_value IS NULL
|
||||
FROM columnar.chunk a, pg_class b
|
||||
WHERE a.storage_id = columnar_relation_storageid(b.oid) AND b.relname='t' ORDER BY 1, 2, 3;
|
||||
WHERE a.storage_id = columnar_test_helpers.columnar_relation_storageid(b.oid) AND b.relname='t' ORDER BY 1, 2, 3;
|
||||
|
||||
VACUUM FULL t;
|
||||
|
||||
SELECT stripe_num, attr_num, chunk_group_num, minimum_value IS NULL, maximum_value IS NULL
|
||||
FROM columnar.chunk a, pg_class b
|
||||
WHERE a.storage_id = columnar_relation_storageid(b.oid) AND b.relname='t' ORDER BY 1, 2, 3;
|
||||
WHERE a.storage_id = columnar_test_helpers.columnar_relation_storageid(b.oid) AND b.relname='t' ORDER BY 1, 2, 3;
|
||||
|
||||
-- Make sure we cleaned-up the transient table metadata after VACUUM FULL commands
|
||||
SELECT count(distinct storage_id) - :columnar_table_count FROM columnar.stripe;
|
||||
|
@ -96,7 +96,7 @@ COMMIT;
|
|||
|
||||
VACUUM VERBOSE t;
|
||||
|
||||
SELECT * FROM chunk_group_consistency;
|
||||
SELECT * FROM columnar_test_helpers.chunk_group_consistency;
|
||||
|
||||
SELECT count(*) FROM t;
|
||||
|
||||
|
@ -114,7 +114,7 @@ SELECT alter_columnar_table_set('t', compression => 'pglz');
|
|||
VACUUM FULL t;
|
||||
VACUUM VERBOSE t;
|
||||
|
||||
SELECT * FROM chunk_group_consistency;
|
||||
SELECT * FROM columnar_test_helpers.chunk_group_consistency;
|
||||
|
||||
DROP TABLE t;
|
||||
DROP VIEW t_stripes;
|
||||
|
@ -131,6 +131,6 @@ INSERT INTO t SELECT 1, 'a', 'xyz' FROM generate_series(1, 1000000) i;
|
|||
|
||||
VACUUM VERBOSE t;
|
||||
|
||||
SELECT * FROM chunk_group_consistency;
|
||||
SELECT * FROM columnar_test_helpers.chunk_group_consistency;
|
||||
|
||||
DROP TABLE t;
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
SELECT compression_type_supported('zstd') AS zstd_supported \gset
|
||||
SELECT columnar_test_helpers.compression_type_supported('zstd') AS zstd_supported \gset
|
||||
\if :zstd_supported
|
||||
\else
|
||||
\q
|
||||
|
|
Loading…
Reference in New Issue