external
/
citus
mirror of https://github.com/citusdata/citus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Improve tests for truncating local data (#5012) 

We have a slightly different behavior when using truncate_local_data_after_distributing_table UDF on metadata synced clusters. This PR aims to add tests to cover such cases.

We allow distributing tables with data that have foreign keys to reference tables only on metadata synced clusters. This is the reason why some of my earlier tests failed when run on a single node Citus cluster.
pull/5021/head
Hanefi Onaldi 2021-06-03 08:51:32 +03:00 committed by GitHub
parent 5f76b93eac
commit 056005db4d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 271 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
src/test/regress/expected/multi_truncate.out
View File

@ -8,7 +8,7 @@ SET search_path TO multi_truncate;
CREATE VIEW table_sizes AS
SELECT
c.relname as name,
pg_catalog.pg_size_pretty(pg_catalog.pg_table_size(c.oid)) as size
pg_catalog.pg_table_size(c.oid) > 0 as has_data
FROM pg_catalog.pg_class c
LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
WHERE c.relkind = 'r'
@ -330,7 +330,7 @@ SELECT * FROM test_local_truncate;
SELECT citus_drop_all_shards('test_local_truncate', 'public', 'test_local_truncate');
citus_drop_all_shards
---------------------------------------------------------------------
4
4
(1 row)
DELETE FROM pg_dist_partition WHERE logicalrelid = 'test_local_truncate'::regclass;
@ -369,7 +369,7 @@ SELECT * FROM test_local_truncate;
SELECT citus_drop_all_shards('test_local_truncate', 'public', 'test_local_truncate');
citus_drop_all_shards
---------------------------------------------------------------------
4
4
(1 row)
DELETE FROM pg_dist_partition WHERE logicalrelid = 'test_local_truncate'::regclass;
@ -427,19 +427,19 @@ NOTICE: truncate cascades to table "referencing_table"
(1 row)
SELECT * FROM table_sizes;
name | size
name | has_data
---------------------------------------------------------------------
referenced_table | 0 bytes
referencing_table | 0 bytes
referenced_table | f
referencing_table | f
(2 rows)
ROLLBACK;
-- observe that none of the tables are truncated
SELECT * FROM table_sizes;
name | size
name | has_data
---------------------------------------------------------------------
referenced_table | 384 kB
referencing_table | 384 kB
referenced_table | t
referencing_table | t
(2 rows)
-- test that if we truncate the referencing table, only said table is affected
@ -451,10 +451,10 @@ SELECT truncate_local_data_after_distributing_table('referencing_table');
(1 row)
SELECT * FROM table_sizes;
name | size
name | has_data
---------------------------------------------------------------------
referenced_table | 384 kB
referencing_table | 0 bytes
referenced_table | t
referencing_table | f
(2 rows)
ROLLBACK;
@ -470,10 +470,10 @@ NOTICE: truncate cascades to table "referencing_table"
(1 row)
SELECT * FROM table_sizes;
name | size
name | has_data
---------------------------------------------------------------------
referenced_table | 0 bytes
referencing_table | 0 bytes
referenced_table | f
referencing_table | f
(2 rows)
ROLLBACK;
@ -498,25 +498,56 @@ INSERT INTO dist SELECT x,x FROM generate_series(1,10000) x;
SELECT truncate_local_data_after_distributing_table('ref');
ERROR: cannot truncate a table referenced in a foreign key constraint by a local table
DETAIL: Table "dist" references "ref"
-- distribute the table and start testing allowed truncation queries
-- test that we do not allow distributing tables that have foreign keys to reference tables
SELECT create_distributed_table('dist','id');
ERROR: cannot distribute "dist" in sequential mode because it is not empty
HINT: If you have manually set citus.multi_shard_modify_mode to 'sequential', try with 'parallel' option. If that is not the case, try distributing local tables when they are empty.
SHOW citus.multi_shard_modify_mode;
citus.multi_shard_modify_mode
---------------------------------------------------------------------
parallel
(1 row)
-- distribute the table after a truncate
TRUNCATE dist;
SELECT create_distributed_table('dist','id');
create_distributed_table
---------------------------------------------------------------------
(1 row)
-- the following should truncate ref and dist
BEGIN;
SELECT truncate_local_data_after_distributing_table('ref');
ERROR: cannot truncate a table referenced in a foreign key constraint by a local table
DETAIL: Table "dist" references "ref"
NOTICE: truncate cascades to table "dist"
truncate_local_data_after_distributing_table
---------------------------------------------------------------------
(1 row)
SELECT * FROM table_sizes;
ERROR: current transaction is aborted, commands ignored until end of transaction block
name | has_data
---------------------------------------------------------------------
dist | f
ref | f
(2 rows)
ROLLBACK;
-- the following should truncate dist table only
BEGIN;
SELECT truncate_local_data_after_distributing_table('dist');
ERROR: supplied parameter is not a distributed relation
DETAIL: This UDF only truncates local records of distributed tables.
truncate_local_data_after_distributing_table
---------------------------------------------------------------------
(1 row)
SELECT * FROM table_sizes;
ERROR: current transaction is aborted, commands ignored until end of transaction block
name | has_data
---------------------------------------------------------------------
dist | f
ref | t
(2 rows)
ROLLBACK;
DROP TABLE ref, dist;
-- tests for issue 1770

147
src/test/regress/expected/single_node_truncate.out Normal file
View File

@ -0,0 +1,147 @@
CREATE SCHEMA single_node_truncate;
SET search_path TO single_node_truncate;
SET citus.shard_replication_factor TO 1;
-- helper view that prints out local table names and sizes in the schema
CREATE VIEW table_sizes AS
SELECT
c.relname as name,
pg_catalog.pg_table_size(c.oid) > 0 as has_data
FROM pg_catalog.pg_class c
LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
WHERE c.relkind = 'r'
AND n.nspname = 'single_node_truncate'
ORDER BY 1;
-- test truncating reference tables
CREATE TABLE ref(id int UNIQUE, data int);
INSERT INTO ref SELECT x,x FROM generate_series(1,10000) x;
SELECT create_reference_table('ref');
NOTICE: Copying data from local table...
NOTICE: copying the data has completed
DETAIL: The local data in the table is no longer visible, but is still on disk.
HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$single_node_truncate.ref$$)
create_reference_table
---------------------------------------------------------------------
(1 row)
CREATE TABLE citus_local(id int, ref_id int REFERENCES ref(id));
INSERT INTO citus_local SELECT x,x FROM generate_series(1,10000) x;
-- verify that shell tables for citus local tables are empty
SELECT * FROM table_sizes;
name | has_data
---------------------------------------------------------------------
citus_local | f
citus_local_102041 | t
ref | t
ref_102040 | t
(4 rows)
-- verify that this UDF is noop on Citus local tables
SELECT truncate_local_data_after_distributing_table('citus_local');
truncate_local_data_after_distributing_table
---------------------------------------------------------------------
(1 row)
SELECT * FROM table_sizes;
name | has_data
---------------------------------------------------------------------
citus_local | f
citus_local_102041 | t
ref | t
ref_102040 | t
(4 rows)
-- test that we allow cascading truncates to citus local tables
BEGIN;
SELECT truncate_local_data_after_distributing_table('ref');
NOTICE: truncate cascades to table "citus_local"
truncate_local_data_after_distributing_table
---------------------------------------------------------------------
(1 row)
SELECT * FROM table_sizes;
name | has_data
---------------------------------------------------------------------
citus_local | f
citus_local_102041 | t
ref | f
ref_102040 | t
(4 rows)
ROLLBACK;
-- test that we allow distributing tables that have foreign keys to reference tables
CREATE TABLE dist(id int, ref_id int REFERENCES ref(id));
INSERT INTO dist SELECT x,x FROM generate_series(1,10000) x;
SELECT create_distributed_table('dist','id');
NOTICE: Copying data from local table...
NOTICE: copying the data has completed
DETAIL: The local data in the table is no longer visible, but is still on disk.
HINT: To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$single_node_truncate.dist$$)
create_distributed_table
---------------------------------------------------------------------
(1 row)
-- the following should truncate ref, dist and citus_local
BEGIN;
SELECT truncate_local_data_after_distributing_table('ref');
NOTICE: truncate cascades to table "citus_local"
NOTICE: truncate cascades to table "dist"
truncate_local_data_after_distributing_table
---------------------------------------------------------------------
(1 row)
SELECT * FROM table_sizes;
name | has_data
---------------------------------------------------------------------
citus_local | f
citus_local_102041 | t
dist | f
dist_102043 | t
dist_102044 | t
dist_102045 | t
dist_102046 | t
ref | f
ref_102040 | t
(9 rows)
ROLLBACK;
-- the following should truncate dist table only
BEGIN;
SELECT truncate_local_data_after_distributing_table('dist');
truncate_local_data_after_distributing_table
---------------------------------------------------------------------
(1 row)
SELECT * FROM table_sizes;
name | has_data
---------------------------------------------------------------------
citus_local | f
citus_local_102041 | t
dist | f
dist_102043 | t
dist_102044 | t
dist_102045 | t
dist_102046 | t
ref | t
ref_102040 | t
(9 rows)
ROLLBACK;
DROP TABLE ref, dist, citus_local;
DROP VIEW table_sizes;
DROP SCHEMA single_node_truncate CASCADE;
-- Remove the coordinator
SELECT 1 FROM master_remove_node('localhost', :master_port);
?column?
---------------------------------------------------------------------
1
(1 row)
-- restart nodeid sequence so that multi_cluster_management still has the same
-- nodeids
ALTER SEQUENCE pg_dist_node_nodeid_seq RESTART 1;

1
src/test/regress/multi_schedule
View File

@ -17,6 +17,7 @@
# ---
test: multi_extension
test: single_node
test: single_node_truncate
test: multi_cluster_management
# below tests are placed right after multi_cluster_management as we do

9
src/test/regress/sql/multi_truncate.sql
View File

@ -11,7 +11,7 @@ SET search_path TO multi_truncate;
CREATE VIEW table_sizes AS
SELECT
c.relname as name,
pg_catalog.pg_size_pretty(pg_catalog.pg_table_size(c.oid)) as size
pg_catalog.pg_table_size(c.oid) > 0 as has_data
FROM pg_catalog.pg_class c
LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
WHERE c.relkind = 'r'
@ -292,7 +292,12 @@ INSERT INTO dist SELECT x,x FROM generate_series(1,10000) x;
-- test that we do not cascade truncates to local referencing tables
SELECT truncate_local_data_after_distributing_table('ref');
-- distribute the table and start testing allowed truncation queries
-- test that we do not allow distributing tables that have foreign keys to reference tables
SELECT create_distributed_table('dist','id');
SHOW citus.multi_shard_modify_mode;
-- distribute the table after a truncate
TRUNCATE dist;
SELECT create_distributed_table('dist','id');
-- the following should truncate ref and dist

63
src/test/regress/sql/single_node_truncate.sql Normal file
View File

@ -0,0 +1,63 @@
CREATE SCHEMA single_node_truncate;
SET search_path TO single_node_truncate;
SET citus.shard_replication_factor TO 1;
-- helper view that prints out local table names and sizes in the schema
CREATE VIEW table_sizes AS
SELECT
c.relname as name,
pg_catalog.pg_table_size(c.oid) > 0 as has_data
FROM pg_catalog.pg_class c
LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
WHERE c.relkind = 'r'
AND n.nspname = 'single_node_truncate'
ORDER BY 1;
-- test truncating reference tables
CREATE TABLE ref(id int UNIQUE, data int);
INSERT INTO ref SELECT x,x FROM generate_series(1,10000) x;
SELECT create_reference_table('ref');
CREATE TABLE citus_local(id int, ref_id int REFERENCES ref(id));
INSERT INTO citus_local SELECT x,x FROM generate_series(1,10000) x;
-- verify that shell tables for citus local tables are empty
SELECT * FROM table_sizes;
-- verify that this UDF is noop on Citus local tables
SELECT truncate_local_data_after_distributing_table('citus_local');
SELECT * FROM table_sizes;
-- test that we allow cascading truncates to citus local tables
BEGIN;
SELECT truncate_local_data_after_distributing_table('ref');
SELECT * FROM table_sizes;
ROLLBACK;
-- test that we allow distributing tables that have foreign keys to reference tables
CREATE TABLE dist(id int, ref_id int REFERENCES ref(id));
INSERT INTO dist SELECT x,x FROM generate_series(1,10000) x;
SELECT create_distributed_table('dist','id');
-- the following should truncate ref, dist and citus_local
BEGIN;
SELECT truncate_local_data_after_distributing_table('ref');
SELECT * FROM table_sizes;
ROLLBACK;
-- the following should truncate dist table only
BEGIN;
SELECT truncate_local_data_after_distributing_table('dist');
SELECT * FROM table_sizes;
ROLLBACK;
DROP TABLE ref, dist, citus_local;
DROP VIEW table_sizes;
DROP SCHEMA single_node_truncate CASCADE;
-- Remove the coordinator
SELECT 1 FROM master_remove_node('localhost', :master_port);
-- restart nodeid sequence so that multi_cluster_management still has the same
-- nodeids
ALTER SEQUENCE pg_dist_node_nodeid_seq RESTART 1;