From ce11ab26acc3d5037ee16809af212886337718a6 Mon Sep 17 00:00:00 2001 From: Nitish Upreti Date: Wed, 22 Jun 2022 10:27:22 -0700 Subject: [PATCH] Local and Remote binary test --- src/test/regress/Makefile | 6 +- ...itus_split_shard_by_split_points_local.out | 0 .../worker_split_binary_copy_test.out | 145 ++++++++++++++++++ src/test/regress/split_schedule | 8 + .../sql/worker_split_binary_copy_test.sql | 95 ++++++++++++ 5 files changed, 253 insertions(+), 1 deletion(-) create mode 100644 src/test/regress/expected/citus_split_shard_by_split_points_local.out create mode 100644 src/test/regress/expected/worker_split_binary_copy_test.out create mode 100644 src/test/regress/split_schedule create mode 100644 src/test/regress/sql/worker_split_binary_copy_test.sql diff --git a/src/test/regress/Makefile b/src/test/regress/Makefile index ec42e0556..5b0135fb2 100644 --- a/src/test/regress/Makefile +++ b/src/test/regress/Makefile @@ -43,7 +43,7 @@ output_files := $(patsubst $(citus_abs_srcdir)/output/%.source,expected/%.out, $ # intermediate, for muscle memory backward compatibility. check: check-full check-enterprise-full # check-full triggers all tests that ought to be run routinely -check-full: check-multi check-multi-mx check-multi-1 check-operations check-follower-cluster check-isolation check-failure +check-full: check-multi check-multi-mx check-multi-1 check-operations check-follower-cluster check-isolation check-failure check-split # check-enterprise-full triggers all enterprise specific tests check-enterprise-full: check-enterprise check-enterprise-isolation check-enterprise-failure @@ -216,6 +216,10 @@ check-columnar-isolation: all $(isolation_test_files) $(pg_regress_multi_check) --load-extension=citus --isolationtester \ -- $(MULTI_REGRESS_OPTS) --inputdir=$(citus_abs_srcdir)/build --schedule=$(citus_abs_srcdir)/columnar_isolation_schedule $(EXTRA_TESTS) +check-split: all + $(pg_regress_multi_check) --load-extension=citus \ + -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/split_schedule $(EXTRA_TESTS) + check-failure: all $(pg_regress_multi_check) --load-extension=citus --mitmproxy \ -- $(MULTI_REGRESS_OPTS) --schedule=$(citus_abs_srcdir)/failure_schedule $(EXTRA_TESTS) diff --git a/src/test/regress/expected/citus_split_shard_by_split_points_local.out b/src/test/regress/expected/citus_split_shard_by_split_points_local.out new file mode 100644 index 000000000..e69de29bb diff --git a/src/test/regress/expected/worker_split_binary_copy_test.out b/src/test/regress/expected/worker_split_binary_copy_test.out new file mode 100644 index 000000000..429a76b1d --- /dev/null +++ b/src/test/regress/expected/worker_split_binary_copy_test.out @@ -0,0 +1,145 @@ +CREATE SCHEMA worker_split_binary_copy_test; +SET search_path TO worker_split_binary_copy_test; +SET citus.shard_count TO 1; +SET citus.shard_replication_factor TO 1; +SET citus.next_shard_id TO 81060000; +-- BEGIN: Create distributed table and insert data. +CREATE TABLE worker_split_binary_copy_test.shard_to_split_copy (id bigserial PRIMARY KEY, value char); +SELECT create_distributed_table('shard_to_split_copy','id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO worker_split_binary_copy_test.shard_to_split_copy (id, value) (SELECT g.id, 'c' FROM generate_series(1, 100) AS g(id)); +-- END: Create distributed table and insert data. +-- BEGIN: Switch to Worker1, Create target shards in worker for local 2-way split copy. +\c - - - :worker_1_port +CREATE TABLE worker_split_binary_copy_test.shard_to_split_copy_81060015 (id bigserial PRIMARY KEY, value char); +CREATE TABLE worker_split_binary_copy_test.shard_to_split_copy_81060016 (id bigserial PRIMARY KEY, value char); +-- End: Switch to Worker1, Create target shards in worker for local 2-way split copy. +-- BEGIN: Switch to Worker2, Create target shards in worker for remote 2-way split copy. +\c - - - :worker_2_port +CREATE TABLE worker_split_binary_copy_test.shard_to_split_copy_81060015 (id bigserial PRIMARY KEY, value char); +CREATE TABLE worker_split_binary_copy_test.shard_to_split_copy_81060016 (id bigserial PRIMARY KEY, value char); +-- End: Switch to Worker2, Create target shards in worker for remote 2-way split copy. +-- BEGIN: List row count for source shard and targets shard in Worker1. +\c - - - :worker_1_port +SELECT COUNT(*) FROM worker_split_binary_copy_test.shard_to_split_copy_81060000; + count +--------------------------------------------------------------------- + 100 +(1 row) + +SELECT COUNT(*) FROM worker_split_binary_copy_test.shard_to_split_copy_81060015; + count +--------------------------------------------------------------------- + 0 +(1 row) + +SELECT COUNT(*) FROM worker_split_binary_copy_test.shard_to_split_copy_81060016; + count +--------------------------------------------------------------------- + 0 +(1 row) + +-- END: List row count for source shard and targets shard in Worker1. +-- BEGIN: List row count for target shard in Worker2. +\c - - - :worker_2_port +SELECT COUNT(*) FROM worker_split_binary_copy_test.shard_to_split_copy_81060015; + count +--------------------------------------------------------------------- + 0 +(1 row) + +SELECT COUNT(*) FROM worker_split_binary_copy_test.shard_to_split_copy_81060016; + count +--------------------------------------------------------------------- + 0 +(1 row) + +-- END: List row count for targets shard in Worker2. +-- BEGIN: Set worker_1_node and worker_2_node +\c - - - :worker_1_port +SELECT nodeid AS worker_1_node FROM pg_dist_node WHERE nodeport=:worker_1_port \gset +SELECT nodeid AS worker_2_node FROM pg_dist_node WHERE nodeport=:worker_2_port \gset +-- END: Set worker_1_node and worker_2_node +-- BEGIN: Trigger 2-way local shard split copy. +SELECT * from worker_split_copy( + 81060000, -- source shard id to copy + ARRAY[ + -- split copy info for split children 1 + ROW(81060015, -- destination shard id + -2147483648, -- split range begin + 1073741823, --split range end + :worker_1_node)::citus.split_copy_info, + -- split copy info for split children 2 + ROW(81060016, --destination shard id + 1073741824, --split range begin + 2147483647, --split range end + :worker_1_node)::citus.split_copy_info + ] + ); + worker_split_copy +--------------------------------------------------------------------- + +(1 row) + +-- END: Trigger 2-way local shard split copy. +-- BEGIN: Trigger 2-way remote shard split copy. +SELECT * from worker_split_copy( + 81060000, -- source shard id to copy + ARRAY[ + -- split copy info for split children 1 + ROW(81060015, -- destination shard id + -2147483648, -- split range begin + 1073741823, --split range end + :worker_2_node)::citus.split_copy_info, + -- split copy info for split children 2 + ROW(81060016, --destination shard id + 1073741824, --split range begin + 2147483647, --split range end + :worker_2_node)::citus.split_copy_info + ] + ); + worker_split_copy +--------------------------------------------------------------------- + +(1 row) + +-- END: Trigger 2-way remote shard split copy. +-- BEGIN: List updated row count for local targets shard. +SELECT COUNT(*) FROM worker_split_binary_copy_test.shard_to_split_copy_81060015; + count +--------------------------------------------------------------------- + 72 +(1 row) + +SELECT COUNT(*) FROM worker_split_binary_copy_test.shard_to_split_copy_81060016; + count +--------------------------------------------------------------------- + 28 +(1 row) + +-- END: List updated row count for local targets shard. +-- BEGIN: List updated row count for remote targets shard. +\c - - - :worker_2_port +SELECT COUNT(*) FROM worker_split_binary_copy_test.shard_to_split_copy_81060015; + count +--------------------------------------------------------------------- + 72 +(1 row) + +SELECT COUNT(*) FROM worker_split_binary_copy_test.shard_to_split_copy_81060016; + count +--------------------------------------------------------------------- + 28 +(1 row) + +-- END: List updated row count for remote targets shard. +-- BEGIN: CLEANUP. +\c - - - :master_port +SET client_min_messages TO WARNING; +DROP SCHEMA citus_split_shard_by_split_points_local CASCADE; +ERROR: schema "citus_split_shard_by_split_points_local" does not exist +-- END: CLEANUP. diff --git a/src/test/regress/split_schedule b/src/test/regress/split_schedule new file mode 100644 index 000000000..1dd462aef --- /dev/null +++ b/src/test/regress/split_schedule @@ -0,0 +1,8 @@ +# Split Shard tests. +# Include tests from 'minimal_schedule' for setup. +test: multi_test_helpers multi_test_helpers_superuser columnar_test_helpers +test: multi_cluster_management +test: multi_test_catalog_views +test: tablespace +# Split tests go here. +test: worker_split_binary_copy_test diff --git a/src/test/regress/sql/worker_split_binary_copy_test.sql b/src/test/regress/sql/worker_split_binary_copy_test.sql new file mode 100644 index 000000000..8e5416bb3 --- /dev/null +++ b/src/test/regress/sql/worker_split_binary_copy_test.sql @@ -0,0 +1,95 @@ +CREATE SCHEMA worker_split_binary_copy_test; +SET search_path TO worker_split_binary_copy_test; +SET citus.shard_count TO 1; +SET citus.shard_replication_factor TO 1; +SET citus.next_shard_id TO 81060000; + +-- BEGIN: Create distributed table and insert data. +CREATE TABLE worker_split_binary_copy_test.shard_to_split_copy (id bigserial PRIMARY KEY, value char); +SELECT create_distributed_table('shard_to_split_copy','id'); +INSERT INTO worker_split_binary_copy_test.shard_to_split_copy (id, value) (SELECT g.id, 'c' FROM generate_series(1, 100) AS g(id)); +-- END: Create distributed table and insert data. + +-- BEGIN: Switch to Worker1, Create target shards in worker for local 2-way split copy. +\c - - - :worker_1_port +CREATE TABLE worker_split_binary_copy_test.shard_to_split_copy_81060015 (id bigserial PRIMARY KEY, value char); +CREATE TABLE worker_split_binary_copy_test.shard_to_split_copy_81060016 (id bigserial PRIMARY KEY, value char); +-- End: Switch to Worker1, Create target shards in worker for local 2-way split copy. + +-- BEGIN: Switch to Worker2, Create target shards in worker for remote 2-way split copy. +\c - - - :worker_2_port +CREATE TABLE worker_split_binary_copy_test.shard_to_split_copy_81060015 (id bigserial PRIMARY KEY, value char); +CREATE TABLE worker_split_binary_copy_test.shard_to_split_copy_81060016 (id bigserial PRIMARY KEY, value char); +-- End: Switch to Worker2, Create target shards in worker for remote 2-way split copy. + +-- BEGIN: List row count for source shard and targets shard in Worker1. +\c - - - :worker_1_port +SELECT COUNT(*) FROM worker_split_binary_copy_test.shard_to_split_copy_81060000; +SELECT COUNT(*) FROM worker_split_binary_copy_test.shard_to_split_copy_81060015; +SELECT COUNT(*) FROM worker_split_binary_copy_test.shard_to_split_copy_81060016; +-- END: List row count for source shard and targets shard in Worker1. + +-- BEGIN: List row count for target shard in Worker2. +\c - - - :worker_2_port +SELECT COUNT(*) FROM worker_split_binary_copy_test.shard_to_split_copy_81060015; +SELECT COUNT(*) FROM worker_split_binary_copy_test.shard_to_split_copy_81060016; +-- END: List row count for targets shard in Worker2. + +-- BEGIN: Set worker_1_node and worker_2_node +\c - - - :worker_1_port +SELECT nodeid AS worker_1_node FROM pg_dist_node WHERE nodeport=:worker_1_port \gset +SELECT nodeid AS worker_2_node FROM pg_dist_node WHERE nodeport=:worker_2_port \gset +-- END: Set worker_1_node and worker_2_node + +-- BEGIN: Trigger 2-way local shard split copy. +SELECT * from worker_split_copy( + 81060000, -- source shard id to copy + ARRAY[ + -- split copy info for split children 1 + ROW(81060015, -- destination shard id + -2147483648, -- split range begin + 1073741823, --split range end + :worker_1_node)::citus.split_copy_info, + -- split copy info for split children 2 + ROW(81060016, --destination shard id + 1073741824, --split range begin + 2147483647, --split range end + :worker_1_node)::citus.split_copy_info + ] + ); +-- END: Trigger 2-way local shard split copy. + +-- BEGIN: Trigger 2-way remote shard split copy. +SELECT * from worker_split_copy( + 81060000, -- source shard id to copy + ARRAY[ + -- split copy info for split children 1 + ROW(81060015, -- destination shard id + -2147483648, -- split range begin + 1073741823, --split range end + :worker_2_node)::citus.split_copy_info, + -- split copy info for split children 2 + ROW(81060016, --destination shard id + 1073741824, --split range begin + 2147483647, --split range end + :worker_2_node)::citus.split_copy_info + ] + ); +-- END: Trigger 2-way remote shard split copy. + +-- BEGIN: List updated row count for local targets shard. +SELECT COUNT(*) FROM worker_split_binary_copy_test.shard_to_split_copy_81060015; +SELECT COUNT(*) FROM worker_split_binary_copy_test.shard_to_split_copy_81060016; +-- END: List updated row count for local targets shard. + +-- BEGIN: List updated row count for remote targets shard. +\c - - - :worker_2_port +SELECT COUNT(*) FROM worker_split_binary_copy_test.shard_to_split_copy_81060015; +SELECT COUNT(*) FROM worker_split_binary_copy_test.shard_to_split_copy_81060016; +-- END: List updated row count for remote targets shard. + +-- BEGIN: CLEANUP. +\c - - - :master_port +SET client_min_messages TO WARNING; +DROP SCHEMA citus_split_shard_by_split_points_local CASCADE; +-- END: CLEANUP.