mirror of https://github.com/citusdata/citus.git
207 lines
6.9 KiB
Plaintext
207 lines
6.9 KiB
Plaintext
--
|
|
-- failure_offline_move_shard_placement
|
|
--
|
|
-- The tests cover moving shard placements without using logical replication.
|
|
CREATE SCHEMA IF NOT EXISTS move_shard_offline;
|
|
SET SEARCH_PATH = move_shard_offline;
|
|
SET citus.shard_count TO 4;
|
|
SET citus.next_shard_id TO 200;
|
|
SET citus.shard_replication_factor TO 1;
|
|
SELECT pg_backend_pid() as pid \gset
|
|
SELECT citus.mitmproxy('conn.allow()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
CREATE TABLE t(id int PRIMARY KEY, int_data int);
|
|
SELECT create_distributed_table('t', 'id');
|
|
create_distributed_table
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
CREATE VIEW shards_in_workers AS
|
|
SELECT shardid,
|
|
(CASE WHEN nodeport = :worker_1_port THEN 'worker1' ELSE 'worker2' END) AS worker
|
|
FROM pg_dist_placement NATURAL JOIN pg_dist_node
|
|
WHERE shardstate != 4
|
|
ORDER BY 1,2 ASC;
|
|
CREATE VIEW indices_on_shard_201 AS
|
|
SELECT * FROM run_command_on_workers( $cmd$
|
|
SELECT CASE WHEN COUNT(*) > 0 THEN TRUE ELSE FALSE END
|
|
FROM pg_index WHERE indrelid = 'move_shard_offline.t_201'::regclass
|
|
$cmd$);
|
|
CREATE VIEW find_index_for_shard_201_in_workers AS
|
|
SELECT CASE nodeport WHEN :worker_1_port THEN 'worker1' ELSE 'worker2' END
|
|
FROM indices_on_shard_201 WHERE result = 't';
|
|
-- Insert some data
|
|
INSERT INTO t SELECT x, x+1 FROM generate_series(1,100) AS f(x);
|
|
-- Initial shard placements
|
|
SELECT * FROM shards_in_workers;
|
|
shardid | worker
|
|
---------------------------------------------------------------------
|
|
200 | worker2
|
|
201 | worker1
|
|
202 | worker2
|
|
203 | worker1
|
|
(4 rows)
|
|
|
|
SELECT * FROM find_index_for_shard_201_in_workers;
|
|
case
|
|
---------------------------------------------------------------------
|
|
worker1
|
|
(1 row)
|
|
|
|
-- failure on sanity checks
|
|
SELECT citus.mitmproxy('conn.onQuery(query="DROP TABLE IF EXISTS move_shard_offline.t CASCADE").kill()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT master_move_shard_placement(201, 'localhost', :worker_1_port, 'localhost', :worker_2_proxy_port, 'block_writes');
|
|
ERROR: connection not open
|
|
CONTEXT: while executing command on localhost:xxxxx
|
|
-- cancellation on sanity checks
|
|
SELECT citus.mitmproxy('conn.onQuery(query="DROP TABLE IF EXISTS move_shard_offline.t CASCADE").cancel(' || :pid || ')');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT master_move_shard_placement(201, 'localhost', :worker_1_port, 'localhost', :worker_2_proxy_port, 'block_writes');
|
|
ERROR: canceling statement due to user request
|
|
-- failure on move_shard table creation
|
|
SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE move_shard_offline.t").kill()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT master_move_shard_placement(201, 'localhost', :worker_1_port, 'localhost', :worker_2_proxy_port, 'block_writes');
|
|
ERROR: connection not open
|
|
CONTEXT: while executing command on localhost:xxxxx
|
|
-- cancellation on move_shard table creation
|
|
SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE move_shard_offline.t").cancel(' || :pid || ')');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT master_move_shard_placement(201, 'localhost', :worker_1_port, 'localhost', :worker_2_proxy_port, 'block_writes');
|
|
ERROR: canceling statement due to user request
|
|
-- failure on blocking COPY operation on target node
|
|
SELECT citus.mitmproxy('conn.onQuery(query="COPY").kill()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT master_move_shard_placement(201, 'localhost', :worker_1_port, 'localhost', :worker_2_proxy_port, 'block_writes');
|
|
ERROR: connection not open
|
|
CONTEXT: while executing command on localhost:xxxxx
|
|
while executing command on localhost:xxxxx
|
|
-- cancellation on blocking COPY operation on target node
|
|
SELECT citus.mitmproxy('conn.onQuery(query="COPY").cancel(' || :pid || ')');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT master_move_shard_placement(201, 'localhost', :worker_1_port, 'localhost', :worker_2_proxy_port, 'block_writes');
|
|
ERROR: canceling statement due to user request
|
|
-- failure on adding constraints on target node
|
|
SELECT citus.mitmproxy('conn.onQuery(query="ADD CONSTRAINT").kill()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT master_move_shard_placement(201, 'localhost', :worker_1_port, 'localhost', :worker_2_proxy_port, 'block_writes');
|
|
ERROR: connection not open
|
|
CONTEXT: while executing command on localhost:xxxxx
|
|
-- cancellation on adding constraints on target node
|
|
SELECT citus.mitmproxy('conn.onQuery(query="ADD CONSTRAINT").cancel(' || :pid || ')');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT master_move_shard_placement(201, 'localhost', :worker_1_port, 'localhost', :worker_2_proxy_port, 'block_writes');
|
|
ERROR: canceling statement due to user request
|
|
SELECT public.wait_for_resource_cleanup();
|
|
wait_for_resource_cleanup
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
-- Verify that the shard is not moved and the number of rows are still 100k
|
|
SELECT citus.mitmproxy('conn.allow()');
|
|
mitmproxy
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT * FROM shards_in_workers;
|
|
shardid | worker
|
|
---------------------------------------------------------------------
|
|
200 | worker2
|
|
201 | worker1
|
|
202 | worker2
|
|
203 | worker1
|
|
(4 rows)
|
|
|
|
SELECT count(*) FROM t;
|
|
count
|
|
---------------------------------------------------------------------
|
|
100
|
|
(1 row)
|
|
|
|
SELECT * FROM find_index_for_shard_201_in_workers;
|
|
case
|
|
---------------------------------------------------------------------
|
|
worker1
|
|
(1 row)
|
|
|
|
-- Verify that shard can be moved after a temporary failure
|
|
SELECT master_move_shard_placement(201, 'localhost', :worker_1_port, 'localhost', :worker_2_proxy_port, 'block_writes');
|
|
master_move_shard_placement
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT public.wait_for_resource_cleanup();
|
|
wait_for_resource_cleanup
|
|
---------------------------------------------------------------------
|
|
|
|
(1 row)
|
|
|
|
SELECT * FROM shards_in_workers;
|
|
shardid | worker
|
|
---------------------------------------------------------------------
|
|
200 | worker2
|
|
201 | worker2
|
|
202 | worker2
|
|
203 | worker1
|
|
(4 rows)
|
|
|
|
SELECT count(*) FROM t;
|
|
count
|
|
---------------------------------------------------------------------
|
|
100
|
|
(1 row)
|
|
|
|
SELECT * FROM find_index_for_shard_201_in_workers;
|
|
case
|
|
---------------------------------------------------------------------
|
|
worker2
|
|
(1 row)
|
|
|
|
DROP SCHEMA move_shard_offline CASCADE;
|
|
NOTICE: drop cascades to 4 other objects
|
|
DETAIL: drop cascades to table t
|
|
drop cascades to view shards_in_workers
|
|
drop cascades to view indices_on_shard_201
|
|
drop cascades to view find_index_for_shard_201_in_workers
|