--
-- failure_offline_move_shard_placement
--
-- The tests cover moving shard placements without using logical replication.
CREATE SCHEMA IF NOT EXISTS move_shard_offline;
SET SEARCH_PATH = move_shard_offline;
SET citus.shard_count TO 4;
SET citus.next_shard_id TO 200;
SET citus.shard_replication_factor TO 1;
SELECT pg_backend_pid() as pid \gset
SELECT citus.mitmproxy('conn.allow()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

CREATE TABLE t(id int PRIMARY KEY, int_data int);
SELECT create_distributed_table('t', 'id');
 create_distributed_table
---------------------------------------------------------------------

(1 row)

CREATE VIEW shards_in_workers AS
  SELECT shardid,
         (CASE WHEN nodeport = :worker_1_port THEN 'worker1' ELSE 'worker2' END) AS worker
  FROM pg_dist_placement NATURAL JOIN pg_dist_node
  WHERE shardstate != 4
  ORDER BY 1,2 ASC;
CREATE VIEW indices_on_shard_201 AS
  SELECT * FROM run_command_on_workers( $cmd$
    SELECT CASE WHEN COUNT(*) > 0 THEN TRUE ELSE FALSE END
    FROM pg_index WHERE indrelid = 'move_shard_offline.t_201'::regclass
  $cmd$);
CREATE VIEW find_index_for_shard_201_in_workers AS
  SELECT CASE nodeport WHEN :worker_1_port THEN 'worker1' ELSE 'worker2' END
  FROM indices_on_shard_201 WHERE result = 't';
-- Insert some data
INSERT INTO t SELECT x, x+1 FROM generate_series(1,100) AS f(x);
-- Initial shard placements
SELECT * FROM shards_in_workers;
 shardid | worker
---------------------------------------------------------------------
     200 | worker2
     201 | worker1
     202 | worker2
     203 | worker1
(4 rows)

SELECT * FROM find_index_for_shard_201_in_workers;
  case
---------------------------------------------------------------------
 worker1
(1 row)

-- failure on sanity checks
SELECT citus.mitmproxy('conn.onQuery(query="DROP TABLE IF EXISTS move_shard_offline.t CASCADE").kill()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT master_move_shard_placement(201, 'localhost', :worker_1_port, 'localhost', :worker_2_proxy_port, 'block_writes');
ERROR:  connection not open
CONTEXT:  while executing command on localhost:xxxxx
-- cancellation on sanity checks
SELECT citus.mitmproxy('conn.onQuery(query="DROP TABLE IF EXISTS move_shard_offline.t CASCADE").cancel(' || :pid || ')');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT master_move_shard_placement(201, 'localhost', :worker_1_port, 'localhost', :worker_2_proxy_port, 'block_writes');
ERROR:  canceling statement due to user request
-- failure on move_shard table creation
SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE move_shard_offline.t").kill()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT master_move_shard_placement(201, 'localhost', :worker_1_port, 'localhost', :worker_2_proxy_port, 'block_writes');
ERROR:  connection not open
CONTEXT:  while executing command on localhost:xxxxx
-- cancellation on move_shard table creation
SELECT citus.mitmproxy('conn.onQuery(query="CREATE TABLE move_shard_offline.t").cancel(' || :pid || ')');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT master_move_shard_placement(201, 'localhost', :worker_1_port, 'localhost', :worker_2_proxy_port, 'block_writes');
ERROR:  canceling statement due to user request
-- failure on blocking COPY operation on target node
SELECT citus.mitmproxy('conn.onQuery(query="COPY").kill()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT master_move_shard_placement(201, 'localhost', :worker_1_port, 'localhost', :worker_2_proxy_port, 'block_writes');
ERROR:  connection not open
CONTEXT:  while executing command on localhost:xxxxx
while executing command on localhost:xxxxx
-- cancellation on blocking COPY operation on target node
SELECT citus.mitmproxy('conn.onQuery(query="COPY").cancel(' || :pid || ')');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT master_move_shard_placement(201, 'localhost', :worker_1_port, 'localhost', :worker_2_proxy_port, 'block_writes');
ERROR:  canceling statement due to user request
-- failure on adding constraints on target node
SELECT citus.mitmproxy('conn.onQuery(query="ADD CONSTRAINT").kill()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT master_move_shard_placement(201, 'localhost', :worker_1_port, 'localhost', :worker_2_proxy_port, 'block_writes');
ERROR:  connection not open
CONTEXT:  while executing command on localhost:xxxxx
-- cancellation on adding constraints on target node
SELECT citus.mitmproxy('conn.onQuery(query="ADD CONSTRAINT").cancel(' || :pid || ')');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT master_move_shard_placement(201, 'localhost', :worker_1_port, 'localhost', :worker_2_proxy_port, 'block_writes');
ERROR:  canceling statement due to user request
SELECT public.wait_for_resource_cleanup();
 wait_for_resource_cleanup
---------------------------------------------------------------------

(1 row)

-- Verify that the shard is not moved and the number of rows are still 100k
SELECT citus.mitmproxy('conn.allow()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SELECT * FROM shards_in_workers;
 shardid | worker
---------------------------------------------------------------------
     200 | worker2
     201 | worker1
     202 | worker2
     203 | worker1
(4 rows)

SELECT count(*) FROM t;
 count
---------------------------------------------------------------------
   100
(1 row)

SELECT * FROM find_index_for_shard_201_in_workers;
  case
---------------------------------------------------------------------
 worker1
(1 row)

-- Verify that shard can be moved after a temporary failure
SELECT master_move_shard_placement(201, 'localhost', :worker_1_port, 'localhost', :worker_2_proxy_port, 'block_writes');
 master_move_shard_placement
---------------------------------------------------------------------

(1 row)

SELECT public.wait_for_resource_cleanup();
 wait_for_resource_cleanup
---------------------------------------------------------------------

(1 row)

SELECT * FROM shards_in_workers;
 shardid | worker
---------------------------------------------------------------------
     200 | worker2
     201 | worker2
     202 | worker2
     203 | worker1
(4 rows)

SELECT count(*) FROM t;
 count
---------------------------------------------------------------------
   100
(1 row)

SELECT * FROM find_index_for_shard_201_in_workers;
  case
---------------------------------------------------------------------
 worker2
(1 row)

DROP SCHEMA move_shard_offline CASCADE;
NOTICE:  drop cascades to 4 other objects
DETAIL:  drop cascades to table t
drop cascades to view shards_in_workers
drop cascades to view indices_on_shard_201
drop cascades to view find_index_for_shard_201_in_workers