-- We have two different output files for this failure test because the
-- failure behaviour of SAVEPOINT and RELEASE commands are different if
-- we use the executor. If we use it, these commands error out if any of
-- the placement commands fail. Otherwise, we might mark the placement
-- as invalid and continue with a WARNING.
SELECT citus.mitmproxy('conn.allow()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

SET citus.shard_count = 2;
SET citus.shard_replication_factor = 1; -- one shard per worker
SET citus.next_shard_id TO 100950;
ALTER SEQUENCE pg_catalog.pg_dist_placement_placementid_seq RESTART 150;
CREATE TABLE artists (
    id bigint NOT NULL,
    name text NOT NULL
);
SELECT create_distributed_table('artists', 'id');
 create_distributed_table
---------------------------------------------------------------------

(1 row)

-- add some data
INSERT INTO artists VALUES (1, 'Pablo Picasso');
INSERT INTO artists VALUES (2, 'Vincent van Gogh');
INSERT INTO artists VALUES (3, 'Claude Monet');
INSERT INTO artists VALUES (4, 'William Kurelek');
-- simply fail at SAVEPOINT
SELECT citus.mitmproxy('conn.onQuery(query="^SAVEPOINT").kill()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

BEGIN;
INSERT INTO artists VALUES (5, 'Asher Lev');
SAVEPOINT s1;
WARNING:  connection not open
CONTEXT:  while executing command on localhost:xxxxx
WARNING:  connection error: localhost:xxxxx
DETAIL:  connection not open
WARNING:  connection not open
CONTEXT:  while executing command on localhost:xxxxx
WARNING:  connection not open
CONTEXT:  while executing command on localhost:xxxxx
ERROR:  connection not open
CONTEXT:  while executing command on localhost:xxxxx
DELETE FROM artists WHERE id=4;
ERROR:  current transaction is aborted, commands ignored until end of transaction block
RELEASE SAVEPOINT s1;
ERROR:  current transaction is aborted, commands ignored until end of transaction block
COMMIT;
SELECT * FROM artists WHERE id IN (4, 5);
 id |      name
---------------------------------------------------------------------
  4 | William Kurelek
(1 row)

-- fail at RELEASE
SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").kill()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

BEGIN;
UPDATE artists SET name='a';
SAVEPOINT s1;
DELETE FROM artists WHERE id=4;
RELEASE SAVEPOINT s1;
WARNING:  AbortSubTransaction while in COMMIT state
WARNING:  connection not open
CONTEXT:  while executing command on localhost:xxxxx
WARNING:  connection error: localhost:xxxxx
DETAIL:  connection not open
WARNING:  connection not open
CONTEXT:  while executing command on localhost:xxxxx
WARNING:  connection not open
CONTEXT:  while executing command on localhost:xxxxx
WARNING:  savepoint "savepoint_2" does not exist
CONTEXT:  while executing command on localhost:xxxxx
ERROR:  connection not open
CONTEXT:  while executing command on localhost:xxxxx
ROLLBACK;
SELECT * FROM artists WHERE id IN (4, 5);
 id |      name
---------------------------------------------------------------------
  4 | William Kurelek
(1 row)

-- fail at ROLLBACK
SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

BEGIN;
INSERT INTO artists VALUES (5, 'Asher Lev');
SAVEPOINT s1;
DELETE FROM artists WHERE id=4;
ROLLBACK TO SAVEPOINT s1;
WARNING:  connection not open
CONTEXT:  while executing command on localhost:xxxxx
WARNING:  connection not open
CONTEXT:  while executing command on localhost:xxxxx
COMMIT;
ERROR:  could not make changes to shard xxxxx on any node
SELECT * FROM artists WHERE id IN (4, 5);
 id |      name
---------------------------------------------------------------------
  4 | William Kurelek
(1 row)

-- fail at second RELEASE
SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").after(1).kill()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

BEGIN;
SAVEPOINT s1;
DELETE FROM artists WHERE id=4;
RELEASE SAVEPOINT s1;
SAVEPOINT s2;
INSERT INTO artists VALUES (5, 'Jacob Kahn');
RELEASE SAVEPOINT s2;
WARNING:  AbortSubTransaction while in COMMIT state
WARNING:  connection not open
CONTEXT:  while executing command on localhost:xxxxx
WARNING:  connection error: localhost:xxxxx
DETAIL:  connection not open
WARNING:  connection not open
CONTEXT:  while executing command on localhost:xxxxx
WARNING:  connection not open
CONTEXT:  while executing command on localhost:xxxxx
ERROR:  connection not open
CONTEXT:  while executing command on localhost:xxxxx
COMMIT;
SELECT * FROM artists WHERE id IN (4, 5);
 id |      name
---------------------------------------------------------------------
  4 | William Kurelek
(1 row)

-- fail at second ROLLBACK
SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").after(1).kill()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

BEGIN;
SAVEPOINT s1;
UPDATE artists SET name='A' WHERE id=4;
ROLLBACK TO SAVEPOINT s1;
SAVEPOINT s2;
DELETE FROM artists WHERE id=5;
ROLLBACK TO SAVEPOINT s2;
WARNING:  connection not open
CONTEXT:  while executing command on localhost:xxxxx
WARNING:  connection not open
CONTEXT:  while executing command on localhost:xxxxx
COMMIT;
ERROR:  could not make changes to shard xxxxx on any node
SELECT * FROM artists WHERE id IN (4, 5);
 id |      name
---------------------------------------------------------------------
  4 | William Kurelek
(1 row)

SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").after(1).kill()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

-- Release after rollback
BEGIN;
SAVEPOINT s1;
ROLLBACK TO s1;
RELEASE SAVEPOINT s1;
SAVEPOINT s2;
INSERT INTO artists VALUES (6, 'John J. Audubon');
INSERT INTO artists VALUES (7, 'Emily Carr');
ROLLBACK TO s2;
RELEASE SAVEPOINT s2;
COMMIT;
SELECT * FROM artists WHERE id=7;
 id | name
---------------------------------------------------------------------
(0 rows)

SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

-- Recover from errors
\set VERBOSITY terse
BEGIN;
SAVEPOINT s1;
SAVEPOINT s2;
INSERT INTO artists VALUES (6, 'John J. Audubon');
INSERT INTO artists VALUES (7, 'Emily Carr');
INSERT INTO artists VALUES (7, 'Emily Carr');
ROLLBACK TO SAVEPOINT s1;
WARNING:  connection not open
WARNING:  connection not open
WARNING:  connection not open
WARNING:  connection error: localhost:xxxxx
WARNING:  connection not open
WARNING:  connection not open
COMMIT;
ERROR:  could not make changes to shard xxxxx on any node
SELECT * FROM artists WHERE id=6;
 id | name
---------------------------------------------------------------------
(0 rows)

-- replication factor > 1
CREATE TABLE researchers (
  id bigint NOT NULL,
  lab_id int NOT NULL,
  name text NOT NULL
);
SET citus.shard_count = 1;
SET citus.shard_replication_factor = 2; -- single shard, on both workers
SELECT create_distributed_table('researchers', 'lab_id', 'hash');
 create_distributed_table
---------------------------------------------------------------------

(1 row)

-- simply fail at SAVEPOINT
SELECT citus.mitmproxy('conn.onQuery(query="^SAVEPOINT").kill()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

BEGIN;
INSERT INTO researchers VALUES (7, 4, 'Jan Plaza');
SAVEPOINT s1;
WARNING:  connection not open
WARNING:  connection error: localhost:xxxxx
WARNING:  connection not open
WARNING:  connection not open
ERROR:  connection not open
INSERT INTO researchers VALUES (8, 4, 'Alonzo Church');
ERROR:  current transaction is aborted, commands ignored until end of transaction block
ROLLBACK TO s1;
ERROR:  savepoint "s1" does not exist
RELEASE SAVEPOINT s1;
ERROR:  current transaction is aborted, commands ignored until end of transaction block
COMMIT;
-- should see correct results from healthy placement and one bad placement
SELECT * FROM researchers WHERE lab_id = 4;
 id | lab_id | name
---------------------------------------------------------------------
(0 rows)

UPDATE pg_dist_shard_placement SET shardstate = 1
WHERE shardstate = 3 AND shardid IN (
  SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'researchers'::regclass
) RETURNING placementid;
 placementid
---------------------------------------------------------------------
(0 rows)

TRUNCATE researchers;
-- fail at rollback
SELECT citus.mitmproxy('conn.onQuery(query="^ROLLBACK").kill()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

BEGIN;
INSERT INTO researchers VALUES (7, 4, 'Jan Plaza');
SAVEPOINT s1;
INSERT INTO researchers VALUES (8, 4, 'Alonzo Church');
ROLLBACK TO s1;
WARNING:  connection not open
WARNING:  connection not open
RELEASE SAVEPOINT s1;
COMMIT;
ERROR:  failure on connection marked as essential: localhost:xxxxx
-- should see correct results from healthy placement and one bad placement
SELECT * FROM researchers WHERE lab_id = 4;
 id | lab_id | name
---------------------------------------------------------------------
(0 rows)

UPDATE pg_dist_shard_placement SET shardstate = 1
WHERE shardstate = 3 AND shardid IN (
  SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'researchers'::regclass
) RETURNING placementid;
 placementid
---------------------------------------------------------------------
(0 rows)

TRUNCATE researchers;
-- fail at release
SELECT citus.mitmproxy('conn.onQuery(query="^RELEASE").kill()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

BEGIN;
INSERT INTO researchers VALUES (7, 4, 'Jan Plaza');
SAVEPOINT s1;
INSERT INTO researchers VALUES (8, 4, 'Alonzo Church');
ROLLBACK TO s1;
RELEASE SAVEPOINT s1;
WARNING:  AbortSubTransaction while in COMMIT state
WARNING:  connection not open
WARNING:  connection error: localhost:xxxxx
WARNING:  connection not open
WARNING:  connection not open
WARNING:  savepoint "savepoint_3" does not exist
ERROR:  connection not open
COMMIT;
-- should see correct results from healthy placement and one bad placement
SELECT * FROM researchers WHERE lab_id = 4;
 id | lab_id | name
---------------------------------------------------------------------
(0 rows)

UPDATE pg_dist_shard_placement SET shardstate = 1
WHERE shardstate = 3 AND shardid IN (
  SELECT shardid FROM pg_dist_shard WHERE logicalrelid = 'researchers'::regclass
) RETURNING placementid;
 placementid
---------------------------------------------------------------------
(0 rows)

TRUNCATE researchers;
-- clean up
SELECT citus.mitmproxy('conn.allow()');
 mitmproxy
---------------------------------------------------------------------

(1 row)

DROP TABLE artists;
DROP TABLE researchers;
