\c - - - :master_port
SET citus.shard_replication_factor TO 2;
CREATE TABLE the_replicated_table (a int, b int, z bigserial);
SELECT create_distributed_table('the_replicated_table', 'a');
 create_distributed_table
---------------------------------------------------------------------

(1 row)

SET citus.shard_replication_factor TO 1;
CREATE TABLE the_table (a int, b int, z bigserial);
SELECT create_distributed_table('the_table', 'a');
 create_distributed_table
---------------------------------------------------------------------

(1 row)

SELECT 1 FROM master_add_node('localhost', :master_port, groupid => 0);
NOTICE:  localhost:xxxxx is the coordinator and already contains metadata, skipping syncing the metadata
 ?column?
---------------------------------------------------------------------
        1
(1 row)

CREATE TABLE reference_table (a int, b int, z bigserial);
SELECT create_reference_table('reference_table');
 create_reference_table
---------------------------------------------------------------------

(1 row)

CREATE TABLE citus_local_table (a int, b int, z bigserial);
SELECT citus_add_local_table_to_metadata('citus_local_table');
 citus_add_local_table_to_metadata
---------------------------------------------------------------------

(1 row)

CREATE TABLE local (a int, b int);
\c - - - :follower_master_port
-- inserts normally do not work on a standby coordinator
INSERT INTO the_table (a, b, z) VALUES (1, 2, 2);
ERROR:  writing to worker nodes is not currently allowed
DETAIL:  the database is read-only
INSERT INTO reference_table (a, b, z) VALUES (1, 2, 2);
ERROR:  writing to worker nodes is not currently allowed
DETAIL:  the database is read-only
INSERT INTO citus_local_table (a, b, z) VALUES (1, 2, 2);
ERROR:  writing to worker nodes is not currently allowed
DETAIL:  the database is read-only
-- We can allow DML on a writable standby coordinator.
-- Note that it doesn't help to enable writes for
--   (a) citus local tables
--   (b) coordinator replicated reference tables.
--   (c) reference tables or replication > 1 distributed tables
-- (a) and (b) is because the data is in the coordinator and will hit
-- read-only tranaction checks on Postgres
-- (c) is because citus uses 2PC, where a transaction record should
-- be inserted to pg_dist_node, which is not allowed
SET citus.writable_standby_coordinator TO on;
INSERT INTO the_table (a, b, z) VALUES (1, 2, 2);
SELECT * FROM the_table;
 a | b | z
---------------------------------------------------------------------
 1 | 2 | 2
(1 row)

INSERT INTO the_replicated_table (a, b, z) VALUES (1, 2, 2);
ERROR:  writing to worker nodes is not currently allowed for replicated tables such as reference tables or hash distributed tables with replication factor greater than 1.
DETAIL:  the database is read-only
HINT:  All modifications to replicated tables happen via 2PC, and 2PC requires the database to be in a writable state.
SELECT * FROM the_replicated_table;
 a | b | z
---------------------------------------------------------------------
(0 rows)

INSERT INTO reference_table (a, b, z) VALUES (1, 2, 2);
ERROR:  writing to worker nodes is not currently allowed for replicated tables such as reference tables or hash distributed tables with replication factor greater than 1.
DETAIL:  the database is read-only
HINT:  All modifications to replicated tables happen via 2PC, and 2PC requires the database to be in a writable state.
SELECT * FROM reference_table;
 a | b | z
---------------------------------------------------------------------
(0 rows)

INSERT INTO citus_local_table (a, b, z) VALUES (1, 2, 2);
ERROR:  cannot execute INSERT in a read-only transaction
SELECT * FROM citus_local_table;
 a | b | z
---------------------------------------------------------------------
(0 rows)

UPDATE the_table SET z = 3 WHERE a = 1;
UPDATE the_replicated_table SET z = 3 WHERE a = 1;
ERROR:  writing to worker nodes is not currently allowed for replicated tables such as reference tables or hash distributed tables with replication factor greater than 1.
DETAIL:  the database is read-only
HINT:  All modifications to replicated tables happen via 2PC, and 2PC requires the database to be in a writable state.
UPDATE reference_table SET z = 3 WHERE a = 1;
ERROR:  writing to worker nodes is not currently allowed for replicated tables such as reference tables or hash distributed tables with replication factor greater than 1.
DETAIL:  the database is read-only
HINT:  All modifications to replicated tables happen via 2PC, and 2PC requires the database to be in a writable state.
UPDATE citus_local_table SET z = 3 WHERE a = 1;
ERROR:  cannot execute UPDATE in a read-only transaction
SELECT * FROM the_table;
 a | b | z
---------------------------------------------------------------------
 1 | 2 | 3
(1 row)

SELECT * FROM reference_table;
 a | b | z
---------------------------------------------------------------------
(0 rows)

SELECT * FROM citus_local_table;
 a | b | z
---------------------------------------------------------------------
(0 rows)

DELETE FROM the_table WHERE a = 1;
DELETE FROM the_replicated_table WHERE a = 1;
ERROR:  writing to worker nodes is not currently allowed for replicated tables such as reference tables or hash distributed tables with replication factor greater than 1.
DETAIL:  the database is read-only
HINT:  All modifications to replicated tables happen via 2PC, and 2PC requires the database to be in a writable state.
DELETE FROM reference_table WHERE a = 1;
ERROR:  writing to worker nodes is not currently allowed for replicated tables such as reference tables or hash distributed tables with replication factor greater than 1.
DETAIL:  the database is read-only
HINT:  All modifications to replicated tables happen via 2PC, and 2PC requires the database to be in a writable state.
DELETE FROM citus_local_table WHERE a = 1;
ERROR:  cannot execute DELETE in a read-only transaction
SELECT * FROM the_table;
 a | b | z
---------------------------------------------------------------------
(0 rows)

SELECT * FROM reference_table;
 a | b | z
---------------------------------------------------------------------
(0 rows)

SELECT * FROM citus_local_table;
 a | b | z
---------------------------------------------------------------------
(0 rows)

-- drawing from a sequence is not possible
INSERT INTO the_table (a, b) VALUES (1, 2);
ERROR:  cannot assign TransactionIds during recovery
INSERT INTO the_replicated_table (a, b) VALUES (1, 2);
ERROR:  writing to worker nodes is not currently allowed for replicated tables such as reference tables or hash distributed tables with replication factor greater than 1.
DETAIL:  the database is read-only
HINT:  All modifications to replicated tables happen via 2PC, and 2PC requires the database to be in a writable state.
INSERT INTO reference_table (a, b) VALUES (1, 2);
ERROR:  writing to worker nodes is not currently allowed for replicated tables such as reference tables or hash distributed tables with replication factor greater than 1.
DETAIL:  the database is read-only
HINT:  All modifications to replicated tables happen via 2PC, and 2PC requires the database to be in a writable state.
INSERT INTO citus_local_table (a, b) VALUES (1, 2);
ERROR:  cannot assign TransactionIds during recovery
-- 2PC is not possible
INSERT INTO the_table (a, b, z) VALUES (2, 3, 4), (5, 6, 7);
ERROR:  cannot assign TransactionIds during recovery
INSERT INTO the_replicated_table (a, b, z) VALUES (2, 3, 4), (5, 6, 7);
ERROR:  writing to worker nodes is not currently allowed for replicated tables such as reference tables or hash distributed tables with replication factor greater than 1.
DETAIL:  the database is read-only
HINT:  All modifications to replicated tables happen via 2PC, and 2PC requires the database to be in a writable state.
INSERT INTO reference_table (a, b, z) VALUES (2, 3, 4), (5, 6, 7);
ERROR:  writing to worker nodes is not currently allowed for replicated tables such as reference tables or hash distributed tables with replication factor greater than 1.
DETAIL:  the database is read-only
HINT:  All modifications to replicated tables happen via 2PC, and 2PC requires the database to be in a writable state.
INSERT INTO citus_local_table (a, b, z) VALUES (2, 3, 4), (5, 6, 7);
ERROR:  cannot execute INSERT in a read-only transaction
-- COPY is not possible because Citus user 2PC
COPY the_table (a, b, z) FROM STDIN WITH CSV;
ERROR:  COPY command to Citus tables is not allowed in read-only mode
DETAIL:  the database is read-only
HINT:  All COPY commands to citus tables happen via 2PC, and 2PC requires the database to be in a writable state.
\.
invalid command \.
COPY the_replicated_table (a, b, z) FROM STDIN WITH CSV;
ERROR:  writing to worker nodes is not currently allowed for replicated tables such as reference tables or hash distributed tables with replication factor greater than 1.
DETAIL:  the database is read-only
HINT:  All modifications to replicated tables happen via 2PC, and 2PC requires the database to be in a writable state.
\.
invalid command \.
COPY reference_table (a, b, z) FROM STDIN WITH CSV;
ERROR:  writing to worker nodes is not currently allowed for replicated tables such as reference tables or hash distributed tables with replication factor greater than 1.
DETAIL:  the database is read-only
HINT:  All modifications to replicated tables happen via 2PC, and 2PC requires the database to be in a writable state.
\.
invalid command \.
COPY citus_local_table (a, b, z) FROM STDIN WITH CSV;
ERROR:  COPY command to Citus tables is not allowed in read-only mode
DETAIL:  the database is read-only
HINT:  All COPY commands to citus tables happen via 2PC, and 2PC requires the database to be in a writable state.
\.
invalid command \.
-- all multi-shard modifications require 2PC hence not supported
INSERT INTO the_table (a, b, z) VALUES (2, 3, 4), (5, 6, 7);
ERROR:  cannot assign TransactionIds during recovery
SELECT * FROM the_table ORDER BY a;
 a | b | z
---------------------------------------------------------------------
(0 rows)

-- all modifications to reference tables use 2PC, hence not supported
INSERT INTO reference_table (a, b, z) VALUES (2, 3, 4), (5, 6, 7);
ERROR:  writing to worker nodes is not currently allowed for replicated tables such as reference tables or hash distributed tables with replication factor greater than 1.
DETAIL:  the database is read-only
HINT:  All modifications to replicated tables happen via 2PC, and 2PC requires the database to be in a writable state.
SELECT * FROM reference_table ORDER BY a;
 a | b | z
---------------------------------------------------------------------
(0 rows)

-- citus local tables are on the coordinator, and coordinator is read-only
INSERT INTO citus_local_table (a, b, z) VALUES (2, 3, 4), (5, 6, 7);
ERROR:  cannot execute INSERT in a read-only transaction
SELECT * FROM citus_local_table ORDER BY a;
 a | b | z
---------------------------------------------------------------------
(0 rows)

-- modifying CTEs are possible
WITH del AS (DELETE FROM the_table RETURNING *)
SELECT * FROM del ORDER BY a;
ERROR:  cannot assign TransactionIds during recovery
WITH del AS (DELETE FROM reference_table RETURNING *)
SELECT * FROM del ORDER BY a;
ERROR:  writing to worker nodes is not currently allowed for replicated tables such as reference tables or hash distributed tables with replication factor greater than 1.
DETAIL:  the database is read-only
HINT:  All modifications to replicated tables happen via 2PC, and 2PC requires the database to be in a writable state.
WITH del AS (DELETE FROM the_replicated_table RETURNING *)
SELECT * FROM del ORDER BY a;
ERROR:  writing to worker nodes is not currently allowed for replicated tables such as reference tables or hash distributed tables with replication factor greater than 1.
DETAIL:  the database is read-only
HINT:  All modifications to replicated tables happen via 2PC, and 2PC requires the database to be in a writable state.
WITH del AS (DELETE FROM citus_local_table RETURNING *)
SELECT * FROM del ORDER BY a;
ERROR:  cannot execute DELETE in a read-only transaction
-- multi-shard COPY is not possible due to 2PC
COPY the_table (a, b, z) FROM STDIN WITH CSV;
ERROR:  COPY command to Citus tables is not allowed in read-only mode
DETAIL:  the database is read-only
HINT:  All COPY commands to citus tables happen via 2PC, and 2PC requires the database to be in a writable state.
\.
invalid command \.
COPY reference_table (a, b, z) FROM STDIN WITH CSV;
ERROR:  writing to worker nodes is not currently allowed for replicated tables such as reference tables or hash distributed tables with replication factor greater than 1.
DETAIL:  the database is read-only
HINT:  All modifications to replicated tables happen via 2PC, and 2PC requires the database to be in a writable state.
\.
invalid command \.
COPY citus_local_table (a, b, z) FROM STDIN WITH CSV;
ERROR:  COPY command to Citus tables is not allowed in read-only mode
DETAIL:  the database is read-only
HINT:  All COPY commands to citus tables happen via 2PC, and 2PC requires the database to be in a writable state.
\.
invalid command \.
SELECT * FROM the_table ORDER BY a;
 a | b | z
---------------------------------------------------------------------
(0 rows)

SELECT * FROM reference_table ORDER BY a;
 a | b | z
---------------------------------------------------------------------
(0 rows)

SELECT * FROM citus_local_table ORDER BY a;
 a | b | z
---------------------------------------------------------------------
(0 rows)

DELETE FROM reference_table;
ERROR:  writing to worker nodes is not currently allowed for replicated tables such as reference tables or hash distributed tables with replication factor greater than 1.
DETAIL:  the database is read-only
HINT:  All modifications to replicated tables happen via 2PC, and 2PC requires the database to be in a writable state.
DELETE FROM citus_local_table;
ERROR:  cannot execute DELETE in a read-only transaction
-- multi-shard modification always uses 2PC, so not supported
DELETE FROM the_table;
ERROR:  cannot assign TransactionIds during recovery
-- DDL is not possible
TRUNCATE the_table;
ERROR:  cannot acquire lock mode AccessExclusiveLock on database objects while recovery is in progress
HINT:  Only RowExclusiveLock or less can be acquired on database objects during recovery.
TRUNCATE reference_table;
ERROR:  cannot acquire lock mode AccessExclusiveLock on database objects while recovery is in progress
HINT:  Only RowExclusiveLock or less can be acquired on database objects during recovery.
TRUNCATE citus_local_table;
ERROR:  cannot acquire lock mode AccessExclusiveLock on database objects while recovery is in progress
HINT:  Only RowExclusiveLock or less can be acquired on database objects during recovery.
ALTER TABLE the_table ADD COLUMN c int;
ERROR:  cannot acquire lock mode AccessExclusiveLock on database objects while recovery is in progress
HINT:  Only RowExclusiveLock or less can be acquired on database objects during recovery.
ALTER TABLE reference_table ADD COLUMN c int;
ERROR:  cannot acquire lock mode AccessExclusiveLock on database objects while recovery is in progress
HINT:  Only RowExclusiveLock or less can be acquired on database objects during recovery.
ALTER TABLE citus_local_table ADD COLUMN c int;
ERROR:  cannot acquire lock mode AccessExclusiveLock on database objects while recovery is in progress
HINT:  Only RowExclusiveLock or less can be acquired on database objects during recovery.
-- rollback is possible
BEGIN;
INSERT INTO the_table (a, b, z) VALUES (1, 2, 2);
ROLLBACK;
BEGIN;
INSERT INTO reference_table (a, b, z) VALUES (1, 2, 2);
ERROR:  writing to worker nodes is not currently allowed for replicated tables such as reference tables or hash distributed tables with replication factor greater than 1.
DETAIL:  the database is read-only
HINT:  All modifications to replicated tables happen via 2PC, and 2PC requires the database to be in a writable state.
ROLLBACK;
BEGIN;
INSERT INTO citus_local_table (a, b, z) VALUES (1, 2, 2);
ERROR:  cannot execute INSERT in a read-only transaction
ROLLBACK;
SELECT * FROM the_table ORDER BY a;
 a | b | z
---------------------------------------------------------------------
(0 rows)

SELECT * FROM reference_table ORDER BY a;
 a | b | z
---------------------------------------------------------------------
(0 rows)

SELECT * FROM citus_local_table ORDER BY a;
 a | b | z
---------------------------------------------------------------------
(0 rows)

-- we should still disallow writes to local tables
INSERT INTO local VALUES (1, 1);
ERROR:  cannot execute INSERT in a read-only transaction
INSERT INTO local SELECT i,i FROM generate_series(0,100)i;
ERROR:  cannot execute INSERT in a read-only transaction
-- we shouldn't be able to create local tables
CREATE TEMP TABLE local_copy_of_the_table AS SELECT * FROM the_table;
ERROR:  cannot execute CREATE TABLE AS in a read-only transaction
\c "port=9070 dbname=regression options='-c\ citus.use_secondary_nodes=always\ -c\ citus.cluster_name=second-cluster'"
-- separate follower formations currently cannot do writes
SET citus.writable_standby_coordinator TO on;
INSERT INTO the_table (a, b, z) VALUES (1, 2, 3);
ERROR:  writing to worker nodes is not currently allowed
DETAIL:  citus.use_secondary_nodes is set to 'always'
SELECT * FROM the_table ORDER BY a;
 a | b | z
---------------------------------------------------------------------
(0 rows)

INSERT INTO reference_table (a, b, z) VALUES (1, 2, 3);
ERROR:  writing to worker nodes is not currently allowed
DETAIL:  citus.use_secondary_nodes is set to 'always'
SELECT * FROM reference_table ORDER BY a;
ERROR:  there is a shard placement in node group 0 but there are no nodes in that group
INSERT INTO citus_local_table (a, b, z) VALUES (1, 2, 3);
ERROR:  writing to worker nodes is not currently allowed
DETAIL:  citus.use_secondary_nodes is set to 'always'
SELECT * FROM citus_local_table ORDER BY a;
ERROR:  there is a shard placement in node group 0 but there are no nodes in that group
\c -reuse-previous=off regression - - :master_port
DROP TABLE the_table;
DROP TABLE reference_table;
DROP TABLE citus_local_table;
SELECT master_remove_node('localhost', :master_port);
 master_remove_node
---------------------------------------------------------------------

(1 row)