-- Test creation of mx tables and metadata syncing
SELECT nextval('pg_catalog.pg_dist_placement_placementid_seq') AS last_placement_id
\gset
SELECT nextval('pg_catalog.pg_dist_groupid_seq') AS last_group_id \gset
SELECT nextval('pg_catalog.pg_dist_node_nodeid_seq') AS last_node_id \gset
SELECT nextval('pg_catalog.pg_dist_colocationid_seq') AS last_colocation_id \gset
SELECT nextval('pg_catalog.pg_dist_shardid_seq') AS last_shard_id \gset
SET citus.replication_model TO streaming;
SET citus.shard_count TO 8;
SET citus.shard_replication_factor TO 1;
SET citus.replicate_reference_tables_on_activate TO off;
\set VERBOSITY terse
-- Simulates a readonly node by setting default_transaction_read_only.
CREATE FUNCTION mark_node_readonly(hostname TEXT, port INTEGER, isreadonly BOOLEAN)
    RETURNS TEXT
    LANGUAGE sql
    AS $$
    SELECT master_run_on_worker(ARRAY[hostname], ARRAY[port],
           ARRAY['ALTER SYSTEM SET default_transaction_read_only TO ' || isreadonly::TEXT], false);
    SELECT result FROM
        master_run_on_worker(ARRAY[hostname], ARRAY[port],
                             ARRAY['SELECT pg_reload_conf()'], false);
$$;
-- add a node to the cluster
SELECT master_add_node('localhost', :worker_1_port) As nodeid_1 \gset
SELECT nodeid, nodename, nodeport, hasmetadata, metadatasynced FROM pg_dist_node;
 nodeid | nodename  | nodeport | hasmetadata | metadatasynced
---------------------------------------------------------------------
      2 | localhost |    57637 | f           | f
(1 row)

-- create couple of tables
CREATE TABLE ref_table(a int primary key);
SELECT create_reference_table('ref_table');
 create_reference_table
---------------------------------------------------------------------

(1 row)

CREATE TABLE dist_table_1(a int primary key, b int references ref_table(a));
SELECT create_distributed_table('dist_table_1', 'a');
 create_distributed_table
---------------------------------------------------------------------

(1 row)

-- update the node
SELECT 1 FROM master_update_node((SELECT nodeid FROM pg_dist_node),
                                 'localhost', :worker_2_port);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

SELECT nodeid, nodename, nodeport, hasmetadata, metadatasynced FROM pg_dist_node;
 nodeid | nodename  | nodeport | hasmetadata | metadatasynced
---------------------------------------------------------------------
      2 | localhost |    57638 | f           | f
(1 row)

-- start syncing metadata to the node
SELECT 1 FROM start_metadata_sync_to_node('localhost', :worker_2_port);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

SELECT nodeid, nodename, nodeport, hasmetadata, metadatasynced FROM pg_dist_node;
 nodeid | nodename  | nodeport | hasmetadata | metadatasynced
---------------------------------------------------------------------
      2 | localhost |    57638 | t           | t
(1 row)

---------------------------------------------------------------------
-- Test that maintenance daemon syncs after master_update_node
---------------------------------------------------------------------
-- Update the node again. We do this as epeatable read, so we just see the
-- changes by master_update_node(). This is to avoid inconsistent results
-- if the maintenance daemon does the metadata sync too fast.
BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ;
SELECT nodeid, nodename, nodeport, hasmetadata, metadatasynced FROM pg_dist_node;
 nodeid | nodename  | nodeport | hasmetadata | metadatasynced
---------------------------------------------------------------------
      2 | localhost |    57638 | t           | t
(1 row)

SELECT 1 FROM master_update_node(:nodeid_1, 'localhost', :worker_1_port);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

SELECT nodeid, nodename, nodeport, hasmetadata, metadatasynced FROM pg_dist_node;
 nodeid | nodename  | nodeport | hasmetadata | metadatasynced
---------------------------------------------------------------------
      2 | localhost |    57637 | t           | f
(1 row)

END;
-- wait until maintenance daemon does the next metadata sync, and then
-- check if metadata is synced again
SELECT wait_until_metadata_sync(30000);
 wait_until_metadata_sync
---------------------------------------------------------------------

(1 row)

SELECT nodeid, hasmetadata, metadatasynced FROM pg_dist_node;
 nodeid | hasmetadata | metadatasynced
---------------------------------------------------------------------
      2 | t           | t
(1 row)

SELECT verify_metadata('localhost', :worker_1_port);
 verify_metadata
---------------------------------------------------------------------
 t
(1 row)

-- Update the node to a non-existent node. This is to simulate updating to
-- a unwriteable node.
BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ;
SELECT nodeid, nodename, nodeport, hasmetadata, metadatasynced FROM pg_dist_node;
 nodeid | nodename  | nodeport | hasmetadata | metadatasynced
---------------------------------------------------------------------
      2 | localhost |    57637 | t           | t
(1 row)

SELECT 1 FROM master_update_node(:nodeid_1, 'localhost', 12345);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

SELECT nodeid, nodename, nodeport, hasmetadata, metadatasynced FROM pg_dist_node;
 nodeid | nodename  | nodeport | hasmetadata | metadatasynced
---------------------------------------------------------------------
      2 | localhost |    12345 | t           | f
(1 row)

END;
-- maintenace daemon metadata sync should fail, because node is still unwriteable.
SELECT wait_until_metadata_sync(30000);
 wait_until_metadata_sync
---------------------------------------------------------------------

(1 row)

SELECT nodeid, hasmetadata, metadatasynced FROM pg_dist_node;
 nodeid | hasmetadata | metadatasynced
---------------------------------------------------------------------
      2 | t           | f
(1 row)

-- update it back to :worker_1_port, now metadata should be synced
SELECT 1 FROM master_update_node(:nodeid_1, 'localhost', :worker_1_port);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

SELECT wait_until_metadata_sync(30000);
 wait_until_metadata_sync
---------------------------------------------------------------------

(1 row)

SELECT nodeid, hasmetadata, metadatasynced FROM pg_dist_node;
 nodeid | hasmetadata | metadatasynced
---------------------------------------------------------------------
      2 | t           | t
(1 row)

---------------------------------------------------------------------
-- Test updating a node when another node is in readonly-mode
---------------------------------------------------------------------
SELECT master_add_node('localhost', :worker_2_port) AS nodeid_2 \gset
SELECT 1 FROM start_metadata_sync_to_node('localhost', :worker_2_port);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

-- Create a table with shards on both nodes
CREATE TABLE dist_table_2(a int);
SELECT create_distributed_table('dist_table_2', 'a');
 create_distributed_table
---------------------------------------------------------------------

(1 row)

INSERT INTO dist_table_2 SELECT i FROM generate_series(1, 100) i;
SELECT mark_node_readonly('localhost', :worker_2_port, TRUE);
 mark_node_readonly
---------------------------------------------------------------------
 t
(1 row)

-- Now updating the other node will mark worker 2 as not synced.
BEGIN;
SELECT 1 FROM master_update_node(:nodeid_1, 'localhost', 12345);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

SELECT nodeid, hasmetadata, metadatasynced FROM pg_dist_node ORDER BY nodeid;
 nodeid | hasmetadata | metadatasynced
---------------------------------------------------------------------
      2 | t           | f
      3 | t           | f
(2 rows)

COMMIT;
-- worker_2 is out of sync, so further updates aren't sent to it and
-- we shouldn't see the warnings.
SELECT 1 FROM master_update_node(:nodeid_1, 'localhost', 23456);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

SELECT nodeid, hasmetadata, metadatasynced FROM pg_dist_node ORDER BY nodeid;
 nodeid | hasmetadata | metadatasynced
---------------------------------------------------------------------
      2 | t           | f
      3 | t           | f
(2 rows)

-- Make the node writeable.
SELECT mark_node_readonly('localhost', :worker_2_port, FALSE);
 mark_node_readonly
---------------------------------------------------------------------
 t
(1 row)

SELECT wait_until_metadata_sync(30000);
 wait_until_metadata_sync
---------------------------------------------------------------------

(1 row)

-- Mark the node readonly again, so the following master_update_node warns
SELECT mark_node_readonly('localhost', :worker_2_port, TRUE);
 mark_node_readonly
---------------------------------------------------------------------
 t
(1 row)

-- Revert the nodeport of worker 1.
BEGIN;
SELECT 1 FROM master_update_node(:nodeid_1, 'localhost', :worker_1_port);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

SELECT count(*) FROM dist_table_2;
 count
---------------------------------------------------------------------
   100
(1 row)

END;
SELECT wait_until_metadata_sync(30000);
 wait_until_metadata_sync
---------------------------------------------------------------------

(1 row)

-- Make the node writeable.
SELECT mark_node_readonly('localhost', :worker_2_port, FALSE);
 mark_node_readonly
---------------------------------------------------------------------
 t
(1 row)

SELECT wait_until_metadata_sync(30000);
 wait_until_metadata_sync
---------------------------------------------------------------------

(1 row)

SELECT 1 FROM master_update_node(:nodeid_1, 'localhost', :worker_1_port);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

SELECT verify_metadata('localhost', :worker_1_port),
       verify_metadata('localhost', :worker_2_port);
 verify_metadata | verify_metadata
---------------------------------------------------------------------
 t               | t
(1 row)

---------------------------------------------------------------------
-- Test that master_update_node rolls back properly
---------------------------------------------------------------------
BEGIN;
SELECT 1 FROM master_update_node(:nodeid_1, 'localhost', 12345);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

ROLLBACK;
SELECT verify_metadata('localhost', :worker_1_port),
       verify_metadata('localhost', :worker_2_port);
 verify_metadata | verify_metadata
---------------------------------------------------------------------
 t               | t
(1 row)

---------------------------------------------------------------------
-- Test that master_update_node invalidates the plan cache
---------------------------------------------------------------------
PREPARE foo AS SELECT COUNT(*) FROM dist_table_1 WHERE a = 1;
SET citus.log_remote_commands = ON;
-- trigger caching for prepared statements
EXECUTE foo;
NOTICE:  issuing SELECT count(*) AS count FROM public.dist_table_1_102010 dist_table_1 WHERE (a OPERATOR(pg_catalog.=) 1)
 count
---------------------------------------------------------------------
     0
(1 row)

EXECUTE foo;
NOTICE:  issuing SELECT count(*) AS count FROM public.dist_table_1_102010 dist_table_1 WHERE (a OPERATOR(pg_catalog.=) 1)
 count
---------------------------------------------------------------------
     0
(1 row)

EXECUTE foo;
NOTICE:  issuing SELECT count(*) AS count FROM public.dist_table_1_102010 dist_table_1 WHERE (a OPERATOR(pg_catalog.=) 1)
 count
---------------------------------------------------------------------
     0
(1 row)

EXECUTE foo;
NOTICE:  issuing SELECT count(*) AS count FROM public.dist_table_1_102010 dist_table_1 WHERE (a OPERATOR(pg_catalog.=) 1)
 count
---------------------------------------------------------------------
     0
(1 row)

EXECUTE foo;
NOTICE:  issuing SELECT count(*) AS count FROM public.dist_table_1_102010 dist_table_1 WHERE (a OPERATOR(pg_catalog.=) 1)
 count
---------------------------------------------------------------------
     0
(1 row)

EXECUTE foo;
NOTICE:  issuing SELECT count(*) AS count FROM public.dist_table_1_102010 dist_table_1 WHERE (a OPERATOR(pg_catalog.=) 1)
 count
---------------------------------------------------------------------
     0
(1 row)

EXECUTE foo;
NOTICE:  issuing SELECT count(*) AS count FROM public.dist_table_1_102010 dist_table_1 WHERE (a OPERATOR(pg_catalog.=) 1)
 count
---------------------------------------------------------------------
     0
(1 row)

SELECT master_update_node(:nodeid_1, '127.0.0.1', :worker_1_port);
 master_update_node
---------------------------------------------------------------------

(1 row)

SELECT wait_until_metadata_sync(30000);
NOTICE:  issuing LISTEN metadata_sync
 wait_until_metadata_sync
---------------------------------------------------------------------

(1 row)

-- make sure the nodename changed.
EXECUTE foo;
NOTICE:  issuing SELECT count(*) AS count FROM public.dist_table_1_102010 dist_table_1 WHERE (a OPERATOR(pg_catalog.=) 1)
 count
---------------------------------------------------------------------
     0
(1 row)

SET citus.log_remote_commands TO OFF;
---------------------------------------------------------------------
-- Test that master_update_node can appear in a prepared transaction.
---------------------------------------------------------------------
BEGIN;
SELECT 1 FROM master_update_node(:nodeid_1, 'localhost', 12345);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

PREPARE TRANSACTION 'tx01';
COMMIT PREPARED 'tx01';
SELECT wait_until_metadata_sync(30000);
 wait_until_metadata_sync
---------------------------------------------------------------------

(1 row)

SELECT nodeid, hasmetadata, metadatasynced FROM pg_dist_node ORDER BY nodeid;
 nodeid | hasmetadata | metadatasynced
---------------------------------------------------------------------
      2 | t           | f
      3 | t           | t
(2 rows)

BEGIN;
SELECT 1 FROM master_update_node(:nodeid_1, 'localhost', :worker_1_port);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

PREPARE TRANSACTION 'tx01';
COMMIT PREPARED 'tx01';
SELECT wait_until_metadata_sync(30000);
 wait_until_metadata_sync
---------------------------------------------------------------------

(1 row)

SELECT nodeid, hasmetadata, metadatasynced FROM pg_dist_node ORDER BY nodeid;
 nodeid | hasmetadata | metadatasynced
---------------------------------------------------------------------
      2 | t           | t
      3 | t           | t
(2 rows)

SELECT verify_metadata('localhost', :worker_1_port),
       verify_metadata('localhost', :worker_2_port);
 verify_metadata | verify_metadata
---------------------------------------------------------------------
 t               | t
(1 row)

---------------------------------------------------------------------
-- Test that changes in isactive is propagated to the metadata nodes
---------------------------------------------------------------------
-- Don't drop the reference table so it has shards on the nodes being disabled
DROP TABLE dist_table_1, dist_table_2;
SELECT 1 FROM master_disable_node('localhost', :worker_2_port);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

SELECT verify_metadata('localhost', :worker_1_port);
 verify_metadata
---------------------------------------------------------------------
 t
(1 row)

SELECT 1 FROM master_activate_node('localhost', :worker_2_port);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

SELECT verify_metadata('localhost', :worker_1_port);
 verify_metadata
---------------------------------------------------------------------
 t
(1 row)

---------------------------------------------------------------------
-- Test master_disable_node() when the node that is being disabled is actually down
---------------------------------------------------------------------
SELECT master_update_node(:nodeid_2, 'localhost', 1);
 master_update_node
---------------------------------------------------------------------

(1 row)

SELECT wait_until_metadata_sync(30000);
 wait_until_metadata_sync
---------------------------------------------------------------------

(1 row)

-- set metadatasynced so we try porpagating metadata changes
UPDATE pg_dist_node SET metadatasynced = TRUE WHERE nodeid IN (:nodeid_1, :nodeid_2);
-- should error out
SELECT 1 FROM master_disable_node('localhost', 1);
ERROR:  Disabling localhost:xxxxx failed
-- try again after stopping metadata sync
SELECT stop_metadata_sync_to_node('localhost', 1);
 stop_metadata_sync_to_node
---------------------------------------------------------------------

(1 row)

SELECT 1 FROM master_disable_node('localhost', 1);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

SELECT verify_metadata('localhost', :worker_1_port);
 verify_metadata
---------------------------------------------------------------------
 t
(1 row)

SELECT master_update_node(:nodeid_2, 'localhost', :worker_2_port);
 master_update_node
---------------------------------------------------------------------

(1 row)

SELECT wait_until_metadata_sync(30000);
 wait_until_metadata_sync
---------------------------------------------------------------------

(1 row)

SELECT 1 FROM master_activate_node('localhost', :worker_2_port);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

SELECT verify_metadata('localhost', :worker_1_port);
 verify_metadata
---------------------------------------------------------------------
 t
(1 row)

---------------------------------------------------------------------
-- Test master_disable_node() when the other node is down
---------------------------------------------------------------------
-- node 1 is down.
SELECT master_update_node(:nodeid_1, 'localhost', 1);
 master_update_node
---------------------------------------------------------------------

(1 row)

SELECT wait_until_metadata_sync(30000);
 wait_until_metadata_sync
---------------------------------------------------------------------

(1 row)

-- set metadatasynced so we try porpagating metadata changes
UPDATE pg_dist_node SET metadatasynced = TRUE WHERE nodeid IN (:nodeid_1, :nodeid_2);
-- should error out
SELECT 1 FROM master_disable_node('localhost', :worker_2_port);
ERROR:  Disabling localhost:xxxxx failed
-- try again after stopping metadata sync
SELECT stop_metadata_sync_to_node('localhost', 1);
 stop_metadata_sync_to_node
---------------------------------------------------------------------

(1 row)

SELECT 1 FROM master_disable_node('localhost', :worker_2_port);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

-- bring up node 1
SELECT master_update_node(:nodeid_1, 'localhost', :worker_1_port);
 master_update_node
---------------------------------------------------------------------

(1 row)

SELECT wait_until_metadata_sync(30000);
 wait_until_metadata_sync
---------------------------------------------------------------------

(1 row)

SELECT 1 FROM master_activate_node('localhost', :worker_2_port);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

SELECT verify_metadata('localhost', :worker_1_port);
 verify_metadata
---------------------------------------------------------------------
 t
(1 row)

-- cleanup
DROP TABLE ref_table;
TRUNCATE pg_dist_colocation;
SELECT count(*) FROM (SELECT master_remove_node(nodename, nodeport) FROM pg_dist_node) t;
 count
---------------------------------------------------------------------
     2
(1 row)

ALTER SEQUENCE pg_catalog.pg_dist_groupid_seq RESTART :last_group_id;
ALTER SEQUENCE pg_catalog.pg_dist_node_nodeid_seq RESTART :last_node_id;
ALTER SEQUENCE pg_catalog.pg_dist_colocationid_seq RESTART :last_colocation_id;
ALTER SEQUENCE pg_catalog.pg_dist_placement_placementid_seq RESTART :last_placement_id;
ALTER SEQUENCE pg_catalog.pg_dist_shardid_seq RESTART :last_shard_id;
RESET citus.shard_count;
RESET citus.shard_replication_factor;
RESET citus.replication_model;