SET citus.next_shard_id TO 1220000;
ALTER SEQUENCE pg_catalog.pg_dist_colocationid_seq RESTART 1390000;
ALTER SEQUENCE pg_catalog.pg_dist_groupid_seq RESTART 1;
-- Tests functions related to cluster membership
-- add the first node to the cluster in transactional mode
SELECT 1 FROM master_add_node('localhost', :worker_1_port);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

-- add the second node in nontransactional mode
SET citus.metadata_sync_mode TO 'nontransactional';
SELECT 1 FROM master_add_node('localhost', :worker_2_port);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

RESET citus.metadata_sync_mode;
-- I am coordinator
SELECT citus_is_coordinator();
 citus_is_coordinator
---------------------------------------------------------------------
 t
(1 row)

-- I am primary node (fails beacuse not set in pg_dist)
select citus_is_primary_node();
WARNING:  could not find the current node in pg_dist_node
DETAIL:  If this is the coordinator node, consider adding it into the metadata by using citus_set_coordinator_host() UDF. Otherwise, if you're going to use this node as a worker node for a new cluster, make sure to add this node into the metadata from the coordinator by using citus_add_node() UDF.
 citus_is_primary_node
---------------------------------------------------------------------

(1 row)

-- make sure coordinator is always in metadata.
SELECT citus_set_coordinator_host('localhost');
 citus_set_coordinator_host
---------------------------------------------------------------------

(1 row)

-- I am primary node
select citus_is_primary_node();
 citus_is_primary_node
---------------------------------------------------------------------
 t
(1 row)

-- workers are not coordinator
SELECT result FROM run_command_on_workers('SELECT citus_is_coordinator()');
 result
---------------------------------------------------------------------
 f
 f
(2 rows)

-- primary workers are primary node
SELECT result FROM run_command_on_workers('SELECT citus_is_primary_node()');
 result
---------------------------------------------------------------------
 t
 t
(2 rows)

-- get the active nodes
SELECT master_get_active_worker_nodes();
 master_get_active_worker_nodes
---------------------------------------------------------------------
 (localhost,57638)
 (localhost,57637)
(2 rows)

-- try to add a node that is already in the cluster
SELECT * FROM master_add_node('localhost', :worker_1_port);
 master_add_node
---------------------------------------------------------------------
               1
(1 row)

-- get the active nodes
SELECT master_get_active_worker_nodes();
 master_get_active_worker_nodes
---------------------------------------------------------------------
 (localhost,57638)
 (localhost,57637)
(2 rows)

-- try to remove a node (with no placements)
SELECT master_remove_node('localhost', :worker_2_port);
 master_remove_node
---------------------------------------------------------------------

(1 row)

-- verify that the node has been deleted
SELECT master_get_active_worker_nodes();
 master_get_active_worker_nodes
---------------------------------------------------------------------
 (localhost,57637)
(1 row)

-- try to disable a node with no placements see that node is s=removed
SELECT 1 FROM master_add_node('localhost', :worker_2_port);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

SELECT citus_disable_node('localhost', :worker_2_port);
 citus_disable_node
---------------------------------------------------------------------

(1 row)

SELECT public.wait_until_metadata_sync(20000);
 wait_until_metadata_sync
---------------------------------------------------------------------

(1 row)

SELECT master_get_active_worker_nodes();
 master_get_active_worker_nodes
---------------------------------------------------------------------
 (localhost,57637)
(1 row)

-- add some shard placements to the cluster
SET citus.shard_count TO 16;
SET citus.shard_replication_factor TO 1;
-- test warnings on setting the deprecated guc for replication model
BEGIN;
SET citus.replication_model to 'statement';
NOTICE:  Setting citus.replication_model has no effect. Please use citus.shard_replication_factor instead.
DETAIL:  Citus determines the replication model based on the replication factor and the replication models of the colocated shards. If a colocated table is present, the replication model is inherited. Otherwise 'streaming' replication is preferred if supported by the replication factor.
ROLLBACK;
-- check that the rebalancer works even if there are no distributed tables
SELECT * FROM get_rebalance_table_shards_plan();
 table_name | shardid | shard_size | sourcename | sourceport | targetname | targetport
---------------------------------------------------------------------
(0 rows)

SELECT * FROM rebalance_table_shards();
 rebalance_table_shards
---------------------------------------------------------------------

(1 row)

-- TODO: Figure out why this is necessary, rebalance_table_shards shouldn't
-- insert stuff into pg_dist_colocation
TRUNCATE pg_dist_colocation;
SELECT run_command_on_workers('TRUNCATE pg_dist_colocation');
        run_command_on_workers
---------------------------------------------------------------------
 (localhost,57637,t,"TRUNCATE TABLE")
(1 row)

ALTER SEQUENCE pg_catalog.pg_dist_colocationid_seq RESTART 1390000;
SELECT 1 FROM citus_activate_node('localhost', :worker_2_port);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

-- disable node with sync/force options
SELECT citus_disable_node('localhost', :worker_1_port);
ERROR:  disabling the first worker node in the metadata is not allowed
DETAIL:  Citus uses the first worker node in the metadata for certain internal operations when replicated tables are modified. Synchronous mode ensures that all nodes have the same view of the first worker node, which is used for certain locking operations.
HINT:  You can force disabling node, SELECT citus_disable_node('localhost', 57637, synchronous:=true);
SELECT citus_disable_node('localhost', :worker_1_port, synchronous:=true);
 citus_disable_node
---------------------------------------------------------------------

(1 row)

SELECT run_command_on_workers($$SELECT array_agg(isactive ORDER BY nodeport) FROM pg_dist_node WHERE hasmetadata and noderole='primary'::noderole AND nodecluster='default'$$);
    run_command_on_workers
---------------------------------------------------------------------
 (localhost,57638,t,"{t,f,t}")
(1 row)

SELECT 1 FROM citus_activate_node('localhost', :worker_1_port);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

-- disable node with sync/force options
SELECT citus_disable_node('localhost', :worker_2_port, synchronous:=true);
 citus_disable_node
---------------------------------------------------------------------

(1 row)

SELECT run_command_on_workers($$SELECT array_agg(isactive ORDER BY nodeport) FROM pg_dist_node WHERE hasmetadata and noderole='primary'::noderole AND nodecluster='default'$$);
    run_command_on_workers
---------------------------------------------------------------------
 (localhost,57637,t,"{t,t,f}")
(1 row)

SELECT 1 FROM citus_activate_node('localhost', :worker_2_port);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

SELECT 1 FROM citus_activate_node('localhost', :worker_1_port);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

CREATE TABLE cluster_management_test (col_1 text, col_2 int);
SELECT create_distributed_table('cluster_management_test', 'col_1', 'hash');
 create_distributed_table
---------------------------------------------------------------------

(1 row)

-- see that there are some active placements in the candidate node
SELECT shardid, shardstate, nodename, nodeport FROM pg_dist_shard_placement WHERE nodeport=:worker_2_port;
 shardid | shardstate | nodename  | nodeport
---------------------------------------------------------------------
 1220001 |          1 | localhost |    57638
 1220003 |          1 | localhost |    57638
 1220005 |          1 | localhost |    57638
 1220007 |          1 | localhost |    57638
 1220009 |          1 | localhost |    57638
 1220011 |          1 | localhost |    57638
 1220013 |          1 | localhost |    57638
 1220015 |          1 | localhost |    57638
(8 rows)

-- try to remove a node with active placements and see that node removal is failed
SELECT master_remove_node('localhost', :worker_2_port);
ERROR:  cannot remove or disable the node localhost:xxxxx because because it contains the only shard placement for shard xxxxx
DETAIL:  One of the table(s) that prevents the operation complete successfully is public.cluster_management_test
HINT:  To proceed, either drop the tables or use undistribute_table() function to convert them to local tables
SELECT master_get_active_worker_nodes();
 master_get_active_worker_nodes
---------------------------------------------------------------------
 (localhost,57638)
 (localhost,57637)
(2 rows)

-- insert a row so that citus_disable_node() exercises closing connections
CREATE TABLE test_reference_table (y int primary key, name text);
SELECT create_reference_table('test_reference_table');
 create_reference_table
---------------------------------------------------------------------

(1 row)

INSERT INTO test_reference_table VALUES (1, '1');
-- try to remove a node with active placements and reference tables
SELECT citus_remove_node('localhost', :worker_2_port);
ERROR:  cannot remove or disable the node localhost:xxxxx because because it contains the only shard placement for shard xxxxx
DETAIL:  One of the table(s) that prevents the operation complete successfully is public.cluster_management_test
HINT:  To proceed, either drop the tables or use undistribute_table() function to convert them to local tables
-- try to disable a node with active placements
-- which should fail because there are some placements
-- which are the only placements for a given shard
SELECT citus_disable_node('localhost', :worker_2_port);
ERROR:  cannot remove or disable the node localhost:xxxxx because because it contains the only shard placement for shard xxxxx
DETAIL:  One of the table(s) that prevents the operation complete successfully is public.cluster_management_test
HINT:  To proceed, either drop the tables or use undistribute_table() function to convert them to local tables
SELECT master_get_active_worker_nodes();
 master_get_active_worker_nodes
---------------------------------------------------------------------
 (localhost,57638)
 (localhost,57637)
(2 rows)

-- try to disable a node which does not exist and see that an error is thrown
SELECT citus_disable_node('localhost.noexist', 2345);
ERROR:  node at "localhost.noexist:2345" does not exist
-- drop the table without leaving a shard placement behind (messes up other tests)
SELECT master_activate_node('localhost', :worker_2_port);
 master_activate_node
---------------------------------------------------------------------
                    4
(1 row)

DROP TABLE test_reference_table, cluster_management_test;
-- create users like this so results of community and enterprise are same
SET client_min_messages TO ERROR;
CREATE USER non_super_user;
CREATE USER node_metadata_user;
SELECT 1 FROM run_command_on_workers('CREATE USER node_metadata_user');
 ?column?
---------------------------------------------------------------------
        1
        1
(2 rows)

RESET client_min_messages;
GRANT EXECUTE ON FUNCTION master_activate_node(text,int) TO node_metadata_user;
GRANT EXECUTE ON FUNCTION master_add_inactive_node(text,int,int,noderole,name) TO node_metadata_user;
GRANT EXECUTE ON FUNCTION master_add_node(text,int,int,noderole,name) TO node_metadata_user;
GRANT EXECUTE ON FUNCTION master_add_secondary_node(text,int,text,int,name) TO node_metadata_user;
GRANT EXECUTE ON FUNCTION citus_disable_node(text,int,bool) TO node_metadata_user;
GRANT EXECUTE ON FUNCTION citus_disable_node_and_wait(text,int,bool) TO node_metadata_user;
GRANT EXECUTE ON FUNCTION master_remove_node(text,int) TO node_metadata_user;
GRANT EXECUTE ON FUNCTION master_update_node(int,text,int,bool,int) TO node_metadata_user;
-- user needs permission for the pg_dist_node and pg_dist_local_group for metadata syncing
SELECT run_command_on_workers('GRANT ALL ON pg_dist_node TO node_metadata_user');
  run_command_on_workers
---------------------------------------------------------------------
 (localhost,57637,t,GRANT)
 (localhost,57638,t,GRANT)
(2 rows)

SELECT run_command_on_workers('GRANT ALL ON pg_dist_local_group TO node_metadata_user');
  run_command_on_workers
---------------------------------------------------------------------
 (localhost,57637,t,GRANT)
 (localhost,57638,t,GRANT)
(2 rows)

SELECT run_command_on_workers('GRANT ALL ON ALL TABLES IN SCHEMA citus TO node_metadata_user');
  run_command_on_workers
---------------------------------------------------------------------
 (localhost,57637,t,GRANT)
 (localhost,57638,t,GRANT)
(2 rows)

SELECT run_command_on_workers('GRANT ALL ON SCHEMA citus TO node_metadata_user');
  run_command_on_workers
---------------------------------------------------------------------
 (localhost,57637,t,GRANT)
 (localhost,57638,t,GRANT)
(2 rows)

SELECT master_remove_node('localhost', :worker_2_port);
 master_remove_node
---------------------------------------------------------------------

(1 row)

-- Removing public schema from pg_dist_object because it breaks the next tests
DELETE FROM pg_catalog.pg_dist_object WHERE objid = 'public'::regnamespace::oid;
DELETE FROM pg_catalog.pg_dist_object WHERE objid = (SELECT oid FROM pg_extension WHERE extname = 'plpgsql');
-- try to manipulate node metadata via non-super user
SET ROLE non_super_user;
SELECT 1 FROM master_add_inactive_node('localhost', :worker_2_port + 1);
ERROR:  permission denied for function master_add_inactive_node
SELECT 1 FROM master_activate_node('localhost', :worker_2_port + 1);
ERROR:  permission denied for function master_activate_node
SELECT 1 FROM citus_disable_node('localhost', :worker_2_port + 1);
ERROR:  permission denied for function citus_disable_node
SELECT 1 FROM master_remove_node('localhost', :worker_2_port + 1);
ERROR:  permission denied for function master_remove_node
SELECT 1 FROM master_add_node('localhost', :worker_2_port + 1);
ERROR:  permission denied for function master_add_node
SELECT 1 FROM master_add_secondary_node('localhost', :worker_2_port + 2, 'localhost', :worker_2_port);
ERROR:  permission denied for function master_add_secondary_node
SELECT master_update_node(nodeid, 'localhost', :worker_2_port + 3) FROM pg_dist_node WHERE nodeport = :worker_2_port;
ERROR:  permission denied for function master_update_node
-- try to manipulate node metadata via privileged user
SET ROLE node_metadata_user;
SELECT 1 FROM master_add_node('localhost', :worker_2_port);
ERROR:  operation is not allowed
HINT:  Run the command with a superuser.
BEGIN;
SELECT 1 FROM master_add_inactive_node('localhost', :worker_2_port);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

SELECT 1 FROM master_remove_node('localhost', :worker_2_port);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

SELECT 1 FROM master_add_secondary_node('localhost', :worker_2_port + 2, 'localhost', :worker_1_port);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

SELECT master_update_node(nodeid, 'localhost', :worker_2_port + 3) FROM pg_dist_node WHERE nodeport = :worker_2_port;
 master_update_node
---------------------------------------------------------------------
(0 rows)

SELECT nodename, nodeport, noderole FROM pg_dist_node ORDER BY nodeport;
 nodename  | nodeport | noderole
---------------------------------------------------------------------
 localhost |    57636 | primary
 localhost |    57637 | primary
 localhost |    57640 | secondary
(3 rows)

ABORT;
\c - postgres - :master_port
SET citus.next_shard_id TO 1220000;
SET citus.shard_count TO 16;
SET citus.shard_replication_factor TO 1;
SELECT master_get_active_worker_nodes();
 master_get_active_worker_nodes
---------------------------------------------------------------------
 (localhost,57637)
(1 row)

-- restore the node for next tests
SELECT * FROM master_add_node('localhost', :worker_2_port);
 master_add_node
---------------------------------------------------------------------
               7
(1 row)

ALTER SEQUENCE pg_dist_node_nodeid_seq RESTART WITH 7;
ALTER SEQUENCE pg_dist_groupid_seq RESTART WITH 6;
CREATE TABLE cluster_management_test (col_1 text, col_2 int);
SELECT create_distributed_table('cluster_management_test', 'col_1', 'hash');
 create_distributed_table
---------------------------------------------------------------------

(1 row)

-- try to remove a node with active placements and see that node removal is failed
SELECT master_remove_node('localhost', :worker_2_port);
ERROR:  cannot remove or disable the node localhost:xxxxx because because it contains the only shard placement for shard xxxxx
DETAIL:  One of the table(s) that prevents the operation complete successfully is public.cluster_management_test
HINT:  To proceed, either drop the tables or use undistribute_table() function to convert them to local tables
-- mark all placements in the candidate node as inactive
SELECT groupid AS worker_2_group FROM pg_dist_node WHERE nodeport=:worker_2_port \gset
UPDATE pg_dist_placement SET shardstate=3 WHERE groupid=:worker_2_group;
-- manual updates to pg_dist* tables are not automatically reflected to the workers, so we manually do that too
SELECT run_command_on_workers('UPDATE pg_dist_placement SET shardstate=3 WHERE groupid=' || :'worker_2_group');
     run_command_on_workers
---------------------------------------------------------------------
 (localhost,57637,t,"UPDATE 8")
 (localhost,57638,t,"UPDATE 8")
(2 rows)

SELECT shardid, shardstate, nodename, nodeport FROM pg_dist_shard_placement WHERE nodeport=:worker_2_port;
 shardid | shardstate | nodename  | nodeport
---------------------------------------------------------------------
 1220001 |          3 | localhost |    57638
 1220003 |          3 | localhost |    57638
 1220005 |          3 | localhost |    57638
 1220007 |          3 | localhost |    57638
 1220009 |          3 | localhost |    57638
 1220011 |          3 | localhost |    57638
 1220013 |          3 | localhost |    57638
 1220015 |          3 | localhost |    57638
(8 rows)

-- try to remove a node with only inactive placements and see that removal still fails
SELECT master_remove_node('localhost', :worker_2_port);
ERROR:  cannot remove or disable the node localhost:xxxxx because because it contains the only shard placement for shard xxxxx
DETAIL:  One of the table(s) that prevents the operation complete successfully is public.cluster_management_test
HINT:  To proceed, either drop the tables or use undistribute_table() function to convert them to local tables
SELECT master_get_active_worker_nodes();
 master_get_active_worker_nodes
---------------------------------------------------------------------
 (localhost,57638)
 (localhost,57637)
(2 rows)

-- clean-up
SELECT 1 FROM master_add_node('localhost', :worker_2_port);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

UPDATE pg_dist_placement SET shardstate=1 WHERE groupid=:worker_2_group;
SELECT run_command_on_workers('UPDATE pg_dist_placement SET shardstate=1 WHERE groupid=' || :'worker_2_group');
     run_command_on_workers
---------------------------------------------------------------------
 (localhost,57637,t,"UPDATE 8")
 (localhost,57638,t,"UPDATE 8")
(2 rows)

-- when there is no primary we should get a pretty error
UPDATE pg_dist_node SET noderole = 'secondary' WHERE nodeport=:worker_2_port;
SELECT * FROM cluster_management_test;
ERROR:  node group 5 does not have a primary node
-- when there is no node at all in the group we should get a different error
DELETE FROM pg_dist_node WHERE nodeport=:worker_2_port;
SELECT run_command_on_workers('DELETE FROM pg_dist_node WHERE nodeport=' || :'worker_2_port');
     run_command_on_workers
---------------------------------------------------------------------
 (localhost,57637,t,"DELETE 1")
(1 row)

SELECT * FROM cluster_management_test;
ERROR:  there is a shard placement in node group 5 but there are no nodes in that group
-- clean-up
SELECT * INTO old_placements FROM pg_dist_placement WHERE groupid = :worker_2_group;
DELETE FROM pg_dist_placement WHERE groupid = :worker_2_group;
SELECT master_add_node('localhost', :worker_2_port) AS new_node \gset
WARNING:  could not find any shard placements for shardId 1220001
WARNING:  could not find any shard placements for shardId 1220003
WARNING:  could not find any shard placements for shardId 1220005
WARNING:  could not find any shard placements for shardId 1220007
WARNING:  could not find any shard placements for shardId 1220009
WARNING:  could not find any shard placements for shardId 1220011
WARNING:  could not find any shard placements for shardId 1220013
WARNING:  could not find any shard placements for shardId 1220015
INSERT INTO pg_dist_placement SELECT * FROM old_placements;
SELECT groupid AS new_group FROM pg_dist_node WHERE nodeid = :new_node \gset
UPDATE pg_dist_placement SET groupid = :new_group WHERE groupid = :worker_2_group;
SELECT run_command_on_workers('UPDATE pg_dist_placement SET groupid = ' || :'new_group' || ' WHERE groupid = ' || :'worker_2_group');
     run_command_on_workers
---------------------------------------------------------------------
 (localhost,57637,t,"UPDATE 8")
 (localhost,57638,t,"UPDATE 0")
(2 rows)

SELECT start_metadata_sync_to_node('localhost', :worker_2_port);
 start_metadata_sync_to_node
---------------------------------------------------------------------

(1 row)

-- test that you are allowed to remove secondary nodes even if there are placements
SELECT 1 FROM master_add_node('localhost', 9990, groupid => :new_group, noderole => 'secondary');
 ?column?
---------------------------------------------------------------------
        1
(1 row)

SELECT master_remove_node('localhost', :worker_2_port);
ERROR:  cannot remove or disable the node localhost:xxxxx because because it contains the only shard placement for shard xxxxx
DETAIL:  One of the table(s) that prevents the operation complete successfully is public.cluster_management_test
HINT:  To proceed, either drop the tables or use undistribute_table() function to convert them to local tables
SELECT master_remove_node('localhost', 9990);
 master_remove_node
---------------------------------------------------------------------

(1 row)

-- clean-up
DROP TABLE cluster_management_test;
-- check that adding/removing nodes are propagated to nodes with metadata
SELECT master_remove_node('localhost', :worker_2_port);
 master_remove_node
---------------------------------------------------------------------

(1 row)

SELECT start_metadata_sync_to_node('localhost', :worker_1_port);
 start_metadata_sync_to_node
---------------------------------------------------------------------

(1 row)

SELECT 1 FROM master_add_node('localhost', :worker_2_port);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

\c - - - :worker_1_port
SELECT nodename, nodeport FROM pg_dist_node WHERE nodename='localhost' AND nodeport=:worker_2_port;
 nodename  | nodeport
---------------------------------------------------------------------
 localhost |    57638
(1 row)

\c - - - :master_port
SELECT master_remove_node('localhost', :worker_2_port);
 master_remove_node
---------------------------------------------------------------------

(1 row)

\c - - - :worker_1_port
SELECT nodename, nodeport FROM pg_dist_node WHERE nodename='localhost' AND nodeport=:worker_2_port;
 nodename | nodeport
---------------------------------------------------------------------
(0 rows)

\c - - - :master_port
-- check that added nodes are not propagated to nodes without metadata
SELECT stop_metadata_sync_to_node('localhost', :worker_1_port);
NOTICE:  dropping metadata on the node (localhost,57637)
 stop_metadata_sync_to_node
---------------------------------------------------------------------

(1 row)

SELECT 1 FROM master_add_node('localhost', :worker_2_port);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

\c - - - :worker_1_port
SELECT nodename, nodeport FROM pg_dist_node WHERE nodename='localhost' AND nodeport=:worker_2_port;
 nodename | nodeport
---------------------------------------------------------------------
(0 rows)

\c - - - :master_port
-- check that removing two nodes in the same transaction works
SELECT
	master_remove_node('localhost', :worker_1_port),
	master_remove_node('localhost', :worker_2_port);
 master_remove_node | master_remove_node
---------------------------------------------------------------------
                    |
(1 row)

SELECT count(1) FROM pg_dist_node;
 count
---------------------------------------------------------------------
     1
(1 row)

-- check that adding two nodes in the same transaction works
SELECT
	master_add_node('localhost', :worker_1_port),
	master_add_node('localhost', :worker_2_port);
 master_add_node | master_add_node
---------------------------------------------------------------------
              11 |              12
(1 row)

SELECT * FROM pg_dist_node ORDER BY nodeid;
 nodeid | groupid | nodename  | nodeport | noderack | hasmetadata | isactive | noderole | nodecluster | metadatasynced | shouldhaveshards
---------------------------------------------------------------------
      3 |       0 | localhost |    57636 | default  | t           | t        | primary  | default     | t              | f
     11 |       9 | localhost |    57637 | default  | t           | t        | primary  | default     | t              | t
     12 |      10 | localhost |    57638 | default  | t           | t        | primary  | default     | t              | t
(3 rows)

-- check that mixed add/remove node commands work fine inside transaction
BEGIN;
SELECT master_remove_node('localhost', :worker_2_port);
 master_remove_node
---------------------------------------------------------------------

(1 row)

SELECT 1 FROM master_add_node('localhost', :worker_2_port);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

SELECT master_remove_node('localhost', :worker_2_port);
 master_remove_node
---------------------------------------------------------------------

(1 row)

COMMIT;
SELECT nodename, nodeport FROM pg_dist_node WHERE nodename='localhost' AND nodeport=:worker_2_port;
 nodename | nodeport
---------------------------------------------------------------------
(0 rows)

SELECT start_metadata_sync_to_node('localhost', :worker_1_port);
 start_metadata_sync_to_node
---------------------------------------------------------------------

(1 row)

BEGIN;
SELECT 1 FROM master_add_node('localhost', :worker_2_port);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

SELECT master_remove_node('localhost', :worker_2_port);
 master_remove_node
---------------------------------------------------------------------

(1 row)

SELECT 1 FROM master_add_node('localhost', :worker_2_port);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

COMMIT;
SELECT nodename, nodeport FROM pg_dist_node WHERE nodename='localhost' AND nodeport=:worker_2_port;
 nodename  | nodeport
---------------------------------------------------------------------
 localhost |    57638
(1 row)

\c - - - :worker_1_port
SELECT nodename, nodeport FROM pg_dist_node WHERE nodename='localhost' AND nodeport=:worker_2_port;
 nodename  | nodeport
---------------------------------------------------------------------
 localhost |    57638
(1 row)

\c - - - :master_port
SELECT master_remove_node(nodename, nodeport) FROM pg_dist_node;
 master_remove_node
---------------------------------------------------------------------



(3 rows)

SELECT citus_set_coordinator_host('localhost');
 citus_set_coordinator_host
---------------------------------------------------------------------

(1 row)

SELECT 1 FROM master_add_node('localhost', :worker_1_port);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

SELECT 1 FROM master_add_node('localhost', :worker_2_port);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

-- check that a distributed table can be created after adding a node in a transaction
SET citus.shard_count TO 4;
SELECT master_remove_node('localhost', :worker_2_port);
 master_remove_node
---------------------------------------------------------------------

(1 row)

BEGIN;
SELECT 1 FROM master_add_node('localhost', :worker_2_port);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

CREATE TABLE temp(col1 text, col2 int);
SELECT create_distributed_table('temp', 'col1');
 create_distributed_table
---------------------------------------------------------------------

(1 row)

INSERT INTO temp VALUES ('row1', 1);
INSERT INTO temp VALUES ('row2', 2);
COMMIT;
SELECT col1, col2 FROM temp ORDER BY col1;
 col1 | col2
---------------------------------------------------------------------
 row1 |    1
 row2 |    2
(2 rows)

SELECT
	count(*)
FROM
	pg_dist_shard_placement, pg_dist_shard
WHERE
	pg_dist_shard_placement.shardid = pg_dist_shard.shardid
	AND pg_dist_shard.logicalrelid = 'temp'::regclass
	AND pg_dist_shard_placement.nodeport = :worker_2_port;
 count
---------------------------------------------------------------------
     4
(1 row)

DROP TABLE temp;
\c - - - :worker_1_port
DELETE FROM pg_dist_partition;
DELETE FROM pg_dist_shard;
DELETE FROM pg_dist_placement;
DELETE FROM pg_dist_node;
\c - - - :master_port
SELECT stop_metadata_sync_to_node('localhost', :worker_1_port);
NOTICE:  dropping metadata on the node (localhost,57637)
 stop_metadata_sync_to_node
---------------------------------------------------------------------

(1 row)

SELECT stop_metadata_sync_to_node('localhost', :worker_2_port);
NOTICE:  dropping metadata on the node (localhost,57638)
 stop_metadata_sync_to_node
---------------------------------------------------------------------

(1 row)

-- check that you can't add a primary to a non-default cluster
SELECT master_add_node('localhost', 9999, nodecluster => 'olap');
ERROR:  primaries must be added to the default cluster
-- check that you can't add more than one primary to a group
SELECT groupid AS worker_1_group FROM pg_dist_node WHERE nodeport = :worker_1_port \gset
SELECT master_add_node('localhost', 9999, groupid => :worker_1_group, noderole => 'primary');
ERROR:  group 14 already has a primary node
-- check that you can add secondaries and unavailable nodes to a group
SELECT groupid AS worker_2_group FROM pg_dist_node WHERE nodeport = :worker_2_port \gset
SELECT 1 FROM master_add_node('localhost', 9998, groupid => :worker_1_group, noderole => 'secondary');
 ?column?
---------------------------------------------------------------------
        1
(1 row)

SELECT 1 FROM master_add_node('localhost', 9997, groupid => :worker_1_group, noderole => 'unavailable');
 ?column?
---------------------------------------------------------------------
        1
(1 row)

-- add_inactive_node also works with secondaries
SELECT 1 FROM master_add_inactive_node('localhost', 9996, groupid => :worker_2_group, noderole => 'secondary');
 ?column?
---------------------------------------------------------------------
        1
(1 row)

-- check that you can add a seconary to a non-default cluster, and activate it, and remove it
SELECT master_add_inactive_node('localhost', 9999, groupid => :worker_2_group, nodecluster => 'olap', noderole => 'secondary');
 master_add_inactive_node
---------------------------------------------------------------------
                       23
(1 row)

SELECT master_activate_node('localhost', 9999);
 master_activate_node
---------------------------------------------------------------------
                   23
(1 row)

SELECT citus_disable_node('localhost', 9999);
 citus_disable_node
---------------------------------------------------------------------

(1 row)

SELECT public.wait_until_metadata_sync(20000);
 wait_until_metadata_sync
---------------------------------------------------------------------

(1 row)

SELECT master_remove_node('localhost', 9999);
 master_remove_node
---------------------------------------------------------------------

(1 row)

-- check that you can't manually add two primaries to a group
INSERT INTO pg_dist_node (nodename, nodeport, groupid, noderole)
  VALUES ('localhost', 5000, :worker_1_group, 'primary');
ERROR:  there cannot be two primary nodes in a group
CONTEXT:  PL/pgSQL function citus_internal.pg_dist_node_trigger_func() line XX at RAISE
UPDATE pg_dist_node SET noderole = 'primary'
  WHERE groupid = :worker_1_group AND nodeport = 9998;
ERROR:  there cannot be two primary nodes in a group
CONTEXT:  PL/pgSQL function citus_internal.pg_dist_node_trigger_func() line XX at RAISE
-- check that you can't manually add a primary to a non-default cluster
INSERT INTO pg_dist_node (nodename, nodeport, groupid, noderole, nodecluster)
  VALUES ('localhost', 5000, 1000, 'primary', 'olap');
ERROR:  new row for relation "pg_dist_node" violates check constraint "primaries_are_only_allowed_in_the_default_cluster"
DETAIL:  Failing row contains (25, 1000, localhost, 5000, default, f, t, primary, olap, f, t).
UPDATE pg_dist_node SET nodecluster = 'olap'
  WHERE nodeport = :worker_1_port;
ERROR:  new row for relation "pg_dist_node" violates check constraint "primaries_are_only_allowed_in_the_default_cluster"
DETAIL:  Failing row contains (17, 14, localhost, 57637, default, f, t, primary, olap, f, t).
-- check that you /can/ add a secondary node to a non-default cluster
SELECT groupid AS worker_2_group FROM pg_dist_node WHERE nodeport = :worker_2_port \gset
SELECT master_add_node('localhost', 8888, groupid => :worker_1_group, noderole => 'secondary', nodecluster=> 'olap');
 master_add_node
---------------------------------------------------------------------
              26
(1 row)

-- check that super-long cluster names are truncated
SELECT master_add_node('localhost', 8887, groupid => :worker_1_group, noderole => 'secondary', nodecluster=>
	'thisisasixtyfourcharacterstringrepeatedfourtimestomake256chars.'
	'thisisasixtyfourcharacterstringrepeatedfourtimestomake256chars.'
	'thisisasixtyfourcharacterstringrepeatedfourtimestomake256chars.'
	'thisisasixtyfourcharacterstringrepeatedfourtimestomake256chars.'
	'overflow'
);
 master_add_node
---------------------------------------------------------------------
              27
(1 row)

SELECT * FROM pg_dist_node WHERE nodeport=8887;
 nodeid | groupid | nodename  | nodeport | noderack | hasmetadata | isactive | noderole  |                           nodecluster                           | metadatasynced | shouldhaveshards
---------------------------------------------------------------------
     27 |      14 | localhost |     8887 | default  | f           | t        | secondary | thisisasixtyfourcharacterstringrepeatedfourtimestomake256chars. | f              | t
(1 row)

-- don't remove the secondary and unavailable nodes, check that no commands are sent to
-- them in any of the remaining tests
-- master_add_secondary_node lets you skip looking up the groupid
SELECT master_add_secondary_node('localhost', 9995, 'localhost', :worker_1_port);
 master_add_secondary_node
---------------------------------------------------------------------
                        28
(1 row)

SELECT master_add_secondary_node('localhost', 9994, primaryname => 'localhost', primaryport => :worker_2_port);
 master_add_secondary_node
---------------------------------------------------------------------
                        29
(1 row)

SELECT master_add_secondary_node('localhost', 9993, 'localhost', 2000);
ERROR:  node at "localhost:xxxxx" does not exist
SELECT master_add_secondary_node('localhost', 9992, 'localhost', :worker_1_port, nodecluster => 'second-cluster');
 master_add_secondary_node
---------------------------------------------------------------------
                        30
(1 row)

SELECT nodeid AS worker_1_node FROM pg_dist_node WHERE nodeport=9992 \gset
-- citus_update_node allows updating a node from the non-default cluster
SELECT citus_update_node(:worker_1_node, 'localhost', 9991);
 citus_update_node
---------------------------------------------------------------------

(1 row)

SELECT citus_nodename_for_nodeid(:worker_1_node);
 citus_nodename_for_nodeid
---------------------------------------------------------------------
 localhost
(1 row)

SELECT citus_nodeport_for_nodeid(:worker_1_node);
 citus_nodeport_for_nodeid
---------------------------------------------------------------------
                      9991
(1 row)

SELECT citus_update_node(:worker_1_node, 'localhost', 9992);
 citus_update_node
---------------------------------------------------------------------

(1 row)

SELECT citus_nodename_for_nodeid(:worker_1_node);
 citus_nodename_for_nodeid
---------------------------------------------------------------------
 localhost
(1 row)

SELECT citus_nodeport_for_nodeid(:worker_1_node);
 citus_nodeport_for_nodeid
---------------------------------------------------------------------
                      9992
(1 row)

SELECT nodeid AS worker_1_node FROM pg_dist_node WHERE nodeport=:worker_1_port \gset
-- master_update_node checks node exists
SELECT master_update_node(100, 'localhost', 8000);
ERROR:  node 100 not found
-- master_update_node disallows aliasing existing node
SELECT master_update_node(:worker_1_node, 'localhost', :worker_2_port);
ERROR:  there is already another node with the specified hostname and port
-- master_update_node moves a node
SELECT master_update_node(:worker_1_node, 'somehost', 9000);
 master_update_node
---------------------------------------------------------------------

(1 row)

SELECT * FROM pg_dist_node WHERE nodeid = :worker_1_node;
 nodeid | groupid | nodename | nodeport | noderack | hasmetadata | isactive | noderole | nodecluster | metadatasynced | shouldhaveshards
---------------------------------------------------------------------
     17 |      14 | somehost |     9000 | default  | f           | t        | primary  | default     | f              | t
(1 row)

-- cleanup
SELECT master_update_node(:worker_1_node, 'localhost', :worker_1_port);
 master_update_node
---------------------------------------------------------------------

(1 row)

SELECT * FROM pg_dist_node WHERE nodeid = :worker_1_node;
 nodeid | groupid | nodename  | nodeport | noderack | hasmetadata | isactive | noderole | nodecluster | metadatasynced | shouldhaveshards
---------------------------------------------------------------------
     17 |      14 | localhost |    57637 | default  | f           | t        | primary  | default     | f              | t
(1 row)

SET client_min_messages TO ERROR;
SELECT start_metadata_sync_to_node(nodename, nodeport) FROM pg_dist_node WHERE isactive = 't' and noderole = 'primary';
 start_metadata_sync_to_node
---------------------------------------------------------------------



(3 rows)

RESET client_min_messages;
SET citus.shard_replication_factor TO 1;
CREATE TABLE test_dist (x int, y int);
SELECT create_distributed_table('test_dist', 'x');
 create_distributed_table
---------------------------------------------------------------------

(1 row)

-- testing behaviour when setting shouldhaveshards to false on partially empty node
SELECT * from master_set_node_property('localhost', :worker_2_port, 'shouldhaveshards', false);
 master_set_node_property
---------------------------------------------------------------------

(1 row)

CREATE TABLE test_dist_colocated (x int, y int);
CREATE TABLE test_dist_non_colocated (x int, y int);
CREATE TABLE test_dist_colocated_with_non_colocated (x int, y int);
CREATE TABLE test_ref (a int, b int);
SELECT create_distributed_table('test_dist_colocated', 'x');
 create_distributed_table
---------------------------------------------------------------------

(1 row)

SELECT create_distributed_table('test_dist_non_colocated', 'x', colocate_with => 'none');
 create_distributed_table
---------------------------------------------------------------------

(1 row)

SELECT create_distributed_table('test_dist_colocated_with_non_colocated', 'x', colocate_with => 'test_dist_non_colocated');
 create_distributed_table
---------------------------------------------------------------------

(1 row)

SELECT create_reference_table('test_ref');
 create_reference_table
---------------------------------------------------------------------

(1 row)

-- colocated tables should still be placed on shouldhaveshards false nodes for safety
SELECT nodeport, count(*)
FROM pg_dist_shard JOIN pg_dist_shard_placement USING (shardid)
WHERE logicalrelid = 'test_dist_colocated'::regclass GROUP BY nodeport ORDER BY nodeport;
 nodeport | count
---------------------------------------------------------------------
    57637 |     2
    57638 |     2
(2 rows)

-- non colocated tables should not be placed on shouldhaveshards false nodes anymore
SELECT nodeport, count(*)
FROM pg_dist_shard JOIN pg_dist_shard_placement USING (shardid)
WHERE logicalrelid = 'test_dist_non_colocated'::regclass GROUP BY nodeport ORDER BY nodeport;
 nodeport | count
---------------------------------------------------------------------
    57637 |     4
(1 row)

-- this table should be colocated with the test_dist_non_colocated table
-- correctly only on nodes with shouldhaveshards true
SELECT nodeport, count(*)
FROM pg_dist_shard JOIN pg_dist_shard_placement USING (shardid)
WHERE logicalrelid = 'test_dist_colocated_with_non_colocated'::regclass GROUP BY nodeport ORDER BY nodeport;
 nodeport | count
---------------------------------------------------------------------
    57637 |     4
(1 row)

-- reference tables should be placed on with shouldhaveshards false
SELECT nodeport, count(*)
FROM pg_dist_shard JOIN pg_dist_shard_placement USING (shardid)
WHERE logicalrelid = 'test_ref'::regclass GROUP BY nodeport ORDER BY nodeport;
 nodeport | count
---------------------------------------------------------------------
    57636 |     1
    57637 |     1
    57638 |     1
(3 rows)

-- cleanup for next test
DROP TABLE test_dist, test_ref, test_dist_colocated, test_dist_non_colocated, test_dist_colocated_with_non_colocated;
-- testing behaviour when setting shouldhaveshards to false on fully empty node
SELECT * from master_set_node_property('localhost', :worker_2_port, 'shouldhaveshards', false);
 master_set_node_property
---------------------------------------------------------------------

(1 row)

CREATE TABLE test_dist (x int, y int);
CREATE TABLE test_dist_colocated (x int, y int);
CREATE TABLE test_dist_non_colocated (x int, y int);
CREATE TABLE test_ref (a int, b int);
SELECT create_distributed_table('test_dist', 'x');
 create_distributed_table
---------------------------------------------------------------------

(1 row)

SELECT create_reference_table('test_ref');
 create_reference_table
---------------------------------------------------------------------

(1 row)

-- distributed tables should not be placed on nodes with shouldhaveshards false
SELECT nodeport, count(*)
FROM pg_dist_shard JOIN pg_dist_shard_placement USING (shardid)
WHERE logicalrelid = 'test_dist'::regclass GROUP BY nodeport ORDER BY nodeport;
 nodeport | count
---------------------------------------------------------------------
    57637 |     4
(1 row)

-- reference tables should be placed on nodes with shouldhaveshards false
SELECT nodeport, count(*)
FROM pg_dist_shard JOIN pg_dist_shard_placement USING (shardid)
WHERE logicalrelid = 'test_ref'::regclass GROUP BY nodeport ORDER BY nodeport;
 nodeport | count
---------------------------------------------------------------------
    57636 |     1
    57637 |     1
    57638 |     1
(3 rows)

SELECT * from master_set_node_property('localhost', :worker_2_port, 'shouldhaveshards', true);
 master_set_node_property
---------------------------------------------------------------------

(1 row)

-- distributed tables should still not be placed on nodes that were switched to
-- shouldhaveshards true
SELECT nodeport, count(*)
FROM pg_dist_shard JOIN pg_dist_shard_placement USING (shardid)
WHERE logicalrelid = 'test_dist'::regclass GROUP BY nodeport ORDER BY nodeport;
 nodeport | count
---------------------------------------------------------------------
    57637 |     4
(1 row)

-- reference tables should still be placed on all nodes with isdatanode 'true'
SELECT nodeport, count(*)
FROM pg_dist_shard JOIN pg_dist_shard_placement USING (shardid)
WHERE logicalrelid = 'test_ref'::regclass GROUP BY nodeport ORDER BY nodeport;
 nodeport | count
---------------------------------------------------------------------
    57636 |     1
    57637 |     1
    57638 |     1
(3 rows)

SELECT create_distributed_table('test_dist_colocated', 'x');
 create_distributed_table
---------------------------------------------------------------------

(1 row)

SELECT create_distributed_table('test_dist_non_colocated', 'x', colocate_with => 'none');
 create_distributed_table
---------------------------------------------------------------------

(1 row)

-- colocated tables should not be placed on nodedes that were switched to
-- shouldhaveshards true
SELECT nodeport, count(*)
FROM pg_dist_shard JOIN pg_dist_shard_placement USING (shardid)
WHERE logicalrelid = 'test_dist_colocated'::regclass GROUP BY nodeport ORDER BY nodeport;
 nodeport | count
---------------------------------------------------------------------
    57637 |     4
(1 row)

-- non colocated tables should be placed on nodedes that were switched to
-- shouldhaveshards true
SELECT nodeport, count(*)
FROM pg_dist_shard JOIN pg_dist_shard_placement USING (shardid)
WHERE logicalrelid = 'test_dist_non_colocated'::regclass GROUP BY nodeport ORDER BY nodeport;
 nodeport | count
---------------------------------------------------------------------
    57637 |     2
    57638 |     2
(2 rows)

SELECT * from master_set_node_property('localhost', :worker_2_port, 'bogusproperty', false);
ERROR:  only the 'shouldhaveshards' property can be set using this function
SELECT nextval('pg_catalog.pg_dist_groupid_seq') AS last_group_id_cls \gset
SELECT nextval('pg_catalog.pg_dist_node_nodeid_seq') AS last_node_id_cls \gset
BEGIN;
	-- show that we do not send any metadata to any nodes if not enabled
	SET LOCAL citus.log_remote_commands TO ON;
	SET LOCAL citus.grep_remote_commands TO '%pg_dist%';
	SET citus.enable_metadata_sync TO OFF;
	SELECT start_metadata_sync_to_all_nodes();
 start_metadata_sync_to_all_nodes
---------------------------------------------------------------------
 t
(1 row)

	DROP TABLE test_dist, test_ref, test_dist_colocated, test_dist_non_colocated;
	SELECT 1 FROM citus_remove_node('localhost', :worker_1_port);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

	SELECT 1 FROM citus_remove_node('localhost', :worker_2_port);
 ?column?
---------------------------------------------------------------------
        1
(1 row)

	SELECT 1 FROM citus_add_node('localhost', :worker_1_port);
NOTICE:  issuing SELECT metadata ->> 'server_id' AS server_id FROM pg_dist_node_metadata
DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 ?column?
---------------------------------------------------------------------
        1
(1 row)

	SELECT 1 FROM citus_add_node('localhost', :worker_2_port);
NOTICE:  issuing SELECT metadata ->> 'server_id' AS server_id FROM pg_dist_node_metadata
DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 ?column?
---------------------------------------------------------------------
        1
(1 row)

ROLLBACK;
-- keep the rest of the tests inact that depends node/group ids
ALTER SEQUENCE pg_catalog.pg_dist_groupid_seq RESTART :last_group_id_cls;
ALTER SEQUENCE pg_catalog.pg_dist_node_nodeid_seq RESTART :last_node_id_cls;
DROP TABLE test_dist, test_ref, test_dist_colocated, test_dist_non_colocated;
BEGIN;
	SELECT start_metadata_sync_to_all_nodes();
 start_metadata_sync_to_all_nodes
---------------------------------------------------------------------
 t
(1 row)

COMMIT;
SELECT start_metadata_sync_to_all_nodes();
 start_metadata_sync_to_all_nodes
---------------------------------------------------------------------
 t
(1 row)

-- nontransactional sync mode tests
SET citus.metadata_sync_mode TO 'nontransactional';
-- do not allow nontransactional sync inside transaction block
BEGIN;
	SELECT start_metadata_sync_to_all_nodes();
ERROR:  do not sync metadata in transaction block when the sync mode is nontransactional
HINT:  resync after SET citus.metadata_sync_mode TO 'transactional'
COMMIT;
SELECT start_metadata_sync_to_all_nodes();
 start_metadata_sync_to_all_nodes
---------------------------------------------------------------------
 t
(1 row)

-- do not allow nontransactional node addition inside transaction block
BEGIN;
	SELECT citus_remove_node('localhost', :worker_1_port);
 citus_remove_node
---------------------------------------------------------------------

(1 row)

	SELECT citus_add_node('localhost', :worker_1_port);
ERROR:  do not add node in transaction block when the sync mode is nontransactional
HINT:  add the node after SET citus.metadata_sync_mode TO 'transactional'
COMMIT;
RESET citus.metadata_sync_mode;
-- verify that at the end of this file, all primary nodes have metadata synced
SELECT bool_and(hasmetadata) AND bool_and(metadatasynced) FROM pg_dist_node WHERE isactive = 't' and noderole = 'primary';
 ?column?
---------------------------------------------------------------------
 t
(1 row)

-- Grant all on public schema to public
--
-- That's the default on Postgres versions < 15 and we want to
-- keep permissions compatible accross versions, in regression
-- tests.
GRANT ALL ON SCHEMA public TO PUBLIC;
