Fix dependent tests

src/test/regress/expected/multi_metadata_sync.out

@@ -1324,7 +1324,8 @@ NOTICE:  Replicating reference table "mx_ref" to all workers
 
 SELECT shardid, nodename, nodeport 
 FROM pg_dist_shard NATURAL JOIN pg_dist_shard_placement
-WHERE logicalrelid='mx_ref'::regclass;
+WHERE logicalrelid='mx_ref'::regclass
+ORDER BY shardid, nodeport;
  shardid | nodename  | nodeport 
 ---------+-----------+----------
  1310184 | localhost |    57637
@@ -1334,7 +1335,8 @@ WHERE logicalrelid='mx_ref'::regclass;
 \c - - - :worker_1_port
 SELECT shardid, nodename, nodeport 
 FROM pg_dist_shard NATURAL JOIN pg_dist_shard_placement
-WHERE logicalrelid='mx_ref'::regclass;
+WHERE logicalrelid='mx_ref'::regclass
+ORDER BY shardid, nodeport;
  shardid | nodename  | nodeport 
 ---------+-----------+----------
  1310184 | localhost |    57637

src/test/regress/expected/multi_modifying_xacts.out

@@ -1176,4 +1176,12 @@ ORDER BY s.logicalrelid, sp.shardstate;
 ALTER USER test_user_new RENAME TO test_user;
 -- connect back to the master with the proper user to continue the tests 
 \c - :default_user - :master_port
-DROP TABLE reference_modifying_xacts, hash_modifying_xacts, hash_modifying_xacts_second;
+DROP TABLE reference_modifying_xacts, hash_modifying_xacts, hash_modifying_xacts_second, reference_failure_test;
+SELECT * FROM run_command_on_workers('DROP USER test_user');
+ nodename  | nodeport | success |  result   
+-----------+----------+---------+-----------
+ localhost |    57637 | t       | DROP ROLE
+ localhost |    57638 | t       | DROP ROLE
+(2 rows)
+
+DROP USER test_user;

src/test/regress/expected/multi_remove_node_reference_table.out

@@ -777,14 +777,15 @@ SELECT
 FROM
     pg_dist_shard_placement
 WHERE
-    nodeport = :worker_2_port;
+    nodeport = :worker_2_port
+ORDER BY
+	shardid;
  shardid | shardstate | shardlength | nodename  | nodeport 
 ---------+------------+-------------+-----------+----------
  1380001 |          1 |           0 | localhost |    57638
  1380002 |          1 |           0 | localhost |    57638
 (2 rows)
 
-    
 \c - - - :master_port
 SELECT master_remove_node('localhost', :worker_2_port);
  master_remove_node 

src/test/regress/input/multi_copy.source

@@ -605,7 +605,10 @@ SELECT shardid, nodename, nodeport
 DROP TABLE numbers_append;
 
 -- Test copy failures against connection failures
--- switch to a test user, it was previously created
+-- create and switch to test user
+CREATE USER test_user;
+SELECT * FROM run_command_on_workers('CREATE USER test_user');
+
 \c - test_user
 SET citus.shard_count to 4;
 CREATE TABLE numbers_hash (a int, b int);
@@ -700,11 +703,17 @@ SELECT shardid, shardstate, nodename, nodeport
 -- re-enable test_user on the first worker
 \c - :default_user - :worker_1_port
 ALTER USER test_user WITH login;
+
+-- there is a dangling shard in worker_2, drop it
+\c - test_user - :worker_2_port
+DROP TABLE numbers_hash_other_560176;
+
 \c - test_user - :master_port
 
 DROP TABLE numbers_hash;
 DROP TABLE numbers_hash_other;
 DROP TABLE numbers_reference;
+
 \c - :default_user
 
 -- test copy failure inside the node
@@ -738,4 +747,5 @@ SELECT shardid, shardstate, nodename, nodeport
 	WHERE logicalrelid = 'numbers_hash'::regclass order by placementid;
 
 DROP TABLE numbers_hash;
-
+SELECT * FROM run_command_on_workers('DROP USER test_user');
+DROP USER test_user;

src/test/regress/output/multi_copy.source

@@ -769,7 +769,6 @@ SELECT shardid, nodename, nodeport
 SELECT master_add_node('localhost', :worker_1_port);
 NOTICE:  Replicating reference table "nation" to all workers
 NOTICE:  Replicating reference table "supplier" to all workers
-NOTICE:  Replicating reference table "reference_failure_test" to all workers
          master_add_node         
 ---------------------------------
  (3,3,localhost,57637,default,f)
@@ -798,7 +797,17 @@ SELECT shardid, nodename, nodeport
 
 DROP TABLE numbers_append;
 -- Test copy failures against connection failures
--- switch to a test user, it was previously created
+-- create and switch to test user
+CREATE USER test_user;
+NOTICE:  not propagating CREATE ROLE/USER commands to worker nodes
+HINT:  Connect to worker nodes directly to manually create all necessary users and roles.
+SELECT * FROM run_command_on_workers('CREATE USER test_user');
+ nodename  | nodeport | success |   result    
+-----------+----------+---------+-------------
+ localhost |    57637 | t       | CREATE ROLE
+ localhost |    57638 | t       | CREATE ROLE
+(2 rows)
+
 \c - test_user
 SET citus.shard_count to 4;
 CREATE TABLE numbers_hash (a int, b int);
@@ -950,6 +959,9 @@ SELECT shardid, shardstate, nodename, nodeport
 -- re-enable test_user on the first worker
 \c - :default_user - :worker_1_port
 ALTER USER test_user WITH login;
+-- there is a dangling shard in worker_2, drop it
+\c - test_user - :worker_2_port
+DROP TABLE numbers_hash_other_560176;
 \c - test_user - :master_port
 DROP TABLE numbers_hash;
 DROP TABLE numbers_hash_other;
@@ -995,3 +1007,11 @@ SELECT shardid, shardstate, nodename, nodeport
 (8 rows)
 
 DROP TABLE numbers_hash;
+SELECT * FROM run_command_on_workers('DROP USER test_user');
+ nodename  | nodeport | success |  result   
+-----------+----------+---------+-----------
+ localhost |    57637 | t       | DROP ROLE
+ localhost |    57638 | t       | DROP ROLE
+(2 rows)
+
+DROP USER test_user;

src/test/regress/sql/multi_metadata_sync.sql

@@ -581,12 +581,14 @@ SELECT master_add_node('localhost', :worker_2_port);
 
 SELECT shardid, nodename, nodeport 
 FROM pg_dist_shard NATURAL JOIN pg_dist_shard_placement
-WHERE logicalrelid='mx_ref'::regclass;
+WHERE logicalrelid='mx_ref'::regclass
+ORDER BY shardid, nodeport;
 
 \c - - - :worker_1_port
 SELECT shardid, nodename, nodeport 
 FROM pg_dist_shard NATURAL JOIN pg_dist_shard_placement
-WHERE logicalrelid='mx_ref'::regclass;
+WHERE logicalrelid='mx_ref'::regclass
+ORDER BY shardid, nodeport;
 
 \c - - - :master_port
 INSERT INTO pg_dist_shard_placement (SELECT * FROM tmp_shard_placement);

src/test/regress/sql/multi_modifying_xacts.sql

@@ -869,4 +869,7 @@ ALTER USER test_user_new RENAME TO test_user;
 
 -- connect back to the master with the proper user to continue the tests 
 \c - :default_user - :master_port
-DROP TABLE reference_modifying_xacts, hash_modifying_xacts, hash_modifying_xacts_second;
+DROP TABLE reference_modifying_xacts, hash_modifying_xacts, hash_modifying_xacts_second, reference_failure_test;
+
+SELECT * FROM run_command_on_workers('DROP USER test_user');
+DROP USER test_user;

src/test/regress/sql/multi_remove_node_reference_table.sql

@@ -467,8 +467,9 @@ SELECT
 FROM
     pg_dist_shard_placement
 WHERE
-    nodeport = :worker_2_port;
-    
+    nodeport = :worker_2_port
+ORDER BY
+	shardid;
 \c - - - :master_port
 
 SELECT master_remove_node('localhost', :worker_2_port);