Adds datacl propagation

src/backend/distributed/commands/common.c

@@ -14,10 +14,8 @@
 #include "postgres.h"
 
 #include "catalog/objectaddress.h"
-#include "catalog/pg_database.h"
 #include "catalog/pg_ts_config.h"
 #include "catalog/pg_ts_dict.h"
-#include "commands/dbcommands.h"
 #include "nodes/parsenodes.h"
 #include "tcop/utility.h"
 

src/backend/distributed/commands/database.c

@@ -658,6 +658,50 @@ GenerateCreateDatabaseStatementFromPgDatabase(Form_pg_database databaseForm)
 	return str.data;
 }
 
+/*
+ * GrantOnDatabaseDDLCommands returns a list of sql statements to idempotently apply a
+ * GRANT on distributed databases.
+ */
+
+List * GenerateGrantDatabaseCommandList(void){
+	List *grantCommands = NIL;
+
+	Relation pgDatabaseRel = table_open(DatabaseRelationId, AccessShareLock);
+	TableScanDesc scan = table_beginscan_catalog(pgDatabaseRel, 0, NULL);
+
+	HeapTuple tuple = NULL;
+	while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL)
+	{
+		Form_pg_database databaseForm = (Form_pg_database) GETSTRUCT(tuple);
+
+		ObjectAddress *dbAddress = GetDatabaseAddressFromDatabaseName(
+			NameStr(databaseForm->datname), false);
+
+		/* skip databases that are not distributed */
+		if (!IsAnyObjectDistributed(list_make1(dbAddress)))
+		{
+			continue;
+		}
+
+		List *dbGrants =  GrantOnDatabaseDDLCommands(databaseForm->oid);
+
+		/* append dbGrants into grantCommands*/
+		grantCommands = list_concat(grantCommands, dbGrants);
+	}
+
+	char *grantCommand = NULL;
+
+	foreach_ptr(grantCommand, grantCommands)
+	{
+		elog(DEBUG1, "grantCommand: %s", grantCommand);
+	}
+
+	heap_endscan(scan);
+	table_close(pgDatabaseRel, AccessShareLock);
+
+	return grantCommands;
+}
+
 
 /*
  * GenerateCreateDatabaseCommandList returns a list of CREATE DATABASE statements
@@ -666,8 +710,7 @@ GenerateCreateDatabaseStatementFromPgDatabase(Form_pg_database databaseForm)
  * Commands in the list are wrapped by citus_internal_database_command() UDF
  * to avoid from transaction block restrictions that apply to database commands
  */
-List *
+List * GenerateCreateDatabaseCommandList(void)
-GenerateCreateDatabaseCommandList(void)
 {
 	List *commands = NIL;
 

src/backend/distributed/commands/dependencies.c

@@ -465,6 +465,13 @@ GetDependencyCreateDDLCommands(const ObjectAddress *dependency)
 				List *ownerDDLCommands = DatabaseOwnerDDLCommands(dependency);
 				databaseDDLCommands = list_concat(databaseDDLCommands, ownerDDLCommands);
 			}
+			//TODO: To reviewer: Having a code block for dependency makes sense
+			// However dependency tree is based on pg metadata; which does not reflect
+			// actual database dependencies. I added this block just to point out the issue.
+			// if(EnableCreateDatabasePropagation){
+			// 	List *dbGrants =  GrantOnDatabaseDDLCommands(dependency->objectId);
+			// 	databaseDDLCommands = list_concat(databaseDDLCommands, dbGrants);
+			// }
 
 			return databaseDDLCommands;
 		}

src/backend/distributed/commands/role.c

@@ -513,6 +513,8 @@ GenerateRoleOptionsList(HeapTuple tuple)
 List *
 GenerateCreateOrAlterRoleCommand(Oid roleOid)
 {
+	elog(LOG,"GenerateCreateOrAlterRoleCommand execution");
+
 	HeapTuple roleTuple = SearchSysCache1(AUTHOID, ObjectIdGetDatum(roleOid));
 	Form_pg_authid role = ((Form_pg_authid) GETSTRUCT(roleTuple));
 

src/backend/distributed/metadata/metadata_sync.c

@@ -123,6 +123,7 @@ static List * GetObjectsForGrantStmt(ObjectType objectType, Oid objectId);
 static AccessPriv * GetAccessPrivObjectForGrantStmt(char *permission);
 static List * GenerateGrantOnSchemaQueriesFromAclItem(Oid schemaOid,
 													  AclItem *aclItem);
+static List * GenerateGrantOnDatabaseFromAclItem(Oid databaseOid, AclItem *aclItem);
 static List * GenerateGrantOnFunctionQueriesFromAclItem(Oid schemaOid,
 														AclItem *aclItem);
 static List * GrantOnSequenceDDLCommands(Oid sequenceOid);
@@ -154,6 +155,7 @@ static char * RemoteSchemaIdExpressionByName(char *schemaName);
 static char * RemoteTypeIdExpression(Oid typeId);
 static char * RemoteCollationIdExpression(Oid colocationId);
 static char * RemoteTableIdExpression(Oid relationId);
+static void SendDatabaseGrantSyncCommands(MetadataSyncContext *context);
 
 
 PG_FUNCTION_INFO_V1(start_metadata_sync_to_all_nodes);
@@ -2046,6 +2048,80 @@ GenerateGrantOnSchemaQueriesFromAclItem(Oid schemaOid, AclItem *aclItem)
 	return queries;
 }
 
+List *
+GrantOnDatabaseDDLCommands(Oid databaseOid)
+{
+	HeapTuple databaseTuple = SearchSysCache1(DATABASEOID, ObjectIdGetDatum(databaseOid));
+	bool isNull = true;
+	Datum aclDatum = SysCacheGetAttr(DATABASEOID, databaseTuple, Anum_pg_database_datacl,
+									 &isNull);
+	if (isNull)
+	{
+		ReleaseSysCache(databaseTuple);
+		return NIL;
+	}
+	Acl *acl = DatumGetAclPCopy(aclDatum);
+	AclItem *aclDat = ACL_DAT(acl);
+	int aclNum = ACL_NUM(acl);
+	List *commands = NIL;
+
+	ReleaseSysCache(databaseTuple);
+
+	for (int i = 0; i < aclNum; i++)
+	{
+		commands = list_concat(commands,
+							   GenerateGrantOnDatabaseFromAclItem(
+								   databaseOid,&aclDat[i]));
+	}
+
+	return commands;
+}
+
+
+List *
+GenerateGrantOnDatabaseFromAclItem(Oid databaseOid, AclItem *aclItem)
+{
+	AclMode permissions = ACLITEM_GET_PRIVS(*aclItem) & ACL_ALL_RIGHTS_DATABASE;
+	AclMode grants = ACLITEM_GET_GOPTIONS(*aclItem) & ACL_ALL_RIGHTS_DATABASE;
+
+	/*
+	 * seems unlikely but we check if there is a grant option in the list without the actual permission
+	 */
+	Assert(!(grants & ACL_CONNECT) || (permissions & ACL_CONNECT));
+	Assert(!(grants & ACL_CREATE) || (permissions & ACL_CREATE));
+	Assert(!(grants & ACL_CREATE_TEMP) || (permissions & ACL_CREATE_TEMP));
+	Oid granteeOid = aclItem->ai_grantee;
+	List *queries = NIL;
+
+	queries = lappend(queries, GenerateSetRoleQuery(aclItem->ai_grantor));
+
+	if (permissions & ACL_CONNECT)
+	{
+		char *query = DeparseTreeNode((Node *) GenerateGrantStmtForRights(
+										  OBJECT_DATABASE ,granteeOid, databaseOid, "CONNECT",
+										  grants & ACL_CONNECT));
+		queries = lappend(queries, query);
+	}
+	if (permissions & ACL_CREATE)
+	{
+		char *query = DeparseTreeNode((Node *) GenerateGrantStmtForRights(
+										  OBJECT_DATABASE, granteeOid, databaseOid, "CREATE",
+										  grants & ACL_CREATE));
+		queries = lappend(queries, query);
+	}
+	if (permissions & ACL_CREATE_TEMP)
+	{
+		char *query = DeparseTreeNode((Node *) GenerateGrantStmtForRights(
+										  OBJECT_DATABASE, granteeOid, databaseOid, "TEMPORARY",
+										  grants & ACL_CREATE_TEMP));
+		queries = lappend(queries, query);
+	}
+
+	queries = lappend(queries, "RESET ROLE");
+
+	return queries;
+}
+
 
 /*
  * GenerateGrantStmtForRights is the function for creating GrantStmt's for all
@@ -2120,6 +2196,11 @@ GetObjectsForGrantStmt(ObjectType objectType, Oid objectId)
 			return list_make1(sequence);
 		}
 
+		case OBJECT_DATABASE:
+		{
+			return list_make1(makeString(get_database_name(objectId)));
+		}
+
 		default:
 		{
 			elog(ERROR, "unsupported object type for GRANT");
@@ -4563,13 +4644,6 @@ PropagateNodeWideObjectsCommandList(void)
 	/* collect all commands */
 	List *ddlCommands = NIL;
 
-	if (EnableCreateDatabasePropagation)
-	{
-		/* get commands for database creation */
-		List *createDatabaseCommands = GenerateCreateDatabaseCommandList();
-		ddlCommands = list_concat(ddlCommands, createDatabaseCommands);
-	}
-
 	if (EnableAlterRoleSetPropagation)
 	{
 		/*
@@ -4580,6 +4654,13 @@ PropagateNodeWideObjectsCommandList(void)
 		ddlCommands = list_concat(ddlCommands, alterRoleSetCommands);
 	}
 
+	if (EnableCreateDatabasePropagation)
+	{
+		/* get commands for database creation */
+		List *createDatabaseCommands = GenerateCreateDatabaseCommandList();
+		ddlCommands = list_concat(ddlCommands, createDatabaseCommands);
+	}
+
 	return ddlCommands;
 }
 
@@ -4611,7 +4692,7 @@ SyncDistributedObjects(MetadataSyncContext *context)
 
 	Assert(ShouldPropagate());
 
-	/* Send systemwide objects, only roles for now */
+	/* send systemwide objects; i.e. roles and databases for now */
 	SendNodeWideObjectsSyncCommands(context);
 
 	/*
@@ -4651,6 +4732,12 @@ SyncDistributedObjects(MetadataSyncContext *context)
 	 * those tables.
 	 */
 	SendInterTableRelationshipCommands(context);
+
+	/*
+	 * After creation of databases and roles, send the grant database commands
+	 * to the workers.
+	*/
+	SendDatabaseGrantSyncCommands(context);
 }
 
 
@@ -4675,6 +4762,29 @@ SendNodeWideObjectsSyncCommands(MetadataSyncContext *context)
 	SendOrCollectCommandListToActivatedNodes(context, commandList);
 }
 
+/*
+ * SendDatabaseGrantSyncCommands sends database grants to roles to workers with
+ * transactional or nontransactional mode according to transactionMode inside
+ * metadataSyncContext.
+ * This function is called after SendNodeWideObjectsSyncCommands and SendDependencyCreationCommands
+ * because we need both databases and roles to be created on the worker.
+ */
+static void
+SendDatabaseGrantSyncCommands(MetadataSyncContext *context)
+{
+	/* propagate node wide objects. It includes only roles for now. */
+	List *commandList = GenerateGrantDatabaseCommandList();
+
+	if (commandList == NIL)
+	{
+		return;
+	}
+
+	commandList = lcons(DISABLE_DDL_PROPAGATION, commandList);
+	commandList = lappend(commandList, ENABLE_DDL_PROPAGATION);
+	SendOrCollectCommandListToActivatedNodes(context, commandList);
+}
+
 
 /*
  * SendShellTableDeletionCommands sends sequence, and shell table deletion

src/include/distributed/commands.h

@@ -244,6 +244,7 @@ extern List * DropDatabaseStmtObjectAddress(Node *node, bool missingOk,
 extern List * CreateDatabaseStmtObjectAddress(Node *node, bool missingOk,
 											  bool isPostprocess);
 extern List * GenerateCreateDatabaseCommandList(void);
+extern List * GenerateGrantDatabaseCommandList(void);
 
 
 /* domain.c - forward declarations */

src/include/distributed/metadata_sync.h

@@ -107,6 +107,7 @@ extern char * ColocationIdUpdateCommand(Oid relationId, uint32 colocationId);
 extern char * CreateSchemaDDLCommand(Oid schemaId);
 extern List * GrantOnSchemaDDLCommands(Oid schemaId);
 extern List * GrantOnFunctionDDLCommands(Oid functionOid);
+extern List * GrantOnDatabaseDDLCommands(Oid databaseOid);
 extern List * GrantOnForeignServerDDLCommands(Oid serverId);
 extern List * GenerateGrantOnForeignServerQueriesFromAclItem(Oid serverId,
 															 AclItem *aclItem);

src/test/regress/expected/create_drop_database_propagation.out

@@ -487,6 +487,7 @@ DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 NOTICE:  issuing DROP DATABASE  db_force_test WITH ( FORCE )
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 reset citus.log_remote_commands;
+reset citus.grep_remote_commands;
 SELECT * FROM public.check_database_on_all_nodes('db_force_test') ORDER BY node_type;
       node_type       |                                                              result
 ---------------------------------------------------------------------
@@ -537,6 +538,186 @@ SELECT * FROM public.check_database_on_all_nodes('distributed_db') ORDER BY node
 drop database distributed_db;
 set citus.enable_create_database_propagation TO off;
 drop database non_distributed_db;
+-- test role grants on DATABASE in metadata sync
+SET citus.enable_create_database_propagation TO on;
+CREATE ROLE db_role_grants_test_role_exists_on_node_2;
+select 1 from citus_remove_node('localhost', :worker_2_port);
+ ?column?
+---------------------------------------------------------------------
+        1
+(1 row)
+
+CREATE DATABASE db_role_grants_test;
+revoke connect,temp,temporary  on database db_role_grants_test from public;
+SET citus.log_remote_commands = true;
+set citus.grep_remote_commands = '%CREATE ROLE%';
+CREATE ROLE db_role_grants_test_role_missing_on_node_2;
+NOTICE:  issuing SELECT worker_create_or_alter_role('db_role_grants_test_role_missing_on_node_2', 'CREATE ROLE db_role_grants_test_role_missing_on_node_2', 'ALTER ROLE db_role_grants_test_role_missing_on_node_2')
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+RESET citus.log_remote_commands ;
+RESET citus.grep_remote_commands;
+-- check the privileges before grant
+SELECT result from run_command_on_all_nodes(
+  $$
+  select has_database_privilege('db_role_grants_test_role_exists_on_node_2','db_role_grants_test', 'CREATE')
+  $$
+) ORDER BY result;
+ result
+---------------------------------------------------------------------
+ f
+ f
+(2 rows)
+
+SELECT result from run_command_on_all_nodes(
+  $$
+  select has_database_privilege('db_role_grants_test_role_exists_on_node_2','db_role_grants_test', 'TEMPORARY')
+  $$
+) ORDER BY result;
+ result
+---------------------------------------------------------------------
+ f
+ f
+(2 rows)
+
+SELECT result from run_command_on_all_nodes(
+  $$
+  select has_database_privilege('db_role_grants_test_role_exists_on_node_2','db_role_grants_test', 'CONNECT')
+  $$
+) ORDER BY result;
+ result
+---------------------------------------------------------------------
+ f
+ f
+(2 rows)
+
+SELECT result from run_command_on_all_nodes(
+  $$
+  select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test', 'CREATE')
+  $$
+) ORDER BY result;
+ result
+---------------------------------------------------------------------
+ f
+ f
+(2 rows)
+
+SELECT result from run_command_on_all_nodes(
+  $$
+  select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test', 'TEMPORARY')
+  $$
+) ORDER BY result;
+ result
+---------------------------------------------------------------------
+ f
+ f
+(2 rows)
+
+SELECT result from run_command_on_all_nodes(
+  $$
+  select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test', 'CONNECT')
+  $$
+) ORDER BY result;
+ result
+---------------------------------------------------------------------
+ f
+ f
+(2 rows)
+
+SET citus.log_remote_commands = true;
+set citus.grep_remote_commands = '%GRANT%';
+grant CONNECT,TEMPORARY,CREATE on DATABASE db_role_grants_test to db_role_grants_test_role_exists_on_node_2;
+NOTICE:  issuing GRANT connect, temporary, create ON DATABASE db_role_grants_test TO db_role_grants_test_role_exists_on_node_2;
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+grant CONNECT,TEMPORARY,CREATE  on DATABASE db_role_grants_test to db_role_grants_test_role_missing_on_node_2;
+NOTICE:  issuing GRANT connect, temporary, create ON DATABASE db_role_grants_test TO db_role_grants_test_role_missing_on_node_2;
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+select 1 from citus_add_node('localhost', :worker_2_port);
+NOTICE:  issuing CREATE SCHEMA IF NOT EXISTS public AUTHORIZATION pg_database_owner;SET ROLE pg_database_owner;GRANT USAGE ON SCHEMA public TO pg_database_owner;;GRANT CREATE ON SCHEMA public TO pg_database_owner;;RESET ROLE;SET ROLE pg_database_owner;GRANT USAGE ON SCHEMA public TO PUBLIC;;RESET ROLE
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing CREATE SCHEMA IF NOT EXISTS information_schema AUTHORIZATION postgres;SET ROLE postgres;GRANT USAGE ON SCHEMA information_schema TO postgres;;GRANT CREATE ON SCHEMA information_schema TO postgres;;RESET ROLE;SET ROLE postgres;GRANT USAGE ON SCHEMA information_schema TO PUBLIC;;RESET ROLE
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+NOTICE:  issuing SET citus.enable_ddl_propagation TO 'off';SET ROLE postgres;GRANT CONNECT ON DATABASE db_role_grants_test TO postgres;;GRANT CREATE ON DATABASE db_role_grants_test TO postgres;;GRANT TEMPORARY ON DATABASE db_role_grants_test TO postgres;;RESET ROLE;SET ROLE postgres;GRANT CONNECT ON DATABASE db_role_grants_test TO db_role_grants_test_role_exists_on_node_2;;GRANT CREATE ON DATABASE db_role_grants_test TO db_role_grants_test_role_exists_on_node_2;;GRANT TEMPORARY ON DATABASE db_role_grants_test TO db_role_grants_test_role_exists_on_node_2;;RESET ROLE;SET ROLE postgres;GRANT CONNECT ON DATABASE db_role_grants_test TO db_role_grants_test_role_missing_on_node_2;;GRANT CREATE ON DATABASE db_role_grants_test TO db_role_grants_test_role_missing_on_node_2;;GRANT TEMPORARY ON DATABASE db_role_grants_test TO db_role_grants_test_role_missing_on_node_2;;RESET ROLE;SET citus.enable_ddl_propagation TO 'on'
+DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
+ ?column?
+---------------------------------------------------------------------
+        1
+(1 row)
+
+SELECT result from run_command_on_all_nodes(
+  $$
+  select has_database_privilege('db_role_grants_test_role_exists_on_node_2','db_role_grants_test', 'CREATE')
+  $$
+) ORDER BY result;
+ result
+---------------------------------------------------------------------
+ t
+ t
+ t
+(3 rows)
+
+SELECT result from run_command_on_all_nodes(
+  $$
+  select has_database_privilege('db_role_grants_test_role_exists_on_node_2','db_role_grants_test', 'TEMPORARY')
+  $$
+) ORDER BY result;
+ result
+---------------------------------------------------------------------
+ t
+ t
+ t
+(3 rows)
+
+SELECT result from run_command_on_all_nodes(
+  $$
+  select has_database_privilege('db_role_grants_test_role_exists_on_node_2','db_role_grants_test', 'CONNECT')
+  $$
+) ORDER BY result;
+ result
+---------------------------------------------------------------------
+ t
+ t
+ t
+(3 rows)
+
+SELECT result from run_command_on_all_nodes(
+  $$
+  select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test', 'CREATE')
+  $$
+) ORDER BY result;
+ result
+---------------------------------------------------------------------
+ t
+ t
+ t
+(3 rows)
+
+SELECT result from run_command_on_all_nodes(
+  $$
+  select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test', 'TEMPORARY')
+  $$
+) ORDER BY result;
+ result
+---------------------------------------------------------------------
+ t
+ t
+ t
+(3 rows)
+
+SELECT result from run_command_on_all_nodes(
+  $$
+  select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test', 'CONNECT')
+  $$
+) ORDER BY result;
+ result
+---------------------------------------------------------------------
+ t
+ t
+ t
+(3 rows)
+
+DROP DATABASE db_role_grants_test;
+DROP ROLE db_role_grants_test_role_exists_on_node_2;
+DROP ROLE db_role_grants_test_role_missing_on_node_2;
 --clean up resources created by this test
 -- DROP TABLESPACE is not supported, so we need to drop it manually.
 SELECT result FROM run_command_on_all_nodes(

src/test/regress/sql/create_drop_database_propagation.sql

@@ -271,6 +271,7 @@ set citus.grep_remote_commands = '%DROP DATABASE%';
 drop database db_force_test with (force);
 
 reset citus.log_remote_commands;
+reset citus.grep_remote_commands;
 
 SELECT * FROM public.check_database_on_all_nodes('db_force_test') ORDER BY node_type;
 
@@ -296,6 +297,121 @@ set citus.enable_create_database_propagation TO off;
 drop database non_distributed_db;
 
 
+
+-- test role grants on DATABASE in metadata sync
+
+
+
+SET citus.enable_create_database_propagation TO on;
+
+
+
+CREATE ROLE db_role_grants_test_role_exists_on_node_2;
+
+
+select 1 from citus_remove_node('localhost', :worker_2_port);
+
+CREATE DATABASE db_role_grants_test;
+
+revoke connect,temp,temporary  on database db_role_grants_test from public;
+
+SET citus.log_remote_commands = true;
+set citus.grep_remote_commands = '%CREATE ROLE%';
+CREATE ROLE db_role_grants_test_role_missing_on_node_2;
+
+RESET citus.log_remote_commands ;
+RESET citus.grep_remote_commands;
+
+-- check the privileges before grant
+
+SELECT result from run_command_on_all_nodes(
+  $$
+  select has_database_privilege('db_role_grants_test_role_exists_on_node_2','db_role_grants_test', 'CREATE')
+  $$
+) ORDER BY result;
+
+SELECT result from run_command_on_all_nodes(
+  $$
+  select has_database_privilege('db_role_grants_test_role_exists_on_node_2','db_role_grants_test', 'TEMPORARY')
+  $$
+) ORDER BY result;
+
+SELECT result from run_command_on_all_nodes(
+  $$
+  select has_database_privilege('db_role_grants_test_role_exists_on_node_2','db_role_grants_test', 'CONNECT')
+  $$
+) ORDER BY result;
+
+SELECT result from run_command_on_all_nodes(
+  $$
+  select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test', 'CREATE')
+  $$
+) ORDER BY result;
+
+
+SELECT result from run_command_on_all_nodes(
+  $$
+  select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test', 'TEMPORARY')
+  $$
+) ORDER BY result;
+
+SELECT result from run_command_on_all_nodes(
+  $$
+  select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test', 'CONNECT')
+  $$
+) ORDER BY result;
+
+SET citus.log_remote_commands = true;
+set citus.grep_remote_commands = '%GRANT%';
+grant CONNECT,TEMPORARY,CREATE on DATABASE db_role_grants_test to db_role_grants_test_role_exists_on_node_2;
+grant CONNECT,TEMPORARY,CREATE  on DATABASE db_role_grants_test to db_role_grants_test_role_missing_on_node_2;
+
+select 1 from citus_add_node('localhost', :worker_2_port);
+
+
+SELECT result from run_command_on_all_nodes(
+  $$
+  select has_database_privilege('db_role_grants_test_role_exists_on_node_2','db_role_grants_test', 'CREATE')
+  $$
+) ORDER BY result;
+
+SELECT result from run_command_on_all_nodes(
+  $$
+  select has_database_privilege('db_role_grants_test_role_exists_on_node_2','db_role_grants_test', 'TEMPORARY')
+  $$
+) ORDER BY result;
+
+SELECT result from run_command_on_all_nodes(
+  $$
+  select has_database_privilege('db_role_grants_test_role_exists_on_node_2','db_role_grants_test', 'CONNECT')
+  $$
+) ORDER BY result;
+
+SELECT result from run_command_on_all_nodes(
+  $$
+  select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test', 'CREATE')
+  $$
+) ORDER BY result;
+
+
+SELECT result from run_command_on_all_nodes(
+  $$
+  select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test', 'TEMPORARY')
+  $$
+) ORDER BY result;
+
+SELECT result from run_command_on_all_nodes(
+  $$
+  select has_database_privilege('db_role_grants_test_role_missing_on_node_2','db_role_grants_test', 'CONNECT')
+  $$
+) ORDER BY result;
+
+
+DROP DATABASE db_role_grants_test;
+DROP ROLE db_role_grants_test_role_exists_on_node_2;
+DROP ROLE db_role_grants_test_role_missing_on_node_2;
+
+
 --clean up resources created by this test
 
 -- DROP TABLESPACE is not supported, so we need to drop it manually.