Merge pull request #4480 from citusdata/create-table-define-fkey

* Convert postgres tables to citus local tables for CREATE TABLE commands defining foreign keys
* Introduce citus.enable_local_reference_table_foreign_keys guc

src/backend/distributed/commands/alter_table.c

@@ -482,6 +482,10 @@ ConvertTable(TableConversionState *con)
 	if (con->conversionType == UNDISTRIBUTE_TABLE && con->cascadeViaForeignKeys &&
 		(TableReferencing(con->relationId) || TableReferenced(con->relationId)))
 	{
+		/*
+		 * Acquire ExclusiveLock as UndistributeTable does in order to
+		 * make sure that no modifications happen on the relations.
+		 */
 		CascadeOperationForConnectedRelations(con->relationId, ExclusiveLock,
 											  CASCADE_FKEY_UNDISTRIBUTE_TABLE);
 

src/backend/distributed/commands/cascade_table_operation_for_connected_relations.c

@@ -44,7 +44,8 @@ static char * GetDropFkeyCascadeCommand(Oid foreignKeyId);
 static void ExecuteCascadeOperationForRelationIdList(List *relationIdList,
 													 CascadeOperationType
 													 cascadeOperationType);
-
+static void ExecuteForeignKeyCreateCommand(const char *commandString,
+										   bool skip_validation);
 
 /*
  * CascadeOperationForConnectedRelations executes citus table function specified
@@ -106,7 +107,8 @@ CascadeOperationForConnectedRelations(Oid relationId, LOCKMODE lockMode,
 											 cascadeOperationType);
 
 	/* now recreate foreign keys on tables */
-	ExecuteAndLogDDLCommandList(fKeyCreationCommands);
+	bool skip_validation = true;
+	ExecuteForeignKeyCreateCommandList(fKeyCreationCommands, skip_validation);
 }
 
 
@@ -426,3 +428,55 @@ ExecuteAndLogDDLCommand(const char *commandString)
 	CitusProcessUtility(parseTree, commandString, PROCESS_UTILITY_TOPLEVEL,
 						NULL, None_Receiver, NULL);
 }
+
+
+/*
+ * ExecuteForeignKeyCreateCommandList takes a list of foreign key creation ddl commands
+ * and calls ExecuteAndLogForeignKeyCreateCommand function for each of them.
+ */
+void
+ExecuteForeignKeyCreateCommandList(List *ddlCommandList, bool skip_validation)
+{
+	char *ddlCommand = NULL;
+	foreach_ptr(ddlCommand, ddlCommandList)
+	{
+		ExecuteForeignKeyCreateCommand(ddlCommand, skip_validation);
+	}
+}
+
+
+/*
+ * ExecuteForeignKeyCreateCommand takes a foreign key creation command
+ * and logs it in DEBUG4 log level.
+ *
+ * Then, parses, sets skip_validation flag to considering the input and
+ * executes the command via CitusProcessUtility.
+ */
+static void
+ExecuteForeignKeyCreateCommand(const char *commandString, bool skip_validation)
+{
+	ereport(DEBUG4, (errmsg("executing foreign key create command \"%s\"",
+							commandString)));
+
+	Node *parseTree = ParseTreeNode(commandString);
+
+	/*
+	 * We might have thrown an error if IsA(parseTree, AlterTableStmt),
+	 * but that doesn't seem to provide any benefits, so assertion is
+	 * fine for this case.
+	 */
+	Assert(IsA(parseTree, AlterTableStmt));
+
+	if (skip_validation && IsA(parseTree, AlterTableStmt))
+	{
+		parseTree =
+			SkipForeignKeyValidationIfConstraintIsFkey((AlterTableStmt *) parseTree,
+													   true);
+
+		ereport(DEBUG4, (errmsg("skipping validation for foreign key create "
+								"command \"%s\"", commandString)));
+	}
+
+	CitusProcessUtility(parseTree, commandString, PROCESS_UTILITY_TOPLEVEL,
+						NULL, None_Receiver, NULL);
+}

src/backend/distributed/commands/create_citus_local_table.c

@@ -138,6 +138,10 @@ CreateCitusLocalTable(Oid relationId, bool cascadeViaForeignKeys)
 	bool tableHasExternalForeignKeys = TableHasExternalForeignKeys(relationId);
 	if (tableHasExternalForeignKeys && cascadeViaForeignKeys)
 	{
+		/*
+		 * By acquiring AccessExclusiveLock, make sure that no modifications happen
+		 * on the relations.
+		 */
 		CascadeOperationForConnectedRelations(relationId, lockMode,
 											  CASCADE_FKEY_CREATE_CITUS_LOCAL_TABLE);
 

src/backend/distributed/commands/create_distributed_table.c

@@ -466,8 +466,14 @@ CreateDistributedTable(Oid relationId, Var *distributionColumn, char distributio
 		}
 	}
 
-	/* now recreate foreign keys that we dropped beforehand */
+	/*
-	ExecuteAndLogDDLCommandList(fKeyCreationCommandsRelationInvolved);
+	 * Now recreate foreign keys that we dropped beforehand. As modifications are not
+	 * allowed on the relations that are involved in the foreign key relationship,
+	 * we can skip the validation of the foreign keys.
+	 */
+	bool skip_validation = true;
+	ExecuteForeignKeyCreateCommandList(fKeyCreationCommandsRelationInvolved,
+									   skip_validation);
 }
 
 

src/backend/distributed/commands/table.c

@@ -42,7 +42,13 @@
 #include "utils/syscache.h"
 
 
+/* controlled via GUC, should be accessed via GetEnableLocalReferenceForeignKeys() */
+bool EnableLocalReferenceForeignKeys = true;
+
+
 /* Local functions forward declarations for unsupported command checks */
+static void PostprocessCreateTableStmtForeignKeys(CreateStmt *createStatement);
+static bool ShouldEnableLocalReferenceForeignKeys(void);
 static void PostprocessCreateTableStmtPartitionOf(CreateStmt *createStatement,
 												  const char *queryString);
 static bool AlterTableDefinesFKeyBetweenPostgresAndNonDistTable(
@@ -83,6 +89,7 @@ static void SetInterShardDDLTaskRelationShardList(Task *task,
 												  ShardInterval *leftShardInterval,
 												  ShardInterval *rightShardInterval);
 
+
 /*
  * We need to run some of the commands sequentially if there is a foreign constraint
  * from/to reference table.
@@ -165,9 +172,9 @@ PreprocessDropTableStmt(Node *node, const char *queryString,
 
 
 /*
- * PostprocessCreateTableStmt takes CreateStmt object as a parameter and errors
+ * PostprocessCreateTableStmt takes CreateStmt object as a parameter and
- * out if it creates a table with a foreign key that references to a citus local
+ * processes foreign keys on relation via PostprocessCreateTableStmtForeignKeys
- * table.
+ * function.
  *
  * This function also processes CREATE TABLE ... PARTITION OF statements via
  * PostprocessCreateTableStmtPartitionOf function.
@@ -175,16 +182,7 @@ PreprocessDropTableStmt(Node *node, const char *queryString,
 void
 PostprocessCreateTableStmt(CreateStmt *createStatement, const char *queryString)
 {
-	/*
+	PostprocessCreateTableStmtForeignKeys(createStatement);
-	 * Relation must exist and it is already locked as standard process utility
-	 * is already executed.
-	 */
-	bool missingOk = false;
-	Oid relationId = RangeVarGetRelid(createStatement->relation, NoLock, missingOk);
-	if (HasForeignKeyToCitusLocalTable(relationId))
-	{
-		ErrorOutForFKeyBetweenPostgresAndCitusLocalTable(relationId);
-	}
 
 	if (createStatement->inhRelations != NIL && createStatement->partbound != NULL)
 	{
@@ -194,6 +192,77 @@ PostprocessCreateTableStmt(CreateStmt *createStatement, const char *queryString)
 }
 
 
+/*
+ * PostprocessCreateTableStmtForeignKeys drops ands re-defines foreign keys
+ * defined by given CREATE TABLE command if command defined any foreign to
+ * reference or citus local tables.
+ */
+static void
+PostprocessCreateTableStmtForeignKeys(CreateStmt *createStatement)
+{
+	if (!ShouldEnableLocalReferenceForeignKeys())
+	{
+		/*
+		 * Either the user disabled foreign keys from/to local/reference tables
+		 * or the coordinator is not in the metadata */
+		return;
+	}
+
+	/*
+	 * Relation must exist and it is already locked as standard process utility
+	 * is already executed.
+	 */
+	bool missingOk = false;
+	Oid relationId = RangeVarGetRelid(createStatement->relation, NoLock, missingOk);
+
+	/*
+	 * As we are just creating the table, we cannot have foreign keys that our
+	 * relation is referenced. So we use INCLUDE_REFERENCING_CONSTRAINTS here.
+	 * Reason behind using other two flags is explained below.
+	 */
+	int nonDistTableFKeysFlag = INCLUDE_REFERENCING_CONSTRAINTS |
+								INCLUDE_CITUS_LOCAL_TABLES |
+								INCLUDE_REFERENCE_TABLES;
+	List *nonDistTableForeignKeyIdList =
+		GetForeignKeyOids(relationId, nonDistTableFKeysFlag);
+	bool hasForeignKeyToNonDistTable = list_length(nonDistTableForeignKeyIdList) != 0;
+	if (hasForeignKeyToNonDistTable)
+	{
+		/*
+		 * To support foreign keys from postgres tables to reference or citus
+		 * local tables, we drop and re-define foreign keys so that our ALTER
+		 * TABLE hook does the necessary job.
+		 */
+		List *relationFKeyCreationCommands =
+			GetForeignConstraintCommandsInternal(relationId, nonDistTableFKeysFlag);
+		DropRelationForeignKeys(relationId, nonDistTableFKeysFlag);
+
+		bool skip_validation = true;
+		ExecuteForeignKeyCreateCommandList(relationFKeyCreationCommands,
+										   skip_validation);
+	}
+}
+
+
+/*
+ * ShouldEnableLocalReferenceForeignKeys is a wrapper around getting the GUC
+ * EnableLocalReferenceForeignKeys. If the coordinator is not added
+ * to the metadata, the function returns false. Else, the function returns
+ * the value set by the user
+ *
+ */
+static bool
+ShouldEnableLocalReferenceForeignKeys(void)
+{
+	if (!EnableLocalReferenceForeignKeys)
+	{
+		return false;
+	}
+
+	return CoordinatorAddedAsWorkerNode();
+}
+
+
 /*
  * PostprocessCreateTableStmtPartitionOf processes CREATE TABLE ... PARTITION OF
  * statements and it checks if user creates the table as a partition of a distributed
@@ -379,9 +448,9 @@ PreprocessAlterTableStmt(Node *node, const char *alterTableCommand,
 		leftRelationId = IndexGetRelation(leftRelationId, missingOk);
 	}
 
-	if (processUtilityContext != PROCESS_UTILITY_SUBCOMMAND &&
+	if (ShouldEnableLocalReferenceForeignKeys() &&
-		AlterTableDefinesFKeyBetweenPostgresAndNonDistTable(alterTableStatement) &&
+		processUtilityContext != PROCESS_UTILITY_SUBCOMMAND &&
-		CoordinatorAddedAsWorkerNode())
+		AlterTableDefinesFKeyBetweenPostgresAndNonDistTable(alterTableStatement))
 	{
 		/*
 		 * We don't process subcommands generated by postgres.
@@ -1032,13 +1101,13 @@ PreprocessAlterTableSchemaStmt(Node *node, const char *queryString,
 
 
 /*
- * WorkerProcessAlterTableStmt checks and processes the alter table statement to be
+ * SkipForeignKeyValidationIfConstraintIsFkey checks and processes the alter table
- * worked on the distributed table of the worker node. Currently, it only processes
+ * statement to be worked on the distributed table. Currently, it only processes
  * ALTER TABLE ... ADD FOREIGN KEY command to skip the validation step.
  */
 Node *
-WorkerProcessAlterTableStmt(AlterTableStmt *alterTableStatement,
+SkipForeignKeyValidationIfConstraintIsFkey(AlterTableStmt *alterTableStatement,
-							const char *alterTableCommand)
+										   bool processLocalRelation)
 {
 	/* first check whether a distributed relation is affected */
 	if (alterTableStatement->relation == NULL)
@@ -1053,8 +1122,7 @@ WorkerProcessAlterTableStmt(AlterTableStmt *alterTableStatement,
 		return (Node *) alterTableStatement;
 	}
 
-	bool isCitusRelation = IsCitusTable(leftRelationId);
+	if (!IsCitusTable(leftRelationId) && !processLocalRelation)
-	if (!isCitusRelation)
 	{
 		return (Node *) alterTableStatement;
 	}

src/backend/distributed/commands/utility_hook.c

@@ -416,7 +416,8 @@ multi_ProcessUtility(PlannedStmt *pstmt,
 				 * Note validation is done on the shard level when DDL propagation
 				 * is enabled. The following eagerly executes some tasks on workers.
 				 */
-				parsetree = WorkerProcessAlterTableStmt(alterTableStmt, queryString);
+				parsetree =
+					SkipForeignKeyValidationIfConstraintIsFkey(alterTableStmt, false);
 			}
 		}
 	}

src/backend/distributed/operations/worker_node_manager.c

@@ -418,17 +418,13 @@ NodeIsPrimaryWorker(WorkerNode *node)
 
 /*
  * CoordinatorAddedAsWorkerNode returns true if coordinator is added to the
- * pg_dist_node. This function also acquires ShareLock on pg_dist_node
+ * pg_dist_node.
- * and does not release it to ensure that existency of the coordinator in
- * metadata won't be changed until the end of transaction.
  */
 bool
 CoordinatorAddedAsWorkerNode()
 {
 	bool groupContainsNodes = false;
 
-	LockRelationOid(DistNodeRelationId(), ShareLock);
-
 	PrimaryNodeForGroup(COORDINATOR_GROUP_ID, &groupContainsNodes);
 
 	return groupContainsNodes;

src/backend/distributed/shared_library_init.c

@@ -585,6 +585,18 @@ RegisterCitusConfigVariables(void)
 		GUC_STANDARD,
 		NULL, NULL, NULL);
 
+	DefineCustomBoolVariable(
+		"citus.enable_local_reference_table_foreign_keys",
+		gettext_noop("Enables foreign keys from/to local tables"),
+		gettext_noop("When enabled, foreign keys between local tables and reference "
+					 "tables supported."),
+		&EnableLocalReferenceForeignKeys,
+		true,
+		PGC_USERSET,
+		GUC_STANDARD,
+		NULL, NULL, NULL);
+
+
 	DefineCustomBoolVariable(
 		"citus.enable_single_hash_repartition_joins",
 		gettext_noop("Enables single hash repartitioning between hash "

src/include/distributed/commands.h

@@ -20,6 +20,11 @@
 #include "tcop/dest.h"
 #include "tcop/utility.h"
 
+
+/* controlled via GUC, should be accessed via EnableLocalReferenceForeignKeys() */
+extern bool EnableLocalReferenceForeignKeys;
+
+
 /*
  * DistributeObjectOps specifies handlers for node/object type pairs.
  * Instances of this type should all be declared in deparse.c.
@@ -363,8 +368,8 @@ extern List * PreprocessAlterTableMoveAllStmt(Node *node, const char *queryStrin
 											  ProcessUtilityContext processUtilityContext);
 extern List * PreprocessAlterTableSchemaStmt(Node *node, const char *queryString,
 											 ProcessUtilityContext processUtilityContext);
-extern Node * WorkerProcessAlterTableStmt(AlterTableStmt *alterTableStatement,
+extern Node * SkipForeignKeyValidationIfConstraintIsFkey(AlterTableStmt *alterTableStmt,
-										  const char *alterTableCommand);
+														 bool processLocalRelation);
 extern bool IsAlterTableRenameStmt(RenameStmt *renameStmt);
 extern void ErrorIfAlterDropsPartitionColumn(AlterTableStmt *alterTableStatement);
 extern void PostprocessAlterTableStmt(AlterTableStmt *pStmt);
@@ -477,6 +482,8 @@ extern void ErrorIfAnyPartitionRelationInvolvedInNonInheritedFKey(List *relation
 extern void DropRelationForeignKeys(Oid relationId, int flags);
 extern void ExecuteAndLogDDLCommandList(List *ddlCommandList);
 extern void ExecuteAndLogDDLCommand(const char *commandString);
+extern void ExecuteForeignKeyCreateCommandList(List *ddlCommandList,
+											   bool skip_validation);
 
 /* create_citus_local_table.c */
 extern void CreateCitusLocalTable(Oid relationId, bool cascadeViaForeignKeys);

src/test/regress/bin/normalize.sed

@@ -22,9 +22,9 @@ s/^-[+-]{2,}$/------------------------------------------------------------------
 
 # In foreign_key_to_reference_table, normalize shard table names, etc in
 # the generated plan
-s/"(foreign_key_2_|fkey_ref_to_dist_|fkey_ref_)[0-9]+"/"\1xxxxxxx"/g
+s/"(foreign_key_2_|fkey_ref_to_dist_|fkey_ref_|fkey_to_ref_)[0-9]+"/"\1xxxxxxx"/g
 s/"(referenced_table_|referencing_table_|referencing_table2_)[0-9]+"/"\1xxxxxxx"/g
-s/"(referencing_table_0_|referenced_table2_)[0-9]+"/"\1xxxxxxx"/g
+s/"(referencing_table_0_|referencing_table_4_|referenced_table2_)[0-9]+"/"\1xxxxxxx"/g
 s/\(id\)=\([0-9]+\)/(id)=(X)/g
 s/\(ref_id\)=\([0-9]+\)/(ref_id)=(X)/g
 

src/test/regress/expected/citus_local_tables.out

@@ -432,7 +432,7 @@ ERROR:  cannot execute ADD COLUMN command with PRIMARY KEY, UNIQUE, FOREIGN and
 CREATE TABLE local_table_4 (
   a int unique references citus_local_table_1(a),
   b int references local_table_4(a));
-ERROR:  cannot create foreign key constraint as "local_table_4" is a postgres local table
+NOTICE:  executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1504038, 'citus_local_tables_test_schema', 1504027, 'citus_local_tables_test_schema', 'ALTER TABLE citus_local_tables_test_schema.local_table_4 ADD CONSTRAINT local_table_4_a_fkey FOREIGN KEY (a) REFERENCES citus_local_tables_test_schema.citus_local_table_1(a)')
 ALTER TABLE citus_local_table_1 ADD COLUMN b int NOT NULL;
 NOTICE:  executing the command locally: SELECT worker_apply_shard_ddl_command (1504027, 'citus_local_tables_test_schema', 'ALTER TABLE citus_local_table_1 ADD COLUMN b int NOT NULL;')
 -- show that we added column with NOT NULL
@@ -473,17 +473,21 @@ ORDER BY 1;
 (2 rows)
 
 -- execute truncate & drop commands for multiple relations to see that we don't break local execution
-TRUNCATE citus_local_table_1, citus_local_table_2, distributed_table, local_table, reference_table;
+TRUNCATE citus_local_table_1, citus_local_table_2, distributed_table, local_table, reference_table, local_table_4;
 NOTICE:  executing the command locally: TRUNCATE TABLE citus_local_tables_test_schema.citus_local_table_1_xxxxx CASCADE
+NOTICE:  truncate cascades to table "local_table_4_xxxxx"
 NOTICE:  executing the command locally: TRUNCATE TABLE citus_local_tables_test_schema.citus_local_table_2_xxxxx CASCADE
 NOTICE:  executing the command locally: TRUNCATE TABLE citus_local_tables_test_schema.local_table_xxxxx CASCADE
 NOTICE:  truncate cascades to table "citus_local_table_1_xxxxxxx"
+NOTICE:  truncate cascades to table "local_table_4_xxxxx"
 NOTICE:  executing the command locally: TRUNCATE TABLE citus_local_tables_test_schema.reference_table_xxxxx CASCADE
+NOTICE:  executing the command locally: TRUNCATE TABLE citus_local_tables_test_schema.local_table_4_xxxxx CASCADE
 -- test vacuum
 VACUUM citus_local_table_1;
 VACUUM citus_local_table_1, distributed_table, local_table, reference_table;
 -- test drop
-DROP TABLE citus_local_table_1, citus_local_table_2, distributed_table, local_table, reference_table;
+DROP TABLE citus_local_table_1, citus_local_table_2, distributed_table, local_table, reference_table, local_table_4;
+NOTICE:  executing the command locally: DROP TABLE IF EXISTS citus_local_tables_test_schema.local_table_4_xxxxx CASCADE
 NOTICE:  executing the command locally: DROP TABLE IF EXISTS citus_local_tables_test_schema.reference_table_xxxxx CASCADE
 NOTICE:  executing the command locally: DROP TABLE IF EXISTS citus_local_tables_test_schema.local_table_xxxxx CASCADE
 NOTICE:  drop cascades to constraint fkey_c_to_local_1504027 on table citus_local_tables_test_schema.citus_local_table_1_1504027
@@ -500,9 +504,9 @@ SELECT create_citus_local_table('citus_local_table_4');
 -- should work --
 -- insert some data & create an index for table size udf's
 INSERT INTO citus_local_table_4 VALUES (1), (2), (3);
-NOTICE:  executing the command locally: INSERT INTO citus_local_tables_test_schema.citus_local_table_4_1504038 AS citus_table_alias (a) VALUES (1), (2), (3)
+NOTICE:  executing the command locally: INSERT INTO citus_local_tables_test_schema.citus_local_table_4_1504039 AS citus_table_alias (a) VALUES (1), (2), (3)
 CREATE INDEX citus_local_table_4_idx ON citus_local_table_4(a);
-NOTICE:  executing the command locally: CREATE  INDEX   citus_local_table_4_idx_1504038 ON citus_local_tables_test_schema.citus_local_table_4_1504038 USING btree (a )
+NOTICE:  executing the command locally: CREATE  INDEX   citus_local_table_4_idx_1504039 ON citus_local_tables_test_schema.citus_local_table_4_1504039 USING btree (a )
 SELECT citus_table_size('citus_local_table_4');
  citus_table_size
 ---------------------------------------------------------------------
@@ -589,7 +593,7 @@ BEGIN;
   SELECT tableName FROM pg_catalog.pg_tables WHERE tablename LIKE 'citus_local_table_4%';
           tablename
 ---------------------------------------------------------------------
- citus_local_table_4_1504038
+ citus_local_table_4_1504039
 (1 row)
 
 ROLLBACK;
@@ -598,7 +602,7 @@ SELECT shardid, get_colocated_shard_array(shardid)
 FROM (SELECT shardid FROM pg_dist_shard WHERE logicalrelid='citus_local_table_4'::regclass) as shardid;
  shardid | get_colocated_shard_array
 ---------------------------------------------------------------------
- 1504038 | {1504038}
+ 1504039 | {1504039}
 (1 row)
 
 BEGIN;
@@ -619,8 +623,6 @@ ROLLBACK;
 -- should fail --
 SELECT update_distributed_table_colocation('citus_local_table_4', colocate_with => 'none');
 ERROR:  relation citus_local_table_4 should be a hash distributed table
-SELECT master_create_worker_shards('citus_local_table_4', 10, 1);
-ERROR:  unsupported table partition type: n
 SELECT master_create_empty_shard('citus_local_table_4');
 ERROR:  relation "citus_local_table_4" is a citus local table
 SELECT master_apply_delete_command('DELETE FROM citus_local_table_4');
@@ -628,7 +630,7 @@ ERROR:  cannot delete from table
 CREATE TABLE postgres_local_table (a int);
 SELECT master_append_table_to_shard(shardId, 'postgres_local_table', 'localhost', :master_port)
 FROM (SELECT shardid FROM pg_dist_shard WHERE logicalrelid='citus_local_table_4'::regclass) as shardid;
-ERROR:  cannot append to shardId 1504038
+ERROR:  cannot append to shardId 1504039
 -- return true
 SELECT citus_table_is_visible('citus_local_table_4'::regclass::oid);
  citus_table_is_visible
@@ -669,7 +671,7 @@ SELECT create_citus_local_table('referenced_table');
 (1 row)
 
 ALTER TABLE referencing_table ADD CONSTRAINT fkey_cl_to_cl FOREIGN KEY (a) REFERENCES referenced_table(a);
-NOTICE:  executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1504039, 'citus_local_tables_test_schema', 1504040, 'citus_local_tables_test_schema', 'ALTER TABLE referencing_table ADD CONSTRAINT fkey_cl_to_cl FOREIGN KEY (a) REFERENCES referenced_table(a);')
+NOTICE:  executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1504040, 'citus_local_tables_test_schema', 1504041, 'citus_local_tables_test_schema', 'ALTER TABLE referencing_table ADD CONSTRAINT fkey_cl_to_cl FOREIGN KEY (a) REFERENCES referenced_table(a);')
 -- observe the debug messages telling that we switch to sequential
 -- execution when truncating a citus local table that is referenced
 -- by another table

src/test/regress/expected/fkeys_between_local_ref.out

@@ -227,5 +227,204 @@ BEGIN;
 
   ALTER TABLE local_table_2 ADD CONSTRAINT fkey_11 FOREIGN KEY (col_1) REFERENCES reference_table_2(col_1) ON UPDATE CASCADE;
 ROLLBACK;
+--
+-- create table tests
+--
+BEGIN;
+  CREATE TABLE local_table_6 (col_1 INT PRIMARY KEY);
+  -- create a table that references to
+  --  * local table graph
+  --  * reference table
+  --  * another local table
+  --  * itself
+  CREATE TABLE local_table_5 (
+    col_1 INT UNIQUE REFERENCES local_table_1(col_1),
+    col_2 INT,
+    FOREIGN KEY (col_1) REFERENCES reference_table_1(col_1),
+    -- not specify column for this foreign key
+    FOREIGN KEY (col_2) REFERENCES local_table_6,
+    -- also have a self reference
+    FOREIGN KEY (col_2) REFERENCES local_table_5(col_1));
+  -- now print metadata to show that all local tables are converted
+  -- to citus local tables
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+     tablename     | partmethod | repmodel
+---------------------------------------------------------------------
+ local_table_1     | n          | c
+ local_table_2     | n          | c
+ local_table_3     | n          | c
+ local_table_4     | n          | c
+ local_table_5     | n          | c
+ local_table_6     | n          | c
+ reference_table_1 | n          | t
+(7 rows)
+
+ROLLBACK;
+CREATE TABLE distributed_table (col_1 INT PRIMARY KEY);
+SELECT create_distributed_table('distributed_table', 'col_1');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+-- Creating a table that both references to a reference table and a
+-- distributed table fails.
+-- This is because, we convert local table to a citus local table
+-- due to its foreign key to reference table.
+-- But citus local tables can't have foreign keys to distributed tables.
+CREATE TABLE local_table_5 (
+  col_1 INT UNIQUE REFERENCES distributed_table(col_1),
+  FOREIGN KEY (col_1) REFERENCES reference_table_1(col_1));
+ERROR:  cannot create foreign key constraint since foreign keys from reference tables and citus local tables to distributed tables are not supported
+BEGIN;
+  ALTER TABLE distributed_table ADD CONSTRAINT fkey_11 FOREIGN KEY (col_1) REFERENCES reference_table_1(col_1);
+  CREATE TABLE local_table_5 (
+    col_1 INT UNIQUE REFERENCES reference_table_1(col_1),
+    FOREIGN KEY (col_1) REFERENCES local_table_1(col_1));
+  INSERT INTO local_table_5 SELECT i FROM generate_series(195, 205) i;
+  -- Now show that when converting local table to a citus local table,
+  -- distributed table (that is referenced by reference table) stays as is.
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+     tablename     | partmethod | repmodel
+---------------------------------------------------------------------
+ distributed_table | h          | c
+ local_table_1     | n          | c
+ local_table_2     | n          | c
+ local_table_3     | n          | c
+ local_table_4     | n          | c
+ local_table_5     | n          | c
+ reference_table_1 | n          | t
+(7 rows)
+
+  -- show that we validate foreign key constraints, errors out
+  INSERT INTO local_table_5 VALUES (300);
+ERROR:  insert or update on table "local_table_5_1518070" violates foreign key constraint "local_table_5_col_1_fkey1_1518070"
+ROLLBACK;
+BEGIN;
+  CREATE SCHEMA another_schema_fkeys_between_local_ref;
+  CREATE TABLE another_schema_fkeys_between_local_ref.local_table_6 (col_1 INT PRIMARY KEY);
+  -- first convert local tables to citus local tables in graph
+  ALTER TABLE local_table_2 ADD CONSTRAINT fkey_11 FOREIGN KEY (col_1) REFERENCES reference_table_1(col_1) ON DELETE CASCADE;
+  CREATE TABLE local_table_5 (
+    col_1 INT UNIQUE REFERENCES another_schema_fkeys_between_local_ref.local_table_6(col_1) CHECK (col_1 > 0),
+    col_2 INT REFERENCES local_table_3(col_1),
+    FOREIGN KEY (col_1) REFERENCES local_table_5(col_1));
+  -- Now show that we converted local_table_5 & 6 to citus local tables
+  -- as local_table_5 has foreign key to a citus local table too
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref' UNION
+                               SELECT 'another_schema_fkeys_between_local_ref.local_table_6')
+  ORDER BY tablename;
+                      tablename                       | partmethod | repmodel
+---------------------------------------------------------------------
+ another_schema_fkeys_between_local_ref.local_table_6 | n          | c
+ distributed_table                                    | h          | c
+ local_table_1                                        | n          | c
+ local_table_2                                        | n          | c
+ local_table_3                                        | n          | c
+ local_table_4                                        | n          | c
+ local_table_5                                        | n          | c
+ reference_table_1                                    | n          | t
+(8 rows)
+
+ROLLBACK;
+BEGIN;
+  CREATE TABLE local_table_6 (col_1 INT PRIMARY KEY);
+  -- first convert local tables to citus local tables in graph
+  ALTER TABLE local_table_2 ADD CONSTRAINT fkey_11 FOREIGN KEY (col_1) REFERENCES reference_table_1(col_1) ON DELETE CASCADE;
+  -- create a table that references to
+  --  * citus local table graph (via local_table_1)
+  --  * another local table (local_table_6)
+  --  * itself
+  CREATE TABLE local_table_5 (
+    col_1 INT UNIQUE REFERENCES local_table_1(col_1),
+    col_2 INT CHECK (col_2 > 0),
+    -- not specify column for this foreign key
+    FOREIGN KEY (col_2) REFERENCES local_table_6,
+    FOREIGN KEY (col_2) REFERENCES local_table_5(col_1));
+  -- now print metadata to show that all local tables are converted
+  -- to citus local tables
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+     tablename     | partmethod | repmodel
+---------------------------------------------------------------------
+ distributed_table | h          | c
+ local_table_1     | n          | c
+ local_table_2     | n          | c
+ local_table_3     | n          | c
+ local_table_4     | n          | c
+ local_table_5     | n          | c
+ local_table_6     | n          | c
+ reference_table_1 | n          | t
+(8 rows)
+
+  -- now make some of them reference tables
+  SELECT create_reference_table('local_table_2');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SELECT create_reference_table('local_table_6');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+     tablename     | partmethod | repmodel
+---------------------------------------------------------------------
+ distributed_table | h          | c
+ local_table_1     | n          | c
+ local_table_2     | n          | t
+ local_table_3     | n          | c
+ local_table_4     | n          | c
+ local_table_5     | n          | c
+ local_table_6     | n          | t
+ reference_table_1 | n          | t
+(8 rows)
+
+ROLLBACK;
+BEGIN;
+  -- disable foreign keys to reference tables
+  SET LOCAL citus.enable_local_reference_table_foreign_keys TO false;
+  CREATE TABLE local_table_6 (col_1 INT PRIMARY KEY);
+  ALTER TABLE local_table_2 ADD CONSTRAINT fkey_11 FOREIGN KEY (col_1) REFERENCES reference_table_1(col_1) ON DELETE CASCADE;
+  CREATE TABLE local_table_5 (
+    col_1 INT UNIQUE REFERENCES local_table_6(col_1),
+    col_2 INT REFERENCES local_table_3(col_1),
+    FOREIGN KEY (col_1) REFERENCES local_table_5(col_1),
+    FOREIGN KEY (col_1) REFERENCES reference_table_1(col_1));
+  -- Now show none of local_table_5 & 6 to should be converted to citus local tables
+  -- as it is disabled
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+     tablename     | partmethod | repmodel
+---------------------------------------------------------------------
+ distributed_table | h          | c
+ reference_table_1 | n          | t
+(2 rows)
+
+ROLLBACK;
+-- this errors out as we don't support creating citus local
+-- tables from partitioned tables
+CREATE TABLE part_local_table (col_1 INT REFERENCES reference_table_1(col_1)) PARTITION BY RANGE (col_1);
+ERROR:  cannot create citus local table "part_local_table", only regular tables and foreign tables are supported for citus local table creation
+-- they fail as col_99 does not exist
+CREATE TABLE local_table_5 (col_1 INT, FOREIGN KEY (col_99) REFERENCES reference_table_1(col_1));
+ERROR:  column "col_99" referenced in foreign key constraint does not exist
+CREATE TABLE local_table_5 (col_1 INT, FOREIGN KEY (col_1) REFERENCES reference_table_1(col_99));
+ERROR:  column "col_99" referenced in foreign key constraint does not exist
+-- fails as referenced table does not exist
+CREATE TABLE local_table_5 (col_1 INT, FOREIGN KEY (col_1) REFERENCES table_does_not_exist(dummy));
+ERROR:  relation "table_does_not_exist" does not exist
 -- cleanup at exit
 DROP SCHEMA fkeys_between_local_ref CASCADE;

src/test/regress/expected/foreign_key_to_reference_table.out

@@ -7,6 +7,7 @@ SET citus.shard_replication_factor TO 1;
 SET citus.shard_count TO 8;
 SET citus.next_shard_id TO 7000000;
 SET citus.next_placement_id TO 7000000;
+SET client_min_messages TO ERROR;
 CREATE TYPE foreign_details AS (name text, relid text, refd_relid text);
 CREATE VIEW table_fkeys_in_workers AS
 SELECT
@@ -143,18 +144,11 @@ SELECT create_distributed_table('referencing_table', 'ref_id');
 (1 row)
 
 ALTER TABLE referencing_table ADD CONSTRAINT fkey_ref FOREIGN KEY(id) REFERENCES referenced_table(id) ON DELETE SET NULL;
-SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
+SELECT COUNT(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
-       name       |                     relid                      |                  refd_relid
+ count
 ---------------------------------------------------------------------
- fkey_ref_7000043 | fkey_reference_table.referencing_table_7000043 | fkey_reference_table.referenced_table_7000042
+     8
- fkey_ref_7000044 | fkey_reference_table.referencing_table_7000044 | fkey_reference_table.referenced_table_7000042
+(1 row)
- fkey_ref_7000045 | fkey_reference_table.referencing_table_7000045 | fkey_reference_table.referenced_table_7000042
- fkey_ref_7000046 | fkey_reference_table.referencing_table_7000046 | fkey_reference_table.referenced_table_7000042
- fkey_ref_7000047 | fkey_reference_table.referencing_table_7000047 | fkey_reference_table.referenced_table_7000042
- fkey_ref_7000048 | fkey_reference_table.referencing_table_7000048 | fkey_reference_table.referenced_table_7000042
- fkey_ref_7000049 | fkey_reference_table.referencing_table_7000049 | fkey_reference_table.referenced_table_7000042
- fkey_ref_7000050 | fkey_reference_table.referencing_table_7000050 | fkey_reference_table.referenced_table_7000042
-(8 rows)
 
 DROP TABLE referencing_table;
 CREATE TABLE referencing_table(id int, ref_id int, FOREIGN KEY(id) REFERENCES referenced_table(id) ON DELETE SET NULL);
@@ -164,18 +158,11 @@ SELECT create_distributed_table('referencing_table', 'ref_id');
 
 (1 row)
 
-SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
+SELECT COUNT(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
-               name                |                     relid                      |                  refd_relid
+ count
 ---------------------------------------------------------------------
- referencing_table_id_fkey_7000051 | fkey_reference_table.referencing_table_7000051 | fkey_reference_table.referenced_table_7000042
+     8
- referencing_table_id_fkey_7000052 | fkey_reference_table.referencing_table_7000052 | fkey_reference_table.referenced_table_7000042
+(1 row)
- referencing_table_id_fkey_7000053 | fkey_reference_table.referencing_table_7000053 | fkey_reference_table.referenced_table_7000042
- referencing_table_id_fkey_7000054 | fkey_reference_table.referencing_table_7000054 | fkey_reference_table.referenced_table_7000042
- referencing_table_id_fkey_7000055 | fkey_reference_table.referencing_table_7000055 | fkey_reference_table.referenced_table_7000042
- referencing_table_id_fkey_7000056 | fkey_reference_table.referencing_table_7000056 | fkey_reference_table.referenced_table_7000042
- referencing_table_id_fkey_7000057 | fkey_reference_table.referencing_table_7000057 | fkey_reference_table.referenced_table_7000042
- referencing_table_id_fkey_7000058 | fkey_reference_table.referencing_table_7000058 | fkey_reference_table.referenced_table_7000042
-(8 rows)
 
 DROP TABLE referencing_table;
 CREATE TABLE referencing_table(id int, ref_id int);
@@ -186,18 +173,11 @@ SELECT create_distributed_table('referencing_table', 'ref_id');
 (1 row)
 
 ALTER TABLE referencing_table ADD CONSTRAINT fkey_ref FOREIGN KEY(id) REFERENCES referenced_table(id) ON DELETE SET DEFAULT;
-SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
+SELECT COUNT(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
-       name       |                     relid                      |                  refd_relid
+ count
 ---------------------------------------------------------------------
- fkey_ref_7000059 | fkey_reference_table.referencing_table_7000059 | fkey_reference_table.referenced_table_7000042
+     8
- fkey_ref_7000060 | fkey_reference_table.referencing_table_7000060 | fkey_reference_table.referenced_table_7000042
+(1 row)
- fkey_ref_7000061 | fkey_reference_table.referencing_table_7000061 | fkey_reference_table.referenced_table_7000042
- fkey_ref_7000062 | fkey_reference_table.referencing_table_7000062 | fkey_reference_table.referenced_table_7000042
- fkey_ref_7000063 | fkey_reference_table.referencing_table_7000063 | fkey_reference_table.referenced_table_7000042
- fkey_ref_7000064 | fkey_reference_table.referencing_table_7000064 | fkey_reference_table.referenced_table_7000042
- fkey_ref_7000065 | fkey_reference_table.referencing_table_7000065 | fkey_reference_table.referenced_table_7000042
- fkey_ref_7000066 | fkey_reference_table.referencing_table_7000066 | fkey_reference_table.referenced_table_7000042
-(8 rows)
 
 DROP TABLE referencing_table;
 BEGIN;
@@ -209,18 +189,11 @@ BEGIN;
 (1 row)
 
 COMMIT;
-SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
+SELECT COUNT(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
-               name                |                     relid                      |                  refd_relid
+ count
 ---------------------------------------------------------------------
- referencing_table_id_fkey_7000067 | fkey_reference_table.referencing_table_7000067 | fkey_reference_table.referenced_table_7000042
+     8
- referencing_table_id_fkey_7000068 | fkey_reference_table.referencing_table_7000068 | fkey_reference_table.referenced_table_7000042
+(1 row)
- referencing_table_id_fkey_7000069 | fkey_reference_table.referencing_table_7000069 | fkey_reference_table.referenced_table_7000042
- referencing_table_id_fkey_7000070 | fkey_reference_table.referencing_table_7000070 | fkey_reference_table.referenced_table_7000042
- referencing_table_id_fkey_7000071 | fkey_reference_table.referencing_table_7000071 | fkey_reference_table.referenced_table_7000042
- referencing_table_id_fkey_7000072 | fkey_reference_table.referencing_table_7000072 | fkey_reference_table.referenced_table_7000042
- referencing_table_id_fkey_7000073 | fkey_reference_table.referencing_table_7000073 | fkey_reference_table.referenced_table_7000042
- referencing_table_id_fkey_7000074 | fkey_reference_table.referencing_table_7000074 | fkey_reference_table.referenced_table_7000042
-(8 rows)
 
 DROP TABLE referencing_table;
 CREATE TABLE referencing_table(id int, ref_id int);
@@ -231,18 +204,11 @@ SELECT create_distributed_table('referencing_table', 'ref_id');
 (1 row)
 
 ALTER TABLE referencing_table ADD CONSTRAINT fkey_ref FOREIGN KEY(id) REFERENCES referenced_table(id) ON UPDATE SET NULL;
-SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
+SELECT COUNT(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
-       name       |                     relid                      |                  refd_relid
+ count
 ---------------------------------------------------------------------
- fkey_ref_7000075 | fkey_reference_table.referencing_table_7000075 | fkey_reference_table.referenced_table_7000042
+     8
- fkey_ref_7000076 | fkey_reference_table.referencing_table_7000076 | fkey_reference_table.referenced_table_7000042
+(1 row)
- fkey_ref_7000077 | fkey_reference_table.referencing_table_7000077 | fkey_reference_table.referenced_table_7000042
- fkey_ref_7000078 | fkey_reference_table.referencing_table_7000078 | fkey_reference_table.referenced_table_7000042
- fkey_ref_7000079 | fkey_reference_table.referencing_table_7000079 | fkey_reference_table.referenced_table_7000042
- fkey_ref_7000080 | fkey_reference_table.referencing_table_7000080 | fkey_reference_table.referenced_table_7000042
- fkey_ref_7000081 | fkey_reference_table.referencing_table_7000081 | fkey_reference_table.referenced_table_7000042
- fkey_ref_7000082 | fkey_reference_table.referencing_table_7000082 | fkey_reference_table.referenced_table_7000042
-(8 rows)
 
 DROP TABLE referencing_table;
 CREATE TABLE referencing_table(id int, ref_id int);
@@ -253,18 +219,11 @@ SELECT create_distributed_table('referencing_table', 'ref_id');
 (1 row)
 
 ALTER TABLE referencing_table ADD CONSTRAINT fkey_ref FOREIGN KEY(id) REFERENCES referenced_table(id) ON UPDATE SET DEFAULT;
-SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
+SELECT COUNT(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
-       name       |                     relid                      |                  refd_relid
+ count
 ---------------------------------------------------------------------
- fkey_ref_7000083 | fkey_reference_table.referencing_table_7000083 | fkey_reference_table.referenced_table_7000042
+     8
- fkey_ref_7000084 | fkey_reference_table.referencing_table_7000084 | fkey_reference_table.referenced_table_7000042
+(1 row)
- fkey_ref_7000085 | fkey_reference_table.referencing_table_7000085 | fkey_reference_table.referenced_table_7000042
- fkey_ref_7000086 | fkey_reference_table.referencing_table_7000086 | fkey_reference_table.referenced_table_7000042
- fkey_ref_7000087 | fkey_reference_table.referencing_table_7000087 | fkey_reference_table.referenced_table_7000042
- fkey_ref_7000088 | fkey_reference_table.referencing_table_7000088 | fkey_reference_table.referenced_table_7000042
- fkey_ref_7000089 | fkey_reference_table.referencing_table_7000089 | fkey_reference_table.referenced_table_7000042
- fkey_ref_7000090 | fkey_reference_table.referencing_table_7000090 | fkey_reference_table.referenced_table_7000042
-(8 rows)
 
 DROP TABLE referencing_table;
 CREATE TABLE referencing_table(id int, ref_id int);
@@ -275,18 +234,11 @@ SELECT create_distributed_table('referencing_table', 'ref_id');
 (1 row)
 
 ALTER TABLE referencing_table ADD CONSTRAINT fkey_ref FOREIGN KEY(id) REFERENCES referenced_table(id) ON UPDATE CASCADE;
-SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
+SELECT COUNT(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
-       name       |                     relid                      |                  refd_relid
+ count
 ---------------------------------------------------------------------
- fkey_ref_7000091 | fkey_reference_table.referencing_table_7000091 | fkey_reference_table.referenced_table_7000042
+     8
- fkey_ref_7000092 | fkey_reference_table.referencing_table_7000092 | fkey_reference_table.referenced_table_7000042
+(1 row)
- fkey_ref_7000093 | fkey_reference_table.referencing_table_7000093 | fkey_reference_table.referenced_table_7000042
- fkey_ref_7000094 | fkey_reference_table.referencing_table_7000094 | fkey_reference_table.referenced_table_7000042
- fkey_ref_7000095 | fkey_reference_table.referencing_table_7000095 | fkey_reference_table.referenced_table_7000042
- fkey_ref_7000096 | fkey_reference_table.referencing_table_7000096 | fkey_reference_table.referenced_table_7000042
- fkey_ref_7000097 | fkey_reference_table.referencing_table_7000097 | fkey_reference_table.referenced_table_7000042
- fkey_ref_7000098 | fkey_reference_table.referencing_table_7000098 | fkey_reference_table.referenced_table_7000042
-(8 rows)
 
 DROP TABLE referencing_table;
 -- check if we can add the foreign key while adding the column
@@ -301,10 +253,11 @@ ALTER TABLE referencing_table ADD COLUMN referencing int REFERENCES referenced_t
 ERROR:  cannot execute ADD COLUMN command with PRIMARY KEY, UNIQUE, FOREIGN and CHECK constraints
 DETAIL:  Adding a column with a constraint in one command is not supported because all constraints in Citus must have explicit names
 HINT:  You can issue each command separately such as ALTER TABLE referencing_table ADD COLUMN referencing data_type; ALTER TABLE referencing_table ADD CONSTRAINT constraint_name FOREIGN KEY (referencing) REFERENCES referenced_table(id) ON UPDATE CASCADE;
-SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
+SELECT COUNT(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
- name | relid | refd_relid
+ count
 ---------------------------------------------------------------------
-(0 rows)
+     0
+(1 row)
 
 DROP TABLE referencing_table;
 -- foreign keys are only supported when the replication factor = 1
@@ -320,10 +273,11 @@ ALTER TABLE referencing_table ADD CONSTRAINT fkey_ref FOREIGN KEY (id) REFERENCE
 ERROR:  cannot create foreign key constraint
 DETAIL:  Citus currently supports foreign key constraints only for "citus.shard_replication_factor = 1".
 HINT:  Please change "citus.shard_replication_factor to 1". To learn more about using foreign keys with other replication factors, please contact us at https://citusdata.com/about/contact_us.
-SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
+SELECT COUNT(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
- name | relid | refd_relid
+ count
 ---------------------------------------------------------------------
-(0 rows)
+     0
+(1 row)
 
 DROP TABLE referencing_table;
 -- should fail when we add the column as well
@@ -338,10 +292,11 @@ ALTER TABLE referencing_table ADD COLUMN referencing_col int REFERENCES referenc
 ERROR:  cannot create foreign key constraint
 DETAIL:  Citus currently supports foreign key constraints only for "citus.shard_replication_factor = 1".
 HINT:  Please change "citus.shard_replication_factor to 1". To learn more about using foreign keys with other replication factors, please contact us at https://citusdata.com/about/contact_us.
-SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
+SELECT COUNT(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
- name | relid | refd_relid
+ count
 ---------------------------------------------------------------------
-(0 rows)
+     0
+(1 row)
 
 DROP TABLE referencing_table;
 SET citus.shard_replication_factor TO 1;
@@ -353,18 +308,11 @@ SELECT create_distributed_table('referencing_table', 'ref_id');
 
 (1 row)
 
-SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
+SELECT COUNT(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
-               name                |                     relid                      |                  refd_relid
+ count
 ---------------------------------------------------------------------
- referencing_table_id_fkey_7000123 | fkey_reference_table.referencing_table_7000123 | fkey_reference_table.referenced_table_7000042
+     8
- referencing_table_id_fkey_7000124 | fkey_reference_table.referencing_table_7000124 | fkey_reference_table.referenced_table_7000042
+(1 row)
- referencing_table_id_fkey_7000125 | fkey_reference_table.referencing_table_7000125 | fkey_reference_table.referenced_table_7000042
- referencing_table_id_fkey_7000126 | fkey_reference_table.referencing_table_7000126 | fkey_reference_table.referenced_table_7000042
- referencing_table_id_fkey_7000127 | fkey_reference_table.referencing_table_7000127 | fkey_reference_table.referenced_table_7000042
- referencing_table_id_fkey_7000128 | fkey_reference_table.referencing_table_7000128 | fkey_reference_table.referenced_table_7000042
- referencing_table_id_fkey_7000129 | fkey_reference_table.referencing_table_7000129 | fkey_reference_table.referenced_table_7000042
- referencing_table_id_fkey_7000130 | fkey_reference_table.referencing_table_7000130 | fkey_reference_table.referenced_table_7000042
-(8 rows)
 
 DROP TABLE referencing_table;
 DROP TABLE referenced_table;
@@ -384,18 +332,11 @@ BEGIN;
 (1 row)
 
 COMMIT;
-SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
+SELECT COUNT(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
-               name                |                     relid                      |                  refd_relid
+ count
 ---------------------------------------------------------------------
- referencing_table_id_fkey_7000132 | fkey_reference_table.referencing_table_7000132 | fkey_reference_table.referenced_table_7000131
+     8
- referencing_table_id_fkey_7000133 | fkey_reference_table.referencing_table_7000133 | fkey_reference_table.referenced_table_7000131
+(1 row)
- referencing_table_id_fkey_7000134 | fkey_reference_table.referencing_table_7000134 | fkey_reference_table.referenced_table_7000131
- referencing_table_id_fkey_7000135 | fkey_reference_table.referencing_table_7000135 | fkey_reference_table.referenced_table_7000131
- referencing_table_id_fkey_7000136 | fkey_reference_table.referencing_table_7000136 | fkey_reference_table.referenced_table_7000131
- referencing_table_id_fkey_7000137 | fkey_reference_table.referencing_table_7000137 | fkey_reference_table.referenced_table_7000131
- referencing_table_id_fkey_7000138 | fkey_reference_table.referencing_table_7000138 | fkey_reference_table.referenced_table_7000131
- referencing_table_id_fkey_7000139 | fkey_reference_table.referencing_table_7000139 | fkey_reference_table.referenced_table_7000131
-(8 rows)
 
 DROP TABLE referencing_table;
 -- foreign keys are supported either in between distributed tables including the
@@ -468,7 +409,6 @@ CONTEXT:  while executing command on localhost:xxxxx
 DELETE FROM referenced_table WHERE id = 501;
 -- test cascading truncate
 TRUNCATE referenced_table CASCADE;
-NOTICE:  truncate cascades to table "referencing_table"
 SELECT count(*) FROM referencing_table;
  count
 ---------------------------------------------------------------------
@@ -795,26 +735,11 @@ SELECT create_distributed_table('referencing_table', 'id');
 
 ALTER TABLE referencing_table ADD CONSTRAINT fkey_ref FOREIGN KEY (id) REFERENCES referenced_table(test_column) ON DELETE CASCADE;
 ALTER TABLE referencing_table ADD CONSTRAINT foreign_key_2 FOREIGN KEY (id) REFERENCES referenced_table2(test_column2) ON DELETE CASCADE;
-SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
+SELECT COUNT(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
-         name          |                     relid                      |                   refd_relid
+ count
 ---------------------------------------------------------------------
- fkey_ref_7000226      | fkey_reference_table.referencing_table_7000226 | fkey_reference_table.referenced_table_7000224
+    16
- fkey_ref_7000227      | fkey_reference_table.referencing_table_7000227 | fkey_reference_table.referenced_table_7000224
+(1 row)
- fkey_ref_7000228      | fkey_reference_table.referencing_table_7000228 | fkey_reference_table.referenced_table_7000224
- fkey_ref_7000229      | fkey_reference_table.referencing_table_7000229 | fkey_reference_table.referenced_table_7000224
- fkey_ref_7000230      | fkey_reference_table.referencing_table_7000230 | fkey_reference_table.referenced_table_7000224
- fkey_ref_7000231      | fkey_reference_table.referencing_table_7000231 | fkey_reference_table.referenced_table_7000224
- fkey_ref_7000232      | fkey_reference_table.referencing_table_7000232 | fkey_reference_table.referenced_table_7000224
- fkey_ref_7000233      | fkey_reference_table.referencing_table_7000233 | fkey_reference_table.referenced_table_7000224
- foreign_key_2_7000226 | fkey_reference_table.referencing_table_7000226 | fkey_reference_table.referenced_table2_7000225
- foreign_key_2_7000227 | fkey_reference_table.referencing_table_7000227 | fkey_reference_table.referenced_table2_7000225
- foreign_key_2_7000228 | fkey_reference_table.referencing_table_7000228 | fkey_reference_table.referenced_table2_7000225
- foreign_key_2_7000229 | fkey_reference_table.referencing_table_7000229 | fkey_reference_table.referenced_table2_7000225
- foreign_key_2_7000230 | fkey_reference_table.referencing_table_7000230 | fkey_reference_table.referenced_table2_7000225
- foreign_key_2_7000231 | fkey_reference_table.referencing_table_7000231 | fkey_reference_table.referenced_table2_7000225
- foreign_key_2_7000232 | fkey_reference_table.referencing_table_7000232 | fkey_reference_table.referenced_table2_7000225
- foreign_key_2_7000233 | fkey_reference_table.referencing_table_7000233 | fkey_reference_table.referenced_table2_7000225
-(16 rows)
 
 INSERT INTO referenced_table SELECT x, x+1 FROM generate_series(0,1000) AS f(x);
 INSERT INTO referenced_table2 SELECT x, x+1 FROM generate_series(500,1500) AS f(x);
@@ -923,26 +848,11 @@ BEGIN;
   ALTER TABLE referencing_table ADD CONSTRAINT fkey_ref FOREIGN KEY (id) REFERENCES referenced_table(test_column) ON DELETE CASCADE;
   ALTER TABLE referencing_table ADD CONSTRAINT foreign_key_2 FOREIGN KEY (ref_id) REFERENCES referenced_table2(test_column2) ON DELETE CASCADE;
 COMMIT;
-SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
+SELECT COUNT(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
-         name          |                     relid                      |                   refd_relid
+ count
 ---------------------------------------------------------------------
- fkey_ref_7000246      | fkey_reference_table.referencing_table_7000246 | fkey_reference_table.referenced_table_7000244
+    16
- fkey_ref_7000247      | fkey_reference_table.referencing_table_7000247 | fkey_reference_table.referenced_table_7000244
+(1 row)
- fkey_ref_7000248      | fkey_reference_table.referencing_table_7000248 | fkey_reference_table.referenced_table_7000244
- fkey_ref_7000249      | fkey_reference_table.referencing_table_7000249 | fkey_reference_table.referenced_table_7000244
- fkey_ref_7000250      | fkey_reference_table.referencing_table_7000250 | fkey_reference_table.referenced_table_7000244
- fkey_ref_7000251      | fkey_reference_table.referencing_table_7000251 | fkey_reference_table.referenced_table_7000244
- fkey_ref_7000252      | fkey_reference_table.referencing_table_7000252 | fkey_reference_table.referenced_table_7000244
- fkey_ref_7000253      | fkey_reference_table.referencing_table_7000253 | fkey_reference_table.referenced_table_7000244
- foreign_key_2_7000246 | fkey_reference_table.referencing_table_7000246 | fkey_reference_table.referenced_table2_7000245
- foreign_key_2_7000247 | fkey_reference_table.referencing_table_7000247 | fkey_reference_table.referenced_table2_7000245
- foreign_key_2_7000248 | fkey_reference_table.referencing_table_7000248 | fkey_reference_table.referenced_table2_7000245
- foreign_key_2_7000249 | fkey_reference_table.referencing_table_7000249 | fkey_reference_table.referenced_table2_7000245
- foreign_key_2_7000250 | fkey_reference_table.referencing_table_7000250 | fkey_reference_table.referenced_table2_7000245
- foreign_key_2_7000251 | fkey_reference_table.referencing_table_7000251 | fkey_reference_table.referenced_table2_7000245
- foreign_key_2_7000252 | fkey_reference_table.referencing_table_7000252 | fkey_reference_table.referenced_table2_7000245
- foreign_key_2_7000253 | fkey_reference_table.referencing_table_7000253 | fkey_reference_table.referenced_table2_7000245
-(16 rows)
 
 INSERT INTO referenced_table SELECT x, x+1 FROM generate_series(0,1000) AS f(x);
 INSERT INTO referenced_table2 SELECT x, x+1 FROM generate_series(500,1500) AS f(x);
@@ -1054,34 +964,11 @@ ALTER TABLE referencing_table ADD CONSTRAINT fkey_ref FOREIGN KEY (id) REFERENCE
 ALTER TABLE referencing_table2 ADD CONSTRAINT fkey_ref FOREIGN KEY (ref_id) REFERENCES referenced_table(test_column2) ON DELETE CASCADE;
 ALTER TABLE referencing_table2 ADD CONSTRAINT fkey_ref_to_dist FOREIGN KEY (id) REFERENCES referencing_table(id) ON DELETE CASCADE;
 COMMIT;
-SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
+SELECT COUNT(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
-           name           |                      relid                      |                   refd_relid
+ count
 ---------------------------------------------------------------------
- fkey_ref_7000265         | fkey_reference_table.referencing_table_7000265  | fkey_reference_table.referenced_table_7000264
+    24
- fkey_ref_7000266         | fkey_reference_table.referencing_table_7000266  | fkey_reference_table.referenced_table_7000264
+(1 row)
- fkey_ref_7000267         | fkey_reference_table.referencing_table_7000267  | fkey_reference_table.referenced_table_7000264
- fkey_ref_7000268         | fkey_reference_table.referencing_table_7000268  | fkey_reference_table.referenced_table_7000264
- fkey_ref_7000269         | fkey_reference_table.referencing_table_7000269  | fkey_reference_table.referenced_table_7000264
- fkey_ref_7000270         | fkey_reference_table.referencing_table_7000270  | fkey_reference_table.referenced_table_7000264
- fkey_ref_7000271         | fkey_reference_table.referencing_table_7000271  | fkey_reference_table.referenced_table_7000264
- fkey_ref_7000272         | fkey_reference_table.referencing_table_7000272  | fkey_reference_table.referenced_table_7000264
- fkey_ref_7000273         | fkey_reference_table.referencing_table2_7000273 | fkey_reference_table.referenced_table_7000264
- fkey_ref_7000274         | fkey_reference_table.referencing_table2_7000274 | fkey_reference_table.referenced_table_7000264
- fkey_ref_7000275         | fkey_reference_table.referencing_table2_7000275 | fkey_reference_table.referenced_table_7000264
- fkey_ref_7000276         | fkey_reference_table.referencing_table2_7000276 | fkey_reference_table.referenced_table_7000264
- fkey_ref_7000277         | fkey_reference_table.referencing_table2_7000277 | fkey_reference_table.referenced_table_7000264
- fkey_ref_7000278         | fkey_reference_table.referencing_table2_7000278 | fkey_reference_table.referenced_table_7000264
- fkey_ref_7000279         | fkey_reference_table.referencing_table2_7000279 | fkey_reference_table.referenced_table_7000264
- fkey_ref_7000280         | fkey_reference_table.referencing_table2_7000280 | fkey_reference_table.referenced_table_7000264
- fkey_ref_to_dist_7000273 | fkey_reference_table.referencing_table2_7000273 | fkey_reference_table.referencing_table_7000265
- fkey_ref_to_dist_7000274 | fkey_reference_table.referencing_table2_7000274 | fkey_reference_table.referencing_table_7000266
- fkey_ref_to_dist_7000275 | fkey_reference_table.referencing_table2_7000275 | fkey_reference_table.referencing_table_7000267
- fkey_ref_to_dist_7000276 | fkey_reference_table.referencing_table2_7000276 | fkey_reference_table.referencing_table_7000268
- fkey_ref_to_dist_7000277 | fkey_reference_table.referencing_table2_7000277 | fkey_reference_table.referencing_table_7000269
- fkey_ref_to_dist_7000278 | fkey_reference_table.referencing_table2_7000278 | fkey_reference_table.referencing_table_7000270
- fkey_ref_to_dist_7000279 | fkey_reference_table.referencing_table2_7000279 | fkey_reference_table.referencing_table_7000271
- fkey_ref_to_dist_7000280 | fkey_reference_table.referencing_table2_7000280 | fkey_reference_table.referencing_table_7000272
-(24 rows)
 
 INSERT INTO referenced_table SELECT x, x+1 FROM generate_series(0,1000) AS f(x);
 -- should fail
@@ -1188,26 +1075,11 @@ SELECT create_distributed_table('referencing_referencing_table', 'id');
 (1 row)
 
 ALTER TABLE referencing_table ADD CONSTRAINT fkey_ref FOREIGN KEY (ref_id, ref_id2) REFERENCES referenced_table(test_column, test_column2) ON DELETE CASCADE;
-SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.referencing%' ORDER BY 1,2,3;
+SELECT COUNT(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.referencing%';
-                         name                         |                           relid                            |                   refd_relid
+ count
 ---------------------------------------------------------------------
- fkey_ref_7000299                                     | fkey_reference_table.referencing_table_7000299             | fkey_reference_table.referenced_table_7000298
+    16
- fkey_ref_7000300                                     | fkey_reference_table.referencing_table_7000300             | fkey_reference_table.referenced_table_7000298
+(1 row)
- fkey_ref_7000301                                     | fkey_reference_table.referencing_table_7000301             | fkey_reference_table.referenced_table_7000298
- fkey_ref_7000302                                     | fkey_reference_table.referencing_table_7000302             | fkey_reference_table.referenced_table_7000298
- fkey_ref_7000303                                     | fkey_reference_table.referencing_table_7000303             | fkey_reference_table.referenced_table_7000298
- fkey_ref_7000304                                     | fkey_reference_table.referencing_table_7000304             | fkey_reference_table.referenced_table_7000298
- fkey_ref_7000305                                     | fkey_reference_table.referencing_table_7000305             | fkey_reference_table.referenced_table_7000298
- fkey_ref_7000306                                     | fkey_reference_table.referencing_table_7000306             | fkey_reference_table.referenced_table_7000298
- referencing_referencing_table_id_fkey_7000307 | fkey_reference_table.referencing_referencing_table_7000307 | fkey_reference_table.referencing_table_7000299
- referencing_referencing_table_id_fkey_7000308 | fkey_reference_table.referencing_referencing_table_7000308 | fkey_reference_table.referencing_table_7000300
- referencing_referencing_table_id_fkey_7000309 | fkey_reference_table.referencing_referencing_table_7000309 | fkey_reference_table.referencing_table_7000301
- referencing_referencing_table_id_fkey_7000310 | fkey_reference_table.referencing_referencing_table_7000310 | fkey_reference_table.referencing_table_7000302
- referencing_referencing_table_id_fkey_7000311 | fkey_reference_table.referencing_referencing_table_7000311 | fkey_reference_table.referencing_table_7000303
- referencing_referencing_table_id_fkey_7000312 | fkey_reference_table.referencing_referencing_table_7000312 | fkey_reference_table.referencing_table_7000304
- referencing_referencing_table_id_fkey_7000313 | fkey_reference_table.referencing_referencing_table_7000313 | fkey_reference_table.referencing_table_7000305
- referencing_referencing_table_id_fkey_7000314 | fkey_reference_table.referencing_referencing_table_7000314 | fkey_reference_table.referencing_table_7000306
-(16 rows)
 
 INSERT INTO referenced_table SELECT x, x+1 FROM generate_series(1,1000) AS f(x);
 INSERT INTO referencing_table SELECT x, x+1, x+2 FROM generate_series(1,999) AS f(x);
@@ -1224,36 +1096,6 @@ NOTICE:  drop cascades to constraint fkey_ref on table referencing_table
 DROP TABLE referencing_table CASCADE;
 NOTICE:  drop cascades to constraint referencing_referencing_table_id_fkey on table referencing_referencing_table
 DROP TABLE referencing_referencing_table;
--- test if create_distributed_table works in transactions with some edge cases
--- the following checks if create_distributed_table works on foreign keys when
--- one of them is a self-referencing table of multiple distributed tables
-BEGIN;
-  CREATE TABLE test_table_1(id int PRIMARY KEY);
-  SELECT create_reference_table('test_table_1');
- create_reference_table
----------------------------------------------------------------------
-
-(1 row)
-
-  CREATE TABLE test_table_2(id int PRIMARY KEY, value_1 int, FOREIGN KEY(id) REFERENCES test_table_1(id));
-  SELECT create_distributed_table('test_table_2', 'id');
- create_distributed_table
----------------------------------------------------------------------
-
-(1 row)
-
-  CREATE TABLE test_table_3(id int PRIMARY KEY, value_1 int, FOREIGN KEY(value_1) REFERENCES test_table_1(id), FOREIGN KEY(id) REFERENCES test_table_2(id));
-  SELECT create_distributed_table('test_table_3', 'id');
- create_distributed_table
----------------------------------------------------------------------
-
-(1 row)
-
-  DROP TABLE test_table_1 CASCADE;
-NOTICE:  drop cascades to 2 other objects
-DETAIL:  drop cascades to constraint test_table_2_id_fkey on table test_table_2
-drop cascades to constraint test_table_3_value_1_fkey on table test_table_3
-ROLLBACK;
 -- create_reference_table, create_distributed_table and ALTER TABLE in the same transaction
 BEGIN;
   CREATE TABLE test_table_1(id int PRIMARY KEY);
@@ -1820,7 +1662,7 @@ ALTER TABLE referencing_table_4 ADD CONSTRAINT fkey FOREIGN KEY (id) REFERENCES
 ALTER TABLE referencing_table_4 ADD CONSTRAINT fkey_to_ref FOREIGN KEY (value_1) REFERENCES referenced_table;
 -- should fail since the data will flow to partitioning_test_4 and it has a foreign constraint to partitioning_test_0 on id column
 INSERT INTO referencing_table VALUES (0, 5);
-ERROR:  insert or update on table "referencing_table_4_7000540" violates foreign key constraint "fkey_xxxxxxx"
+ERROR:  insert or update on table "referencing_table_4_xxxxxxx" violates foreign key constraint "fkey_xxxxxxx"
 DETAIL:  Key (id)=(X) is not present in table "referencing_table_0_xxxxxxx".
 CONTEXT:  while executing command on localhost:xxxxx
 -- should succeed on partitioning_test_0
@@ -1833,7 +1675,7 @@ SELECT * FROM referencing_table;
 
 -- should fail since partitioning_test_4 has foreign constraint to referenced_table on value_1 column
 INSERT INTO referencing_table VALUES (0, 5);
-ERROR:  insert or update on table "referencing_table_4_7000540" violates foreign key constraint "fkey_to_ref_7000540"
+ERROR:  insert or update on table "referencing_table_4_xxxxxxx" violates foreign key constraint "fkey_to_ref_xxxxxxx"
 DETAIL:  Key (value_1)=(5) is not present in table "referenced_table_xxxxxxx".
 CONTEXT:  while executing command on localhost:xxxxx
 INSERT INTO referenced_table VALUES(5,5);

src/test/regress/sql/citus_local_tables.sql

@@ -351,14 +351,14 @@ WHERE indrelid::regclass::text LIKE 'citus_local_table_1%' AND indexrelid::regcl
 ORDER BY 1;
 
 -- execute truncate & drop commands for multiple relations to see that we don't break local execution
-TRUNCATE citus_local_table_1, citus_local_table_2, distributed_table, local_table, reference_table;
+TRUNCATE citus_local_table_1, citus_local_table_2, distributed_table, local_table, reference_table, local_table_4;
 
 -- test vacuum
 VACUUM citus_local_table_1;
 VACUUM citus_local_table_1, distributed_table, local_table, reference_table;
 
 -- test drop
-DROP TABLE citus_local_table_1, citus_local_table_2, distributed_table, local_table, reference_table;
+DROP TABLE citus_local_table_1, citus_local_table_2, distributed_table, local_table, reference_table, local_table_4;
 
 -- test some other udf's with citus local tables
 
@@ -418,7 +418,6 @@ ROLLBACK;
 
 SELECT update_distributed_table_colocation('citus_local_table_4', colocate_with => 'none');
 
-SELECT master_create_worker_shards('citus_local_table_4', 10, 1);
 SELECT master_create_empty_shard('citus_local_table_4');
 SELECT master_apply_delete_command('DELETE FROM citus_local_table_4');
 

src/test/regress/sql/fkeys_between_local_ref.sql

@@ -175,5 +175,146 @@ BEGIN;
   ALTER TABLE local_table_2 ADD CONSTRAINT fkey_11 FOREIGN KEY (col_1) REFERENCES reference_table_2(col_1) ON UPDATE CASCADE;
 ROLLBACK;
 
+--
+-- create table tests
+--
+
+BEGIN;
+  CREATE TABLE local_table_6 (col_1 INT PRIMARY KEY);
+
+  -- create a table that references to
+  --  * local table graph
+  --  * reference table
+  --  * another local table
+  --  * itself
+  CREATE TABLE local_table_5 (
+    col_1 INT UNIQUE REFERENCES local_table_1(col_1),
+    col_2 INT,
+    FOREIGN KEY (col_1) REFERENCES reference_table_1(col_1),
+    -- not specify column for this foreign key
+    FOREIGN KEY (col_2) REFERENCES local_table_6,
+    -- also have a self reference
+    FOREIGN KEY (col_2) REFERENCES local_table_5(col_1));
+
+  -- now print metadata to show that all local tables are converted
+  -- to citus local tables
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ROLLBACK;
+
+CREATE TABLE distributed_table (col_1 INT PRIMARY KEY);
+SELECT create_distributed_table('distributed_table', 'col_1');
+
+-- Creating a table that both references to a reference table and a
+-- distributed table fails.
+-- This is because, we convert local table to a citus local table
+-- due to its foreign key to reference table.
+-- But citus local tables can't have foreign keys to distributed tables.
+CREATE TABLE local_table_5 (
+  col_1 INT UNIQUE REFERENCES distributed_table(col_1),
+  FOREIGN KEY (col_1) REFERENCES reference_table_1(col_1));
+
+BEGIN;
+  ALTER TABLE distributed_table ADD CONSTRAINT fkey_11 FOREIGN KEY (col_1) REFERENCES reference_table_1(col_1);
+
+  CREATE TABLE local_table_5 (
+    col_1 INT UNIQUE REFERENCES reference_table_1(col_1),
+    FOREIGN KEY (col_1) REFERENCES local_table_1(col_1));
+
+  INSERT INTO local_table_5 SELECT i FROM generate_series(195, 205) i;
+
+  -- Now show that when converting local table to a citus local table,
+  -- distributed table (that is referenced by reference table) stays as is.
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+
+  -- show that we validate foreign key constraints, errors out
+  INSERT INTO local_table_5 VALUES (300);
+ROLLBACK;
+
+BEGIN;
+  CREATE SCHEMA another_schema_fkeys_between_local_ref;
+  CREATE TABLE another_schema_fkeys_between_local_ref.local_table_6 (col_1 INT PRIMARY KEY);
+
+  -- first convert local tables to citus local tables in graph
+  ALTER TABLE local_table_2 ADD CONSTRAINT fkey_11 FOREIGN KEY (col_1) REFERENCES reference_table_1(col_1) ON DELETE CASCADE;
+
+  CREATE TABLE local_table_5 (
+    col_1 INT UNIQUE REFERENCES another_schema_fkeys_between_local_ref.local_table_6(col_1) CHECK (col_1 > 0),
+    col_2 INT REFERENCES local_table_3(col_1),
+    FOREIGN KEY (col_1) REFERENCES local_table_5(col_1));
+
+  -- Now show that we converted local_table_5 & 6 to citus local tables
+  -- as local_table_5 has foreign key to a citus local table too
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref' UNION
+                               SELECT 'another_schema_fkeys_between_local_ref.local_table_6')
+  ORDER BY tablename;
+ROLLBACK;
+
+BEGIN;
+  CREATE TABLE local_table_6 (col_1 INT PRIMARY KEY);
+  -- first convert local tables to citus local tables in graph
+  ALTER TABLE local_table_2 ADD CONSTRAINT fkey_11 FOREIGN KEY (col_1) REFERENCES reference_table_1(col_1) ON DELETE CASCADE;
+
+  -- create a table that references to
+  --  * citus local table graph (via local_table_1)
+  --  * another local table (local_table_6)
+  --  * itself
+  CREATE TABLE local_table_5 (
+    col_1 INT UNIQUE REFERENCES local_table_1(col_1),
+    col_2 INT CHECK (col_2 > 0),
+    -- not specify column for this foreign key
+    FOREIGN KEY (col_2) REFERENCES local_table_6,
+    FOREIGN KEY (col_2) REFERENCES local_table_5(col_1));
+
+  -- now print metadata to show that all local tables are converted
+  -- to citus local tables
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+
+  -- now make some of them reference tables
+  SELECT create_reference_table('local_table_2');
+  SELECT create_reference_table('local_table_6');
+
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ROLLBACK;
+
+BEGIN;
+  -- disable foreign keys to reference tables
+  SET LOCAL citus.enable_local_reference_table_foreign_keys TO false;
+  CREATE TABLE local_table_6 (col_1 INT PRIMARY KEY);
+
+  ALTER TABLE local_table_2 ADD CONSTRAINT fkey_11 FOREIGN KEY (col_1) REFERENCES reference_table_1(col_1) ON DELETE CASCADE;
+
+  CREATE TABLE local_table_5 (
+    col_1 INT UNIQUE REFERENCES local_table_6(col_1),
+    col_2 INT REFERENCES local_table_3(col_1),
+    FOREIGN KEY (col_1) REFERENCES local_table_5(col_1),
+    FOREIGN KEY (col_1) REFERENCES reference_table_1(col_1));
+
+  -- Now show none of local_table_5 & 6 to should be converted to citus local tables
+  -- as it is disabled
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ROLLBACK;
+
+-- this errors out as we don't support creating citus local
+-- tables from partitioned tables
+CREATE TABLE part_local_table (col_1 INT REFERENCES reference_table_1(col_1)) PARTITION BY RANGE (col_1);
+
+-- they fail as col_99 does not exist
+CREATE TABLE local_table_5 (col_1 INT, FOREIGN KEY (col_99) REFERENCES reference_table_1(col_1));
+CREATE TABLE local_table_5 (col_1 INT, FOREIGN KEY (col_1) REFERENCES reference_table_1(col_99));
+
+-- fails as referenced table does not exist
+CREATE TABLE local_table_5 (col_1 INT, FOREIGN KEY (col_1) REFERENCES table_does_not_exist(dummy));
+
 -- cleanup at exit
 DROP SCHEMA fkeys_between_local_ref CASCADE;

src/test/regress/sql/foreign_key_to_reference_table.sql

@@ -9,6 +9,8 @@ SET citus.shard_count TO 8;
 SET citus.next_shard_id TO 7000000;
 SET citus.next_placement_id TO 7000000;
 
+SET client_min_messages TO ERROR;
+
 CREATE TYPE foreign_details AS (name text, relid text, refd_relid text);
 
 CREATE VIEW table_fkeys_in_workers AS
@@ -95,50 +97,50 @@ SELECT create_reference_table('referenced_table');
 CREATE TABLE referencing_table(id int, ref_id int);
 SELECT create_distributed_table('referencing_table', 'ref_id');
 ALTER TABLE referencing_table ADD CONSTRAINT fkey_ref FOREIGN KEY(id) REFERENCES referenced_table(id) ON DELETE SET NULL;
-SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
+SELECT COUNT(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
 DROP TABLE referencing_table;
 
 CREATE TABLE referencing_table(id int, ref_id int, FOREIGN KEY(id) REFERENCES referenced_table(id) ON DELETE SET NULL);
 SELECT create_distributed_table('referencing_table', 'ref_id');
-SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
+SELECT COUNT(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
 DROP TABLE referencing_table;
 
 CREATE TABLE referencing_table(id int, ref_id int);
 SELECT create_distributed_table('referencing_table', 'ref_id');
 ALTER TABLE referencing_table ADD CONSTRAINT fkey_ref FOREIGN KEY(id) REFERENCES referenced_table(id) ON DELETE SET DEFAULT;
-SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
+SELECT COUNT(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
 DROP TABLE referencing_table;
 
 BEGIN;
   CREATE TABLE referencing_table(id int, ref_id int, FOREIGN KEY(id) REFERENCES referenced_table(id) ON DELETE SET DEFAULT);
   SELECT create_distributed_table('referencing_table', 'ref_id');
 COMMIT;
-SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
+SELECT COUNT(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
 DROP TABLE referencing_table;
 
 CREATE TABLE referencing_table(id int, ref_id int);
 SELECT create_distributed_table('referencing_table', 'ref_id');
 ALTER TABLE referencing_table ADD CONSTRAINT fkey_ref FOREIGN KEY(id) REFERENCES referenced_table(id) ON UPDATE SET NULL;
-SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
+SELECT COUNT(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
 DROP TABLE referencing_table;
 
 CREATE TABLE referencing_table(id int, ref_id int);
 SELECT create_distributed_table('referencing_table', 'ref_id');
 ALTER TABLE referencing_table ADD CONSTRAINT fkey_ref FOREIGN KEY(id) REFERENCES referenced_table(id) ON UPDATE SET DEFAULT;
-SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
+SELECT COUNT(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
 DROP TABLE referencing_table;
 
 CREATE TABLE referencing_table(id int, ref_id int);
 SELECT create_distributed_table('referencing_table', 'ref_id');
 ALTER TABLE referencing_table ADD CONSTRAINT fkey_ref FOREIGN KEY(id) REFERENCES referenced_table(id) ON UPDATE CASCADE;
-SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
+SELECT COUNT(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
 DROP TABLE referencing_table;
 
 -- check if we can add the foreign key while adding the column
 CREATE TABLE referencing_table(id int, ref_id int);
 SELECT create_distributed_table('referencing_table', 'ref_id');
 ALTER TABLE referencing_table ADD COLUMN referencing int REFERENCES referenced_table(id) ON UPDATE CASCADE;
-SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
+SELECT COUNT(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
 DROP TABLE referencing_table;
 
 -- foreign keys are only supported when the replication factor = 1
@@ -146,21 +148,21 @@ SET citus.shard_replication_factor TO 2;
 CREATE TABLE referencing_table(id int, ref_id int);
 SELECT create_distributed_table('referencing_table', 'ref_id');
 ALTER TABLE referencing_table ADD CONSTRAINT fkey_ref FOREIGN KEY (id) REFERENCES referenced_table(id);
-SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
+SELECT COUNT(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
 DROP TABLE referencing_table;
 
 -- should fail when we add the column as well
 CREATE TABLE referencing_table(id int, ref_id int);
 SELECT create_distributed_table('referencing_table', 'ref_id');
 ALTER TABLE referencing_table ADD COLUMN referencing_col int REFERENCES referenced_table(id) ON DELETE SET NULL;
-SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
+SELECT COUNT(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
 DROP TABLE referencing_table;
 SET citus.shard_replication_factor TO 1;
 
 -- simple create_distributed_table should work in/out transactions on tables with foreign key to reference tables
 CREATE TABLE referencing_table(id int, ref_id int, FOREIGN KEY (id) REFERENCES referenced_table(id));
 SELECT create_distributed_table('referencing_table', 'ref_id');
-SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
+SELECT COUNT(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
 DROP TABLE referencing_table;
 DROP TABLE referenced_table;
 
@@ -170,7 +172,7 @@ BEGIN;
   CREATE TABLE referencing_table(id int, ref_id int, FOREIGN KEY (id) REFERENCES referenced_table(id));
   SELECT create_distributed_table('referencing_table', 'ref_id');
 COMMIT;
-SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
+SELECT COUNT(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
 DROP TABLE referencing_table;
 
 -- foreign keys are supported either in between distributed tables including the
@@ -416,7 +418,7 @@ SELECT create_distributed_table('referencing_table', 'id');
 ALTER TABLE referencing_table ADD CONSTRAINT fkey_ref FOREIGN KEY (id) REFERENCES referenced_table(test_column) ON DELETE CASCADE;
 ALTER TABLE referencing_table ADD CONSTRAINT foreign_key_2 FOREIGN KEY (id) REFERENCES referenced_table2(test_column2) ON DELETE CASCADE;
 
-SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
+SELECT COUNT(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
 
 INSERT INTO referenced_table SELECT x, x+1 FROM generate_series(0,1000) AS f(x);
 INSERT INTO referenced_table2 SELECT x, x+1 FROM generate_series(500,1500) AS f(x);
@@ -473,7 +475,7 @@ BEGIN;
   ALTER TABLE referencing_table ADD CONSTRAINT foreign_key_2 FOREIGN KEY (ref_id) REFERENCES referenced_table2(test_column2) ON DELETE CASCADE;
 COMMIT;
 
-SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
+SELECT COUNT(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
 
 INSERT INTO referenced_table SELECT x, x+1 FROM generate_series(0,1000) AS f(x);
 INSERT INTO referenced_table2 SELECT x, x+1 FROM generate_series(500,1500) AS f(x);
@@ -537,7 +539,7 @@ ALTER TABLE referencing_table2 ADD CONSTRAINT fkey_ref FOREIGN KEY (ref_id) REFE
 ALTER TABLE referencing_table2 ADD CONSTRAINT fkey_ref_to_dist FOREIGN KEY (id) REFERENCES referencing_table(id) ON DELETE CASCADE;
 COMMIT;
 
-SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%' ORDER BY 1,2,3;
+SELECT COUNT(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
 
 INSERT INTO referenced_table SELECT x, x+1 FROM generate_series(0,1000) AS f(x);
 -- should fail
@@ -593,7 +595,7 @@ SELECT create_distributed_table('referencing_table', 'id');
 SELECT create_distributed_table('referencing_referencing_table', 'id');
 ALTER TABLE referencing_table ADD CONSTRAINT fkey_ref FOREIGN KEY (ref_id, ref_id2) REFERENCES referenced_table(test_column, test_column2) ON DELETE CASCADE;
 
-SELECT * FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.referencing%' ORDER BY 1,2,3;
+SELECT COUNT(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.referencing%';
 
 INSERT INTO referenced_table SELECT x, x+1 FROM generate_series(1,1000) AS f(x);
 INSERT INTO referencing_table SELECT x, x+1, x+2 FROM generate_series(1,999) AS f(x);
@@ -606,22 +608,6 @@ DROP TABLE referenced_table CASCADE;
 DROP TABLE referencing_table CASCADE;
 DROP TABLE referencing_referencing_table;
 
--- test if create_distributed_table works in transactions with some edge cases
--- the following checks if create_distributed_table works on foreign keys when
--- one of them is a self-referencing table of multiple distributed tables
-BEGIN;
-  CREATE TABLE test_table_1(id int PRIMARY KEY);
-  SELECT create_reference_table('test_table_1');
-
-  CREATE TABLE test_table_2(id int PRIMARY KEY, value_1 int, FOREIGN KEY(id) REFERENCES test_table_1(id));
-  SELECT create_distributed_table('test_table_2', 'id');
-
-  CREATE TABLE test_table_3(id int PRIMARY KEY, value_1 int, FOREIGN KEY(value_1) REFERENCES test_table_1(id), FOREIGN KEY(id) REFERENCES test_table_2(id));
-  SELECT create_distributed_table('test_table_3', 'id');
-
-  DROP TABLE test_table_1 CASCADE;
-ROLLBACK;
-
 -- create_reference_table, create_distributed_table and ALTER TABLE in the same transaction
 BEGIN;
   CREATE TABLE test_table_1(id int PRIMARY KEY);