Convert postgres tables to citus local when creating reference table having fkeys

src/backend/distributed/commands/create_distributed_table.c

@@ -44,6 +44,7 @@
 #include "distributed/deparser.h"
 #include "distributed/distribution_column.h"
 #include "distributed/listutils.h"
+#include "distributed/local_executor.h"
 #include "distributed/metadata_utility.h"
 #include "distributed/coordinator_protocol.h"
 #include "distributed/metadata/dependency.h"
@@ -365,6 +366,40 @@ CreateDistributedTable(Oid relationId, Var *distributionColumn, char distributio
 		relationId = DropFKeysAndUndistributeTable(relationId);
 	}
 
+	/*
+	 * To support foreign keys between reference tables and local tables,
+	 * we drop & re-define foreign keys at the end of this function so
+	 * that ALTER TABLE hook does the necessary job, which means converting
+	 * local tables to citus local tables to properly support such foreign
+	 * keys.
+	 *
+	 * This function does not expect to create Citus local table, so we blindly
+	 * create reference table when the method is DISTRIBUTE_BY_NONE.
+	 */
+	else if (distributionMethod == DISTRIBUTE_BY_NONE &&
+			 ShouldEnableLocalReferenceForeignKeys() &&
+			 HasForeignKeyWithLocalTable(relationId))
+	{
+		/*
+		 * Store foreign key creation commands for foreign key relationships
+		 * that relation has with postgres tables.
+		 */
+		originalForeignKeyRecreationCommands =
+			GetFKeyCreationCommandsRelationInvolvedWithTableType(relationId,
+																 INCLUDE_LOCAL_TABLES);
+
+		/*
+		 * Soon we will convert local tables to citus local tables. As
+		 * CreateCitusLocalTable needs to use local execution, now we
+		 * switch to local execution beforehand so that reference table
+		 * creation doesn't use remote execution and we don't error out
+		 * in CreateCitusLocalTable.
+		 */
+		SetLocalExecutionStatus(LOCAL_EXECUTION_REQUIRED);
+
+		DropFKeysRelationInvolvedWithTableType(relationId, INCLUDE_LOCAL_TABLES);
+	}
+
 	/*
 	 * distributed tables might have dependencies on different objects, since we create
 	 * shards for a distributed table via multiple sessions these objects will be created
@@ -428,6 +463,10 @@ CreateDistributedTable(Oid relationId, Var *distributionColumn, char distributio
 	}
 	else if (distributionMethod == DISTRIBUTE_BY_NONE)
 	{
+		/*
+		 * This function does not expect to create Citus local table, so we blindly
+		 * create reference table when the method is DISTRIBUTE_BY_NONE.
+		 */
 		CreateReferenceTableShard(relationId);
 	}
 
@@ -474,7 +513,8 @@ CreateDistributedTable(Oid relationId, Var *distributionColumn, char distributio
 	 * we can skip the validation of the foreign keys.
 	 */
 	bool skip_validation = true;
-	ExecuteForeignKeyCreateCommandList(droppedFKeyCreationCommands, skip_validation);
+	ExecuteForeignKeyCreateCommandList(originalForeignKeyRecreationCommands,
+									   skip_validation);
 }
 
 

src/backend/distributed/commands/foreign_constraint.c

@@ -78,6 +78,7 @@ static void ForeignConstraintFindDistKeys(HeapTuple pgConstraintTuple,
 static List * GetForeignKeyIdsForColumn(char *columnName, Oid relationId,
 										int searchForeignKeyColumnFlags);
 static Oid get_relation_constraint_oid_compat(HeapTuple heapTuple);
+static List * GetForeignKeysWithLocalTables(Oid relationId);
 static bool IsTableTypeIncluded(Oid relationId, int flags);
 static void UpdateConstraintIsValid(Oid constraintId, bool isValid);
 
@@ -711,6 +712,38 @@ get_relation_constraint_oid_compat(HeapTuple heapTuple)
 }
 
 
+/*
+ * HasForeignKeyToLocalTable returns true if relation has foreign key
+ * relationship with a local table.
+ */
+bool
+HasForeignKeyWithLocalTable(Oid relationId)
+{
+	List *foreignKeysWithLocalTables = GetForeignKeysWithLocalTables(relationId);
+	return list_length(foreignKeysWithLocalTables) > 0;
+}
+
+
+/*
+ * GetForeignKeysWithLocalTables returns a list foreign keys for foreign key
+ * relationaships that relation has with local tables.
+ */
+static List *
+GetForeignKeysWithLocalTables(Oid relationId)
+{
+	int referencingFKeysFlag = INCLUDE_REFERENCING_CONSTRAINTS |
+							   INCLUDE_LOCAL_TABLES;
+	List *referencingFKeyList = GetForeignKeyOids(relationId, referencingFKeysFlag);
+
+	/* already captured self referencing foreign keys, so use EXCLUDE_SELF_REFERENCES */
+	int referencedFKeysFlag = INCLUDE_REFERENCED_CONSTRAINTS |
+							  EXCLUDE_SELF_REFERENCES |
+							  INCLUDE_LOCAL_TABLES;
+	List *referencedFKeyList = GetForeignKeyOids(relationId, referencedFKeysFlag);
+	return list_concat(referencingFKeyList, referencedFKeyList);
+}
+
+
 /*
  * HasForeignKeyToCitusLocalTable returns true if any of the foreign key constraints
  * on the relation with relationId references to a citus local table.

src/include/distributed/commands.h

@@ -187,6 +187,7 @@ extern List * GetForeignConstraintToReferenceTablesCommands(Oid relationId);
 extern List * GetForeignConstraintToDistributedTablesCommands(Oid relationId);
 extern List * GetForeignConstraintFromDistributedTablesCommands(Oid relationId);
 extern List * GetForeignConstraintCommandsInternal(Oid relationId, int flags);
+extern bool HasForeignKeyWithLocalTable(Oid relationId);
 extern bool HasForeignKeyToCitusLocalTable(Oid relationId);
 extern bool HasForeignKeyToReferenceTable(Oid relationOid);
 extern bool TableReferenced(Oid relationOid);

src/test/regress/expected/fkeys_between_local_ref.out

@@ -481,6 +481,145 @@ BEGIN;
 (2 rows)
 
 ROLLBACK;
+-- converting any local table to a citus local table in graph converts
+-- other tables to citus local tables, test this in below xact blocks
+BEGIN;
+  SELECT create_reference_table('local_table_1');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SELECT create_distributed_table('local_table_2', 'col_1');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+     tablename     | partmethod | repmodel
+---------------------------------------------------------------------
+ distributed_table | h          | c
+ local_table_1     | n          | t
+ local_table_2     | h          | c
+ local_table_3     | n          | c
+ local_table_4     | n          | c
+ reference_table_1 | n          | t
+(6 rows)
+
+ROLLBACK;
+BEGIN;
+  SELECT create_reference_table('local_table_4');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SELECT create_reference_table('local_table_3');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+     tablename     | partmethod | repmodel
+---------------------------------------------------------------------
+ distributed_table | h          | c
+ local_table_1     | n          | c
+ local_table_2     | n          | c
+ local_table_3     | n          | t
+ local_table_4     | n          | t
+ reference_table_1 | n          | t
+(6 rows)
+
+ROLLBACK;
+BEGIN;
+  CREATE TABLE local_table_5 (col_1 INT REFERENCES local_table_1(col_1));
+  SELECT create_reference_table('local_table_1');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SELECT create_distributed_table('local_table_2', 'col_1');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SELECT create_distributed_table('local_table_5', 'col_1');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+     tablename     | partmethod | repmodel
+---------------------------------------------------------------------
+ distributed_table | h          | c
+ local_table_1     | n          | t
+ local_table_2     | h          | c
+ local_table_3     | n          | c
+ local_table_4     | n          | c
+ local_table_5     | h          | c
+ reference_table_1 | n          | t
+(7 rows)
+
+ROLLBACK;
+BEGIN;
+  ALTER TABLE local_table_1 ADD CONSTRAINT fkey_13 FOREIGN KEY (col_1) REFERENCES local_table_2(col_1) ON DELETE CASCADE;
+  -- errors out as foreign keys from reference tables to citus local tables
+  -- cannot have CASCADE behavior
+  SELECT create_reference_table('local_table_1');
+ERROR:  cannot define foreign key constraint, foreign keys from reference tables to citus local tables can only be defined with NO ACTION or RESTRICT behaviors
+ROLLBACK;
+SET citus.enable_local_execution TO OFF;
+-- show that this errors out as it tries to convert connected relations to citus
+-- local tables and creating citus local table requires local execution but local
+-- execution is disabled
+SELECT create_reference_table('local_table_1');
+ERROR:  cannot execute command because a local execution has accessed a placement in the transaction
+SET citus.enable_local_execution TO ON;
+-- test behavior when outside of the xact block
+CREATE TABLE local_table_6 (col_1 INT REFERENCES local_table_1(col_1));
+SELECT create_reference_table('local_table_1');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+SELECT create_distributed_table('local_table_2', 'col_1');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+SELECT create_distributed_table('local_table_6', 'col_1');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+ORDER BY tablename;
+     tablename     | partmethod | repmodel
+---------------------------------------------------------------------
+ distributed_table | h          | c
+ local_table_1     | n          | t
+ local_table_2     | h          | c
+ local_table_3     | n          | c
+ local_table_4     | n          | c
+ local_table_6     | h          | c
+ reference_table_1 | n          | t
+(7 rows)
+
 -- this errors out as we don't support creating citus local
 -- tables from partitioned tables
 CREATE TABLE part_local_table (col_1 INT REFERENCES reference_table_1(col_1)) PARTITION BY RANGE (col_1);
@@ -493,5 +632,148 @@ ERROR:  column "col_99" referenced in foreign key constraint does not exist
 -- fails as referenced table does not exist
 CREATE TABLE local_table_5 (col_1 INT, FOREIGN KEY (col_1) REFERENCES table_does_not_exist(dummy));
 ERROR:  relation "table_does_not_exist" does not exist
+-- drop & recreate schema to prevent noise in next test outputs
+DROP SCHEMA fkeys_between_local_ref CASCADE;
+CREATE SCHEMA fkeys_between_local_ref;
+SET search_path TO fkeys_between_local_ref;
+-- now have some tests to test behavior before/after enabling foreign keys
+-- between local tables & reference tables
+BEGIN;
+  SET citus.enable_local_reference_table_foreign_keys TO OFF;
+  CREATE TABLE ref_1(a int PRIMARY KEY);
+  CREATE TABLE pg_local_1(a int PRIMARY KEY REFERENCES ref_1(a));
+  SELECT create_reference_table('ref_1');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SET citus.enable_local_reference_table_foreign_keys TO ON;
+  CREATE TABLE ref_2(a int PRIMARY KEY);
+  SELECT create_reference_table('ref_2');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  ALTER TABLE pg_local_1 ADD CONSTRAINT c1 FOREIGN KEY(a) REFERENCES ref_2(a);
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ tablename  | partmethod | repmodel
+---------------------------------------------------------------------
+ pg_local_1 | n          | c
+ ref_1      | n          | t
+ ref_2      | n          | t
+(3 rows)
+
+ROLLBACK;
+BEGIN;
+  SET citus.enable_local_reference_table_foreign_keys TO OFF;
+  CREATE TABLE ref_1(a int PRIMARY KEY);
+  CREATE TABLE pg_local_1(a int PRIMARY KEY REFERENCES ref_1(a));
+  SELECT create_reference_table('ref_1');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SET citus.enable_local_reference_table_foreign_keys TO ON;
+  CREATE TABLE ref_2(a int PRIMARY KEY REFERENCES pg_local_1(a));
+  SELECT create_reference_table('ref_2');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ tablename  | partmethod | repmodel
+---------------------------------------------------------------------
+ pg_local_1 | n          | c
+ ref_1      | n          | t
+ ref_2      | n          | t
+(3 rows)
+
+ROLLBACK;
+BEGIN;
+  SET citus.enable_local_reference_table_foreign_keys TO OFF;
+  CREATE TABLE ref_1(a int PRIMARY KEY);
+  CREATE TABLE pg_local_1(a int PRIMARY KEY REFERENCES ref_1(a));
+  SELECT create_reference_table('ref_1');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SET citus.enable_local_reference_table_foreign_keys TO ON;
+  CREATE TABLE ref_2(a int PRIMARY KEY REFERENCES pg_local_1(a));
+  SELECT create_reference_table('ref_2');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ tablename  | partmethod | repmodel
+---------------------------------------------------------------------
+ pg_local_1 | n          | c
+ ref_1      | n          | t
+ ref_2      | n          | t
+(3 rows)
+
+ROLLBACK;
+BEGIN;
+  SET citus.enable_local_reference_table_foreign_keys TO OFF;
+  CREATE TABLE ref_1(a int PRIMARY KEY);
+  CREATE TABLE pg_local_1(a int PRIMARY KEY REFERENCES ref_1(a));
+  SELECT create_reference_table('ref_1');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SET citus.enable_local_reference_table_foreign_keys TO ON;
+  CREATE TABLE pg_local_2(a int PRIMARY KEY REFERENCES pg_local_1(a));
+  -- we still didn't convert local tables to citus local tables
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ tablename | partmethod | repmodel
+---------------------------------------------------------------------
+ ref_1     | n          | t
+(1 row)
+
+  CREATE TABLE pg_local_3(a int PRIMARY KEY REFERENCES ref_1(a));
+  -- pg_local_3 is not connected to other local tables, so we will just
+  -- convert pg_local_3 to a citus local table
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ tablename  | partmethod | repmodel
+---------------------------------------------------------------------
+ pg_local_3 | n          | c
+ ref_1      | n          | t
+(2 rows)
+
+  CREATE TABLE pg_local_4(a int PRIMARY KEY REFERENCES ref_1(a), FOREIGN KEY (a) REFERENCES pg_local_2(a));
+  -- pg_local_4 is connected to ref_1, pg_local_1 and pg_local_2,
+  -- so we will convert those two local tables to citus local tables too
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ tablename  | partmethod | repmodel
+---------------------------------------------------------------------
+ pg_local_1 | n          | c
+ pg_local_2 | n          | c
+ pg_local_3 | n          | c
+ pg_local_4 | n          | c
+ ref_1      | n          | t
+(5 rows)
+
+ROLLBACK;
 -- cleanup at exit
 DROP SCHEMA fkeys_between_local_ref CASCADE;

src/test/regress/sql/fkeys_between_local_ref.sql

@@ -341,6 +341,70 @@ BEGIN;
   ORDER BY tablename;
 ROLLBACK;
 
+-- converting any local table to a citus local table in graph converts
+-- other tables to citus local tables, test this in below xact blocks
+
+BEGIN;
+  SELECT create_reference_table('local_table_1');
+
+  SELECT create_distributed_table('local_table_2', 'col_1');
+
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ROLLBACK;
+
+BEGIN;
+  SELECT create_reference_table('local_table_4');
+
+  SELECT create_reference_table('local_table_3');
+
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ROLLBACK;
+
+BEGIN;
+  CREATE TABLE local_table_5 (col_1 INT REFERENCES local_table_1(col_1));
+
+  SELECT create_reference_table('local_table_1');
+
+  SELECT create_distributed_table('local_table_2', 'col_1');
+  SELECT create_distributed_table('local_table_5', 'col_1');
+
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ROLLBACK;
+
+BEGIN;
+  ALTER TABLE local_table_1 ADD CONSTRAINT fkey_13 FOREIGN KEY (col_1) REFERENCES local_table_2(col_1) ON DELETE CASCADE;
+
+  -- errors out as foreign keys from reference tables to citus local tables
+  -- cannot have CASCADE behavior
+  SELECT create_reference_table('local_table_1');
+ROLLBACK;
+
+SET citus.enable_local_execution TO OFF;
+-- show that this errors out as it tries to convert connected relations to citus
+-- local tables and creating citus local table requires local execution but local
+-- execution is disabled
+SELECT create_reference_table('local_table_1');
+SET citus.enable_local_execution TO ON;
+
+-- test behavior when outside of the xact block
+
+CREATE TABLE local_table_6 (col_1 INT REFERENCES local_table_1(col_1));
+
+SELECT create_reference_table('local_table_1');
+
+SELECT create_distributed_table('local_table_2', 'col_1');
+SELECT create_distributed_table('local_table_6', 'col_1');
+
+SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+ORDER BY tablename;
+
 -- this errors out as we don't support creating citus local
 -- tables from partitioned tables
 CREATE TABLE part_local_table (col_1 INT REFERENCES reference_table_1(col_1)) PARTITION BY RANGE (col_1);
@@ -352,5 +416,98 @@ CREATE TABLE local_table_5 (col_1 INT, FOREIGN KEY (col_1) REFERENCES reference_
 -- fails as referenced table does not exist
 CREATE TABLE local_table_5 (col_1 INT, FOREIGN KEY (col_1) REFERENCES table_does_not_exist(dummy));
 
+-- drop & recreate schema to prevent noise in next test outputs
+DROP SCHEMA fkeys_between_local_ref CASCADE;
+CREATE SCHEMA fkeys_between_local_ref;
+SET search_path TO fkeys_between_local_ref;
+
+-- now have some tests to test behavior before/after enabling foreign keys
+-- between local tables & reference tables
+
+BEGIN;
+  SET citus.enable_local_reference_table_foreign_keys TO OFF;
+
+  CREATE TABLE ref_1(a int PRIMARY KEY);
+  CREATE TABLE pg_local_1(a int PRIMARY KEY REFERENCES ref_1(a));
+  SELECT create_reference_table('ref_1');
+
+  SET citus.enable_local_reference_table_foreign_keys TO ON;
+
+  CREATE TABLE ref_2(a int PRIMARY KEY);
+  SELECT create_reference_table('ref_2');
+  ALTER TABLE pg_local_1 ADD CONSTRAINT c1 FOREIGN KEY(a) REFERENCES ref_2(a);
+
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ROLLBACK;
+
+BEGIN;
+  SET citus.enable_local_reference_table_foreign_keys TO OFF;
+
+  CREATE TABLE ref_1(a int PRIMARY KEY);
+  CREATE TABLE pg_local_1(a int PRIMARY KEY REFERENCES ref_1(a));
+  SELECT create_reference_table('ref_1');
+
+  SET citus.enable_local_reference_table_foreign_keys TO ON;
+
+  CREATE TABLE ref_2(a int PRIMARY KEY REFERENCES pg_local_1(a));
+  SELECT create_reference_table('ref_2');
+
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ROLLBACK;
+
+BEGIN;
+  SET citus.enable_local_reference_table_foreign_keys TO OFF;
+
+  CREATE TABLE ref_1(a int PRIMARY KEY);
+  CREATE TABLE pg_local_1(a int PRIMARY KEY REFERENCES ref_1(a));
+  SELECT create_reference_table('ref_1');
+
+  SET citus.enable_local_reference_table_foreign_keys TO ON;
+
+  CREATE TABLE ref_2(a int PRIMARY KEY REFERENCES pg_local_1(a));
+  SELECT create_reference_table('ref_2');
+
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ROLLBACK;
+
+BEGIN;
+  SET citus.enable_local_reference_table_foreign_keys TO OFF;
+
+  CREATE TABLE ref_1(a int PRIMARY KEY);
+  CREATE TABLE pg_local_1(a int PRIMARY KEY REFERENCES ref_1(a));
+  SELECT create_reference_table('ref_1');
+
+  SET citus.enable_local_reference_table_foreign_keys TO ON;
+
+  CREATE TABLE pg_local_2(a int PRIMARY KEY REFERENCES pg_local_1(a));
+
+  -- we still didn't convert local tables to citus local tables
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+
+  CREATE TABLE pg_local_3(a int PRIMARY KEY REFERENCES ref_1(a));
+
+  -- pg_local_3 is not connected to other local tables, so we will just
+  -- convert pg_local_3 to a citus local table
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+
+  CREATE TABLE pg_local_4(a int PRIMARY KEY REFERENCES ref_1(a), FOREIGN KEY (a) REFERENCES pg_local_2(a));
+
+  -- pg_local_4 is connected to ref_1, pg_local_1 and pg_local_2,
+  -- so we will convert those two local tables to citus local tables too
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ROLLBACK;
+
 -- cleanup at exit
 DROP SCHEMA fkeys_between_local_ref CASCADE;