Merge pull request #4539 from citusdata/auto-citus-local-when-create-ref

Convert postgres tables to citus local when creating reference table having fkeys

src/backend/distributed/commands/create_distributed_table.c

@@ -44,6 +44,7 @@
 #include "distributed/deparser.h"
 #include "distributed/distribution_column.h"
 #include "distributed/listutils.h"
+#include "distributed/local_executor.h"
 #include "distributed/metadata_utility.h"
 #include "distributed/coordinator_protocol.h"
 #include "distributed/metadata/dependency.h"
@@ -114,9 +115,10 @@ static void EnsureLocalTableEmptyIfNecessary(Oid relationId, char distributionMe
 static bool ShouldLocalTableBeEmpty(Oid relationId, char distributionMethod, bool
 									viaDeprecatedAPI);
 static void EnsureCitusTableCanBeCreated(Oid relationOid);
-static List * GetFKeyCreationCommandsRelationInvolved(Oid relationId);
+static List * GetFKeyCreationCommandsRelationInvolvedWithTableType(Oid relationId,
+																   int tableTypeFlag);
 static Oid DropFKeysAndUndistributeTable(Oid relationId);
-static void DropFKeysRelationInvolved(Oid relationId);
+static void DropFKeysRelationInvolvedWithTableType(Oid relationId, int tableTypeFlag);
 static bool LocalTableEmpty(Oid tableId);
 static void CopyLocalDataIntoShards(Oid relationId);
 static List * TupleDescColumnNameList(TupleDesc tupleDescriptor);
@@ -354,15 +356,50 @@ CreateDistributedTable(Oid relationId, Var *distributionColumn, char distributio
 	 * given relation is involved, then we undistribute the relation and finally
 	 * we re-create dropped foreign keys at the end of this function.
 	 */
-	List *fKeyCreationCommandsRelationInvolved = NIL;
+	List *originalForeignKeyRecreationCommands = NIL;
 	if (IsCitusTableType(relationId, CITUS_LOCAL_TABLE))
 	{
 		/* store foreign key creation commands that relation is involved */
-		fKeyCreationCommandsRelationInvolved =
+		originalForeignKeyRecreationCommands =
-			GetFKeyCreationCommandsRelationInvolved(relationId);
+			GetFKeyCreationCommandsRelationInvolvedWithTableType(relationId,
+																 INCLUDE_ALL_TABLE_TYPES);
 		relationId = DropFKeysAndUndistributeTable(relationId);
 	}
 
+	/*
+	 * To support foreign keys between reference tables and local tables,
+	 * we drop & re-define foreign keys at the end of this function so
+	 * that ALTER TABLE hook does the necessary job, which means converting
+	 * local tables to citus local tables to properly support such foreign
+	 * keys.
+	 *
+	 * This function does not expect to create Citus local table, so we blindly
+	 * create reference table when the method is DISTRIBUTE_BY_NONE.
+	 */
+	else if (distributionMethod == DISTRIBUTE_BY_NONE &&
+			 ShouldEnableLocalReferenceForeignKeys() &&
+			 HasForeignKeyWithLocalTable(relationId))
+	{
+		/*
+		 * Store foreign key creation commands for foreign key relationships
+		 * that relation has with postgres tables.
+		 */
+		originalForeignKeyRecreationCommands =
+			GetFKeyCreationCommandsRelationInvolvedWithTableType(relationId,
+																 INCLUDE_LOCAL_TABLES);
+
+		/*
+		 * Soon we will convert local tables to citus local tables. As
+		 * CreateCitusLocalTable needs to use local execution, now we
+		 * switch to local execution beforehand so that reference table
+		 * creation doesn't use remote execution and we don't error out
+		 * in CreateCitusLocalTable.
+		 */
+		SetLocalExecutionStatus(LOCAL_EXECUTION_REQUIRED);
+
+		DropFKeysRelationInvolvedWithTableType(relationId, INCLUDE_LOCAL_TABLES);
+	}
+
 	/*
 	 * distributed tables might have dependencies on different objects, since we create
 	 * shards for a distributed table via multiple sessions these objects will be created
@@ -426,6 +463,10 @@ CreateDistributedTable(Oid relationId, Var *distributionColumn, char distributio
 	}
 	else if (distributionMethod == DISTRIBUTE_BY_NONE)
 	{
+		/*
+		 * This function does not expect to create Citus local table, so we blindly
+		 * create reference table when the method is DISTRIBUTE_BY_NONE.
+		 */
 		CreateReferenceTableShard(relationId);
 	}
 
@@ -472,27 +513,28 @@ CreateDistributedTable(Oid relationId, Var *distributionColumn, char distributio
 	 * we can skip the validation of the foreign keys.
 	 */
 	bool skip_validation = true;
-	ExecuteForeignKeyCreateCommandList(fKeyCreationCommandsRelationInvolved,
+	ExecuteForeignKeyCreateCommandList(originalForeignKeyRecreationCommands,
 									   skip_validation);
 }
 
 
 /*
- * GetFKeyCreationCommandsRelationInvolved returns a list of DDL commands to
+ * GetFKeyCreationCommandsRelationInvolvedWithTableType returns a list of DDL
- * recreate the foreign keys that relation with relationId is involved.
+ * commands to recreate the foreign keys that relation with relationId is involved
+ * with given table type.
  */
 static List *
-GetFKeyCreationCommandsRelationInvolved(Oid relationId)
+GetFKeyCreationCommandsRelationInvolvedWithTableType(Oid relationId, int tableTypeFlag)
 {
 	int referencingFKeysFlag = INCLUDE_REFERENCING_CONSTRAINTS |
-							   INCLUDE_ALL_TABLE_TYPES;
+							   tableTypeFlag;
 	List *referencingFKeyCreationCommands =
 		GetForeignConstraintCommandsInternal(relationId, referencingFKeysFlag);
 
 	/* already captured self referencing foreign keys, so use EXCLUDE_SELF_REFERENCES */
 	int referencedFKeysFlag = INCLUDE_REFERENCED_CONSTRAINTS |
 							  EXCLUDE_SELF_REFERENCES |
-							  INCLUDE_ALL_TABLE_TYPES;
+							  tableTypeFlag;
 	List *referencedFKeyCreationCommands =
 		GetForeignConstraintCommandsInternal(relationId, referencedFKeysFlag);
 	return list_concat(referencingFKeyCreationCommands, referencedFKeyCreationCommands);
@@ -510,7 +552,7 @@ GetFKeyCreationCommandsRelationInvolved(Oid relationId)
 static Oid
 DropFKeysAndUndistributeTable(Oid relationId)
 {
-	DropFKeysRelationInvolved(relationId);
+	DropFKeysRelationInvolvedWithTableType(relationId, INCLUDE_ALL_TABLE_TYPES);
 
 	/* store them before calling UndistributeTable as it changes relationId */
 	char *relationName = get_rel_name(relationId);
@@ -535,20 +577,20 @@ DropFKeysAndUndistributeTable(Oid relationId)
 
 
 /*
- * DropFKeysRelationInvolved drops all foreign keys that relation with
+ * DropFKeysRelationInvolvedWithTableType drops foreign keys that relation
- * relationId is involved.
+ * with relationId is involved with given table type.
  */
 static void
-DropFKeysRelationInvolved(Oid relationId)
+DropFKeysRelationInvolvedWithTableType(Oid relationId, int tableTypeFlag)
 {
 	int referencingFKeysFlag = INCLUDE_REFERENCING_CONSTRAINTS |
-							   INCLUDE_ALL_TABLE_TYPES;
+							   tableTypeFlag;
 	DropRelationForeignKeys(relationId, referencingFKeysFlag);
 
 	/* already captured self referencing foreign keys, so use EXCLUDE_SELF_REFERENCES */
 	int referencedFKeysFlag = INCLUDE_REFERENCED_CONSTRAINTS |
 							  EXCLUDE_SELF_REFERENCES |
-							  INCLUDE_ALL_TABLE_TYPES;
+							  tableTypeFlag;
 	DropRelationForeignKeys(relationId, referencedFKeysFlag);
 }
 

src/backend/distributed/commands/foreign_constraint.c

@@ -78,6 +78,7 @@ static void ForeignConstraintFindDistKeys(HeapTuple pgConstraintTuple,
 static List * GetForeignKeyIdsForColumn(char *columnName, Oid relationId,
 										int searchForeignKeyColumnFlags);
 static Oid get_relation_constraint_oid_compat(HeapTuple heapTuple);
+static List * GetForeignKeysWithLocalTables(Oid relationId);
 static bool IsTableTypeIncluded(Oid relationId, int flags);
 static void UpdateConstraintIsValid(Oid constraintId, bool isValid);
 
@@ -711,6 +712,38 @@ get_relation_constraint_oid_compat(HeapTuple heapTuple)
 }
 
 
+/*
+ * HasForeignKeyToLocalTable returns true if relation has foreign key
+ * relationship with a local table.
+ */
+bool
+HasForeignKeyWithLocalTable(Oid relationId)
+{
+	List *foreignKeysWithLocalTables = GetForeignKeysWithLocalTables(relationId);
+	return list_length(foreignKeysWithLocalTables) > 0;
+}
+
+
+/*
+ * GetForeignKeysWithLocalTables returns a list foreign keys for foreign key
+ * relationaships that relation has with local tables.
+ */
+static List *
+GetForeignKeysWithLocalTables(Oid relationId)
+{
+	int referencingFKeysFlag = INCLUDE_REFERENCING_CONSTRAINTS |
+							   INCLUDE_LOCAL_TABLES;
+	List *referencingFKeyList = GetForeignKeyOids(relationId, referencingFKeysFlag);
+
+	/* already captured self referencing foreign keys, so use EXCLUDE_SELF_REFERENCES */
+	int referencedFKeysFlag = INCLUDE_REFERENCED_CONSTRAINTS |
+							  EXCLUDE_SELF_REFERENCES |
+							  INCLUDE_LOCAL_TABLES;
+	List *referencedFKeyList = GetForeignKeyOids(relationId, referencedFKeysFlag);
+	return list_concat(referencingFKeyList, referencedFKeyList);
+}
+
+
 /*
  * HasForeignKeyToCitusLocalTable returns true if any of the foreign key constraints
  * on the relation with relationId references to a citus local table.

src/include/distributed/commands.h

@@ -187,6 +187,7 @@ extern List * GetForeignConstraintToReferenceTablesCommands(Oid relationId);
 extern List * GetForeignConstraintToDistributedTablesCommands(Oid relationId);
 extern List * GetForeignConstraintFromDistributedTablesCommands(Oid relationId);
 extern List * GetForeignConstraintCommandsInternal(Oid relationId, int flags);
+extern bool HasForeignKeyWithLocalTable(Oid relationId);
 extern bool HasForeignKeyToCitusLocalTable(Oid relationId);
 extern bool HasForeignKeyToReferenceTable(Oid relationOid);
 extern bool TableReferenced(Oid relationOid);

src/test/regress/expected/fkeys_between_local_ref.out

@@ -481,6 +481,145 @@ BEGIN;
 (2 rows)
 
 ROLLBACK;
+-- converting any local table to a citus local table in graph converts
+-- other tables to citus local tables, test this in below xact blocks
+BEGIN;
+  SELECT create_reference_table('local_table_1');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SELECT create_distributed_table('local_table_2', 'col_1');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+     tablename     | partmethod | repmodel
+---------------------------------------------------------------------
+ distributed_table | h          | c
+ local_table_1     | n          | t
+ local_table_2     | h          | c
+ local_table_3     | n          | c
+ local_table_4     | n          | c
+ reference_table_1 | n          | t
+(6 rows)
+
+ROLLBACK;
+BEGIN;
+  SELECT create_reference_table('local_table_4');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SELECT create_reference_table('local_table_3');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+     tablename     | partmethod | repmodel
+---------------------------------------------------------------------
+ distributed_table | h          | c
+ local_table_1     | n          | c
+ local_table_2     | n          | c
+ local_table_3     | n          | t
+ local_table_4     | n          | t
+ reference_table_1 | n          | t
+(6 rows)
+
+ROLLBACK;
+BEGIN;
+  CREATE TABLE local_table_5 (col_1 INT REFERENCES local_table_1(col_1));
+  SELECT create_reference_table('local_table_1');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SELECT create_distributed_table('local_table_2', 'col_1');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SELECT create_distributed_table('local_table_5', 'col_1');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+     tablename     | partmethod | repmodel
+---------------------------------------------------------------------
+ distributed_table | h          | c
+ local_table_1     | n          | t
+ local_table_2     | h          | c
+ local_table_3     | n          | c
+ local_table_4     | n          | c
+ local_table_5     | h          | c
+ reference_table_1 | n          | t
+(7 rows)
+
+ROLLBACK;
+BEGIN;
+  ALTER TABLE local_table_1 ADD CONSTRAINT fkey_13 FOREIGN KEY (col_1) REFERENCES local_table_2(col_1) ON DELETE CASCADE;
+  -- errors out as foreign keys from reference tables to citus local tables
+  -- cannot have CASCADE behavior
+  SELECT create_reference_table('local_table_1');
+ERROR:  cannot define foreign key constraint, foreign keys from reference tables to citus local tables can only be defined with NO ACTION or RESTRICT behaviors
+ROLLBACK;
+SET citus.enable_local_execution TO OFF;
+-- show that this errors out as it tries to convert connected relations to citus
+-- local tables and creating citus local table requires local execution but local
+-- execution is disabled
+SELECT create_reference_table('local_table_1');
+ERROR:  cannot execute command because a local execution has accessed a placement in the transaction
+SET citus.enable_local_execution TO ON;
+-- test behavior when outside of the xact block
+CREATE TABLE local_table_6 (col_1 INT REFERENCES local_table_1(col_1));
+SELECT create_reference_table('local_table_1');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+SELECT create_distributed_table('local_table_2', 'col_1');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+SELECT create_distributed_table('local_table_6', 'col_1');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+ORDER BY tablename;
+     tablename     | partmethod | repmodel
+---------------------------------------------------------------------
+ distributed_table | h          | c
+ local_table_1     | n          | t
+ local_table_2     | h          | c
+ local_table_3     | n          | c
+ local_table_4     | n          | c
+ local_table_6     | h          | c
+ reference_table_1 | n          | t
+(7 rows)
+
 -- this errors out as we don't support creating citus local
 -- tables from partitioned tables
 CREATE TABLE part_local_table (col_1 INT REFERENCES reference_table_1(col_1)) PARTITION BY RANGE (col_1);
@@ -493,5 +632,148 @@ ERROR:  column "col_99" referenced in foreign key constraint does not exist
 -- fails as referenced table does not exist
 CREATE TABLE local_table_5 (col_1 INT, FOREIGN KEY (col_1) REFERENCES table_does_not_exist(dummy));
 ERROR:  relation "table_does_not_exist" does not exist
+-- drop & recreate schema to prevent noise in next test outputs
+DROP SCHEMA fkeys_between_local_ref CASCADE;
+CREATE SCHEMA fkeys_between_local_ref;
+SET search_path TO fkeys_between_local_ref;
+-- now have some tests to test behavior before/after enabling foreign keys
+-- between local tables & reference tables
+BEGIN;
+  SET citus.enable_local_reference_table_foreign_keys TO OFF;
+  CREATE TABLE ref_1(a int PRIMARY KEY);
+  CREATE TABLE pg_local_1(a int PRIMARY KEY REFERENCES ref_1(a));
+  SELECT create_reference_table('ref_1');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SET citus.enable_local_reference_table_foreign_keys TO ON;
+  CREATE TABLE ref_2(a int PRIMARY KEY);
+  SELECT create_reference_table('ref_2');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  ALTER TABLE pg_local_1 ADD CONSTRAINT c1 FOREIGN KEY(a) REFERENCES ref_2(a);
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ tablename  | partmethod | repmodel
+---------------------------------------------------------------------
+ pg_local_1 | n          | c
+ ref_1      | n          | t
+ ref_2      | n          | t
+(3 rows)
+
+ROLLBACK;
+BEGIN;
+  SET citus.enable_local_reference_table_foreign_keys TO OFF;
+  CREATE TABLE ref_1(a int PRIMARY KEY);
+  CREATE TABLE pg_local_1(a int PRIMARY KEY REFERENCES ref_1(a));
+  SELECT create_reference_table('ref_1');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SET citus.enable_local_reference_table_foreign_keys TO ON;
+  CREATE TABLE ref_2(a int PRIMARY KEY REFERENCES pg_local_1(a));
+  SELECT create_reference_table('ref_2');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ tablename  | partmethod | repmodel
+---------------------------------------------------------------------
+ pg_local_1 | n          | c
+ ref_1      | n          | t
+ ref_2      | n          | t
+(3 rows)
+
+ROLLBACK;
+BEGIN;
+  SET citus.enable_local_reference_table_foreign_keys TO OFF;
+  CREATE TABLE ref_1(a int PRIMARY KEY);
+  CREATE TABLE pg_local_1(a int PRIMARY KEY REFERENCES ref_1(a));
+  SELECT create_reference_table('ref_1');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SET citus.enable_local_reference_table_foreign_keys TO ON;
+  CREATE TABLE ref_2(a int PRIMARY KEY REFERENCES pg_local_1(a));
+  SELECT create_reference_table('ref_2');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ tablename  | partmethod | repmodel
+---------------------------------------------------------------------
+ pg_local_1 | n          | c
+ ref_1      | n          | t
+ ref_2      | n          | t
+(3 rows)
+
+ROLLBACK;
+BEGIN;
+  SET citus.enable_local_reference_table_foreign_keys TO OFF;
+  CREATE TABLE ref_1(a int PRIMARY KEY);
+  CREATE TABLE pg_local_1(a int PRIMARY KEY REFERENCES ref_1(a));
+  SELECT create_reference_table('ref_1');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SET citus.enable_local_reference_table_foreign_keys TO ON;
+  CREATE TABLE pg_local_2(a int PRIMARY KEY REFERENCES pg_local_1(a));
+  -- we still didn't convert local tables to citus local tables
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ tablename | partmethod | repmodel
+---------------------------------------------------------------------
+ ref_1     | n          | t
+(1 row)
+
+  CREATE TABLE pg_local_3(a int PRIMARY KEY REFERENCES ref_1(a));
+  -- pg_local_3 is not connected to other local tables, so we will just
+  -- convert pg_local_3 to a citus local table
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ tablename  | partmethod | repmodel
+---------------------------------------------------------------------
+ pg_local_3 | n          | c
+ ref_1      | n          | t
+(2 rows)
+
+  CREATE TABLE pg_local_4(a int PRIMARY KEY REFERENCES ref_1(a), FOREIGN KEY (a) REFERENCES pg_local_2(a));
+  -- pg_local_4 is connected to ref_1, pg_local_1 and pg_local_2,
+  -- so we will convert those two local tables to citus local tables too
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ tablename  | partmethod | repmodel
+---------------------------------------------------------------------
+ pg_local_1 | n          | c
+ pg_local_2 | n          | c
+ pg_local_3 | n          | c
+ pg_local_4 | n          | c
+ ref_1      | n          | t
+(5 rows)
+
+ROLLBACK;
 -- cleanup at exit
 DROP SCHEMA fkeys_between_local_ref CASCADE;

src/test/regress/expected/foreign_key_to_reference_table.out

@@ -467,10 +467,8 @@ SELECT count(*) FROM referencing_schema.referencing_table;
    800
 (1 row)
 
-SET client_min_messages TO ERROR;
 DROP SCHEMA referenced_schema CASCADE;
 DROP SCHEMA referencing_schema CASCADE;
-RESET client_min_messages;
 -- on delete set update cascades properly
 CREATE TABLE referenced_table(test_column int, test_column2 int, PRIMARY KEY(test_column));
 CREATE TABLE referencing_table(id int, ref_id int DEFAULT 1);
@@ -525,7 +523,6 @@ SELECT count(*) FROM referencing_table;
 (1 row)
 
 DROP TABLE referenced_table CASCADE;
-NOTICE:  drop cascades to constraint fkey_ref on table referencing_table
 DROP TABLE referencing_table CASCADE;
 -- In the following test, we'll use a SERIAL column as the referenced column
 -- in the foreign constraint. We'll first show that and insert on non-serial
@@ -557,7 +554,6 @@ SELECT count(*) FROM referencing_table;
 (1 row)
 
 DROP TABLE referenced_table CASCADE;
-NOTICE:  drop cascades to constraint fkey_ref on table referencing_table
 DROP TABLE referencing_table CASCADE;
 -- In the following test, we'll use a SERIAL column as the referencing column
 -- in the foreign constraint. We'll first show that the values that exist
@@ -587,7 +583,6 @@ INSERT INTO referencing_table(id) SELECT x FROM generate_series(1,10) AS f(x);
 ERROR:  insert or update on table "referencing_table_xxxxxxx" violates foreign key constraint "fkey_ref_xxxxxxx"
 DETAIL:  Key (ref_id)=(X) is not present in table "referenced_table_xxxxxxx".
 DROP TABLE referenced_table CASCADE;
-NOTICE:  drop cascades to constraint fkey_ref on table referencing_table
 DROP TABLE referencing_table CASCADE;
 -- In the following test, we'll use a SERIAL column as the referencing column
 -- and referenced columns in a foreign constraint. We'll first show that the
@@ -618,7 +613,6 @@ INSERT INTO referencing_table(id) SELECT x FROM generate_series(1,10) AS f(x);
 ERROR:  insert or update on table "referencing_table_xxxxxxx" violates foreign key constraint "fkey_ref_xxxxxxx"
 DETAIL:  Key (ref_id)=(X) is not present in table "referenced_table_xxxxxxx".
 DROP TABLE referenced_table CASCADE;
-NOTICE:  drop cascades to constraint fkey_ref on table referencing_table
 DROP TABLE referencing_table CASCADE;
 -- In the following test, we use a volatile function in the referencing
 -- column in a foreign constraint. We show that if the data exists in the
@@ -642,7 +636,6 @@ ALTER TABLE referencing_table ADD CONSTRAINT fkey_ref FOREIGN KEY (ref_id) REFER
 INSERT INTO referenced_table SELECT x, x FROM generate_series(0,1000) AS f(x);
 INSERT INTO referencing_table SELECT x,(random()*1000)::int FROM generate_series(0,1000) AS f(x);
 DROP TABLE referenced_table CASCADE;
-NOTICE:  drop cascades to constraint fkey_ref on table referencing_table
 DROP TABLE referencing_table CASCADE;
 -- In the following tests, we create a foreign constraint with
 -- ON UPDATE CASCADE and see if it works properly with cascading upsert
@@ -677,7 +670,6 @@ SELECT * FROM referencing_table WHERE ref_id < 0 ORDER BY 1;
 (4 rows)
 
 DROP TABLE referenced_table CASCADE;
-NOTICE:  drop cascades to constraint fkey_ref on table referencing_table
 DROP TABLE referencing_table CASCADE;
 -- create_distributed_table should fail for tables with data if fkey exists to reference table
 CREATE TABLE referenced_table(test_column int, test_column2 int, PRIMARY KEY(test_column));
@@ -685,25 +677,12 @@ CREATE TABLE referencing_table(id int, ref_id int DEFAULT -1, FOREIGN KEY (ref_i
 INSERT INTO referenced_table VALUES (1,1), (2,2), (3,3);
 INSERT INTO referencing_table VALUES (1,1), (2,2), (3,3);
 SELECT create_reference_table('referenced_table');
-NOTICE:  Copying data from local table...
-NOTICE:  copying the data has completed
-DETAIL:  The local data in the table is no longer visible, but is still on disk.
-HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$fkey_reference_table.referenced_table$$)
  create_reference_table
 ---------------------------------------------------------------------
 
 (1 row)
 
-SELECT create_distributed_table('referencing_table', 'id');
-ERROR:  cannot distribute "referencing_table" in sequential mode because it is not empty
-HINT:  If you have manually set citus.multi_shard_modify_mode to 'sequential', try with 'parallel' option. If that is not the case, try distributing local tables when they are empty.
-BEGIN;
-  SELECT create_distributed_table('referencing_table', 'id');
-ERROR:  cannot distribute "referencing_table" in sequential mode because it is not empty
-HINT:  If you have manually set citus.multi_shard_modify_mode to 'sequential', try with 'parallel' option. If that is not the case, try distributing local tables when they are empty.
-COMMIT;
 DROP TABLE referenced_table CASCADE;
-NOTICE:  drop cascades to constraint referencing_table_ref_id_fkey on table referencing_table
 DROP TABLE referencing_table CASCADE;
 -- Chained references
 -- In the following test, we create foreign keys from one column in a distributed
@@ -778,9 +757,7 @@ SELECT count(*) FROM referencing_table;
 (1 row)
 
 DROP TABLE referenced_table CASCADE;
-NOTICE:  drop cascades to constraint fkey_ref on table referencing_table
 DROP TABLE referenced_table2 CASCADE;
-NOTICE:  drop cascades to constraint foreign_key_2 on table referencing_table
 DROP TABLE referencing_table CASCADE;
 -- check if the above fkeys are created with create_distributed_table
 CREATE TABLE referenced_table(test_column int, test_column2 int, PRIMARY KEY(test_column));
@@ -812,9 +789,7 @@ SELECT count(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_tab
 
 \set VERBOSITY terse
 DROP TABLE referenced_table CASCADE;
-NOTICE:  drop cascades to constraint referencing_table_id_fkey on table referencing_table
 DROP TABLE referenced_table2 CASCADE;
-NOTICE:  drop cascades to constraint referencing_table_id_fkey1 on table referencing_table
 DROP TABLE referencing_table CASCADE;
 -- In the following test, we create foreign keys from two columns in a distributed
 -- table to two reference tables separately. We expect to see that even if a data
@@ -888,9 +863,7 @@ SELECT count(*) FROM referencing_table;
 (1 row)
 
 DROP TABLE referenced_table CASCADE;
-NOTICE:  drop cascades to constraint fkey_ref on table referencing_table
 DROP TABLE referenced_table2 CASCADE;
-NOTICE:  drop cascades to constraint foreign_key_2 on table referencing_table
 DROP TABLE referencing_table CASCADE;
 -- check if the above fkeys are created when create_distributed_table is used for 1 foreign key and alter table for the other
 CREATE TABLE referenced_table(test_column int, test_column2 int, PRIMARY KEY(test_column));
@@ -924,9 +897,7 @@ SELECT count(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_tab
 (1 row)
 
 DROP TABLE referenced_table CASCADE;
-NOTICE:  drop cascades to constraint referencing_table_id_fkey on table referencing_table
 DROP TABLE referenced_table2 CASCADE;
-NOTICE:  drop cascades to constraint foreign_key_2 on table referencing_table
 DROP TABLE referencing_table CASCADE;
 \set VERBOSITY default
 -- two distributed tables are referencing to one reference table and
@@ -1005,48 +976,25 @@ SELECT count(*) FROM referencing_table2;
 
 \set VERBOSITY terse
 DROP TABLE referenced_table CASCADE;
-NOTICE:  drop cascades to 2 other objects
 DROP TABLE referencing_table CASCADE;
-NOTICE:  drop cascades to constraint fkey_ref_to_dist on table referencing_table2
 DROP TABLE referencing_table2 CASCADE;
 \set VERBOSITY default
 -- Check if the above fkeys are created with create_distributed_table
 CREATE TABLE referenced_table(test_column int, test_column2 int UNIQUE, PRIMARY KEY(test_column));
-CREATE TABLE referencing_table(id int PRIMARY KEY, ref_id int, FOREIGN KEY (id) REFERENCES referenced_table(test_column) ON DELETE CASCADE);
-CREATE TABLE referencing_table2(id int, ref_id int, FOREIGN KEY (ref_id) REFERENCES referenced_table(test_column2) ON DELETE CASCADE, FOREIGN KEY (id) REFERENCES referencing_table(id) ON DELETE CASCADE);
 SELECT create_reference_table('referenced_table');
  create_reference_table
 ---------------------------------------------------------------------
 
 (1 row)
 
-BEGIN;
-  SET LOCAL citus.multi_shard_modify_mode TO 'sequential';
-  SELECT create_distributed_table('referencing_table', 'id');
- create_distributed_table
----------------------------------------------------------------------
-
-(1 row)
-
-  SELECT create_distributed_table('referencing_table2', 'id');
- create_distributed_table
----------------------------------------------------------------------
-
-(1 row)
-
-COMMIT;
 SELECT count(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
  count
 ---------------------------------------------------------------------
-    24
+     0
 (1 row)
 
 \set VERBOSITY terse
 DROP TABLE referenced_table CASCADE;
-NOTICE:  drop cascades to 2 other objects
-DROP TABLE referencing_table CASCADE;
-NOTICE:  drop cascades to constraint referencing_table2_id_fkey on table referencing_table2
-DROP TABLE referencing_table2 CASCADE;
 \set VERBOSITY default
 -- In this test we have a chained relationship in form of
 -- distributed table (referencing_referencing_table) has a foreign key with two columns
@@ -1092,11 +1040,8 @@ SELECT max(ref_id) FROM referencing_referencing_table;
 (1 row)
 
 DROP TABLE referenced_table CASCADE;
-NOTICE:  drop cascades to constraint fkey_ref on table referencing_table
 DROP TABLE referencing_table CASCADE;
-NOTICE:  drop cascades to constraint referencing_referencing_table_id_fkey on table referencing_referencing_table
 DROP TABLE referencing_referencing_table;
--- create_reference_table, create_distributed_table and ALTER TABLE in the same transaction
 BEGIN;
   CREATE TABLE test_table_1(id int PRIMARY KEY);
   SELECT create_reference_table('test_table_1');
@@ -1106,28 +1051,11 @@ BEGIN;
 (1 row)
 
   CREATE TABLE test_table_2(id int PRIMARY KEY, value_1 int);
-  SELECT create_distributed_table('test_table_2', 'id');
- create_distributed_table
----------------------------------------------------------------------
-
-(1 row)
-
   ALTER TABLE test_table_2 ADD CONSTRAINT c_check FOREIGN KEY (value_1) REFERENCES test_table_1(id);
-ERROR:  cannot modify table "test_table_2" because there was a parallel operation on a distributed table in the transaction
-DETAIL:  When there is a foreign key to a reference table, Citus needs to perform all operations over a single connection per node to ensure consistency.
-HINT:  Try re-running the transaction with "SET LOCAL citus.multi_shard_modify_mode TO 'sequential';"
   DROP TABLE test_table_1, test_table_2;
-ERROR:  current transaction is aborted, commands ignored until end of transaction block
 COMMIT;
--- the order of create_reference_table and create_distributed_table is changed
 BEGIN;
   CREATE TABLE test_table_1(id int PRIMARY KEY, value_1 int);
-  SELECT create_distributed_table('test_table_1', 'id');
- create_distributed_table
----------------------------------------------------------------------
-
-(1 row)
-
   CREATE TABLE test_table_2(id int PRIMARY KEY);
   SELECT create_reference_table('test_table_2');
  create_reference_table
@@ -1136,33 +1064,20 @@ BEGIN;
 (1 row)
 
   ALTER TABLE test_table_1 ADD CONSTRAINT c_check FOREIGN KEY (value_1) REFERENCES test_table_2(id);
-ERROR:  cannot modify table "test_table_1" because there was a parallel operation on a distributed table in the transaction
-DETAIL:  When there is a foreign key to a reference table, Citus needs to perform all operations over a single connection per node to ensure consistency.
-HINT:  Try re-running the transaction with "SET LOCAL citus.multi_shard_modify_mode TO 'sequential';"
   DROP TABLE test_table_2 CASCADE;
-ERROR:  current transaction is aborted, commands ignored until end of transaction block
 ROLLBACK;
--- make sure that we fail if we need parallel data load
 BEGIN;
   CREATE TABLE test_table_1(id int PRIMARY KEY);
   INSERT INTO test_table_1 SELECT i FROM generate_series(0,100) i;
   CREATE TABLE test_table_2(id int PRIMARY KEY, value_1 int, FOREIGN KEY(value_1) REFERENCES test_table_1(id));
   INSERT INTO test_table_2 SELECT i, i FROM generate_series(0,100) i;
   SELECT create_reference_table('test_table_1');
-NOTICE:  Copying data from local table...
-NOTICE:  copying the data has completed
-DETAIL:  The local data in the table is no longer visible, but is still on disk.
-HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$fkey_reference_table.test_table_1$$)
  create_reference_table
 ---------------------------------------------------------------------
 
 (1 row)
 
-  SELECT create_distributed_table('test_table_2', 'id');
-ERROR:  cannot distribute "test_table_2" in sequential mode because it is not empty
-HINT:  If you have manually set citus.multi_shard_modify_mode to 'sequential', try with 'parallel' option. If that is not the case, try distributing local tables when they are empty.
   DROP TABLE test_table_2, test_table_1;
-ERROR:  current transaction is aborted, commands ignored until end of transaction block
 COMMIT;
 -- make sure that other DDLs/DMLs also work fine
 BEGIN;
@@ -1261,7 +1176,6 @@ SELECT create_distributed_table('test_table_2', 'id');
 (1 row)
 
 ALTER TABLE test_table_1 DROP CONSTRAINT test_table_1_pkey CASCADE;
-NOTICE:  drop cascades to constraint test_table_2_value_1_fkey on table test_table_2
 SELECT count(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
  count
 ---------------------------------------------------------------------
@@ -1286,7 +1200,6 @@ BEGIN;
 (1 row)
 
   ALTER TABLE test_table_1 DROP CONSTRAINT test_table_1_pkey CASCADE;
-NOTICE:  drop cascades to constraint test_table_2_value_1_fkey on table test_table_2
 COMMIT;
 SELECT count(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
  count
@@ -1359,7 +1272,6 @@ SELECT create_distributed_table('test_table_2', 'id');
 (1 row)
 
 ALTER TABLE test_table_1 DROP COLUMN id CASCADE;
-NOTICE:  drop cascades to constraint test_table_2_value_1_fkey on table test_table_2
 SELECT count(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
  count
 ---------------------------------------------------------------------
@@ -1384,7 +1296,6 @@ BEGIN;
 (1 row)
 
   ALTER TABLE test_table_1 DROP COLUMN id CASCADE;
-NOTICE:  drop cascades to constraint test_table_2_value_1_fkey on table test_table_2
 COMMIT;
 SELECT count(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
  count
@@ -1426,7 +1337,6 @@ SELECT count(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_tab
 (1 row)
 
 DROP TABLE test_table_1 CASCADE;
-NOTICE:  drop cascades to constraint test_table_2_value_1_fkey on table test_table_2
 DROP TABLE test_table_2;
 -- check if we can alter the column type and drop it which foreign key is referencing to in a transaction block
 CREATE TABLE test_table_1(id int PRIMARY KEY);
@@ -1446,7 +1356,6 @@ BEGIN;
 
   ALTER TABLE test_table_2 ALTER COLUMN value_1 SET DATA TYPE bigint;
   ALTER TABLE test_table_1 DROP COLUMN id CASCADE;
-NOTICE:  drop cascades to constraint test_table_2_value_1_fkey on table test_table_2
 COMMIT;
 SELECT count(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
  count
@@ -1473,7 +1382,6 @@ SELECT create_distributed_table('test_table_2', 'id');
 INSERT INTO test_table_1 VALUES (1),(2),(3);
 INSERT INTO test_table_2 VALUES (1,1),(2,2),(3,3);
 TRUNCATE test_table_1 CASCADE;
-NOTICE:  truncate cascades to table "test_table_2"
 SELECT * FROM test_table_2;
  id | value_1
 ---------------------------------------------------------------------
@@ -1499,7 +1407,6 @@ INSERT INTO test_table_1 VALUES (1),(2),(3);
 INSERT INTO test_table_2 VALUES (1,1),(2,2),(3,3);
 BEGIN;
   TRUNCATE test_table_1 CASCADE;
-NOTICE:  truncate cascades to table "test_table_2"
 COMMIT;
 SELECT * FROM test_table_2;
  id | value_1
@@ -1526,7 +1433,6 @@ BEGIN;
   INSERT INTO test_table_1 VALUES (1),(2),(3);
   INSERT INTO test_table_2 VALUES (1,1),(2,2),(3,3);
   TRUNCATE test_table_1 CASCADE;
-NOTICE:  truncate cascades to table "test_table_2"
 COMMIT;
 SELECT * FROM test_table_2;
  id | value_1
@@ -1627,7 +1533,6 @@ BEGIN;
   ALTER TABLE test_table_2 ADD CONSTRAINT fkey FOREIGN KEY (value_1) REFERENCES test_table_1(id);
   ALTER TABLE test_table_3 ADD COLUMN test_column int;
   ALTER TABLE test_table_1 DROP COLUMN id CASCADE;
-NOTICE:  drop cascades to constraint fkey on table test_table_2
   ALTER TABLE test_table_1 ADD COLUMN id int;
 COMMIT;
 DROP TABLE test_table_1, test_table_2, test_table_3;
@@ -1703,9 +1608,7 @@ BEGIN;
 
 ROLLBACK;
 DROP TABLE referenced_table CASCADE;
-NOTICE:  drop cascades to constraint fkey_to_ref on table referencing_table_4
 DROP TABLE referencing_table;
-SET client_min_messages TO ERROR;
 DROP SCHEMA fkey_reference_table CASCADE;
 SET search_path TO DEFAULT;
 RESET client_min_messages;

src/test/regress/sql/fkeys_between_local_ref.sql

@@ -341,6 +341,70 @@ BEGIN;
   ORDER BY tablename;
 ROLLBACK;
 
+-- converting any local table to a citus local table in graph converts
+-- other tables to citus local tables, test this in below xact blocks
+
+BEGIN;
+  SELECT create_reference_table('local_table_1');
+
+  SELECT create_distributed_table('local_table_2', 'col_1');
+
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ROLLBACK;
+
+BEGIN;
+  SELECT create_reference_table('local_table_4');
+
+  SELECT create_reference_table('local_table_3');
+
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ROLLBACK;
+
+BEGIN;
+  CREATE TABLE local_table_5 (col_1 INT REFERENCES local_table_1(col_1));
+
+  SELECT create_reference_table('local_table_1');
+
+  SELECT create_distributed_table('local_table_2', 'col_1');
+  SELECT create_distributed_table('local_table_5', 'col_1');
+
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ROLLBACK;
+
+BEGIN;
+  ALTER TABLE local_table_1 ADD CONSTRAINT fkey_13 FOREIGN KEY (col_1) REFERENCES local_table_2(col_1) ON DELETE CASCADE;
+
+  -- errors out as foreign keys from reference tables to citus local tables
+  -- cannot have CASCADE behavior
+  SELECT create_reference_table('local_table_1');
+ROLLBACK;
+
+SET citus.enable_local_execution TO OFF;
+-- show that this errors out as it tries to convert connected relations to citus
+-- local tables and creating citus local table requires local execution but local
+-- execution is disabled
+SELECT create_reference_table('local_table_1');
+SET citus.enable_local_execution TO ON;
+
+-- test behavior when outside of the xact block
+
+CREATE TABLE local_table_6 (col_1 INT REFERENCES local_table_1(col_1));
+
+SELECT create_reference_table('local_table_1');
+
+SELECT create_distributed_table('local_table_2', 'col_1');
+SELECT create_distributed_table('local_table_6', 'col_1');
+
+SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+ORDER BY tablename;
+
 -- this errors out as we don't support creating citus local
 -- tables from partitioned tables
 CREATE TABLE part_local_table (col_1 INT REFERENCES reference_table_1(col_1)) PARTITION BY RANGE (col_1);
@@ -352,5 +416,98 @@ CREATE TABLE local_table_5 (col_1 INT, FOREIGN KEY (col_1) REFERENCES reference_
 -- fails as referenced table does not exist
 CREATE TABLE local_table_5 (col_1 INT, FOREIGN KEY (col_1) REFERENCES table_does_not_exist(dummy));
 
+-- drop & recreate schema to prevent noise in next test outputs
+DROP SCHEMA fkeys_between_local_ref CASCADE;
+CREATE SCHEMA fkeys_between_local_ref;
+SET search_path TO fkeys_between_local_ref;
+
+-- now have some tests to test behavior before/after enabling foreign keys
+-- between local tables & reference tables
+
+BEGIN;
+  SET citus.enable_local_reference_table_foreign_keys TO OFF;
+
+  CREATE TABLE ref_1(a int PRIMARY KEY);
+  CREATE TABLE pg_local_1(a int PRIMARY KEY REFERENCES ref_1(a));
+  SELECT create_reference_table('ref_1');
+
+  SET citus.enable_local_reference_table_foreign_keys TO ON;
+
+  CREATE TABLE ref_2(a int PRIMARY KEY);
+  SELECT create_reference_table('ref_2');
+  ALTER TABLE pg_local_1 ADD CONSTRAINT c1 FOREIGN KEY(a) REFERENCES ref_2(a);
+
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ROLLBACK;
+
+BEGIN;
+  SET citus.enable_local_reference_table_foreign_keys TO OFF;
+
+  CREATE TABLE ref_1(a int PRIMARY KEY);
+  CREATE TABLE pg_local_1(a int PRIMARY KEY REFERENCES ref_1(a));
+  SELECT create_reference_table('ref_1');
+
+  SET citus.enable_local_reference_table_foreign_keys TO ON;
+
+  CREATE TABLE ref_2(a int PRIMARY KEY REFERENCES pg_local_1(a));
+  SELECT create_reference_table('ref_2');
+
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ROLLBACK;
+
+BEGIN;
+  SET citus.enable_local_reference_table_foreign_keys TO OFF;
+
+  CREATE TABLE ref_1(a int PRIMARY KEY);
+  CREATE TABLE pg_local_1(a int PRIMARY KEY REFERENCES ref_1(a));
+  SELECT create_reference_table('ref_1');
+
+  SET citus.enable_local_reference_table_foreign_keys TO ON;
+
+  CREATE TABLE ref_2(a int PRIMARY KEY REFERENCES pg_local_1(a));
+  SELECT create_reference_table('ref_2');
+
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ROLLBACK;
+
+BEGIN;
+  SET citus.enable_local_reference_table_foreign_keys TO OFF;
+
+  CREATE TABLE ref_1(a int PRIMARY KEY);
+  CREATE TABLE pg_local_1(a int PRIMARY KEY REFERENCES ref_1(a));
+  SELECT create_reference_table('ref_1');
+
+  SET citus.enable_local_reference_table_foreign_keys TO ON;
+
+  CREATE TABLE pg_local_2(a int PRIMARY KEY REFERENCES pg_local_1(a));
+
+  -- we still didn't convert local tables to citus local tables
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+
+  CREATE TABLE pg_local_3(a int PRIMARY KEY REFERENCES ref_1(a));
+
+  -- pg_local_3 is not connected to other local tables, so we will just
+  -- convert pg_local_3 to a citus local table
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+
+  CREATE TABLE pg_local_4(a int PRIMARY KEY REFERENCES ref_1(a), FOREIGN KEY (a) REFERENCES pg_local_2(a));
+
+  -- pg_local_4 is connected to ref_1, pg_local_1 and pg_local_2,
+  -- so we will convert those two local tables to citus local tables too
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ROLLBACK;
+
 -- cleanup at exit
 DROP SCHEMA fkeys_between_local_ref CASCADE;

src/test/regress/sql/foreign_key_to_reference_table.sql

@@ -251,10 +251,8 @@ INSERT INTO referencing_schema.referencing_table SELECT x, x from generate_serie
 DELETE FROM referenced_schema.referenced_table WHERE id > 800;
 SELECT count(*) FROM referencing_schema.referencing_table;
 
-SET client_min_messages TO ERROR;
 DROP SCHEMA referenced_schema CASCADE;
 DROP SCHEMA referencing_schema CASCADE;
-RESET client_min_messages;
 
 -- on delete set update cascades properly
 CREATE TABLE referenced_table(test_column int, test_column2 int, PRIMARY KEY(test_column));
@@ -393,11 +391,6 @@ CREATE TABLE referencing_table(id int, ref_id int DEFAULT -1, FOREIGN KEY (ref_i
 INSERT INTO referenced_table VALUES (1,1), (2,2), (3,3);
 INSERT INTO referencing_table VALUES (1,1), (2,2), (3,3);
 SELECT create_reference_table('referenced_table');
-SELECT create_distributed_table('referencing_table', 'id');
-
-BEGIN;
-  SELECT create_distributed_table('referencing_table', 'id');
-COMMIT;
 
 DROP TABLE referenced_table CASCADE;
 DROP TABLE referencing_table CASCADE;
@@ -565,21 +558,12 @@ DROP TABLE referencing_table2 CASCADE;
 
 -- Check if the above fkeys are created with create_distributed_table
 CREATE TABLE referenced_table(test_column int, test_column2 int UNIQUE, PRIMARY KEY(test_column));
-CREATE TABLE referencing_table(id int PRIMARY KEY, ref_id int, FOREIGN KEY (id) REFERENCES referenced_table(test_column) ON DELETE CASCADE);
-CREATE TABLE referencing_table2(id int, ref_id int, FOREIGN KEY (ref_id) REFERENCES referenced_table(test_column2) ON DELETE CASCADE, FOREIGN KEY (id) REFERENCES referencing_table(id) ON DELETE CASCADE);
 SELECT create_reference_table('referenced_table');
-BEGIN;
-  SET LOCAL citus.multi_shard_modify_mode TO 'sequential';
-  SELECT create_distributed_table('referencing_table', 'id');
-  SELECT create_distributed_table('referencing_table2', 'id');
-COMMIT;
 
 SELECT count(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
 
 \set VERBOSITY terse
 DROP TABLE referenced_table CASCADE;
-DROP TABLE referencing_table CASCADE;
-DROP TABLE referencing_table2 CASCADE;
 \set VERBOSITY default
 
 -- In this test we have a chained relationship in form of
@@ -608,23 +592,19 @@ DROP TABLE referenced_table CASCADE;
 DROP TABLE referencing_table CASCADE;
 DROP TABLE referencing_referencing_table;
 
--- create_reference_table, create_distributed_table and ALTER TABLE in the same transaction
 BEGIN;
   CREATE TABLE test_table_1(id int PRIMARY KEY);
   SELECT create_reference_table('test_table_1');
 
   CREATE TABLE test_table_2(id int PRIMARY KEY, value_1 int);
-  SELECT create_distributed_table('test_table_2', 'id');
 
   ALTER TABLE test_table_2 ADD CONSTRAINT c_check FOREIGN KEY (value_1) REFERENCES test_table_1(id);
 
   DROP TABLE test_table_1, test_table_2;
 COMMIT;
 
--- the order of create_reference_table and create_distributed_table is changed
 BEGIN;
   CREATE TABLE test_table_1(id int PRIMARY KEY, value_1 int);
-  SELECT create_distributed_table('test_table_1', 'id');
 
   CREATE TABLE test_table_2(id int PRIMARY KEY);
   SELECT create_reference_table('test_table_2');
@@ -634,7 +614,6 @@ BEGIN;
   DROP TABLE test_table_2 CASCADE;
 ROLLBACK;
 
--- make sure that we fail if we need parallel data load
 BEGIN;
 
   CREATE TABLE test_table_1(id int PRIMARY KEY);
@@ -644,7 +623,6 @@ BEGIN;
   INSERT INTO test_table_2 SELECT i, i FROM generate_series(0,100) i;
 
   SELECT create_reference_table('test_table_1');
-  SELECT create_distributed_table('test_table_2', 'id');
   DROP TABLE test_table_2, test_table_1;
 COMMIT;
 
@@ -940,7 +918,6 @@ ROLLBACK;
 DROP TABLE referenced_table CASCADE;
 DROP TABLE referencing_table;
 
-SET client_min_messages TO ERROR;
 DROP SCHEMA fkey_reference_table CASCADE;
 SET search_path TO DEFAULT;
 RESET client_min_messages;