Merge branch 'master' into fix-maintenance-daemon-crash

src/backend/columnar/cstore.c

@@ -1,6 +1,6 @@
 /*-------------------------------------------------------------------------
  *
- * cstore.c
+ * columnar.c
  *
  * This file contains...
  *
@@ -21,7 +21,7 @@
 #include "utils/rel.h"
 
 #include "citus_version.h"
-#include "columnar/cstore.h"
+#include "columnar/columnar.h"
 
 /* Default values for option parameters */
 #define DEFAULT_STRIPE_ROW_COUNT 150000
@@ -57,7 +57,7 @@ void
 columnar_init_gucs()
 {
 	DefineCustomEnumVariable("columnar.compression",
-							 "Compression type for cstore.",
+							 "Compression type for columnar.",
 							 NULL,
 							 &columnar_compression,
 							 DEFAULT_COMPRESSION_TYPE,

src/backend/columnar/cstore_compression.c

@@ -1,9 +1,9 @@
 /*-------------------------------------------------------------------------
  *
- * cstore_compression.c
+ * columnar_compression.c
  *
  * This file contains compression/decompression functions definitions
- * used in cstore_fdw.
+ * used for columnar.
  *
  * Copyright (c) 2016, Citus Data, Inc.
  *
@@ -14,7 +14,7 @@
 #include "postgres.h"
 
 #include "citus_version.h"
-#include "columnar/cstore.h"
+#include "columnar/columnar.h"
 #include "common/pg_lzcompress.h"
 
 #if HAVE_LIBLZ4
@@ -29,21 +29,22 @@
  *	The information at the start of the compressed data. This decription is taken
  *	from pg_lzcompress in pre-9.5 version of PostgreSQL.
  */
-typedef struct CStoreCompressHeader
+typedef struct ColumnarCompressHeader
 {
 	int32 vl_len_;              /* varlena header (do not touch directly!) */
 	int32 rawsize;
-} CStoreCompressHeader;
+} ColumnarCompressHeader;
 
 /*
  * Utilities for manipulation of header information for compressed data
  */
 
-#define CSTORE_COMPRESS_HDRSZ ((int32) sizeof(CStoreCompressHeader))
+#define COLUMNAR_COMPRESS_HDRSZ ((int32) sizeof(ColumnarCompressHeader))
-#define CSTORE_COMPRESS_RAWSIZE(ptr) (((CStoreCompressHeader *) (ptr))->rawsize)
+#define COLUMNAR_COMPRESS_RAWSIZE(ptr) (((ColumnarCompressHeader *) (ptr))->rawsize)
-#define CSTORE_COMPRESS_RAWDATA(ptr) (((char *) (ptr)) + CSTORE_COMPRESS_HDRSZ)
+#define COLUMNAR_COMPRESS_RAWDATA(ptr) (((char *) (ptr)) + COLUMNAR_COMPRESS_HDRSZ)
-#define CSTORE_COMPRESS_SET_RAWSIZE(ptr, len) (((CStoreCompressHeader *) (ptr))->rawsize = \
+#define COLUMNAR_COMPRESS_SET_RAWSIZE(ptr, \
-												   (len))
+									  len) (((ColumnarCompressHeader *) (ptr))->rawsize = \
+												(len))
 
 
 /*
@@ -116,7 +117,7 @@ CompressBuffer(StringInfo inputBuffer,
 		case COMPRESSION_PG_LZ:
 		{
 			uint64 maximumLength = PGLZ_MAX_OUTPUT(inputBuffer->len) +
-								   CSTORE_COMPRESS_HDRSZ;
+								   COLUMNAR_COMPRESS_HDRSZ;
 			bool compressionResult = false;
 
 			resetStringInfo(outputBuffer);
@@ -124,14 +125,14 @@ CompressBuffer(StringInfo inputBuffer,
 
 			int32 compressedByteCount = pglz_compress((const char *) inputBuffer->data,
 													  inputBuffer->len,
-													  CSTORE_COMPRESS_RAWDATA(
+													  COLUMNAR_COMPRESS_RAWDATA(
 														  outputBuffer->data),
 													  PGLZ_strategy_always);
 			if (compressedByteCount >= 0)
 			{
-				CSTORE_COMPRESS_SET_RAWSIZE(outputBuffer->data, inputBuffer->len);
+				COLUMNAR_COMPRESS_SET_RAWSIZE(outputBuffer->data, inputBuffer->len);
 				SET_VARSIZE_COMPRESSED(outputBuffer->data,
-									   compressedByteCount + CSTORE_COMPRESS_HDRSZ);
+									   compressedByteCount + COLUMNAR_COMPRESS_HDRSZ);
 				compressionResult = true;
 			}
 
@@ -224,11 +225,11 @@ DecompressBuffer(StringInfo buffer,
 		case COMPRESSION_PG_LZ:
 		{
 			StringInfo decompressedBuffer = NULL;
-			uint32 compressedDataSize = VARSIZE(buffer->data) - CSTORE_COMPRESS_HDRSZ;
+			uint32 compressedDataSize = VARSIZE(buffer->data) - COLUMNAR_COMPRESS_HDRSZ;
-			uint32 decompressedDataSize = CSTORE_COMPRESS_RAWSIZE(buffer->data);
+			uint32 decompressedDataSize = COLUMNAR_COMPRESS_RAWSIZE(buffer->data);
 			int32 decompressedByteCount = 0;
 
-			if (compressedDataSize + CSTORE_COMPRESS_HDRSZ != buffer->len)
+			if (compressedDataSize + COLUMNAR_COMPRESS_HDRSZ != buffer->len)
 			{
 				ereport(ERROR, (errmsg("cannot decompress the buffer"),
 								errdetail("Expected %u bytes, but received %u bytes",
@@ -238,11 +239,13 @@ DecompressBuffer(StringInfo buffer,
 			char *decompressedData = palloc0(decompressedDataSize);
 
 	#if PG_VERSION_NUM >= 120000
-			decompressedByteCount = pglz_decompress(CSTORE_COMPRESS_RAWDATA(buffer->data),
+			decompressedByteCount = pglz_decompress(COLUMNAR_COMPRESS_RAWDATA(
+														buffer->data),
 													compressedDataSize, decompressedData,
 													decompressedDataSize, true);
 	#else
-			decompressedByteCount = pglz_decompress(CSTORE_COMPRESS_RAWDATA(buffer->data),
+			decompressedByteCount = pglz_decompress(COLUMNAR_COMPRESS_RAWDATA(
+														buffer->data),
 													compressedDataSize, decompressedData,
 													decompressedDataSize);
 	#endif

src/backend/columnar/cstore_customscan.c

@@ -25,9 +25,9 @@
 #include "optimizer/restrictinfo.h"
 #include "utils/relcache.h"
 
-#include "columnar/cstore.h"
+#include "columnar/columnar.h"
-#include "columnar/cstore_customscan.h"
+#include "columnar/columnar_customscan.h"
-#include "columnar/cstore_tableam.h"
+#include "columnar/columnar_tableam.h"
 
 typedef struct ColumnarScanPath
 {
@@ -102,7 +102,7 @@ const struct CustomExecMethods ColumnarExecuteMethods = {
 
 /*
  * columnar_customscan_init installs the hook required to intercept the postgres planner and
- * provide extra paths for cstore tables
+ * provide extra paths for columnar tables
  */
 void
 columnar_customscan_init()
@@ -158,7 +158,7 @@ ColumnarSetRelPathlistHook(PlannerInfo *root, RelOptInfo *rel, Index rti,
 	}
 
 	/*
-	 * Here we want to inspect if this relation pathlist hook is accessing a cstore table.
+	 * Here we want to inspect if this relation pathlist hook is accessing a columnar table.
 	 * If that is the case we want to insert an extra path that pushes down the projection
 	 * into the scan of the table to minimize the data read.
 	 */
@@ -173,7 +173,7 @@ ColumnarSetRelPathlistHook(PlannerInfo *root, RelOptInfo *rel, Index rti,
 
 		Path *customPath = CreateColumnarScanPath(rel, rte);
 
-		ereport(DEBUG1, (errmsg("pathlist hook for cstore table am")));
+		ereport(DEBUG1, (errmsg("pathlist hook for columnar table am")));
 
 		/* we propose a new path that will be the only path for scanning this relation */
 		clear_paths(rel);
@@ -204,7 +204,7 @@ CreateColumnarScanPath(RelOptInfo *rel, RangeTblEntry *rte)
 	path->pathtarget = rel->reltarget;
 
 	/*
-	 * Add cost estimates for a cstore table scan, row count is the rows estimated by
+	 * Add cost estimates for a columnar table scan, row count is the rows estimated by
 	 * postgres' planner.
 	 */
 	path->rows = rel->rows;
@@ -216,7 +216,7 @@ CreateColumnarScanPath(RelOptInfo *rel, RangeTblEntry *rte)
 
 
 /*
- * ColumnarScanCost calculates the cost of scanning the cstore table. The cost is estimated
+ * ColumnarScanCost calculates the cost of scanning the columnar table. The cost is estimated
  * by using all stripe metadata to estimate based on the columns to read how many pages
  * need to be read.
  */
@@ -277,13 +277,13 @@ ColumnarScanPath_PlanCustomPath(PlannerInfo *root,
 static Node *
 ColumnarScan_CreateCustomScanState(CustomScan *cscan)
 {
-	ColumnarScanState *cstorescanstate = (ColumnarScanState *) newNode(
+	ColumnarScanState *columnarScanState = (ColumnarScanState *) newNode(
 		sizeof(ColumnarScanState), T_CustomScanState);
 
-	CustomScanState *cscanstate = &cstorescanstate->custom_scanstate;
+	CustomScanState *cscanstate = &columnarScanState->custom_scanstate;
 	cscanstate->methods = &ColumnarExecuteMethods;
 
-	cstorescanstate->qual = cscan->scan.plan.qual;
+	columnarScanState->qual = cscan->scan.plan.qual;
 
 	return (Node *) cscanstate;
 }
@@ -338,9 +338,9 @@ ColumnarAttrNeeded(ScanState *ss)
 
 
 static TupleTableSlot *
-ColumnarScanNext(ColumnarScanState *cstorescanstate)
+ColumnarScanNext(ColumnarScanState *columnarScanState)
 {
-	CustomScanState *node = (CustomScanState *) cstorescanstate;
+	CustomScanState *node = (CustomScanState *) columnarScanState;
 
 	/*
 	 * get information from the estate and scan state
@@ -352,7 +352,7 @@ ColumnarScanNext(ColumnarScanState *cstorescanstate)
 
 	if (scandesc == NULL)
 	{
-		/* the cstore access method does not use the flags, they are specific to heap */
+		/* the columnar access method does not use the flags, they are specific to heap */
 		uint32 flags = 0;
 		Bitmapset *attr_needed = ColumnarAttrNeeded(&node->ss);
 
@@ -363,7 +363,7 @@ ColumnarScanNext(ColumnarScanState *cstorescanstate)
 		scandesc = columnar_beginscan_extended(node->ss.ss_currentRelation,
 											   estate->es_snapshot,
 											   0, NULL, NULL, flags, attr_needed,
-											   cstorescanstate->qual);
+											   columnarScanState->qual);
 		bms_free(attr_needed);
 
 		node->ss.ss_currentScanDesc = scandesc;

src/backend/columnar/cstore_debug.c

@@ -1,6 +1,6 @@
 /*-------------------------------------------------------------------------
  *
- * cstore_debug.c
+ * columnar_debug.c
  *
  * Helper functions to debug column store.
  *
@@ -24,8 +24,8 @@
 #include "utils/rel.h"
 #include "utils/tuplestore.h"
 
-#include "columnar/cstore.h"
+#include "columnar/columnar.h"
-#include "columnar/cstore_version_compat.h"
+#include "columnar/columnar_version_compat.h"
 
 static void MemoryContextTotals(MemoryContext context, MemoryContextCounters *counters);
 

src/backend/columnar/cstore_metadata_tables.c

@@ -13,8 +13,8 @@
 #include "safe_lib.h"
 
 #include "citus_version.h"
-#include "columnar/cstore.h"
+#include "columnar/columnar.h"
-#include "columnar/cstore_version_compat.h"
+#include "columnar/columnar_version_compat.h"
 
 #include <sys/stat.h>
 #include "access/heapam.h"
@@ -668,7 +668,7 @@ GetHighestUsedAddressAndId(uint64 storageId,
 	*highestUsedId = 0;
 
 	/* file starts with metapage */
-	*highestUsedAddress = CSTORE_BYTES_PER_PAGE;
+	*highestUsedAddress = COLUMNAR_BYTES_PER_PAGE;
 
 	foreach(stripeMetadataCell, stripeMetadataList)
 	{
@@ -1182,8 +1182,8 @@ InitMetapage(Relation relation)
 
 	ColumnarMetapage *metapage = palloc0(sizeof(ColumnarMetapage));
 	metapage->storageId = GetNextStorageId();
-	metapage->versionMajor = CSTORE_VERSION_MAJOR;
+	metapage->versionMajor = COLUMNAR_VERSION_MAJOR;
-	metapage->versionMinor = CSTORE_VERSION_MINOR;
+	metapage->versionMinor = COLUMNAR_VERSION_MINOR;
 
 	/* create the first block */
 	Buffer newBuffer = ReadBuffer(relation, P_NEW);

src/backend/columnar/cstore_reader.c

@@ -1,8 +1,8 @@
 /*-------------------------------------------------------------------------
  *
- * cstore_reader.c
+ * columnar_reader.c
  *
- * This file contains function definitions for reading cstore files. This
+ * This file contains function definitions for reading columnar tables. This
  * includes the logic for reading file level metadata, reading row stripes,
  * and skipping unrelated row chunks and columns.
  *
@@ -36,8 +36,8 @@
 #include "utils/lsyscache.h"
 #include "utils/rel.h"
 
-#include "columnar/cstore.h"
+#include "columnar/columnar.h"
-#include "columnar/cstore_version_compat.h"
+#include "columnar/columnar_version_compat.h"
 
 /* static function declarations */
 static StripeBuffers * LoadFilteredStripeBuffers(Relation relation,
@@ -80,7 +80,7 @@ static Datum ColumnDefaultValue(TupleConstr *tupleConstraints,
 								Form_pg_attribute attributeForm);
 
 /*
- * ColumnarBeginRead initializes a cstore read operation. This function returns a
+ * ColumnarBeginRead initializes a columnar read operation. This function returns a
  * read handle that's used during reading rows and finishing the read operation.
  */
 TableReadState *
@@ -117,7 +117,7 @@ ColumnarBeginRead(Relation relation, TupleDesc tupleDescriptor,
 
 
 /*
- * ColumnarReadNextRow tries to read a row from the cstore file. On success, it sets
+ * ColumnarReadNextRow tries to read a row from the columnar table. On success, it sets
  * column values and nulls, and returns true. If there are no more rows to read,
  * the function returns false.
  */
@@ -178,18 +178,8 @@ ColumnarReadNextRow(TableReadState *readState, Datum *columnValues, bool *column
 
 	if (chunkIndex != readState->deserializedChunkIndex)
 	{
-		uint32 chunkRowCount = 0;
+		uint32 chunkRowCount =
-
+			readState->stripeBuffers->selectedChunkRowCount[chunkIndex];
-		uint32 stripeRowCount = stripeMetadata->rowCount;
-		uint32 lastChunkIndex = stripeRowCount / stripeMetadata->chunkRowCount;
-		if (chunkIndex == lastChunkIndex)
-		{
-			chunkRowCount = stripeRowCount % stripeMetadata->chunkRowCount;
-		}
-		else
-		{
-			chunkRowCount = stripeMetadata->chunkRowCount;
-		}
 
 		oldContext = MemoryContextSwitchTo(readState->stripeReadContext);
 
@@ -235,7 +225,7 @@ ColumnarRescan(TableReadState *readState)
 }
 
 
-/* Finishes a cstore read operation. */
+/* Finishes a columnar read operation. */
 void
 ColumnarEndRead(TableReadState *readState)
 {
@@ -356,6 +346,14 @@ LoadFilteredStripeBuffers(Relation relation, StripeMetadata *stripeMetadata,
 		SelectedChunkSkipList(stripeSkipList, projectedColumnMask,
 							  selectedChunkMask);
 
+	uint32 selectedChunkCount = selectedChunkSkipList->chunkCount;
+	uint32 *selectedChunkRowCount = palloc0(selectedChunkCount * sizeof(uint32));
+	for (int chunkIndex = 0; chunkIndex < selectedChunkCount; chunkIndex++)
+	{
+		selectedChunkRowCount[chunkIndex] =
+			selectedChunkSkipList->chunkSkipNodeArray[0][chunkIndex].rowCount;
+	}
+
 	/* load column data for projected columns */
 	ColumnBuffers **columnBuffersArray = palloc0(columnCount * sizeof(ColumnBuffers *));
 
@@ -381,6 +379,8 @@ LoadFilteredStripeBuffers(Relation relation, StripeMetadata *stripeMetadata,
 	stripeBuffers->columnCount = columnCount;
 	stripeBuffers->rowCount = StripeSkipListRowCount(selectedChunkSkipList);
 	stripeBuffers->columnBuffersArray = columnBuffersArray;
+	stripeBuffers->selectedChunks = selectedChunkCount;
+	stripeBuffers->selectedChunkRowCount = selectedChunkRowCount;
 
 	return stripeBuffers;
 }

src/backend/columnar/cstore_tableam.c

@@ -14,7 +14,7 @@
 #include "access/tableam.h"
 #include "access/tsmapi.h"
 #if PG_VERSION_NUM >= 130000
-#include "access/heaptoast.h"
+#include "access/detoast.h"
 #else
 #include "access/tuptoaster.h"
 #endif
@@ -50,16 +50,14 @@
 #include "utils/lsyscache.h"
 #include "utils/syscache.h"
 
-#include "columnar/cstore.h"
+#include "columnar/columnar.h"
-#include "columnar/cstore_customscan.h"
+#include "columnar/columnar_customscan.h"
-#include "columnar/cstore_tableam.h"
+#include "columnar/columnar_tableam.h"
-#include "columnar/cstore_version_compat.h"
+#include "columnar/columnar_version_compat.h"
 #include "distributed/commands.h"
 #include "distributed/commands/utility_hook.h"
 #include "distributed/metadata_cache.h"
 
-#define CSTORE_TABLEAM_NAME "columnar"
-
 /*
  * Timing parameters for truncate locking heuristics.
  *
@@ -108,9 +106,10 @@ static void LogRelationStats(Relation rel, int elevel);
 static void TruncateColumnar(Relation rel, int elevel);
 static HeapTuple ColumnarSlotCopyHeapTuple(TupleTableSlot *slot);
 static void ColumnarCheckLogicalReplication(Relation rel);
+static Datum * detoast_values(TupleDesc tupleDesc, Datum *orig_values, bool *isnull);
 
 /* Custom tuple slot ops used for columnar. Initialized in columnar_tableam_init(). */
-TupleTableSlotOps TTSOpsColumnar;
+static TupleTableSlotOps TTSOpsColumnar;
 
 static List *
 RelationColumnList(Relation rel)
@@ -159,7 +158,7 @@ columnar_beginscan(Relation relation, Snapshot snapshot,
 
 	attr_needed = bms_add_range(attr_needed, 0, natts - 1);
 
-	/* the cstore access method does not use the flags, they are specific to heap */
+	/* the columnar access method does not use the flags, they are specific to heap */
 	flags = 0;
 
 	TableScanDesc scandesc = columnar_beginscan_extended(relation, snapshot, nkeys, key,
@@ -437,24 +436,16 @@ columnar_tuple_insert(Relation relation, TupleTableSlot *slot, CommandId cid,
 	TableWriteState *writeState = columnar_init_write_state(relation,
 															RelationGetDescr(relation),
 															GetCurrentSubTransactionId());
-
 	MemoryContext oldContext = MemoryContextSwitchTo(writeState->perTupleContext);
 
-	HeapTuple heapTuple = ExecCopySlotHeapTuple(slot);
-
 	ColumnarCheckLogicalReplication(relation);
-	if (HeapTupleHasExternal(heapTuple))
-	{
-		/* detoast any toasted attributes */
-		HeapTuple newTuple = toast_flatten_tuple(heapTuple,
-												 slot->tts_tupleDescriptor);
-
-		ExecForceStoreHeapTuple(newTuple, slot, true);
-	}
 
 	slot_getallattrs(slot);
 
-	ColumnarWriteRow(writeState, slot->tts_values, slot->tts_isnull);
+	Datum *values = detoast_values(slot->tts_tupleDescriptor,
+								   slot->tts_values, slot->tts_isnull);
+
+	ColumnarWriteRow(writeState, values, slot->tts_isnull);
 
 	MemoryContextSwitchTo(oldContext);
 	MemoryContextReset(writeState->perTupleContext);
@@ -487,28 +478,23 @@ columnar_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
 															GetCurrentSubTransactionId());
 
 	ColumnarCheckLogicalReplication(relation);
+
+	MemoryContext oldContext = MemoryContextSwitchTo(writeState->perTupleContext);
+
 	for (int i = 0; i < ntuples; i++)
 	{
 		TupleTableSlot *tupleSlot = slots[i];
-		MemoryContext oldContext = MemoryContextSwitchTo(writeState->perTupleContext);
-		HeapTuple heapTuple = ExecCopySlotHeapTuple(tupleSlot);
-
-		if (HeapTupleHasExternal(heapTuple))
-		{
-			/* detoast any toasted attributes */
-			HeapTuple newTuple = toast_flatten_tuple(heapTuple,
-													 tupleSlot->tts_tupleDescriptor);
-
-			ExecForceStoreHeapTuple(newTuple, tupleSlot, true);
-		}
 
 		slot_getallattrs(tupleSlot);
 
-		ColumnarWriteRow(writeState, tupleSlot->tts_values, tupleSlot->tts_isnull);
+		Datum *values = detoast_values(tupleSlot->tts_tupleDescriptor,
-		MemoryContextSwitchTo(oldContext);
+									   tupleSlot->tts_values, tupleSlot->tts_isnull);
+
+		ColumnarWriteRow(writeState, values, tupleSlot->tts_isnull);
+		MemoryContextReset(writeState->perTupleContext);
 	}
 
-	MemoryContextReset(writeState->perTupleContext);
+	MemoryContextSwitchTo(oldContext);
 }
 
 
@@ -618,7 +604,7 @@ columnar_relation_copy_data(Relation rel, const RelFileNode *newrnode)
  * we should copy data from OldHeap to NewHeap.
  *
  * In general TableAM case this can also be called for the CLUSTER command
- * which is not applicable for cstore since it doesn't support indexes.
+ * which is not applicable for columnar since it doesn't support indexes.
  */
 static void
 columnar_relation_copy_for_cluster(Relation OldHeap, Relation NewHeap,
@@ -647,11 +633,11 @@ columnar_relation_copy_for_cluster(Relation OldHeap, Relation NewHeap,
 	Assert(sourceDesc->natts == targetDesc->natts);
 
 	/* read settings from old heap, relfilenode will be swapped at the end */
-	ColumnarOptions cstoreOptions = { 0 };
+	ColumnarOptions columnarOptions = { 0 };
-	ReadColumnarOptions(OldHeap->rd_id, &cstoreOptions);
+	ReadColumnarOptions(OldHeap->rd_id, &columnarOptions);
 
 	TableWriteState *writeState = ColumnarBeginWrite(NewHeap->rd_node,
-													 cstoreOptions,
+													 columnarOptions,
 													 targetDesc);
 
 	TableReadState *readState = ColumnarBeginRead(OldHeap, sourceDesc,
@@ -814,7 +800,7 @@ LogRelationStats(Relation rel, int elevel)
 
 /*
  * TruncateColumnar truncates the unused space at the end of main fork for
- * a cstore table. This unused space can be created by aborted transactions.
+ * a columnar table. This unused space can be created by aborted transactions.
  *
  * This implementation is based on heap_vacuum_rel in vacuumlazy.c with some
  * changes so it suits columnar store relations.
@@ -1162,9 +1148,10 @@ columnar_tableam_finish()
 int64
 ColumnarGetChunksFiltered(TableScanDesc scanDesc)
 {
-	ColumnarScanDesc cstoreScanDesc = (ColumnarScanDesc) scanDesc;
+	ColumnarScanDesc columnarScanDesc = (ColumnarScanDesc) scanDesc;
-	TableReadState *readState = cstoreScanDesc->cs_readState;
+	TableReadState *readState = columnarScanDesc->cs_readState;
 
+	/* readState is initialized lazily */
 	if (readState != NULL)
 	{
 		return readState->chunksFiltered;
@@ -1401,6 +1388,45 @@ columnar_handler(PG_FUNCTION_ARGS)
 }
 
 
+/*
+ * detoast_values
+ *
+ * Detoast and decompress all values. If there's no work to do, return
+ * original pointer; otherwise return a newly-allocated values array. Should
+ * be called in per-tuple context.
+ */
+static Datum *
+detoast_values(TupleDesc tupleDesc, Datum *orig_values, bool *isnull)
+{
+	int natts = tupleDesc->natts;
+
+	/* copy on write to optimize for case where nothing is toasted */
+	Datum *values = orig_values;
+
+	for (int i = 0; i < tupleDesc->natts; i++)
+	{
+		if (!isnull[i] && tupleDesc->attrs[i].attlen == -1 &&
+			VARATT_IS_EXTENDED(values[i]))
+		{
+			/* make a copy */
+			if (values == orig_values)
+			{
+				values = palloc(sizeof(Datum) * natts);
+				memcpy_s(values, sizeof(Datum) * natts,
+						 orig_values, sizeof(Datum) * natts);
+			}
+
+			/* will be freed when per-tuple context is reset */
+			struct varlena *new_value = (struct varlena *) DatumGetPointer(values[i]);
+			new_value = detoast_attr(new_value);
+			values[i] = PointerGetDatum(new_value);
+		}
+	}
+
+	return values;
+}
+
+
 /*
  * ColumnarCheckLogicalReplication throws an error if the relation is
  * part of any publication. This should be called before any write to
@@ -1609,7 +1635,7 @@ alter_columnar_table_set(PG_FUNCTION_ARGS)
 		options.compressionType = ParseCompressionType(NameStr(*compressionName));
 		if (options.compressionType == COMPRESSION_TYPE_INVALID)
 		{
-			ereport(ERROR, (errmsg("unknown compression type for cstore table: %s",
+			ereport(ERROR, (errmsg("unknown compression type for columnar table: %s",
 								   quote_identifier(NameStr(*compressionName)))));
 		}
 		ereport(DEBUG1, (errmsg("updating compression to %s",

src/backend/columnar/cstore_writer.c

@@ -1,8 +1,8 @@
 /*-------------------------------------------------------------------------
  *
- * cstore_writer.c
+ * columnar_writer.c
  *
- * This file contains function definitions for writing cstore files. This
+ * This file contains function definitions for writing columnar tables. This
  * includes the logic for writing file level metadata, writing row stripes,
  * and calculating chunk skip nodes.
  *
@@ -29,8 +29,8 @@
 #include "utils/rel.h"
 #include "utils/relfilenodemap.h"
 
-#include "columnar/cstore.h"
+#include "columnar/columnar.h"
-#include "columnar/cstore_version_compat.h"
+#include "columnar/columnar_version_compat.h"
 
 static StripeBuffers * CreateEmptyStripeBuffers(uint32 stripeMaxRowCount,
 												uint32 chunkRowCount,
@@ -53,11 +53,9 @@ static Datum DatumCopy(Datum datum, bool datumTypeByValue, int datumTypeLength);
 static StringInfo CopyStringInfo(StringInfo sourceString);
 
 /*
- * ColumnarBeginWrite initializes a cstore data load operation and returns a table
+ * ColumnarBeginWrite initializes a columnar data load operation and returns a table
  * handle. This handle should be used for adding the row values and finishing the
- * data load operation. If the cstore footer file already exists, we read the
+ * data load operation.
- * footer and then seek to right after the last stripe  where the new stripes
- * will be added.
  */
 TableWriteState *
 ColumnarBeginWrite(RelFileNode relfilenode,
@@ -118,7 +116,7 @@ ColumnarBeginWrite(RelFileNode relfilenode,
 
 
 /*
- * ColumnarWriteRow adds a row to the cstore file. If the stripe is not initialized,
+ * ColumnarWriteRow adds a row to the columnar table. If the stripe is not initialized,
  * we create structures to hold stripe data and skip list. Then, we serialize and
  * append data to serialized value buffer for each of the columns and update
  * corresponding skip nodes. Then, whole chunk data is compressed at every
@@ -214,10 +212,8 @@ ColumnarWriteRow(TableWriteState *writeState, Datum *columnValues, bool *columnN
 
 
 /*
- * ColumnarEndWrite finishes a cstore data load operation. If we have an unflushed
+ * ColumnarEndWrite finishes a columnar data load operation. If we have an unflushed
- * stripe, we flush it. Then, we sync and close the cstore data file. Last, we
+ * stripe, we flush it.
- * flush the footer to a temporary file, and atomically rename this temporary
- * file to the original footer file.
  */
 void
 ColumnarEndWrite(TableWriteState *writeState)
@@ -373,7 +369,7 @@ WriteToSmgr(Relation rel, uint64 logicalOffset, char *data, uint32 dataLength)
 			XLogBeginInsert();
 
 			/*
-			 * Since cstore will mostly write whole pages we force the transmission of the
+			 * Since columnar will mostly write whole pages we force the transmission of the
 			 * whole image in the buffer
 			 */
 			XLogRegisterBuffer(0, buffer, REGBUF_FORCE_IMAGE);

src/backend/columnar/mod.c

@@ -17,11 +17,11 @@
 
 #include "citus_version.h"
 
-#include "columnar/cstore.h"
+#include "columnar/columnar.h"
 #include "columnar/mod.h"
 
 #ifdef HAS_TABLEAM
-#include "columnar/cstore_tableam.h"
+#include "columnar/columnar_tableam.h"
 #endif
 
 void

src/backend/columnar/sql/udfs/alter_columnar_table_reset/10.0-1.sql

@@ -14,4 +14,4 @@ COMMENT ON FUNCTION pg_catalog.alter_columnar_table_reset(
     stripe_row_count bool,
     compression bool,
     compression_level bool)
-IS 'reset on or more options on a cstore table to the system defaults';
+IS 'reset on or more options on a columnar table to the system defaults';

src/backend/columnar/sql/udfs/alter_columnar_table_reset/latest.sql

@@ -14,4 +14,4 @@ COMMENT ON FUNCTION pg_catalog.alter_columnar_table_reset(
     stripe_row_count bool,
     compression bool,
     compression_level bool)
-IS 'reset on or more options on a cstore table to the system defaults';
+IS 'reset on or more options on a columnar table to the system defaults';

src/backend/columnar/sql/udfs/alter_columnar_table_set/10.0-1.sql

@@ -14,4 +14,4 @@ COMMENT ON FUNCTION pg_catalog.alter_columnar_table_set(
     stripe_row_count int,
     compression name,
     compression_level int)
-IS 'set one or more options on a cstore table, when set to NULL no change is made';
+IS 'set one or more options on a columnar table, when set to NULL no change is made';

src/backend/columnar/sql/udfs/alter_columnar_table_set/latest.sql

@@ -14,4 +14,4 @@ COMMENT ON FUNCTION pg_catalog.alter_columnar_table_set(
     stripe_row_count int,
     compression name,
     compression_level int)
-IS 'set one or more options on a cstore table, when set to NULL no change is made';
+IS 'set one or more options on a columnar table, when set to NULL no change is made';

src/backend/columnar/write_state_management.c

@@ -2,7 +2,7 @@
 #include "citus_version.h"
 
 #include "postgres.h"
-#include "columnar/cstore.h"
+#include "columnar/columnar.h"
 
 #if HAS_TABLEAM
 
@@ -47,9 +47,9 @@
 #include "utils/rel.h"
 #include "utils/syscache.h"
 
-#include "columnar/cstore_customscan.h"
+#include "columnar/columnar_customscan.h"
-#include "columnar/cstore_tableam.h"
+#include "columnar/columnar_tableam.h"
-#include "columnar/cstore_version_compat.h"
+#include "columnar/columnar_version_compat.h"
 
 
 /*
@@ -178,12 +178,12 @@ columnar_init_write_state(Relation relation, TupleDesc tupdesc,
 	 */
 	MemoryContext oldContext = MemoryContextSwitchTo(WriteStateContext);
 
-	ColumnarOptions cstoreOptions = { 0 };
+	ColumnarOptions columnarOptions = { 0 };
-	ReadColumnarOptions(relation->rd_id, &cstoreOptions);
+	ReadColumnarOptions(relation->rd_id, &columnarOptions);
 
 	SubXidWriteState *stackEntry = palloc0(sizeof(SubXidWriteState));
 	stackEntry->writeState = ColumnarBeginWrite(relation->rd_node,
-												cstoreOptions,
+												columnarOptions,
 												tupdesc);
 	stackEntry->subXid = currentSubXid;
 	stackEntry->next = hashEntry->writeStateStack;

src/backend/distributed/commands/alter_table.c

@@ -32,8 +32,8 @@
 #include "access/xact.h"
 #include "catalog/dependency.h"
 #include "catalog/pg_am.h"
-#include "columnar/cstore.h"
+#include "columnar/columnar.h"
-#include "columnar/cstore_tableam.h"
+#include "columnar/columnar_tableam.h"
 #include "distributed/colocation_utils.h"
 #include "distributed/commands.h"
 #include "distributed/commands/utility_hook.h"
@@ -41,6 +41,7 @@
 #include "distributed/deparser.h"
 #include "distributed/distribution_column.h"
 #include "distributed/listutils.h"
+#include "distributed/local_executor.h"
 #include "distributed/metadata/dependency.h"
 #include "distributed/metadata_cache.h"
 #include "distributed/metadata_sync.h"
@@ -487,6 +488,15 @@ AlterTableSetAccessMethod(TableConversionParameters *params)
 TableConversionReturn *
 ConvertTable(TableConversionState *con)
 {
+	/*
+	 * We undistribute citus local tables that are not chained with any reference
+	 * tables via foreign keys at the end of the utility hook.
+	 * Here we temporarily set the related GUC to off to disable the logic for
+	 * internally executed DDL's that might invoke this mechanism unnecessarily.
+	 */
+	bool oldEnableLocalReferenceForeignKeys = EnableLocalReferenceForeignKeys;
+	SetLocalEnableLocalReferenceForeignKeys(false);
+
 	if (con->conversionType == UNDISTRIBUTE_TABLE && con->cascadeViaForeignKeys &&
 		(TableReferencing(con->relationId) || TableReferenced(con->relationId)))
 	{
@@ -501,6 +511,7 @@ ConvertTable(TableConversionState *con)
 		 * Undistributed every foreign key connected relation in our foreign key
 		 * subgraph including itself, so return here.
 		 */
+		SetLocalEnableLocalReferenceForeignKeys(oldEnableLocalReferenceForeignKeys);
 		return NULL;
 	}
 	char *newAccessMethod = con->accessMethod ? con->accessMethod :
@@ -742,6 +753,8 @@ ConvertTable(TableConversionState *con)
 	/* increment command counter so that next command can see the new table */
 	CommandCounterIncrement();
 
+	SetLocalEnableLocalReferenceForeignKeys(oldEnableLocalReferenceForeignKeys);
+
 	return ret;
 }
 

src/backend/distributed/commands/cascade_table_operation_for_connected_relations.c

@@ -28,13 +28,13 @@
 #include "distributed/reference_table_utils.h"
 #include "distributed/relation_access_tracking.h"
 #include "distributed/worker_protocol.h"
+#include "miscadmin.h"
 #include "utils/builtins.h"
 #include "utils/lsyscache.h"
 #include "utils/syscache.h"
 
 
 static void EnsureSequentialModeForCitusTableCascadeFunction(List *relationIdList);
-static bool RelationIdListHasReferenceTable(List *relationIdList);
 static void LockRelationsWithLockMode(List *relationIdList, LOCKMODE lockMode);
 static List * RemovePartitionRelationIds(List *relationIdList);
 static List * GetFKeyCreationCommandsForRelationIdList(List *relationIdList);
@@ -222,7 +222,7 @@ EnsureSequentialModeForCitusTableCascadeFunction(List *relationIdList)
  * RelationIdListHasReferenceTable returns true if relationIdList has a relation
  * id that belongs to a reference table.
  */
-static bool
+bool
 RelationIdListHasReferenceTable(List *relationIdList)
 {
 	Oid relationId = InvalidOid;
@@ -282,8 +282,34 @@ DropRelationIdListForeignKeys(List *relationIdList, int fKeyFlags)
 void
 DropRelationForeignKeys(Oid relationId, int fKeyFlags)
 {
+	/*
+	 * We undistribute citus local tables that are not chained with any reference
+	 * tables via foreign keys at the end of the utility hook.
+	 * Here we temporarily set the related GUC to off to disable the logic for
+	 * internally executed DDL's that might invoke this mechanism unnecessarily.
+	 */
+	bool oldEnableLocalReferenceForeignKeys = EnableLocalReferenceForeignKeys;
+	SetLocalEnableLocalReferenceForeignKeys(false);
+
 	List *dropFkeyCascadeCommandList = GetRelationDropFkeyCommands(relationId, fKeyFlags);
 	ExecuteAndLogDDLCommandList(dropFkeyCascadeCommandList);
+
+	SetLocalEnableLocalReferenceForeignKeys(oldEnableLocalReferenceForeignKeys);
+}
+
+
+/*
+ * SetLocalEnableLocalReferenceForeignKeys is simply a C interface for setting
+ * the following:
+ *      SET LOCAL citus.enable_local_reference_table_foreign_keys = 'on'|'off';
+ */
+void
+SetLocalEnableLocalReferenceForeignKeys(bool state)
+{
+	char *stateStr = state ? "on" : "off";
+	set_config_option("citus.enable_local_reference_table_foreign_keys", stateStr,
+					  (superuser() ? PGC_SUSET : PGC_USERSET), PGC_S_SESSION,
+					  GUC_ACTION_LOCAL, true, 0, false);
 }
 
 

src/backend/distributed/commands/create_citus_local_table.c

@@ -72,6 +72,7 @@ static void FinalizeCitusLocalTableCreation(Oid relationId);
 
 
 PG_FUNCTION_INFO_V1(create_citus_local_table);
+PG_FUNCTION_INFO_V1(remove_local_tables_from_metadata);
 
 
 /*
@@ -84,6 +85,23 @@ create_citus_local_table(PG_FUNCTION_ARGS)
 {
 	CheckCitusVersion(ERROR);
 
+	if (ShouldEnableLocalReferenceForeignKeys())
+	{
+		/*
+		 * When foreign keys between reference tables and postgres tables are
+		 * enabled, we automatically undistribute citus local tables that are
+		 * not chained with any reference tables back to postgres tables.
+		 * So give a warning to user for that.
+		 */
+		ereport(WARNING, (errmsg("citus local tables that are not chained with "
+								 "reference tables via foreign keys might be "
+								 "automatically converted back to postgres tables"),
+						  errhint("Consider setting "
+								  "citus.enable_local_reference_table_foreign_keys "
+								  "to 'off' to disable automatically undistributing "
+								  "citus local tables")));
+	}
+
 	Oid relationId = PG_GETARG_OID(0);
 	bool cascadeViaForeignKeys = PG_GETARG_BOOL(1);
 
@@ -93,6 +111,22 @@ create_citus_local_table(PG_FUNCTION_ARGS)
 }
 
 
+/*
+ * remove_local_tables_from_metadata undistributes citus local
+ * tables that are not chained with any reference tables via foreign keys.
+ */
+Datum
+remove_local_tables_from_metadata(PG_FUNCTION_ARGS)
+{
+	CheckCitusVersion(ERROR);
+	EnsureCoordinator();
+
+	UndistributeDisconnectedCitusLocalTables();
+
+	PG_RETURN_VOID();
+}
+
+
 /*
  * CreateCitusLocalTable is the internal method that creates a citus table
  * from the table with relationId. The created table would have the following

src/backend/distributed/commands/create_distributed_table.c

@@ -44,6 +44,7 @@
 #include "distributed/deparser.h"
 #include "distributed/distribution_column.h"
 #include "distributed/listutils.h"
+#include "distributed/local_executor.h"
 #include "distributed/metadata_utility.h"
 #include "distributed/coordinator_protocol.h"
 #include "distributed/metadata/dependency.h"
@@ -114,9 +115,10 @@ static void EnsureLocalTableEmptyIfNecessary(Oid relationId, char distributionMe
 static bool ShouldLocalTableBeEmpty(Oid relationId, char distributionMethod, bool
 									viaDeprecatedAPI);
 static void EnsureCitusTableCanBeCreated(Oid relationOid);
-static List * GetFKeyCreationCommandsRelationInvolved(Oid relationId);
+static List * GetFKeyCreationCommandsRelationInvolvedWithTableType(Oid relationId,
+																   int tableTypeFlag);
 static Oid DropFKeysAndUndistributeTable(Oid relationId);
-static void DropFKeysRelationInvolved(Oid relationId);
+static void DropFKeysRelationInvolvedWithTableType(Oid relationId, int tableTypeFlag);
 static bool LocalTableEmpty(Oid tableId);
 static void CopyLocalDataIntoShards(Oid relationId);
 static List * TupleDescColumnNameList(TupleDesc tupleDescriptor);
@@ -354,15 +356,50 @@ CreateDistributedTable(Oid relationId, Var *distributionColumn, char distributio
 	 * given relation is involved, then we undistribute the relation and finally
 	 * we re-create dropped foreign keys at the end of this function.
 	 */
-	List *fKeyCreationCommandsRelationInvolved = NIL;
+	List *originalForeignKeyRecreationCommands = NIL;
 	if (IsCitusTableType(relationId, CITUS_LOCAL_TABLE))
 	{
 		/* store foreign key creation commands that relation is involved */
-		fKeyCreationCommandsRelationInvolved =
+		originalForeignKeyRecreationCommands =
-			GetFKeyCreationCommandsRelationInvolved(relationId);
+			GetFKeyCreationCommandsRelationInvolvedWithTableType(relationId,
+																 INCLUDE_ALL_TABLE_TYPES);
 		relationId = DropFKeysAndUndistributeTable(relationId);
 	}
 
+	/*
+	 * To support foreign keys between reference tables and local tables,
+	 * we drop & re-define foreign keys at the end of this function so
+	 * that ALTER TABLE hook does the necessary job, which means converting
+	 * local tables to citus local tables to properly support such foreign
+	 * keys.
+	 *
+	 * This function does not expect to create Citus local table, so we blindly
+	 * create reference table when the method is DISTRIBUTE_BY_NONE.
+	 */
+	else if (distributionMethod == DISTRIBUTE_BY_NONE &&
+			 ShouldEnableLocalReferenceForeignKeys() &&
+			 HasForeignKeyWithLocalTable(relationId))
+	{
+		/*
+		 * Store foreign key creation commands for foreign key relationships
+		 * that relation has with postgres tables.
+		 */
+		originalForeignKeyRecreationCommands =
+			GetFKeyCreationCommandsRelationInvolvedWithTableType(relationId,
+																 INCLUDE_LOCAL_TABLES);
+
+		/*
+		 * Soon we will convert local tables to citus local tables. As
+		 * CreateCitusLocalTable needs to use local execution, now we
+		 * switch to local execution beforehand so that reference table
+		 * creation doesn't use remote execution and we don't error out
+		 * in CreateCitusLocalTable.
+		 */
+		SetLocalExecutionStatus(LOCAL_EXECUTION_REQUIRED);
+
+		DropFKeysRelationInvolvedWithTableType(relationId, INCLUDE_LOCAL_TABLES);
+	}
+
 	/*
 	 * distributed tables might have dependencies on different objects, since we create
 	 * shards for a distributed table via multiple sessions these objects will be created
@@ -426,6 +463,10 @@ CreateDistributedTable(Oid relationId, Var *distributionColumn, char distributio
 	}
 	else if (distributionMethod == DISTRIBUTE_BY_NONE)
 	{
+		/*
+		 * This function does not expect to create Citus local table, so we blindly
+		 * create reference table when the method is DISTRIBUTE_BY_NONE.
+		 */
 		CreateReferenceTableShard(relationId);
 	}
 
@@ -472,27 +513,28 @@ CreateDistributedTable(Oid relationId, Var *distributionColumn, char distributio
 	 * we can skip the validation of the foreign keys.
 	 */
 	bool skip_validation = true;
-	ExecuteForeignKeyCreateCommandList(fKeyCreationCommandsRelationInvolved,
+	ExecuteForeignKeyCreateCommandList(originalForeignKeyRecreationCommands,
 									   skip_validation);
 }
 
 
 /*
- * GetFKeyCreationCommandsRelationInvolved returns a list of DDL commands to
+ * GetFKeyCreationCommandsRelationInvolvedWithTableType returns a list of DDL
- * recreate the foreign keys that relation with relationId is involved.
+ * commands to recreate the foreign keys that relation with relationId is involved
+ * with given table type.
  */
 static List *
-GetFKeyCreationCommandsRelationInvolved(Oid relationId)
+GetFKeyCreationCommandsRelationInvolvedWithTableType(Oid relationId, int tableTypeFlag)
 {
 	int referencingFKeysFlag = INCLUDE_REFERENCING_CONSTRAINTS |
-							   INCLUDE_ALL_TABLE_TYPES;
+							   tableTypeFlag;
 	List *referencingFKeyCreationCommands =
 		GetForeignConstraintCommandsInternal(relationId, referencingFKeysFlag);
 
 	/* already captured self referencing foreign keys, so use EXCLUDE_SELF_REFERENCES */
 	int referencedFKeysFlag = INCLUDE_REFERENCED_CONSTRAINTS |
 							  EXCLUDE_SELF_REFERENCES |
-							  INCLUDE_ALL_TABLE_TYPES;
+							  tableTypeFlag;
 	List *referencedFKeyCreationCommands =
 		GetForeignConstraintCommandsInternal(relationId, referencedFKeysFlag);
 	return list_concat(referencingFKeyCreationCommands, referencedFKeyCreationCommands);
@@ -510,7 +552,7 @@ GetFKeyCreationCommandsRelationInvolved(Oid relationId)
 static Oid
 DropFKeysAndUndistributeTable(Oid relationId)
 {
-	DropFKeysRelationInvolved(relationId);
+	DropFKeysRelationInvolvedWithTableType(relationId, INCLUDE_ALL_TABLE_TYPES);
 
 	/* store them before calling UndistributeTable as it changes relationId */
 	char *relationName = get_rel_name(relationId);
@@ -535,20 +577,20 @@ DropFKeysAndUndistributeTable(Oid relationId)
 
 
 /*
- * DropFKeysRelationInvolved drops all foreign keys that relation with
+ * DropFKeysRelationInvolvedWithTableType drops foreign keys that relation
- * relationId is involved.
+ * with relationId is involved with given table type.
  */
 static void
-DropFKeysRelationInvolved(Oid relationId)
+DropFKeysRelationInvolvedWithTableType(Oid relationId, int tableTypeFlag)
 {
 	int referencingFKeysFlag = INCLUDE_REFERENCING_CONSTRAINTS |
-							   INCLUDE_ALL_TABLE_TYPES;
+							   tableTypeFlag;
 	DropRelationForeignKeys(relationId, referencingFKeysFlag);
 
 	/* already captured self referencing foreign keys, so use EXCLUDE_SELF_REFERENCES */
 	int referencedFKeysFlag = INCLUDE_REFERENCED_CONSTRAINTS |
 							  EXCLUDE_SELF_REFERENCES |
-							  INCLUDE_ALL_TABLE_TYPES;
+							  tableTypeFlag;
 	DropRelationForeignKeys(relationId, referencedFKeysFlag);
 }
 

src/backend/distributed/commands/drop_distributed_table.c

@@ -12,6 +12,7 @@
 #include "miscadmin.h"
 
 #include "distributed/commands/utility_hook.h"
+#include "distributed/commands.h"
 #include "distributed/metadata_utility.h"
 #include "distributed/coordinator_protocol.h"
 #include "distributed/metadata_sync.h"
@@ -30,6 +31,7 @@ static void MasterRemoveDistributedTableMetadataFromWorkers(Oid relationId,
 PG_FUNCTION_INFO_V1(master_drop_distributed_table_metadata);
 PG_FUNCTION_INFO_V1(master_remove_partition_metadata);
 PG_FUNCTION_INFO_V1(master_remove_distributed_table_metadata_from_workers);
+PG_FUNCTION_INFO_V1(notify_constraint_dropped);
 
 
 /*
@@ -148,3 +150,30 @@ MasterRemoveDistributedTableMetadataFromWorkers(Oid relationId, char *schemaName
 	char *deleteDistributionCommand = DistributionDeleteCommand(schemaName, tableName);
 	SendCommandToWorkersWithMetadata(deleteDistributionCommand);
 }
+
+
+/*
+ * notify_constraint_dropped simply calls NotifyUtilityHookConstraintDropped
+ * to set ConstraintDropped to true.
+ * This udf is designed to be called from citus_drop_trigger to tell us we
+ * dropped a table constraint.
+ */
+Datum
+notify_constraint_dropped(PG_FUNCTION_ARGS)
+{
+	CheckCitusVersion(ERROR);
+
+	/*
+	 * We reset this only in utility hook, so we should not set this flag
+	 * otherwise if we are not in utility hook.
+	 * In some cases -where dropping foreign key not issued via utility
+	 * hook-, we would not be able to undistribute such citus local tables
+	 * but we are ok with that.
+	 */
+	if (UtilityHookLevel >= 1)
+	{
+		NotifyUtilityHookConstraintDropped();
+	}
+
+	PG_RETURN_VOID();
+}

src/backend/distributed/commands/foreign_constraint.c

@@ -78,6 +78,7 @@ static void ForeignConstraintFindDistKeys(HeapTuple pgConstraintTuple,
 static List * GetForeignKeyIdsForColumn(char *columnName, Oid relationId,
 										int searchForeignKeyColumnFlags);
 static Oid get_relation_constraint_oid_compat(HeapTuple heapTuple);
+static List * GetForeignKeysWithLocalTables(Oid relationId);
 static bool IsTableTypeIncluded(Oid relationId, int flags);
 static void UpdateConstraintIsValid(Oid constraintId, bool isValid);
 
@@ -711,6 +712,38 @@ get_relation_constraint_oid_compat(HeapTuple heapTuple)
 }
 
 
+/*
+ * HasForeignKeyToLocalTable returns true if relation has foreign key
+ * relationship with a local table.
+ */
+bool
+HasForeignKeyWithLocalTable(Oid relationId)
+{
+	List *foreignKeysWithLocalTables = GetForeignKeysWithLocalTables(relationId);
+	return list_length(foreignKeysWithLocalTables) > 0;
+}
+
+
+/*
+ * GetForeignKeysWithLocalTables returns a list foreign keys for foreign key
+ * relationaships that relation has with local tables.
+ */
+static List *
+GetForeignKeysWithLocalTables(Oid relationId)
+{
+	int referencingFKeysFlag = INCLUDE_REFERENCING_CONSTRAINTS |
+							   INCLUDE_LOCAL_TABLES;
+	List *referencingFKeyList = GetForeignKeyOids(relationId, referencingFKeysFlag);
+
+	/* already captured self referencing foreign keys, so use EXCLUDE_SELF_REFERENCES */
+	int referencedFKeysFlag = INCLUDE_REFERENCED_CONSTRAINTS |
+							  EXCLUDE_SELF_REFERENCES |
+							  INCLUDE_LOCAL_TABLES;
+	List *referencedFKeyList = GetForeignKeyOids(relationId, referencedFKeysFlag);
+	return list_concat(referencingFKeyList, referencedFKeyList);
+}
+
+
 /*
  * HasForeignKeyToCitusLocalTable returns true if any of the foreign key constraints
  * on the relation with relationId references to a citus local table.

src/backend/distributed/commands/table.c

@@ -48,7 +48,6 @@ bool EnableLocalReferenceForeignKeys = true;
 
 /* Local functions forward declarations for unsupported command checks */
 static void PostprocessCreateTableStmtForeignKeys(CreateStmt *createStatement);
-static bool ShouldEnableLocalReferenceForeignKeys(void);
 static void PostprocessCreateTableStmtPartitionOf(CreateStmt *createStatement,
 												  const char *queryString);
 static bool AlterTableDefinesFKeyBetweenPostgresAndNonDistTable(
@@ -251,7 +250,7 @@ PostprocessCreateTableStmtForeignKeys(CreateStmt *createStatement)
  * the value set by the user
  *
  */
-static bool
+bool
 ShouldEnableLocalReferenceForeignKeys(void)
 {
 	if (!EnableLocalReferenceForeignKeys)

src/backend/distributed/commands/utility_hook.c

@@ -46,6 +46,7 @@
 #include "distributed/commands/utility_hook.h" /* IWYU pragma: keep */
 #include "distributed/deparser.h"
 #include "distributed/deparse_shard_query.h"
+#include "distributed/foreign_key_relationship.h"
 #include "distributed/listutils.h"
 #include "distributed/local_executor.h"
 #include "distributed/maintenanced.h"
@@ -55,6 +56,7 @@
 #include "distributed/multi_executor.h"
 #include "distributed/multi_explain.h"
 #include "distributed/multi_physical_planner.h"
+#include "distributed/reference_table_utils.h"
 #include "distributed/resource_lock.h"
 #include "distributed/transmit.h"
 #include "distributed/version_compat.h"
@@ -72,6 +74,10 @@ PropSetCmdBehavior PropagateSetCommands = PROPSETCMD_NONE; /* SET prop off */
 static bool shouldInvalidateForeignKeyGraph = false;
 static int activeAlterTables = 0;
 static int activeDropSchemaOrDBs = 0;
+static bool ConstraintDropped = false;
+
+
+int UtilityHookLevel = 0;
 
 
 /* Local functions forward declarations for helper functions */
@@ -88,6 +94,7 @@ static void IncrementUtilityHookCountersIfNecessary(Node *parsetree);
 static void PostStandardProcessUtility(Node *parsetree);
 static void DecrementUtilityHookCountersIfNecessary(Node *parsetree);
 static bool IsDropSchemaOrDB(Node *parsetree);
+static bool ShouldUndistributeCitusLocalTables(void);
 
 
 /*
@@ -237,8 +244,41 @@ multi_ProcessUtility(PlannedStmt *pstmt,
 		return;
 	}
 
-	ProcessUtilityInternal(pstmt, queryString, context,
+	UtilityHookLevel++;
-						   params, queryEnv, dest, completionTag);
+
+	PG_TRY();
+	{
+		ProcessUtilityInternal(pstmt, queryString, context, params, queryEnv, dest,
+							   completionTag);
+
+		if (UtilityHookLevel == 1)
+		{
+			/*
+			 * When Citus local tables are disconnected from the foreign key graph, which
+			 * can happen due to various kinds of drop commands, we immediately
+			 * undistribute them at the end of the command.
+			 */
+			if (ShouldUndistributeCitusLocalTables())
+			{
+				UndistributeDisconnectedCitusLocalTables();
+			}
+			ResetConstraintDropped();
+		}
+
+		UtilityHookLevel--;
+	}
+	PG_CATCH();
+	{
+		if (UtilityHookLevel == 1)
+		{
+			ResetConstraintDropped();
+		}
+
+		UtilityHookLevel--;
+
+		PG_RE_THROW();
+	}
+	PG_END_TRY();
 }
 
 
@@ -647,6 +687,138 @@ ProcessUtilityInternal(PlannedStmt *pstmt,
 }
 
 
+/*
+ * UndistributeDisconnectedCitusLocalTables undistributes citus local tables that
+ * are not connected to any reference tables via their individual foreign key
+ * subgraphs.
+ */
+void
+UndistributeDisconnectedCitusLocalTables(void)
+{
+	List *citusLocalTableIdList = CitusTableTypeIdList(CITUS_LOCAL_TABLE);
+	citusLocalTableIdList = SortList(citusLocalTableIdList, CompareOids);
+
+	Oid citusLocalTableId = InvalidOid;
+	foreach_oid(citusLocalTableId, citusLocalTableIdList)
+	{
+		/* acquire ShareRowExclusiveLock to prevent concurrent foreign key creation */
+		LOCKMODE lockMode = ShareRowExclusiveLock;
+		LockRelationOid(citusLocalTableId, lockMode);
+
+		HeapTuple heapTuple =
+			SearchSysCache1(RELOID, ObjectIdGetDatum(citusLocalTableId));
+		if (!HeapTupleIsValid(heapTuple))
+		{
+			/*
+			 * UndistributeTable drops relation, skip if already undistributed
+			 * via cascade.
+			 */
+			continue;
+		}
+		ReleaseSysCache(heapTuple);
+
+		if (ConnectedToReferenceTableViaFKey(citusLocalTableId))
+		{
+			/* still connected to a reference table, skip it */
+			UnlockRelationOid(citusLocalTableId, lockMode);
+			continue;
+		}
+
+		/*
+		 * Citus local table is not connected to any reference tables, then
+		 * undistribute it via cascade. Here, instead of first dropping foreing
+		 * keys then undistributing the table, we just set cascadeViaForeignKeys
+		 * to true for simplicity.
+		 */
+		TableConversionParameters params = {
+			.relationId = citusLocalTableId,
+			.cascadeViaForeignKeys = true
+		};
+		UndistributeTable(&params);
+	}
+}
+
+
+/*
+ * ShouldUndistributeCitusLocalTables returns true if we might need to check
+ * citus local tables for their connectivity to reference tables.
+ */
+static bool
+ShouldUndistributeCitusLocalTables(void)
+{
+	if (!ConstraintDropped)
+	{
+		/*
+		 * citus_drop_trigger executes notify_constraint_dropped to set
+		 * ConstraintDropped to true, which means that last command dropped
+		 * a table constraint.
+		 */
+		return false;
+	}
+
+	if (!CitusHasBeenLoaded())
+	{
+		/*
+		 * If we are dropping citus, we should not try to undistribute citus
+		 * local tables as they will also be dropped.
+		 */
+		return false;
+	}
+
+	if (!InCoordinatedTransaction())
+	{
+		/* not interacting with any Citus objects */
+		return false;
+	}
+
+	if (IsCitusInitiatedRemoteBackend())
+	{
+		/* connection from the coordinator operating on a shard */
+		return false;
+	}
+
+	if (!ShouldEnableLocalReferenceForeignKeys())
+	{
+		/*
+		 * If foreign keys between reference tables and local tables are
+		 * disabled, then user might be using create_citus_local_table for
+		 * their own purposes. In that case, we should not undistribute
+		 * citus local tables.
+		 */
+		return false;
+	}
+
+	if (!IsCoordinator())
+	{
+		/* we should not perform this operation in worker nodes */
+		return false;
+	}
+
+	return true;
+}
+
+
+/*
+ * NotifyUtilityHookConstraintDropped sets ConstraintDropped to true to tell us
+ * last command dropped a table constraint.
+ */
+void
+NotifyUtilityHookConstraintDropped(void)
+{
+	ConstraintDropped = true;
+}
+
+
+/*
+ * ResetConstraintDropped sets ConstraintDropped to false.
+ */
+void
+ResetConstraintDropped(void)
+{
+	ConstraintDropped = false;
+}
+
+
 /*
  * IsDropSchemaOrDB returns true if parsetree represents DROP SCHEMA ...or
  * a DROP DATABASE.

src/backend/distributed/connection/connection_management.c

@@ -1292,20 +1292,13 @@ AfterXactHostConnectionHandling(ConnectionHashEntry *entry, bool isCommit)
 static bool
 ShouldShutdownConnection(MultiConnection *connection, const int cachedConnectionCount)
 {
-	bool isCitusInitiatedBackend = false;
-
 	/*
 	 * When we are in a backend that was created to serve an internal connection
 	 * from the coordinator or another worker, we disable connection caching to avoid
 	 * escalating the number of cached connections. We can recognize such backends
 	 * from their application name.
 	 */
-	if (application_name != NULL && strcmp(application_name, CITUS_APPLICATION_NAME) == 0)
+	return IsCitusInitiatedRemoteBackend() ||
-	{
-		isCitusInitiatedBackend = true;
-	}
-
-	return isCitusInitiatedBackend ||
 		   connection->initilizationState != POOL_STATE_INITIALIZED ||
 		   cachedConnectionCount >= MaxCachedConnectionsPerWorker ||
 		   connection->forceCloseAtTransactionEnd ||
@@ -1314,6 +1307,17 @@ ShouldShutdownConnection(MultiConnection *connection, const int cachedConnection
 }
 
 
+/*
+ * IsCitusInitiatedRemoteBackend returns true if we are in a backend that citus
+ * initiated via remote connection.
+ */
+bool
+IsCitusInitiatedRemoteBackend(void)
+{
+	return application_name && strcmp(application_name, CITUS_APPLICATION_NAME) == 0;
+}
+
+
 /*
  * ResetConnection preserves the given connection for later usage by
  * resetting its states.

src/backend/distributed/executor/adaptive_executor.c

@@ -3377,6 +3377,25 @@ TransactionStateMachine(WorkerSession *session)
 			case REMOTE_TRANS_SENT_COMMAND:
 			{
 				TaskPlacementExecution *placementExecution = session->currentTask;
+				if (placementExecution == NULL)
+				{
+					/*
+					 * We have seen accounts in production where the placementExecution
+					 * could inadvertently be not set. Investigation documented on
+					 * https://github.com/citusdata/citus-enterprise/issues/493
+					 * (due to sensitive data in the initial report it is not discussed
+					 * in our community repository)
+					 *
+					 * Currently we don't have a reliable way of reproducing this issue.
+					 * Erroring here seems to be a more desirable approach compared to a
+					 * SEGFAULT on the dereference of placementExecution, with a possible
+					 * crash recovery as a result.
+					 */
+					ereport(ERROR, (errmsg(
+										"unable to recover from inconsistent state in "
+										"the connection state machine on coordinator")));
+				}
+
 				ShardCommandExecution *shardCommandExecution =
 					placementExecution->shardCommandExecution;
 				Task *task = shardCommandExecution->task;

src/backend/distributed/operations/node_protocol.c

@@ -64,7 +64,7 @@
 #include "utils/ruleutils.h"
 #include "utils/varlena.h"
 
-#include "columnar/cstore_tableam.h"
+#include "columnar/columnar_tableam.h"
 
 /* Shard related configuration */
 int ShardCount = 32;

src/backend/distributed/sql/citus--9.5-1--10.0-1.sql

@@ -27,6 +27,7 @@ DROP FUNCTION IF EXISTS pg_catalog.citus_total_relation_size(regclass);
 #include "udfs/citus_move_shard_placement/10.0-1.sql"
 #include "udfs/citus_drop_trigger/10.0-1.sql"
 #include "udfs/worker_change_sequence_dependency/10.0-1.sql"
+#include "udfs/remove_local_tables_from_metadata/10.0-1.sql"
 
 #include "../../columnar/sql/columnar--9.5-1--10.0-1.sql"
 

src/backend/distributed/sql/downgrades/citus--10.0-1--9.5-1.sql

@@ -1,5 +1,18 @@
 -- citus--10.0-1--9.5-1
--- this is an empty downgrade path since citus--9.5-1--10.0-1.sql is empty for now
+
+-- In Citus 10.0, we added another internal udf (notify_constraint_dropped)
+-- to be called by citus_drop_trigger. Since this script is executed when
+-- downgrading Citus, we don't have notify_constraint_dropped in citus.so.
+-- For this reason, we first need to downgrade citus_drop_trigger so it doesn't
+-- call notify_constraint_dropped.
+-- To downgrade citus_drop_trigger, we first need to have the old version of
+-- citus_drop_all_shards as we renamed it in Citus 10.0.
+ALTER FUNCTION pg_catalog.citus_drop_all_shards(regclass, text, text)
+RENAME TO master_drop_all_shards;
+#include "../udfs/citus_drop_trigger/9.5-1.sql"
+
+-- Now we can safely drop notify_constraint_dropped as we downgraded citus_drop_trigger.
+DROP FUNCTION pg_catalog.notify_constraint_dropped();
 
 #include "../udfs/citus_finish_pg_upgrade/9.5-1.sql"
 
@@ -40,8 +53,6 @@ ALTER FUNCTION pg_catalog.citus_dist_placement_cache_invalidate()
 RENAME TO master_dist_placement_cache_invalidate;
 ALTER FUNCTION pg_catalog.citus_dist_shard_cache_invalidate()
 RENAME TO master_dist_shard_cache_invalidate;
-ALTER FUNCTION pg_catalog.citus_drop_all_shards(regclass, text, text)
-RENAME TO master_drop_all_shards;
 
 #include "../udfs/citus_conninfo_cache_invalidate/9.5-1.sql"
 #include "../udfs/citus_dist_local_group_cache_invalidate/9.5-1.sql"
@@ -89,7 +100,8 @@ CREATE FUNCTION pg_catalog.master_create_worker_shards(table_name text, shard_co
     AS 'MODULE_PATHNAME'
     LANGUAGE C STRICT;
 
-#include "../udfs/citus_drop_trigger/9.5-1.sql"
+DROP FUNCTION pg_catalog.remove_local_tables_from_metadata();
+
 #include "../udfs/citus_total_relation_size/7.0-1.sql"
 #include "../udfs/upgrade_to_reference_table/8.0-1.sql"
 #include "../udfs/undistribute_table/9.5-1.sql"

src/backend/distributed/sql/udfs/citus_drop_trigger/10.0-1.sql

@@ -1,9 +1,15 @@
+CREATE OR REPLACE FUNCTION pg_catalog.notify_constraint_dropped()
+    RETURNS void
+    LANGUAGE C STRICT
+    AS 'MODULE_PATHNAME', $$notify_constraint_dropped$$;
+
 CREATE OR REPLACE FUNCTION pg_catalog.citus_drop_trigger()
     RETURNS event_trigger
     LANGUAGE plpgsql
     SET search_path = pg_catalog
     AS $cdbdt$
 DECLARE
+    constraint_event_count INTEGER;
     v_obj record;
     sequence_names text[] := '{}';
     table_colocation_id integer;
@@ -25,6 +31,18 @@ BEGIN
     LOOP
         PERFORM master_unmark_object_distributed(v_obj.classid, v_obj.objid, v_obj.objsubid);
     END LOOP;
+
+    SELECT COUNT(*) INTO constraint_event_count
+    FROM pg_event_trigger_dropped_objects()
+    WHERE object_type IN ('table constraint');
+
+    IF constraint_event_count > 0
+    THEN
+        -- Tell utility hook that a table constraint is dropped so we might
+        -- need to undistribute some of the citus local tables that are not
+        -- connected to any reference tables.
+        PERFORM notify_constraint_dropped();
+    END IF;
 END;
 $cdbdt$;
 COMMENT ON FUNCTION pg_catalog.citus_drop_trigger()

src/backend/distributed/sql/udfs/citus_drop_trigger/latest.sql

@@ -1,9 +1,15 @@
+CREATE OR REPLACE FUNCTION pg_catalog.notify_constraint_dropped()
+    RETURNS void
+    LANGUAGE C STRICT
+    AS 'MODULE_PATHNAME', $$notify_constraint_dropped$$;
+
 CREATE OR REPLACE FUNCTION pg_catalog.citus_drop_trigger()
     RETURNS event_trigger
     LANGUAGE plpgsql
     SET search_path = pg_catalog
     AS $cdbdt$
 DECLARE
+    constraint_event_count INTEGER;
     v_obj record;
     sequence_names text[] := '{}';
     table_colocation_id integer;
@@ -25,6 +31,18 @@ BEGIN
     LOOP
         PERFORM master_unmark_object_distributed(v_obj.classid, v_obj.objid, v_obj.objsubid);
     END LOOP;
+
+    SELECT COUNT(*) INTO constraint_event_count
+    FROM pg_event_trigger_dropped_objects()
+    WHERE object_type IN ('table constraint');
+
+    IF constraint_event_count > 0
+    THEN
+        -- Tell utility hook that a table constraint is dropped so we might
+        -- need to undistribute some of the citus local tables that are not
+        -- connected to any reference tables.
+        PERFORM notify_constraint_dropped();
+    END IF;
 END;
 $cdbdt$;
 COMMENT ON FUNCTION pg_catalog.citus_drop_trigger()

src/backend/distributed/sql/udfs/remove_local_tables_from_metadata/10.0-1.sql

@@ -0,0 +1,6 @@
+CREATE OR REPLACE FUNCTION pg_catalog.remove_local_tables_from_metadata()
+	RETURNS void
+	LANGUAGE C STRICT
+	AS 'MODULE_PATHNAME', $$remove_local_tables_from_metadata$$;
+COMMENT ON FUNCTION pg_catalog.remove_local_tables_from_metadata()
+	IS 'undistribute citus local tables that are not chained with any reference tables via foreign keys';

src/backend/distributed/sql/udfs/remove_local_tables_from_metadata/latest.sql

@@ -0,0 +1,6 @@
+CREATE OR REPLACE FUNCTION pg_catalog.remove_local_tables_from_metadata()
+	RETURNS void
+	LANGUAGE C STRICT
+	AS 'MODULE_PATHNAME', $$remove_local_tables_from_metadata$$;
+COMMENT ON FUNCTION pg_catalog.remove_local_tables_from_metadata()
+	IS 'undistribute citus local tables that are not chained with any reference tables via foreign keys';

src/backend/distributed/test/foreign_key_relationship_query.c

@@ -14,12 +14,15 @@
 #include "fmgr.h"
 #include "funcapi.h"
 
+#include "catalog/dependency.h"
+#include "catalog/pg_constraint.h"
 #include "distributed/foreign_key_relationship.h"
 #include "distributed/coordinator_protocol.h"
 #include "distributed/listutils.h"
 #include "distributed/metadata_cache.h"
 #include "distributed/tuplestore.h"
 #include "distributed/version_compat.h"
+#include "utils/builtins.h"
 
 
 #define GET_FKEY_CONNECTED_RELATIONS_COLUMNS 1
@@ -29,6 +32,42 @@
 PG_FUNCTION_INFO_V1(get_referencing_relation_id_list);
 PG_FUNCTION_INFO_V1(get_referenced_relation_id_list);
 PG_FUNCTION_INFO_V1(get_foreign_key_connected_relations);
+PG_FUNCTION_INFO_V1(drop_constraint_cascade_via_perform_deletion);
+
+
+/*
+ * drop_constraint_cascade_via_perform_deletion simply drops constraint on
+ * relation via performDeletion.
+ */
+Datum
+drop_constraint_cascade_via_perform_deletion(PG_FUNCTION_ARGS)
+{
+	Oid relationId = PG_GETARG_OID(0);
+
+	if (PG_ARGISNULL(1))
+	{
+		/* avoid unexpected crashes in regression tests */
+		ereport(ERROR, (errmsg("cannot perform operation without constraint "
+							   "name argument")));
+	}
+
+	text *constraintNameText = PG_GETARG_TEXT_P(1);
+	char *constraintName = text_to_cstring(constraintNameText);
+
+	/* error if constraint does not exist */
+	bool missingOk = false;
+	Oid constraintId = get_relation_constraint_oid(relationId, constraintName, missingOk);
+
+	ObjectAddress constraintObjectAddress;
+	constraintObjectAddress.classId = ConstraintRelationId;
+	constraintObjectAddress.objectId = constraintId;
+	constraintObjectAddress.objectSubId = 0;
+
+	performDeletion(&constraintObjectAddress, DROP_CASCADE, 0);
+
+	PG_RETURN_VOID();
+}
+
 
 /*
  * get_referencing_relation_id_list returns the list of table oids that is referencing

src/backend/distributed/utils/foreign_key_relationship.c

@@ -23,9 +23,11 @@
 #include "access/table.h"
 #endif
 #include "catalog/pg_constraint.h"
+#include "distributed/commands.h"
 #include "distributed/foreign_key_relationship.h"
 #include "distributed/hash_helpers.h"
 #include "distributed/listutils.h"
+#include "distributed/metadata_cache.h"
 #include "distributed/version_compat.h"
 #include "nodes/pg_list.h"
 #include "storage/lockdefs.h"
@@ -144,6 +146,24 @@ GetForeignKeyConnectedRelationIdList(Oid relationId)
 }
 
 
+/*
+ * ConnectedToReferenceTableViaFKey returns true if given relationId is
+ * connected to a reference table via its foreign key subgraph.
+ */
+bool
+ConnectedToReferenceTableViaFKey(Oid relationId)
+{
+	/*
+	 * As we will operate on foreign key connected relations, here we
+	 * invalidate foreign key graph so that we act on fresh graph.
+	 */
+	InvalidateForeignKeyGraph();
+
+	List *fkeyConnectedRelations = GetForeignKeyConnectedRelationIdList(relationId);
+	return RelationIdListHasReferenceTable(fkeyConnectedRelations);
+}
+
+
 /*
  * GetRelationshipNodesForFKeyConnectedRelations performs breadth-first search
  * starting from input ForeignConstraintRelationshipNode and returns a list

src/include/columnar/columnar.h

@@ -1,6 +1,6 @@
 /*-------------------------------------------------------------------------
  *
- * cstore.h
+ * columnar.h
  *
  * Type and function declarations for Columnar
  *
@@ -11,8 +11,8 @@
  *-------------------------------------------------------------------------
  */
 
-#ifndef CSTORE_H
+#ifndef COLUMNAR_H
-#define CSTORE_H
+#define COLUMNAR_H
 #include "postgres.h"
 
 #include "fmgr.h"
@@ -37,22 +37,17 @@
 #define COMPRESSION_LEVEL_MIN 1
 #define COMPRESSION_LEVEL_MAX 19
 
-/* String representations of compression types */
-#define COMPRESSION_STRING_NONE "none"
-#define COMPRESSION_STRING_PG_LZ "pglz"
-
 /* Columnar file signature */
-#define CSTORE_MAGIC_NUMBER "citus_cstore"
+#define COLUMNAR_VERSION_MAJOR 1
-#define CSTORE_VERSION_MAJOR 1
+#define COLUMNAR_VERSION_MINOR 7
-#define CSTORE_VERSION_MINOR 7
 
 /* miscellaneous defines */
-#define CSTORE_TUPLE_COST_MULTIPLIER 10
+#define COLUMNAR_TUPLE_COST_MULTIPLIER 10
-#define CSTORE_POSTSCRIPT_SIZE_LENGTH 1
+#define COLUMNAR_POSTSCRIPT_SIZE_LENGTH 1
-#define CSTORE_POSTSCRIPT_SIZE_MAX 256
+#define COLUMNAR_POSTSCRIPT_SIZE_MAX 256
-#define CSTORE_BYTES_PER_PAGE (BLCKSZ - SizeOfPageHeaderData)
+#define COLUMNAR_BYTES_PER_PAGE (BLCKSZ - SizeOfPageHeaderData)
 
-/* Enumaration for cstore file's compression method */
+/* Enumaration for columnar table's compression method */
 typedef enum
 {
 	COMPRESSION_TYPE_INVALID = -1,
@@ -67,7 +62,7 @@ typedef enum
 
 /*
  * ColumnarOptions holds the option values to be used when reading or writing
- * a cstore file. To resolve these values, we first check foreign table's options,
+ * a columnar table. To resolve these values, we first check foreign table's options,
  * and if not present, we then fall back to the default values specified above.
  */
 typedef struct ColumnarOptions
@@ -93,7 +88,7 @@ typedef struct ColumnarTableDDLContext
 
 /*
  * StripeMetadata represents information about a stripe. This information is
- * stored in the cstore file's footer.
+ * stored in the metadata table "columnar.stripe".
  */
 typedef struct StripeMetadata
 {
@@ -107,13 +102,6 @@ typedef struct StripeMetadata
 } StripeMetadata;
 
 
-/* DataFileMetadata represents the metadata of a cstore file. */
-typedef struct DataFileMetadata
-{
-	List *stripeMetadataList;
-} DataFileMetadata;
-
-
 /* ColumnChunkSkipNode contains statistics for a ColumnChunkData. */
 typedef struct ColumnChunkSkipNode
 {
@@ -207,16 +195,24 @@ typedef struct ColumnBuffers
 } ColumnBuffers;
 
 
-/* StripeBuffers represents data for a row stripe in a cstore file. */
+/* StripeBuffers represents data for a row stripe. */
 typedef struct StripeBuffers
 {
 	uint32 columnCount;
 	uint32 rowCount;
 	ColumnBuffers **columnBuffersArray;
+
+	/*
+	 * We might skip reading some chunks because they're refuted by the
+	 * WHERE clause. We keep number of selected chunks and number of rows
+	 * in each of them.
+	 */
+	uint32 selectedChunks;
+	uint32 *selectedChunkRowCount;
 } StripeBuffers;
 
 
-/* TableReadState represents state of a cstore file read operation. */
+/* TableReadState represents state of a columnar scan. */
 typedef struct TableReadState
 {
 	List *stripeList;
@@ -242,7 +238,7 @@ typedef struct TableReadState
 } TableReadState;
 
 
-/* TableWriteState represents state of a cstore file write operation. */
+/* TableWriteState represents state of a columnar write operation. */
 typedef struct TableWriteState
 {
 	TupleDesc tupleDescriptor;
@@ -274,7 +270,7 @@ extern void columnar_init_gucs(void);
 
 extern CompressionType ParseCompressionType(const char *compressionTypeString);
 
-/* Function declarations for writing to a cstore file */
+/* Function declarations for writing to a columnar table */
 extern TableWriteState * ColumnarBeginWrite(RelFileNode relfilenode,
 											ColumnarOptions options,
 											TupleDesc tupleDescriptor);
@@ -284,7 +280,7 @@ extern void ColumnarFlushPendingWrites(TableWriteState *state);
 extern void ColumnarEndWrite(TableWriteState *state);
 extern bool ContainsPendingWrites(TableWriteState *state);
 
-/* Function declarations for reading from a cstore file */
+/* Function declarations for reading from columnar table */
 extern TableReadState * ColumnarBeginRead(Relation relation,
 										  TupleDesc tupleDescriptor,
 										  List *projectedColumnList,
@@ -366,8 +362,8 @@ logical_to_smgr(uint64 logicalOffset)
 {
 	SmgrAddr addr;
 
-	addr.blockno = logicalOffset / CSTORE_BYTES_PER_PAGE;
+	addr.blockno = logicalOffset / COLUMNAR_BYTES_PER_PAGE;
-	addr.offset = SizeOfPageHeaderData + (logicalOffset % CSTORE_BYTES_PER_PAGE);
+	addr.offset = SizeOfPageHeaderData + (logicalOffset % COLUMNAR_BYTES_PER_PAGE);
 
 	return addr;
 }
@@ -379,7 +375,7 @@ logical_to_smgr(uint64 logicalOffset)
 static inline uint64
 smgr_to_logical(SmgrAddr addr)
 {
-	return CSTORE_BYTES_PER_PAGE * addr.blockno + addr.offset - SizeOfPageHeaderData;
+	return COLUMNAR_BYTES_PER_PAGE * addr.blockno + addr.offset - SizeOfPageHeaderData;
 }
 
 
@@ -398,4 +394,4 @@ next_block_start(SmgrAddr addr)
 }
 
 
-#endif /* CSTORE_H */
+#endif /* COLUMNAR_H */

src/include/columnar/columnar_customscan.h

@@ -1,9 +1,9 @@
 /*-------------------------------------------------------------------------
  *
- * cstore_customscan.h
+ * columnar_customscan.h
  *
  * Forward declarations of functions to hookup the custom scan feature of
- * cstore.
+ * columnar.
  *
  * $Id$
  *

src/include/columnar/columnar_version_compat.h

@@ -1,6 +1,6 @@
 /*-------------------------------------------------------------------------
  *
- * cstore_version_compat.h
+ * columnar_version_compat.h
  *
  *  Compatibility macros for writing code agnostic to PostgreSQL versions
  *
@@ -11,8 +11,8 @@
  *-------------------------------------------------------------------------
  */
 
-#ifndef CSTORE_COMPAT_H
+#ifndef COLUMNAR_COMPAT_H
-#define CSTORE_COMPAT_H
+#define COLUMNAR_COMPAT_H
 
 #if PG_VERSION_NUM < 100000
 
@@ -43,4 +43,8 @@
 #define table_endscan heap_endscan
 #endif
 
-#endif /* CSTORE_COMPAT_H */
+#if PG_VERSION_NUM < 130000
+#define detoast_attr(X) heap_tuple_untoast_attr(X)
+#endif
+
+#endif /* COLUMNAR_COMPAT_H */

src/include/columnar/mod.h

@@ -2,7 +2,7 @@
  *
  * mod.h
  *
- * Type and function declarations for CStore
+ * Type and function declarations for columnar
  *
  * Copyright (c) 2016, Citus Data, Inc.
  *

src/include/distributed/commands.h

@@ -187,6 +187,7 @@ extern List * GetForeignConstraintToReferenceTablesCommands(Oid relationId);
 extern List * GetForeignConstraintToDistributedTablesCommands(Oid relationId);
 extern List * GetForeignConstraintFromDistributedTablesCommands(Oid relationId);
 extern List * GetForeignConstraintCommandsInternal(Oid relationId, int flags);
+extern bool HasForeignKeyWithLocalTable(Oid relationId);
 extern bool HasForeignKeyToCitusLocalTable(Oid relationId);
 extern bool HasForeignKeyToReferenceTable(Oid relationOid);
 extern bool TableReferenced(Oid relationOid);
@@ -360,6 +361,7 @@ extern List * PreprocessDropTableStmt(Node *stmt, const char *queryString,
 									  ProcessUtilityContext processUtilityContext);
 extern void PostprocessCreateTableStmt(CreateStmt *createStatement,
 									   const char *queryString);
+extern bool ShouldEnableLocalReferenceForeignKeys(void);
 extern List * PostprocessAlterTableStmtAttachPartition(
 	AlterTableStmt *alterTableStatement,
 	const char *queryString);
@@ -481,7 +483,9 @@ extern void CascadeOperationForConnectedRelations(Oid relationId, LOCKMODE relLo
 												  CascadeOperationType
 												  cascadeOperationType);
 extern void ErrorIfAnyPartitionRelationInvolvedInNonInheritedFKey(List *relationIdList);
+extern bool RelationIdListHasReferenceTable(List *relationIdList);
 extern void DropRelationForeignKeys(Oid relationId, int flags);
+extern void SetLocalEnableLocalReferenceForeignKeys(bool state);
 extern void ExecuteAndLogDDLCommandList(List *ddlCommandList);
 extern void ExecuteAndLogDDLCommand(const char *commandString);
 extern void ExecuteForeignKeyCreateCommandList(List *ddlCommandList,

src/include/distributed/commands/utility_hook.h

@@ -35,6 +35,8 @@ extern bool EnableDependencyCreation;
 extern bool EnableCreateTypePropagation;
 extern bool EnableAlterRolePropagation;
 extern bool EnableAlterRoleSetPropagation;
+extern int UtilityHookLevel;
+
 
 /*
  * A DDLJob encapsulates the remote tasks and commands needed to process all or
@@ -77,6 +79,9 @@ extern List * DDLTaskList(Oid relationId, const char *commandString);
 extern List * NodeDDLTaskList(TargetWorkerSet targets, List *commands);
 extern bool AlterTableInProgress(void);
 extern bool DropSchemaOrDBInProgress(void);
+extern void UndistributeDisconnectedCitusLocalTables(void);
+extern void NotifyUtilityHookConstraintDropped(void);
+extern void ResetConstraintDropped(void);
 extern void ExecuteDistributedDDLJob(DDLJob *ddlJob);
 
 /* forward declarations for sending custom commands to a distributed table */

src/include/distributed/connection_management.h

@@ -252,6 +252,7 @@ extern void FinishConnectionListEstablishment(List *multiConnectionList);
 extern void FinishConnectionEstablishment(MultiConnection *connection);
 extern void ClaimConnectionExclusively(MultiConnection *connection);
 extern void UnclaimConnection(MultiConnection *connection);
+extern bool IsCitusInitiatedRemoteBackend(void);
 
 /* time utilities */
 extern double MillisecondsPassedSince(instr_time moment);

src/include/distributed/foreign_key_relationship.h

@@ -16,6 +16,7 @@
 #include "nodes/primnodes.h"
 
 extern List * GetForeignKeyConnectedRelationIdList(Oid relationId);
+extern bool ConnectedToReferenceTableViaFKey(Oid relationId);
 extern List * ReferencedRelationIdList(Oid relationId);
 extern List * ReferencingRelationIdList(Oid relationId);
 extern void SetForeignConstraintRelationshipGraphInvalid(void);

src/test/regress/bin/normalize.sed

@@ -200,3 +200,7 @@ s/(NOTICE:  executing.*)\([0-9]+, 'citus_local_tables_test_schema', [0-9]+(.*)/\
 s/citus_local_table_4_idx_[0-9]+/citus_local_table_4_idx_xxxxxx/g
 s/citus_local_table_4_[0-9]+/citus_local_table_4_xxxxxx/g
 s/ERROR:  cannot append to shardId [0-9]+/ERROR:  cannot append to shardId xxxxxx/g
+
+# hide warning/hint message that we get when executing create_citus_local_table
+/citus local tables that are not chained with reference tables via foreign keys might be automatically converted back to postgres tables$/d
+/Consider setting citus.enable_local_reference_table_foreign_keys to 'off' to disable automatically undistributing citus local tables$/d

src/test/regress/expected/am_insert.out

@@ -96,3 +96,38 @@ DROP PUBLICATION test_columnar_publication;
 -- should succeed
 INSERT INTO test_logical_replication VALUES (3);
 DROP TABLE test_logical_replication;
+--
+-- test toast interactions
+--
+-- row table with data in different storage formats
+CREATE TABLE test_toast_row(plain TEXT, main TEXT, external TEXT, extended TEXT);
+ALTER TABLE test_toast_row ALTER COLUMN plain SET STORAGE plain; -- inline, uncompressed
+ALTER TABLE test_toast_row ALTER COLUMN main SET STORAGE main; -- inline, compressed
+ALTER TABLE test_toast_row ALTER COLUMN external SET STORAGE external; -- out-of-line, uncompressed
+ALTER TABLE test_toast_row ALTER COLUMN extended SET STORAGE extended; -- out-of-line, compressed
+INSERT INTO test_toast_row VALUES(
+       repeat('w', 5000), repeat('x', 5000), repeat('y', 5000), repeat('z', 5000));
+SELECT
+  pg_column_size(plain), pg_column_size(main),
+  pg_column_size(external), pg_column_size(extended)
+FROM test_toast_row;
+ pg_column_size | pg_column_size | pg_column_size | pg_column_size
+---------------------------------------------------------------------
+           5004 |             69 |           5000 |             65
+(1 row)
+
+CREATE TABLE test_toast_columnar(plain TEXT, main TEXT, external TEXT, extended TEXT)
+  USING columnar;
+INSERT INTO test_toast_columnar SELECT plain, main, external, extended
+  FROM test_toast_row;
+SELECT
+  pg_column_size(plain), pg_column_size(main),
+  pg_column_size(external), pg_column_size(extended)
+FROM test_toast_columnar;
+ pg_column_size | pg_column_size | pg_column_size | pg_column_size
+---------------------------------------------------------------------
+           5004 |           5004 |           5004 |           5004
+(1 row)
+
+DROP TABLE test_toast_row;
+DROP TABLE test_toast_columnar;

src/test/regress/expected/am_tableoptions.out

@@ -246,7 +246,7 @@ SELECT alter_columnar_table_reset('not_a_columnar_table', compression => true);
 ERROR:  table not_a_columnar_table is not a columnar table
 -- verify you can't use a compression that is not known
 SELECT alter_columnar_table_set('table_options', compression => 'foobar');
-ERROR:  unknown compression type for cstore table: foobar
+ERROR:  unknown compression type for columnar table: foobar
 -- verify cannot set out of range compression levels
 SELECT alter_columnar_table_set('table_options', compression_level => 0);
 ERROR:  compression level out of range

src/test/regress/expected/auto_undist_citus_local.out

@@ -0,0 +1,635 @@
+-- regression tests regarding foreign key
+-- drops cascading into undistributing Citus
+-- local tables to Postgres local tables
+CREATE SCHEMA drop_fkey_cascade;
+SET search_path TO drop_fkey_cascade;
+SET client_min_messages TO WARNING;
+SET citus.next_shard_id TO 1810000;
+SELECT 1 FROM master_add_node('localhost', :master_port, groupId => 0);
+ ?column?
+---------------------------------------------------------------------
+        1
+(1 row)
+
+-- show that DROP CONSTRAINT cascades to undistributing citus_local_table
+CREATE TABLE citus_local_table(l1 int);
+SELECT create_citus_local_table('citus_local_table');
+ create_citus_local_table
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE TABLE reference_table(r1 int primary key);
+SELECT create_reference_table('reference_table');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+ALTER TABLE citus_local_table ADD CONSTRAINT fkey_local_to_ref FOREIGN KEY(l1) REFERENCES reference_table(r1) ON DELETE CASCADE;
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table'::regclass) ORDER BY logicalrelid;
+   logicalrelid    | partmethod | repmodel
+---------------------------------------------------------------------
+ citus_local_table | n          | c
+ reference_table   | n          | t
+(2 rows)
+
+CREATE OR REPLACE FUNCTION drop_constraint_cascade_via_perform_deletion(IN table_name regclass, IN constraint_name text)
+RETURNS VOID
+LANGUAGE C STRICT
+AS 'citus', $$drop_constraint_cascade_via_perform_deletion$$;
+BEGIN;
+  SELECT drop_constraint_cascade_via_perform_deletion('citus_local_table', 'fkey_local_to_ref');
+ drop_constraint_cascade_via_perform_deletion
+---------------------------------------------------------------------
+
+(1 row)
+
+  -- we dropped constraint without going through utility hook,
+  -- so we should still see citus_local_table
+  SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table'::regclass) ORDER BY logicalrelid;
+   logicalrelid    | partmethod | repmodel
+---------------------------------------------------------------------
+ citus_local_table | n          | c
+ reference_table   | n          | t
+(2 rows)
+
+ROLLBACK;
+ALTER TABLE citus_local_table DROP CONSTRAINT fkey_local_to_ref;
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table'::regclass) ORDER BY logicalrelid;
+  logicalrelid   | partmethod | repmodel
+---------------------------------------------------------------------
+ reference_table | n          | t
+(1 row)
+
+DROP TABLE citus_local_table, reference_table;
+-- show that DROP COLUMN cascades to undistributing citus_local_table
+CREATE TABLE reference_table(r1 int primary key, r2 int);
+SELECT create_reference_table('reference_table');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE TABLE citus_local_table(l1 int REFERENCES reference_table(r1), l2 int);
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table'::regclass) ORDER BY logicalrelid;
+   logicalrelid    | partmethod | repmodel
+---------------------------------------------------------------------
+ reference_table   | n          | t
+ citus_local_table | n          | c
+(2 rows)
+
+ALTER TABLE reference_table DROP COLUMN r1 CASCADE;
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table'::regclass) ORDER BY logicalrelid;
+  logicalrelid   | partmethod | repmodel
+---------------------------------------------------------------------
+ reference_table | n          | t
+(1 row)
+
+DROP TABLE citus_local_table, reference_table;
+-- show that DROP COLUMN that cascades into drop foreign key undistributes local table
+CREATE TABLE reference_table(r1 int primary key, r2 int);
+SELECT create_reference_table('reference_table');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE TABLE citus_local_table(l1 int REFERENCES reference_table(r1), l2 int);
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table'::regclass) ORDER BY logicalrelid;
+   logicalrelid    | partmethod | repmodel
+---------------------------------------------------------------------
+ reference_table   | n          | t
+ citus_local_table | n          | c
+(2 rows)
+
+ALTER TABLE citus_local_table DROP COLUMN l1 CASCADE;
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table'::regclass) ORDER BY logicalrelid;
+  logicalrelid   | partmethod | repmodel
+---------------------------------------------------------------------
+ reference_table | n          | t
+(1 row)
+
+DROP TABLE citus_local_table, reference_table;
+-- show that PRIMARY KEY that cascades into drop foreign key undistributes local table
+CREATE TABLE reference_table(r1 int primary key, r2 int);
+SELECT create_reference_table('reference_table');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE TABLE citus_local_table(l1 int REFERENCES reference_table(r1), l2 int);
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table'::regclass) ORDER BY logicalrelid;
+   logicalrelid    | partmethod | repmodel
+---------------------------------------------------------------------
+ reference_table   | n          | t
+ citus_local_table | n          | c
+(2 rows)
+
+ALTER TABLE reference_table DROP CONSTRAINT reference_table_pkey CASCADE;
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table'::regclass) ORDER BY logicalrelid;
+  logicalrelid   | partmethod | repmodel
+---------------------------------------------------------------------
+ reference_table | n          | t
+(1 row)
+
+-- show that DROP UNIQUE INDEX that cascades into drop foreign key undistributes local table
+DROP TABLE citus_local_table, reference_table;
+CREATE TABLE reference_table(r1 int, r2 int);
+SELECT create_reference_table('reference_table');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE UNIQUE INDEX ref_unique ON reference_table(r1);
+CREATE TABLE citus_local_table(l1 int REFERENCES reference_table(r1), l2 int);
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table'::regclass) ORDER BY logicalrelid;
+   logicalrelid    | partmethod | repmodel
+---------------------------------------------------------------------
+ reference_table   | n          | t
+ citus_local_table | n          | c
+(2 rows)
+
+DROP INDEX ref_unique CASCADE;
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table'::regclass) ORDER BY logicalrelid;
+  logicalrelid   | partmethod | repmodel
+---------------------------------------------------------------------
+ reference_table | n          | t
+(1 row)
+
+-- show that UNIQUE CONSTRAINT that cascades into drop foreign key undistributes local table
+DROP TABLE citus_local_table, reference_table;
+CREATE TABLE reference_table(r1 int UNIQUE, r2 int);
+SELECT create_reference_table('reference_table');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE TABLE citus_local_table(l1 int REFERENCES reference_table(r1), l2 int);
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table'::regclass) ORDER BY logicalrelid;
+   logicalrelid    | partmethod | repmodel
+---------------------------------------------------------------------
+ reference_table   | n          | t
+ citus_local_table | n          | c
+(2 rows)
+
+ALTER TABLE reference_table DROP CONSTRAINT reference_table_r1_key  CASCADE;
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table'::regclass) ORDER BY logicalrelid;
+  logicalrelid   | partmethod | repmodel
+---------------------------------------------------------------------
+ reference_table | n          | t
+(1 row)
+
+-- show that DROP TABLE that cascades into drop foreign key undistributes local table
+DROP TABLE citus_local_table, reference_table;
+CREATE TABLE reference_table(r1 int UNIQUE, r2 int);
+SELECT create_reference_table('reference_table');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE TABLE citus_local_table(l1 int REFERENCES reference_table(r1), l2 int);
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table'::regclass) ORDER BY logicalrelid;
+   logicalrelid    | partmethod | repmodel
+---------------------------------------------------------------------
+ reference_table   | n          | t
+ citus_local_table | n          | c
+(2 rows)
+
+DROP TABLE reference_table CASCADE;
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass) ORDER BY logicalrelid;
+ logicalrelid | partmethod | repmodel
+---------------------------------------------------------------------
+(0 rows)
+
+-- show that UNIQUE CONSTRAINT that cascades into drop foreign key undistributes local table
+DROP TABLE citus_local_table;
+CREATE TABLE reference_table(r1 int UNIQUE, r2 int);
+SELECT create_reference_table('reference_table');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE TABLE citus_local_table(l1 int REFERENCES reference_table(r1), l2 int);
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table'::regclass) ORDER BY logicalrelid;
+   logicalrelid    | partmethod | repmodel
+---------------------------------------------------------------------
+ reference_table   | n          | t
+ citus_local_table | n          | c
+(2 rows)
+
+ALTER TABLE reference_table DROP CONSTRAINT reference_table_r1_key  CASCADE;
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table'::regclass) ORDER BY logicalrelid;
+  logicalrelid   | partmethod | repmodel
+---------------------------------------------------------------------
+ reference_table | n          | t
+(1 row)
+
+-- show that DROP SCHEMA that cascades into drop foreign key undistributes local table
+DROP TABLE citus_local_table, reference_table;
+CREATE SCHEMA ref_table_drop_schema;
+CREATE TABLE ref_table_drop_schema.reference_table(r1 int UNIQUE, r2 int);
+SELECT create_reference_table('ref_table_drop_schema.reference_table');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE TABLE citus_local_table(l1 int REFERENCES ref_table_drop_schema.reference_table(r1), l2 int);
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'ref_table_drop_schema.reference_table'::regclass) ORDER BY logicalrelid;
+             logicalrelid              | partmethod | repmodel
+---------------------------------------------------------------------
+ ref_table_drop_schema.reference_table | n          | t
+ citus_local_table                     | n          | c
+(2 rows)
+
+DROP SCHEMA ref_table_drop_schema CASCADE;
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass) ORDER BY logicalrelid;
+ logicalrelid | partmethod | repmodel
+---------------------------------------------------------------------
+(0 rows)
+
+-- drop column cascade that doesn't cascade into citus local table
+DROP TABLE IF EXISTS citus_local_table, reference_table_1, reference_table_2;
+CREATE TABLE reference_table_1(r1 int UNIQUE, r2 int);
+SELECT create_reference_table('reference_table_1');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE TABLE reference_table_2(r1 int UNIQUE REFERENCES reference_table_1(r1), r2 int);
+SELECT create_reference_table('reference_table_2');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE TABLE citus_local_table(l1 int REFERENCES reference_table_2(r1), l2 int);
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table_1'::regclass, 'reference_table_2'::regclass) ORDER BY logicalrelid;
+   logicalrelid    | partmethod | repmodel
+---------------------------------------------------------------------
+ reference_table_1 | n          | t
+ reference_table_2 | n          | t
+ citus_local_table | n          | c
+(3 rows)
+
+ALTER TABLE reference_table_1 DROP COLUMN r1 CASCADE;
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table_1'::regclass, 'reference_table_2'::regclass) ORDER BY logicalrelid;
+   logicalrelid    | partmethod | repmodel
+---------------------------------------------------------------------
+ reference_table_1 | n          | t
+ reference_table_2 | n          | t
+ citus_local_table | n          | c
+(3 rows)
+
+-- local table has multiple foreign keys to two tables
+-- drop one at a time
+DROP TABLE IF EXISTS citus_local_table, reference_table_1, reference_table_2;
+CREATE TABLE reference_table_1(r1 int UNIQUE, r2 int);
+SELECT create_reference_table('reference_table_1');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE TABLE reference_table_2(r1 int UNIQUE, r2 int);
+SELECT create_reference_table('reference_table_2');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE TABLE citus_local_table(l1 int REFERENCES reference_table_1(r1), l2 int REFERENCES reference_table_2(r1));
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table_1'::regclass, 'reference_table_2'::regclass) ORDER BY logicalrelid;
+   logicalrelid    | partmethod | repmodel
+---------------------------------------------------------------------
+ reference_table_1 | n          | t
+ reference_table_2 | n          | t
+ citus_local_table | n          | c
+(3 rows)
+
+DROP TABLE reference_table_1 CASCADE;
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table_2'::regclass) ORDER BY logicalrelid;
+   logicalrelid    | partmethod | repmodel
+---------------------------------------------------------------------
+ reference_table_2 | n          | t
+ citus_local_table | n          | c
+(2 rows)
+
+CREATE TABLE distributed_table (d1 int);
+SELECT create_distributed_table('distributed_table', 'd1');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+-- drop an unrelated distributed table too
+DROP TABLE reference_table_2, distributed_table CASCADE;
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass) ORDER BY logicalrelid;
+ logicalrelid | partmethod | repmodel
+---------------------------------------------------------------------
+(0 rows)
+
+-- local table has multiple foreign keys to two tables
+-- drop both at the same time
+DROP TABLE IF EXISTS citus_local_table, reference_table_1, reference_table_2;
+CREATE TABLE reference_table_1(r1 int UNIQUE, r2 int);
+SELECT create_reference_table('reference_table_1');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE TABLE reference_table_2(r1 int UNIQUE, r2 int);
+SELECT create_reference_table('reference_table_2');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE TABLE citus_local_table(l1 int REFERENCES reference_table_1(r1), l2 int REFERENCES reference_table_2(r1));
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table_1'::regclass, 'reference_table_2'::regclass) ORDER BY logicalrelid;
+   logicalrelid    | partmethod | repmodel
+---------------------------------------------------------------------
+ reference_table_1 | n          | t
+ reference_table_2 | n          | t
+ citus_local_table | n          | c
+(3 rows)
+
+DROP TABLE reference_table_1, reference_table_2 CASCADE;
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass) ORDER BY logicalrelid;
+ logicalrelid | partmethod | repmodel
+---------------------------------------------------------------------
+(0 rows)
+
+-- local table has multiple foreign keys to two tables
+-- drop one at a time
+DROP TABLE IF EXISTS citus_local_table, reference_table_1, reference_table_2;
+CREATE TABLE reference_table_1(r1 int UNIQUE, r2 int);
+SELECT create_reference_table('reference_table_1');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE TABLE reference_table_2(r1 int UNIQUE, r2 int);
+SELECT create_reference_table('reference_table_2');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE TABLE citus_local_table(l1 int REFERENCES reference_table_1(r1), l2 int REFERENCES reference_table_2(r1));
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table_1'::regclass, 'reference_table_2'::regclass) ORDER BY logicalrelid;
+   logicalrelid    | partmethod | repmodel
+---------------------------------------------------------------------
+ reference_table_1 | n          | t
+ reference_table_2 | n          | t
+ citus_local_table | n          | c
+(3 rows)
+
+BEGIN;
+  ALTER TABLE citus_local_table DROP CONSTRAINT citus_local_table_l1_fkey;
+  SAVEPOINT sp1;
+  -- this should undistribute citus_local_table
+  ALTER TABLE citus_local_table DROP CONSTRAINT citus_local_table_l2_fkey;
+  SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table_1'::regclass, 'reference_table_2'::regclass) ORDER BY logicalrelid;
+   logicalrelid    | partmethod | repmodel
+---------------------------------------------------------------------
+ reference_table_1 | n          | t
+ reference_table_2 | n          | t
+(2 rows)
+
+  ROLLBACK TO SAVEPOINT sp1;
+  -- rollback'ed second drop constraint, so we should still see citus_local_table
+  SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table_1'::regclass, 'reference_table_2'::regclass) ORDER BY logicalrelid;
+   logicalrelid    | partmethod | repmodel
+---------------------------------------------------------------------
+ reference_table_1 | n          | t
+ reference_table_2 | n          | t
+ citus_local_table | n          | c
+(3 rows)
+
+  -- this should undistribute citus_local_table again
+  ALTER TABLE citus_local_table DROP CONSTRAINT citus_local_table_l2_fkey;
+  SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table_1'::regclass, 'reference_table_2'::regclass) ORDER BY logicalrelid;
+   logicalrelid    | partmethod | repmodel
+---------------------------------------------------------------------
+ reference_table_1 | n          | t
+ reference_table_2 | n          | t
+(2 rows)
+
+COMMIT;
+-- a single drop column cascades into multiple undistributes
+DROP TABLE IF EXISTS citus_local_table_1, citus_local_table_2, reference_table_1;
+CREATE TABLE reference_table_1(r1 int UNIQUE, r2 int);
+SELECT create_reference_table('reference_table_1');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE TABLE citus_local_table_1(l1 int REFERENCES reference_table_1(r1), l2 int UNIQUE);
+CREATE TABLE citus_local_table_2(l1 int REFERENCES reference_table_1(r1), l2 int UNIQUE REFERENCES citus_local_table_1(l2));
+CREATE TABLE citus_local_table_3(l1 int REFERENCES reference_table_1(r1), l2 int REFERENCES citus_local_table_2(l2));
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('reference_table_1'::regclass, 'citus_local_table_1'::regclass, 'citus_local_table_2'::regclass, 'citus_local_table_3'::regclass) ORDER BY logicalrelid;
+    logicalrelid     | partmethod | repmodel
+---------------------------------------------------------------------
+ reference_table_1   | n          | t
+ citus_local_table_1 | n          | c
+ citus_local_table_2 | n          | c
+ citus_local_table_3 | n          | c
+(4 rows)
+
+ALTER TABLE reference_table_1 DROP COLUMN r1 CASCADE;
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('reference_table_1'::regclass, 'citus_local_table_1'::regclass, 'citus_local_table_2'::regclass, 'citus_local_table_3'::regclass) ORDER BY logicalrelid;
+   logicalrelid    | partmethod | repmodel
+---------------------------------------------------------------------
+ reference_table_1 | n          | t
+(1 row)
+
+-- a single drop table cascades into multiple undistributes
+DROP TABLE IF EXISTS citus_local_table_1, citus_local_table_2, citus_local_table_3, citus_local_table_2, reference_table_1;
+CREATE TABLE reference_table_1(r1 int UNIQUE, r2 int);
+SELECT create_reference_table('reference_table_1');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE TABLE citus_local_table_1(l1 int REFERENCES reference_table_1(r1), l2 int UNIQUE);
+CREATE TABLE citus_local_table_2(l1 int REFERENCES reference_table_1(r1), l2 int UNIQUE REFERENCES citus_local_table_1(l2));
+CREATE TABLE citus_local_table_3(l1 int REFERENCES reference_table_1(r1), l2 int REFERENCES citus_local_table_2(l2));
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('reference_table_1'::regclass, 'citus_local_table_1'::regclass, 'citus_local_table_2'::regclass, 'citus_local_table_3'::regclass) ORDER BY logicalrelid;
+    logicalrelid     | partmethod | repmodel
+---------------------------------------------------------------------
+ reference_table_1   | n          | t
+ citus_local_table_1 | n          | c
+ citus_local_table_2 | n          | c
+ citus_local_table_3 | n          | c
+(4 rows)
+
+-- test DROP OWNED BY
+-- Citus does not support "ALTER TABLE OWNER TO" commands. Also, not to deal with tests output
+-- difference between community and enterprise, let's disable enable_ddl_propagation here.
+SET citus.enable_ddl_propagation to OFF;
+CREATE USER another_user;
+SELECT run_command_on_workers('CREATE USER another_user');
+      run_command_on_workers
+---------------------------------------------------------------------
+ (localhost,57637,t,"CREATE ROLE")
+ (localhost,57638,t,"CREATE ROLE")
+(2 rows)
+
+ALTER TABLE reference_table_1 OWNER TO another_user;
+SELECT run_command_on_placements('reference_table_1', 'ALTER TABLE %s OWNER TO another_user');
+         run_command_on_placements
+---------------------------------------------------------------------
+ (localhost,57636,1810039,t,"ALTER TABLE")
+ (localhost,57637,1810039,t,"ALTER TABLE")
+ (localhost,57638,1810039,t,"ALTER TABLE")
+(3 rows)
+
+SET citus.enable_ddl_propagation to ON;
+BEGIN;
+  DROP OWNED BY another_user cascade;
+  SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ( 'citus_local_table_1'::regclass, 'citus_local_table_2'::regclass, 'citus_local_table_3'::regclass) ORDER BY logicalrelid;
+ logicalrelid | partmethod | repmodel
+---------------------------------------------------------------------
+(0 rows)
+
+ROLLBACK;
+DROP TABLE reference_table_1 CASCADE;
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ( 'citus_local_table_1'::regclass, 'citus_local_table_2'::regclass, 'citus_local_table_3'::regclass) ORDER BY logicalrelid;
+ logicalrelid | partmethod | repmodel
+---------------------------------------------------------------------
+(0 rows)
+
+-- dropping constraints inside a plpgsql procedure should be fine
+DROP TABLE IF EXISTS citus_local_table_1, reference_table_1 CASCADE;
+CREATE TABLE reference_table_1(r1 int UNIQUE, r2 int);
+SELECT create_reference_table('reference_table_1');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE TABLE citus_local_table_1(l1 int REFERENCES reference_table_1(r1), l2 int UNIQUE);
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('reference_table_1'::regclass, 'citus_local_table_1'::regclass) ORDER BY logicalrelid;
+    logicalrelid     | partmethod | repmodel
+---------------------------------------------------------------------
+ reference_table_1   | n          | t
+ citus_local_table_1 | n          | c
+(2 rows)
+
+CREATE OR REPLACE FUNCTION drop_constraint_via_func()
+RETURNS void LANGUAGE plpgsql AS $$
+BEGIN
+ALTER TABLE citus_local_table_1 DROP CONSTRAINT citus_local_table_1_l1_fkey;
+END;$$;
+BEGIN;
+  SELECT drop_constraint_via_func();
+ drop_constraint_via_func
+---------------------------------------------------------------------
+
+(1 row)
+
+  SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('reference_table_1'::regclass, 'citus_local_table_1'::regclass) ORDER BY logicalrelid;
+   logicalrelid    | partmethod | repmodel
+---------------------------------------------------------------------
+ reference_table_1 | n          | t
+(1 row)
+
+ROLLBACK;
+create or replace procedure drop_constraint_via_proc()
+language plpgsql
+as $$
+DECLARE
+    res INT := 0;
+begin
+        ALTER TABLE citus_local_table_1 DROP CONSTRAINT citus_local_table_1_l1_fkey;
+    commit;
+end;$$;
+call drop_constraint_via_proc();
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('reference_table_1'::regclass, 'citus_local_table_1'::regclass) ORDER BY logicalrelid;
+   logicalrelid    | partmethod | repmodel
+---------------------------------------------------------------------
+ reference_table_1 | n          | t
+(1 row)
+
+-- even if the procedure is called from another procedure
+DROP TABLE IF EXISTS citus_local_table_1, reference_table_1 CASCADE;
+CREATE TABLE reference_table_1(r1 int UNIQUE, r2 int);
+SELECT create_reference_table('reference_table_1');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE TABLE citus_local_table_1(l1 int REFERENCES reference_table_1(r1), l2 int UNIQUE);
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('reference_table_1'::regclass, 'citus_local_table_1'::regclass) ORDER BY logicalrelid;
+    logicalrelid     | partmethod | repmodel
+---------------------------------------------------------------------
+ reference_table_1   | n          | t
+ citus_local_table_1 | n          | c
+(2 rows)
+
+create or replace procedure drop_constraint_via_proc_top_level()
+language plpgsql
+as $$
+DECLARE
+    res INT := 0;
+begin
+        CALL drop_constraint_via_proc();
+    commit;
+end;$$;
+CALL drop_constraint_via_proc_top_level();
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('reference_table_1'::regclass, 'citus_local_table_1'::regclass) ORDER BY logicalrelid;
+   logicalrelid    | partmethod | repmodel
+---------------------------------------------------------------------
+ reference_table_1 | n          | t
+(1 row)
+
+-- even if the procedure is called from an exception handler
+DROP TABLE IF EXISTS citus_local_table_1, reference_table_1 CASCADE;
+CREATE TABLE reference_table_1(r1 int UNIQUE, r2 int);
+SELECT create_reference_table('reference_table_1');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE TABLE citus_local_table_1(l1 int REFERENCES reference_table_1(r1), l2 int UNIQUE);
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('reference_table_1'::regclass, 'citus_local_table_1'::regclass) ORDER BY logicalrelid;
+    logicalrelid     | partmethod | repmodel
+---------------------------------------------------------------------
+ reference_table_1   | n          | t
+ citus_local_table_1 | n          | c
+(2 rows)
+
+create or replace procedure drop_constraint_via_proc_exception()
+language plpgsql
+as $$
+DECLARE
+    res INT := 0;
+begin
+		PERFORM 1/0;
+    	EXCEPTION
+             when others then
+        CALL drop_constraint_via_proc();
+    commit;
+end;$$;
+CALL drop_constraint_via_proc_exception();
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('reference_table_1'::regclass, 'citus_local_table_1'::regclass) ORDER BY logicalrelid;
+   logicalrelid    | partmethod | repmodel
+---------------------------------------------------------------------
+ reference_table_1 | n          | t
+(1 row)
+
+DROP SCHEMA drop_fkey_cascade CASCADE;

src/test/regress/expected/citus_local_tables.out

@@ -259,6 +259,8 @@ NOTICE:  foreign-data wrapper "fake_fdw" does not have an extension defined
 
 (1 row)
 
+DROP FOREIGN TABLE foreign_table;
+NOTICE:  executing the command locally: DROP FOREIGN TABLE IF EXISTS citus_local_tables_test_schema.foreign_table_xxxxx CASCADE
 -- drop them for next tests
 DROP TABLE citus_local_table_1, citus_local_table_2, distributed_table;
 NOTICE:  executing the command locally: DROP TABLE IF EXISTS citus_local_tables_test_schema.citus_local_table_2_xxxxx CASCADE
@@ -413,6 +415,67 @@ NOTICE:  executing the command locally: CREATE UNIQUE INDEX   uniqueindex2_15040
 ---- utility command execution ----
 ---------------------------------------------------------------------
 SET search_path TO citus_local_tables_test_schema;
+CREATE TABLE dummy_reference_table (a INT PRIMARY KEY);
+SELECT create_reference_table('dummy_reference_table');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+BEGIN;
+  SET client_min_messages TO ERROR;
+  SELECT remove_local_tables_from_metadata();
+ remove_local_tables_from_metadata
+---------------------------------------------------------------------
+
+(1 row)
+
+  -- should not see any citus local tables
+  SELECT logicalrelid::regclass::text FROM pg_dist_partition, pg_tables
+  WHERE tablename=logicalrelid::regclass::text AND
+        schemaname='citus_local_tables_test_schema' AND
+        partmethod = 'n' AND repmodel = 'c'
+  ORDER BY 1;
+ logicalrelid
+---------------------------------------------------------------------
+(0 rows)
+
+ROLLBACK;
+-- define foreign keys between dummy_reference_table and citus local tables
+-- not to undistribute them automatically
+ALTER TABLE citus_local_table_1 ADD CONSTRAINT fkey_to_dummy_ref FOREIGN KEY (a) REFERENCES dummy_reference_table(a);
+NOTICE:  executing the command locally: SELECT worker_apply_inter_shard_ddl_command (xxxxx, 'citus_local_tables_test_schema', xxxxx, 'citus_local_tables_test_schema', 'ALTER TABLE citus_local_table_1 ADD CONSTRAINT fkey_to_dummy_ref FOREIGN KEY (a) REFERENCES dummy_reference_table(a);')
+ALTER TABLE citus_local_table_2 ADD CONSTRAINT fkey_to_dummy_ref FOREIGN KEY (a) REFERENCES dummy_reference_table(a);
+NOTICE:  executing the command locally: SELECT worker_apply_inter_shard_ddl_command (xxxxx, 'citus_local_tables_test_schema', xxxxx, 'citus_local_tables_test_schema', 'ALTER TABLE citus_local_table_2 ADD CONSTRAINT fkey_to_dummy_ref FOREIGN KEY (a) REFERENCES dummy_reference_table(a);')
+ALTER TABLE unlogged_table ADD CONSTRAINT fkey_to_dummy_ref FOREIGN KEY (a) REFERENCES dummy_reference_table(a);
+NOTICE:  executing the command locally: SELECT worker_apply_inter_shard_ddl_command (xxxxx, 'citus_local_tables_test_schema', xxxxx, 'citus_local_tables_test_schema', 'ALTER TABLE unlogged_table ADD CONSTRAINT fkey_to_dummy_ref FOREIGN KEY (a) REFERENCES dummy_reference_table(a);')
+ALTER TABLE local_table_3 ADD CONSTRAINT fkey_to_dummy_ref FOREIGN KEY (a) REFERENCES dummy_reference_table(a);
+NOTICE:  executing the command locally: SELECT worker_apply_inter_shard_ddl_command (xxxxx, 'citus_local_tables_test_schema', xxxxx, 'citus_local_tables_test_schema', 'ALTER TABLE local_table_3 ADD CONSTRAINT fkey_to_dummy_ref FOREIGN KEY (a) REFERENCES dummy_reference_table(a);')
+ALTER TABLE dummy_reference_table ADD CONSTRAINT fkey_from_dummy_ref FOREIGN KEY (a) REFERENCES "CiTUS!LocalTables"."LocalTabLE.1!?!"(id);
+NOTICE:  executing the command locally: SELECT worker_apply_inter_shard_ddl_command (xxxxx, 'citus_local_tables_test_schema', xxxxx, 'CiTUS!LocalTables', 'ALTER TABLE dummy_reference_table ADD CONSTRAINT fkey_from_dummy_ref FOREIGN KEY (a) REFERENCES "CiTUS!LocalTables"."LocalTabLE.1!?!"(id);')
+BEGIN;
+  SET client_min_messages TO ERROR;
+  SELECT remove_local_tables_from_metadata();
+ remove_local_tables_from_metadata
+---------------------------------------------------------------------
+
+(1 row)
+
+  -- now we defined foreign keys with above citus local tables, we should still see them
+  SELECT logicalrelid::regclass::text FROM pg_dist_partition, pg_tables
+  WHERE tablename=logicalrelid::regclass::text AND
+        schemaname='citus_local_tables_test_schema' AND
+        partmethod = 'n' AND repmodel = 'c'
+  ORDER BY 1;
+    logicalrelid
+---------------------------------------------------------------------
+ citus_local_table_1
+ citus_local_table_2
+ local_table_3
+ unlogged_table
+(4 rows)
+
+ROLLBACK;
 -- between citus local tables and distributed tables
 ALTER TABLE citus_local_table_1 ADD CONSTRAINT fkey_c_to_dist FOREIGN KEY(a) references distributed_table(a);
 ERROR:  cannot create foreign key constraint since foreign keys from reference tables and citus local tables to distributed tables are not supported
@@ -675,21 +738,17 @@ NOTICE:  executing the command locally: SELECT worker_apply_inter_shard_ddl_comm
 -- verify creating citus local table with extended statistics
 CREATE TABLE test_citus_local_table_with_stats(a int, b int);
 CREATE STATISTICS stx1 ON a, b FROM test_citus_local_table_with_stats;
-SELECT create_citus_local_table('test_citus_local_table_with_stats');
+ALTER TABLE test_citus_local_table_with_stats ADD CONSTRAINT fkey_to_dummy_ref FOREIGN KEY (a) REFERENCES dummy_reference_table(a);
- create_citus_local_table
+NOTICE:  executing the command locally: SELECT worker_apply_inter_shard_ddl_command (xxxxx, 'citus_local_tables_test_schema', xxxxx, 'citus_local_tables_test_schema', 'ALTER TABLE test_citus_local_table_with_stats ADD CONSTRAINT fkey_to_dummy_ref FOREIGN KEY (a) REFERENCES dummy_reference_table(a);')
----------------------------------------------------------------------
-
-(1 row)
-
 CREATE STATISTICS "CiTUS!LocalTables"."Bad\'StatName" ON a, b FROM test_citus_local_table_with_stats;
-NOTICE:  executing the command locally: SELECT worker_apply_shard_ddl_command (1504042, 'citus_local_tables_test_schema', E'CREATE STATISTICS "CiTUS!LocalTables"."Bad\\''StatName" ON a, b FROM citus_local_tables_test_schema.test_citus_local_table_with_stats')
+NOTICE:  executing the command locally: SELECT worker_apply_shard_ddl_command (1504043, 'citus_local_tables_test_schema', E'CREATE STATISTICS "CiTUS!LocalTables"."Bad\\''StatName" ON a, b FROM citus_local_tables_test_schema.test_citus_local_table_with_stats')
 SELECT stxname FROM pg_statistic_ext ORDER BY stxname;
         stxname
 ---------------------------------------------------------------------
  Bad\'StatName
- Bad\'StatName_1504042
+ Bad\'StatName_1504043
  stx1
- stx1_1504042
+ stx1_1504043
 (4 rows)
 
 -- observe the debug messages telling that we switch to sequential

src/test/regress/expected/fkeys_between_local_ref.out

@@ -114,6 +114,14 @@ BEGIN;
  t
 (1 row)
 
+  -- dropping that column would undistribute those 4 citus local tables
+  ALTER TABLE local_table_1 DROP COLUMN col_1 CASCADE;
+  SELECT COUNT(*)=0 FROM citus_local_tables_in_schema;
+ ?column?
+---------------------------------------------------------------------
+ t
+(1 row)
+
 ROLLBACK;
 -- this actually attempts to convert local tables to citus local tables but errors out
 -- as citus doesn't support defining foreign keys via add column commands
@@ -132,6 +140,25 @@ BEGIN;
  t
 (1 row)
 
+  -- dropping foreign key from local_table_2 would only undistribute local_table_2 & local_table_5
+  ALTER TABLE local_table_2 DROP CONSTRAINT fkey_1;
+  SELECT logicalrelid::regclass::text FROM citus_local_tables_in_schema ORDER BY logicalrelid;
+ logicalrelid
+---------------------------------------------------------------------
+ local_table_1
+ local_table_3
+ local_table_4
+(3 rows)
+
+    -- dropping local_table_1 would undistribute last two citus local tables as local_table_1
+    -- was the bridge to reference table
+  DROP TABLE local_table_1 CASCADE;
+  SELECT COUNT(*)=0 FROM citus_local_tables_in_schema;
+ ?column?
+---------------------------------------------------------------------
+ t
+(1 row)
+
 ROLLBACK;
 -- they fail as local_table_99 does not exist
 ALTER TABLE local_table_99 ADD CONSTRAINT fkey FOREIGN KEY (col_1) REFERENCES reference_table_1(col_1);
@@ -331,6 +358,21 @@ BEGIN;
  reference_table_1                                    | n          | t
 (8 rows)
 
+  DROP TABLE local_table_3 CASCADE;
+  DROP SCHEMA another_schema_fkeys_between_local_ref CASCADE;
+  -- now we shouldn't see local_table_5 since now it is not connected to any reference tables
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+     tablename     | partmethod | repmodel
+---------------------------------------------------------------------
+ distributed_table | h          | c
+ local_table_1     | n          | c
+ local_table_2     | n          | c
+ local_table_4     | n          | c
+ reference_table_1 | n          | t
+(5 rows)
+
 ROLLBACK;
 BEGIN;
   CREATE TABLE local_table_6 (col_1 INT PRIMARY KEY);
@@ -391,6 +433,31 @@ BEGIN;
  reference_table_1 | n          | t
 (8 rows)
 
+  CREATE SCHEMA another_schema_fkeys_between_local_ref;
+  CREATE TABLE another_schema_fkeys_between_local_ref.reference_table_3 (col_1 INT UNIQUE);
+  SELECT create_reference_table('another_schema_fkeys_between_local_ref.reference_table_3');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  TRUNCATE local_table_4 CASCADE;
+  ALTER TABLE local_table_4 ADD CONSTRAINT fkey_12 FOREIGN KEY (col_1) REFERENCES another_schema_fkeys_between_local_ref.reference_table_3(col_1);
+  DROP TABLE local_table_5 CASCADE;
+  ALTER TABLE local_table_2 DROP CONSTRAINT fkey_1;
+  DROP SCHEMA another_schema_fkeys_between_local_ref CASCADE;
+  -- now we shouldn't see any citus local tables
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+     tablename     | partmethod | repmodel
+---------------------------------------------------------------------
+ distributed_table | h          | c
+ local_table_2     | n          | t
+ local_table_6     | n          | t
+ reference_table_1 | n          | t
+(4 rows)
+
 ROLLBACK;
 BEGIN;
   -- disable foreign keys to reference tables
@@ -414,6 +481,145 @@ BEGIN;
 (2 rows)
 
 ROLLBACK;
+-- converting any local table to a citus local table in graph converts
+-- other tables to citus local tables, test this in below xact blocks
+BEGIN;
+  SELECT create_reference_table('local_table_1');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SELECT create_distributed_table('local_table_2', 'col_1');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+     tablename     | partmethod | repmodel
+---------------------------------------------------------------------
+ distributed_table | h          | c
+ local_table_1     | n          | t
+ local_table_2     | h          | c
+ local_table_3     | n          | c
+ local_table_4     | n          | c
+ reference_table_1 | n          | t
+(6 rows)
+
+ROLLBACK;
+BEGIN;
+  SELECT create_reference_table('local_table_4');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SELECT create_reference_table('local_table_3');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+     tablename     | partmethod | repmodel
+---------------------------------------------------------------------
+ distributed_table | h          | c
+ local_table_1     | n          | c
+ local_table_2     | n          | c
+ local_table_3     | n          | t
+ local_table_4     | n          | t
+ reference_table_1 | n          | t
+(6 rows)
+
+ROLLBACK;
+BEGIN;
+  CREATE TABLE local_table_5 (col_1 INT REFERENCES local_table_1(col_1));
+  SELECT create_reference_table('local_table_1');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SELECT create_distributed_table('local_table_2', 'col_1');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SELECT create_distributed_table('local_table_5', 'col_1');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+     tablename     | partmethod | repmodel
+---------------------------------------------------------------------
+ distributed_table | h          | c
+ local_table_1     | n          | t
+ local_table_2     | h          | c
+ local_table_3     | n          | c
+ local_table_4     | n          | c
+ local_table_5     | h          | c
+ reference_table_1 | n          | t
+(7 rows)
+
+ROLLBACK;
+BEGIN;
+  ALTER TABLE local_table_1 ADD CONSTRAINT fkey_13 FOREIGN KEY (col_1) REFERENCES local_table_2(col_1) ON DELETE CASCADE;
+  -- errors out as foreign keys from reference tables to citus local tables
+  -- cannot have CASCADE behavior
+  SELECT create_reference_table('local_table_1');
+ERROR:  cannot define foreign key constraint, foreign keys from reference tables to citus local tables can only be defined with NO ACTION or RESTRICT behaviors
+ROLLBACK;
+SET citus.enable_local_execution TO OFF;
+-- show that this errors out as it tries to convert connected relations to citus
+-- local tables and creating citus local table requires local execution but local
+-- execution is disabled
+SELECT create_reference_table('local_table_1');
+ERROR:  cannot execute command because a local execution has accessed a placement in the transaction
+SET citus.enable_local_execution TO ON;
+-- test behavior when outside of the xact block
+CREATE TABLE local_table_6 (col_1 INT REFERENCES local_table_1(col_1));
+SELECT create_reference_table('local_table_1');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+SELECT create_distributed_table('local_table_2', 'col_1');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+SELECT create_distributed_table('local_table_6', 'col_1');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+ORDER BY tablename;
+     tablename     | partmethod | repmodel
+---------------------------------------------------------------------
+ distributed_table | h          | c
+ local_table_1     | n          | t
+ local_table_2     | h          | c
+ local_table_3     | n          | c
+ local_table_4     | n          | c
+ local_table_6     | h          | c
+ reference_table_1 | n          | t
+(7 rows)
+
 -- this errors out as we don't support creating citus local
 -- tables from partitioned tables
 CREATE TABLE part_local_table (col_1 INT REFERENCES reference_table_1(col_1)) PARTITION BY RANGE (col_1);
@@ -426,5 +632,148 @@ ERROR:  column "col_99" referenced in foreign key constraint does not exist
 -- fails as referenced table does not exist
 CREATE TABLE local_table_5 (col_1 INT, FOREIGN KEY (col_1) REFERENCES table_does_not_exist(dummy));
 ERROR:  relation "table_does_not_exist" does not exist
+-- drop & recreate schema to prevent noise in next test outputs
+DROP SCHEMA fkeys_between_local_ref CASCADE;
+CREATE SCHEMA fkeys_between_local_ref;
+SET search_path TO fkeys_between_local_ref;
+-- now have some tests to test behavior before/after enabling foreign keys
+-- between local tables & reference tables
+BEGIN;
+  SET citus.enable_local_reference_table_foreign_keys TO OFF;
+  CREATE TABLE ref_1(a int PRIMARY KEY);
+  CREATE TABLE pg_local_1(a int PRIMARY KEY REFERENCES ref_1(a));
+  SELECT create_reference_table('ref_1');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SET citus.enable_local_reference_table_foreign_keys TO ON;
+  CREATE TABLE ref_2(a int PRIMARY KEY);
+  SELECT create_reference_table('ref_2');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  ALTER TABLE pg_local_1 ADD CONSTRAINT c1 FOREIGN KEY(a) REFERENCES ref_2(a);
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ tablename  | partmethod | repmodel
+---------------------------------------------------------------------
+ pg_local_1 | n          | c
+ ref_1      | n          | t
+ ref_2      | n          | t
+(3 rows)
+
+ROLLBACK;
+BEGIN;
+  SET citus.enable_local_reference_table_foreign_keys TO OFF;
+  CREATE TABLE ref_1(a int PRIMARY KEY);
+  CREATE TABLE pg_local_1(a int PRIMARY KEY REFERENCES ref_1(a));
+  SELECT create_reference_table('ref_1');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SET citus.enable_local_reference_table_foreign_keys TO ON;
+  CREATE TABLE ref_2(a int PRIMARY KEY REFERENCES pg_local_1(a));
+  SELECT create_reference_table('ref_2');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ tablename  | partmethod | repmodel
+---------------------------------------------------------------------
+ pg_local_1 | n          | c
+ ref_1      | n          | t
+ ref_2      | n          | t
+(3 rows)
+
+ROLLBACK;
+BEGIN;
+  SET citus.enable_local_reference_table_foreign_keys TO OFF;
+  CREATE TABLE ref_1(a int PRIMARY KEY);
+  CREATE TABLE pg_local_1(a int PRIMARY KEY REFERENCES ref_1(a));
+  SELECT create_reference_table('ref_1');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SET citus.enable_local_reference_table_foreign_keys TO ON;
+  CREATE TABLE ref_2(a int PRIMARY KEY REFERENCES pg_local_1(a));
+  SELECT create_reference_table('ref_2');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ tablename  | partmethod | repmodel
+---------------------------------------------------------------------
+ pg_local_1 | n          | c
+ ref_1      | n          | t
+ ref_2      | n          | t
+(3 rows)
+
+ROLLBACK;
+BEGIN;
+  SET citus.enable_local_reference_table_foreign_keys TO OFF;
+  CREATE TABLE ref_1(a int PRIMARY KEY);
+  CREATE TABLE pg_local_1(a int PRIMARY KEY REFERENCES ref_1(a));
+  SELECT create_reference_table('ref_1');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+  SET citus.enable_local_reference_table_foreign_keys TO ON;
+  CREATE TABLE pg_local_2(a int PRIMARY KEY REFERENCES pg_local_1(a));
+  -- we still didn't convert local tables to citus local tables
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ tablename | partmethod | repmodel
+---------------------------------------------------------------------
+ ref_1     | n          | t
+(1 row)
+
+  CREATE TABLE pg_local_3(a int PRIMARY KEY REFERENCES ref_1(a));
+  -- pg_local_3 is not connected to other local tables, so we will just
+  -- convert pg_local_3 to a citus local table
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ tablename  | partmethod | repmodel
+---------------------------------------------------------------------
+ pg_local_3 | n          | c
+ ref_1      | n          | t
+(2 rows)
+
+  CREATE TABLE pg_local_4(a int PRIMARY KEY REFERENCES ref_1(a), FOREIGN KEY (a) REFERENCES pg_local_2(a));
+  -- pg_local_4 is connected to ref_1, pg_local_1 and pg_local_2,
+  -- so we will convert those two local tables to citus local tables too
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ tablename  | partmethod | repmodel
+---------------------------------------------------------------------
+ pg_local_1 | n          | c
+ pg_local_2 | n          | c
+ pg_local_3 | n          | c
+ pg_local_4 | n          | c
+ ref_1      | n          | t
+(5 rows)
+
+ROLLBACK;
 -- cleanup at exit
 DROP SCHEMA fkeys_between_local_ref CASCADE;

src/test/regress/expected/foreign_key_to_reference_table.out

@@ -467,10 +467,8 @@ SELECT count(*) FROM referencing_schema.referencing_table;
    800
 (1 row)
 
-SET client_min_messages TO ERROR;
 DROP SCHEMA referenced_schema CASCADE;
 DROP SCHEMA referencing_schema CASCADE;
-RESET client_min_messages;
 -- on delete set update cascades properly
 CREATE TABLE referenced_table(test_column int, test_column2 int, PRIMARY KEY(test_column));
 CREATE TABLE referencing_table(id int, ref_id int DEFAULT 1);
@@ -525,7 +523,6 @@ SELECT count(*) FROM referencing_table;
 (1 row)
 
 DROP TABLE referenced_table CASCADE;
-NOTICE:  drop cascades to constraint fkey_ref on table referencing_table
 DROP TABLE referencing_table CASCADE;
 -- In the following test, we'll use a SERIAL column as the referenced column
 -- in the foreign constraint. We'll first show that and insert on non-serial
@@ -557,7 +554,6 @@ SELECT count(*) FROM referencing_table;
 (1 row)
 
 DROP TABLE referenced_table CASCADE;
-NOTICE:  drop cascades to constraint fkey_ref on table referencing_table
 DROP TABLE referencing_table CASCADE;
 -- In the following test, we'll use a SERIAL column as the referencing column
 -- in the foreign constraint. We'll first show that the values that exist
@@ -587,7 +583,6 @@ INSERT INTO referencing_table(id) SELECT x FROM generate_series(1,10) AS f(x);
 ERROR:  insert or update on table "referencing_table_xxxxxxx" violates foreign key constraint "fkey_ref_xxxxxxx"
 DETAIL:  Key (ref_id)=(X) is not present in table "referenced_table_xxxxxxx".
 DROP TABLE referenced_table CASCADE;
-NOTICE:  drop cascades to constraint fkey_ref on table referencing_table
 DROP TABLE referencing_table CASCADE;
 -- In the following test, we'll use a SERIAL column as the referencing column
 -- and referenced columns in a foreign constraint. We'll first show that the
@@ -618,7 +613,6 @@ INSERT INTO referencing_table(id) SELECT x FROM generate_series(1,10) AS f(x);
 ERROR:  insert or update on table "referencing_table_xxxxxxx" violates foreign key constraint "fkey_ref_xxxxxxx"
 DETAIL:  Key (ref_id)=(X) is not present in table "referenced_table_xxxxxxx".
 DROP TABLE referenced_table CASCADE;
-NOTICE:  drop cascades to constraint fkey_ref on table referencing_table
 DROP TABLE referencing_table CASCADE;
 -- In the following test, we use a volatile function in the referencing
 -- column in a foreign constraint. We show that if the data exists in the
@@ -642,7 +636,6 @@ ALTER TABLE referencing_table ADD CONSTRAINT fkey_ref FOREIGN KEY (ref_id) REFER
 INSERT INTO referenced_table SELECT x, x FROM generate_series(0,1000) AS f(x);
 INSERT INTO referencing_table SELECT x,(random()*1000)::int FROM generate_series(0,1000) AS f(x);
 DROP TABLE referenced_table CASCADE;
-NOTICE:  drop cascades to constraint fkey_ref on table referencing_table
 DROP TABLE referencing_table CASCADE;
 -- In the following tests, we create a foreign constraint with
 -- ON UPDATE CASCADE and see if it works properly with cascading upsert
@@ -677,7 +670,6 @@ SELECT * FROM referencing_table WHERE ref_id < 0 ORDER BY 1;
 (4 rows)
 
 DROP TABLE referenced_table CASCADE;
-NOTICE:  drop cascades to constraint fkey_ref on table referencing_table
 DROP TABLE referencing_table CASCADE;
 -- create_distributed_table should fail for tables with data if fkey exists to reference table
 CREATE TABLE referenced_table(test_column int, test_column2 int, PRIMARY KEY(test_column));
@@ -685,25 +677,12 @@ CREATE TABLE referencing_table(id int, ref_id int DEFAULT -1, FOREIGN KEY (ref_i
 INSERT INTO referenced_table VALUES (1,1), (2,2), (3,3);
 INSERT INTO referencing_table VALUES (1,1), (2,2), (3,3);
 SELECT create_reference_table('referenced_table');
-NOTICE:  Copying data from local table...
-NOTICE:  copying the data has completed
-DETAIL:  The local data in the table is no longer visible, but is still on disk.
-HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$fkey_reference_table.referenced_table$$)
  create_reference_table
 ---------------------------------------------------------------------
 
 (1 row)
 
-SELECT create_distributed_table('referencing_table', 'id');
-ERROR:  cannot distribute "referencing_table" in sequential mode because it is not empty
-HINT:  If you have manually set citus.multi_shard_modify_mode to 'sequential', try with 'parallel' option. If that is not the case, try distributing local tables when they are empty.
-BEGIN;
-  SELECT create_distributed_table('referencing_table', 'id');
-ERROR:  cannot distribute "referencing_table" in sequential mode because it is not empty
-HINT:  If you have manually set citus.multi_shard_modify_mode to 'sequential', try with 'parallel' option. If that is not the case, try distributing local tables when they are empty.
-COMMIT;
 DROP TABLE referenced_table CASCADE;
-NOTICE:  drop cascades to constraint referencing_table_ref_id_fkey on table referencing_table
 DROP TABLE referencing_table CASCADE;
 -- Chained references
 -- In the following test, we create foreign keys from one column in a distributed
@@ -778,9 +757,7 @@ SELECT count(*) FROM referencing_table;
 (1 row)
 
 DROP TABLE referenced_table CASCADE;
-NOTICE:  drop cascades to constraint fkey_ref on table referencing_table
 DROP TABLE referenced_table2 CASCADE;
-NOTICE:  drop cascades to constraint foreign_key_2 on table referencing_table
 DROP TABLE referencing_table CASCADE;
 -- check if the above fkeys are created with create_distributed_table
 CREATE TABLE referenced_table(test_column int, test_column2 int, PRIMARY KEY(test_column));
@@ -812,9 +789,7 @@ SELECT count(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_tab
 
 \set VERBOSITY terse
 DROP TABLE referenced_table CASCADE;
-NOTICE:  drop cascades to constraint referencing_table_id_fkey on table referencing_table
 DROP TABLE referenced_table2 CASCADE;
-NOTICE:  drop cascades to constraint referencing_table_id_fkey1 on table referencing_table
 DROP TABLE referencing_table CASCADE;
 -- In the following test, we create foreign keys from two columns in a distributed
 -- table to two reference tables separately. We expect to see that even if a data
@@ -888,9 +863,7 @@ SELECT count(*) FROM referencing_table;
 (1 row)
 
 DROP TABLE referenced_table CASCADE;
-NOTICE:  drop cascades to constraint fkey_ref on table referencing_table
 DROP TABLE referenced_table2 CASCADE;
-NOTICE:  drop cascades to constraint foreign_key_2 on table referencing_table
 DROP TABLE referencing_table CASCADE;
 -- check if the above fkeys are created when create_distributed_table is used for 1 foreign key and alter table for the other
 CREATE TABLE referenced_table(test_column int, test_column2 int, PRIMARY KEY(test_column));
@@ -924,9 +897,7 @@ SELECT count(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_tab
 (1 row)
 
 DROP TABLE referenced_table CASCADE;
-NOTICE:  drop cascades to constraint referencing_table_id_fkey on table referencing_table
 DROP TABLE referenced_table2 CASCADE;
-NOTICE:  drop cascades to constraint foreign_key_2 on table referencing_table
 DROP TABLE referencing_table CASCADE;
 \set VERBOSITY default
 -- two distributed tables are referencing to one reference table and
@@ -1005,48 +976,25 @@ SELECT count(*) FROM referencing_table2;
 
 \set VERBOSITY terse
 DROP TABLE referenced_table CASCADE;
-NOTICE:  drop cascades to 2 other objects
 DROP TABLE referencing_table CASCADE;
-NOTICE:  drop cascades to constraint fkey_ref_to_dist on table referencing_table2
 DROP TABLE referencing_table2 CASCADE;
 \set VERBOSITY default
 -- Check if the above fkeys are created with create_distributed_table
 CREATE TABLE referenced_table(test_column int, test_column2 int UNIQUE, PRIMARY KEY(test_column));
-CREATE TABLE referencing_table(id int PRIMARY KEY, ref_id int, FOREIGN KEY (id) REFERENCES referenced_table(test_column) ON DELETE CASCADE);
-CREATE TABLE referencing_table2(id int, ref_id int, FOREIGN KEY (ref_id) REFERENCES referenced_table(test_column2) ON DELETE CASCADE, FOREIGN KEY (id) REFERENCES referencing_table(id) ON DELETE CASCADE);
 SELECT create_reference_table('referenced_table');
  create_reference_table
 ---------------------------------------------------------------------
 
 (1 row)
 
-BEGIN;
-  SET LOCAL citus.multi_shard_modify_mode TO 'sequential';
-  SELECT create_distributed_table('referencing_table', 'id');
- create_distributed_table
----------------------------------------------------------------------
-
-(1 row)
-
-  SELECT create_distributed_table('referencing_table2', 'id');
- create_distributed_table
----------------------------------------------------------------------
-
-(1 row)
-
-COMMIT;
 SELECT count(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
  count
 ---------------------------------------------------------------------
-    24
+     0
 (1 row)
 
 \set VERBOSITY terse
 DROP TABLE referenced_table CASCADE;
-NOTICE:  drop cascades to 2 other objects
-DROP TABLE referencing_table CASCADE;
-NOTICE:  drop cascades to constraint referencing_table2_id_fkey on table referencing_table2
-DROP TABLE referencing_table2 CASCADE;
 \set VERBOSITY default
 -- In this test we have a chained relationship in form of
 -- distributed table (referencing_referencing_table) has a foreign key with two columns
@@ -1092,11 +1040,8 @@ SELECT max(ref_id) FROM referencing_referencing_table;
 (1 row)
 
 DROP TABLE referenced_table CASCADE;
-NOTICE:  drop cascades to constraint fkey_ref on table referencing_table
 DROP TABLE referencing_table CASCADE;
-NOTICE:  drop cascades to constraint referencing_referencing_table_id_fkey on table referencing_referencing_table
 DROP TABLE referencing_referencing_table;
--- create_reference_table, create_distributed_table and ALTER TABLE in the same transaction
 BEGIN;
   CREATE TABLE test_table_1(id int PRIMARY KEY);
   SELECT create_reference_table('test_table_1');
@@ -1106,28 +1051,11 @@ BEGIN;
 (1 row)
 
   CREATE TABLE test_table_2(id int PRIMARY KEY, value_1 int);
-  SELECT create_distributed_table('test_table_2', 'id');
- create_distributed_table
----------------------------------------------------------------------
-
-(1 row)
-
   ALTER TABLE test_table_2 ADD CONSTRAINT c_check FOREIGN KEY (value_1) REFERENCES test_table_1(id);
-ERROR:  cannot modify table "test_table_2" because there was a parallel operation on a distributed table in the transaction
-DETAIL:  When there is a foreign key to a reference table, Citus needs to perform all operations over a single connection per node to ensure consistency.
-HINT:  Try re-running the transaction with "SET LOCAL citus.multi_shard_modify_mode TO 'sequential';"
   DROP TABLE test_table_1, test_table_2;
-ERROR:  current transaction is aborted, commands ignored until end of transaction block
 COMMIT;
--- the order of create_reference_table and create_distributed_table is changed
 BEGIN;
   CREATE TABLE test_table_1(id int PRIMARY KEY, value_1 int);
-  SELECT create_distributed_table('test_table_1', 'id');
- create_distributed_table
----------------------------------------------------------------------
-
-(1 row)
-
   CREATE TABLE test_table_2(id int PRIMARY KEY);
   SELECT create_reference_table('test_table_2');
  create_reference_table
@@ -1136,33 +1064,20 @@ BEGIN;
 (1 row)
 
   ALTER TABLE test_table_1 ADD CONSTRAINT c_check FOREIGN KEY (value_1) REFERENCES test_table_2(id);
-ERROR:  cannot modify table "test_table_1" because there was a parallel operation on a distributed table in the transaction
-DETAIL:  When there is a foreign key to a reference table, Citus needs to perform all operations over a single connection per node to ensure consistency.
-HINT:  Try re-running the transaction with "SET LOCAL citus.multi_shard_modify_mode TO 'sequential';"
   DROP TABLE test_table_2 CASCADE;
-ERROR:  current transaction is aborted, commands ignored until end of transaction block
 ROLLBACK;
--- make sure that we fail if we need parallel data load
 BEGIN;
   CREATE TABLE test_table_1(id int PRIMARY KEY);
   INSERT INTO test_table_1 SELECT i FROM generate_series(0,100) i;
   CREATE TABLE test_table_2(id int PRIMARY KEY, value_1 int, FOREIGN KEY(value_1) REFERENCES test_table_1(id));
   INSERT INTO test_table_2 SELECT i, i FROM generate_series(0,100) i;
   SELECT create_reference_table('test_table_1');
-NOTICE:  Copying data from local table...
-NOTICE:  copying the data has completed
-DETAIL:  The local data in the table is no longer visible, but is still on disk.
-HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$fkey_reference_table.test_table_1$$)
  create_reference_table
 ---------------------------------------------------------------------
 
 (1 row)
 
-  SELECT create_distributed_table('test_table_2', 'id');
-ERROR:  cannot distribute "test_table_2" in sequential mode because it is not empty
-HINT:  If you have manually set citus.multi_shard_modify_mode to 'sequential', try with 'parallel' option. If that is not the case, try distributing local tables when they are empty.
   DROP TABLE test_table_2, test_table_1;
-ERROR:  current transaction is aborted, commands ignored until end of transaction block
 COMMIT;
 -- make sure that other DDLs/DMLs also work fine
 BEGIN;
@@ -1261,7 +1176,6 @@ SELECT create_distributed_table('test_table_2', 'id');
 (1 row)
 
 ALTER TABLE test_table_1 DROP CONSTRAINT test_table_1_pkey CASCADE;
-NOTICE:  drop cascades to constraint test_table_2_value_1_fkey on table test_table_2
 SELECT count(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
  count
 ---------------------------------------------------------------------
@@ -1286,7 +1200,6 @@ BEGIN;
 (1 row)
 
   ALTER TABLE test_table_1 DROP CONSTRAINT test_table_1_pkey CASCADE;
-NOTICE:  drop cascades to constraint test_table_2_value_1_fkey on table test_table_2
 COMMIT;
 SELECT count(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
  count
@@ -1359,7 +1272,6 @@ SELECT create_distributed_table('test_table_2', 'id');
 (1 row)
 
 ALTER TABLE test_table_1 DROP COLUMN id CASCADE;
-NOTICE:  drop cascades to constraint test_table_2_value_1_fkey on table test_table_2
 SELECT count(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
  count
 ---------------------------------------------------------------------
@@ -1384,7 +1296,6 @@ BEGIN;
 (1 row)
 
   ALTER TABLE test_table_1 DROP COLUMN id CASCADE;
-NOTICE:  drop cascades to constraint test_table_2_value_1_fkey on table test_table_2
 COMMIT;
 SELECT count(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
  count
@@ -1426,7 +1337,6 @@ SELECT count(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_tab
 (1 row)
 
 DROP TABLE test_table_1 CASCADE;
-NOTICE:  drop cascades to constraint test_table_2_value_1_fkey on table test_table_2
 DROP TABLE test_table_2;
 -- check if we can alter the column type and drop it which foreign key is referencing to in a transaction block
 CREATE TABLE test_table_1(id int PRIMARY KEY);
@@ -1446,7 +1356,6 @@ BEGIN;
 
   ALTER TABLE test_table_2 ALTER COLUMN value_1 SET DATA TYPE bigint;
   ALTER TABLE test_table_1 DROP COLUMN id CASCADE;
-NOTICE:  drop cascades to constraint test_table_2_value_1_fkey on table test_table_2
 COMMIT;
 SELECT count(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
  count
@@ -1473,7 +1382,6 @@ SELECT create_distributed_table('test_table_2', 'id');
 INSERT INTO test_table_1 VALUES (1),(2),(3);
 INSERT INTO test_table_2 VALUES (1,1),(2,2),(3,3);
 TRUNCATE test_table_1 CASCADE;
-NOTICE:  truncate cascades to table "test_table_2"
 SELECT * FROM test_table_2;
  id | value_1
 ---------------------------------------------------------------------
@@ -1499,7 +1407,6 @@ INSERT INTO test_table_1 VALUES (1),(2),(3);
 INSERT INTO test_table_2 VALUES (1,1),(2,2),(3,3);
 BEGIN;
   TRUNCATE test_table_1 CASCADE;
-NOTICE:  truncate cascades to table "test_table_2"
 COMMIT;
 SELECT * FROM test_table_2;
  id | value_1
@@ -1526,7 +1433,6 @@ BEGIN;
   INSERT INTO test_table_1 VALUES (1),(2),(3);
   INSERT INTO test_table_2 VALUES (1,1),(2,2),(3,3);
   TRUNCATE test_table_1 CASCADE;
-NOTICE:  truncate cascades to table "test_table_2"
 COMMIT;
 SELECT * FROM test_table_2;
  id | value_1
@@ -1627,7 +1533,6 @@ BEGIN;
   ALTER TABLE test_table_2 ADD CONSTRAINT fkey FOREIGN KEY (value_1) REFERENCES test_table_1(id);
   ALTER TABLE test_table_3 ADD COLUMN test_column int;
   ALTER TABLE test_table_1 DROP COLUMN id CASCADE;
-NOTICE:  drop cascades to constraint fkey on table test_table_2
   ALTER TABLE test_table_1 ADD COLUMN id int;
 COMMIT;
 DROP TABLE test_table_1, test_table_2, test_table_3;
@@ -1703,9 +1608,7 @@ BEGIN;
 
 ROLLBACK;
 DROP TABLE referenced_table CASCADE;
-NOTICE:  drop cascades to constraint fkey_to_ref on table referencing_table_4
 DROP TABLE referencing_table;
-SET client_min_messages TO ERROR;
 DROP SCHEMA fkey_reference_table CASCADE;
 SET search_path TO DEFAULT;
 RESET client_min_messages;

src/test/regress/expected/multi_extension.out

@@ -495,6 +495,8 @@ SELECT * FROM print_extension_changes();
                                                                                  | function citus_update_table_statistics(regclass)
                                                                                  | function columnar.columnar_handler(internal)
                                                                                  | function create_citus_local_table(regclass,boolean)
+                                                                                 | function notify_constraint_dropped()
+                                                                                 | function remove_local_tables_from_metadata()
                                                                                  | function time_partition_range(regclass)
                                                                                  | function undistribute_table(regclass,boolean)
                                                                                  | function worker_change_sequence_dependency(regclass,regclass,regclass)
@@ -506,7 +508,7 @@ SELECT * FROM print_extension_changes();
                                                                                  | view citus_shards
                                                                                  | view citus_tables
                                                                                  | view time_partitions
-(61 rows)
+(63 rows)
 
 DROP TABLE prev_objects, extension_diff;
 -- show running version

src/test/regress/expected/multi_extension_0.out

@@ -491,6 +491,8 @@ SELECT * FROM print_extension_changes();
                                                                                  | function citus_update_shard_statistics(bigint)
                                                                                  | function citus_update_table_statistics(regclass)
                                                                                  | function create_citus_local_table(regclass,boolean)
+                                                                                 | function notify_constraint_dropped()
+                                                                                 | function remove_local_tables_from_metadata()
                                                                                  | function time_partition_range(regclass)
                                                                                  | function undistribute_table(regclass,boolean)
                                                                                  | function worker_change_sequence_dependency(regclass,regclass,regclass)
@@ -502,7 +504,7 @@ SELECT * FROM print_extension_changes();
                                                                                  | view citus_shards
                                                                                  | view citus_tables
                                                                                  | view time_partitions
-(57 rows)
+(59 rows)
 
 DROP TABLE prev_objects, extension_diff;
 -- show running version

src/test/regress/expected/ref_citus_local_fkeys.out

@@ -52,19 +52,25 @@ NOTICE:  executing the command locally: SELECT l1 FROM ref_citus_local_fkeys.cit
 -- show that we support drop constraint
 ALTER TABLE citus_local_table DROP CONSTRAINT fkey_local_to_ref;
 NOTICE:  executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506000, 'ref_citus_local_fkeys', 1506001, 'ref_citus_local_fkeys', 'ALTER TABLE citus_local_table DROP CONSTRAINT fkey_local_to_ref;')
+NOTICE:  creating a new table for ref_citus_local_fkeys.citus_local_table
+NOTICE:  Moving the data of ref_citus_local_fkeys.citus_local_table
+NOTICE:  executing the command locally: SELECT l1 FROM ref_citus_local_fkeys.citus_local_table_1506000 citus_local_table
+NOTICE:  Dropping the old ref_citus_local_fkeys.citus_local_table
+NOTICE:  executing the command locally: DROP TABLE IF EXISTS ref_citus_local_fkeys.citus_local_table_xxxxx CASCADE
+NOTICE:  Renaming the new table to ref_citus_local_fkeys.citus_local_table
 -- we support ON UPDATE CASCADE behaviour in "ALTER TABLE ADD fkey citus_local_table (to reference table)" commands
 ALTER TABLE citus_local_table ADD CONSTRAINT fkey_local_to_ref FOREIGN KEY(l1) REFERENCES reference_table(r1) ON UPDATE CASCADE;
-NOTICE:  executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506000, 'ref_citus_local_fkeys', 1506001, 'ref_citus_local_fkeys', 'ALTER TABLE citus_local_table ADD CONSTRAINT fkey_local_to_ref FOREIGN KEY(l1) REFERENCES reference_table(r1) ON UPDATE CASCADE;')
+NOTICE:  executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506002, 'ref_citus_local_fkeys', 1506001, 'ref_citus_local_fkeys', 'ALTER TABLE citus_local_table ADD CONSTRAINT fkey_local_to_ref FOREIGN KEY(l1) REFERENCES reference_table(r1) ON UPDATE CASCADE;')
 -- show that on update cascade works
 INSERT INTO reference_table VALUES (12);
 NOTICE:  executing the command locally: INSERT INTO ref_citus_local_fkeys.reference_table_1506001 (r1) VALUES (12)
 INSERT INTO citus_local_table VALUES (12);
-NOTICE:  executing the command locally: INSERT INTO ref_citus_local_fkeys.citus_local_table_1506000 (l1) VALUES (12)
+NOTICE:  executing the command locally: INSERT INTO ref_citus_local_fkeys.citus_local_table_1506002 (l1) VALUES (12)
 UPDATE reference_table SET r1=13 WHERE r1=12;
 NOTICE:  executing the command locally: UPDATE ref_citus_local_fkeys.reference_table_1506001 reference_table SET r1 = 13 WHERE (r1 OPERATOR(pg_catalog.=) 12)
 -- should print a row with 13
 SELECT * FROM citus_local_table ORDER BY l1;
-NOTICE:  executing the command locally: SELECT l1 FROM ref_citus_local_fkeys.citus_local_table_1506000 citus_local_table ORDER BY l1
+NOTICE:  executing the command locally: SELECT l1 FROM ref_citus_local_fkeys.citus_local_table_1506002 citus_local_table ORDER BY l1
  l1
 ---------------------------------------------------------------------
  13
@@ -72,32 +78,43 @@ NOTICE:  executing the command locally: SELECT l1 FROM ref_citus_local_fkeys.cit
 
 -- drop constraint for next commands
 ALTER TABLE citus_local_table DROP CONSTRAINT fkey_local_to_ref;
-NOTICE:  executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506000, 'ref_citus_local_fkeys', 1506001, 'ref_citus_local_fkeys', 'ALTER TABLE citus_local_table DROP CONSTRAINT fkey_local_to_ref;')
+NOTICE:  executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506002, 'ref_citus_local_fkeys', 1506001, 'ref_citus_local_fkeys', 'ALTER TABLE citus_local_table DROP CONSTRAINT fkey_local_to_ref;')
+NOTICE:  creating a new table for ref_citus_local_fkeys.citus_local_table
+NOTICE:  Moving the data of ref_citus_local_fkeys.citus_local_table
+NOTICE:  executing the command locally: SELECT l1 FROM ref_citus_local_fkeys.citus_local_table_1506002 citus_local_table
+NOTICE:  Dropping the old ref_citus_local_fkeys.citus_local_table
+NOTICE:  executing the command locally: DROP TABLE IF EXISTS ref_citus_local_fkeys.citus_local_table_xxxxx CASCADE
+NOTICE:  Renaming the new table to ref_citus_local_fkeys.citus_local_table
 INSERT INTO citus_local_table VALUES (2);
-NOTICE:  executing the command locally: INSERT INTO ref_citus_local_fkeys.citus_local_table_1506000 (l1) VALUES (2)
 -- show that we are checking for foreign key constraint while defining, below should fail
 ALTER TABLE citus_local_table ADD CONSTRAINT fkey_local_to_ref FOREIGN KEY(l1) REFERENCES reference_table(r1);
-NOTICE:  executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506000, 'ref_citus_local_fkeys', 1506001, 'ref_citus_local_fkeys', 'ALTER TABLE citus_local_table ADD CONSTRAINT fkey_local_to_ref FOREIGN KEY(l1) REFERENCES reference_table(r1);')
+NOTICE:  executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506003, 'ref_citus_local_fkeys', 1506001, 'ref_citus_local_fkeys', 'ALTER TABLE citus_local_table ADD CONSTRAINT fkey_local_to_ref FOREIGN KEY(l1) REFERENCES reference_table(r1);')
-ERROR:  insert or update on table "citus_local_table_1506000" violates foreign key constraint "fkey_local_to_ref_1506000"
+ERROR:  insert or update on table "citus_local_table_1506003" violates foreign key constraint "fkey_local_to_ref_1506003"
 INSERT INTO reference_table VALUES (2);
 NOTICE:  executing the command locally: INSERT INTO ref_citus_local_fkeys.reference_table_1506001 (r1) VALUES (2)
 -- this should work
 ALTER TABLE citus_local_table ADD CONSTRAINT fkey_local_to_ref FOREIGN KEY(l1) REFERENCES reference_table(r1);
-NOTICE:  executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506000, 'ref_citus_local_fkeys', 1506001, 'ref_citus_local_fkeys', 'ALTER TABLE citus_local_table ADD CONSTRAINT fkey_local_to_ref FOREIGN KEY(l1) REFERENCES reference_table(r1);')
+NOTICE:  executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506004, 'ref_citus_local_fkeys', 1506001, 'ref_citus_local_fkeys', 'ALTER TABLE citus_local_table ADD CONSTRAINT fkey_local_to_ref FOREIGN KEY(l1) REFERENCES reference_table(r1);')
 -- show that we are checking for foreign key constraint after defining, this should fail
 INSERT INTO citus_local_table VALUES (1);
-NOTICE:  executing the command locally: INSERT INTO ref_citus_local_fkeys.citus_local_table_1506000 (l1) VALUES (1)
+NOTICE:  executing the command locally: INSERT INTO ref_citus_local_fkeys.citus_local_table_1506004 (l1) VALUES (1)
-ERROR:  insert or update on table "citus_local_table_1506000" violates foreign key constraint "fkey_local_to_ref_1506000"
+ERROR:  insert or update on table "citus_local_table_1506004" violates foreign key constraint "fkey_local_to_ref_1506004"
 INSERT INTO reference_table VALUES (1);
 NOTICE:  executing the command locally: INSERT INTO ref_citus_local_fkeys.reference_table_1506001 (r1) VALUES (1)
 -- this should work
 INSERT INTO citus_local_table VALUES (1);
-NOTICE:  executing the command locally: INSERT INTO ref_citus_local_fkeys.citus_local_table_1506000 (l1) VALUES (1)
+NOTICE:  executing the command locally: INSERT INTO ref_citus_local_fkeys.citus_local_table_1506004 (l1) VALUES (1)
 -- drop and add constraint for next commands
 ALTER TABLE citus_local_table DROP CONSTRAINT fkey_local_to_ref;
-NOTICE:  executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506000, 'ref_citus_local_fkeys', 1506001, 'ref_citus_local_fkeys', 'ALTER TABLE citus_local_table DROP CONSTRAINT fkey_local_to_ref;')
+NOTICE:  executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506004, 'ref_citus_local_fkeys', 1506001, 'ref_citus_local_fkeys', 'ALTER TABLE citus_local_table DROP CONSTRAINT fkey_local_to_ref;')
+NOTICE:  creating a new table for ref_citus_local_fkeys.citus_local_table
+NOTICE:  Moving the data of ref_citus_local_fkeys.citus_local_table
+NOTICE:  executing the command locally: SELECT l1 FROM ref_citus_local_fkeys.citus_local_table_1506004 citus_local_table
+NOTICE:  Dropping the old ref_citus_local_fkeys.citus_local_table
+NOTICE:  executing the command locally: DROP TABLE IF EXISTS ref_citus_local_fkeys.citus_local_table_xxxxx CASCADE
+NOTICE:  Renaming the new table to ref_citus_local_fkeys.citus_local_table
 ALTER TABLE citus_local_table ADD CONSTRAINT fkey_local_to_ref FOREIGN KEY(l1) REFERENCES reference_table(r1);
-NOTICE:  executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506000, 'ref_citus_local_fkeys', 1506001, 'ref_citus_local_fkeys', 'ALTER TABLE citus_local_table ADD CONSTRAINT fkey_local_to_ref FOREIGN KEY(l1) REFERENCES reference_table(r1);')
+NOTICE:  executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506005, 'ref_citus_local_fkeys', 1506001, 'ref_citus_local_fkeys', 'ALTER TABLE citus_local_table ADD CONSTRAINT fkey_local_to_ref FOREIGN KEY(l1) REFERENCES reference_table(r1);')
 -- show that drop table without CASCADE errors out
 DROP TABLE reference_table;
 ERROR:  cannot drop table reference_table because other objects depend on it
@@ -106,12 +123,18 @@ BEGIN;
   DROP TABLE reference_table CASCADE;
 NOTICE:  drop cascades to constraint fkey_local_to_ref on table citus_local_table
 NOTICE:  executing the command locally: DROP TABLE IF EXISTS ref_citus_local_fkeys.reference_table_xxxxx CASCADE
-NOTICE:  drop cascades to constraint fkey_local_to_ref_1506000 on table ref_citus_local_fkeys.citus_local_table_1506000
+NOTICE:  drop cascades to constraint fkey_local_to_ref_1506005 on table ref_citus_local_fkeys.citus_local_table_1506005
+NOTICE:  creating a new table for ref_citus_local_fkeys.citus_local_table
+NOTICE:  Moving the data of ref_citus_local_fkeys.citus_local_table
+NOTICE:  executing the command locally: SELECT l1 FROM ref_citus_local_fkeys.citus_local_table_1506005 citus_local_table
+NOTICE:  Dropping the old ref_citus_local_fkeys.citus_local_table
+NOTICE:  executing the command locally: DROP TABLE IF EXISTS ref_citus_local_fkeys.citus_local_table_xxxxx CASCADE
+NOTICE:  Renaming the new table to ref_citus_local_fkeys.citus_local_table
 ROLLBACK;
 -- drop tables finally
 DROP TABLE citus_local_table, reference_table;
 NOTICE:  executing the command locally: DROP TABLE IF EXISTS ref_citus_local_fkeys.reference_table_xxxxx CASCADE
-NOTICE:  drop cascades to constraint fkey_local_to_ref_1506000 on table ref_citus_local_fkeys.citus_local_table_1506000
+NOTICE:  drop cascades to constraint fkey_local_to_ref_1506005 on table ref_citus_local_fkeys.citus_local_table_1506005
 NOTICE:  executing the command locally: DROP TABLE IF EXISTS ref_citus_local_fkeys.citus_local_table_xxxxx CASCADE
 ---------------------------------------------------------------------
 -- foreign key from reference table to citus local table --
@@ -141,11 +164,11 @@ SELECT create_reference_table('reference_table');
 (1 row)
 
 INSERT INTO reference_table VALUES (3);
-NOTICE:  executing the command locally: INSERT INTO ref_citus_local_fkeys.reference_table_1506003 (r1) VALUES (3)
+NOTICE:  executing the command locally: INSERT INTO ref_citus_local_fkeys.reference_table_1506007 (r1) VALUES (3)
 -- show that we are checking for foreign key constraint while defining, this should fail
 ALTER TABLE reference_table ADD CONSTRAINT fkey_ref_to_local FOREIGN KEY(r1) REFERENCES citus_local_table(l1);
-NOTICE:  executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506003, 'ref_citus_local_fkeys', 1506002, 'ref_citus_local_fkeys', 'ALTER TABLE reference_table ADD CONSTRAINT fkey_ref_to_local FOREIGN KEY(r1) REFERENCES citus_local_table(l1);')
+NOTICE:  executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506007, 'ref_citus_local_fkeys', 1506006, 'ref_citus_local_fkeys', 'ALTER TABLE reference_table ADD CONSTRAINT fkey_ref_to_local FOREIGN KEY(r1) REFERENCES citus_local_table(l1);')
-ERROR:  insert or update on table "reference_table_1506003" violates foreign key constraint "fkey_ref_to_local_1506003"
+ERROR:  insert or update on table "reference_table_1506007" violates foreign key constraint "fkey_ref_to_local_1506007"
 -- we do not support CASCADE / SET NULL / SET DEFAULT behavior in "ALTER TABLE ADD fkey reference_table (to citus_local_table)" commands
 ALTER TABLE reference_table ADD CONSTRAINT fkey_ref_to_local FOREIGN KEY(r1) REFERENCES citus_local_table(l1) ON DELETE CASCADE;
 ERROR:  cannot define foreign key constraint, foreign keys from reference tables to citus local tables can only be defined with NO ACTION or RESTRICT behaviors
@@ -160,25 +183,32 @@ ERROR:  cannot define foreign key constraint, foreign keys from reference tables
 ALTER TABLE reference_table ADD CONSTRAINT fkey_ref_to_local FOREIGN KEY(r1) REFERENCES citus_local_table(l1) ON UPDATE SET DEFAULT;
 ERROR:  cannot define foreign key constraint, foreign keys from reference tables to citus local tables can only be defined with NO ACTION or RESTRICT behaviors
 INSERT INTO citus_local_table VALUES (3);
-NOTICE:  executing the command locally: INSERT INTO ref_citus_local_fkeys.citus_local_table_1506002 (l1) VALUES (3)
+NOTICE:  executing the command locally: INSERT INTO ref_citus_local_fkeys.citus_local_table_1506006 (l1) VALUES (3)
 -- .. but we allow such foreign keys with RESTRICT behavior
 BEGIN;
   ALTER TABLE reference_table ADD CONSTRAINT fkey_ref_to_local FOREIGN KEY(r1) REFERENCES citus_local_table(l1) ON DELETE RESTRICT;
-NOTICE:  executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506003, 'ref_citus_local_fkeys', 1506002, 'ref_citus_local_fkeys', 'ALTER TABLE reference_table ADD CONSTRAINT fkey_ref_to_local FOREIGN KEY(r1) REFERENCES citus_local_table(l1) ON DELETE RESTRICT;')
+NOTICE:  executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506007, 'ref_citus_local_fkeys', 1506006, 'ref_citus_local_fkeys', 'ALTER TABLE reference_table ADD CONSTRAINT fkey_ref_to_local FOREIGN KEY(r1) REFERENCES citus_local_table(l1) ON DELETE RESTRICT;')
 ROLLBACK;
 -- .. and we allow such foreign keys with NO ACTION behavior
 ALTER TABLE reference_table ADD CONSTRAINT fkey_ref_to_local FOREIGN KEY(r1) REFERENCES citus_local_table(l1) ON DELETE NO ACTION;
-NOTICE:  executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506003, 'ref_citus_local_fkeys', 1506002, 'ref_citus_local_fkeys', 'ALTER TABLE reference_table ADD CONSTRAINT fkey_ref_to_local FOREIGN KEY(r1) REFERENCES citus_local_table(l1) ON DELETE NO ACTION;')
+NOTICE:  executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506007, 'ref_citus_local_fkeys', 1506006, 'ref_citus_local_fkeys', 'ALTER TABLE reference_table ADD CONSTRAINT fkey_ref_to_local FOREIGN KEY(r1) REFERENCES citus_local_table(l1) ON DELETE NO ACTION;')
 -- show that adding/dropping foreign keys from reference to citus local
 -- tables works fine with remote execution too
 SET citus.enable_local_execution TO OFF;
 ALTER TABLE reference_table DROP CONSTRAINT fkey_ref_to_local;
+NOTICE:  creating a new table for ref_citus_local_fkeys.citus_local_table
+NOTICE:  Moving the data of ref_citus_local_fkeys.citus_local_table
+NOTICE:  Dropping the old ref_citus_local_fkeys.citus_local_table
+NOTICE:  Renaming the new table to ref_citus_local_fkeys.citus_local_table
 ALTER TABLE reference_table ADD CONSTRAINT fkey_ref_to_local FOREIGN KEY(r1) REFERENCES citus_local_table(l1) ON DELETE NO ACTION;
+ERROR:  cannot execute command because a local execution has accessed a placement in the transaction
 SET citus.enable_local_execution TO ON;
+ALTER TABLE reference_table ADD CONSTRAINT fkey_ref_to_local FOREIGN KEY(r1) REFERENCES citus_local_table(l1) ON DELETE NO ACTION;
+NOTICE:  executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506007, 'ref_citus_local_fkeys', 1506009, 'ref_citus_local_fkeys', 'ALTER TABLE reference_table ADD CONSTRAINT fkey_ref_to_local FOREIGN KEY(r1) REFERENCES citus_local_table(l1) ON DELETE NO ACTION;')
 -- show that we are checking for foreign key constraint after defining, this should fail
 INSERT INTO reference_table VALUES (4);
-NOTICE:  executing the command locally: INSERT INTO ref_citus_local_fkeys.reference_table_1506003 (r1) VALUES (4)
+NOTICE:  executing the command locally: INSERT INTO ref_citus_local_fkeys.reference_table_1506007 (r1) VALUES (4)
-ERROR:  insert or update on table "reference_table_1506003" violates foreign key constraint "fkey_ref_to_local_1506003"
+ERROR:  insert or update on table "reference_table_1506007" violates foreign key constraint "fkey_ref_to_local_1506007"
 -- enable the worker_2 to show that we don't try to set up the foreign keys
 -- between reference tables and citus local tables in worker_2 placements of
 -- the reference tables
@@ -192,7 +222,13 @@ NOTICE:  Replicating reference table "reference_table" to the node localhost:xxx
 -- show that we support drop constraint
 BEGIN;
   ALTER TABLE reference_table DROP CONSTRAINT fkey_ref_to_local;
-NOTICE:  executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506003, 'ref_citus_local_fkeys', 1506002, 'ref_citus_local_fkeys', 'ALTER TABLE reference_table DROP CONSTRAINT fkey_ref_to_local;')
+NOTICE:  executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506007, 'ref_citus_local_fkeys', 1506009, 'ref_citus_local_fkeys', 'ALTER TABLE reference_table DROP CONSTRAINT fkey_ref_to_local;')
+NOTICE:  creating a new table for ref_citus_local_fkeys.citus_local_table
+NOTICE:  Moving the data of ref_citus_local_fkeys.citus_local_table
+NOTICE:  executing the command locally: SELECT l1 FROM ref_citus_local_fkeys.citus_local_table_1506009 citus_local_table
+NOTICE:  Dropping the old ref_citus_local_fkeys.citus_local_table
+NOTICE:  executing the command locally: DROP TABLE IF EXISTS ref_citus_local_fkeys.citus_local_table_xxxxx CASCADE
+NOTICE:  Renaming the new table to ref_citus_local_fkeys.citus_local_table
 ROLLBACK;
 -- show that drop table errors as expected
 DROP TABLE citus_local_table;
@@ -201,7 +237,7 @@ ERROR:  cannot drop table citus_local_table because other objects depend on it
 DROP TABLE citus_local_table CASCADE;
 NOTICE:  drop cascades to constraint fkey_ref_to_local on table reference_table
 NOTICE:  executing the command locally: DROP TABLE IF EXISTS ref_citus_local_fkeys.citus_local_table_xxxxx CASCADE
-NOTICE:  drop cascades to constraint fkey_ref_to_local_1506003 on table ref_citus_local_fkeys.reference_table_1506003
+NOTICE:  drop cascades to constraint fkey_ref_to_local_1506007 on table ref_citus_local_fkeys.reference_table_1506007
 BEGIN;
   CREATE TABLE citus_local_table_1(a int, b int, unique (a,b));
   CREATE TABLE citus_local_table_2(a int, b int, unique (a,b));
@@ -219,7 +255,7 @@ BEGIN;
 
   -- show that we properly handle multi column foreign keys
   ALTER TABLE citus_local_table_1 ADD CONSTRAINT multi_fkey FOREIGN KEY (a, b) REFERENCES citus_local_table_2(a, b);
-NOTICE:  executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506004, 'ref_citus_local_fkeys', 1506005, 'ref_citus_local_fkeys', 'ALTER TABLE citus_local_table_1 ADD CONSTRAINT multi_fkey FOREIGN KEY (a, b) REFERENCES citus_local_table_2(a, b);')
+NOTICE:  executing the command locally: SELECT worker_apply_inter_shard_ddl_command (1506010, 'ref_citus_local_fkeys', 1506011, 'ref_citus_local_fkeys', 'ALTER TABLE citus_local_table_1 ADD CONSTRAINT multi_fkey FOREIGN KEY (a, b) REFERENCES citus_local_table_2(a, b);')
 COMMIT;
 -- when local execution is disabled, citus local table cannot be created
 BEGIN;

src/test/regress/expected/upgrade_list_citus_objects.out

@@ -151,12 +151,14 @@ ORDER BY 1;
  function master_update_node(integer,text,integer,boolean,integer)
  function master_update_shard_statistics(bigint)
  function master_update_table_statistics(regclass)
+ function notify_constraint_dropped()
  function poolinfo_valid(text)
  function read_intermediate_result(text,citus_copy_format)
  function read_intermediate_results(text[],citus_copy_format)
  function rebalance_table_shards(regclass,real,integer,bigint[],citus.shard_transfer_mode,boolean,name)
  function recover_prepared_transactions()
  function relation_is_a_known_shard(regclass)
+ function remove_local_tables_from_metadata()
  function replicate_reference_tables()
  function replicate_table_shards(regclass,integer,integer,bigint[],citus.shard_transfer_mode)
  function role_exists(name)
@@ -236,5 +238,5 @@ ORDER BY 1;
  view citus_worker_stat_activity
  view pg_dist_shard_placement
  view time_partitions
-(220 rows)
+(222 rows)
 

src/test/regress/expected/upgrade_list_citus_objects_0.out

@@ -147,12 +147,14 @@ ORDER BY 1;
  function master_update_node(integer,text,integer,boolean,integer)
  function master_update_shard_statistics(bigint)
  function master_update_table_statistics(regclass)
+ function notify_constraint_dropped()
  function poolinfo_valid(text)
  function read_intermediate_result(text,citus_copy_format)
  function read_intermediate_results(text[],citus_copy_format)
  function rebalance_table_shards(regclass,real,integer,bigint[],citus.shard_transfer_mode,boolean,name)
  function recover_prepared_transactions()
  function relation_is_a_known_shard(regclass)
+ function remove_local_tables_from_metadata()
  function replicate_reference_tables()
  function replicate_table_shards(regclass,integer,integer,bigint[],citus.shard_transfer_mode)
  function role_exists(name)
@@ -232,5 +234,5 @@ ORDER BY 1;
  view citus_worker_stat_activity
  view pg_dist_shard_placement
  view time_partitions
-(216 rows)
+(218 rows)
 

src/test/regress/input/am_chunk_filtering.source

@@ -76,4 +76,12 @@ CREATE TABLE simple_chunk_filtering(i int) USING COLUMNAR;
 INSERT INTO simple_chunk_filtering SELECT generate_series(0,234567);
 EXPLAIN (analyze on, costs off, timing off, summary off)
   SELECT * FROM simple_chunk_filtering WHERE i > 123456;
+
+-- https://github.com/citusdata/citus/issues/4555
+TRUNCATE simple_chunk_filtering;
+INSERT INTO simple_chunk_filtering SELECT generate_series(0,200000);
+COPY (SELECT * FROM simple_chunk_filtering WHERE i > 180000) TO '/dev/null';
+EXPLAIN (analyze on, costs off, timing off, summary off)
+  SELECT * FROM simple_chunk_filtering WHERE i > 180000;
+
 DROP TABLE simple_chunk_filtering;

src/test/regress/multi_schedule

@@ -319,7 +319,8 @@ test: multi_remove_node_reference_table
 # --------
 test: add_coordinator
 test: multi_reference_table citus_local_tables_queries
-test: foreign_key_to_reference_table citus_local_table_triggers
+test: foreign_key_to_reference_table
+test: citus_local_table_triggers
 test: replicate_reference_tables_to_coordinator
 test: coordinator_shouldhaveshards
 test: local_shard_utility_command_execution
@@ -328,6 +329,7 @@ test: multi_row_router_insert mixed_relkind_tests create_ref_dist_from_citus_loc
 test: undistribute_table_cascade
 test: create_citus_local_table_cascade
 test: fkeys_between_local_ref
+test: auto_undist_citus_local
 
 test: remove_coordinator
 

src/test/regress/output/am_chunk_filtering.source

@@ -130,4 +130,18 @@ EXPLAIN (analyze on, costs off, timing off, summary off)
    Columnar Chunks Removed by Filter: 12
 (4 rows)
 
+-- https://github.com/citusdata/citus/issues/4555
+TRUNCATE simple_chunk_filtering;
+INSERT INTO simple_chunk_filtering SELECT generate_series(0,200000);
+COPY (SELECT * FROM simple_chunk_filtering WHERE i > 180000) TO '/dev/null';
+EXPLAIN (analyze on, costs off, timing off, summary off)
+  SELECT * FROM simple_chunk_filtering WHERE i > 180000;
+                                    QUERY PLAN
+---------------------------------------------------------------------
+ Custom Scan (ColumnarScan) on simple_chunk_filtering (actual rows=20000 loops=1)
+   Filter: (i > 180000)
+   Rows Removed by Filter: 1
+   Columnar Chunks Removed by Filter: 18
+(4 rows)
+
 DROP TABLE simple_chunk_filtering;

src/test/regress/sql/am_insert.sql

@@ -66,3 +66,34 @@ DROP PUBLICATION test_columnar_publication;
 -- should succeed
 INSERT INTO test_logical_replication VALUES (3);
 DROP TABLE test_logical_replication;
+
+--
+-- test toast interactions
+--
+
+-- row table with data in different storage formats
+CREATE TABLE test_toast_row(plain TEXT, main TEXT, external TEXT, extended TEXT);
+ALTER TABLE test_toast_row ALTER COLUMN plain SET STORAGE plain; -- inline, uncompressed
+ALTER TABLE test_toast_row ALTER COLUMN main SET STORAGE main; -- inline, compressed
+ALTER TABLE test_toast_row ALTER COLUMN external SET STORAGE external; -- out-of-line, uncompressed
+ALTER TABLE test_toast_row ALTER COLUMN extended SET STORAGE extended; -- out-of-line, compressed
+
+INSERT INTO test_toast_row VALUES(
+       repeat('w', 5000), repeat('x', 5000), repeat('y', 5000), repeat('z', 5000));
+
+SELECT
+  pg_column_size(plain), pg_column_size(main),
+  pg_column_size(external), pg_column_size(extended)
+FROM test_toast_row;
+
+CREATE TABLE test_toast_columnar(plain TEXT, main TEXT, external TEXT, extended TEXT)
+  USING columnar;
+INSERT INTO test_toast_columnar SELECT plain, main, external, extended
+  FROM test_toast_row;
+SELECT
+  pg_column_size(plain), pg_column_size(main),
+  pg_column_size(external), pg_column_size(extended)
+FROM test_toast_columnar;
+
+DROP TABLE test_toast_row;
+DROP TABLE test_toast_columnar;

src/test/regress/sql/auto_undist_citus_local.sql

@@ -0,0 +1,320 @@
+-- regression tests regarding foreign key
+-- drops cascading into undistributing Citus
+-- local tables to Postgres local tables
+CREATE SCHEMA drop_fkey_cascade;
+SET search_path TO drop_fkey_cascade;
+SET client_min_messages TO WARNING;
+SET citus.next_shard_id TO 1810000;
+
+SELECT 1 FROM master_add_node('localhost', :master_port, groupId => 0);
+
+-- show that DROP CONSTRAINT cascades to undistributing citus_local_table
+CREATE TABLE citus_local_table(l1 int);
+SELECT create_citus_local_table('citus_local_table');
+CREATE TABLE reference_table(r1 int primary key);
+SELECT create_reference_table('reference_table');
+ALTER TABLE citus_local_table ADD CONSTRAINT fkey_local_to_ref FOREIGN KEY(l1) REFERENCES reference_table(r1) ON DELETE CASCADE;
+
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table'::regclass) ORDER BY logicalrelid;
+
+CREATE OR REPLACE FUNCTION drop_constraint_cascade_via_perform_deletion(IN table_name regclass, IN constraint_name text)
+RETURNS VOID
+LANGUAGE C STRICT
+AS 'citus', $$drop_constraint_cascade_via_perform_deletion$$;
+
+BEGIN;
+  SELECT drop_constraint_cascade_via_perform_deletion('citus_local_table', 'fkey_local_to_ref');
+  -- we dropped constraint without going through utility hook,
+  -- so we should still see citus_local_table
+  SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table'::regclass) ORDER BY logicalrelid;
+ROLLBACK;
+
+ALTER TABLE citus_local_table DROP CONSTRAINT fkey_local_to_ref;
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table'::regclass) ORDER BY logicalrelid;
+
+DROP TABLE citus_local_table, reference_table;
+
+-- show that DROP COLUMN cascades to undistributing citus_local_table
+CREATE TABLE reference_table(r1 int primary key, r2 int);
+SELECT create_reference_table('reference_table');
+
+CREATE TABLE citus_local_table(l1 int REFERENCES reference_table(r1), l2 int);
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table'::regclass) ORDER BY logicalrelid;
+
+ALTER TABLE reference_table DROP COLUMN r1 CASCADE;
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table'::regclass) ORDER BY logicalrelid;
+DROP TABLE citus_local_table, reference_table;
+
+-- show that DROP COLUMN that cascades into drop foreign key undistributes local table
+CREATE TABLE reference_table(r1 int primary key, r2 int);
+SELECT create_reference_table('reference_table');
+
+CREATE TABLE citus_local_table(l1 int REFERENCES reference_table(r1), l2 int);
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table'::regclass) ORDER BY logicalrelid;
+ALTER TABLE citus_local_table DROP COLUMN l1 CASCADE;
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table'::regclass) ORDER BY logicalrelid;
+
+
+DROP TABLE citus_local_table, reference_table;
+
+-- show that PRIMARY KEY that cascades into drop foreign key undistributes local table
+CREATE TABLE reference_table(r1 int primary key, r2 int);
+SELECT create_reference_table('reference_table');
+
+CREATE TABLE citus_local_table(l1 int REFERENCES reference_table(r1), l2 int);
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table'::regclass) ORDER BY logicalrelid;
+ALTER TABLE reference_table DROP CONSTRAINT reference_table_pkey CASCADE;
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table'::regclass) ORDER BY logicalrelid;
+
+-- show that DROP UNIQUE INDEX that cascades into drop foreign key undistributes local table
+DROP TABLE citus_local_table, reference_table;
+
+CREATE TABLE reference_table(r1 int, r2 int);
+SELECT create_reference_table('reference_table');
+CREATE UNIQUE INDEX ref_unique ON reference_table(r1);
+
+CREATE TABLE citus_local_table(l1 int REFERENCES reference_table(r1), l2 int);
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table'::regclass) ORDER BY logicalrelid;
+DROP INDEX ref_unique CASCADE;
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table'::regclass) ORDER BY logicalrelid;
+
+-- show that UNIQUE CONSTRAINT that cascades into drop foreign key undistributes local table
+DROP TABLE citus_local_table, reference_table;
+
+CREATE TABLE reference_table(r1 int UNIQUE, r2 int);
+SELECT create_reference_table('reference_table');
+
+CREATE TABLE citus_local_table(l1 int REFERENCES reference_table(r1), l2 int);
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table'::regclass) ORDER BY logicalrelid;
+ALTER TABLE reference_table DROP CONSTRAINT reference_table_r1_key  CASCADE;
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table'::regclass) ORDER BY logicalrelid;
+
+-- show that DROP TABLE that cascades into drop foreign key undistributes local table
+DROP TABLE citus_local_table, reference_table;
+
+CREATE TABLE reference_table(r1 int UNIQUE, r2 int);
+SELECT create_reference_table('reference_table');
+CREATE TABLE citus_local_table(l1 int REFERENCES reference_table(r1), l2 int);
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table'::regclass) ORDER BY logicalrelid;
+DROP TABLE reference_table CASCADE;
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass) ORDER BY logicalrelid;
+
+
+-- show that UNIQUE CONSTRAINT that cascades into drop foreign key undistributes local table
+DROP TABLE citus_local_table;
+
+CREATE TABLE reference_table(r1 int UNIQUE, r2 int);
+SELECT create_reference_table('reference_table');
+
+CREATE TABLE citus_local_table(l1 int REFERENCES reference_table(r1), l2 int);
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table'::regclass) ORDER BY logicalrelid;
+ALTER TABLE reference_table DROP CONSTRAINT reference_table_r1_key  CASCADE;
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table'::regclass) ORDER BY logicalrelid;
+
+
+-- show that DROP SCHEMA that cascades into drop foreign key undistributes local table
+DROP TABLE citus_local_table, reference_table;
+
+CREATE SCHEMA ref_table_drop_schema;
+CREATE TABLE ref_table_drop_schema.reference_table(r1 int UNIQUE, r2 int);
+SELECT create_reference_table('ref_table_drop_schema.reference_table');
+CREATE TABLE citus_local_table(l1 int REFERENCES ref_table_drop_schema.reference_table(r1), l2 int);
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'ref_table_drop_schema.reference_table'::regclass) ORDER BY logicalrelid;
+DROP SCHEMA ref_table_drop_schema CASCADE;
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass) ORDER BY logicalrelid;
+
+
+-- drop column cascade that doesn't cascade into citus local table
+DROP TABLE IF EXISTS citus_local_table, reference_table_1, reference_table_2;
+
+CREATE TABLE reference_table_1(r1 int UNIQUE, r2 int);
+SELECT create_reference_table('reference_table_1');
+CREATE TABLE reference_table_2(r1 int UNIQUE REFERENCES reference_table_1(r1), r2 int);
+SELECT create_reference_table('reference_table_2');
+CREATE TABLE citus_local_table(l1 int REFERENCES reference_table_2(r1), l2 int);
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table_1'::regclass, 'reference_table_2'::regclass) ORDER BY logicalrelid;
+ALTER TABLE reference_table_1 DROP COLUMN r1 CASCADE;
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table_1'::regclass, 'reference_table_2'::regclass) ORDER BY logicalrelid;
+
+
+-- local table has multiple foreign keys to two tables
+-- drop one at a time
+DROP TABLE IF EXISTS citus_local_table, reference_table_1, reference_table_2;
+CREATE TABLE reference_table_1(r1 int UNIQUE, r2 int);
+SELECT create_reference_table('reference_table_1');
+CREATE TABLE reference_table_2(r1 int UNIQUE, r2 int);
+SELECT create_reference_table('reference_table_2');
+CREATE TABLE citus_local_table(l1 int REFERENCES reference_table_1(r1), l2 int REFERENCES reference_table_2(r1));
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table_1'::regclass, 'reference_table_2'::regclass) ORDER BY logicalrelid;
+
+DROP TABLE reference_table_1 CASCADE;
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table_2'::regclass) ORDER BY logicalrelid;
+
+CREATE TABLE distributed_table (d1 int);
+SELECT create_distributed_table('distributed_table', 'd1');
+
+-- drop an unrelated distributed table too
+DROP TABLE reference_table_2, distributed_table CASCADE;
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass) ORDER BY logicalrelid;
+
+-- local table has multiple foreign keys to two tables
+-- drop both at the same time
+DROP TABLE IF EXISTS citus_local_table, reference_table_1, reference_table_2;
+CREATE TABLE reference_table_1(r1 int UNIQUE, r2 int);
+SELECT create_reference_table('reference_table_1');
+CREATE TABLE reference_table_2(r1 int UNIQUE, r2 int);
+SELECT create_reference_table('reference_table_2');
+CREATE TABLE citus_local_table(l1 int REFERENCES reference_table_1(r1), l2 int REFERENCES reference_table_2(r1));
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table_1'::regclass, 'reference_table_2'::regclass) ORDER BY logicalrelid;
+DROP TABLE reference_table_1, reference_table_2 CASCADE;
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass) ORDER BY logicalrelid;
+
+-- local table has multiple foreign keys to two tables
+-- drop one at a time
+DROP TABLE IF EXISTS citus_local_table, reference_table_1, reference_table_2;
+CREATE TABLE reference_table_1(r1 int UNIQUE, r2 int);
+SELECT create_reference_table('reference_table_1');
+CREATE TABLE reference_table_2(r1 int UNIQUE, r2 int);
+SELECT create_reference_table('reference_table_2');
+CREATE TABLE citus_local_table(l1 int REFERENCES reference_table_1(r1), l2 int REFERENCES reference_table_2(r1));
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table_1'::regclass, 'reference_table_2'::regclass) ORDER BY logicalrelid;
+
+BEGIN;
+  ALTER TABLE citus_local_table DROP CONSTRAINT citus_local_table_l1_fkey;
+  SAVEPOINT sp1;
+
+  -- this should undistribute citus_local_table
+  ALTER TABLE citus_local_table DROP CONSTRAINT citus_local_table_l2_fkey;
+  SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table_1'::regclass, 'reference_table_2'::regclass) ORDER BY logicalrelid;
+  ROLLBACK TO SAVEPOINT sp1;
+
+  -- rollback'ed second drop constraint, so we should still see citus_local_table
+  SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table_1'::regclass, 'reference_table_2'::regclass) ORDER BY logicalrelid;
+
+  -- this should undistribute citus_local_table again
+  ALTER TABLE citus_local_table DROP CONSTRAINT citus_local_table_l2_fkey;
+  SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('citus_local_table'::regclass, 'reference_table_1'::regclass, 'reference_table_2'::regclass) ORDER BY logicalrelid;
+COMMIT;
+
+-- a single drop column cascades into multiple undistributes
+DROP TABLE IF EXISTS citus_local_table_1, citus_local_table_2, reference_table_1;
+
+CREATE TABLE reference_table_1(r1 int UNIQUE, r2 int);
+SELECT create_reference_table('reference_table_1');
+
+CREATE TABLE citus_local_table_1(l1 int REFERENCES reference_table_1(r1), l2 int UNIQUE);
+CREATE TABLE citus_local_table_2(l1 int REFERENCES reference_table_1(r1), l2 int UNIQUE REFERENCES citus_local_table_1(l2));
+CREATE TABLE citus_local_table_3(l1 int REFERENCES reference_table_1(r1), l2 int REFERENCES citus_local_table_2(l2));
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('reference_table_1'::regclass, 'citus_local_table_1'::regclass, 'citus_local_table_2'::regclass, 'citus_local_table_3'::regclass) ORDER BY logicalrelid;
+ALTER TABLE reference_table_1 DROP COLUMN r1 CASCADE;
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('reference_table_1'::regclass, 'citus_local_table_1'::regclass, 'citus_local_table_2'::regclass, 'citus_local_table_3'::regclass) ORDER BY logicalrelid;
+
+-- a single drop table cascades into multiple undistributes
+DROP TABLE IF EXISTS citus_local_table_1, citus_local_table_2, citus_local_table_3, citus_local_table_2, reference_table_1;
+CREATE TABLE reference_table_1(r1 int UNIQUE, r2 int);
+SELECT create_reference_table('reference_table_1');
+CREATE TABLE citus_local_table_1(l1 int REFERENCES reference_table_1(r1), l2 int UNIQUE);
+CREATE TABLE citus_local_table_2(l1 int REFERENCES reference_table_1(r1), l2 int UNIQUE REFERENCES citus_local_table_1(l2));
+CREATE TABLE citus_local_table_3(l1 int REFERENCES reference_table_1(r1), l2 int REFERENCES citus_local_table_2(l2));
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('reference_table_1'::regclass, 'citus_local_table_1'::regclass, 'citus_local_table_2'::regclass, 'citus_local_table_3'::regclass) ORDER BY logicalrelid;
+
+-- test DROP OWNED BY
+
+-- Citus does not support "ALTER TABLE OWNER TO" commands. Also, not to deal with tests output
+-- difference between community and enterprise, let's disable enable_ddl_propagation here.
+SET citus.enable_ddl_propagation to OFF;
+
+CREATE USER another_user;
+SELECT run_command_on_workers('CREATE USER another_user');
+
+ALTER TABLE reference_table_1 OWNER TO another_user;
+SELECT run_command_on_placements('reference_table_1', 'ALTER TABLE %s OWNER TO another_user');
+
+SET citus.enable_ddl_propagation to ON;
+
+BEGIN;
+  DROP OWNED BY another_user cascade;
+  SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ( 'citus_local_table_1'::regclass, 'citus_local_table_2'::regclass, 'citus_local_table_3'::regclass) ORDER BY logicalrelid;
+ROLLBACK;
+
+DROP TABLE reference_table_1 CASCADE;
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ( 'citus_local_table_1'::regclass, 'citus_local_table_2'::regclass, 'citus_local_table_3'::regclass) ORDER BY logicalrelid;
+
+
+-- dropping constraints inside a plpgsql procedure should be fine
+DROP TABLE IF EXISTS citus_local_table_1, reference_table_1 CASCADE;
+CREATE TABLE reference_table_1(r1 int UNIQUE, r2 int);
+SELECT create_reference_table('reference_table_1');
+CREATE TABLE citus_local_table_1(l1 int REFERENCES reference_table_1(r1), l2 int UNIQUE);
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('reference_table_1'::regclass, 'citus_local_table_1'::regclass) ORDER BY logicalrelid;
+
+
+CREATE OR REPLACE FUNCTION drop_constraint_via_func()
+RETURNS void LANGUAGE plpgsql AS $$
+BEGIN
+ALTER TABLE citus_local_table_1 DROP CONSTRAINT citus_local_table_1_l1_fkey;
+END;$$;
+
+BEGIN;
+  SELECT drop_constraint_via_func();
+  SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('reference_table_1'::regclass, 'citus_local_table_1'::regclass) ORDER BY logicalrelid;
+ROLLBACK;
+
+create or replace procedure drop_constraint_via_proc()
+language plpgsql
+as $$
+DECLARE
+    res INT := 0;
+begin
+        ALTER TABLE citus_local_table_1 DROP CONSTRAINT citus_local_table_1_l1_fkey;
+    commit;
+end;$$;
+call drop_constraint_via_proc();
+
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('reference_table_1'::regclass, 'citus_local_table_1'::regclass) ORDER BY logicalrelid;
+
+-- even if the procedure is called from another procedure
+DROP TABLE IF EXISTS citus_local_table_1, reference_table_1 CASCADE;
+CREATE TABLE reference_table_1(r1 int UNIQUE, r2 int);
+SELECT create_reference_table('reference_table_1');
+CREATE TABLE citus_local_table_1(l1 int REFERENCES reference_table_1(r1), l2 int UNIQUE);
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('reference_table_1'::regclass, 'citus_local_table_1'::regclass) ORDER BY logicalrelid;
+
+create or replace procedure drop_constraint_via_proc_top_level()
+language plpgsql
+as $$
+DECLARE
+    res INT := 0;
+begin
+        CALL drop_constraint_via_proc();
+    commit;
+end;$$;
+
+CALL drop_constraint_via_proc_top_level();
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('reference_table_1'::regclass, 'citus_local_table_1'::regclass) ORDER BY logicalrelid;
+
+
+-- even if the procedure is called from an exception handler
+DROP TABLE IF EXISTS citus_local_table_1, reference_table_1 CASCADE;
+CREATE TABLE reference_table_1(r1 int UNIQUE, r2 int);
+SELECT create_reference_table('reference_table_1');
+CREATE TABLE citus_local_table_1(l1 int REFERENCES reference_table_1(r1), l2 int UNIQUE);
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('reference_table_1'::regclass, 'citus_local_table_1'::regclass) ORDER BY logicalrelid;
+
+create or replace procedure drop_constraint_via_proc_exception()
+language plpgsql
+as $$
+DECLARE
+    res INT := 0;
+begin
+		PERFORM 1/0;
+    	EXCEPTION
+             when others then
+        CALL drop_constraint_via_proc();
+    commit;
+end;$$;
+
+CALL drop_constraint_via_proc_exception();
+SELECT logicalrelid, partmethod, repmodel FROM pg_dist_partition WHERE logicalrelid IN ('reference_table_1'::regclass, 'citus_local_table_1'::regclass) ORDER BY logicalrelid;
+
+DROP SCHEMA drop_fkey_cascade CASCADE;

src/test/regress/sql/citus_local_tables.sql

@@ -201,6 +201,8 @@ CREATE FOREIGN TABLE foreign_table (
 -- & shell relation points to the same same server object
 SELECT create_citus_local_table('foreign_table');
 
+DROP FOREIGN TABLE foreign_table;
+
 -- drop them for next tests
 DROP TABLE citus_local_table_1, citus_local_table_2, distributed_table;
 
@@ -314,6 +316,41 @@ CREATE UNIQUE INDEX uniqueIndex2 ON "LocalTabLE.1!?!"(id);
 
 SET search_path TO citus_local_tables_test_schema;
 
+CREATE TABLE dummy_reference_table (a INT PRIMARY KEY);
+SELECT create_reference_table('dummy_reference_table');
+
+BEGIN;
+  SET client_min_messages TO ERROR;
+  SELECT remove_local_tables_from_metadata();
+
+  -- should not see any citus local tables
+  SELECT logicalrelid::regclass::text FROM pg_dist_partition, pg_tables
+  WHERE tablename=logicalrelid::regclass::text AND
+        schemaname='citus_local_tables_test_schema' AND
+        partmethod = 'n' AND repmodel = 'c'
+  ORDER BY 1;
+ROLLBACK;
+
+-- define foreign keys between dummy_reference_table and citus local tables
+-- not to undistribute them automatically
+ALTER TABLE citus_local_table_1 ADD CONSTRAINT fkey_to_dummy_ref FOREIGN KEY (a) REFERENCES dummy_reference_table(a);
+ALTER TABLE citus_local_table_2 ADD CONSTRAINT fkey_to_dummy_ref FOREIGN KEY (a) REFERENCES dummy_reference_table(a);
+ALTER TABLE unlogged_table ADD CONSTRAINT fkey_to_dummy_ref FOREIGN KEY (a) REFERENCES dummy_reference_table(a);
+ALTER TABLE local_table_3 ADD CONSTRAINT fkey_to_dummy_ref FOREIGN KEY (a) REFERENCES dummy_reference_table(a);
+ALTER TABLE dummy_reference_table ADD CONSTRAINT fkey_from_dummy_ref FOREIGN KEY (a) REFERENCES "CiTUS!LocalTables"."LocalTabLE.1!?!"(id);
+
+BEGIN;
+  SET client_min_messages TO ERROR;
+  SELECT remove_local_tables_from_metadata();
+
+  -- now we defined foreign keys with above citus local tables, we should still see them
+  SELECT logicalrelid::regclass::text FROM pg_dist_partition, pg_tables
+  WHERE tablename=logicalrelid::regclass::text AND
+        schemaname='citus_local_tables_test_schema' AND
+        partmethod = 'n' AND repmodel = 'c'
+  ORDER BY 1;
+ROLLBACK;
+
 -- between citus local tables and distributed tables
 ALTER TABLE citus_local_table_1 ADD CONSTRAINT fkey_c_to_dist FOREIGN KEY(a) references distributed_table(a);
 ALTER TABLE distributed_table ADD CONSTRAINT fkey_dist_to_c FOREIGN KEY(a) references citus_local_table_1(a);
@@ -450,7 +487,7 @@ ALTER TABLE referencing_table ADD CONSTRAINT fkey_cl_to_cl FOREIGN KEY (a) REFER
 -- verify creating citus local table with extended statistics
 CREATE TABLE test_citus_local_table_with_stats(a int, b int);
 CREATE STATISTICS stx1 ON a, b FROM test_citus_local_table_with_stats;
-SELECT create_citus_local_table('test_citus_local_table_with_stats');
+ALTER TABLE test_citus_local_table_with_stats ADD CONSTRAINT fkey_to_dummy_ref FOREIGN KEY (a) REFERENCES dummy_reference_table(a);
 CREATE STATISTICS "CiTUS!LocalTables"."Bad\'StatName" ON a, b FROM test_citus_local_table_with_stats;
 SELECT stxname FROM pg_statistic_ext ORDER BY stxname;
 

src/test/regress/sql/fkeys_between_local_ref.sql

@@ -95,6 +95,10 @@ BEGIN;
 
   -- show that we converted all 4 local tables in this schema to citus local tables
   SELECT COUNT(*)=4 FROM citus_local_tables_in_schema;
+
+  -- dropping that column would undistribute those 4 citus local tables
+  ALTER TABLE local_table_1 DROP COLUMN col_1 CASCADE;
+  SELECT COUNT(*)=0 FROM citus_local_tables_in_schema;
 ROLLBACK;
 
 -- this actually attempts to convert local tables to citus local tables but errors out
@@ -111,6 +115,15 @@ BEGIN;
 
   -- now we have 5 citus local tables in this schema
   SELECT COUNT(*)=5 FROM citus_local_tables_in_schema;
+
+  -- dropping foreign key from local_table_2 would only undistribute local_table_2 & local_table_5
+  ALTER TABLE local_table_2 DROP CONSTRAINT fkey_1;
+  SELECT logicalrelid::regclass::text FROM citus_local_tables_in_schema ORDER BY logicalrelid;
+
+    -- dropping local_table_1 would undistribute last two citus local tables as local_table_1
+    -- was the bridge to reference table
+  DROP TABLE local_table_1 CASCADE;
+  SELECT COUNT(*)=0 FROM citus_local_tables_in_schema;
 ROLLBACK;
 
 -- they fail as local_table_99 does not exist
@@ -252,6 +265,14 @@ BEGIN;
   WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref' UNION
                                SELECT 'another_schema_fkeys_between_local_ref.local_table_6')
   ORDER BY tablename;
+
+  DROP TABLE local_table_3 CASCADE;
+  DROP SCHEMA another_schema_fkeys_between_local_ref CASCADE;
+
+  -- now we shouldn't see local_table_5 since now it is not connected to any reference tables
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
 ROLLBACK;
 
 BEGIN;
@@ -283,6 +304,21 @@ BEGIN;
   SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
   WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
   ORDER BY tablename;
+
+  CREATE SCHEMA another_schema_fkeys_between_local_ref;
+  CREATE TABLE another_schema_fkeys_between_local_ref.reference_table_3 (col_1 INT UNIQUE);
+  SELECT create_reference_table('another_schema_fkeys_between_local_ref.reference_table_3');
+  TRUNCATE local_table_4 CASCADE;
+  ALTER TABLE local_table_4 ADD CONSTRAINT fkey_12 FOREIGN KEY (col_1) REFERENCES another_schema_fkeys_between_local_ref.reference_table_3(col_1);
+
+  DROP TABLE local_table_5 CASCADE;
+  ALTER TABLE local_table_2 DROP CONSTRAINT fkey_1;
+  DROP SCHEMA another_schema_fkeys_between_local_ref CASCADE;
+
+  -- now we shouldn't see any citus local tables
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
 ROLLBACK;
 
 BEGIN;
@@ -305,6 +341,70 @@ BEGIN;
   ORDER BY tablename;
 ROLLBACK;
 
+-- converting any local table to a citus local table in graph converts
+-- other tables to citus local tables, test this in below xact blocks
+
+BEGIN;
+  SELECT create_reference_table('local_table_1');
+
+  SELECT create_distributed_table('local_table_2', 'col_1');
+
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ROLLBACK;
+
+BEGIN;
+  SELECT create_reference_table('local_table_4');
+
+  SELECT create_reference_table('local_table_3');
+
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ROLLBACK;
+
+BEGIN;
+  CREATE TABLE local_table_5 (col_1 INT REFERENCES local_table_1(col_1));
+
+  SELECT create_reference_table('local_table_1');
+
+  SELECT create_distributed_table('local_table_2', 'col_1');
+  SELECT create_distributed_table('local_table_5', 'col_1');
+
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ROLLBACK;
+
+BEGIN;
+  ALTER TABLE local_table_1 ADD CONSTRAINT fkey_13 FOREIGN KEY (col_1) REFERENCES local_table_2(col_1) ON DELETE CASCADE;
+
+  -- errors out as foreign keys from reference tables to citus local tables
+  -- cannot have CASCADE behavior
+  SELECT create_reference_table('local_table_1');
+ROLLBACK;
+
+SET citus.enable_local_execution TO OFF;
+-- show that this errors out as it tries to convert connected relations to citus
+-- local tables and creating citus local table requires local execution but local
+-- execution is disabled
+SELECT create_reference_table('local_table_1');
+SET citus.enable_local_execution TO ON;
+
+-- test behavior when outside of the xact block
+
+CREATE TABLE local_table_6 (col_1 INT REFERENCES local_table_1(col_1));
+
+SELECT create_reference_table('local_table_1');
+
+SELECT create_distributed_table('local_table_2', 'col_1');
+SELECT create_distributed_table('local_table_6', 'col_1');
+
+SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+ORDER BY tablename;
+
 -- this errors out as we don't support creating citus local
 -- tables from partitioned tables
 CREATE TABLE part_local_table (col_1 INT REFERENCES reference_table_1(col_1)) PARTITION BY RANGE (col_1);
@@ -316,5 +416,98 @@ CREATE TABLE local_table_5 (col_1 INT, FOREIGN KEY (col_1) REFERENCES reference_
 -- fails as referenced table does not exist
 CREATE TABLE local_table_5 (col_1 INT, FOREIGN KEY (col_1) REFERENCES table_does_not_exist(dummy));
 
+-- drop & recreate schema to prevent noise in next test outputs
+DROP SCHEMA fkeys_between_local_ref CASCADE;
+CREATE SCHEMA fkeys_between_local_ref;
+SET search_path TO fkeys_between_local_ref;
+
+-- now have some tests to test behavior before/after enabling foreign keys
+-- between local tables & reference tables
+
+BEGIN;
+  SET citus.enable_local_reference_table_foreign_keys TO OFF;
+
+  CREATE TABLE ref_1(a int PRIMARY KEY);
+  CREATE TABLE pg_local_1(a int PRIMARY KEY REFERENCES ref_1(a));
+  SELECT create_reference_table('ref_1');
+
+  SET citus.enable_local_reference_table_foreign_keys TO ON;
+
+  CREATE TABLE ref_2(a int PRIMARY KEY);
+  SELECT create_reference_table('ref_2');
+  ALTER TABLE pg_local_1 ADD CONSTRAINT c1 FOREIGN KEY(a) REFERENCES ref_2(a);
+
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ROLLBACK;
+
+BEGIN;
+  SET citus.enable_local_reference_table_foreign_keys TO OFF;
+
+  CREATE TABLE ref_1(a int PRIMARY KEY);
+  CREATE TABLE pg_local_1(a int PRIMARY KEY REFERENCES ref_1(a));
+  SELECT create_reference_table('ref_1');
+
+  SET citus.enable_local_reference_table_foreign_keys TO ON;
+
+  CREATE TABLE ref_2(a int PRIMARY KEY REFERENCES pg_local_1(a));
+  SELECT create_reference_table('ref_2');
+
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ROLLBACK;
+
+BEGIN;
+  SET citus.enable_local_reference_table_foreign_keys TO OFF;
+
+  CREATE TABLE ref_1(a int PRIMARY KEY);
+  CREATE TABLE pg_local_1(a int PRIMARY KEY REFERENCES ref_1(a));
+  SELECT create_reference_table('ref_1');
+
+  SET citus.enable_local_reference_table_foreign_keys TO ON;
+
+  CREATE TABLE ref_2(a int PRIMARY KEY REFERENCES pg_local_1(a));
+  SELECT create_reference_table('ref_2');
+
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ROLLBACK;
+
+BEGIN;
+  SET citus.enable_local_reference_table_foreign_keys TO OFF;
+
+  CREATE TABLE ref_1(a int PRIMARY KEY);
+  CREATE TABLE pg_local_1(a int PRIMARY KEY REFERENCES ref_1(a));
+  SELECT create_reference_table('ref_1');
+
+  SET citus.enable_local_reference_table_foreign_keys TO ON;
+
+  CREATE TABLE pg_local_2(a int PRIMARY KEY REFERENCES pg_local_1(a));
+
+  -- we still didn't convert local tables to citus local tables
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+
+  CREATE TABLE pg_local_3(a int PRIMARY KEY REFERENCES ref_1(a));
+
+  -- pg_local_3 is not connected to other local tables, so we will just
+  -- convert pg_local_3 to a citus local table
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+
+  CREATE TABLE pg_local_4(a int PRIMARY KEY REFERENCES ref_1(a), FOREIGN KEY (a) REFERENCES pg_local_2(a));
+
+  -- pg_local_4 is connected to ref_1, pg_local_1 and pg_local_2,
+  -- so we will convert those two local tables to citus local tables too
+  SELECT logicalrelid::text AS tablename, partmethod, repmodel FROM pg_dist_partition
+  WHERE logicalrelid::text IN (SELECT tablename FROM pg_tables WHERE schemaname='fkeys_between_local_ref')
+  ORDER BY tablename;
+ROLLBACK;
+
 -- cleanup at exit
 DROP SCHEMA fkeys_between_local_ref CASCADE;

src/test/regress/sql/foreign_key_to_reference_table.sql

@@ -251,10 +251,8 @@ INSERT INTO referencing_schema.referencing_table SELECT x, x from generate_serie
 DELETE FROM referenced_schema.referenced_table WHERE id > 800;
 SELECT count(*) FROM referencing_schema.referencing_table;
 
-SET client_min_messages TO ERROR;
 DROP SCHEMA referenced_schema CASCADE;
 DROP SCHEMA referencing_schema CASCADE;
-RESET client_min_messages;
 
 -- on delete set update cascades properly
 CREATE TABLE referenced_table(test_column int, test_column2 int, PRIMARY KEY(test_column));
@@ -393,11 +391,6 @@ CREATE TABLE referencing_table(id int, ref_id int DEFAULT -1, FOREIGN KEY (ref_i
 INSERT INTO referenced_table VALUES (1,1), (2,2), (3,3);
 INSERT INTO referencing_table VALUES (1,1), (2,2), (3,3);
 SELECT create_reference_table('referenced_table');
-SELECT create_distributed_table('referencing_table', 'id');
-
-BEGIN;
-  SELECT create_distributed_table('referencing_table', 'id');
-COMMIT;
 
 DROP TABLE referenced_table CASCADE;
 DROP TABLE referencing_table CASCADE;
@@ -565,21 +558,12 @@ DROP TABLE referencing_table2 CASCADE;
 
 -- Check if the above fkeys are created with create_distributed_table
 CREATE TABLE referenced_table(test_column int, test_column2 int UNIQUE, PRIMARY KEY(test_column));
-CREATE TABLE referencing_table(id int PRIMARY KEY, ref_id int, FOREIGN KEY (id) REFERENCES referenced_table(test_column) ON DELETE CASCADE);
-CREATE TABLE referencing_table2(id int, ref_id int, FOREIGN KEY (ref_id) REFERENCES referenced_table(test_column2) ON DELETE CASCADE, FOREIGN KEY (id) REFERENCES referencing_table(id) ON DELETE CASCADE);
 SELECT create_reference_table('referenced_table');
-BEGIN;
-  SET LOCAL citus.multi_shard_modify_mode TO 'sequential';
-  SELECT create_distributed_table('referencing_table', 'id');
-  SELECT create_distributed_table('referencing_table2', 'id');
-COMMIT;
 
 SELECT count(*) FROM table_fkeys_in_workers WHERE relid LIKE 'fkey_reference_table.%' AND refd_relid LIKE 'fkey_reference_table.%';
 
 \set VERBOSITY terse
 DROP TABLE referenced_table CASCADE;
-DROP TABLE referencing_table CASCADE;
-DROP TABLE referencing_table2 CASCADE;
 \set VERBOSITY default
 
 -- In this test we have a chained relationship in form of
@@ -608,23 +592,19 @@ DROP TABLE referenced_table CASCADE;
 DROP TABLE referencing_table CASCADE;
 DROP TABLE referencing_referencing_table;
 
--- create_reference_table, create_distributed_table and ALTER TABLE in the same transaction
 BEGIN;
   CREATE TABLE test_table_1(id int PRIMARY KEY);
   SELECT create_reference_table('test_table_1');
 
   CREATE TABLE test_table_2(id int PRIMARY KEY, value_1 int);
-  SELECT create_distributed_table('test_table_2', 'id');
 
   ALTER TABLE test_table_2 ADD CONSTRAINT c_check FOREIGN KEY (value_1) REFERENCES test_table_1(id);
 
   DROP TABLE test_table_1, test_table_2;
 COMMIT;
 
--- the order of create_reference_table and create_distributed_table is changed
 BEGIN;
   CREATE TABLE test_table_1(id int PRIMARY KEY, value_1 int);
-  SELECT create_distributed_table('test_table_1', 'id');
 
   CREATE TABLE test_table_2(id int PRIMARY KEY);
   SELECT create_reference_table('test_table_2');
@@ -634,7 +614,6 @@ BEGIN;
   DROP TABLE test_table_2 CASCADE;
 ROLLBACK;
 
--- make sure that we fail if we need parallel data load
 BEGIN;
 
   CREATE TABLE test_table_1(id int PRIMARY KEY);
@@ -644,7 +623,6 @@ BEGIN;
   INSERT INTO test_table_2 SELECT i, i FROM generate_series(0,100) i;
 
   SELECT create_reference_table('test_table_1');
-  SELECT create_distributed_table('test_table_2', 'id');
   DROP TABLE test_table_2, test_table_1;
 COMMIT;
 
@@ -940,7 +918,6 @@ ROLLBACK;
 DROP TABLE referenced_table CASCADE;
 DROP TABLE referencing_table;
 
-SET client_min_messages TO ERROR;
 DROP SCHEMA fkey_reference_table CASCADE;
 SET search_path TO DEFAULT;
 RESET client_min_messages;

src/test/regress/sql/ref_citus_local_fkeys.sql

@@ -120,6 +120,8 @@ ALTER TABLE reference_table DROP CONSTRAINT fkey_ref_to_local;
 ALTER TABLE reference_table ADD CONSTRAINT fkey_ref_to_local FOREIGN KEY(r1) REFERENCES citus_local_table(l1) ON DELETE NO ACTION;
 SET citus.enable_local_execution TO ON;
 
+ALTER TABLE reference_table ADD CONSTRAINT fkey_ref_to_local FOREIGN KEY(r1) REFERENCES citus_local_table(l1) ON DELETE NO ACTION;
+
 -- show that we are checking for foreign key constraint after defining, this should fail
 INSERT INTO reference_table VALUES (4);