Bump Citus version to 11.1.6

(cherry picked from commit c36adc8426)

.circleci/config.yml

@@ -6,19 +6,19 @@ orbs:
 parameters:
   image_suffix:
     type: string
-    default: '-v0eef34d'
+    default: '-vc4b1573'
   pg13_version:
     type: string
-    default: '13.8'
+    default: '13.10'
   pg14_version:
     type: string
-    default: '14.5'
+    default: '14.7'
   pg15_version:
     type: string
-    default: '15beta4'
+    default: '15.2'
   upgrade_pg_versions:
     type: string
-    default: '13.8-14.5-15beta4'
+    default: '13.10-14.7-15.2'
   style_checker_tools_version:
     type: string
     default: '0.8.18'

.github/packaging/packaging_ignore.yml

@@ -0,0 +1,3 @@
+base:
+  - ".* warning: ignoring old recipe for target [`']check'"
+  - ".* warning: overriding recipe for target [`']check'"

.github/packaging/validate_build_output.sh

@@ -0,0 +1,20 @@
+package_type=${1}
+
+# Since $HOME is set in GH_Actions as /github/home, pyenv fails to create virtualenvs.
+# For this script, we set $HOME to /root and then set it back to /github/home.
+GITHUB_HOME="${HOME}"
+export HOME="/root"
+
+eval "$(pyenv init -)"
+pyenv versions
+pyenv virtualenv ${PACKAGING_PYTHON_VERSION} packaging_env
+pyenv activate packaging_env
+
+git clone -b v0.8.25 --depth=1  https://github.com/citusdata/tools.git tools
+python3 -m pip install -r tools/packaging_automation/requirements.txt
+python3 -m tools.packaging_automation.validate_build_output --output_file output.log \
+                                                            --ignore_file .github/packaging/packaging_ignore.yml \
+                                                            --package_type ${package_type}
+pyenv deactivate
+# Set $HOME back to /github/home
+export HOME=${GITHUB_HOME}

.github/workflows/packaging-test-pipelines.yml

@@ -0,0 +1,161 @@
+name: Build tests in packaging images
+
+on:
+  push:
+    branches: "**"
+
+  workflow_dispatch:
+
+jobs:
+
+  get_postgres_versions_from_file:
+    runs-on: ubuntu-latest
+    outputs:
+      pg_versions: ${{ steps.get-postgres-versions.outputs.pg_versions }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+      - name: Get Postgres Versions
+        id: get-postgres-versions
+        run: |
+          # Postgres versions are stored in .circleci/config.yml file in "build-[pg-version] format. Below command
+          # extracts the versions and get the unique values.
+          pg_versions=`grep -Eo 'build-[[:digit:]]{2}' .circleci/config.yml|sed -e "s/^build-//"|sort|uniq|tr '\n' ','| head -c -1`
+          pg_versions_array="[ ${pg_versions} ]"
+          echo "Supported PG Versions: ${pg_versions_array}"
+          # Below line is needed to set the output variable to be used in the next job
+          echo "pg_versions=${pg_versions_array}" >> $GITHUB_OUTPUT
+
+  rpm_build_tests:
+    name: rpm_build_tests
+    needs: get_postgres_versions_from_file
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # While we use separate images for different Postgres versions in rpm
+        # based distros
+        # For this reason, we need to use a "matrix" to generate names of
+        # rpm images, e.g. citus/packaging:centos-7-pg12
+        packaging_docker_image:
+          - oraclelinux-7
+          - oraclelinux-8
+          - centos-7
+          - centos-8
+          - almalinux-9
+        POSTGRES_VERSION: ${{ fromJson(needs.get_postgres_versions_from_file.outputs.pg_versions) }}
+
+    container:
+      image: citus/packaging:${{ matrix.packaging_docker_image }}-pg${{ matrix.POSTGRES_VERSION }}
+      options: --user root
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set Postgres and python parameters for rpm based distros
+        run: |
+          echo "/usr/pgsql-${{ matrix.POSTGRES_VERSION }}/bin" >> $GITHUB_PATH
+          echo "/root/.pyenv/bin:$PATH" >> $GITHUB_PATH
+          echo "PACKAGING_PYTHON_VERSION=3.8.16" >> $GITHUB_ENV
+
+      - name: Configure
+        run: |
+          echo "Current Shell:$0"
+          echo "GCC Version: $(gcc --version)"
+          ./configure 2>&1 | tee output.log
+
+      - name: Make clean
+        run: |
+          make clean
+
+      - name: Make
+        run: |
+          make CFLAGS="-Wno-missing-braces" -sj$(cat /proc/cpuinfo | grep "core id" | wc -l) 2>&1 | tee -a output.log
+
+      - name: Make install
+        run: |
+          make CFLAGS="-Wno-missing-braces" install 2>&1 | tee -a output.log
+
+      - name: Validate output
+        env:
+          POSTGRES_VERSION: ${{ matrix.POSTGRES_VERSION }}
+          PACKAGING_DOCKER_IMAGE: ${{ matrix.packaging_docker_image }}
+        run: |
+          echo "Postgres version: ${POSTGRES_VERSION}"
+
+          ## Install required packages to execute packaging tools for rpm based distros
+          yum install python3-pip python3-devel postgresql-devel -y
+          python3 -m pip install wheel
+
+          ./.github/packaging/validate_build_output.sh "rpm"
+
+  deb_build_tests:
+    name: deb_build_tests
+    needs: get_postgres_versions_from_file
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # On deb based distros, we use the same docker image for
+        # builds based on different Postgres versions because deb
+        # based images include all postgres installations.
+        # For this reason, we have multiple runs --which is 3 today--
+        # for each deb based image and we use POSTGRES_VERSION to set
+        # PG_CONFIG variable in each of those runs.
+        packaging_docker_image:
+          - debian-buster-all
+          - debian-bookworm-all
+          - debian-bullseye-all
+          - ubuntu-bionic-all
+          - ubuntu-focal-all
+          - ubuntu-jammy-all
+          - ubuntu-kinetic-all
+
+        POSTGRES_VERSION: ${{ fromJson(needs.get_postgres_versions_from_file.outputs.pg_versions) }}
+
+    container:
+      image: citus/packaging:${{ matrix.packaging_docker_image }}
+      options: --user root
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set pg_config path and python parameters for deb based distros
+        run: |
+          echo "PG_CONFIG=/usr/lib/postgresql/${{ matrix.POSTGRES_VERSION }}/bin/pg_config" >> $GITHUB_ENV
+          echo "/root/.pyenv/bin:$PATH" >> $GITHUB_PATH
+          echo "PACKAGING_PYTHON_VERSION=3.8.16" >> $GITHUB_ENV
+
+      - name: Configure
+        run: |
+          echo "Current Shell:$0"
+          echo "GCC Version: $(gcc --version)"
+          ./configure 2>&1 | tee output.log
+
+      - name: Make clean
+        run: |
+          make clean
+
+      - name: Make
+        run: |
+          make -sj$(cat /proc/cpuinfo | grep "core id" | wc -l) 2>&1 | tee -a output.log
+
+      - name: Make install
+        run: |
+          make install 2>&1 | tee -a output.log
+
+      - name: Validate output
+        env:
+          POSTGRES_VERSION: ${{ matrix.POSTGRES_VERSION }}
+          PACKAGING_DOCKER_IMAGE: ${{ matrix.packaging_docker_image }}
+        run: |
+          echo "Postgres version: ${POSTGRES_VERSION}"
+
+          sudo apt-get update -y
+          ## Install required packages to execute packaging tools for deb based distros
+          sudo apt-get purge -y python3-yaml
+          ./.github/packaging/validate_build_output.sh "deb"

CHANGELOG.md

@@ -1,3 +1,212 @@
+### citus v11.1.6 (April 20, 2023) ###
+
+* Correctly reports shard size in `citus_shards` view (#6748)
+
+* Fixes a bug in shard copy operations (#6721)
+
+* Fixes a bug that breaks pg upgrades if the user has a columnar table (#6624)
+
+* Fixes a bug that causes background rebalancer to fail when a reference table
+  doesn't have a primary key (#6682)
+
+* Fixes a regression in allowed foreign keys on distributed tables (#6550)
+
+* Fixes a use-after-free bug in connection management (#6685)
+
+* Fixes an unexpected foreign table error by disallowing to drop the
+  `table_name` option (#6669)
+
+* Fixes an uninitialized memory access in shard split API (#6845)
+
+* Fixes compilation for PG13.10 and PG14.7 (#6711)
+
+* Fixes crash that happens when trying to replicate a reference table that is
+  actually dropped (#6595)
+
+* Fixes memory leak issue with query results that returns single row (#6724)
+
+* Fixes the modifiers for subscription and role creation (#6603)
+
+* Makes sure to quote all identifiers used for logical replication to prevent
+  potential issues (#6604)
+
+* Makes sure to skip foreign key validations at the end of shard moves (#6640)
+
+### citus v11.1.5 (December 12, 2022) ###
+
+* Fixes two potential dangling pointer issues
+
+* Fixes compilation warning on PG13 + OpenSSL 3.0
+
+### citus v11.1.4 (October 24, 2022) ###
+
+* Fixes an upgrade problem for `worker_fetch_foreign_file` when upgrade path
+  starts from 8.3 up to 11.1
+
+* Fixes an upgrade problem for `worker_repartition_cleanup` when upgrade path
+  starts from 9.1 up to 11.1
+
+### citus v11.1.3 (October 14, 2022) ###
+
+* Adds support for PostgreSQL 15.0
+
+* Fixes a bug in `ALTER EXTENSION citus UPDATE`
+
+* Fixes a bug that causes a crash with empty/null password
+
+* Fixes a bug that causes not retaining trigger enable/disable settings when
+  re-creating them on shards
+
+* Fixes a bug that prevents retaining columnar table options after a
+  table-rewrite
+
+* Raises memory limits in columnar from 256MB to 1GB for reads and writes
+
+### citus v11.1.2 (September 30, 2022) ###
+
+* Adds support for PostgreSQL 15rc1
+
+* Disallows having `ON DELETE/UPDATE SET DEFAULT` actions on columns that
+  default to sequences
+
+* Fixes a bug that might cause inserting incorrect `DEFAULT` values when
+  applying foreign key actions
+
+* Fixes a performance issue related to shard-moves by creating replica
+  identities before copying shards
+
+* Improves logging during shard-splits and resource cleanup
+
+* Makes sure to reuse connections for shard-splits and logical replication
+
+* Makes sure to try dropping replication slots a few more times after a failure
+  at the end of the shard-split
+
+### citus v11.1.1 (September 16, 2022) ###
+
+* Fixes a bug that prevents `create_distributed_table_concurrently()` working
+  on an empty node
+
+### citus v11.1.0 (September 15, 2022) ###
+
+* Adds support for PostgreSQL 15beta4
+
+* Adds ability to run shard rebalancer in the background
+
+* Adds `create_distributed_table_concurrently()` UDF to distribute tables
+  without interrupting the application
+
+* Adds `citus_split_shard_by_split_points()` UDF that allows
+  splitting a shard to specified set of nodes without blocking writes
+  and based on given split points
+
+* Adds support for non-blocking tenant isolation
+
+* Adds support for isolation tenants that use partitioned tables
+  or columnar tables
+
+* Separates columnar table access method into a separate logical extension
+
+* Adds support for online replication in `replicate_reference_tables()`
+
+* Improves performance of blocking shard moves
+
+* Improves non-blocking shard moves with a faster custom copy logic
+
+* Creates all foreign keys quickly at the end of a shard move
+
+* Limits `get_rebalance_progress()` to show shards in moving state
+
+* Makes `citus_move_shard_placement()` idempotent if shard already exists
+  on target node
+
+* Shows `citus_copy_shard_placement()` progress in `get_rebalance_progres()`
+
+* Supports changing CPU priorities for backends and shard moves
+
+* Adds the GUC `citus.allow_unsafe_constraints` to allow unique/exclusion/
+  primary key constraints without distribution column
+
+* Introduces GUC `citus.skip_constraint_validation`
+
+* Introduces `citus_locks` view
+
+* Improves `citus_tables` view by showing local tables added to metadata
+
+* Improves columnar table access method by moving old catalog tables into
+  an internal schema and introduces more secure & informative views based
+  on them
+
+* Adds support for `GRANT/REVOKE` on aggregates
+
+* Adds support for `NULLS NOT DISTINCT` clauses for indexes for PG15+
+
+* Adds support for setting relation options for columnar tables using
+  `ALTER TABLE`
+
+* Adds support for unlogged distributed sequences
+
+* Removes `do_repair` option from `citus_copy_shard_placement()`
+
+* Removes deprecated re-partitioning functions like
+  `worker_hash_partition_table()`
+
+* Drops support for isolation tenants that use replicated tables
+
+* Checks existence of the shards before insert, delete, and update
+
+* Hides tables owned by extensions from `citus_tables` and `citus_shards`
+
+* Propagates `VACUUM` and `ANALYZE` to worker nodes
+
+* Makes non-partitioned table size calculation quicker
+
+* Improves `create_distributed_table()` by creating new colocation entries when
+  using `colocate_with => 'none'`
+
+* Ensures that `SELECT .. FOR UPDATE` opens a transaction block when used in
+  a function call
+
+* Prevents a segfault by disallowing usage of SQL functions referencing to a
+  distributed table
+
+* Prevents creating a new colocation entry when replicating reference tables
+
+* Fixes a bug in query escaping in `undistribute_table()` and
+  `alter_distributed_table()`
+
+* Fixes a bug preventing the usage of `isolate_tenant_to_new_shard()` with text
+  column
+
+* Fixes a bug that may cause `GRANT` to propagate within `CREATE EXTENSION`
+
+* Fixes a bug that causes incorrectly marking `metadatasynced` flag for
+  coordinator
+
+* Fixes a bug that may prevent Citus from creating function in transaction
+  block properly
+
+* Fixes a bug that prevents promoting read-replicas as primaries
+
+* Fixes a bug that prevents setting colocation group of a partitioned
+  distributed table to `none`
+
+* Fixes a bug that prevents using `AUTO` option for `VACUUM (INDEX_CLEANUP)`
+  operation
+
+* Fixes a segfault in `citus_copy_shard_placement()`
+
+* Fixes an issue that can cause logical reference table replication to fail
+
+* Fixes schema name qualification for `RENAME SEQUENCE` statement
+
+* Fixes several small memory leaks
+
+* Fixes the transaction timestamp column of the `get_current_transaction_id()`
+  on coordinator
+
+* Maps any unused parameters to a generic type in prepared statements
+
 ### citus v10.2.8 (August 19, 2022) ###
 
 * Fixes compilation warning caused by latest upgrade script changes

ci/fix_gitignore.sh

@@ -5,7 +5,8 @@ source ci/ci_helpers.sh
 
 # Remove all the ignored files from git tree, and error out
 # find all ignored files in git tree, and use quotation marks to prevent word splitting on filenames with spaces in them
-ignored_lines_in_git_tree=$(git ls-files --ignored --exclude-standard | sed 's/.*/"&"/')
+# NOTE: Option --cached is needed to avoid a bug in git ls-files command.
+ignored_lines_in_git_tree=$(git ls-files --ignored --cached --exclude-standard | sed 's/.*/"&"/')
 
 if [[ -n $ignored_lines_in_git_tree ]]
 then

configure

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for Citus 11.1devel.
+# Generated by GNU Autoconf 2.69 for Citus 11.1.6.
 #
 #
 # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@@ -579,8 +579,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='Citus'
 PACKAGE_TARNAME='citus'
-PACKAGE_VERSION='11.1devel'
-PACKAGE_STRING='Citus 11.1devel'
+PACKAGE_VERSION='11.1.6'
+PACKAGE_STRING='Citus 11.1.6'
 PACKAGE_BUGREPORT=''
 PACKAGE_URL=''
 
@@ -1262,7 +1262,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures Citus 11.1devel to adapt to many kinds of systems.
+\`configure' configures Citus 11.1.6 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1324,7 +1324,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of Citus 11.1devel:";;
+     short | recursive ) echo "Configuration of Citus 11.1.6:";;
    esac
   cat <<\_ACEOF
 
@@ -1429,7 +1429,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-Citus configure 11.1devel
+Citus configure 11.1.6
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -1912,7 +1912,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by Citus $as_me 11.1devel, which was
+It was created by Citus $as_me 11.1.6, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -5393,7 +5393,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by Citus $as_me 11.1devel, which was
+This file was extended by Citus $as_me 11.1.6, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -5455,7 +5455,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-Citus config.status 11.1devel
+Citus config.status 11.1.6
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 

configure.ac

@@ -5,7 +5,7 @@
 # everyone needing autoconf installed, the resulting files are checked
 # into the SCM.
 
-AC_INIT([Citus], [11.1devel])
+AC_INIT([Citus], [11.1.6])
 AC_COPYRIGHT([Copyright (c) Citus Data, Inc.])
 
 # we'll need sed and awk for some of the version commands

src/backend/columnar/columnar_metadata.c

@@ -60,6 +60,10 @@
 #include "utils/relfilenodemap.h"
 
 #define COLUMNAR_RELOPTION_NAMESPACE "columnar"
+#define SLOW_METADATA_ACCESS_WARNING \
+	"Metadata index %s is not available, this might mean slower read/writes " \
+	"on columnar tables. This is expected during Postgres upgrades and not " \
+	"expected otherwise."
 
 typedef struct
 {
@@ -701,15 +705,23 @@ ReadStripeSkipList(RelFileNode relfilenode, uint64 stripe, TupleDesc tupleDescri
 
 	Oid columnarChunkOid = ColumnarChunkRelationId();
 	Relation columnarChunk = table_open(columnarChunkOid, AccessShareLock);
-	Relation index = index_open(ColumnarChunkIndexRelationId(), AccessShareLock);
 
 	ScanKeyInit(&scanKey[0], Anum_columnar_chunk_storageid,
 				BTEqualStrategyNumber, F_OIDEQ, UInt64GetDatum(storageId));
 	ScanKeyInit(&scanKey[1], Anum_columnar_chunk_stripe,
 				BTEqualStrategyNumber, F_OIDEQ, Int32GetDatum(stripe));
 
-	SysScanDesc scanDescriptor = systable_beginscan_ordered(columnarChunk, index,
-															snapshot, 2, scanKey);
+	Oid indexId = ColumnarChunkIndexRelationId();
+	bool indexOk = OidIsValid(indexId);
+	SysScanDesc scanDescriptor = systable_beginscan(columnarChunk, indexId,
+													indexOk, snapshot, 2, scanKey);
+
+	static bool loggedSlowMetadataAccessWarning = false;
+	if (!indexOk && !loggedSlowMetadataAccessWarning)
+	{
+		ereport(WARNING, (errmsg(SLOW_METADATA_ACCESS_WARNING, "chunk_pkey")));
+		loggedSlowMetadataAccessWarning = true;
+	}
 
 	StripeSkipList *chunkList = palloc0(sizeof(StripeSkipList));
 	chunkList->chunkCount = chunkCount;
@@ -721,8 +733,7 @@ ReadStripeSkipList(RelFileNode relfilenode, uint64 stripe, TupleDesc tupleDescri
 			palloc0(chunkCount * sizeof(ColumnChunkSkipNode));
 	}
 
-	while (HeapTupleIsValid(heapTuple = systable_getnext_ordered(scanDescriptor,
-																 ForwardScanDirection)))
+	while (HeapTupleIsValid(heapTuple = systable_getnext(scanDescriptor)))
 	{
 		Datum datumArray[Natts_columnar_chunk];
 		bool isNullArray[Natts_columnar_chunk];
@@ -787,8 +798,7 @@ ReadStripeSkipList(RelFileNode relfilenode, uint64 stripe, TupleDesc tupleDescri
 		}
 	}
 
-	systable_endscan_ordered(scanDescriptor);
-	index_close(index, AccessShareLock);
+	systable_endscan(scanDescriptor);
 	table_close(columnarChunk, AccessShareLock);
 
 	chunkList->chunkGroupRowCounts =
@@ -799,9 +809,9 @@ ReadStripeSkipList(RelFileNode relfilenode, uint64 stripe, TupleDesc tupleDescri
 
 
 /*
- * FindStripeByRowNumber returns StripeMetadata for the stripe whose
- * firstRowNumber is greater than given rowNumber. If no such stripe
- * exists, then returns NULL.
+ * FindStripeByRowNumber returns StripeMetadata for the stripe that has the
+ * smallest firstRowNumber among the stripes whose firstRowNumber is grater
+ * than given rowNumber. If no such stripe exists, then returns NULL.
  */
 StripeMetadata *
 FindNextStripeByRowNumber(Relation relation, uint64 rowNumber, Snapshot snapshot)
@@ -891,8 +901,7 @@ StripeGetHighestRowNumber(StripeMetadata *stripeMetadata)
 /*
  * StripeMetadataLookupRowNumber returns StripeMetadata for the stripe whose
  * firstRowNumber is less than or equal to (FIND_LESS_OR_EQUAL), or is
- * greater than (FIND_GREATER) given rowNumber by doing backward index
- * scan on stripe_first_row_number_idx.
+ * greater than (FIND_GREATER) given rowNumber.
  * If no such stripe exists, then returns NULL.
  */
 static StripeMetadata *
@@ -923,14 +932,23 @@ StripeMetadataLookupRowNumber(Relation relation, uint64 rowNumber, Snapshot snap
 	ScanKeyInit(&scanKey[1], Anum_columnar_stripe_first_row_number,
 				strategyNumber, procedure, UInt64GetDatum(rowNumber));
 
-
 	Relation columnarStripes = table_open(ColumnarStripeRelationId(), AccessShareLock);
-	Relation index = index_open(ColumnarStripeFirstRowNumberIndexRelationId(),
-								AccessShareLock);
-	SysScanDesc scanDescriptor = systable_beginscan_ordered(columnarStripes, index,
-															snapshot, 2,
-															scanKey);
 
+	Oid indexId = ColumnarStripeFirstRowNumberIndexRelationId();
+	bool indexOk = OidIsValid(indexId);
+	SysScanDesc scanDescriptor = systable_beginscan(columnarStripes, indexId, indexOk,
+													snapshot, 2, scanKey);
+
+	static bool loggedSlowMetadataAccessWarning = false;
+	if (!indexOk && !loggedSlowMetadataAccessWarning)
+	{
+		ereport(WARNING, (errmsg(SLOW_METADATA_ACCESS_WARNING,
+								 "stripe_first_row_number_idx")));
+		loggedSlowMetadataAccessWarning = true;
+	}
+
+	if (indexOk)
+	{
 		ScanDirection scanDirection = NoMovementScanDirection;
 		if (lookupMode == FIND_LESS_OR_EQUAL)
 		{
@@ -945,9 +963,40 @@ StripeMetadataLookupRowNumber(Relation relation, uint64 rowNumber, Snapshot snap
 		{
 			foundStripeMetadata = BuildStripeMetadata(columnarStripes, heapTuple);
 		}
+	}
+	else
+	{
+		HeapTuple heapTuple = NULL;
+		while (HeapTupleIsValid(heapTuple = systable_getnext(scanDescriptor)))
+		{
+			StripeMetadata *stripe = BuildStripeMetadata(columnarStripes, heapTuple);
+			if (!foundStripeMetadata)
+			{
+				/* first match */
+				foundStripeMetadata = stripe;
+			}
+			else if (lookupMode == FIND_LESS_OR_EQUAL &&
+					 stripe->firstRowNumber > foundStripeMetadata->firstRowNumber)
+			{
+				/*
+				 * Among the stripes with firstRowNumber less-than-or-equal-to given,
+				 * we're looking for the one with the greatest firstRowNumber.
+				 */
+				foundStripeMetadata = stripe;
+			}
+			else if (lookupMode == FIND_GREATER &&
+					 stripe->firstRowNumber < foundStripeMetadata->firstRowNumber)
+			{
+				/*
+				 * Among the stripes with firstRowNumber greater-than given,
+				 * we're looking for the one with the smallest firstRowNumber.
+				 */
+				foundStripeMetadata = stripe;
+			}
+		}
+	}
 
-	systable_endscan_ordered(scanDescriptor);
-	index_close(index, AccessShareLock);
+	systable_endscan(scanDescriptor);
 	table_close(columnarStripes, AccessShareLock);
 
 	return foundStripeMetadata;
@@ -1021,8 +1070,8 @@ CheckStripeMetadataConsistency(StripeMetadata *stripeMetadata)
 
 /*
  * FindStripeWithHighestRowNumber returns StripeMetadata for the stripe that
- * has the row with highest rowNumber by doing backward index scan on
- * stripe_first_row_number_idx. If given relation is empty, then returns NULL.
+ * has the row with highest rowNumber. If given relation is empty, then returns
+ * NULL.
  */
 StripeMetadata *
 FindStripeWithHighestRowNumber(Relation relation, Snapshot snapshot)
@@ -1035,19 +1084,46 @@ FindStripeWithHighestRowNumber(Relation relation, Snapshot snapshot)
 				BTEqualStrategyNumber, F_OIDEQ, Int32GetDatum(storageId));
 
 	Relation columnarStripes = table_open(ColumnarStripeRelationId(), AccessShareLock);
-	Relation index = index_open(ColumnarStripeFirstRowNumberIndexRelationId(),
-								AccessShareLock);
-	SysScanDesc scanDescriptor = systable_beginscan_ordered(columnarStripes, index,
+
+	Oid indexId = ColumnarStripeFirstRowNumberIndexRelationId();
+	bool indexOk = OidIsValid(indexId);
+	SysScanDesc scanDescriptor = systable_beginscan(columnarStripes, indexId, indexOk,
 													snapshot, 1, scanKey);
 
-	HeapTuple heapTuple = systable_getnext_ordered(scanDescriptor, BackwardScanDirection);
+	static bool loggedSlowMetadataAccessWarning = false;
+	if (!indexOk && !loggedSlowMetadataAccessWarning)
+	{
+		ereport(WARNING, (errmsg(SLOW_METADATA_ACCESS_WARNING,
+								 "stripe_first_row_number_idx")));
+		loggedSlowMetadataAccessWarning = true;
+	}
+
+	if (indexOk)
+	{
+		/* do one-time fetch using the index */
+		HeapTuple heapTuple = systable_getnext_ordered(scanDescriptor,
+													   BackwardScanDirection);
 		if (HeapTupleIsValid(heapTuple))
 		{
 			stripeWithHighestRowNumber = BuildStripeMetadata(columnarStripes, heapTuple);
 		}
+	}
+	else
+	{
+		HeapTuple heapTuple = NULL;
+		while (HeapTupleIsValid(heapTuple = systable_getnext(scanDescriptor)))
+		{
+			StripeMetadata *stripe = BuildStripeMetadata(columnarStripes, heapTuple);
+			if (!stripeWithHighestRowNumber ||
+				stripe->firstRowNumber > stripeWithHighestRowNumber->firstRowNumber)
+			{
+				/* first or a greater match */
+				stripeWithHighestRowNumber = stripe;
+			}
+		}
+	}
 
-	systable_endscan_ordered(scanDescriptor);
-	index_close(index, AccessShareLock);
+	systable_endscan(scanDescriptor);
 	table_close(columnarStripes, AccessShareLock);
 
 	return stripeWithHighestRowNumber;
@@ -1064,7 +1140,6 @@ ReadChunkGroupRowCounts(uint64 storageId, uint64 stripe, uint32 chunkGroupCount,
 {
 	Oid columnarChunkGroupOid = ColumnarChunkGroupRelationId();
 	Relation columnarChunkGroup = table_open(columnarChunkGroupOid, AccessShareLock);
-	Relation index = index_open(ColumnarChunkGroupIndexRelationId(), AccessShareLock);
 
 	ScanKeyData scanKey[2];
 	ScanKeyInit(&scanKey[0], Anum_columnar_chunkgroup_storageid,
@@ -1072,15 +1147,22 @@ ReadChunkGroupRowCounts(uint64 storageId, uint64 stripe, uint32 chunkGroupCount,
 	ScanKeyInit(&scanKey[1], Anum_columnar_chunkgroup_stripe,
 				BTEqualStrategyNumber, F_OIDEQ, Int32GetDatum(stripe));
 
+	Oid indexId = ColumnarChunkGroupIndexRelationId();
+	bool indexOk = OidIsValid(indexId);
 	SysScanDesc scanDescriptor =
-		systable_beginscan_ordered(columnarChunkGroup, index, snapshot, 2, scanKey);
+		systable_beginscan(columnarChunkGroup, indexId, indexOk, snapshot, 2, scanKey);
+
+	static bool loggedSlowMetadataAccessWarning = false;
+	if (!indexOk && !loggedSlowMetadataAccessWarning)
+	{
+		ereport(WARNING, (errmsg(SLOW_METADATA_ACCESS_WARNING, "chunk_group_pkey")));
+		loggedSlowMetadataAccessWarning = true;
+	}
 
-	uint32 chunkGroupIndex = 0;
 	HeapTuple heapTuple = NULL;
 	uint32 *chunkGroupRowCounts = palloc0(chunkGroupCount * sizeof(uint32));
 
-	while (HeapTupleIsValid(heapTuple = systable_getnext_ordered(scanDescriptor,
-																 ForwardScanDirection)))
+	while (HeapTupleIsValid(heapTuple = systable_getnext(scanDescriptor)))
 	{
 		Datum datumArray[Natts_columnar_chunkgroup];
 		bool isNullArray[Natts_columnar_chunkgroup];
@@ -1091,24 +1173,16 @@ ReadChunkGroupRowCounts(uint64 storageId, uint64 stripe, uint32 chunkGroupCount,
 
 		uint32 tupleChunkGroupIndex =
 			DatumGetUInt32(datumArray[Anum_columnar_chunkgroup_chunk - 1]);
-		if (chunkGroupIndex >= chunkGroupCount ||
-			tupleChunkGroupIndex != chunkGroupIndex)
+		if (tupleChunkGroupIndex >= chunkGroupCount)
 		{
 			elog(ERROR, "unexpected chunk group");
 		}
 
-		chunkGroupRowCounts[chunkGroupIndex] =
+		chunkGroupRowCounts[tupleChunkGroupIndex] =
 			(uint32) DatumGetUInt64(datumArray[Anum_columnar_chunkgroup_row_count - 1]);
-		chunkGroupIndex++;
 	}
 
-	if (chunkGroupIndex != chunkGroupCount)
-	{
-		elog(ERROR, "unexpected chunk group count");
-	}
-
-	systable_endscan_ordered(scanDescriptor);
-	index_close(index, AccessShareLock);
+	systable_endscan(scanDescriptor);
 	table_close(columnarChunkGroup, AccessShareLock);
 
 	return chunkGroupRowCounts;
@@ -1305,14 +1379,20 @@ UpdateStripeMetadataRow(uint64 storageId, uint64 stripeId, bool *update,
 	Oid columnarStripesOid = ColumnarStripeRelationId();
 
 	Relation columnarStripes = table_open(columnarStripesOid, AccessShareLock);
-	Relation columnarStripePkeyIndex = index_open(ColumnarStripePKeyIndexRelationId(),
-												  AccessShareLock);
 
-	SysScanDesc scanDescriptor = systable_beginscan_ordered(columnarStripes,
-															columnarStripePkeyIndex,
+	Oid indexId = ColumnarStripePKeyIndexRelationId();
+	bool indexOk = OidIsValid(indexId);
+	SysScanDesc scanDescriptor = systable_beginscan(columnarStripes, indexId, indexOk,
 													&dirtySnapshot, 2, scanKey);
 
-	HeapTuple oldTuple = systable_getnext_ordered(scanDescriptor, ForwardScanDirection);
+	static bool loggedSlowMetadataAccessWarning = false;
+	if (!indexOk && !loggedSlowMetadataAccessWarning)
+	{
+		ereport(WARNING, (errmsg(SLOW_METADATA_ACCESS_WARNING, "stripe_pkey")));
+		loggedSlowMetadataAccessWarning = true;
+	}
+
+	HeapTuple oldTuple = systable_getnext(scanDescriptor);
 	if (!HeapTupleIsValid(oldTuple))
 	{
 		ereport(ERROR, (errmsg("attempted to modify an unexpected stripe, "
@@ -1347,8 +1427,7 @@ UpdateStripeMetadataRow(uint64 storageId, uint64 stripeId, bool *update,
 
 	CommandCounterIncrement();
 
-	systable_endscan_ordered(scanDescriptor);
-	index_close(columnarStripePkeyIndex, AccessShareLock);
+	systable_endscan(scanDescriptor);
 	table_close(columnarStripes, AccessShareLock);
 
 	/* return StripeMetadata object built from modified tuple */
@@ -1359,6 +1438,10 @@ UpdateStripeMetadataRow(uint64 storageId, uint64 stripeId, bool *update,
 /*
  * ReadDataFileStripeList reads the stripe list for a given storageId
  * in the given snapshot.
+ *
+ * Doesn't sort the stripes by their ids before returning if
+ * stripe_first_row_number_idx is not available --normally can only happen
+ * during pg upgrades.
  */
 static List *
 ReadDataFileStripeList(uint64 storageId, Snapshot snapshot)
@@ -1373,22 +1456,27 @@ ReadDataFileStripeList(uint64 storageId, Snapshot snapshot)
 	Oid columnarStripesOid = ColumnarStripeRelationId();
 
 	Relation columnarStripes = table_open(columnarStripesOid, AccessShareLock);
-	Relation index = index_open(ColumnarStripeFirstRowNumberIndexRelationId(),
-								AccessShareLock);
 
-	SysScanDesc scanDescriptor = systable_beginscan_ordered(columnarStripes, index,
-															snapshot, 1,
-															scanKey);
+	Oid indexId = ColumnarStripeFirstRowNumberIndexRelationId();
+	bool indexOk = OidIsValid(indexId);
+	SysScanDesc scanDescriptor = systable_beginscan(columnarStripes, indexId,
+													indexOk, snapshot, 1, scanKey);
 
-	while (HeapTupleIsValid(heapTuple = systable_getnext_ordered(scanDescriptor,
-																 ForwardScanDirection)))
+	static bool loggedSlowMetadataAccessWarning = false;
+	if (!indexOk && !loggedSlowMetadataAccessWarning)
+	{
+		ereport(WARNING, (errmsg(SLOW_METADATA_ACCESS_WARNING,
+								 "stripe_first_row_number_idx")));
+		loggedSlowMetadataAccessWarning = true;
+	}
+
+	while (HeapTupleIsValid(heapTuple = systable_getnext(scanDescriptor)))
 	{
 		StripeMetadata *stripeMetadata = BuildStripeMetadata(columnarStripes, heapTuple);
 		stripeMetadataList = lappend(stripeMetadataList, stripeMetadata);
 	}
 
-	systable_endscan_ordered(scanDescriptor);
-	index_close(index, AccessShareLock);
+	systable_endscan(scanDescriptor);
 	table_close(columnarStripes, AccessShareLock);
 
 	return stripeMetadataList;
@@ -1499,25 +1587,30 @@ DeleteStorageFromColumnarMetadataTable(Oid metadataTableId,
 		return;
 	}
 
-	Relation index = index_open(storageIdIndexId, AccessShareLock);
+	bool indexOk = OidIsValid(storageIdIndexId);
+	SysScanDesc scanDescriptor = systable_beginscan(metadataTable, storageIdIndexId,
+													indexOk, NULL, 1, scanKey);
 
-	SysScanDesc scanDescriptor = systable_beginscan_ordered(metadataTable, index, NULL,
-															1, scanKey);
+	static bool loggedSlowMetadataAccessWarning = false;
+	if (!indexOk && !loggedSlowMetadataAccessWarning)
+	{
+		ereport(WARNING, (errmsg(SLOW_METADATA_ACCESS_WARNING,
+								 "on a columnar metadata table")));
+		loggedSlowMetadataAccessWarning = true;
+	}
 
 	ModifyState *modifyState = StartModifyRelation(metadataTable);
 
 	HeapTuple heapTuple;
-	while (HeapTupleIsValid(heapTuple = systable_getnext_ordered(scanDescriptor,
-																 ForwardScanDirection)))
+	while (HeapTupleIsValid(heapTuple = systable_getnext(scanDescriptor)))
 	{
 		DeleteTupleAndEnforceConstraints(modifyState, heapTuple);
 	}
 
-	systable_endscan_ordered(scanDescriptor);
+	systable_endscan(scanDescriptor);
 
 	FinishModifyRelation(modifyState);
 
-	index_close(index, AccessShareLock);
 	table_close(metadataTable, AccessShareLock);
 }
 
@@ -1650,6 +1743,9 @@ create_estate_for_relation(Relation rel)
 
 /*
  * DatumToBytea serializes a datum into a bytea value.
+ *
+ * Since we don't want to limit datum size to RSIZE_MAX unnecessarily,
+ * we use memcpy instead of memcpy_s several places in this function.
  */
 static bytea *
 DatumToBytea(Datum value, Form_pg_attribute attrForm)
@@ -1666,19 +1762,16 @@ DatumToBytea(Datum value, Form_pg_attribute attrForm)
 			Datum tmp;
 			store_att_byval(&tmp, value, attrForm->attlen);
 
-			memcpy_s(VARDATA(result), datumLength + VARHDRSZ,
-					 &tmp, attrForm->attlen);
+			memcpy(VARDATA(result), &tmp, attrForm->attlen); /* IGNORE-BANNED */
 		}
 		else
 		{
-			memcpy_s(VARDATA(result), datumLength + VARHDRSZ,
-					 DatumGetPointer(value), attrForm->attlen);
+			memcpy(VARDATA(result), DatumGetPointer(value), attrForm->attlen); /* IGNORE-BANNED */
 		}
 	}
 	else
 	{
-		memcpy_s(VARDATA(result), datumLength + VARHDRSZ,
-				 DatumGetPointer(value), datumLength);
+		memcpy(VARDATA(result), DatumGetPointer(value), datumLength); /* IGNORE-BANNED */
 	}
 
 	return result;
@@ -1697,8 +1790,12 @@ ByteaToDatum(bytea *bytes, Form_pg_attribute attrForm)
 	 * after the byteaDatum is freed.
 	 */
 	char *binaryDataCopy = palloc0(VARSIZE_ANY_EXHDR(bytes));
-	memcpy_s(binaryDataCopy, VARSIZE_ANY_EXHDR(bytes),
-			 VARDATA_ANY(bytes), VARSIZE_ANY_EXHDR(bytes));
+
+	/*
+	 * We use IGNORE-BANNED here since we don't want to limit datum size to
+	 * RSIZE_MAX unnecessarily.
+	 */
+	memcpy(binaryDataCopy, VARDATA_ANY(bytes), VARSIZE_ANY_EXHDR(bytes)); /* IGNORE-BANNED */
 
 	return fetch_att(binaryDataCopy, attrForm->attbyval, attrForm->attlen);
 }

src/backend/columnar/columnar_tableam.c

@@ -739,7 +739,9 @@ columnar_tuple_insert(Relation relation, TupleTableSlot *slot, CommandId cid,
 	 */
 	ColumnarWriteState *writeState = columnar_init_write_state(relation,
 															   RelationGetDescr(relation),
+															   slot->tts_tableOid,
 															   GetCurrentSubTransactionId());
+
 	MemoryContext oldContext = MemoryContextSwitchTo(ColumnarWritePerTupleContext(
 														 writeState));
 
@@ -781,8 +783,14 @@ columnar_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
 {
 	CheckCitusColumnarVersion(ERROR);
 
+	/*
+	 * The callback to .multi_insert is table_multi_insert() and this is only used for the COPY
+	 * command, so slot[i]->tts_tableoid will always be equal to relation->id. Thus, we can send
+	 * RelationGetRelid(relation) as the tupSlotTableOid
+	 */
 	ColumnarWriteState *writeState = columnar_init_write_state(relation,
 															   RelationGetDescr(relation),
+															   RelationGetRelid(relation),
 															   GetCurrentSubTransactionId());
 
 	ColumnarCheckLogicalReplication(relation);
@@ -2568,8 +2576,13 @@ detoast_values(TupleDesc tupleDesc, Datum *orig_values, bool *isnull)
 			if (values == orig_values)
 			{
 				values = palloc(sizeof(Datum) * natts);
-				memcpy_s(values, sizeof(Datum) * natts,
-						 orig_values, sizeof(Datum) * natts);
+
+				/*
+				 * We use IGNORE-BANNED here since we don't want to limit
+				 * size of the buffer that holds the datum array to RSIZE_MAX
+				 * unnecessarily.
+				 */
+				memcpy(values, orig_values, sizeof(Datum) * natts); /* IGNORE-BANNED */
 			}
 
 			/* will be freed when per-tuple context is reset */

src/backend/columnar/columnar_writer.c

@@ -531,6 +531,9 @@ SerializeBoolArray(bool *boolArray, uint32 boolArrayLength)
 /*
  * SerializeSingleDatum serializes the given datum value and appends it to the
  * provided string info buffer.
+ *
+ * Since we don't want to limit datum buffer size to RSIZE_MAX unnecessarily,
+ * we use memcpy instead of memcpy_s several places in this function.
  */
 static void
 SerializeSingleDatum(StringInfo datumBuffer, Datum datum, bool datumTypeByValue,
@@ -552,15 +555,13 @@ SerializeSingleDatum(StringInfo datumBuffer, Datum datum, bool datumTypeByValue,
 		}
 		else
 		{
-			memcpy_s(currentDatumDataPointer, datumBuffer->maxlen - datumBuffer->len,
-					 DatumGetPointer(datum), datumTypeLength);
+			memcpy(currentDatumDataPointer, DatumGetPointer(datum), datumTypeLength); /* IGNORE-BANNED */
 		}
 	}
 	else
 	{
 		Assert(!datumTypeByValue);
-		memcpy_s(currentDatumDataPointer, datumBuffer->maxlen - datumBuffer->len,
-				 DatumGetPointer(datum), datumLength);
+		memcpy(currentDatumDataPointer, DatumGetPointer(datum), datumLength); /* IGNORE-BANNED */
 	}
 
 	datumBuffer->len += datumLengthAligned;
@@ -714,7 +715,12 @@ DatumCopy(Datum datum, bool datumTypeByValue, int datumTypeLength)
 	{
 		uint32 datumLength = att_addlength_datum(0, datumTypeLength, datum);
 		char *datumData = palloc0(datumLength);
-		memcpy_s(datumData, datumLength, DatumGetPointer(datum), datumLength);
+
+		/*
+		 * We use IGNORE-BANNED here since we don't want to limit datum size to
+		 * RSIZE_MAX unnecessarily.
+		 */
+		memcpy(datumData, DatumGetPointer(datum), datumLength); /* IGNORE-BANNED */
 
 		datumCopy = PointerGetDatum(datumData);
 	}
@@ -737,8 +743,12 @@ CopyStringInfo(StringInfo sourceString)
 		targetString->data = palloc0(sourceString->len);
 		targetString->len = sourceString->len;
 		targetString->maxlen = sourceString->len;
-		memcpy_s(targetString->data, sourceString->len,
-				 sourceString->data, sourceString->len);
+
+		/*
+		 * We use IGNORE-BANNED here since we don't want to limit string
+		 * buffer size to RSIZE_MAX unnecessarily.
+		 */
+		memcpy(targetString->data, sourceString->data, sourceString->len); /* IGNORE-BANNED */
 	}
 
 	return targetString;

src/backend/columnar/sql/columnar--10.2-2--10.2-3.sql

@@ -10,6 +10,17 @@
 --
 -- To do that, drop stripe_first_row_number_idx and create a unique
 -- constraint with the same name to keep the code change at minimum.
+--
+-- If we have a pg_depend entry for this index, we can not drop it as
+-- the extension depends on it. Remove the pg_depend entry if it exists.
+DELETE FROM pg_depend
+WHERE classid = 'pg_am'::regclass::oid
+    AND objid IN (select oid from pg_am where amname = 'columnar')
+    AND objsubid = 0
+    AND refclassid = 'pg_class'::regclass::oid
+    AND refobjid = 'columnar.stripe_first_row_number_idx'::regclass::oid
+    AND refobjsubid = 0
+    AND deptype = 'n';
 DROP INDEX columnar.stripe_first_row_number_idx;
 ALTER TABLE columnar.stripe ADD CONSTRAINT stripe_first_row_number_idx
 UNIQUE (storage_id, first_row_number);

src/backend/columnar/sql/downgrades/columnar--10.2-1--10.1-1.sql

@@ -8,5 +8,16 @@ DROP FUNCTION citus_internal.upgrade_columnar_storage(regclass);
 DROP FUNCTION citus_internal.downgrade_columnar_storage(regclass);
 
 -- drop "first_row_number" column and the index defined on it
+--
+-- If we have a pg_depend entry for this index, we can not drop it as
+-- the extension depends on it. Remove the pg_depend entry if it exists.
+DELETE FROM pg_depend
+WHERE classid = 'pg_am'::regclass::oid
+    AND objid IN (select oid from pg_am where amname = 'columnar')
+    AND objsubid = 0
+    AND refclassid = 'pg_class'::regclass::oid
+    AND refobjid = 'columnar.stripe_first_row_number_idx'::regclass::oid
+    AND refobjsubid = 0
+    AND deptype = 'n';
 DROP INDEX columnar.stripe_first_row_number_idx;
 ALTER TABLE columnar.stripe DROP COLUMN first_row_number;

src/backend/columnar/sql/downgrades/columnar--10.2-3--10.2-2.sql

@@ -1,4 +1,14 @@
 -- columnar--10.2-3--10.2-2.sql
-
+--
+-- If we have a pg_depend entry for this index, we can not drop it as
+-- the extension depends on it. Remove the pg_depend entry if it exists.
+DELETE FROM pg_depend
+WHERE classid = 'pg_am'::regclass::oid
+    AND objid IN (select oid from pg_am where amname = 'columnar')
+    AND objsubid = 0
+    AND refclassid = 'pg_class'::regclass::oid
+    AND refobjid = 'columnar.stripe_first_row_number_idx'::regclass::oid
+    AND refobjsubid = 0
+    AND deptype = 'n';
 ALTER TABLE columnar.stripe DROP CONSTRAINT stripe_first_row_number_idx;
 CREATE INDEX stripe_first_row_number_idx ON columnar.stripe USING BTREE(storage_id, first_row_number);

src/backend/columnar/write_state_management.c

@@ -113,6 +113,7 @@ CleanupWriteStateMap(void *arg)
 
 ColumnarWriteState *
 columnar_init_write_state(Relation relation, TupleDesc tupdesc,
+						  Oid tupSlotRelationId,
 						  SubTransactionId currentSubXid)
 {
 	bool found;
@@ -176,7 +177,16 @@ columnar_init_write_state(Relation relation, TupleDesc tupdesc,
 	MemoryContext oldContext = MemoryContextSwitchTo(WriteStateContext);
 
 	ColumnarOptions columnarOptions = { 0 };
-	ReadColumnarOptions(relation->rd_id, &columnarOptions);
+
+	/*
+	 * In case of a table rewrite, we need to fetch table options based on the
+	 * relation id of the source tuple slot.
+	 *
+	 * For this reason, we always pass tupSlotRelationId here; which should be
+	 * same as the target table if the write operation is not related to a table
+	 * rewrite etc.
+	 */
+	ReadColumnarOptions(tupSlotRelationId, &columnarOptions);
 
 	SubXidWriteState *stackEntry = palloc0(sizeof(SubXidWriteState));
 	stackEntry->writeState = ColumnarBeginWrite(relation->rd_node,

src/backend/distributed/commands/alter_table.c

@@ -1223,8 +1223,15 @@ CreateDistributedTableLike(TableConversionState *con)
 		newShardCount = con->shardCount;
 	}
 
+	/*
+	 * To get the correct column name, we use the original relation id, not the
+	 * new relation id. The reason is that the cached attributes of the original
+	 * and newly created tables are not the same if the original table has
+	 * dropped columns (dropped columns are still present in the attribute cache)
+	 * Detailed example in https://github.com/citusdata/citus/pull/6387
+	 */
 	char *distributionColumnName =
-		ColumnToColumnName(con->newRelationId, (Node *) newDistributionKey);
+		ColumnToColumnName(con->relationId, (Node *) newDistributionKey);
 
 	Oid originalRelationId = con->relationId;
 	if (con->originalDistributionKey != NULL && PartitionTable(originalRelationId))
@@ -1604,6 +1611,8 @@ ReplaceTable(Oid sourceId, Oid targetId, List *justBeforeDropCommands,
 		}
 		else if (ShouldSyncTableMetadata(sourceId))
 		{
+			char *qualifiedTableName = quote_qualified_identifier(schemaName, sourceName);
+
 			/*
 			 * We are converting a citus local table to a distributed/reference table,
 			 * so we should prevent dropping the sequence on the table. Otherwise, we'd
@@ -1612,8 +1621,8 @@ ReplaceTable(Oid sourceId, Oid targetId, List *justBeforeDropCommands,
 			StringInfo command = makeStringInfo();
 
 			appendStringInfo(command,
-							 "SELECT pg_catalog.worker_drop_sequence_dependency('%s');",
-							 quote_qualified_identifier(schemaName, sourceName));
+							 "SELECT pg_catalog.worker_drop_sequence_dependency(%s);",
+							 quote_literal_cstr(qualifiedTableName));
 
 			SendCommandToWorkersWithMetadata(command->data);
 		}
@@ -1903,11 +1912,17 @@ CreateWorkerChangeSequenceDependencyCommand(char *sequenceSchemaName, char *sequ
 											char *sourceSchemaName, char *sourceName,
 											char *targetSchemaName, char *targetName)
 {
+	char *qualifiedSchemaName = quote_qualified_identifier(sequenceSchemaName,
+														   sequenceName);
+	char *qualifiedSourceName = quote_qualified_identifier(sourceSchemaName, sourceName);
+	char *qualifiedTargetName = quote_qualified_identifier(targetSchemaName, targetName);
+
 	StringInfo query = makeStringInfo();
-	appendStringInfo(query, "SELECT worker_change_sequence_dependency('%s', '%s', '%s')",
-					 quote_qualified_identifier(sequenceSchemaName, sequenceName),
-					 quote_qualified_identifier(sourceSchemaName, sourceName),
-					 quote_qualified_identifier(targetSchemaName, targetName));
+	appendStringInfo(query, "SELECT worker_change_sequence_dependency(%s, %s, %s)",
+					 quote_literal_cstr(qualifiedSchemaName),
+					 quote_literal_cstr(qualifiedSourceName),
+					 quote_literal_cstr(qualifiedTargetName));
+
 	return query->data;
 }
 

src/backend/distributed/commands/cascade_table_operation_for_connected_relations.c

@@ -623,18 +623,13 @@ ExecuteForeignKeyCreateCommand(const char *commandString, bool skip_validation)
 	 */
 	Assert(IsA(parseTree, AlterTableStmt));
 
-	bool oldSkipConstraintsValidationValue = SkipConstraintValidation;
-
 	if (skip_validation && IsA(parseTree, AlterTableStmt))
 	{
-		EnableSkippingConstraintValidation();
-
+		SkipForeignKeyValidationIfConstraintIsFkey((AlterTableStmt *) parseTree, true);
 		ereport(DEBUG4, (errmsg("skipping validation for foreign key create "
 								"command \"%s\"", commandString)));
 	}
 
 	ProcessUtilityParseTree(parseTree, commandString, PROCESS_UTILITY_QUERY,
 							NULL, None_Receiver, NULL);
-
-	SkipConstraintValidation = oldSkipConstraintsValidationValue;
 }

src/backend/distributed/commands/citus_add_local_table_to_metadata.c

@@ -46,6 +46,7 @@
 #include "utils/lsyscache.h"
 #include "utils/ruleutils.h"
 #include "utils/syscache.h"
+#include "foreign/foreign.h"
 
 
 /*
@@ -61,6 +62,8 @@ static void ErrorIfAddingPartitionTableToMetadata(Oid relationId);
 static void ErrorIfUnsupportedCreateCitusLocalTable(Relation relation);
 static void ErrorIfUnsupportedCitusLocalTableKind(Oid relationId);
 static void ErrorIfUnsupportedCitusLocalColumnDefinition(Relation relation);
+static void EnsureIfPostgresFdwHasTableName(Oid relationId);
+static void ErrorIfOptionListHasNoTableName(List *optionList);
 static void NoticeIfAutoConvertingLocalTables(bool autoConverted, Oid relationId);
 static CascadeOperationType GetCascadeTypeForCitusLocalTables(bool autoConverted);
 static List * GetShellTableDDLEventsForCitusLocalTable(Oid relationId);
@@ -87,7 +90,7 @@ static List * ReversedOidList(List *oidList);
 static void AppendExplicitIndexIdsToList(Form_pg_index indexForm,
 										 List **explicitIndexIdList,
 										 int flags);
-static void DropDefaultExpressionsAndMoveOwnedSequenceOwnerships(Oid sourceRelationId,
+static void DropNextValExprsAndMoveOwnedSeqOwnerships(Oid sourceRelationId,
 													  Oid targetRelationId);
 static void DropDefaultColumnDefinition(Oid relationId, char *columnName);
 static void TransferSequenceOwnership(Oid ownedSequenceId, Oid targetRelationId,
@@ -128,6 +131,9 @@ citus_add_local_table_to_metadata_internal(Oid relationId, bool cascadeViaForeig
 {
 	CheckCitusVersion(ERROR);
 
+	/* enable citus_add_local_table_to_metadata on an empty node */
+	InsertCoordinatorIfClusterEmpty();
+
 	bool autoConverted = false;
 	CreateCitusLocalTable(relationId, cascadeViaForeignKeys, autoConverted);
 }
@@ -363,10 +369,10 @@ CreateCitusLocalTable(Oid relationId, bool cascadeViaForeignKeys, bool autoConve
 
 	/*
 	 * Move sequence ownerships from shard table to shell table and also drop
-	 * DEFAULT expressions from shard relation as we should evaluate such columns
-	 * in shell table when needed.
+	 * DEFAULT expressions based on sequences from shard relation as we should
+	 * evaluate such columns in shell table when needed.
 	 */
-	DropDefaultExpressionsAndMoveOwnedSequenceOwnerships(shardRelationId,
+	DropNextValExprsAndMoveOwnedSeqOwnerships(shardRelationId,
 											  shellRelationId);
 
 	InsertMetadataForCitusLocalTable(shellRelationId, shardId, autoConverted);
@@ -485,6 +491,16 @@ ErrorIfUnsupportedCreateCitusLocalTable(Relation relation)
 	EnsureTableNotDistributed(relationId);
 	ErrorIfUnsupportedCitusLocalColumnDefinition(relation);
 
+	/*
+	 * Error out with a hint if the foreign table is using postgres_fdw and
+	 * the option table_name is not provided.
+	 * Citus relays all the Citus local foreign table logic to the placement of the
+	 * Citus local table. If table_name is NOT provided, Citus would try to talk to
+	 * the foreign postgres table over the shard's table name, which would not exist
+	 * on the remote server.
+	 */
+	EnsureIfPostgresFdwHasTableName(relationId);
+
 	/*
 	 * When creating other citus table types, we don't need to check that case as
 	 * EnsureTableNotDistributed already errors out if the given relation implies
@@ -500,6 +516,93 @@ ErrorIfUnsupportedCreateCitusLocalTable(Relation relation)
 }
 
 
+/*
+ * ServerUsesPostgresFdw gets a foreign server Oid and returns true if the FDW that
+ * the server depends on is postgres_fdw. Returns false otherwise.
+ */
+bool
+ServerUsesPostgresFdw(Oid serverId)
+{
+	ForeignServer *server = GetForeignServer(serverId);
+	ForeignDataWrapper *fdw = GetForeignDataWrapper(server->fdwid);
+
+	if (strcmp(fdw->fdwname, "postgres_fdw") == 0)
+	{
+		return true;
+	}
+
+	return false;
+}
+
+
+/*
+ * EnsureIfPostgresFdwHasTableName errors out with a hint if the foreign table is using postgres_fdw and
+ * the option table_name is not provided.
+ */
+static void
+EnsureIfPostgresFdwHasTableName(Oid relationId)
+{
+	char relationKind = get_rel_relkind(relationId);
+	if (relationKind == RELKIND_FOREIGN_TABLE)
+	{
+		ForeignTable *foreignTable = GetForeignTable(relationId);
+		if (ServerUsesPostgresFdw(foreignTable->serverid))
+		{
+			ErrorIfOptionListHasNoTableName(foreignTable->options);
+		}
+	}
+}
+
+
+/*
+ * ErrorIfOptionListHasNoTableName gets an option list (DefElem) and errors out
+ * if the list does not contain a table_name element.
+ */
+static void
+ErrorIfOptionListHasNoTableName(List *optionList)
+{
+	char *table_nameString = "table_name";
+	DefElem *option = NULL;
+	foreach_ptr(option, optionList)
+	{
+		char *optionName = option->defname;
+		if (strcmp(optionName, table_nameString) == 0)
+		{
+			return;
+		}
+	}
+
+	ereport(ERROR, (errmsg(
+						"table_name option must be provided when using postgres_fdw with Citus"),
+					errhint("Provide the option \"table_name\" with value target table's"
+							" name")));
+}
+
+
+/*
+ * ForeignTableDropsTableNameOption returns true if given option list contains
+ * (DROP table_name).
+ */
+bool
+ForeignTableDropsTableNameOption(List *optionList)
+{
+	char *table_nameString = "table_name";
+	DefElem *option = NULL;
+	foreach_ptr(option, optionList)
+	{
+		char *optionName = option->defname;
+		DefElemAction optionAction = option->defaction;
+		if (strcmp(optionName, table_nameString) == 0 &&
+			optionAction == DEFELEM_DROP)
+		{
+			return true;
+		}
+	}
+
+	return false;
+}
+
+
 /*
  * ErrorIfUnsupportedCitusLocalTableKind errors out if the relation kind of
  * relation with relationId is not supported for citus local table creation.
@@ -1158,13 +1261,14 @@ GetRenameStatsCommandList(List *statsOidList, uint64 shardId)
 
 
 /*
- * DropDefaultExpressionsAndMoveOwnedSequenceOwnerships drops default column
- * definitions for relation with sourceRelationId. Also, for each column that
- * defaults to an owned sequence, it grants ownership to the same named column
- * of the relation with targetRelationId.
+ * DropNextValExprsAndMoveOwnedSeqOwnerships drops default column definitions
+ * that are based on sequences for relation with sourceRelationId.
+ *
+ * Also, for each such column that owns a sequence, it grants ownership to the
+ * same named column of the relation with targetRelationId.
  */
 static void
-DropDefaultExpressionsAndMoveOwnedSequenceOwnerships(Oid sourceRelationId,
+DropNextValExprsAndMoveOwnedSeqOwnerships(Oid sourceRelationId,
 										  Oid targetRelationId)
 {
 	List *columnNameList = NIL;
@@ -1175,10 +1279,29 @@ DropDefaultExpressionsAndMoveOwnedSequenceOwnerships(Oid sourceRelationId,
 	char *columnName = NULL;
 	Oid ownedSequenceId = InvalidOid;
 	forboth_ptr_oid(columnName, columnNameList, ownedSequenceId, ownedSequenceIdList)
+	{
+		/*
+		 * We drop nextval() expressions because Citus currently evaluates
+		 * nextval() on the shell table, not on the shards. Hence, there is
+		 * no reason for keeping nextval(). Also, distributed/reference table
+		 * shards do not have - so be consistent with those.
+		 *
+		 * Note that we keep other kind of DEFAULT expressions on shards
+		 * because we still want to be able to evaluate DEFAULT expressions
+		 * that are not based on sequences on shards, e.g., for foreign key
+		 * - SET DEFAULT actions.
+		 */
+		AttrNumber columnAttrNumber = get_attnum(sourceRelationId, columnName);
+		if (ColumnDefaultsToNextVal(sourceRelationId, columnAttrNumber))
 		{
 			DropDefaultColumnDefinition(sourceRelationId, columnName);
+		}
 
-		/* column might not own a sequence */
+		/*
+		 * Column might own a sequence without having a nextval() expr on it
+		 * --e.g., due to ALTER SEQUENCE OWNED BY .. --, so check if that is
+		 * the case even if the column doesn't have a DEFAULT.
+		 */
 		if (OidIsValid(ownedSequenceId))
 		{
 			TransferSequenceOwnership(ownedSequenceId, targetRelationId, columnName);

src/backend/distributed/commands/create_distributed_table.c

@@ -382,7 +382,6 @@ CreateDistributedTableConcurrently(Oid relationId, char *distributionColumnName,
 							   "citus.shard_replication_factor > 1")));
 	}
 
-	EnsureCoordinatorIsInMetadata();
 	EnsureCitusTableCanBeCreated(relationId);
 
 	EnsureValidDistributionColumn(relationId, distributionColumnName);
@@ -528,6 +527,14 @@ CreateDistributedTableConcurrently(Oid relationId, char *distributionColumnName,
 		colocatedTableId = ColocatedTableId(colocationId);
 	}
 
+	List *workerNodeList = DistributedTablePlacementNodeList(NoLock);
+	if (workerNodeList == NIL)
+	{
+		ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+						errmsg("no worker nodes are available for placing shards"),
+						errhint("Add more worker nodes.")));
+	}
+
 	List *workersForPlacementList;
 	List *shardSplitPointsList;
 
@@ -555,7 +562,6 @@ CreateDistributedTableConcurrently(Oid relationId, char *distributionColumnName,
 		/*
 		 * Place shards in a round-robin fashion across all data nodes.
 		 */
-		List *workerNodeList = DistributedTablePlacementNodeList(NoLock);
 		workersForPlacementList = RoundRobinWorkerNodeList(workerNodeList, shardCount);
 	}
 
@@ -856,6 +862,8 @@ WorkerNodesForShardList(List *shardList)
 static List *
 RoundRobinWorkerNodeList(List *workerNodeList, int listLength)
 {
+	Assert(workerNodeList != NIL);
+
 	List *nodeIdList = NIL;
 
 	for (int idx = 0; idx < listLength; idx++)

src/backend/distributed/commands/foreign_constraint.c

@@ -23,12 +23,14 @@
 #include "catalog/pg_type.h"
 #include "distributed/colocation_utils.h"
 #include "distributed/commands.h"
+#include "distributed/commands/sequence.h"
 #include "distributed/coordinator_protocol.h"
 #include "distributed/listutils.h"
 #include "distributed/coordinator_protocol.h"
 #include "distributed/multi_join_order.h"
 #include "distributed/namespace_utils.h"
 #include "distributed/reference_table_utils.h"
+#include "distributed/utils/array_type.h"
 #include "distributed/version_compat.h"
 #include "miscadmin.h"
 #include "utils/builtins.h"
@@ -57,6 +59,8 @@ typedef bool (*CheckRelationFunc)(Oid);
 /* Local functions forward declarations */
 static void EnsureReferencingTableNotReplicated(Oid referencingTableId);
 static void EnsureSupportedFKeyOnDistKey(Form_pg_constraint constraintForm);
+static bool ForeignKeySetsNextValColumnToDefault(HeapTuple pgConstraintTuple);
+static List * ForeignKeyGetDefaultingAttrs(HeapTuple pgConstraintTuple);
 static void EnsureSupportedFKeyBetweenCitusLocalAndRefTable(Form_pg_constraint
 															constraintForm,
 															char
@@ -256,6 +260,23 @@ ErrorIfUnsupportedForeignConstraintExists(Relation relation, char referencingDis
 			referencedReplicationModel = referencingReplicationModel;
 		}
 
+		/*
+		 * Given that we drop DEFAULT nextval('sequence') expressions from
+		 * shard relation columns, allowing ON DELETE/UPDATE SET DEFAULT
+		 * on such columns causes inserting NULL values to referencing relation
+		 * as a result of a delete/update operation on referenced relation.
+		 *
+		 * For this reason, we disallow ON DELETE/UPDATE SET DEFAULT actions
+		 * on columns that default to sequences.
+		 */
+		if (ForeignKeySetsNextValColumnToDefault(heapTuple))
+		{
+			ereport(ERROR, (errmsg("cannot create foreign key constraint "
+								   "since Citus does not support ON DELETE "
+								   "/ UPDATE SET DEFAULT actions on the "
+								   "columns that default to sequences")));
+		}
+
 		bool referencingIsCitusLocalOrRefTable =
 			(referencingDistMethod == DISTRIBUTE_BY_NONE);
 		bool referencedIsCitusLocalOrRefTable =
@@ -358,6 +379,104 @@ ErrorIfUnsupportedForeignConstraintExists(Relation relation, char referencingDis
 }
 
 
+/*
+ * ForeignKeySetsNextValColumnToDefault returns true if at least one of the
+ * columns specified in ON DELETE / UPDATE SET DEFAULT clauses default to
+ * nextval().
+ */
+static bool
+ForeignKeySetsNextValColumnToDefault(HeapTuple pgConstraintTuple)
+{
+	Form_pg_constraint pgConstraintForm =
+		(Form_pg_constraint) GETSTRUCT(pgConstraintTuple);
+
+	List *setDefaultAttrs = ForeignKeyGetDefaultingAttrs(pgConstraintTuple);
+	AttrNumber setDefaultAttr = InvalidAttrNumber;
+	foreach_int(setDefaultAttr, setDefaultAttrs)
+	{
+		if (ColumnDefaultsToNextVal(pgConstraintForm->conrelid, setDefaultAttr))
+		{
+			return true;
+		}
+	}
+
+	return false;
+}
+
+
+/*
+ * ForeignKeyGetDefaultingAttrs returns a list of AttrNumbers
+ * might be set to default ON DELETE or ON UPDATE.
+ *
+ * For example; if the foreign key has SET DEFAULT clause for
+ * both actions, then returns a superset of the attributes that
+ * might be set to DEFAULT on either of those actions.
+ */
+static List *
+ForeignKeyGetDefaultingAttrs(HeapTuple pgConstraintTuple)
+{
+	bool isNull = false;
+	Datum referencingColumnsDatum = SysCacheGetAttr(CONSTROID, pgConstraintTuple,
+													Anum_pg_constraint_conkey, &isNull);
+	if (isNull)
+	{
+		ereport(ERROR, (errmsg("got NULL conkey from catalog")));
+	}
+
+	List *referencingColumns =
+		IntegerArrayTypeToList(DatumGetArrayTypeP(referencingColumnsDatum));
+
+	Form_pg_constraint pgConstraintForm =
+		(Form_pg_constraint) GETSTRUCT(pgConstraintTuple);
+	if (pgConstraintForm->confupdtype == FKCONSTR_ACTION_SETDEFAULT)
+	{
+		/*
+		 * Postgres doesn't allow specifying SET DEFAULT for a subset of
+		 * the referencing columns for ON UPDATE action, so in that case
+		 * we return all referencing columns regardless of what ON DELETE
+		 * action says.
+		 */
+		return referencingColumns;
+	}
+
+	if (pgConstraintForm->confdeltype != FKCONSTR_ACTION_SETDEFAULT)
+	{
+		return NIL;
+	}
+
+	List *onDeleteSetDefColumnList = NIL;
+#if PG_VERSION_NUM >= PG_VERSION_15
+	Datum onDeleteSetDefColumnsDatum = SysCacheGetAttr(CONSTROID, pgConstraintTuple,
+													   Anum_pg_constraint_confdelsetcols,
+													   &isNull);
+
+	/*
+	 * confdelsetcols being NULL means that "ON DELETE SET DEFAULT" doesn't
+	 * specify which subset of columns should be set to DEFAULT, so fetching
+	 * NULL from the catalog is also possible.
+	 */
+	if (!isNull)
+	{
+		onDeleteSetDefColumnList =
+			IntegerArrayTypeToList(DatumGetArrayTypeP(onDeleteSetDefColumnsDatum));
+	}
+#endif
+
+	if (list_length(onDeleteSetDefColumnList) == 0)
+	{
+		/*
+		 * That means that all referencing columns need to be set to
+		 * DEFAULT.
+		 */
+		return referencingColumns;
+	}
+	else
+	{
+		return onDeleteSetDefColumnList;
+	}
+}
+
+
 /*
  * EnsureSupportedFKeyBetweenCitusLocalAndRefTable is a helper function that
  * takes a foreign key constraint form for a foreign key between two citus
@@ -1192,19 +1311,6 @@ IsTableTypeIncluded(Oid relationId, int flags)
 }
 
 
-/*
- * EnableSkippingConstraintValidation is simply a C interface for setting the following:
- *      SET LOCAL citus.skip_constraint_validation TO on;
- */
-void
-EnableSkippingConstraintValidation()
-{
-	set_config_option("citus.skip_constraint_validation", "true",
-					  PGC_SUSET, PGC_S_SESSION,
-					  GUC_ACTION_LOCAL, true, 0, false);
-}
-
-
 /*
  * RelationInvolvedInAnyNonInheritedForeignKeys returns true if relation involved
  * in a foreign key that is not inherited from its parent relation.

src/backend/distributed/commands/role.c

@@ -354,17 +354,23 @@ ExtractEncryptedPassword(Oid roleOid)
 
 	Datum passwordDatum = heap_getattr(tuple, Anum_pg_authid_rolpassword,
 									   pgAuthIdDescription, &isNull);
-	char *passwordCstring = TextDatumGetCString(passwordDatum);
+
+	/*
+	 * In PG, an empty password is treated the same as NULL.
+	 * So we propagate NULL password to the other nodes, even if
+	 * the user supplied an empty password
+	 */
+
+	char *passwordCstring = NULL;
+	if (!isNull)
+	{
+		passwordCstring = pstrdup(TextDatumGetCString(passwordDatum));
+	}
 
 	table_close(pgAuthId, AccessShareLock);
 	ReleaseSysCache(tuple);
 
-	if (isNull)
-	{
-		return NULL;
-	}
-
-	return pstrdup(passwordCstring);
+	return passwordCstring;
 }
 
 

src/backend/distributed/commands/sequence.c

@@ -27,6 +27,7 @@
 #include "nodes/makefuncs.h"
 #include "distributed/worker_create_or_replace.h"
 #include "nodes/parsenodes.h"
+#include "rewrite/rewriteHandler.h"
 #include "utils/builtins.h"
 #include "utils/lsyscache.h"
 
@@ -213,6 +214,29 @@ ExtractDefaultColumnsAndOwnedSequences(Oid relationId, List **columnNameList,
 }
 
 
+/*
+ * ColumnDefaultsToNextVal returns true if the column with attrNumber
+ * has a default expression that contains nextval().
+ */
+bool
+ColumnDefaultsToNextVal(Oid relationId, AttrNumber attrNumber)
+{
+	AssertArg(AttributeNumberIsValid(attrNumber));
+
+	Relation relation = RelationIdGetRelation(relationId);
+	Node *defExpr = build_column_default(relation, attrNumber);
+	RelationClose(relation);
+
+	if (defExpr == NULL)
+	{
+		/* column doesn't have a DEFAULT expression */
+		return false;
+	}
+
+	return contain_nextval_expression_walker(defExpr, NULL);
+}
+
+
 /*
  * PreprocessDropSequenceStmt gets called during the planning phase of a DROP SEQUENCE statement
  * and returns a list of DDLJob's that will drop any distributed sequences from the

src/backend/distributed/commands/table.c

@@ -40,6 +40,7 @@
 #include "distributed/resource_lock.h"
 #include "distributed/version_compat.h"
 #include "distributed/worker_shard_visibility.h"
+#include "foreign/foreign.h"
 #include "lib/stringinfo.h"
 #include "nodes/parsenodes.h"
 #include "parser/parse_expr.h"
@@ -117,6 +118,8 @@ static char * GetAlterColumnWithNextvalDefaultCmd(Oid sequenceOid, Oid relationI
 												  char *colname);
 static char * GetAddColumnWithNextvalDefaultCmd(Oid sequenceOid, Oid relationId,
 												char *colname, TypeName *typeName);
+static void ErrorIfAlterTableDropTableNameFromPostgresFdw(List *optionList, Oid
+														  relationId);
 
 
 /*
@@ -1875,7 +1878,8 @@ PreprocessAlterTableSchemaStmt(Node *node, const char *queryString,
  * ALTER TABLE ... ADD FOREIGN KEY command to skip the validation step.
  */
 void
-SkipForeignKeyValidationIfConstraintIsFkey(AlterTableStmt *alterTableStatement)
+SkipForeignKeyValidationIfConstraintIsFkey(AlterTableStmt *alterTableStatement,
+										   bool processLocalRelation)
 {
 	/* first check whether a distributed relation is affected */
 	if (alterTableStatement->relation == NULL)
@@ -1890,11 +1894,17 @@ SkipForeignKeyValidationIfConstraintIsFkey(AlterTableStmt *alterTableStatement)
 		return;
 	}
 
-	if (!IsCitusTable(leftRelationId))
+	if (!IsCitusTable(leftRelationId) && !processLocalRelation)
 	{
 		return;
 	}
 
+	/*
+	 * We check if there is a ADD FOREIGN CONSTRAINT command in sub commands
+	 * list. We set skip_validation to true to prevent PostgreSQL to verify
+	 * validity of the foreign constraint. Validity will be checked on the
+	 * shards anyway.
+	 */
 	AlterTableCmd *command = NULL;
 	foreach_ptr(command, alterTableStatement->cmds)
 	{
@@ -1906,9 +1916,8 @@ SkipForeignKeyValidationIfConstraintIsFkey(AlterTableStmt *alterTableStatement)
 			Constraint *constraint = (Constraint *) command->def;
 			if (constraint->contype == CONSTR_FOREIGN)
 			{
-				/* set the GUC skip_constraint_validation to on */
-				EnableSkippingConstraintValidation();
-				return;
+				/* foreign constraint validations will be done in shards. */
+				constraint->skip_validation = true;
 			}
 		}
 	}
@@ -2643,6 +2652,42 @@ ErrorIfUnsupportedConstraint(Relation relation, char distributionMethod,
 }
 
 
+/*
+ * ErrorIfAlterTableDropTableNameFromPostgresFdw errors if given alter foreign table
+ * option list drops 'table_name' from a postgresfdw foreign table which is
+ * inside metadata.
+ */
+static void
+ErrorIfAlterTableDropTableNameFromPostgresFdw(List *optionList, Oid relationId)
+{
+	char relationKind PG_USED_FOR_ASSERTS_ONLY =
+		get_rel_relkind(relationId);
+	Assert(relationKind == RELKIND_FOREIGN_TABLE);
+
+	ForeignTable *foreignTable = GetForeignTable(relationId);
+	Oid serverId = foreignTable->serverid;
+	if (!ServerUsesPostgresFdw(serverId))
+	{
+		return;
+	}
+
+	if (IsCitusTableType(relationId, CITUS_LOCAL_TABLE) &&
+		ForeignTableDropsTableNameOption(optionList))
+	{
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg(
+					 "alter foreign table alter options (drop table_name) command "
+					 "is not allowed for Citus tables"),
+				 errdetail(
+					 "Table_name option can not be dropped from a foreign table "
+					 "which is inside metadata."),
+				 errhint(
+					 "Try to undistribute foreign table before dropping table_name option.")));
+	}
+}
+
+
 /*
  * ErrorIfUnsupportedAlterTableStmt checks if the corresponding alter table
  * statement is supported for distributed tables and errors out if it is not.
@@ -3031,6 +3076,8 @@ ErrorIfUnsupportedAlterTableStmt(AlterTableStmt *alterTableStatement)
 			{
 				if (IsForeignTable(relationId))
 				{
+					List *optionList = (List *) command->def;
+					ErrorIfAlterTableDropTableNameFromPostgresFdw(optionList, relationId);
 					break;
 				}
 			}

src/backend/distributed/commands/trigger.c

@@ -43,6 +43,7 @@
 
 
 /* local function forward declarations */
+static char * GetAlterTriggerStateCommand(Oid triggerId);
 static bool IsCreateCitusTruncateTriggerStmt(CreateTrigStmt *createTriggerStmt);
 static String * GetAlterTriggerDependsTriggerNameValue(AlterObjectDependsStmt *
 													   alterTriggerDependsStmt);
@@ -99,6 +100,18 @@ GetExplicitTriggerCommandList(Oid relationId)
 		createTriggerCommandList = lappend(
 			createTriggerCommandList,
 			makeTableDDLCommandString(createTriggerCommand));
+
+		/*
+		 * Appends the commands for the trigger settings that are not covered
+		 * by CREATE TRIGGER command, such as ALTER TABLE ENABLE/DISABLE <trigger>.
+		 */
+
+		char *alterTriggerStateCommand =
+			GetAlterTriggerStateCommand(triggerId);
+
+		createTriggerCommandList = lappend(
+			createTriggerCommandList,
+			makeTableDDLCommandString(alterTriggerStateCommand));
 	}
 
 	/* revert back to original search_path */
@@ -108,6 +121,72 @@ GetExplicitTriggerCommandList(Oid relationId)
 }
 
 
+/*
+ * GetAlterTriggerStateCommand returns the DDL command to set enable/disable
+ * state for given trigger. Throws an error if no such trigger exists.
+ */
+static char *
+GetAlterTriggerStateCommand(Oid triggerId)
+{
+	StringInfo alterTriggerStateCommand = makeStringInfo();
+
+	bool missingOk = false;
+	HeapTuple triggerTuple = GetTriggerTupleById(triggerId, missingOk);
+
+	Form_pg_trigger triggerForm = (Form_pg_trigger) GETSTRUCT(triggerTuple);
+
+	char *qualifiedRelName = generate_qualified_relation_name(triggerForm->tgrelid);
+	const char *quotedTrigName = quote_identifier(NameStr(triggerForm->tgname));
+	char enableDisableState = triggerForm->tgenabled;
+
+	const char *alterTriggerStateStr = NULL;
+	switch (enableDisableState)
+	{
+		case TRIGGER_FIRES_ON_ORIGIN:
+		{
+			/* default mode */
+			alterTriggerStateStr = "ENABLE";
+			break;
+		}
+
+		case TRIGGER_FIRES_ALWAYS:
+		{
+			alterTriggerStateStr = "ENABLE ALWAYS";
+			break;
+		}
+
+		case TRIGGER_FIRES_ON_REPLICA:
+		{
+			alterTriggerStateStr = "ENABLE REPLICA";
+			break;
+		}
+
+		case TRIGGER_DISABLED:
+		{
+			alterTriggerStateStr = "DISABLE";
+			break;
+		}
+
+		default:
+		{
+			elog(ERROR, "unexpected trigger state");
+		}
+	}
+
+	appendStringInfo(alterTriggerStateCommand, "ALTER TABLE %s %s TRIGGER %s;",
+					 qualifiedRelName, alterTriggerStateStr, quotedTrigName);
+
+	/*
+	 * Free triggerTuple at the end since quote_identifier() might not return
+	 * a palloc'd string if given identifier doesn't need to be quoted, and in
+	 * that case quotedTrigName would still be bound to triggerTuple.
+	 */
+	heap_freetuple(triggerTuple);
+
+	return alterTriggerStateCommand->data;
+}
+
+
 /*
  * GetTriggerTupleById returns copy of the heap tuple from pg_trigger for
  * the trigger with triggerId. If no such trigger exists, this function returns

src/backend/distributed/commands/utility_hook.c

@@ -117,9 +117,6 @@ static void DecrementUtilityHookCountersIfNecessary(Node *parsetree);
 static bool IsDropSchemaOrDB(Node *parsetree);
 static bool ShouldCheckUndistributeCitusLocalTables(void);
 static bool ShouldAddNewTableToMetadata(Node *parsetree);
-static bool ServerUsesPostgresFDW(char *serverName);
-static void ErrorIfOptionListHasNoTableName(List *optionList);
-
 
 /*
  * ProcessUtilityParseTree is a convenience method to create a PlannedStmt out of
@@ -378,7 +375,6 @@ ProcessUtilityInternal(PlannedStmt *pstmt,
 	Node *parsetree = pstmt->utilityStmt;
 	List *ddlJobs = NIL;
 	DistOpsValidationState distOpsValidationState = HasNoneValidObject;
-	bool oldSkipConstraintsValidationValue = SkipConstraintValidation;
 
 	if (IsA(parsetree, ExplainStmt) &&
 		IsA(((ExplainStmt *) parsetree)->query, Query))
@@ -597,7 +593,9 @@ ProcessUtilityInternal(PlannedStmt *pstmt,
 		 * Citus intervening. The only exception is partition column drop, in
 		 * which case we error out. Advanced Citus users use this to implement their
 		 * own DDL propagation. We also use it to avoid re-propagating DDL commands
-		 * when changing MX tables on workers.
+		 * when changing MX tables on workers. Below, we also make sure that DDL
+		 * commands don't run queries that might get intercepted by Citus and error
+		 * out during planning, specifically we skip validation in foreign keys.
 		 */
 
 		if (IsA(parsetree, AlterTableStmt))
@@ -616,7 +614,33 @@ ProcessUtilityInternal(PlannedStmt *pstmt,
 				 * Note validation is done on the shard level when DDL propagation
 				 * is enabled. The following eagerly executes some tasks on workers.
 				 */
-				SkipForeignKeyValidationIfConstraintIsFkey(alterTableStmt);
+				SkipForeignKeyValidationIfConstraintIsFkey(alterTableStmt, false);
+			}
+		}
+	}
+
+	/*
+	 * If we've explicitly set the citus.skip_constraint_validation GUC, then
+	 * we skip validation of any added constraints.
+	 */
+	if (IsA(parsetree, AlterTableStmt) && SkipConstraintValidation)
+	{
+		AlterTableStmt *alterTableStmt = (AlterTableStmt *) parsetree;
+		AlterTableCmd *command = NULL;
+		foreach_ptr(command, alterTableStmt->cmds)
+		{
+			AlterTableType alterTableType = command->subtype;
+
+			/*
+			 * XXX: In theory we could probably use this GUC to skip validation
+			 * of VALIDATE CONSTRAINT and ALTER CONSTRAINT too. But currently
+			 * this is not needed, so we make its behaviour only apply to ADD
+			 * CONSTRAINT.
+			 */
+			if (alterTableType == AT_AddConstraint)
+			{
+				Constraint *constraint = (Constraint *) command->def;
+				constraint->skip_validation = true;
 			}
 		}
 	}
@@ -787,18 +811,6 @@ ProcessUtilityInternal(PlannedStmt *pstmt,
 
 		CreateStmt *createTableStmt = (CreateStmt *) (&createForeignTableStmt->base);
 
-		/*
-		 * Error out with a hint if the foreign table is using postgres_fdw and
-		 * the option table_name is not provided.
-		 * Citus relays all the Citus local foreign table logic to the placement of the
-		 * Citus local table. If table_name is NOT provided, Citus would try to talk to
-		 * the foreign postgres table over the shard's table name, which would not exist
-		 * on the remote server.
-		 */
-		if (ServerUsesPostgresFDW(createForeignTableStmt->servername))
-		{
-			ErrorIfOptionListHasNoTableName(createForeignTableStmt->options);
-		}
 
 		PostprocessCreateTableStmt(createTableStmt, queryString);
 	}
@@ -902,8 +914,6 @@ ProcessUtilityInternal(PlannedStmt *pstmt,
 		 */
 		CitusHasBeenLoaded(); /* lgtm[cpp/return-value-ignored] */
 	}
-
-	SkipConstraintValidation = oldSkipConstraintsValidationValue;
 }
 
 
@@ -1088,50 +1098,6 @@ ShouldAddNewTableToMetadata(Node *parsetree)
 }
 
 
-/*
- * ServerUsesPostgresFDW gets a foreign server name and returns true if the FDW that
- * the server depends on is postgres_fdw. Returns false otherwise.
- */
-static bool
-ServerUsesPostgresFDW(char *serverName)
-{
-	ForeignServer *server = GetForeignServerByName(serverName, false);
-	ForeignDataWrapper *fdw = GetForeignDataWrapper(server->fdwid);
-
-	if (strcmp(fdw->fdwname, "postgres_fdw") == 0)
-	{
-		return true;
-	}
-
-	return false;
-}
-
-
-/*
- * ErrorIfOptionListHasNoTableName gets an option list (DefElem) and errors out
- * if the list does not contain a table_name element.
- */
-static void
-ErrorIfOptionListHasNoTableName(List *optionList)
-{
-	char *table_nameString = "table_name";
-	DefElem *option = NULL;
-	foreach_ptr(option, optionList)
-	{
-		char *optionName = option->defname;
-		if (strcmp(optionName, table_nameString) == 0)
-		{
-			return;
-		}
-	}
-
-	ereport(ERROR, (errmsg(
-						"table_name option must be provided when using postgres_fdw with Citus"),
-					errhint("Provide the option \"table_name\" with value target table's"
-							" name")));
-}
-
-
 /*
  * NotifyUtilityHookConstraintDropped sets ConstraintDropped to true to tell us
  * last command dropped a table constraint.

src/backend/distributed/connection/connection_management.c

@@ -16,7 +16,7 @@
 #include "miscadmin.h"
 
 #include "safe_lib.h"
-
+#include "postmaster/postmaster.h"
 #include "access/hash.h"
 #include "commands/dbcommands.h"
 #include "distributed/backend_data.h"
@@ -63,7 +63,6 @@ static void FreeConnParamsHashEntryFields(ConnParamsHashEntry *entry);
 static void AfterXactHostConnectionHandling(ConnectionHashEntry *entry, bool isCommit);
 static bool ShouldShutdownConnection(MultiConnection *connection, const int
 									 cachedConnectionCount);
-static void ResetConnection(MultiConnection *connection);
 static bool RemoteTransactionIdle(MultiConnection *connection);
 static int EventSetSizeForConnectionList(List *connections);
 
@@ -244,6 +243,23 @@ GetNodeUserDatabaseConnection(uint32 flags, const char *hostname, int32 port,
 }
 
 
+/*
+ * GetConnectionForLocalQueriesOutsideTransaction returns a localhost connection for
+ * subtransaction. To avoid creating excessive connections, we reuse an
+ * existing connection.
+ */
+MultiConnection *
+GetConnectionForLocalQueriesOutsideTransaction(char *userName)
+{
+	int connectionFlag = OUTSIDE_TRANSACTION;
+	MultiConnection *connection =
+		GetNodeUserDatabaseConnection(connectionFlag, LocalHostName, PostPortNumber,
+									  userName, get_database_name(MyDatabaseId));
+
+	return connection;
+}
+
+
 /*
  * StartNodeUserDatabaseConnection() initiates a connection to a remote node.
  *
@@ -688,8 +704,8 @@ CloseConnection(MultiConnection *connection)
 		dlist_delete(&connection->connectionNode);
 
 		/* same for transaction state and shard/placement machinery */
-		CloseRemoteTransaction(connection);
 		CloseShardPlacementAssociation(connection);
+		ResetRemoteTransaction(connection);
 
 		/* we leave the per-host entry alive */
 		pfree(connection);
@@ -1433,6 +1449,9 @@ AfterXactHostConnectionHandling(ConnectionHashEntry *entry, bool isCommit)
 		{
 			ShutdownConnection(connection);
 
+			/* remove from transactionlist before free-ing */
+			ResetRemoteTransaction(connection);
+
 			/* unlink from list */
 			dlist_delete(iter.cur);
 
@@ -1443,7 +1462,10 @@ AfterXactHostConnectionHandling(ConnectionHashEntry *entry, bool isCommit)
 			/*
 			 * reset healthy session lifespan connections.
 			 */
-			ResetConnection(connection);
+			ResetRemoteTransaction(connection);
+
+			UnclaimConnection(connection);
+
 
 			cachedConnectionCount++;
 		}
@@ -1482,24 +1504,6 @@ ShouldShutdownConnection(MultiConnection *connection, const int cachedConnection
 }
 
 
-/*
- * ResetConnection preserves the given connection for later usage by
- * resetting its states.
- */
-static void
-ResetConnection(MultiConnection *connection)
-{
-	/* reset per-transaction state */
-	ResetRemoteTransaction(connection);
-	ResetShardPlacementAssociation(connection);
-
-	/* reset copy state */
-	connection->copyBytesWrittenSinceLastFlush = 0;
-
-	UnclaimConnection(connection);
-}
-
-
 /*
  * RemoteTransactionIdle function returns true if we manually
  * set flag on run_commands_on_session_level_connection_to_node to true to

src/backend/distributed/deparser/ruleutils_14.c

@@ -482,7 +482,7 @@ get_merged_argument_list(CallStmt *stmt, List **mergedNamedArgList,
 	Oid  functionOid = stmt->funcexpr->funcid;
 	List *namedArgList = NIL;
 	List *finalArgumentList = NIL;
-	Oid  finalArgTypes[FUNC_MAX_ARGS];
+	Oid  *finalArgTypes;
 	Oid  *argTypes = NULL;
 	char *argModes = NULL;
 	char **argNames = NULL;
@@ -519,6 +519,7 @@ get_merged_argument_list(CallStmt *stmt, List **mergedNamedArgList,
 
 	/* Remove the duplicate INOUT counting */
 	numberOfArgs = numberOfArgs - totalInoutArgs;
+	finalArgTypes = palloc0(sizeof(Oid) * numberOfArgs);
 
 	ListCell *inArgCell = list_head(stmt->funcexpr->args);
 	ListCell *outArgCell = list_head(stmt->outargs);

src/backend/distributed/deparser/ruleutils_15.c

@@ -491,7 +491,7 @@ get_merged_argument_list(CallStmt *stmt, List **mergedNamedArgList,
 	Oid  functionOid = stmt->funcexpr->funcid;
 	List *namedArgList = NIL;
 	List *finalArgumentList = NIL;
-	Oid  finalArgTypes[FUNC_MAX_ARGS];
+	Oid  *finalArgTypes;
 	Oid  *argTypes = NULL;
 	char *argModes = NULL;
 	char **argNames = NULL;
@@ -528,6 +528,7 @@ get_merged_argument_list(CallStmt *stmt, List **mergedNamedArgList,
 
 	/* Remove the duplicate INOUT counting */
 	numberOfArgs = numberOfArgs - totalInoutArgs;
+	finalArgTypes = palloc0(sizeof(Oid) * numberOfArgs);
 
 	ListCell *inArgCell = list_head(stmt->funcexpr->args);
 	ListCell *outArgCell = list_head(stmt->outargs);
@@ -610,6 +611,7 @@ get_merged_argument_list(CallStmt *stmt, List **mergedNamedArgList,
 
 	return true;
 }
+
 /*
  * pg_get_rule_expr deparses an expression and returns the result as a string.
  */

src/backend/distributed/executor/adaptive_executor.c

@@ -4558,6 +4558,7 @@ ReceiveResults(WorkerSession *session, bool storeRows)
 		TupleDesc tupleDescriptor = tupleDest->tupleDescForQuery(tupleDest, queryIndex);
 		if (tupleDescriptor == NULL)
 		{
+			PQclear(result);
 			continue;
 		}
 

src/backend/distributed/executor/multi_executor.c

@@ -861,11 +861,6 @@ AlterTableConstraintCheck(QueryDesc *queryDesc)
 		return false;
 	}
 
-	if (SkipConstraintValidation)
-	{
-		return true;
-	}
-
 	/*
 	 * While an ALTER TABLE is in progress, we might do SELECTs on some
 	 * catalog tables too. For example, when dropping a column, citus_drop_trigger()

src/backend/distributed/metadata/metadata_utility.c

@@ -985,7 +985,7 @@ AppendShardSizeQuery(StringInfo selectQuery, ShardInterval *shardInterval)
 
 	appendStringInfo(selectQuery, "SELECT " UINT64_FORMAT " AS shard_id, ", shardId);
 	appendStringInfo(selectQuery, "%s AS shard_name, ", quotedShardName);
-	appendStringInfo(selectQuery, PG_RELATION_SIZE_FUNCTION, quotedShardName);
+	appendStringInfo(selectQuery, PG_TOTAL_RELATION_SIZE_FUNCTION, quotedShardName);
 }
 
 
@@ -4022,7 +4022,7 @@ CancelTasksForJob(int64 jobid)
 		}
 
 		/* make sure the current user has the rights to cancel this task */
-		Oid taskOwner = DatumGetObjectId(values[Anum_pg_dist_background_task_owner]);
+		Oid taskOwner = DatumGetObjectId(values[Anum_pg_dist_background_task_owner - 1]);
 		if (superuser_arg(taskOwner) && !superuser())
 		{
 			/* must be a superuser to cancel tasks owned by superuser */

src/backend/distributed/operations/shard_cleaner.c

@@ -303,21 +303,40 @@ DropOrphanedShardsForCleanup()
 										   workerNode->workerName,
 										   workerNode->workerPort))
 		{
+			if (record->policy == CLEANUP_DEFERRED_ON_SUCCESS)
+			{
+				ereport(LOG, (errmsg("deferred drop of orphaned shard %s on %s:%d "
+									 "completed",
+									 qualifiedTableName,
+									 workerNode->workerName, workerNode->workerPort)));
+			}
+			else
+			{
+				ereport(LOG, (errmsg("cleaned up orphaned shard %s on %s:%d which "
+									 "was left behind after a failed operation",
+									 qualifiedTableName,
+									 workerNode->workerName, workerNode->workerPort)));
+			}
+
 			/* delete the cleanup record */
 			DeleteCleanupRecordByRecordId(record->recordId);
 			removedShardCountForCleanup++;
 		}
 		else
 		{
+			/*
+			 * We log failures at the end, since they occur repeatedly
+			 * for a large number of objects.
+			 */
 			failedShardCountForCleanup++;
 		}
 	}
 
 	if (failedShardCountForCleanup > 0)
 	{
-		ereport(WARNING, (errmsg("Failed to cleanup %d shards out of %d",
-								 failedShardCountForCleanup, list_length(
-									 cleanupRecordList))));
+		ereport(WARNING, (errmsg("failed to clean up %d orphaned shards out of %d",
+								 failedShardCountForCleanup,
+								 list_length(cleanupRecordList))));
 	}
 
 	return removedShardCountForCleanup;
@@ -396,19 +415,29 @@ DropOrphanedShardsForMove(bool waitForLocks)
 										   shardPlacement->nodeName,
 										   shardPlacement->nodePort))
 		{
+			ereport(LOG, (errmsg("deferred drop of orphaned shard %s on %s:%d "
+								 "after a move completed",
+								 qualifiedTableName,
+								 shardPlacement->nodeName,
+								 shardPlacement->nodePort)));
+
 			/* delete the actual placement */
 			DeleteShardPlacementRow(placement->placementId);
 			removedShardCount++;
 		}
 		else
 		{
+			/*
+			 * We log failures at the end, since they occur repeatedly
+			 * for a large number of objects.
+			 */
 			failedShardDropCount++;
 		}
 	}
 
 	if (failedShardDropCount > 0)
 	{
-		ereport(WARNING, (errmsg("Failed to drop %d orphaned shards out of %d",
+		ereport(WARNING, (errmsg("failed to clean up %d orphaned shards out of %d",
 								 failedShardDropCount, list_length(shardPlacementList))));
 	}
 
@@ -436,7 +465,7 @@ RegisterOperationNeedingCleanup(void)
  * completion with failure. This will trigger cleanup of appropriate resources.
  */
 void
-FinalizeOperationNeedingCleanupOnFailure()
+FinalizeOperationNeedingCleanupOnFailure(const char *operationName)
 {
 	/* We must have a valid OperationId. Any operation requring cleanup
 	 * will call RegisterOperationNeedingCleanup.
@@ -454,7 +483,8 @@ FinalizeOperationNeedingCleanupOnFailure()
 		/* We only supporting cleaning shards right now */
 		if (record->objectType != CLEANUP_OBJECT_SHARD_PLACEMENT)
 		{
-			ereport(WARNING, (errmsg("Invalid object type %d for cleanup record ",
+			ereport(WARNING, (errmsg(
+								  "Invalid object type %d on failed operation cleanup",
 								  record->objectType)));
 			continue;
 		}
@@ -473,6 +503,12 @@ FinalizeOperationNeedingCleanupOnFailure()
 											   workerNode->workerName,
 											   workerNode->workerPort))
 			{
+				ereport(LOG, (errmsg("cleaned up orphaned shard %s on %s:%d after a "
+									 "%s operation failed",
+									 qualifiedTableName,
+									 workerNode->workerName, workerNode->workerPort,
+									 operationName)));
+
 				/*
 				 * Given the operation is failing and we will abort its transaction, we cannot delete
 				 * records in the current transaction. Delete these records outside of the
@@ -483,23 +519,22 @@ FinalizeOperationNeedingCleanupOnFailure()
 			}
 			else
 			{
+				/*
+				 * We log failures at the end, since they occur repeatedly
+				 * for a large number of objects.
+				 */
 				failedShardCountOnComplete++;
 			}
 		}
 	}
 
-	if (list_length(currentOperationRecordList) > 0)
-	{
-		ereport(LOG, (errmsg("Removed %d orphaned shards out of %d",
-							 removedShardCountOnComplete, list_length(
-								 currentOperationRecordList))));
-
 	if (failedShardCountOnComplete > 0)
 	{
-			ereport(WARNING, (errmsg("Failed to cleanup %d shards out of %d",
-									 failedShardCountOnComplete, list_length(
-										 currentOperationRecordList))));
-		}
+		ereport(WARNING, (errmsg("failed to clean up %d orphaned shards out of %d after "
+								 "a %s operation failed",
+								 failedShardCountOnComplete,
+								 list_length(currentOperationRecordList),
+								 operationName)));
 	}
 }
 
@@ -509,7 +544,7 @@ FinalizeOperationNeedingCleanupOnFailure()
  * completion with success. This will trigger cleanup of appropriate resources.
  */
 void
-FinalizeOperationNeedingCleanupOnSuccess()
+FinalizeOperationNeedingCleanupOnSuccess(const char *operationName)
 {
 	/* We must have a valid OperationId. Any operation requring cleanup
 	 * will call RegisterOperationNeedingCleanup.
@@ -527,7 +562,8 @@ FinalizeOperationNeedingCleanupOnSuccess()
 		/* We only supporting cleaning shards right now */
 		if (record->objectType != CLEANUP_OBJECT_SHARD_PLACEMENT)
 		{
-			ereport(WARNING, (errmsg("Invalid object type %d for cleanup record ",
+			ereport(WARNING, (errmsg(
+								  "Invalid object type %d on operation cleanup",
 								  record->objectType)));
 			continue;
 		}
@@ -546,6 +582,12 @@ FinalizeOperationNeedingCleanupOnSuccess()
 											   workerNode->workerName,
 											   workerNode->workerPort))
 			{
+				ereport(LOG, (errmsg("cleaned up orphaned shard %s on %s:%d after a "
+									 "%s operation completed",
+									 qualifiedTableName,
+									 workerNode->workerName, workerNode->workerPort,
+									 operationName)));
+
 				/*
 				 * Delete cleanup records outside transaction as:
 				 * The resources are marked as 'CLEANUP_ALWAYS' and should be cleaned no matter
@@ -556,6 +598,10 @@ FinalizeOperationNeedingCleanupOnSuccess()
 			}
 			else
 			{
+				/*
+				 * We log failures at the end, since they occur repeatedly
+				 * for a large number of objects.
+				 */
 				failedShardCountOnComplete++;
 			}
 		}
@@ -570,18 +616,14 @@ FinalizeOperationNeedingCleanupOnSuccess()
 		}
 	}
 
-	if (list_length(currentOperationRecordList) > 0)
-	{
-		ereport(LOG, (errmsg("Removed %d orphaned shards out of %d",
-							 removedShardCountOnComplete, list_length(
-								 currentOperationRecordList))));
-
 	if (failedShardCountOnComplete > 0)
 	{
-			ereport(WARNING, (errmsg("Failed to cleanup %d shards out of %d",
-									 failedShardCountOnComplete, list_length(
-										 currentOperationRecordList))));
-		}
+		ereport(WARNING, (errmsg(
+							  "failed to clean up %d orphaned shards out of %d after "
+							  "a %s operation completed",
+							  failedShardCountOnComplete,
+							  list_length(currentOperationRecordList),
+							  operationName)));
 	}
 }
 
@@ -670,9 +712,9 @@ InsertCleanupRecordInSubtransaction(CleanupObject objectType,
 					 nodeGroupId,
 					 policy);
 
-	SendCommandListToWorkerOutsideTransaction(LocalHostName,
-											  PostPortNumber,
-											  CitusExtensionOwnerName(),
+	MultiConnection *connection =
+		GetConnectionForLocalQueriesOutsideTransaction(CitusExtensionOwnerName());
+	SendCommandListToWorkerOutsideTransactionWithConnection(connection,
 															list_make1(command->data));
 }
 
@@ -691,9 +733,9 @@ DeleteCleanupRecordByRecordIdOutsideTransaction(uint64 recordId)
 					 PG_DIST_CLEANUP,
 					 recordId);
 
-	SendCommandListToWorkerOutsideTransaction(LocalHostName,
-											  PostPortNumber,
-											  CitusExtensionOwnerName(),
+	MultiConnection *connection = GetConnectionForLocalQueriesOutsideTransaction(
+		CitusExtensionOwnerName());
+	SendCommandListToWorkerOutsideTransactionWithConnection(connection,
 															list_make1(command->data));
 }
 
@@ -727,18 +769,11 @@ TryLockRelationAndPlacementCleanup(Oid relationId, LOCKMODE lockmode)
  * true on success.
  */
 static bool
-TryDropShardOutsideTransaction(OperationId operationId, char *qualifiedTableName,
-							   char *nodeName, int nodePort)
+TryDropShardOutsideTransaction(OperationId operationId,
+							   char *qualifiedTableName,
+							   char *nodeName,
+							   int nodePort)
 {
-	char *operation = (operationId == INVALID_OPERATION_ID) ? "move" : "cleanup";
-
-	ereport(LOG, (errmsg("cleaning up %s on %s:%d which was left "
-						 "after a %s",
-						 qualifiedTableName,
-						 nodeName,
-						 nodePort,
-						 operation)));
-
 	/* prepare sql query to execute to drop the shard */
 	StringInfo dropQuery = makeStringInfo();
 	appendStringInfo(dropQuery, DROP_REGULAR_TABLE_COMMAND, qualifiedTableName);
@@ -756,9 +791,13 @@ TryDropShardOutsideTransaction(OperationId operationId, char *qualifiedTableName
 									   dropQuery->data);
 
 	/* remove the shard from the node */
-	bool success = SendOptionalCommandListToWorkerOutsideTransaction(nodeName,
-																	 nodePort,
-																	 NULL,
+	int connectionFlags = OUTSIDE_TRANSACTION;
+	MultiConnection *workerConnection = GetNodeUserDatabaseConnection(connectionFlags,
+																	  nodeName, nodePort,
+																	  CurrentUserName(),
+																	  NULL);
+	bool success = SendOptionalCommandListToWorkerOutsideTransactionWithConnection(
+		workerConnection,
 		dropCommandList);
 
 	return success;
@@ -800,13 +839,8 @@ GetNextOperationId()
 	appendStringInfo(nextValueCommand, "SELECT nextval(%s);",
 					 quote_literal_cstr(sequenceName->data));
 
-	int connectionFlag = FORCE_NEW_CONNECTION;
-	MultiConnection *connection = GetNodeUserDatabaseConnection(connectionFlag,
-																LocalHostName,
-																PostPortNumber,
-																CitusExtensionOwnerName(),
-																get_database_name(
-																	MyDatabaseId));
+	MultiConnection *connection = GetConnectionForLocalQueriesOutsideTransaction(
+		CitusExtensionOwnerName());
 
 	PGresult *result = NULL;
 	int queryResult = ExecuteOptionalRemoteCommand(connection, nextValueCommand->data,
@@ -821,7 +855,6 @@ GetNextOperationId()
 
 	PQclear(result);
 	ForgetResults(connection);
-	CloseConnection(connection);
 
 	return operationdId;
 }

src/backend/distributed/operations/shard_rebalancer.c

@@ -1817,8 +1817,11 @@ RebalanceTableShardsBackground(RebalanceOptions *options, Oid shardReplicationMo
 	List *referenceTableIdList = NIL;
 
 	if (HasNodesWithMissingReferenceTables(&referenceTableIdList))
+	{
+		if (shardTransferMode == TRANSFER_MODE_AUTOMATIC)
 		{
 			VerifyTablesHaveReplicaIdentity(referenceTableIdList);
+		}
 
 		/*
 		 * Reference tables need to be copied to (newly-added) nodes, this needs to be the

src/backend/distributed/operations/shard_split.c

@@ -151,7 +151,7 @@ static List * ExecuteSplitShardReplicationSetupUDF(WorkerNode *sourceWorkerNode,
 												   List *destinationWorkerNodesList,
 												   DistributionColumnMap *
 												   distributionColumnOverrides);
-static void ExecuteSplitShardReleaseSharedMemory(WorkerNode *sourceWorkerNode);
+static void ExecuteSplitShardReleaseSharedMemory(MultiConnection *sourceConnection);
 static void AddDummyShardEntryInMap(HTAB *mapOfPlacementToDummyShardList, uint32
 									targetNodeId,
 									ShardInterval *shardInterval);
@@ -169,6 +169,12 @@ static const char *const SplitOperationName[] =
 	[ISOLATE_TENANT_TO_NEW_SHARD] = "isolate",
 	[CREATE_DISTRIBUTED_TABLE] = "create"
 };
+static const char *const SplitOperationAPIName[] =
+{
+	[SHARD_SPLIT_API] = "citus_split_shard_by_split_points",
+	[ISOLATE_TENANT_TO_NEW_SHARD] = "isolate_tenant_to_new_shard",
+	[CREATE_DISTRIBUTED_TABLE] = "create_distributed_table_concurrently"
+};
 static const char *const SplitTargetName[] =
 {
 	[SHARD_SPLIT_API] = "shard",
@@ -469,6 +475,8 @@ SplitShard(SplitMode splitMode,
 		   List *colocatedShardIntervalList,
 		   uint32 targetColocationId)
 {
+	const char *operationName = SplitOperationAPIName[splitOperation];
+
 	ErrorIfModificationAndSplitInTheSameTransaction(splitOperation);
 
 	ShardInterval *shardIntervalToSplit = LoadShardInterval(shardIdToSplit);
@@ -526,6 +534,8 @@ SplitShard(SplitMode splitMode,
 
 	if (splitMode == BLOCKING_SPLIT)
 	{
+		ereport(LOG, (errmsg("performing blocking %s ", operationName)));
+
 		BlockingShardSplit(
 			splitOperation,
 			splitWorkflowId,
@@ -536,6 +546,8 @@ SplitShard(SplitMode splitMode,
 	}
 	else
 	{
+		ereport(LOG, (errmsg("performing non-blocking %s ", operationName)));
+
 		NonBlockingShardSplit(
 			splitOperation,
 			splitWorkflowId,
@@ -548,7 +560,10 @@ SplitShard(SplitMode splitMode,
 		PlacementMovedUsingLogicalReplicationInTX = true;
 	}
 
-	FinalizeOperationNeedingCleanupOnSuccess();
+	/*
+	 * Drop temporary objects that were marked as CLEANUP_ALWAYS.
+	 */
+	FinalizeOperationNeedingCleanupOnSuccess(operationName);
 }
 
 
@@ -569,6 +584,8 @@ BlockingShardSplit(SplitOperation splitOperation,
 				   List *workersForPlacementList,
 				   DistributionColumnMap *distributionColumnOverrides)
 {
+	const char *operationName = SplitOperationAPIName[splitOperation];
+
 	BlockWritesToShardList(sourceColocatedShardIntervalList);
 
 	/* First create shard interval metadata for split children */
@@ -583,10 +600,14 @@ BlockingShardSplit(SplitOperation splitOperation,
 
 	PG_TRY();
 	{
+		ereport(LOG, (errmsg("creating child shards for %s", operationName)));
+
 		/* Physically create split children. */
 		CreateSplitShardsForShardGroup(shardGroupSplitIntervalListList,
 									   workersForPlacementList);
 
+		ereport(LOG, (errmsg("performing copy for %s", operationName)));
+
 		/* For Blocking split, copy isn't snapshotted */
 		char *snapshotName = NULL;
 		DoSplitCopy(sourceShardNode, sourceColocatedShardIntervalList,
@@ -596,6 +617,10 @@ BlockingShardSplit(SplitOperation splitOperation,
 		/* Used for testing */
 		ConflictOnlyWithIsolationTesting();
 
+		ereport(LOG, (errmsg(
+						  "creating auxillary structures (indexes, stats, replicaindentities, triggers) for %s",
+						  operationName)));
+
 		/* Create auxiliary structures (indexes, stats, replicaindentities, triggers) */
 		CreateAuxiliaryStructuresForShardGroup(shardGroupSplitIntervalListList,
 											   workersForPlacementList,
@@ -617,10 +642,16 @@ BlockingShardSplit(SplitOperation splitOperation,
 		 */
 		if (DeferShardDeleteOnSplit)
 		{
+			ereport(LOG, (errmsg("marking deferred cleanup of source shard(s) for %s",
+								 operationName)));
+
 			InsertDeferredDropCleanupRecordsForShards(sourceColocatedShardIntervalList);
 		}
 		else
 		{
+			ereport(LOG, (errmsg("performing cleanup of source shard(s) for %s",
+								 operationName)));
+
 			DropShardList(sourceColocatedShardIntervalList);
 		}
 
@@ -635,6 +666,9 @@ BlockingShardSplit(SplitOperation splitOperation,
 			shardGroupSplitIntervalListList,
 			workersForPlacementList);
 
+		ereport(LOG, (errmsg("creating foreign key constraints (if any) for %s",
+							 operationName)));
+
 		/*
 		 * Create foreign keys if exists after the metadata changes happening in
 		 * DropShardList() and InsertSplitChildrenShardMetadata() because the foreign
@@ -649,7 +683,7 @@ BlockingShardSplit(SplitOperation splitOperation,
 		ShutdownAllConnections();
 
 		/* Do a best effort cleanup of shards created on workers in the above block */
-		FinalizeOperationNeedingCleanupOnFailure();
+		FinalizeOperationNeedingCleanupOnFailure(operationName);
 
 		PG_RE_THROW();
 	}
@@ -670,10 +704,15 @@ CheckIfRelationWithSameNameExists(ShardInterval *shardInterval, WorkerNode *work
 	AppendShardIdToName(&shardName, shardInterval->shardId);
 
 	StringInfo checkShardExistsQuery = makeStringInfo();
+
+	/*
+	 * We pass schemaName and shardName without quote_identifier, since
+	 * they are used as strings here.
+	 */
 	appendStringInfo(checkShardExistsQuery,
-					 "SELECT EXISTS (SELECT FROM pg_catalog.pg_tables WHERE schemaname = '%s' AND tablename = '%s');",
-					 schemaName,
-					 shardName);
+					 "SELECT EXISTS (SELECT FROM pg_catalog.pg_tables WHERE schemaname = %s AND tablename = %s);",
+					 quote_literal_cstr(schemaName),
+					 quote_literal_cstr(shardName));
 
 	int connectionFlags = 0;
 	MultiConnection *connection = GetNodeUserDatabaseConnection(connectionFlags,
@@ -691,11 +730,13 @@ CheckIfRelationWithSameNameExists(ShardInterval *shardInterval, WorkerNode *work
 		ReportResultError(connection, result, ERROR);
 	}
 
-	char *checkExists = PQgetvalue(result, 0, 0);
+	char *existsString = PQgetvalue(result, 0, 0);
+	bool tableExists = strcmp(existsString, "t") == 0;
+
 	PQclear(result);
 	ForgetResults(connection);
 
-	return strcmp(checkExists, "t") == 0;
+	return tableExists;
 }
 
 
@@ -1015,10 +1056,12 @@ static void
 CreateObjectOnPlacement(List *objectCreationCommandList,
 						WorkerNode *workerPlacementNode)
 {
-	char *currentUser = CurrentUserName();
-	SendCommandListToWorkerOutsideTransaction(workerPlacementNode->workerName,
+	MultiConnection *connection =
+		GetNodeUserDatabaseConnection(OUTSIDE_TRANSACTION,
+									  workerPlacementNode->workerName,
 									  workerPlacementNode->workerPort,
-											  currentUser,
+									  NULL, NULL);
+	SendCommandListToWorkerOutsideTransactionWithConnection(connection,
 															objectCreationCommandList);
 }
 
@@ -1487,6 +1530,8 @@ NonBlockingShardSplit(SplitOperation splitOperation,
 					  DistributionColumnMap *distributionColumnOverrides,
 					  uint32 targetColocationId)
 {
+	const char *operationName = SplitOperationAPIName[splitOperation];
+
 	ErrorIfMultipleNonblockingMoveSplitInTheSameTransaction();
 
 	char *superUser = CitusExtensionOwnerName();
@@ -1529,6 +1574,9 @@ NonBlockingShardSplit(SplitOperation splitOperation,
 	/* Non-Blocking shard split workflow starts here */
 	PG_TRY();
 	{
+		ereport(LOG, (errmsg("creating child shards for %s",
+							 operationName)));
+
 		/* 1) Physically create split children. */
 		CreateSplitShardsForShardGroup(shardGroupSplitIntervalListList,
 									   workersForPlacementList);
@@ -1558,6 +1606,10 @@ NonBlockingShardSplit(SplitOperation splitOperation,
 		 */
 		CreateReplicaIdentitiesForDummyShards(mapOfPlacementToDummyShardList);
 
+		ereport(LOG, (errmsg(
+						  "creating replication artifacts (publications, replication slots, subscriptions for %s",
+						  operationName)));
+
 		/* 4) Create Publications. */
 		CreatePublications(sourceConnection, publicationInfoHash);
 
@@ -1606,11 +1658,35 @@ NonBlockingShardSplit(SplitOperation splitOperation,
 			databaseName,
 			logicalRepTargetList);
 
+		/*
+		 * We have to create the primary key (or any other replica identity)
+		 * before the update/delete operations that are queued will be
+		 * replicated. Because if the replica identity does not exist on the
+		 * target, the replication would fail.
+		 *
+		 * So the latest possible moment we could do this is right after the
+		 * initial data COPY, but before enabling the susbcriptions. It might
+		 * seem like a good idea to it after the initial data COPY, since
+		 * it's generally the rule that it's cheaper to build an index at once
+		 * than to create it incrementally. This general rule, is why we create
+		 * all the regular indexes as late during the move as possible.
+		 *
+		 * But as it turns out in practice it's not as clear cut, and we saw a
+		 * speed degradation in the time it takes to move shards when doing the
+		 * replica identity creation after the initial COPY. So, instead we
+		 * keep it before the COPY.
+		 */
+		CreateReplicaIdentities(logicalRepTargetList);
+
+		ereport(LOG, (errmsg("performing copy for %s", operationName)));
+
 		/* 8) Do snapshotted Copy */
 		DoSplitCopy(sourceShardToCopyNode, sourceColocatedShardIntervalList,
 					shardGroupSplitIntervalListList, workersForPlacementList,
 					snapshot, distributionColumnOverrides);
 
+		ereport(LOG, (errmsg("replicating changes for %s", operationName)));
+
 		/*
 		 * 9) Logically replicate all the changes and do most of the table DDL,
 		 * like index and foreign key creation.
@@ -1631,10 +1707,16 @@ NonBlockingShardSplit(SplitOperation splitOperation,
 		 */
 		if (DeferShardDeleteOnSplit)
 		{
+			ereport(LOG, (errmsg("marking deferred cleanup of source shard(s) for %s",
+								 operationName)));
+
 			InsertDeferredDropCleanupRecordsForShards(sourceColocatedShardIntervalList);
 		}
 		else
 		{
+			ereport(LOG, (errmsg("performing cleanup of source shard(s) for %s",
+								 operationName)));
+
 			DropShardList(sourceColocatedShardIntervalList);
 		}
 
@@ -1683,6 +1765,9 @@ NonBlockingShardSplit(SplitOperation splitOperation,
 		 */
 		CreatePartitioningHierarchy(logicalRepTargetList);
 
+		ereport(LOG, (errmsg("creating foreign key constraints (if any) for %s",
+							 operationName)));
+
 		/*
 		 * 14) Create foreign keys if exists after the metadata changes happening in
 		 * DropShardList() and InsertSplitChildrenShardMetadata() because the foreign
@@ -1694,7 +1779,7 @@ NonBlockingShardSplit(SplitOperation splitOperation,
 		 * 15) Release shared memory allocated by worker_split_shard_replication_setup udf
 		 * at source node.
 		 */
-		ExecuteSplitShardReleaseSharedMemory(sourceShardToCopyNode);
+		ExecuteSplitShardReleaseSharedMemory(sourceConnection);
 
 		/* 16) Close source connection */
 		CloseConnection(sourceConnection);
@@ -1716,7 +1801,11 @@ NonBlockingShardSplit(SplitOperation splitOperation,
 		 */
 		DropAllLogicalReplicationLeftovers(SHARD_SPLIT);
 
-		FinalizeOperationNeedingCleanupOnFailure();
+		/*
+		 * Drop temporary objects that were marked as CLEANUP_ON_FAILURE
+		 * or CLEANUP_ALWAYS.
+		 */
+		FinalizeOperationNeedingCleanupOnFailure(operationName);
 
 		PG_RE_THROW();
 	}
@@ -1900,7 +1989,7 @@ CreateWorkerForPlacementSet(List *workersForPlacementList)
 	/* we don't have value field as it's a set */
 	info.entrysize = info.keysize;
 
-	uint32 hashFlags = (HASH_ELEM | HASH_FUNCTION | HASH_CONTEXT);
+	uint32 hashFlags = (HASH_ELEM | HASH_FUNCTION | HASH_CONTEXT | HASH_COMPARE);
 
 	HTAB *workerForPlacementSet = hash_create("worker placement set", 32, &info,
 											  hashFlags);
@@ -1987,19 +2076,8 @@ ExecuteSplitShardReplicationSetupUDF(WorkerNode *sourceWorkerNode,
  * shared memory to store split information. This has to be released after split completes(or fails).
  */
 static void
-ExecuteSplitShardReleaseSharedMemory(WorkerNode *sourceWorkerNode)
+ExecuteSplitShardReleaseSharedMemory(MultiConnection *sourceConnection)
 {
-	char *superUser = CitusExtensionOwnerName();
-	char *databaseName = get_database_name(MyDatabaseId);
-
-	int connectionFlag = FORCE_NEW_CONNECTION;
-	MultiConnection *sourceConnection = GetNodeUserDatabaseConnection(
-		connectionFlag,
-		sourceWorkerNode->workerName,
-		sourceWorkerNode->workerPort,
-		superUser,
-		databaseName);
-
 	StringInfo splitShardReleaseMemoryUDF = makeStringInfo();
 	appendStringInfo(splitShardReleaseMemoryUDF,
 					 "SELECT pg_catalog.worker_split_shard_release_dsm();");
@@ -2214,14 +2292,8 @@ GetNextShardIdForSplitChild()
 	appendStringInfo(nextValueCommand, "SELECT nextval(%s);", quote_literal_cstr(
 						 "pg_catalog.pg_dist_shardid_seq"));
 
-	int connectionFlag = FORCE_NEW_CONNECTION;
-	MultiConnection *connection = GetNodeUserDatabaseConnection(connectionFlag,
-																LocalHostName,
-																PostPortNumber,
-																CitusExtensionOwnerName(),
-																get_database_name(
-																	MyDatabaseId));
-
+	MultiConnection *connection = GetConnectionForLocalQueriesOutsideTransaction(
+		CitusExtensionOwnerName());
 	PGresult *result = NULL;
 	int queryResult = ExecuteOptionalRemoteCommand(connection, nextValueCommand->data,
 												   &result);
@@ -2238,7 +2310,8 @@ GetNextShardIdForSplitChild()
 	}
 
 	shardId = SafeStringToUint64(PQgetvalue(result, 0, 0 /* nodeId column*/));
-	CloseConnection(connection);
+	PQclear(result);
+	ForgetResults(connection);
 
 	return shardId;
 }

src/backend/distributed/operations/shard_transfer.c

@@ -1485,6 +1485,15 @@ EnsureShardCanBeCopied(int64 shardId, const char *sourceNodeName, int32 sourceNo
 								shardId)));
 		}
 	}
+
+	/*
+	 * Make sure the relation exists. In some cases the relation is actually dropped but
+	 * the metadata remains, such as dropping table while citus.enable_ddl_propagation
+	 * is set to off.
+	 */
+	ShardInterval *shardInterval = LoadShardInterval(shardId);
+	Oid distributedTableId = shardInterval->relationId;
+	EnsureRelationExists(distributedTableId);
 }
 
 

src/backend/distributed/operations/worker_copy_table_to_node_udf.c

@@ -53,8 +53,14 @@ worker_copy_table_to_node(PG_FUNCTION_ARGS)
 		targetNodeId);
 
 	StringInfo selectShardQueryForCopy = makeStringInfo();
+
+	/*
+	 * Even though we do COPY(SELECT ...) all the columns, we can't just do SELECT * because we need to not COPY generated colums.
+	 */
+	const char *columnList = CopyableColumnNamesFromRelationName(relationSchemaName,
+																 relationName);
 	appendStringInfo(selectShardQueryForCopy,
-					 "SELECT * FROM %s;", relationQualifiedName);
+					 "SELECT %s FROM %s;", columnList, relationQualifiedName);
 
 	ParamListInfo params = NULL;
 	ExecuteQueryStringIntoDestReceiver(selectShardQueryForCopy->data, params,

src/backend/distributed/operations/worker_node_manager.c

@@ -267,8 +267,11 @@ ErrorIfCoordinatorNotAddedAsWorkerNode()
 		return;
 	}
 
-	ereport(ERROR, (errmsg("could not find the coordinator node in "
-						   "metadata as it is not added as a worker")));
+	ereport(ERROR, (errmsg("operation is not allowed when coordinator "
+						   "is not added into metadata"),
+					errhint("Use \"SELECT citus_set_coordinator_host('"
+							"<hostname>', '<port>')\" to configure the "
+							"coordinator hostname and port")));
 }
 
 

src/backend/distributed/operations/worker_shard_copy.c

@@ -73,7 +73,7 @@ static void ShardCopyDestReceiverDestroy(DestReceiver *destReceiver);
 static bool CanUseLocalCopy(uint32_t destinationNodeId);
 static StringInfo ConstructShardCopyStatement(List *destinationShardFullyQualifiedName,
 											  bool
-											  useBinaryFormat);
+											  useBinaryFormat, TupleDesc tupleDesc);
 static void WriteLocalTuple(TupleTableSlot *slot, ShardCopyDestReceiver *copyDest);
 static int ReadFromLocalBufferCallback(void *outBuf, int minRead, int maxRead);
 static void LocalCopyToShard(ShardCopyDestReceiver *copyDest, CopyOutState
@@ -105,7 +105,8 @@ ConnectToRemoteAndStartCopy(ShardCopyDestReceiver *copyDest)
 
 	StringInfo copyStatement = ConstructShardCopyStatement(
 		copyDest->destinationShardFullyQualifiedName,
-		copyDest->copyOutState->binary);
+		copyDest->copyOutState->binary,
+		copyDest->tupleDescriptor);
 
 	if (!SendRemoteCommand(copyDest->connection, copyStatement->data))
 	{
@@ -344,21 +345,80 @@ ShardCopyDestReceiverDestroy(DestReceiver *dest)
 }
 
 
+/*
+ *  CopyableColumnNamesFromTupleDesc function creates and returns a comma seperated column names string  to be used in COPY
+ *  and SELECT statements when copying a table. The COPY and SELECT statements should filter out the GENERATED columns since COPY
+ *  statement fails to handle them. Iterating over the attributes of the table we also need to skip the dropped columns.
+ */
+const char *
+CopyableColumnNamesFromTupleDesc(TupleDesc tupDesc)
+{
+	StringInfo columnList = makeStringInfo();
+	bool firstInList = true;
+
+	for (int i = 0; i < tupDesc->natts; i++)
+	{
+		Form_pg_attribute att = TupleDescAttr(tupDesc, i);
+		if (att->attgenerated || att->attisdropped)
+		{
+			continue;
+		}
+		if (!firstInList)
+		{
+			appendStringInfo(columnList, ",");
+		}
+
+		firstInList = false;
+
+		appendStringInfo(columnList, "%s", quote_identifier(NameStr(att->attname)));
+	}
+
+	return columnList->data;
+}
+
+
+/*
+ *  CopyableColumnNamesFromRelationName function is a wrapper for CopyableColumnNamesFromTupleDesc.
+ */
+const char *
+CopyableColumnNamesFromRelationName(const char *schemaName, const char *relationName)
+{
+	Oid namespaceOid = get_namespace_oid(schemaName, true);
+
+	Oid relationId = get_relname_relid(relationName, namespaceOid);
+
+	Relation relation = relation_open(relationId, AccessShareLock);
+
+	TupleDesc tupleDesc = RelationGetDescr(relation);
+
+	const char *columnList = CopyableColumnNamesFromTupleDesc(tupleDesc);
+
+	relation_close(relation, NoLock);
+
+	return columnList;
+}
+
+
 /*
  * ConstructShardCopyStatement constructs the text of a COPY statement
  * for copying into a result table
  */
 static StringInfo
 ConstructShardCopyStatement(List *destinationShardFullyQualifiedName, bool
-							useBinaryFormat)
+							useBinaryFormat,
+							TupleDesc tupleDesc)
 {
 	char *destinationShardSchemaName = linitial(destinationShardFullyQualifiedName);
 	char *destinationShardRelationName = lsecond(destinationShardFullyQualifiedName);
 
+
 	StringInfo command = makeStringInfo();
-	appendStringInfo(command, "COPY %s.%s FROM STDIN",
+
+	const char *columnList = CopyableColumnNamesFromTupleDesc(tupleDesc);
+
+	appendStringInfo(command, "COPY %s.%s (%s) FROM STDIN",
 					 quote_identifier(destinationShardSchemaName), quote_identifier(
-						 destinationShardRelationName));
+						 destinationShardRelationName), columnList);
 
 	if (useBinaryFormat)
 	{

src/backend/distributed/operations/worker_split_copy_udf.c

@@ -43,6 +43,10 @@ static DestReceiver *  CreatePartitionedSplitCopyDestReceiver(EState *executor,
 															  List *splitCopyInfoList);
 static void BuildMinMaxRangeArrays(List *splitCopyInfoList, ArrayType **minValueArray,
 								   ArrayType **maxValueArray);
+static char * TraceWorkerSplitCopyUdf(char *sourceShardToCopySchemaName,
+									  char *sourceShardToCopyPrefix,
+									  char *sourceShardToCopyQualifiedName,
+									  List *splitCopyInfoList);
 
 /*
  * worker_split_copy(source_shard_id bigint, splitCopyInfo pg_catalog.split_copy_info[])
@@ -93,15 +97,26 @@ worker_split_copy(PG_FUNCTION_ARGS)
 	Oid sourceShardToCopySchemaOId = get_rel_namespace(
 		shardIntervalToSplitCopy->relationId);
 	char *sourceShardToCopySchemaName = get_namespace_name(sourceShardToCopySchemaOId);
-	char *sourceShardToCopyName = get_rel_name(shardIntervalToSplitCopy->relationId);
+	char *sourceShardPrefix = get_rel_name(shardIntervalToSplitCopy->relationId);
+	char *sourceShardToCopyName = pstrdup(sourceShardPrefix);
 	AppendShardIdToName(&sourceShardToCopyName, shardIdToSplitCopy);
 	char *sourceShardToCopyQualifiedName = quote_qualified_identifier(
 		sourceShardToCopySchemaName,
 		sourceShardToCopyName);
 
+	ereport(LOG, (errmsg("%s", TraceWorkerSplitCopyUdf(sourceShardToCopySchemaName,
+													   sourceShardPrefix,
+													   sourceShardToCopyQualifiedName,
+													   splitCopyInfoList))));
+
 	StringInfo selectShardQueryForCopy = makeStringInfo();
+	const char *columnList = CopyableColumnNamesFromRelationName(
+		sourceShardToCopySchemaName,
+		sourceShardToCopyName);
+
 	appendStringInfo(selectShardQueryForCopy,
-					 "SELECT * FROM %s;", sourceShardToCopyQualifiedName);
+					 "SELECT %s FROM %s;", columnList,
+					 sourceShardToCopyQualifiedName);
 
 	ParamListInfo params = NULL;
 	ExecuteQueryStringIntoDestReceiver(selectShardQueryForCopy->data, params,
@@ -113,6 +128,48 @@ worker_split_copy(PG_FUNCTION_ARGS)
 }
 
 
+/* Trace split copy udf */
+static char *
+TraceWorkerSplitCopyUdf(char *sourceShardToCopySchemaName,
+						char *sourceShardToCopyPrefix,
+						char *sourceShardToCopyQualifiedName,
+						List *splitCopyInfoList)
+{
+	StringInfo splitCopyTrace = makeStringInfo();
+	appendStringInfo(splitCopyTrace, "performing copy from shard %s to [",
+					 sourceShardToCopyQualifiedName);
+
+	/* split copy always has atleast two destinations */
+	int index = 1;
+	int splitWayCount = list_length(splitCopyInfoList);
+	SplitCopyInfo *splitCopyInfo = NULL;
+	foreach_ptr(splitCopyInfo, splitCopyInfoList)
+	{
+		char *shardNameCopy = pstrdup(sourceShardToCopyPrefix);
+		AppendShardIdToName(&shardNameCopy, splitCopyInfo->destinationShardId);
+
+		char *shardNameCopyQualifiedName = quote_qualified_identifier(
+			sourceShardToCopySchemaName,
+			shardNameCopy);
+
+		appendStringInfo(splitCopyTrace, "%s (nodeId: %u)", shardNameCopyQualifiedName,
+						 splitCopyInfo->destinationShardNodeId);
+		pfree(shardNameCopy);
+
+		if (index < splitWayCount)
+		{
+			appendStringInfo(splitCopyTrace, ", ");
+		}
+
+		index++;
+	}
+
+	appendStringInfo(splitCopyTrace, "]");
+
+	return splitCopyTrace->data;
+}
+
+
 /* Parse a single SplitCopyInfo Tuple */
 static void
 ParseSplitCopyInfoDatum(Datum splitCopyInfoDatum, SplitCopyInfo **splitCopyInfo)

src/backend/distributed/operations/worker_split_shard_replication_setup_udf.c

@@ -281,7 +281,8 @@ PopulateShardSplitInfoInSM(ShardSplitInfoSMHeader *shardSplitInfoSMHeader)
 	{
 		uint32_t nodeId = entry->key.nodeId;
 		uint32_t tableOwnerId = entry->key.tableOwnerId;
-		char *derivedSlotName = ReplicationSlotName(SHARD_SPLIT, nodeId, tableOwnerId);
+		char *derivedSlotName = ReplicationSlotNameForNodeAndOwner(SHARD_SPLIT, nodeId,
+																   tableOwnerId);
 
 		List *shardSplitInfoList = entry->shardSplitInfoList;
 		ShardSplitInfo *splitShardInfo = NULL;
@@ -389,7 +390,8 @@ ReturnReplicationSlotInfo(Tuplestorestate *tupleStore, TupleDesc
 		char *tableOwnerName = GetUserNameFromId(entry->key.tableOwnerId, false);
 		values[1] = CStringGetTextDatum(tableOwnerName);
 
-		char *slotName = ReplicationSlotName(SHARD_SPLIT, entry->key.nodeId,
+		char *slotName = ReplicationSlotNameForNodeAndOwner(SHARD_SPLIT,
+															entry->key.nodeId,
 															entry->key.tableOwnerId);
 		values[2] = CStringGetTextDatum(slotName);
 

src/backend/distributed/planner/distributed_planner.c

@@ -1897,14 +1897,14 @@ multi_relation_restriction_hook(PlannerInfo *root, RelOptInfo *relOptInfo,
 	MemoryContext restrictionsMemoryContext = plannerRestrictionContext->memoryContext;
 	MemoryContext oldMemoryContext = MemoryContextSwitchTo(restrictionsMemoryContext);
 
-	bool distributedTable = IsCitusTable(rte->relid);
+	bool isCitusTable = IsCitusTable(rte->relid);
 
 	RelationRestriction *relationRestriction = palloc0(sizeof(RelationRestriction));
 	relationRestriction->index = restrictionIndex;
 	relationRestriction->relationId = rte->relid;
 	relationRestriction->rte = rte;
 	relationRestriction->relOptInfo = relOptInfo;
-	relationRestriction->distributedRelation = distributedTable;
+	relationRestriction->citusTable = isCitusTable;
 	relationRestriction->plannerInfo = root;
 
 	/* see comments on GetVarFromAssignedParam() */
@@ -1919,10 +1919,42 @@ multi_relation_restriction_hook(PlannerInfo *root, RelOptInfo *relOptInfo,
 	 * We're also keeping track of whether all participant
 	 * tables are reference tables.
 	 */
-	if (distributedTable)
+	if (isCitusTable)
 	{
 		cacheEntry = GetCitusTableCacheEntry(rte->relid);
 
+		/*
+		 * The statistics objects of the distributed table are not relevant
+		 * for the distributed planning, so we can override it.
+		 *
+		 * Normally, we should not need this. However, the combination of
+		 * Postgres commit 269b532aef55a579ae02a3e8e8df14101570dfd9 and
+		 * Citus function AdjustPartitioningForDistributedPlanning()
+		 * forces us to do this. The commit expects statistics objects
+		 * of partitions to have "inh" flag set properly. Whereas, the
+		 * function overrides "inh" flag. To avoid Postgres to throw error,
+		 * we override statlist such that Postgres does not try to process
+		 * any statistics objects during the standard_planner() on the
+		 * coordinator. In the end, we do not need the standard_planner()
+		 * on the coordinator to generate an optimized plan. We call
+		 * into standard_planner() for other purposes, such as generating the
+		 * relationRestrictionContext here.
+		 *
+		 * AdjustPartitioningForDistributedPlanning() is a hack that we use
+		 * to prevent Postgres' standard_planner() to expand all the partitions
+		 * for the distributed planning when a distributed partitioned table
+		 * is queried. It is required for both correctness and performance
+		 * reasons. Although we can eliminate the use of the function for
+		 * the correctness (e.g., make sure that rest of the planner can handle
+		 * partitions), it's performance implication is hard to avoid. Certain
+		 * planning logic of Citus (such as router or query pushdown) relies
+		 * heavily on the relationRestrictionList. If
+		 * AdjustPartitioningForDistributedPlanning() is removed, all the
+		 * partitions show up in the, causing high planning times for
+		 * such queries.
+		 */
+		relOptInfo->statlist = NIL;
+
 		relationRestrictionContext->allReferenceTables &=
 			IsCitusTableTypeCacheEntry(cacheEntry, REFERENCE_TABLE);
 	}

src/backend/distributed/planner/multi_router_planner.c

@@ -3692,7 +3692,7 @@ CopyRelationRestrictionContext(RelationRestrictionContext *oldContext)
 
 		newRestriction->index = oldRestriction->index;
 		newRestriction->relationId = oldRestriction->relationId;
-		newRestriction->distributedRelation = oldRestriction->distributedRelation;
+		newRestriction->citusTable = oldRestriction->citusTable;
 		newRestriction->rte = copyObject(oldRestriction->rte);
 
 		/* can't be copied, we copy (flatly) a RelOptInfo, and then decouple baserestrictinfo */

src/backend/distributed/planner/relation_restriction_equivalence.c

@@ -224,7 +224,7 @@ ContextContainsLocalRelation(RelationRestrictionContext *restrictionContext)
 	{
 		RelationRestriction *relationRestriction = lfirst(relationRestrictionCell);
 
-		if (!relationRestriction->distributedRelation)
+		if (!relationRestriction->citusTable)
 		{
 			return true;
 		}

src/backend/distributed/replication/multi_logical_replication.c

@@ -66,6 +66,7 @@
 #include "utils/syscache.h"
 
 #define STR_ERRCODE_UNDEFINED_OBJECT "42704"
+#define STR_ERRCODE_OBJECT_IN_USE "55006"
 
 
 #define REPLICATION_SLOT_CATALOG_TABLE_NAME "pg_replication_slots"
@@ -156,6 +157,10 @@ static void WaitForGroupedLogicalRepTargetsToBecomeReady(
 static void WaitForGroupedLogicalRepTargetsToCatchUp(XLogRecPtr sourcePosition,
 													 GroupedLogicalRepTargets *
 													 groupedLogicalRepTargets);
+static void RecreateGroupedLogicalRepTargetsConnections(
+	HTAB *groupedLogicalRepTargetsHash,
+	char *user,
+	char *databaseName);
 
 /*
  * LogicallyReplicateShards replicates a list of shards from one node to another
@@ -233,6 +238,26 @@ LogicallyReplicateShards(List *shardList, char *sourceNodeName, int sourceNodePo
 		/* only useful for isolation testing, see the function comment for the details */
 		ConflictOnlyWithIsolationTesting();
 
+		/*
+		 * We have to create the primary key (or any other replica identity)
+		 * before the update/delete operations that are queued will be
+		 * replicated. Because if the replica identity does not exist on the
+		 * target, the replication would fail.
+		 *
+		 * So the latest possible moment we could do this is right after the
+		 * initial data COPY, but before enabling the susbcriptions. It might
+		 * seem like a good idea to it after the initial data COPY, since
+		 * it's generally the rule that it's cheaper to build an index at once
+		 * than to create it incrementally. This general rule, is why we create
+		 * all the regular indexes as late during the move as possible.
+		 *
+		 * But as it turns out in practice it's not as clear cut, and we saw a
+		 * speed degradation in the time it takes to move shards when doing the
+		 * replica identity creation after the initial COPY. So, instead we
+		 * keep it before the COPY.
+		 */
+		CreateReplicaIdentities(logicalRepTargetList);
+
 		CopyShardsToNode(sourceNode, targetNode, shardList, snapshot);
 
 		/*
@@ -346,20 +371,6 @@ CompleteNonBlockingShardTransfer(List *shardList,
 								 HTAB *groupedLogicalRepTargetsHash,
 								 LogicalRepType type)
 {
-	/*
-	 * We have to create the primary key (or any other replica identity)
-	 * before the update/delete operations that are queued will be
-	 * replicated. Because if the replica identity does not exist on the
-	 * target, the replication would fail.
-	 *
-	 * So we it right after the initial data COPY, but before enabling the
-	 * susbcriptions. We do it at this latest possible moment, because its
-	 * much cheaper to build an index at once than to create it
-	 * incrementally. So this way we create the primary key index in one go
-	 * for all data from the initial COPY.
-	 */
-	CreateReplicaIdentities(logicalRepTargetList);
-
 	/* Start applying the changes from the replication slots to catch up. */
 	EnableSubscriptions(logicalRepTargetList);
 
@@ -490,7 +501,7 @@ CreateShardMoveLogicalRepTargetList(HTAB *publicationInfoHash, List *shardList)
 		target->newShards = NIL;
 		target->subscriptionOwnerName = SubscriptionRoleName(SHARD_MOVE, ownerId);
 		target->replicationSlot = palloc0(sizeof(ReplicationSlotInfo));
-		target->replicationSlot->name = ReplicationSlotName(SHARD_MOVE,
+		target->replicationSlot->name = ReplicationSlotNameForNodeAndOwner(SHARD_MOVE,
 																		   nodeId,
 																		   ownerId);
 		target->replicationSlot->targetNodeId = nodeId;
@@ -559,10 +570,10 @@ DropAllLogicalReplicationLeftovers(LogicalRepType type)
 	char *databaseName = get_database_name(MyDatabaseId);
 
 	/*
-	 * We open new connections to all nodes. The reason for this is that
-	 * operations on subscriptions, publications and replication slotscannot be
-	 * run in a transaction. By forcing a new connection we make sure no
-	 * transaction is active on the connection.
+	 * We need connections that are not currently inside a transaction. The
+	 * reason for this is that operations on subscriptions, publications and
+	 * replication slots cannot be run in a transaction. By forcing a new
+	 * connection we make sure no transaction is active on the connection.
 	 */
 	int connectionFlags = FORCE_NEW_CONNECTION;
 
@@ -600,7 +611,9 @@ DropAllLogicalReplicationLeftovers(LogicalRepType type)
 		/*
 		 * We close all connections that we opened for the dropping here. That
 		 * way we don't keep these connections open unnecessarily during the
-		 * 'LogicalRepType' operation (which can take a long time).
+		 * 'LogicalRepType' operation (which can take a long time). We might
+		 * need to reopen a few later on, but that seems better than keeping
+		 * many open for no reason for a long time.
 		 */
 		CloseConnection(cleanupConnection);
 	}
@@ -1150,11 +1163,14 @@ CreatePartitioningHierarchy(List *logicalRepTargetList)
 				 * parallel, so create them sequentially. Also attaching partition
 				 * is a quick operation, so it is fine to execute sequentially.
 				 */
-				SendCommandListToWorkerOutsideTransaction(
+
+				MultiConnection *connection =
+					GetNodeUserDatabaseConnection(OUTSIDE_TRANSACTION,
 												  target->superuserConnection->hostname,
 												  target->superuserConnection->port,
-					tableOwner,
-					list_make1(attachPartitionCommand));
+												  tableOwner, NULL);
+				ExecuteCriticalRemoteCommand(connection, attachPartitionCommand);
+
 				MemoryContextReset(localContext);
 			}
 		}
@@ -1203,10 +1219,8 @@ CreateUncheckedForeignKeyConstraints(List *logicalRepTargetList)
 				list_make1("SET LOCAL citus.skip_constraint_validation TO ON;"),
 				commandList);
 
-			SendCommandListToWorkerOutsideTransaction(
-				target->superuserConnection->hostname,
-				target->superuserConnection->port,
-				target->superuserConnection->user,
+			SendCommandListToWorkerOutsideTransactionWithConnection(
+				target->superuserConnection,
 				commandList);
 
 			MemoryContextReset(localContext);
@@ -1281,18 +1295,64 @@ DropPublications(MultiConnection *sourceConnection, HTAB *publicationInfoHash)
 
 /*
  * DropReplicationSlot drops the replication slot with the given name
- * if it exists.
+ * if it exists. It retries if the command fails with an OBJECT_IN_USE error.
  */
 static void
 DropReplicationSlot(MultiConnection *connection, char *replicationSlotName)
 {
-	ExecuteCriticalRemoteCommand(
+	int maxSecondsToTryDropping = 20;
+	bool raiseInterrupts = true;
+	PGresult *result = NULL;
+
+	/* we'll retry in case of an OBJECT_IN_USE error */
+	while (maxSecondsToTryDropping >= 0)
+	{
+		int querySent = SendRemoteCommand(
 			connection,
 			psprintf(
 				"select pg_drop_replication_slot(slot_name) from "
 				REPLICATION_SLOT_CATALOG_TABLE_NAME
 				" where slot_name = %s",
-			quote_literal_cstr(replicationSlotName)));
+				quote_literal_cstr(replicationSlotName))
+			);
+
+		if (querySent == 0)
+		{
+			ReportConnectionError(connection, ERROR);
+		}
+
+		result = GetRemoteCommandResult(connection, raiseInterrupts);
+
+		if (IsResponseOK(result))
+		{
+			/* no error, we are good to go */
+			break;
+		}
+
+		char *errorcode = PQresultErrorField(result, PG_DIAG_SQLSTATE);
+		if (errorcode != NULL && strcmp(errorcode, STR_ERRCODE_OBJECT_IN_USE) == 0 &&
+			maxSecondsToTryDropping > 0)
+		{
+			/* retry dropping the replication slot after sleeping for one sec */
+			maxSecondsToTryDropping--;
+			pg_usleep(1000);
+		}
+		else
+		{
+			/*
+			 * Report error if:
+			 * - Error code is not 55006 (Object In Use)
+			 * - Or, we have made enough number of retries (currently 20), but didn't work
+			 */
+			ReportResultError(connection, result, ERROR);
+		}
+
+		PQclear(result);
+		ForgetResults(connection);
+	}
+
+	PQclear(result);
+	ForgetResults(connection);
 }
 
 
@@ -1321,11 +1381,14 @@ PublicationName(LogicalRepType type, uint32_t nodeId, Oid ownerId)
 
 
 /*
- * ReplicationSlotName returns the name of the replication slot for the given
- * node and table owner.
+ * ReplicationSlotNameForNodeAndOwner returns the name of the replication slot for the
+ * given node and table owner.
+ *
+ * Note that PG15 introduced a new ReplicationSlotName function that caused name conflicts
+ * and we renamed this function.
  */
 char *
-ReplicationSlotName(LogicalRepType type, uint32_t nodeId, Oid ownerId)
+ReplicationSlotNameForNodeAndOwner(LogicalRepType type, uint32_t nodeId, Oid ownerId)
 {
 	StringInfo slotName = makeStringInfo();
 	appendStringInfo(slotName, "%s%u_%u", replicationSlotPrefix[type], nodeId,
@@ -1348,7 +1411,7 @@ ReplicationSlotName(LogicalRepType type, uint32_t nodeId, Oid ownerId)
 char *
 SubscriptionName(LogicalRepType type, Oid ownerId)
 {
-	return psprintf("%s%i", subscriptionPrefix[type], ownerId);
+	return psprintf("%s%u", subscriptionPrefix[type], ownerId);
 }
 
 
@@ -1359,7 +1422,7 @@ SubscriptionName(LogicalRepType type, Oid ownerId)
 char *
 SubscriptionRoleName(LogicalRepType type, Oid ownerId)
 {
-	return psprintf("%s%i", subscriptionRolePrefix[type], ownerId);
+	return psprintf("%s%u", subscriptionRolePrefix[type], ownerId);
 }
 
 
@@ -1585,11 +1648,11 @@ DropUser(MultiConnection *connection, char *username)
 	 * The DROP USER command should not propagate, so we temporarily disable
 	 * DDL propagation.
 	 */
-	SendCommandListToWorkerOutsideTransaction(
-		connection->hostname, connection->port, connection->user,
+	SendCommandListToWorkerOutsideTransactionWithConnection(
+		connection,
 		list_make2(
 			"SET LOCAL citus.enable_ddl_propagation TO OFF;",
-			psprintf("DROP USER IF EXISTS %s",
+			psprintf("DROP USER IF EXISTS %s;",
 					 quote_identifier(username))));
 }
 
@@ -1611,7 +1674,7 @@ CreatePublications(MultiConnection *connection,
 		bool prefixWithComma = false;
 
 		appendStringInfo(createPublicationCommand, "CREATE PUBLICATION %s FOR TABLE ",
-						 entry->name);
+						 quote_identifier(entry->name));
 
 		ShardInterval *shard = NULL;
 		foreach_ptr(shard, entry->shardIntervals)
@@ -1771,16 +1834,14 @@ CreateSubscriptions(MultiConnection *sourceConnection,
 		 * create a user with SUPERUSER permissions and then alter it to NOSUPERUSER.
 		 * This prevents permission escalations.
 		 */
-		SendCommandListToWorkerOutsideTransaction(
-			target->superuserConnection->hostname,
-			target->superuserConnection->port,
-			target->superuserConnection->user,
+		SendCommandListToWorkerOutsideTransactionWithConnection(
+			target->superuserConnection,
 			list_make2(
 				"SET LOCAL citus.enable_ddl_propagation TO OFF;",
 				psprintf(
-					"CREATE USER %s SUPERUSER IN ROLE %s",
-					target->subscriptionOwnerName,
-					GetUserNameFromId(ownerId, false)
+					"CREATE USER %s SUPERUSER IN ROLE %s;",
+					quote_identifier(target->subscriptionOwnerName),
+					quote_identifier(GetUserNameFromId(ownerId, false))
 					)));
 
 		StringInfo conninfo = makeStringInfo();
@@ -1824,23 +1885,21 @@ CreateSubscriptions(MultiConnection *sourceConnection,
 		pfree(createSubscriptionCommand);
 		ExecuteCriticalRemoteCommand(target->superuserConnection, psprintf(
 										 "ALTER SUBSCRIPTION %s OWNER TO %s",
-										 target->subscriptionName,
-										 target->subscriptionOwnerName
+										 quote_identifier(target->subscriptionName),
+										 quote_identifier(target->subscriptionOwnerName)
 										 ));
 
 		/*
 		 * The ALTER ROLE command should not propagate, so we temporarily
 		 * disable DDL propagation.
 		 */
-		SendCommandListToWorkerOutsideTransaction(
-			target->superuserConnection->hostname,
-			target->superuserConnection->port,
-			target->superuserConnection->user,
+		SendCommandListToWorkerOutsideTransactionWithConnection(
+			target->superuserConnection,
 			list_make2(
 				"SET LOCAL citus.enable_ddl_propagation TO OFF;",
 				psprintf(
-					"ALTER ROLE %s NOSUPERUSER",
-					target->subscriptionOwnerName
+					"ALTER ROLE %s NOSUPERUSER;",
+					quote_identifier(target->subscriptionOwnerName)
 					)));
 	}
 }
@@ -2001,8 +2060,12 @@ CreateGroupedLogicalRepTargetsConnections(HTAB *groupedLogicalRepTargetsHash,
  * RecreateGroupedLogicalRepTargetsConnections recreates connections for all of the
  * nodes in the groupedLogicalRepTargetsHash where the old connection is broken or
  * currently running a query.
+ *
+ * IMPORTANT: When it recreates the connection, it doesn't close the existing
+ * connection. This means that this function should only be called when we know
+ * we'll throw an error afterwards, otherwise we would leak these connections.
  */
-void
+static void
 RecreateGroupedLogicalRepTargetsConnections(HTAB *groupedLogicalRepTargetsHash,
 											char *user,
 											char *databaseName)
@@ -2012,10 +2075,11 @@ RecreateGroupedLogicalRepTargetsConnections(HTAB *groupedLogicalRepTargetsHash,
 	GroupedLogicalRepTargets *groupedLogicalRepTargets = NULL;
 	foreach_htab(groupedLogicalRepTargets, &status, groupedLogicalRepTargetsHash)
 	{
-		if (groupedLogicalRepTargets->superuserConnection &&
-			PQstatus(groupedLogicalRepTargets->superuserConnection->pgConn) ==
-			CONNECTION_OK &&
-			!PQisBusy(groupedLogicalRepTargets->superuserConnection->pgConn)
+		MultiConnection *superuserConnection =
+			groupedLogicalRepTargets->superuserConnection;
+		if (superuserConnection &&
+			PQstatus(superuserConnection->pgConn) == CONNECTION_OK &&
+			!PQisBusy(superuserConnection->pgConn)
 			)
 		{
 			continue;
@@ -2023,8 +2087,8 @@ RecreateGroupedLogicalRepTargetsConnections(HTAB *groupedLogicalRepTargetsHash,
 		WorkerNode *targetWorkerNode = FindNodeWithNodeId(
 			groupedLogicalRepTargets->nodeId,
 			false);
-		MultiConnection *superuserConnection =
-			GetNodeUserDatabaseConnection(connectionFlags,
+		superuserConnection = GetNodeUserDatabaseConnection(
+			connectionFlags,
 			targetWorkerNode->workerName,
 			targetWorkerNode->workerPort,
 			user,

src/backend/distributed/shardsplit/shardsplit_decoder.c

@@ -34,6 +34,10 @@ static Oid FindTargetRelationOid(Relation sourceShardRelation,
 								 HeapTuple tuple,
 								 char *currentSlotName);
 
+static HeapTuple GetTupleForTargetSchema(HeapTuple sourceRelationTuple,
+										 TupleDesc sourceTupleDesc,
+										 TupleDesc targetTupleDesc);
+
 /*
  * Postgres uses 'pgoutput' as default plugin for logical replication.
  * We want to reuse Postgres pgoutput's functionality as much as possible.
@@ -129,6 +133,71 @@ split_change_cb(LogicalDecodingContext *ctx, ReorderBufferTXN *txn,
 	}
 
 	Relation targetRelation = RelationIdGetRelation(targetRelationOid);
+
+	/*
+	 * If any columns from source relation have been dropped, then the tuple needs to
+	 * be formatted according to the target relation.
+	 */
+	TupleDesc sourceRelationDesc = RelationGetDescr(relation);
+	TupleDesc targetRelationDesc = RelationGetDescr(targetRelation);
+	if (sourceRelationDesc->natts > targetRelationDesc->natts)
+	{
+		switch (change->action)
+		{
+			case REORDER_BUFFER_CHANGE_INSERT:
+			{
+				HeapTuple sourceRelationNewTuple = &(change->data.tp.newtuple->tuple);
+				HeapTuple targetRelationNewTuple = GetTupleForTargetSchema(
+					sourceRelationNewTuple, sourceRelationDesc, targetRelationDesc);
+
+				change->data.tp.newtuple->tuple = *targetRelationNewTuple;
+				break;
+			}
+
+			case REORDER_BUFFER_CHANGE_UPDATE:
+			{
+				HeapTuple sourceRelationNewTuple = &(change->data.tp.newtuple->tuple);
+				HeapTuple targetRelationNewTuple = GetTupleForTargetSchema(
+					sourceRelationNewTuple, sourceRelationDesc, targetRelationDesc);
+
+				change->data.tp.newtuple->tuple = *targetRelationNewTuple;
+
+				/*
+				 * Format oldtuple according to the target relation. If the column values of replica
+				 * identiy change, then the old tuple is non-null and needs to be formatted according
+				 * to the target relation schema.
+				 */
+				if (change->data.tp.oldtuple != NULL)
+				{
+					HeapTuple sourceRelationOldTuple = &(change->data.tp.oldtuple->tuple);
+					HeapTuple targetRelationOldTuple = GetTupleForTargetSchema(
+						sourceRelationOldTuple,
+						sourceRelationDesc,
+						targetRelationDesc);
+
+					change->data.tp.oldtuple->tuple = *targetRelationOldTuple;
+				}
+				break;
+			}
+
+			case REORDER_BUFFER_CHANGE_DELETE:
+			{
+				HeapTuple sourceRelationOldTuple = &(change->data.tp.oldtuple->tuple);
+				HeapTuple targetRelationOldTuple = GetTupleForTargetSchema(
+					sourceRelationOldTuple, sourceRelationDesc, targetRelationDesc);
+
+				change->data.tp.oldtuple->tuple = *targetRelationOldTuple;
+				break;
+			}
+
+			/* Only INSERT/DELETE/UPDATE actions are visible in the replication path of split shard */
+			default:
+				ereport(ERROR, errmsg(
+							"Unexpected Action :%d. Expected action is INSERT/DELETE/UPDATE",
+							change->action));
+		}
+	}
+
 	pgoutputChangeCB(ctx, txn, targetRelation, change);
 	RelationClose(targetRelation);
 }
@@ -223,3 +292,51 @@ GetHashValueForIncomingTuple(Relation sourceShardRelation,
 
 	return DatumGetInt32(hashedValueDatum);
 }
+
+
+/*
+ * GetTupleForTargetSchema returns a tuple with the schema of the target relation.
+ * If some columns within the source relations are dropped, we would have to reformat
+ * the tuple to match the schema of the target relation.
+ *
+ * Consider the below scenario:
+ * Session1 : Drop column followed by create_distributed_table_concurrently
+ * Session2 : Concurrent insert workload
+ *
+ * The child shards created by create_distributed_table_concurrently will have less columns
+ * than the source shard because some column were dropped.
+ * The incoming tuple from session2 will have more columns as the writes
+ * happened on source shard. But now the tuple needs to be applied on child shard. So we need to format
+ * it according to child schema.
+ */
+static HeapTuple
+GetTupleForTargetSchema(HeapTuple sourceRelationTuple,
+						TupleDesc sourceRelDesc,
+						TupleDesc targetRelDesc)
+{
+	/* Deform the tuple */
+	Datum *oldValues = (Datum *) palloc0(sourceRelDesc->natts * sizeof(Datum));
+	bool *oldNulls = (bool *) palloc0(sourceRelDesc->natts * sizeof(bool));
+	heap_deform_tuple(sourceRelationTuple, sourceRelDesc, oldValues,
+					  oldNulls);
+
+
+	/* Create new tuple by skipping dropped columns */
+	int nextAttributeIndex = 0;
+	Datum *newValues = (Datum *) palloc0(targetRelDesc->natts * sizeof(Datum));
+	bool *newNulls = (bool *) palloc0(targetRelDesc->natts * sizeof(bool));
+	for (int i = 0; i < sourceRelDesc->natts; i++)
+	{
+		if (TupleDescAttr(sourceRelDesc, i)->attisdropped)
+		{
+			continue;
+		}
+
+		newValues[nextAttributeIndex] = oldValues[i];
+		newNulls[nextAttributeIndex] = oldNulls[i];
+		nextAttributeIndex++;
+	}
+
+	HeapTuple targetRelationTuple = heap_form_tuple(targetRelDesc, newValues, newNulls);
+	return targetRelationTuple;
+}

src/backend/distributed/sql/citus--11.0-4--11.1-1.sql

@@ -5,14 +5,23 @@
 #include "udfs/citus_internal_delete_partition_metadata/11.1-1.sql"
 #include "udfs/citus_copy_shard_placement/11.1-1.sql"
 
+-- We should not introduce breaking sql changes to upgrade files after they are released.
+-- We did that for worker_fetch_foreign_file in v9.0.0 and worker_repartition_cleanup in v9.2.0.
+-- When we try to drop those udfs in that file, they were missing for some clients unexpectedly
+-- due to buggy changes in old upgrade scripts. For that case, the fix is to change DROP statements
+-- with DROP IF EXISTS for those 2 udfs in 11.0-4--11.1-1.
+-- Fixes an upgrade problem for worker_fetch_foreign_file when upgrade starts from 8.3 up to 11.1
+-- Fixes an upgrade problem for worker_repartition_cleanup when upgrade starts from 9.1 up to 11.1
+-- Refer the related PR https://github.com/citusdata/citus/pull/6441 for more information
+DROP FUNCTION IF EXISTS pg_catalog.worker_fetch_foreign_file(text, text, bigint, text[], integer[]);
+DROP FUNCTION IF EXISTS pg_catalog.worker_repartition_cleanup(bigint);
+
 DROP FUNCTION pg_catalog.worker_create_schema(bigint,text);
 DROP FUNCTION pg_catalog.worker_cleanup_job_schema_cache();
-DROP FUNCTION pg_catalog.worker_fetch_foreign_file(text, text, bigint, text[], integer[]);
 DROP FUNCTION pg_catalog.worker_fetch_partition_file(bigint, integer, integer, integer, text, integer);
 DROP FUNCTION pg_catalog.worker_hash_partition_table(bigint, integer, text, text, oid, anyarray);
 DROP FUNCTION pg_catalog.worker_merge_files_into_table(bigint, integer, text[], text[]);
 DROP FUNCTION pg_catalog.worker_range_partition_table(bigint, integer, text, text, oid, anyarray);
-DROP FUNCTION pg_catalog.worker_repartition_cleanup(bigint);
 
 DO $check_columnar$
 BEGIN

src/backend/distributed/transaction/remote_transaction.c

@@ -66,6 +66,7 @@ StartRemoteTransactionBegin(struct MultiConnection *connection)
 
 	/* remember transaction as being in-progress */
 	dlist_push_tail(&InProgressTransactions, &connection->transactionNode);
+	connection->transactionInProgress = true;
 
 	transaction->transactionState = REMOTE_TRANS_STARTING;
 
@@ -750,26 +751,6 @@ MarkRemoteTransactionCritical(struct MultiConnection *connection)
 }
 
 
-/*
- * CloseRemoteTransaction handles closing a connection that, potentially, is
- * part of a coordinated transaction.  This should only ever be called from
- * connection_management.c, while closing a connection during a transaction.
- */
-void
-CloseRemoteTransaction(struct MultiConnection *connection)
-{
-	RemoteTransaction *transaction = &connection->remoteTransaction;
-
-	/* unlink from list of open transactions, if necessary */
-	if (transaction->transactionState != REMOTE_TRANS_NOT_STARTED)
-	{
-		/* XXX: Should we error out for a critical transaction? */
-
-		dlist_delete(&connection->transactionNode);
-	}
-}
-
-
 /*
  * ResetRemoteTransaction resets the state of the transaction after the end of
  * the main transaction, if the connection is being reused.
@@ -779,8 +760,23 @@ ResetRemoteTransaction(struct MultiConnection *connection)
 {
 	RemoteTransaction *transaction = &connection->remoteTransaction;
 
+	/* unlink from list of open transactions, if necessary */
+	if (connection->transactionInProgress)
+	{
+		/* XXX: Should we error out for a critical transaction? */
+
+		dlist_delete(&connection->transactionNode);
+		connection->transactionInProgress = false;
+		memset(&connection->transactionNode, 0, sizeof(connection->transactionNode));
+	}
+
 	/* just reset the entire state, relying on 0 being invalid/false */
 	memset(transaction, 0, sizeof(*transaction));
+
+	ResetShardPlacementAssociation(connection);
+
+	/* reset copy state */
+	connection->copyBytesWrittenSinceLastFlush = 0;
 }
 
 

src/backend/distributed/transaction/worker_transaction.c

@@ -340,6 +340,25 @@ SendCommandListToWorkerOutsideTransaction(const char *nodeName, int32 nodePort,
 																	  nodeName, nodePort,
 																	  nodeUser, NULL);
 
+	SendCommandListToWorkerOutsideTransactionWithConnection(workerConnection,
+															commandList);
+	CloseConnection(workerConnection);
+}
+
+
+/*
+ * SendCommandListToWorkerOutsideTransactionWithConnection sends the command list
+ * over the specified connection. This opens a new transaction on the
+ * connection, thus it's important that no transaction is currently open.
+ * This function is mainly useful to avoid opening an closing
+ * connections excessively by allowing reusing a single connection to send
+ * multiple separately committing transactions. The function raises an error if
+ * any of the queries fail.
+ */
+void
+SendCommandListToWorkerOutsideTransactionWithConnection(MultiConnection *workerConnection,
+														List *commandList)
+{
 	MarkRemoteTransactionCritical(workerConnection);
 	RemoteTransactionBegin(workerConnection);
 
@@ -351,7 +370,7 @@ SendCommandListToWorkerOutsideTransaction(const char *nodeName, int32 nodePort,
 	}
 
 	RemoteTransactionCommit(workerConnection);
-	CloseConnection(workerConnection);
+	ResetRemoteTransaction(workerConnection);
 }
 
 
@@ -430,21 +449,18 @@ SendMetadataCommandListToWorkerListInCoordinatedTransaction(List *workerNodeList
 
 
 /*
- * SendOptionalCommandListToWorkerOutsideTransaction sends the given command
- * list to the given worker in a single transaction that is outside of the
- * coordinated tranaction. If any of the commands fail, it rollbacks the
- * transaction, and otherwise commits.
+ * SendOptionalCommandListToWorkerOutsideTransactionWithConnection sends the
+ * given command list over a specified connection in a single transaction that
+ * is outside of the coordinated tranaction.
+ *
+ * If any of the commands fail, it rollbacks the transaction, and otherwise commits.
+ * A successful commit is indicated by returning true, and a failed commit by returning
+ * false.
  */
 bool
-SendOptionalCommandListToWorkerOutsideTransaction(const char *nodeName, int32 nodePort,
-												  const char *nodeUser, List *commandList)
+SendOptionalCommandListToWorkerOutsideTransactionWithConnection(
+	MultiConnection *workerConnection, List *commandList)
 {
-	int connectionFlags = FORCE_NEW_CONNECTION;
-	bool failed = false;
-
-	MultiConnection *workerConnection = GetNodeUserDatabaseConnection(connectionFlags,
-																	  nodeName, nodePort,
-																	  nodeUser, NULL);
 	if (PQstatus(workerConnection->pgConn) != CONNECTION_OK)
 	{
 		return false;
@@ -452,6 +468,7 @@ SendOptionalCommandListToWorkerOutsideTransaction(const char *nodeName, int32 no
 	RemoteTransactionBegin(workerConnection);
 
 	/* iterate over the commands and execute them in the same connection */
+	bool failed = false;
 	const char *commandString = NULL;
 	foreach_ptr(commandString, commandList)
 	{
@@ -471,6 +488,30 @@ SendOptionalCommandListToWorkerOutsideTransaction(const char *nodeName, int32 no
 		RemoteTransactionCommit(workerConnection);
 	}
 
+	ResetRemoteTransaction(workerConnection);
+
+	return !failed;
+}
+
+
+/*
+ * SendOptionalCommandListToWorkerOutsideTransaction sends the given command
+ * list to the given worker in a single transaction that is outside of the
+ * coordinated tranaction. If any of the commands fail, it rollbacks the
+ * transaction, and otherwise commits.
+ */
+bool
+SendOptionalCommandListToWorkerOutsideTransaction(const char *nodeName, int32 nodePort,
+												  const char *nodeUser, List *commandList)
+{
+	int connectionFlags = FORCE_NEW_CONNECTION;
+
+	MultiConnection *workerConnection = GetNodeUserDatabaseConnection(connectionFlags,
+																	  nodeName, nodePort,
+																	  nodeUser, NULL);
+	bool failed = SendOptionalCommandListToWorkerOutsideTransactionWithConnection(
+		workerConnection,
+		commandList);
 	CloseConnection(workerConnection);
 
 	return !failed;

src/backend/distributed/utils/enable_ssl.c

@@ -9,20 +9,22 @@
  */
 
 
+#include "postgres.h"
+
 /*
  * Make sure that functions marked as deprecated in OpenSSL 3.0 don't trigger
  * deprecation warnings by indicating that we're using the OpenSSL 1.0.1
  * compatibile API. Postgres does this by already in PG14, so we should not do
- * it otherwise we get warnings about redefining this value.
+ * it otherwise we get warnings about redefining this value. This needs to be
+ * done before including libpq.h.
  */
+#include "distributed/pg_version_constants.h"
 #if PG_VERSION_NUM < PG_VERSION_14
 #ifndef OPENSSL_API_COMPAT
 #define OPENSSL_API_COMPAT 0x1000100L
 #endif
 #endif
 
-#include "postgres.h"
-
 #include "distributed/connection_management.h"
 #include "distributed/memutils.h"
 #include "distributed/worker_protocol.h"

src/backend/distributed/utils/reference_table_utils.c

@@ -723,6 +723,12 @@ ReplicateAllReferenceTablesToNode(WorkerNode *workerNode)
 		ShardInterval *shardInterval = NULL;
 		foreach_ptr(shardInterval, referenceShardIntervalList)
 		{
+			/*
+			 * Make sure the relation exists. In some cases the relation is
+			 * actually dropped but the metadata remains, such as dropping table
+			 * while citus.enable_ddl_propagation is set to off.
+			 */
+			EnsureRelationExists(shardInterval->relationId);
 			uint64 shardId = shardInterval->shardId;
 
 			LockShardDistributionMetadata(shardId, ExclusiveLock);

src/backend/distributed/utils/resource_lock.c

@@ -781,14 +781,7 @@ SerializeNonCommutativeWrites(List *shardIntervalList, LOCKMODE lockMode)
 static bool
 AnyTableReplicated(List *shardIntervalList, List **replicatedShardIntervalList)
 {
-	if (replicatedShardIntervalList == NULL)
-	{
-		/* the caller is not interested in the replicatedShardIntervalList */
 	List *localList = NIL;
-		replicatedShardIntervalList = &localList;
-	}
-
-	*replicatedShardIntervalList = NIL;
 
 	ShardInterval *shardInterval = NULL;
 	foreach_ptr(shardInterval, shardIntervalList)
@@ -798,17 +791,22 @@ AnyTableReplicated(List *shardIntervalList, List **replicatedShardIntervalList)
 		Oid relationId = RelationIdForShard(shardId);
 		if (ReferenceTableShardId(shardId))
 		{
-			*replicatedShardIntervalList =
-				lappend(*replicatedShardIntervalList, LoadShardInterval(shardId));
+			localList =
+				lappend(localList, LoadShardInterval(shardId));
 		}
 		else if (!SingleReplicatedTable(relationId))
 		{
-			*replicatedShardIntervalList =
-				lappend(*replicatedShardIntervalList, LoadShardInterval(shardId));
+			localList =
+				lappend(localList, LoadShardInterval(shardId));
 		}
 	}
 
-	return list_length(*replicatedShardIntervalList) > 0;
+	if (replicatedShardIntervalList != NULL)
+	{
+		*replicatedShardIntervalList = localList;
+	}
+
+	return list_length(localList) > 0;
 }
 
 

src/bin/pg_send_cancellation/Makefile

@@ -10,7 +10,11 @@ PG_LDFLAGS += $(LDFLAGS)
 include $(citus_top_builddir)/Makefile.global
 
 # We reuse all the Citus flags (incl. security flags), but we are building a program not a shared library
-override CFLAGS := $(filter-out -shared,$(CFLAGS))
+# We sometimes build Citus with a newer version of gcc than Postgres was built
+# with and this breaks LTO (link-time optimization). Even if disabling it can
+# have some perf impact this is ok because pg_send_cancellation is only used
+# for tests anyway.
+override CFLAGS := $(filter-out -shared, $(CFLAGS)) -fno-lto
 
 # Filter out unneeded dependencies
 override LIBS := $(filter-out -lz -lreadline -ledit -ltermcap -lncurses -lcurses -lpam, $(LIBS))

src/include/columnar/columnar.h

@@ -314,6 +314,7 @@ extern Datum columnar_relation_storageid(PG_FUNCTION_ARGS);
 /* write_state_management.c */
 extern ColumnarWriteState * columnar_init_write_state(Relation relation, TupleDesc
 													  tupdesc,
+													  Oid tupSlotRelationId,
 													  SubTransactionId currentSubXid);
 extern void FlushWriteStateForRelfilenode(Oid relfilenode, SubTransactionId
 										  currentSubXid);

src/include/distributed/commands.h

@@ -285,7 +285,6 @@ extern bool TableHasExternalForeignKeys(Oid relationId);
 extern List * GetForeignKeyOids(Oid relationId, int flags);
 extern Oid GetReferencedTableId(Oid foreignKeyId);
 extern Oid GetReferencingTableId(Oid foreignKeyId);
-extern void EnableSkippingConstraintValidation(void);
 extern bool RelationInvolvedInAnyNonInheritedForeignKeys(Oid relationId);
 
 
@@ -541,7 +540,8 @@ extern List * PreprocessAlterTableMoveAllStmt(Node *node, const char *queryStrin
 											  ProcessUtilityContext processUtilityContext);
 extern List * PreprocessAlterTableSchemaStmt(Node *node, const char *queryString,
 											 ProcessUtilityContext processUtilityContext);
-extern void SkipForeignKeyValidationIfConstraintIsFkey(AlterTableStmt *alterTableStmt);
+extern void SkipForeignKeyValidationIfConstraintIsFkey(AlterTableStmt *alterTableStmt,
+													   bool processLocalRelation);
 extern bool IsAlterTableRenameStmt(RenameStmt *renameStmt);
 extern void ErrorIfAlterDropsPartitionColumn(AlterTableStmt *alterTableStatement);
 extern void PostprocessAlterTableStmt(AlterTableStmt *pStmt);

src/include/distributed/commands/sequence.h

@@ -10,9 +10,11 @@
 #ifndef CITUS_SEQUENCE_H
 #define CITUS_SEQUENCE_H
 
+#include "access/attnum.h"
 #include "nodes/pg_list.h"
 
 
+extern bool ColumnDefaultsToNextVal(Oid relationId, AttrNumber attrNumber);
 extern void ExtractDefaultColumnsAndOwnedSequences(Oid relationId,
 												   List **columnNameList,
 												   List **ownedSequenceIdList);

src/include/distributed/connection_management.h

@@ -189,8 +189,12 @@ typedef struct MultiConnection
 	/* information about the associated remote transaction */
 	RemoteTransaction remoteTransaction;
 
-	/* membership in list of in-progress transactions */
+	/*
+	 * membership in list of in-progress transactions and a flag to indicate
+	 * that the connection was added to this list
+	 */
 	dlist_node transactionNode;
+	bool transactionInProgress;
 
 	/* list of all placements referenced by this connection */
 	dlist_head referencedPlacements;
@@ -289,6 +293,7 @@ extern MultiConnection * StartNodeConnection(uint32 flags, const char *hostname,
 extern MultiConnection * GetNodeUserDatabaseConnection(uint32 flags, const char *hostname,
 													   int32 port, const char *user,
 													   const char *database);
+extern MultiConnection * GetConnectionForLocalQueriesOutsideTransaction(char *userName);
 extern MultiConnection * StartNodeUserDatabaseConnection(uint32 flags,
 														 const char *hostname,
 														 int32 port,

src/include/distributed/distributed_planner.h

@@ -56,7 +56,7 @@ typedef struct RelationRestriction
 {
 	Index index;
 	Oid relationId;
-	bool distributedRelation;
+	bool citusTable;
 	RangeTblEntry *rte;
 	RelOptInfo *relOptInfo;
 	PlannerInfo *plannerInfo;

src/include/distributed/metadata_utility.h

@@ -358,6 +358,8 @@ extern bool RegularTable(Oid relationId);
 extern bool TableEmpty(Oid tableId);
 extern bool IsForeignTable(Oid relationId);
 extern bool RelationUsesIdentityColumns(TupleDesc relationDesc);
+extern bool ForeignTableDropsTableNameOption(List *optionList);
+extern bool ServerUsesPostgresFdw(Oid serverId);
 extern char * ConstructQualifiedShardName(ShardInterval *shardInterval);
 extern uint64 GetFirstShardId(Oid relationId);
 extern Datum StringToDatum(char *inputString, Oid dataType);

src/include/distributed/multi_logical_replication.h

@@ -157,7 +157,8 @@ extern void DropPublications(MultiConnection *sourceConnection,
 extern void DropAllLogicalReplicationLeftovers(LogicalRepType type);
 
 extern char * PublicationName(LogicalRepType type, uint32_t nodeId, Oid ownerId);
-extern char * ReplicationSlotName(LogicalRepType type, uint32_t nodeId, Oid ownerId);
+extern char * ReplicationSlotNameForNodeAndOwner(LogicalRepType type, uint32_t nodeId, Oid
+												 ownerId);
 extern char * SubscriptionName(LogicalRepType type, Oid ownerId);
 extern char * SubscriptionRoleName(LogicalRepType type, Oid ownerId);
 
@@ -172,10 +173,6 @@ extern HTAB * CreateGroupedLogicalRepTargetsHash(List *subscriptionInfoList);
 extern void CreateGroupedLogicalRepTargetsConnections(HTAB *groupedLogicalRepTargetsHash,
 													  char *user,
 													  char *databaseName);
-extern void RecreateGroupedLogicalRepTargetsConnections(
-	HTAB *groupedLogicalRepTargetsHash,
-	char *user,
-	char *databaseName);
 extern void CloseGroupedLogicalRepTargetsConnections(HTAB *groupedLogicalRepTargetsHash);
 extern void CompleteNonBlockingShardTransfer(List *shardList,
 											 MultiConnection *sourceConnection,

src/include/distributed/remote_transaction.h

@@ -130,7 +130,6 @@ extern void MarkRemoteTransactionCritical(struct MultiConnection *connection);
  * transaction managment code.
  */
 
-extern void CloseRemoteTransaction(struct MultiConnection *connection);
 extern void ResetRemoteTransaction(struct MultiConnection *connection);
 
 /* perform handling for all in-progress transactions */

src/include/distributed/shard_cleaner.h

@@ -103,13 +103,13 @@ extern void InsertCleanupRecordInSubtransaction(CleanupObject objectType,
  * completion on failure. This will trigger cleanup of appropriate resources
  * and cleanup records.
  */
-extern void FinalizeOperationNeedingCleanupOnFailure(void);
+extern void FinalizeOperationNeedingCleanupOnFailure(const char *operationName);
 
 /*
  * FinalizeOperationNeedingCleanupOnSuccess is be called by an operation to signal
  * completion on success. This will trigger cleanup of appropriate resources
  * and cleanup records.
  */
-extern void FinalizeOperationNeedingCleanupOnSuccess(void);
+extern void FinalizeOperationNeedingCleanupOnSuccess(const char *operationName);
 
 #endif /*CITUS_SHARD_CLEANER_H */

src/include/distributed/worker_shard_copy.h

@@ -19,4 +19,9 @@ extern DestReceiver * CreateShardCopyDestReceiver(EState *executorState,
 												  List *destinationShardFullyQualifiedName,
 												  uint32_t destinationNodeId);
 
+extern const char * CopyableColumnNamesFromRelationName(const char *schemaName, const
+														char *relationName);
+
+extern const char * CopyableColumnNamesFromTupleDesc(TupleDesc tupdesc);
+
 #endif /* WORKER_SHARD_COPY_H_ */

src/include/distributed/worker_transaction.h

@@ -12,6 +12,7 @@
 #ifndef WORKER_TRANSACTION_H
 #define WORKER_TRANSACTION_H
 
+#include "distributed/connection_management.h"
 #include "distributed/worker_manager.h"
 #include "storage/lockdefs.h"
 
@@ -59,6 +60,10 @@ extern bool SendOptionalCommandListToWorkerOutsideTransaction(const char *nodeNa
 															  int32 nodePort,
 															  const char *nodeUser,
 															  List *commandList);
+extern bool SendOptionalCommandListToWorkerOutsideTransactionWithConnection(
+	MultiConnection *workerConnection,
+	List *
+	commandList);
 extern bool SendOptionalMetadataCommandListToWorkerInCoordinatedTransaction(const
 																			char *nodeName,
 																			int32 nodePort,
@@ -74,6 +79,9 @@ extern void SendCommandListToWorkerOutsideTransaction(const char *nodeName,
 													  int32 nodePort,
 													  const char *nodeUser,
 													  List *commandList);
+extern void SendCommandListToWorkerOutsideTransactionWithConnection(
+	MultiConnection *workerConnection,
+	List *commandList);
 extern void SendMetadataCommandListToWorkerListInCoordinatedTransaction(
 	List *workerNodeList,
 	const char *

src/include/pg_version_compat.h

@@ -55,6 +55,14 @@ pg_strtoint64(char *s)
 }
 
 
+/*
+ * RelationGetSmgr got backported in 13.10 and 14.7 so redefining it for any
+ * version higher causes compilation errors due to redefining of the function.
+ * We want to use it in all versions. So we backport it ourselves in earlier
+ * versions, and rely on the Postgres provided version in the later versions.
+ */
+#if PG_VERSION_NUM >= PG_VERSION_13 && PG_VERSION_NUM < 130010 \
+	|| PG_VERSION_NUM >= PG_VERSION_14 && PG_VERSION_NUM < 140007
 static inline SMgrRelation
 RelationGetSmgr(Relation rel)
 {
@@ -66,6 +74,9 @@ RelationGetSmgr(Relation rel)
 }
 
 
+#endif
+
+
 #define CREATE_SEQUENCE_COMMAND \
 	"CREATE SEQUENCE IF NOT EXISTS %s AS %s INCREMENT BY " INT64_FORMAT \
 	" MINVALUE " INT64_FORMAT " MAXVALUE " INT64_FORMAT \

src/test/regress/enterprise_isolation_logicalrep_1_schedule

@@ -6,7 +6,9 @@ test: isolation_setup
 test: isolation_cluster_management
 
 test: isolation_logical_replication_single_shard_commands
+test: isolation_logical_replication_nonsu_nonbypassrls
 test: isolation_logical_replication_multi_shard_commands
 test: isolation_non_blocking_shard_split
+test: isolation_create_distributed_concurrently_after_drop_column
 test: isolation_non_blocking_shard_split_with_index_as_replicaIdentity
 test: isolation_non_blocking_shard_split_fkey

src/test/regress/enterprise_isolation_logicalrep_2_schedule

@@ -7,3 +7,4 @@ test: isolation_cluster_management
 
 test: isolation_logical_replication_single_shard_commands_on_mx
 test: isolation_logical_replication_multi_shard_commands_on_mx
+test: isolation_logical_replication_skip_fk_validation

src/test/regress/expected/alter_distributed_table.out

@@ -100,6 +100,161 @@ SELECT STRING_AGG(table_name::text, ', ' ORDER BY 1) AS "Colocation Groups" FROM
  dist_table
 (3 rows)
 
+-- right now dist_table has columns a, b, dist_column is b, it has 6 shards
+-- column cache is: a pos 1, b pos 2
+-- let's add another column
+ALTER TABLE dist_table ADD COLUMN c int DEFAULT 1;
+-- right now column cache is: a pos 1, b pos 2, c pos 3
+-- test using alter_distributed_table to change shard count after dropping one column
+ALTER TABLE dist_table DROP COLUMN a;
+-- right now column cache is: a pos 1 attisdropped=true, b pos 2, c pos 3
+-- let's try changing the shard count
+SELECT alter_distributed_table('dist_table', shard_count := 7, cascade_to_colocated := false);
+NOTICE:  creating a new table for alter_distributed_table.dist_table
+NOTICE:  moving the data of alter_distributed_table.dist_table
+NOTICE:  dropping the old alter_distributed_table.dist_table
+NOTICE:  renaming the new table to alter_distributed_table.dist_table
+ alter_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+-- right now column cache is: b pos 1, c pos 2 because a new table has been created
+-- check that b is still distribution column
+SELECT table_name, citus_table_type, distribution_column, shard_count FROM public.citus_tables
+    WHERE table_name = 'dist_table'::regclass;
+ table_name | citus_table_type | distribution_column | shard_count
+---------------------------------------------------------------------
+ dist_table | distributed      | b                   |           7
+(1 row)
+
+-- let's add another column
+ALTER TABLE dist_table ADD COLUMN d int DEFAULT 2;
+-- right now column cache is: b pos 1, c pos 2, d pos 3, dist_column is b
+-- test using alter_distributed_table to change dist. column after dropping one column
+ALTER TABLE dist_table DROP COLUMN c;
+-- right now column cache is: b pos 1, c pos 2 attisdropped=true, d pos 3
+-- let's try changing the distribution column
+SELECT alter_distributed_table('dist_table', distribution_column := 'd');
+NOTICE:  creating a new table for alter_distributed_table.dist_table
+NOTICE:  moving the data of alter_distributed_table.dist_table
+NOTICE:  dropping the old alter_distributed_table.dist_table
+NOTICE:  renaming the new table to alter_distributed_table.dist_table
+ alter_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+-- right now column cache is: b pos 1, d pos 2 because a new table has been created
+-- check that d is the distribution column
+SELECT table_name, citus_table_type, distribution_column, shard_count FROM public.citus_tables
+    WHERE table_name = 'dist_table'::regclass;
+ table_name | citus_table_type | distribution_column | shard_count
+---------------------------------------------------------------------
+ dist_table | distributed      | d                   |           7
+(1 row)
+
+-- add another column and undistribute
+ALTER TABLE dist_table ADD COLUMN e int DEFAULT 3;
+SELECT undistribute_table('dist_table');
+NOTICE:  creating a new table for alter_distributed_table.dist_table
+NOTICE:  moving the data of alter_distributed_table.dist_table
+NOTICE:  dropping the old alter_distributed_table.dist_table
+NOTICE:  renaming the new table to alter_distributed_table.dist_table
+ undistribute_table
+---------------------------------------------------------------------
+
+(1 row)
+
+-- right now column cache is: b pos 1, d pos 2, e pos 3, table is not Citus table
+-- try dropping column and then distributing
+ALTER TABLE dist_table DROP COLUMN b;
+-- right now column cache is: b pos 1 attisdropped=true, d pos 2, e pos 3
+-- distribute with d
+SELECT create_distributed_table ('dist_table', 'd', colocate_with := 'none');
+NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is no longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$alter_distributed_table.dist_table$$)
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+-- check that d is the distribution column
+SELECT table_name, citus_table_type, distribution_column, shard_count FROM public.citus_tables
+    WHERE table_name = 'dist_table'::regclass;
+ table_name | citus_table_type | distribution_column | shard_count
+---------------------------------------------------------------------
+ dist_table | distributed      | d                   |           4
+(1 row)
+
+-- alter distribution column to e
+SELECT alter_distributed_table('dist_table', distribution_column := 'e');
+NOTICE:  creating a new table for alter_distributed_table.dist_table
+NOTICE:  moving the data of alter_distributed_table.dist_table
+NOTICE:  dropping the old alter_distributed_table.dist_table
+NOTICE:  renaming the new table to alter_distributed_table.dist_table
+ alter_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+-- right now column cache is: d pos 1, e pos 2
+-- check that e is the distribution column
+SELECT table_name, citus_table_type, distribution_column, shard_count FROM public.citus_tables
+    WHERE table_name = 'dist_table'::regclass;
+ table_name | citus_table_type | distribution_column | shard_count
+---------------------------------------------------------------------
+ dist_table | distributed      | e                   |           4
+(1 row)
+
+ALTER TABLE dist_table ADD COLUMN a int DEFAULT 4;
+ALTER TABLE dist_table ADD COLUMN b int DEFAULT 5;
+-- right now column cache is: d pos 1, e pos 2, a pos 3, b pos 4
+-- alter distribution column to a
+SELECT alter_distributed_table('dist_table', distribution_column := 'a');
+NOTICE:  creating a new table for alter_distributed_table.dist_table
+NOTICE:  moving the data of alter_distributed_table.dist_table
+NOTICE:  dropping the old alter_distributed_table.dist_table
+NOTICE:  renaming the new table to alter_distributed_table.dist_table
+ alter_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+-- right now column cache hasn't changed
+-- check that a is the distribution column
+SELECT table_name, citus_table_type, distribution_column, shard_count FROM public.citus_tables
+    WHERE table_name = 'dist_table'::regclass;
+ table_name | citus_table_type | distribution_column | shard_count
+---------------------------------------------------------------------
+ dist_table | distributed      | a                   |           4
+(1 row)
+
+ALTER TABLE dist_table DROP COLUMN d;
+ALTER TABLE dist_table DROP COLUMN e;
+-- right now column cache is: d pos 1 attisdropped=true, e pos 2 attisdropped=true, a pos 3, b pos 4
+-- alter distribution column to b
+SELECT alter_distributed_table('dist_table', distribution_column := 'b');
+NOTICE:  creating a new table for alter_distributed_table.dist_table
+NOTICE:  moving the data of alter_distributed_table.dist_table
+NOTICE:  dropping the old alter_distributed_table.dist_table
+NOTICE:  renaming the new table to alter_distributed_table.dist_table
+ alter_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+-- column cache is: a pos 1, b pos 2 -> configuration with which we started these drop column tests
+-- check that b is the distribution column
+SELECT table_name, citus_table_type, distribution_column, shard_count FROM public.citus_tables
+    WHERE table_name = 'dist_table'::regclass;
+ table_name | citus_table_type | distribution_column | shard_count
+---------------------------------------------------------------------
+ dist_table | distributed      | b                   |           4
+(1 row)
+
 -- test altering colocation, note that shard count will also change
 SELECT alter_distributed_table('dist_table', colocate_with := 'alter_distributed_table.colocation_table');
 NOTICE:  creating a new table for alter_distributed_table.dist_table

src/test/regress/expected/alter_role_propagation.out

@@ -319,5 +319,42 @@ SELECT COUNT(*) FROM public.test_search_path;
 (1 row)
 
 ALTER USER current_user RESET search_path;
+-- test empty/null password: it is treated the same as no password
+SET password_encryption TO md5;
+CREATE ROLE new_role;
+SELECT workers.result AS worker_password, pg_authid.rolpassword AS coord_password FROM run_command_on_workers($$SELECT rolpassword FROM pg_authid WHERE rolname = 'new_role'$$) workers, pg_authid WHERE pg_authid.rolname = 'new_role';
+ worker_password | coord_password
+---------------------------------------------------------------------
+                 |
+                 |
+(2 rows)
+
+ALTER ROLE new_role PASSWORD '';
+NOTICE:  empty string is not a valid password, clearing password
+SELECT workers.result AS worker_password, pg_authid.rolpassword AS coord_password FROM run_command_on_workers($$SELECT rolpassword FROM pg_authid WHERE rolname = 'new_role'$$) workers, pg_authid WHERE pg_authid.rolname = 'new_role';
+ worker_password | coord_password
+---------------------------------------------------------------------
+                 |
+                 |
+(2 rows)
+
+ALTER ROLE new_role PASSWORD 'new_password';
+SELECT workers.result AS worker_password, pg_authid.rolpassword AS coord_password, workers.result = pg_authid.rolpassword AS password_is_same FROM run_command_on_workers($$SELECT rolpassword FROM pg_authid WHERE rolname = 'new_role'$$) workers, pg_authid WHERE pg_authid.rolname = 'new_role';
+           worker_password           |           coord_password            | password_is_same
+---------------------------------------------------------------------
+ md51a28da0f1a2416525eec435bdce8cbbe | md51a28da0f1a2416525eec435bdce8cbbe | t
+ md51a28da0f1a2416525eec435bdce8cbbe | md51a28da0f1a2416525eec435bdce8cbbe | t
+(2 rows)
+
+ALTER ROLE new_role PASSWORD NULL;
+SELECT workers.result AS worker_password, pg_authid.rolpassword AS coord_password FROM run_command_on_workers($$SELECT rolpassword FROM pg_authid WHERE rolname = 'new_role'$$) workers, pg_authid WHERE pg_authid.rolname = 'new_role';
+ worker_password | coord_password
+---------------------------------------------------------------------
+                 |
+                 |
+(2 rows)
+
+RESET password_encryption;
+DROP ROLE new_role;
 DROP TABLE test_search_path;
 DROP SCHEMA alter_role, ",CitUs,.TeeN!?", test_sp CASCADE;

src/test/regress/expected/background_rebalance.out

@@ -176,5 +176,128 @@ SELECT citus_rebalance_wait();
 
 (1 row)
 
+DROP TABLE t1;
+-- make sure a non-super user can stop rebalancing
+CREATE USER non_super_user_rebalance WITH LOGIN;
+GRANT ALL ON SCHEMA background_rebalance TO non_super_user_rebalance;
+SET ROLE non_super_user_rebalance;
+CREATE TABLE non_super_user_t1 (a int PRIMARY KEY);
+SELECT create_distributed_table('non_super_user_t1', 'a', shard_count => 4, colocate_with => 'none');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+SELECT citus_move_shard_placement(85674008, 'localhost', :worker_1_port, 'localhost', :worker_2_port, shard_transfer_mode => 'block_writes');
+ citus_move_shard_placement
+---------------------------------------------------------------------
+
+(1 row)
+
+SELECT 1 FROM citus_rebalance_start();
+NOTICE:  Scheduled 1 moves as job xxx
+DETAIL:  Rebalance scheduled as background job
+HINT:  To monitor progress, run: SELECT * FROM pg_dist_background_task WHERE job_id = xxx ORDER BY task_id ASC; or SELECT * FROM get_rebalance_progress();
+ ?column?
+---------------------------------------------------------------------
+        1
+(1 row)
+
+SELECT citus_rebalance_stop();
+ citus_rebalance_stop
+---------------------------------------------------------------------
+
+(1 row)
+
+RESET ROLE;
+SET citus.replicate_reference_tables_on_activate = false;
+CREATE TABLE ref_no_pk(a int);
+SELECT create_reference_table('ref_no_pk');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE TABLE ref_with_pk(a int primary key);
+SELECT create_reference_table('ref_with_pk');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+-- Add coordinator so there's a node which doesn't have the reference tables
+SELECT 1 FROM citus_add_node('localhost', :master_port, groupId=>0);
+NOTICE:  localhost:xxxxx is the coordinator and already contains metadata, skipping syncing the metadata
+ ?column?
+---------------------------------------------------------------------
+        1
+(1 row)
+
+-- fails
+BEGIN;
+SELECT 1 FROM citus_rebalance_start();
+ERROR:  cannot use logical replication to transfer shards of the relation ref_no_pk since it doesn't have a REPLICA IDENTITY or PRIMARY KEY
+DETAIL:  UPDATE and DELETE commands on the shard will error out during logical replication unless there is a REPLICA IDENTITY or PRIMARY KEY.
+HINT:  If you wish to continue without a replica identity set the shard_transfer_mode to 'force_logical' or 'block_writes'.
+ROLLBACK;
+-- success
+BEGIN;
+SELECT 1 FROM citus_rebalance_start(shard_transfer_mode := 'force_logical');
+NOTICE:  Scheduled 1 moves as job xxx
+DETAIL:  Rebalance scheduled as background job
+HINT:  To monitor progress, run: SELECT * FROM pg_dist_background_task WHERE job_id = xxx ORDER BY task_id ASC; or SELECT * FROM get_rebalance_progress();
+ ?column?
+---------------------------------------------------------------------
+        1
+(1 row)
+
+ROLLBACK;
+-- success
+BEGIN;
+SELECT 1 FROM citus_rebalance_start(shard_transfer_mode := 'block_writes');
+NOTICE:  Scheduled 1 moves as job xxx
+DETAIL:  Rebalance scheduled as background job
+HINT:  To monitor progress, run: SELECT * FROM pg_dist_background_task WHERE job_id = xxx ORDER BY task_id ASC; or SELECT * FROM get_rebalance_progress();
+ ?column?
+---------------------------------------------------------------------
+        1
+(1 row)
+
+ROLLBACK;
+-- fails
+SELECT 1 FROM citus_rebalance_start();
+ERROR:  cannot use logical replication to transfer shards of the relation ref_no_pk since it doesn't have a REPLICA IDENTITY or PRIMARY KEY
+DETAIL:  UPDATE and DELETE commands on the shard will error out during logical replication unless there is a REPLICA IDENTITY or PRIMARY KEY.
+HINT:  If you wish to continue without a replica identity set the shard_transfer_mode to 'force_logical' or 'block_writes'.
+-- succeeds
+SELECT 1 FROM citus_rebalance_start(shard_transfer_mode := 'force_logical');
+NOTICE:  Scheduled 1 moves as job xxx
+DETAIL:  Rebalance scheduled as background job
+HINT:  To monitor progress, run: SELECT * FROM pg_dist_background_task WHERE job_id = xxx ORDER BY task_id ASC; or SELECT * FROM get_rebalance_progress();
+ ?column?
+---------------------------------------------------------------------
+        1
+(1 row)
+
+-- wait for success
+SELECT citus_rebalance_wait();
+ citus_rebalance_wait
+---------------------------------------------------------------------
+
+(1 row)
+
+-- Remove coordinator again to allow rerunning of this test
+SELECT 1 FROM citus_remove_node('localhost', :master_port);
+ ?column?
+---------------------------------------------------------------------
+        1
+(1 row)
+
+SELECT public.wait_until_metadata_sync(30000);
+ wait_until_metadata_sync
+---------------------------------------------------------------------
+
+(1 row)
+
 SET client_min_messages TO WARNING;
 DROP SCHEMA background_rebalance CASCADE;

src/test/regress/expected/citus_local_table_triggers.out

@@ -232,6 +232,30 @@ SELECT * FROM citus_local_table_triggers
  truncate_trigger_xxxxxxx  | "interesting!schema"."citus_local!_table"         | O
 (3 rows)
 
+-- ALTER TABLE ENABLE REPLICA trigger
+ALTER TABLE "interesting!schema"."citus_local!_table" ENABLE REPLICA TRIGGER "trigger\'name22";
+NOTICE:  executing the command locally: SELECT worker_apply_shard_ddl_command (1507008, 'interesting!schema', E'ALTER TABLE "interesting!schema"."citus_local!_table" ENABLE REPLICA TRIGGER "trigger\\''name22";')
+SELECT * FROM citus_local_table_triggers
+    WHERE tgname NOT LIKE 'RI_ConstraintTrigger%';
+         tgname          |                      tgrelid                      | tgenabled
+---------------------------------------------------------------------
+ trigger\'name22         | "interesting!schema"."citus_local!_table"         | R
+ trigger\'name22_1507008 | "interesting!schema"."citus_local!_table_1507008" | R
+ truncate_trigger_xxxxxxx  | "interesting!schema"."citus_local!_table"         | O
+(3 rows)
+
+-- ALTER TABLE ENABLE ALWAYS trigger
+ALTER TABLE "interesting!schema"."citus_local!_table" ENABLE ALWAYS TRIGGER "trigger\'name22";
+NOTICE:  executing the command locally: SELECT worker_apply_shard_ddl_command (1507008, 'interesting!schema', E'ALTER TABLE "interesting!schema"."citus_local!_table" ENABLE ALWAYS TRIGGER "trigger\\''name22";')
+SELECT * FROM citus_local_table_triggers
+    WHERE tgname NOT LIKE 'RI_ConstraintTrigger%';
+         tgname          |                      tgrelid                      | tgenabled
+---------------------------------------------------------------------
+ trigger\'name22         | "interesting!schema"."citus_local!_table"         | A
+ trigger\'name22_1507008 | "interesting!schema"."citus_local!_table_1507008" | A
+ truncate_trigger_xxxxxxx  | "interesting!schema"."citus_local!_table"         | O
+(3 rows)
+
 -- ALTER TABLE DISABLE trigger
 ALTER TABLE "interesting!schema"."citus_local!_table" DISABLE TRIGGER "trigger\'name22";
 NOTICE:  executing the command locally: SELECT worker_apply_shard_ddl_command (1507008, 'interesting!schema', E'ALTER TABLE "interesting!schema"."citus_local!_table" DISABLE TRIGGER "trigger\\''name22";')

src/test/regress/expected/citus_local_tables.out

@@ -40,7 +40,13 @@ SELECT 1 FROM master_remove_node('localhost', :master_port);
 CREATE TABLE citus_local_table_1 (a int primary key);
 -- this should fail as coordinator is removed from pg_dist_node
 SELECT citus_add_local_table_to_metadata('citus_local_table_1');
-ERROR:  could not find the coordinator node in metadata as it is not added as a worker
+ERROR:  operation is not allowed when coordinator is not added into metadata
+-- This should also fail as coordinator is removed from pg_dist_node.
+--
+-- This is not a great place to test this but is one of those places that we
+-- have workers in metadata but not the coordinator.
+SELECT create_distributed_table_concurrently('citus_local_table_1', 'a');
+ERROR:  operation is not allowed when coordinator is not added into metadata
 -- let coordinator have citus local tables again for next tests
 set client_min_messages to ERROR;
 SELECT 1 FROM master_add_node('localhost', :master_port, groupId => 0);
@@ -223,7 +229,7 @@ ROLLBACK;
 CREATE FOREIGN TABLE foreign_table (
   id bigint not null,
   full_name text not null default ''
-) SERVER fake_fdw_server OPTIONS (encoding 'utf-8', compression 'true');
+) SERVER fake_fdw_server OPTIONS (encoding 'utf-8', compression 'true', table_name 'foreign_table');
 -- observe that we do not create fdw server for shell table, both shard relation
 -- & shell relation points to the same same server object
 -- Disable metadata sync since citus doesn't support distributing

src/test/regress/expected/citus_non_blocking_split_shards.out

@@ -60,7 +60,7 @@ SELECT create_reference_table('reference_table');
 
 (1 row)
 
-CREATE TABLE colocated_dist_table (measureid integer PRIMARY KEY);
+CREATE TABLE colocated_dist_table (measureid integer PRIMARY KEY, genid integer GENERATED ALWAYS AS ( measureid + 3 ) stored, value varchar(44), col_todrop integer);
 CLUSTER colocated_dist_table USING colocated_dist_table_pkey;
 SELECT create_distributed_table('colocated_dist_table', 'measureid', colocate_with:='sensors');
  create_distributed_table
@@ -84,8 +84,9 @@ ALTER TABLE sensors ADD CONSTRAINT fkey_table_to_dist FOREIGN KEY (measureid) RE
 -- END : Create Foreign key constraints.
 -- BEGIN : Load data into tables.
 INSERT INTO reference_table SELECT i FROM generate_series(0,1000)i;
-INSERT INTO colocated_dist_table SELECT i FROM generate_series(0,1000)i;
+INSERT INTO colocated_dist_table(measureid, value, col_todrop) SELECT i,'Value',i FROM generate_series(0,1000)i;
 INSERT INTO sensors SELECT i, '2020-01-05', '{}', 11011.10, 'A', 'I <3 Citus' FROM generate_series(0,1000)i;
+ALTER TABLE colocated_dist_table DROP COLUMN col_todrop;
 SELECT COUNT(*) FROM sensors;
  count
 ---------------------------------------------------------------------

src/test/regress/expected/citus_split_shard_by_split_points.out

@@ -56,7 +56,7 @@ SELECT create_reference_table('reference_table');
 
 (1 row)
 
-CREATE TABLE colocated_dist_table (measureid integer PRIMARY KEY);
+CREATE TABLE colocated_dist_table (measureid integer PRIMARY KEY, genid integer GENERATED ALWAYS AS ( measureid + 3 ) stored, value varchar(44), col_todrop integer);
 CLUSTER colocated_dist_table USING colocated_dist_table_pkey;
 SELECT create_distributed_table('colocated_dist_table', 'measureid', colocate_with:='sensors');
  create_distributed_table
@@ -80,8 +80,9 @@ ALTER TABLE sensors ADD CONSTRAINT fkey_table_to_dist FOREIGN KEY (measureid) RE
 -- END : Create Foreign key constraints.
 -- BEGIN : Load data into tables.
 INSERT INTO reference_table SELECT i FROM generate_series(0,1000)i;
-INSERT INTO colocated_dist_table SELECT i FROM generate_series(0,1000)i;
+INSERT INTO colocated_dist_table(measureid, value, col_todrop) SELECT i,'Value',i FROM generate_series(0,1000)i;
 INSERT INTO sensors SELECT i, '2020-01-05', '{}', 11011.10, 'A', 'I <3 Citus' FROM generate_series(0,1000)i;
+ALTER TABLE colocated_dist_table DROP COLUMN col_todrop;
 SELECT COUNT(*) FROM sensors;
  count
 ---------------------------------------------------------------------

src/test/regress/expected/citus_table_triggers.out

@@ -146,9 +146,12 @@ SELECT master_get_table_ddl_events('test_table');
  CREATE TABLE table_triggers_schema.test_table (id integer, text_number text, text_col text)
  ALTER TABLE table_triggers_schema.test_table OWNER TO postgres
  CREATE TRIGGER test_table_delete AFTER DELETE ON table_triggers_schema.test_table FOR EACH STATEMENT EXECUTE FUNCTION table_triggers_schema.test_table_trigger_function()
+ ALTER TABLE table_triggers_schema.test_table ENABLE TRIGGER test_table_delete;
  CREATE CONSTRAINT TRIGGER test_table_insert AFTER INSERT ON table_triggers_schema.test_table DEFERRABLE INITIALLY IMMEDIATE FOR EACH ROW WHEN (((new.id > 5) OR ((new.text_col IS NOT NULL) AND ((new.id)::numeric < to_number(new.text_number, '9999'::text))))) EXECUTE FUNCTION table_triggers_schema.test_table_trigger_function()
+ ALTER TABLE table_triggers_schema.test_table ENABLE TRIGGER test_table_insert;
  CREATE CONSTRAINT TRIGGER test_table_update AFTER UPDATE OF id ON table_triggers_schema.test_table NOT DEFERRABLE INITIALLY IMMEDIATE FOR EACH ROW WHEN (((NOT (old.* IS DISTINCT FROM new.*)) AND (old.text_number IS NOT NULL))) EXECUTE FUNCTION table_triggers_schema.test_table_trigger_function()
-(5 rows)
+ ALTER TABLE table_triggers_schema.test_table ENABLE TRIGGER test_table_update;
+(8 rows)
 
 -- cleanup at exit
 DROP SCHEMA table_triggers_schema CASCADE;

src/test/regress/expected/citus_update_table_statistics.out

@@ -64,11 +64,11 @@ SET citus.multi_shard_modify_mode TO sequential;
 SELECT citus_update_table_statistics('test_table_statistics_hash');
 NOTICE:  issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx');
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-NOTICE:  issuing SELECT 981000 AS shard_id, 'public.test_table_statistics_hash_981000' AS shard_name, pg_relation_size('public.test_table_statistics_hash_981000') UNION ALL SELECT 981001 AS shard_id, 'public.test_table_statistics_hash_981001' AS shard_name, pg_relation_size('public.test_table_statistics_hash_981001') UNION ALL SELECT 981002 AS shard_id, 'public.test_table_statistics_hash_981002' AS shard_name, pg_relation_size('public.test_table_statistics_hash_981002') UNION ALL SELECT 981003 AS shard_id, 'public.test_table_statistics_hash_981003' AS shard_name, pg_relation_size('public.test_table_statistics_hash_981003') UNION ALL SELECT 981004 AS shard_id, 'public.test_table_statistics_hash_981004' AS shard_name, pg_relation_size('public.test_table_statistics_hash_981004') UNION ALL SELECT 981005 AS shard_id, 'public.test_table_statistics_hash_981005' AS shard_name, pg_relation_size('public.test_table_statistics_hash_981005') UNION ALL SELECT 981006 AS shard_id, 'public.test_table_statistics_hash_981006' AS shard_name, pg_relation_size('public.test_table_statistics_hash_981006') UNION ALL SELECT 981007 AS shard_id, 'public.test_table_statistics_hash_981007' AS shard_name, pg_relation_size('public.test_table_statistics_hash_981007') UNION ALL SELECT 0::bigint, NULL::text, 0::bigint;
+NOTICE:  issuing SELECT 981000 AS shard_id, 'public.test_table_statistics_hash_981000' AS shard_name, pg_total_relation_size('public.test_table_statistics_hash_981000') UNION ALL SELECT 981001 AS shard_id, 'public.test_table_statistics_hash_981001' AS shard_name, pg_total_relation_size('public.test_table_statistics_hash_981001') UNION ALL SELECT 981002 AS shard_id, 'public.test_table_statistics_hash_981002' AS shard_name, pg_total_relation_size('public.test_table_statistics_hash_981002') UNION ALL SELECT 981003 AS shard_id, 'public.test_table_statistics_hash_981003' AS shard_name, pg_total_relation_size('public.test_table_statistics_hash_981003') UNION ALL SELECT 981004 AS shard_id, 'public.test_table_statistics_hash_981004' AS shard_name, pg_total_relation_size('public.test_table_statistics_hash_981004') UNION ALL SELECT 981005 AS shard_id, 'public.test_table_statistics_hash_981005' AS shard_name, pg_total_relation_size('public.test_table_statistics_hash_981005') UNION ALL SELECT 981006 AS shard_id, 'public.test_table_statistics_hash_981006' AS shard_name, pg_total_relation_size('public.test_table_statistics_hash_981006') UNION ALL SELECT 981007 AS shard_id, 'public.test_table_statistics_hash_981007' AS shard_name, pg_total_relation_size('public.test_table_statistics_hash_981007') UNION ALL SELECT 0::bigint, NULL::text, 0::bigint;
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 NOTICE:  issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx');
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-NOTICE:  issuing SELECT 981000 AS shard_id, 'public.test_table_statistics_hash_981000' AS shard_name, pg_relation_size('public.test_table_statistics_hash_981000') UNION ALL SELECT 981001 AS shard_id, 'public.test_table_statistics_hash_981001' AS shard_name, pg_relation_size('public.test_table_statistics_hash_981001') UNION ALL SELECT 981002 AS shard_id, 'public.test_table_statistics_hash_981002' AS shard_name, pg_relation_size('public.test_table_statistics_hash_981002') UNION ALL SELECT 981003 AS shard_id, 'public.test_table_statistics_hash_981003' AS shard_name, pg_relation_size('public.test_table_statistics_hash_981003') UNION ALL SELECT 981004 AS shard_id, 'public.test_table_statistics_hash_981004' AS shard_name, pg_relation_size('public.test_table_statistics_hash_981004') UNION ALL SELECT 981005 AS shard_id, 'public.test_table_statistics_hash_981005' AS shard_name, pg_relation_size('public.test_table_statistics_hash_981005') UNION ALL SELECT 981006 AS shard_id, 'public.test_table_statistics_hash_981006' AS shard_name, pg_relation_size('public.test_table_statistics_hash_981006') UNION ALL SELECT 981007 AS shard_id, 'public.test_table_statistics_hash_981007' AS shard_name, pg_relation_size('public.test_table_statistics_hash_981007') UNION ALL SELECT 0::bigint, NULL::text, 0::bigint;
+NOTICE:  issuing SELECT 981000 AS shard_id, 'public.test_table_statistics_hash_981000' AS shard_name, pg_total_relation_size('public.test_table_statistics_hash_981000') UNION ALL SELECT 981001 AS shard_id, 'public.test_table_statistics_hash_981001' AS shard_name, pg_total_relation_size('public.test_table_statistics_hash_981001') UNION ALL SELECT 981002 AS shard_id, 'public.test_table_statistics_hash_981002' AS shard_name, pg_total_relation_size('public.test_table_statistics_hash_981002') UNION ALL SELECT 981003 AS shard_id, 'public.test_table_statistics_hash_981003' AS shard_name, pg_total_relation_size('public.test_table_statistics_hash_981003') UNION ALL SELECT 981004 AS shard_id, 'public.test_table_statistics_hash_981004' AS shard_name, pg_total_relation_size('public.test_table_statistics_hash_981004') UNION ALL SELECT 981005 AS shard_id, 'public.test_table_statistics_hash_981005' AS shard_name, pg_total_relation_size('public.test_table_statistics_hash_981005') UNION ALL SELECT 981006 AS shard_id, 'public.test_table_statistics_hash_981006' AS shard_name, pg_total_relation_size('public.test_table_statistics_hash_981006') UNION ALL SELECT 981007 AS shard_id, 'public.test_table_statistics_hash_981007' AS shard_name, pg_total_relation_size('public.test_table_statistics_hash_981007') UNION ALL SELECT 0::bigint, NULL::text, 0::bigint;
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 NOTICE:  issuing COMMIT
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
@@ -152,11 +152,11 @@ SET citus.multi_shard_modify_mode TO sequential;
 SELECT citus_update_table_statistics('test_table_statistics_append');
 NOTICE:  issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx');
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-NOTICE:  issuing SELECT 981008 AS shard_id, 'public.test_table_statistics_append_981008' AS shard_name, pg_relation_size('public.test_table_statistics_append_981008') UNION ALL SELECT 981009 AS shard_id, 'public.test_table_statistics_append_981009' AS shard_name, pg_relation_size('public.test_table_statistics_append_981009') UNION ALL SELECT 0::bigint, NULL::text, 0::bigint;
+NOTICE:  issuing SELECT 981008 AS shard_id, 'public.test_table_statistics_append_981008' AS shard_name, pg_total_relation_size('public.test_table_statistics_append_981008') UNION ALL SELECT 981009 AS shard_id, 'public.test_table_statistics_append_981009' AS shard_name, pg_total_relation_size('public.test_table_statistics_append_981009') UNION ALL SELECT 0::bigint, NULL::text, 0::bigint;
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 NOTICE:  issuing BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED;SELECT assign_distributed_transaction_id(xx, xx, 'xxxxxxx');
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
-NOTICE:  issuing SELECT 981008 AS shard_id, 'public.test_table_statistics_append_981008' AS shard_name, pg_relation_size('public.test_table_statistics_append_981008') UNION ALL SELECT 981009 AS shard_id, 'public.test_table_statistics_append_981009' AS shard_name, pg_relation_size('public.test_table_statistics_append_981009') UNION ALL SELECT 0::bigint, NULL::text, 0::bigint;
+NOTICE:  issuing SELECT 981008 AS shard_id, 'public.test_table_statistics_append_981008' AS shard_name, pg_total_relation_size('public.test_table_statistics_append_981008') UNION ALL SELECT 981009 AS shard_id, 'public.test_table_statistics_append_981009' AS shard_name, pg_total_relation_size('public.test_table_statistics_append_981009') UNION ALL SELECT 0::bigint, NULL::text, 0::bigint;
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx
 NOTICE:  issuing COMMIT
 DETAIL:  on server postgres@localhost:xxxxx connectionId: xxxxxxx

src/test/regress/expected/columnar_alter_set_type.out

@@ -50,3 +50,28 @@ SELECT * FROM test_alter_table ORDER BY a;
 (4 rows)
 
 DROP TABLE test_alter_table;
+--  Make sure that the correct table options are used when rewriting the table.
+-- This is reflected by the VACUUM VERBOSE output right after a rewrite showing
+-- that all chunks are compressed with the configured compression algorithm
+-- https://github.com/citusdata/citus/issues/5927
+CREATE TABLE test(i int) USING columnar;
+ALTER TABLE test SET (columnar.compression = lz4);
+INSERT INTO test VALUES(1);
+VACUUM VERBOSE test;
+INFO:  statistics for "test":
+storage id: xxxxx
+total file size: 24576, total data size: 6
+compression rate: 0.83x
+total row count: 1, stripe count: 1, average rows per stripe: 1
+chunk count: 1, containing data for dropped columns: 0, lz4 compressed: 1
+
+ALTER TABLE test ALTER COLUMN i TYPE int8;
+VACUUM VERBOSE test;
+INFO:  statistics for "test":
+storage id: xxxxx
+total file size: 24576, total data size: 10
+compression rate: 0.90x
+total row count: 1, stripe count: 1, average rows per stripe: 1
+chunk count: 1, containing data for dropped columns: 0, lz4 compressed: 1
+
+DROP TABLE test;

src/test/regress/expected/columnar_create.out

@@ -55,6 +55,89 @@ SELECT columnar_test_helpers.columnar_metadata_has_storage_id(:columnar_table_1_
  t
 (1 row)
 
+BEGIN;
+  INSERT INTO columnar_table_1 VALUES (2);
+ROLLBACK;
+INSERT INTO columnar_table_1 VALUES (3),(4);
+INSERT INTO columnar_table_1 VALUES (5),(6);
+INSERT INTO columnar_table_1 VALUES (7),(8);
+-- Test whether columnar metadata accessors are still fine even
+-- when the metadata indexes are not available to them.
+BEGIN;
+  ALTER INDEX columnar_internal.stripe_first_row_number_idx RENAME TO new_index_name;
+  ALTER INDEX columnar_internal.chunk_pkey RENAME TO new_index_name_1;
+  ALTER INDEX columnar_internal.stripe_pkey RENAME TO new_index_name_2;
+  ALTER INDEX columnar_internal.chunk_group_pkey RENAME TO new_index_name_3;
+  CREATE INDEX columnar_table_1_idx ON columnar_table_1(a);
+WARNING:  Metadata index stripe_first_row_number_idx is not available, this might mean slower read/writes on columnar tables. This is expected during Postgres upgrades and not expected otherwise.
+WARNING:  Metadata index stripe_first_row_number_idx is not available, this might mean slower read/writes on columnar tables. This is expected during Postgres upgrades and not expected otherwise.
+WARNING:  Metadata index chunk_pkey is not available, this might mean slower read/writes on columnar tables. This is expected during Postgres upgrades and not expected otherwise.
+WARNING:  Metadata index chunk_group_pkey is not available, this might mean slower read/writes on columnar tables. This is expected during Postgres upgrades and not expected otherwise.
+  -- make sure that we test index scan
+  SET LOCAL columnar.enable_custom_scan TO 'off';
+  SET LOCAL enable_seqscan TO off;
+  SET LOCAL seq_page_cost TO 10000000;
+  SELECT * FROM columnar_table_1 WHERE a = 6;
+WARNING:  Metadata index stripe_first_row_number_idx is not available, this might mean slower read/writes on columnar tables. This is expected during Postgres upgrades and not expected otherwise.
+ a
+---------------------------------------------------------------------
+ 6
+(1 row)
+
+  SELECT * FROM columnar_table_1 WHERE a = 5;
+ a
+---------------------------------------------------------------------
+ 5
+(1 row)
+
+  SELECT * FROM columnar_table_1 WHERE a = 7;
+ a
+---------------------------------------------------------------------
+ 7
+(1 row)
+
+  SELECT * FROM columnar_table_1 WHERE a = 3;
+ a
+---------------------------------------------------------------------
+ 3
+(1 row)
+
+  DROP INDEX columnar_table_1_idx;
+  -- Re-shuffle some metadata records to test whether we can
+  -- rely on sequential metadata scan when the metadata records
+  -- are not ordered by their "first_row_number"s.
+  WITH cte AS (
+      DELETE FROM columnar_internal.stripe
+      WHERE storage_id = columnar.get_storage_id('columnar_table_1')
+      RETURNING *
+  )
+  INSERT INTO columnar_internal.stripe SELECT * FROM cte ORDER BY first_row_number DESC;
+  SELECT SUM(a) FROM columnar_table_1;
+ sum
+---------------------------------------------------------------------
+  34
+(1 row)
+
+  SELECT * FROM columnar_table_1 WHERE a = 6;
+ a
+---------------------------------------------------------------------
+ 6
+(1 row)
+
+  -- Run a SELECT query after the INSERT command to force flushing the
+  -- data within the xact block.
+  INSERT INTO columnar_table_1 VALUES (20);
+  SELECT COUNT(*) FROM columnar_table_1;
+WARNING:  Metadata index stripe_pkey is not available, this might mean slower read/writes on columnar tables. This is expected during Postgres upgrades and not expected otherwise.
+ count
+---------------------------------------------------------------------
+     8
+(1 row)
+
+  DROP TABLE columnar_table_1 CASCADE;
+NOTICE:  drop cascades to materialized view columnar_table_1_mv
+WARNING:  Metadata index on a columnar metadata table is not available, this might mean slower read/writes on columnar tables. This is expected during Postgres upgrades and not expected otherwise.
+ROLLBACK;
 -- test dropping columnar table
 DROP TABLE columnar_table_1 CASCADE;
 NOTICE:  drop cascades to materialized view columnar_table_1_mv

src/test/regress/expected/columnar_indexes.out

@@ -257,6 +257,32 @@ SELECT SUM(a)=48000 FROM columnar_table WHERE a = 16000 OR a = 32000;
  t
 (1 row)
 
+BEGIN;
+  ALTER INDEX columnar_internal.stripe_first_row_number_idx RENAME TO new_index_name;
+  ALTER INDEX columnar_internal.chunk_pkey RENAME TO new_index_name_1;
+  -- same queries but this time some metadata indexes are not available
+  SELECT SUM(a)=312487500 FROM columnar_table WHERE a < 25000;
+WARNING:  Metadata index stripe_first_row_number_idx is not available, this might mean slower read/writes on columnar tables. This is expected during Postgres upgrades and not expected otherwise.
+WARNING:  Metadata index stripe_first_row_number_idx is not available, this might mean slower read/writes on columnar tables. This is expected during Postgres upgrades and not expected otherwise.
+WARNING:  Metadata index chunk_pkey is not available, this might mean slower read/writes on columnar tables. This is expected during Postgres upgrades and not expected otherwise.
+ ?column?
+---------------------------------------------------------------------
+ t
+(1 row)
+
+  SELECT SUM(a)=167000 FROM columnar_table WHERE a = 16000 OR a = 151000;
+ ?column?
+---------------------------------------------------------------------
+ t
+(1 row)
+
+  SELECT SUM(a)=48000 FROM columnar_table WHERE a = 16000 OR a = 32000;
+ ?column?
+---------------------------------------------------------------------
+ t
+(1 row)
+
+ROLLBACK;
 TRUNCATE columnar_table;
 ALTER TABLE columnar_table DROP CONSTRAINT columnar_table_pkey;
 -- hash --

src/test/regress/expected/create_distributed_table_concurrently.out

@@ -57,6 +57,35 @@ ERROR:  cannot colocate tables nocolo and test
 DETAIL:  Distribution column types don't match for nocolo and test.
 select create_distributed_table_concurrently('test','key', colocate_with := 'noexists');
 ERROR:  relation "noexists" does not exist
+select citus_set_node_property('localhost', :worker_1_port, 'shouldhaveshards', false);
+ citus_set_node_property
+---------------------------------------------------------------------
+
+(1 row)
+
+select citus_set_node_property('localhost', :worker_2_port, 'shouldhaveshards', false);
+ citus_set_node_property
+---------------------------------------------------------------------
+
+(1 row)
+
+select create_distributed_table_concurrently('test','key');
+NOTICE:  relation test does not have a REPLICA IDENTITY or PRIMARY KEY
+DETAIL:  UPDATE and DELETE commands on the relation will error out during create_distributed_table_concurrently unless there is a REPLICA IDENTITY or PRIMARY KEY. INSERT commands will still work.
+ERROR:  no worker nodes are available for placing shards
+HINT:  Add more worker nodes.
+select citus_set_node_property('localhost', :worker_1_port, 'shouldhaveshards', true);
+ citus_set_node_property
+---------------------------------------------------------------------
+
+(1 row)
+
+select citus_set_node_property('localhost', :worker_2_port, 'shouldhaveshards', true);
+ citus_set_node_property
+---------------------------------------------------------------------
+
+(1 row)
+
 -- use colocate_with "default"
 select create_distributed_table_concurrently('test','key', shard_count := 11);
 NOTICE:  relation test does not have a REPLICA IDENTITY or PRIMARY KEY

src/test/regress/expected/cte_inline.out

@@ -463,29 +463,6 @@ DEBUG:  Creating router plan
 (5 rows)
 
 \set VERBOSITY default
--- enable_group_by_reordering is a new GUC introduced in PG15
--- it does some optimization of the order of group by keys which results
--- in a different explain output plan between PG13/14 and PG15
--- Hence we set that GUC to off.
-SHOW server_version \gset
-SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15
-\gset
-\if :server_version_ge_15
-SET enable_group_by_reordering TO off;
-\endif
-SELECT DISTINCT 1 FROM run_command_on_workers($$ALTER SYSTEM SET enable_group_by_reordering TO off;$$);
- ?column?
----------------------------------------------------------------------
-        1
-(1 row)
-
-SELECT run_command_on_workers($$SELECT pg_reload_conf()$$);
- run_command_on_workers
----------------------------------------------------------------------
- (localhost,57637,t,t)
- (localhost,57638,t,t)
-(2 rows)
-
 EXPLAIN (COSTS OFF) WITH cte_1 AS NOT MATERIALIZED (SELECT * FROM test_table)
 SELECT
 	count(*)
@@ -524,22 +501,6 @@ DEBUG:  join prunable for intervals [1073741824,2147483647] and [0,1073741823]
                                  ->  Seq Scan on test_table_1960000 test_table_1
 (12 rows)
 
-\if :server_version_ge_15
-RESET enable_group_by_reordering;
-\endif
-SELECT DISTINCT 1 FROM run_command_on_workers($$ALTER SYSTEM RESET enable_group_by_reordering;$$);
- ?column?
----------------------------------------------------------------------
-        1
-(1 row)
-
-SELECT run_command_on_workers($$SELECT pg_reload_conf()$$);
- run_command_on_workers
----------------------------------------------------------------------
- (localhost,57637,t,t)
- (localhost,57638,t,t)
-(2 rows)
-
 -- ctes with volatile functions are not
 -- inlined
 WITH cte_1 AS (SELECT *, random() FROM test_table)

src/test/regress/expected/cte_inline_0.out

@@ -463,29 +463,6 @@ DEBUG:  Creating router plan
 (5 rows)
 
 \set VERBOSITY default
--- enable_group_by_reordering is a new GUC introduced in PG15
--- it does some optimization of the order of group by keys which results
--- in a different explain output plan between PG13/14 and PG15
--- Hence we set that GUC to off.
-SHOW server_version \gset
-SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15
-\gset
-\if :server_version_ge_15
-SET enable_group_by_reordering TO off;
-\endif
-SELECT DISTINCT 1 FROM run_command_on_workers($$ALTER SYSTEM SET enable_group_by_reordering TO off;$$);
- ?column?
----------------------------------------------------------------------
-        1
-(1 row)
-
-SELECT run_command_on_workers($$SELECT pg_reload_conf()$$);
- run_command_on_workers
----------------------------------------------------------------------
- (localhost,57637,t,t)
- (localhost,57638,t,t)
-(2 rows)
-
 EXPLAIN (COSTS OFF) WITH cte_1 AS NOT MATERIALIZED (SELECT * FROM test_table)
 SELECT
 	count(*)
@@ -524,22 +501,6 @@ DEBUG:  join prunable for intervals [1073741824,2147483647] and [0,1073741823]
                                  ->  Seq Scan on test_table_1960000 test_table_1
 (12 rows)
 
-\if :server_version_ge_15
-RESET enable_group_by_reordering;
-\endif
-SELECT DISTINCT 1 FROM run_command_on_workers($$ALTER SYSTEM RESET enable_group_by_reordering;$$);
- ?column?
----------------------------------------------------------------------
-        1
-(1 row)
-
-SELECT run_command_on_workers($$SELECT pg_reload_conf()$$);
- run_command_on_workers
----------------------------------------------------------------------
- (localhost,57637,t,t)
- (localhost,57638,t,t)
-(2 rows)
-
 -- ctes with volatile functions are not
 -- inlined
 WITH cte_1 AS (SELECT *, random() FROM test_table)

src/test/regress/expected/distributed_triggers.out

@@ -855,7 +855,145 @@ SELECT run_command_on_workers($$SELECT count(*) FROM pg_trigger WHERE tgname lik
  (localhost,57638,t,1)
 (2 rows)
 
-RESET client_min_messages;
+CREATE TABLE "dist_\'table"(a int);
+CREATE FUNCTION trigger_func()
+  RETURNS trigger
+  LANGUAGE plpgsql
+AS $function$
+BEGIN
+	RETURN NULL;
+END;
+$function$;
+CREATE TRIGGER default_mode_trigger
+AFTER UPDATE OR DELETE ON "dist_\'table"
+FOR STATEMENT EXECUTE FUNCTION trigger_func();
+CREATE TRIGGER "disabled_trigger\'"
+AFTER UPDATE OR DELETE ON "dist_\'table"
+FOR STATEMENT EXECUTE FUNCTION trigger_func();
+ALTER TABLE "dist_\'table" DISABLE trigger "disabled_trigger\'";
+CREATE TRIGGER replica_trigger
+AFTER UPDATE OR DELETE ON "dist_\'table"
+FOR STATEMENT EXECUTE FUNCTION trigger_func();
+ALTER TABLE "dist_\'table" ENABLE REPLICA trigger replica_trigger;
+CREATE TRIGGER always_enabled_trigger
+AFTER UPDATE OR DELETE ON "dist_\'table"
+FOR STATEMENT EXECUTE FUNCTION trigger_func();
+ALTER TABLE "dist_\'table" ENABLE ALWAYS trigger always_enabled_trigger;
+CREATE TRIGGER noop_enabled_trigger
+AFTER UPDATE OR DELETE ON "dist_\'table"
+FOR STATEMENT EXECUTE FUNCTION trigger_func();
+ALTER TABLE "dist_\'table" ENABLE trigger noop_enabled_trigger;
+SELECT create_distributed_table('dist_\''table', 'a');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+SELECT bool_and(tgenabled = 'O') FROM pg_trigger WHERE tgname LIKE 'default_mode_trigger%';
+ bool_and
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SELECT run_command_on_workers($$SELECT bool_and(tgenabled = 'O') FROM pg_trigger WHERE tgname LIKE 'default_mode_trigger%'$$);
+ run_command_on_workers
+---------------------------------------------------------------------
+ (localhost,57637,t,t)
+ (localhost,57638,t,t)
+(2 rows)
+
+SELECT bool_and(tgenabled = 'D') FROM pg_trigger WHERE tgname LIKE 'disabled_trigger%';
+ bool_and
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SELECT run_command_on_workers($$SELECT bool_and(tgenabled = 'D') FROM pg_trigger WHERE tgname LIKE 'disabled_trigger%'$$);
+ run_command_on_workers
+---------------------------------------------------------------------
+ (localhost,57637,t,t)
+ (localhost,57638,t,t)
+(2 rows)
+
+SELECT bool_and(tgenabled = 'R') FROM pg_trigger WHERE tgname LIKE 'replica_trigger%';
+ bool_and
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SELECT run_command_on_workers($$SELECT bool_and(tgenabled = 'R') FROM pg_trigger WHERE tgname LIKE 'replica_trigger%'$$);
+ run_command_on_workers
+---------------------------------------------------------------------
+ (localhost,57637,t,t)
+ (localhost,57638,t,t)
+(2 rows)
+
+SELECT bool_and(tgenabled = 'A') FROM pg_trigger WHERE tgname LIKE 'always_enabled_trigger%';
+ bool_and
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SELECT run_command_on_workers($$SELECT bool_and(tgenabled = 'A') FROM pg_trigger WHERE tgname LIKE 'always_enabled_trigger%'$$);
+ run_command_on_workers
+---------------------------------------------------------------------
+ (localhost,57637,t,t)
+ (localhost,57638,t,t)
+(2 rows)
+
+SELECT bool_and(tgenabled = 'O') FROM pg_trigger WHERE tgname LIKE 'noop_enabled_trigger%';
+ bool_and
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SELECT run_command_on_workers($$SELECT bool_and(tgenabled = 'O') FROM pg_trigger WHERE tgname LIKE 'noop_enabled_trigger%'$$);
+ run_command_on_workers
+---------------------------------------------------------------------
+ (localhost,57637,t,t)
+ (localhost,57638,t,t)
+(2 rows)
+
+CREATE TABLE citus_local(a int);
+CREATE FUNCTION citus_local_trig_func()
+  RETURNS trigger
+  LANGUAGE plpgsql
+AS $function$
+BEGIN
+	RETURN NULL;
+END;
+$function$;
+CREATE TRIGGER citus_local_trig
+AFTER UPDATE OR DELETE ON citus_local
+FOR STATEMENT EXECUTE FUNCTION citus_local_trig_func();
+-- make sure that trigger is initially not disabled
+SELECT tgenabled = 'D' FROM pg_trigger WHERE tgname LIKE 'citus_local_trig%';
+ ?column?
+---------------------------------------------------------------------
+ f
+(1 row)
+
+ALTER TABLE citus_local DISABLE trigger citus_local_trig;
+SELECT citus_add_local_table_to_metadata('citus_local');
+ citus_add_local_table_to_metadata
+---------------------------------------------------------------------
+
+(1 row)
+
+SELECT bool_and(tgenabled = 'D') FROM pg_trigger WHERE tgname LIKE 'citus_local_trig%';
+ bool_and
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SELECT run_command_on_workers($$SELECT bool_and(tgenabled = 'D') FROM pg_trigger WHERE tgname LIKE 'citus_local_trig%'$$);
+ run_command_on_workers
+---------------------------------------------------------------------
+ (localhost,57637,t,t)
+ (localhost,57638,t,t)
+(2 rows)
+
+SET client_min_messages TO ERROR;
 RESET citus.enable_unsafe_triggers;
 SELECT run_command_on_workers('ALTER SYSTEM RESET citus.enable_unsafe_triggers;');
        run_command_on_workers
@@ -873,25 +1011,3 @@ SELECT run_command_on_workers('SELECT pg_reload_conf();');
 
 SET citus.log_remote_commands TO off;
 DROP SCHEMA distributed_triggers CASCADE;
-NOTICE:  drop cascades to 21 other objects
-DETAIL:  drop cascades to table data
-drop cascades to function record_change()
-drop cascades to function insert_delete_document(text,text)
-drop cascades to function bad_shardkey_record_change()
-drop cascades to function remote_shardkey_record_change()
-drop cascades to function insert_document(text,text)
-drop cascades to table emptest
-drop cascades to table emptest_audit
-drop cascades to function process_emp_audit()
-drop cascades to view emp_triggers
-drop cascades to table record_op
-drop cascades to function record_emp()
-drop cascades to table data_changes
-drop cascades to table sale
-drop cascades to table record_sale
-drop cascades to function record_sale()
-drop cascades to view sale_triggers
-drop cascades to extension seg
-drop cascades to table distributed_table
-drop cascades to table distributed_table_change
-drop cascades to function insert_99()

src/test/regress/expected/failure_on_create_subscription.out

@@ -41,8 +41,13 @@ SELECT * FROM shards_in_workers;
      103 | worker1
 (4 rows)
 
--- failure on creating the subscription
-SELECT citus.mitmproxy('conn.onQuery(query="CREATE SUBSCRIPTION").kill()');
+-- Failure on creating the subscription
+-- Failing exactly on CREATE SUBSCRIPTION is causing flaky test where we fail with either:
+-- 1) ERROR:  connection to the remote node localhost:xxxxx failed with the following error: ERROR:  subscription "citus_shard_move_subscription_xxxxxxx" does not exist
+-- another command is already in progress
+-- 2) ERROR:  connection to the remote node localhost:xxxxx failed with the following error: another command is already in progress
+-- Instead fail on the next step (ALTER SUBSCRIPTION) instead which is also required logically as part of uber CREATE SUBSCRIPTION operation.
+SELECT citus.mitmproxy('conn.onQuery(query="ALTER SUBSCRIPTION").kill()');
  mitmproxy
 ---------------------------------------------------------------------
 

src/test/regress/expected/foreign_tables_mx.out

@@ -4,6 +4,15 @@ SET citus.shard_replication_factor TO 1;
 SET citus.enable_local_execution TO ON;
 CREATE SCHEMA foreign_tables_schema_mx;
 SET search_path TO foreign_tables_schema_mx;
+SET client_min_messages to ERROR;
+-- ensure that coordinator is added to pg_dist_node
+SELECT 1 FROM master_add_node('localhost', :master_port, groupId => 0);
+ ?column?
+---------------------------------------------------------------------
+        1
+(1 row)
+
+RESET client_min_messages;
 -- test adding foreign table to metadata with the guc
 SET citus.use_citus_managed_tables TO ON;
 CREATE TABLE foreign_table_test (id integer NOT NULL, data text, a bigserial);
@@ -379,15 +388,93 @@ SELECT * FROM ref_tbl d JOIN foreign_table_local f ON d.a=f.id ORDER BY f.id;
 
 \c - - - :master_port
 SET search_path TO foreign_tables_schema_mx;
--- should error out because doesn't have a table_name field
 CREATE FOREIGN TABLE foreign_table_local_fails (
         id integer NOT NULL,
         data text
 )
         SERVER foreign_server_local
         OPTIONS (schema_name 'foreign_tables_schema_mx');
+-- should error out because doesn't have a table_name field
+SELECT citus_add_local_table_to_metadata('foreign_table_local_fails');
 ERROR:  table_name option must be provided when using postgres_fdw with Citus
+-- should work since it has a table_name
+ALTER FOREIGN TABLE foreign_table_local_fails OPTIONS (table_name 'foreign_table_test');
+SELECT citus_add_local_table_to_metadata('foreign_table_local_fails');
+ citus_add_local_table_to_metadata
+---------------------------------------------------------------------
+
+(1 row)
+
+INSERT INTO foreign_table_test VALUES (1, 'test');
+SELECT undistribute_table('foreign_table_local_fails');
+NOTICE:  creating a new table for foreign_tables_schema_mx.foreign_table_local_fails
+NOTICE:  dropping the old foreign_tables_schema_mx.foreign_table_local_fails
+NOTICE:  renaming the new table to foreign_tables_schema_mx.foreign_table_local_fails
+ undistribute_table
+---------------------------------------------------------------------
+
+(1 row)
+
 DROP FOREIGN TABLE foreign_table_local;
+-- disallow dropping table_name when foreign table is in metadata
+CREATE TABLE table_name_drop(id int);
+CREATE FOREIGN TABLE foreign_table_name_drop_fails (
+        id INT
+)
+        SERVER foreign_server_local
+        OPTIONS (schema_name 'foreign_tables_schema_mx', table_name 'table_name_drop');
+SELECT citus_add_local_table_to_metadata('foreign_table_name_drop_fails');
+ citus_add_local_table_to_metadata
+---------------------------------------------------------------------
+
+(1 row)
+
+-- table_name option is already added
+ALTER FOREIGN TABLE foreign_table_name_drop_fails OPTIONS (ADD table_name 'table_name_drop');
+ERROR:  option "table_name" provided more than once
+-- throw error if user tries to drop table_name option from a foreign table inside metadata
+ALTER FOREIGN TABLE foreign_table_name_drop_fails OPTIONS (DROP table_name);
+ERROR:  alter foreign table alter options (drop table_name) command is not allowed for Citus tables
+-- case sensitive option name
+ALTER FOREIGN TABLE foreign_table_name_drop_fails OPTIONS (DROP Table_Name);
+ERROR:  alter foreign table alter options (drop table_name) command is not allowed for Citus tables
+-- other options are allowed to drop
+ALTER FOREIGN TABLE foreign_table_name_drop_fails OPTIONS (DROP schema_name);
+CREATE FOREIGN TABLE foreign_table_name_drop (
+        id INT
+)
+        SERVER foreign_server_local
+        OPTIONS (schema_name 'foreign_tables_schema_mx', table_name 'table_name_drop');
+-- user can drop table_option if foreign table is not in metadata
+ALTER FOREIGN TABLE foreign_table_name_drop OPTIONS (DROP table_name);
+-- we should not intercept data wrappers other than postgres_fdw
+CREATE EXTENSION file_fdw;
+-- remove validator method to add table_name option; otherwise, table_name option is not allowed
+SELECT result FROM run_command_on_all_nodes('ALTER FOREIGN DATA WRAPPER file_fdw NO VALIDATOR');
+           result
+---------------------------------------------------------------------
+ ALTER FOREIGN DATA WRAPPER
+ ALTER FOREIGN DATA WRAPPER
+ ALTER FOREIGN DATA WRAPPER
+(3 rows)
+
+CREATE SERVER citustest FOREIGN DATA WRAPPER file_fdw;
+\copy (select i from generate_series(0,100)i) to '/tmp/test_file_fdw.data';
+CREATE FOREIGN TABLE citustest_filefdw (
+        data text
+)
+        SERVER citustest
+        OPTIONS ( filename '/tmp/test_file_fdw.data');
+-- add non-postgres_fdw table into metadata even if it does not have table_name option
+SELECT citus_add_local_table_to_metadata('citustest_filefdw');
+ citus_add_local_table_to_metadata
+---------------------------------------------------------------------
+
+(1 row)
+
+ALTER FOREIGN TABLE citustest_filefdw OPTIONS (ADD table_name 'unused_table_name_option');
+-- drop table_name option of non-postgres_fdw table even if it is inside metadata
+ALTER FOREIGN TABLE citustest_filefdw OPTIONS (DROP table_name);
 -- cleanup at exit
 set client_min_messages to error;
 DROP SCHEMA foreign_tables_schema_mx CASCADE;

src/test/regress/expected/isolation_create_citus_local_table.out

@@ -239,7 +239,7 @@ master_remove_node
 step s2-create-citus-local-table-1: SELECT citus_add_local_table_to_metadata('citus_local_table_1'); <waiting ...>
 step s1-commit: COMMIT;
 step s2-create-citus-local-table-1: <... completed>
-ERROR:  could not find the coordinator node in metadata as it is not added as a worker
+ERROR:  operation is not allowed when coordinator is not added into metadata
 step s2-commit: COMMIT;
 master_remove_node
 ---------------------------------------------------------------------

src/test/regress/expected/isolation_create_distributed_concurrently_after_drop_column.out

@@ -0,0 +1,667 @@
+Parsed test spec with 3 sessions
+
+starting permutation: s2-print-cluster-1 s3-acquire-advisory-lock s2-begin s1-alter-table s1-set-factor-1 s1-create-distributed-table-observations_with_pk-concurrently s2-insert-observations_with_pk s2-update-observations_with_pk s2-end s2-print-cluster-1 s3-release-advisory-lock s2-print-cluster-1
+step s2-print-cluster-1:
+ -- row count per shard
+ SELECT
+  nodeport, shardid, success, result
+ FROM
+  run_command_on_placements('observations_with_pk', 'select count(*) from %s')
+ ORDER BY
+  nodeport, shardid;
+ SELECT *
+ FROM
+  observations_with_pk
+ ORDER BY
+ measurement_id;
+
+nodeport|shardid|success|result
+---------------------------------------------------------------------
+(0 rows)
+
+tenant_id|dummy|measurement_id|payload|observation_time
+---------------------------------------------------------------------
+(0 rows)
+
+step s3-acquire-advisory-lock:
+    SELECT pg_advisory_lock(44000, 55152);
+
+pg_advisory_lock
+---------------------------------------------------------------------
+
+(1 row)
+
+step s2-begin:
+    BEGIN;
+
+step s1-alter-table:
+ ALTER TABLE observations_with_pk DROP COLUMN dummy;
+ ALTER TABLE observations_with_full_replica_identity DROP COLUMN dummy;
+
+step s1-set-factor-1:
+ SET citus.shard_replication_factor TO 1;
+ SELECT citus_set_coordinator_host('localhost');
+
+citus_set_coordinator_host
+---------------------------------------------------------------------
+
+(1 row)
+
+step s1-create-distributed-table-observations_with_pk-concurrently:
+ SELECT create_distributed_table_concurrently('observations_with_pk','tenant_id');
+ <waiting ...>
+step s2-insert-observations_with_pk: 
+ INSERT INTO observations_with_pk(tenant_id, payload) SELECT 'tenant_id', jsonb_build_object('name', 29.3);
+ INSERT INTO observations_with_pk(tenant_id, payload) SELECT 'tenant_id', jsonb_build_object('name', 29.3);
+ INSERT INTO observations_with_pk(tenant_id, payload) SELECT 'tenant_id', jsonb_build_object('name', 29.3);
+ INSERT INTO observations_with_pk(tenant_id, payload) SELECT 'tenant_id', jsonb_build_object('name', 29.3);
+
+step s2-update-observations_with_pk:
+ UPDATE observations_with_pk set observation_time='03/11/2019 02:00:00'::TIMESTAMP where tenant_id = 'tenant_id' and measurement_id = 3;
+
+step s2-end:
+   COMMIT;
+
+step s2-print-cluster-1:
+ -- row count per shard
+ SELECT
+  nodeport, shardid, success, result
+ FROM
+  run_command_on_placements('observations_with_pk', 'select count(*) from %s')
+ ORDER BY
+  nodeport, shardid;
+ SELECT *
+ FROM
+  observations_with_pk
+ ORDER BY
+ measurement_id;
+
+nodeport|shardid|success|result
+---------------------------------------------------------------------
+   57636|1500004|t      |     4
+(1 row)
+
+tenant_id|measurement_id|payload       |observation_time
+---------------------------------------------------------------------
+tenant_id|             1|{"name": 29.3}|Sun Mar 11 03:00:00 2018 PDT
+tenant_id|             2|{"name": 29.3}|Sun Mar 11 03:00:00 2018 PDT
+tenant_id|             3|{"name": 29.3}|Mon Mar 11 02:00:00 2019 PDT
+tenant_id|             4|{"name": 29.3}|Sun Mar 11 03:00:00 2018 PDT
+(4 rows)
+
+step s3-release-advisory-lock:
+    SELECT pg_advisory_unlock(44000, 55152);
+
+pg_advisory_unlock
+---------------------------------------------------------------------
+t
+(1 row)
+
+step s1-create-distributed-table-observations_with_pk-concurrently: <... completed>
+create_distributed_table_concurrently
+---------------------------------------------------------------------
+
+(1 row)
+
+step s2-print-cluster-1:
+ -- row count per shard
+ SELECT
+  nodeport, shardid, success, result
+ FROM
+  run_command_on_placements('observations_with_pk', 'select count(*) from %s')
+ ORDER BY
+  nodeport, shardid;
+ SELECT *
+ FROM
+  observations_with_pk
+ ORDER BY
+ measurement_id;
+
+nodeport|shardid|success|result
+---------------------------------------------------------------------
+   57637|1500006|t      |     4
+   57637|1500008|t      |     0
+   57638|1500005|t      |     0
+   57638|1500007|t      |     0
+(4 rows)
+
+tenant_id|measurement_id|payload       |observation_time
+---------------------------------------------------------------------
+tenant_id|             1|{"name": 29.3}|Sun Mar 11 03:00:00 2018 PDT
+tenant_id|             2|{"name": 29.3}|Sun Mar 11 03:00:00 2018 PDT
+tenant_id|             3|{"name": 29.3}|Mon Mar 11 02:00:00 2019 PDT
+tenant_id|             4|{"name": 29.3}|Sun Mar 11 03:00:00 2018 PDT
+(4 rows)
+
+
+starting permutation: s2-print-cluster-1 s3-acquire-advisory-lock s2-begin s1-alter-table s1-set-factor-1 s1-create-distributed-table-observations_with_pk-concurrently s2-insert-observations_with_pk s2-update-primary-key-observations_with_pk s2-end s2-print-cluster-1 s3-release-advisory-lock s2-print-cluster-1
+step s2-print-cluster-1:
+ -- row count per shard
+ SELECT
+  nodeport, shardid, success, result
+ FROM
+  run_command_on_placements('observations_with_pk', 'select count(*) from %s')
+ ORDER BY
+  nodeport, shardid;
+ SELECT *
+ FROM
+  observations_with_pk
+ ORDER BY
+ measurement_id;
+
+nodeport|shardid|success|result
+---------------------------------------------------------------------
+(0 rows)
+
+tenant_id|dummy|measurement_id|payload|observation_time
+---------------------------------------------------------------------
+(0 rows)
+
+step s3-acquire-advisory-lock:
+    SELECT pg_advisory_lock(44000, 55152);
+
+pg_advisory_lock
+---------------------------------------------------------------------
+
+(1 row)
+
+step s2-begin:
+    BEGIN;
+
+step s1-alter-table:
+ ALTER TABLE observations_with_pk DROP COLUMN dummy;
+ ALTER TABLE observations_with_full_replica_identity DROP COLUMN dummy;
+
+step s1-set-factor-1:
+ SET citus.shard_replication_factor TO 1;
+ SELECT citus_set_coordinator_host('localhost');
+
+citus_set_coordinator_host
+---------------------------------------------------------------------
+
+(1 row)
+
+step s1-create-distributed-table-observations_with_pk-concurrently:
+ SELECT create_distributed_table_concurrently('observations_with_pk','tenant_id');
+ <waiting ...>
+step s2-insert-observations_with_pk: 
+ INSERT INTO observations_with_pk(tenant_id, payload) SELECT 'tenant_id', jsonb_build_object('name', 29.3);
+ INSERT INTO observations_with_pk(tenant_id, payload) SELECT 'tenant_id', jsonb_build_object('name', 29.3);
+ INSERT INTO observations_with_pk(tenant_id, payload) SELECT 'tenant_id', jsonb_build_object('name', 29.3);
+ INSERT INTO observations_with_pk(tenant_id, payload) SELECT 'tenant_id', jsonb_build_object('name', 29.3);
+
+step s2-update-primary-key-observations_with_pk:
+ UPDATE observations_with_pk set measurement_id=100 where tenant_id = 'tenant_id' and measurement_id = 4 ;
+
+step s2-end:
+   COMMIT;
+
+step s2-print-cluster-1:
+ -- row count per shard
+ SELECT
+  nodeport, shardid, success, result
+ FROM
+  run_command_on_placements('observations_with_pk', 'select count(*) from %s')
+ ORDER BY
+  nodeport, shardid;
+ SELECT *
+ FROM
+  observations_with_pk
+ ORDER BY
+ measurement_id;
+
+nodeport|shardid|success|result
+---------------------------------------------------------------------
+   57636|1500009|t      |     4
+(1 row)
+
+tenant_id|measurement_id|payload       |observation_time
+---------------------------------------------------------------------
+tenant_id|             1|{"name": 29.3}|Sun Mar 11 03:00:00 2018 PDT
+tenant_id|             2|{"name": 29.3}|Sun Mar 11 03:00:00 2018 PDT
+tenant_id|             3|{"name": 29.3}|Sun Mar 11 03:00:00 2018 PDT
+tenant_id|           100|{"name": 29.3}|Sun Mar 11 03:00:00 2018 PDT
+(4 rows)
+
+step s3-release-advisory-lock:
+    SELECT pg_advisory_unlock(44000, 55152);
+
+pg_advisory_unlock
+---------------------------------------------------------------------
+t
+(1 row)
+
+step s1-create-distributed-table-observations_with_pk-concurrently: <... completed>
+create_distributed_table_concurrently
+---------------------------------------------------------------------
+
+(1 row)
+
+step s2-print-cluster-1:
+ -- row count per shard
+ SELECT
+  nodeport, shardid, success, result
+ FROM
+  run_command_on_placements('observations_with_pk', 'select count(*) from %s')
+ ORDER BY
+  nodeport, shardid;
+ SELECT *
+ FROM
+  observations_with_pk
+ ORDER BY
+ measurement_id;
+
+nodeport|shardid|success|result
+---------------------------------------------------------------------
+   57637|1500011|t      |     4
+   57637|1500013|t      |     0
+   57638|1500010|t      |     0
+   57638|1500012|t      |     0
+(4 rows)
+
+tenant_id|measurement_id|payload       |observation_time
+---------------------------------------------------------------------
+tenant_id|             1|{"name": 29.3}|Sun Mar 11 03:00:00 2018 PDT
+tenant_id|             2|{"name": 29.3}|Sun Mar 11 03:00:00 2018 PDT
+tenant_id|             3|{"name": 29.3}|Sun Mar 11 03:00:00 2018 PDT
+tenant_id|           100|{"name": 29.3}|Sun Mar 11 03:00:00 2018 PDT
+(4 rows)
+
+
+starting permutation: s2-print-cluster-1 s3-acquire-advisory-lock s2-begin s1-alter-table s1-set-factor-1 s1-create-distributed-table-observations_with_pk-concurrently s2-insert-observations_with_pk s2-update-observations_with_pk s2-delete-observations_with_pk s2-end s2-print-cluster-1 s3-release-advisory-lock s2-print-cluster-1
+step s2-print-cluster-1:
+ -- row count per shard
+ SELECT
+  nodeport, shardid, success, result
+ FROM
+  run_command_on_placements('observations_with_pk', 'select count(*) from %s')
+ ORDER BY
+  nodeport, shardid;
+ SELECT *
+ FROM
+  observations_with_pk
+ ORDER BY
+ measurement_id;
+
+nodeport|shardid|success|result
+---------------------------------------------------------------------
+(0 rows)
+
+tenant_id|dummy|measurement_id|payload|observation_time
+---------------------------------------------------------------------
+(0 rows)
+
+step s3-acquire-advisory-lock:
+    SELECT pg_advisory_lock(44000, 55152);
+
+pg_advisory_lock
+---------------------------------------------------------------------
+
+(1 row)
+
+step s2-begin:
+    BEGIN;
+
+step s1-alter-table:
+ ALTER TABLE observations_with_pk DROP COLUMN dummy;
+ ALTER TABLE observations_with_full_replica_identity DROP COLUMN dummy;
+
+step s1-set-factor-1:
+ SET citus.shard_replication_factor TO 1;
+ SELECT citus_set_coordinator_host('localhost');
+
+citus_set_coordinator_host
+---------------------------------------------------------------------
+
+(1 row)
+
+step s1-create-distributed-table-observations_with_pk-concurrently:
+ SELECT create_distributed_table_concurrently('observations_with_pk','tenant_id');
+ <waiting ...>
+step s2-insert-observations_with_pk: 
+ INSERT INTO observations_with_pk(tenant_id, payload) SELECT 'tenant_id', jsonb_build_object('name', 29.3);
+ INSERT INTO observations_with_pk(tenant_id, payload) SELECT 'tenant_id', jsonb_build_object('name', 29.3);
+ INSERT INTO observations_with_pk(tenant_id, payload) SELECT 'tenant_id', jsonb_build_object('name', 29.3);
+ INSERT INTO observations_with_pk(tenant_id, payload) SELECT 'tenant_id', jsonb_build_object('name', 29.3);
+
+step s2-update-observations_with_pk:
+ UPDATE observations_with_pk set observation_time='03/11/2019 02:00:00'::TIMESTAMP where tenant_id = 'tenant_id' and measurement_id = 3;
+
+step s2-delete-observations_with_pk:
+ DELETE FROM observations_with_pk where tenant_id = 'tenant_id' and measurement_id = 3 ;
+
+step s2-end:
+   COMMIT;
+
+step s2-print-cluster-1:
+ -- row count per shard
+ SELECT
+  nodeport, shardid, success, result
+ FROM
+  run_command_on_placements('observations_with_pk', 'select count(*) from %s')
+ ORDER BY
+  nodeport, shardid;
+ SELECT *
+ FROM
+  observations_with_pk
+ ORDER BY
+ measurement_id;
+
+nodeport|shardid|success|result
+---------------------------------------------------------------------
+   57636|1500014|t      |     3
+(1 row)
+
+tenant_id|measurement_id|payload       |observation_time
+---------------------------------------------------------------------
+tenant_id|             1|{"name": 29.3}|Sun Mar 11 03:00:00 2018 PDT
+tenant_id|             2|{"name": 29.3}|Sun Mar 11 03:00:00 2018 PDT
+tenant_id|             4|{"name": 29.3}|Sun Mar 11 03:00:00 2018 PDT
+(3 rows)
+
+step s3-release-advisory-lock:
+    SELECT pg_advisory_unlock(44000, 55152);
+
+pg_advisory_unlock
+---------------------------------------------------------------------
+t
+(1 row)
+
+step s1-create-distributed-table-observations_with_pk-concurrently: <... completed>
+create_distributed_table_concurrently
+---------------------------------------------------------------------
+
+(1 row)
+
+step s2-print-cluster-1:
+ -- row count per shard
+ SELECT
+  nodeport, shardid, success, result
+ FROM
+  run_command_on_placements('observations_with_pk', 'select count(*) from %s')
+ ORDER BY
+  nodeport, shardid;
+ SELECT *
+ FROM
+  observations_with_pk
+ ORDER BY
+ measurement_id;
+
+nodeport|shardid|success|result
+---------------------------------------------------------------------
+   57637|1500016|t      |     3
+   57637|1500018|t      |     0
+   57638|1500015|t      |     0
+   57638|1500017|t      |     0
+(4 rows)
+
+tenant_id|measurement_id|payload       |observation_time
+---------------------------------------------------------------------
+tenant_id|             1|{"name": 29.3}|Sun Mar 11 03:00:00 2018 PDT
+tenant_id|             2|{"name": 29.3}|Sun Mar 11 03:00:00 2018 PDT
+tenant_id|             4|{"name": 29.3}|Sun Mar 11 03:00:00 2018 PDT
+(3 rows)
+
+
+starting permutation: s2-print-cluster-2 s3-acquire-advisory-lock s2-begin s1-alter-table s1-set-factor-1 s1-create-distributed-table-observations-2-concurrently s2-insert-observations_with_full_replica_identity s2-update-observations_with_full_replica_identity s2-end s2-print-cluster-2 s3-release-advisory-lock s2-print-cluster-2
+step s2-print-cluster-2:
+ -- row count per shard
+ SELECT
+  nodeport, shardid, success, result
+ FROM
+  run_command_on_placements('observations_with_full_replica_identity', 'select count(*) from %s')
+ ORDER BY
+  nodeport, shardid;
+ SELECT *
+ FROM
+  observations_with_full_replica_identity
+ ORDER BY
+ measurement_id;
+
+nodeport|shardid|success|result
+---------------------------------------------------------------------
+(0 rows)
+
+tenant_id|dummy|measurement_id|payload|observation_time
+---------------------------------------------------------------------
+(0 rows)
+
+step s3-acquire-advisory-lock:
+    SELECT pg_advisory_lock(44000, 55152);
+
+pg_advisory_lock
+---------------------------------------------------------------------
+
+(1 row)
+
+step s2-begin:
+    BEGIN;
+
+step s1-alter-table:
+ ALTER TABLE observations_with_pk DROP COLUMN dummy;
+ ALTER TABLE observations_with_full_replica_identity DROP COLUMN dummy;
+
+step s1-set-factor-1:
+ SET citus.shard_replication_factor TO 1;
+ SELECT citus_set_coordinator_host('localhost');
+
+citus_set_coordinator_host
+---------------------------------------------------------------------
+
+(1 row)
+
+step s1-create-distributed-table-observations-2-concurrently:
+ SELECT create_distributed_table_concurrently('observations_with_full_replica_identity','tenant_id');
+ <waiting ...>
+step s2-insert-observations_with_full_replica_identity: 
+ INSERT INTO observations_with_full_replica_identity(tenant_id, payload) SELECT 'tenant_id', jsonb_build_object('name', 29.3);
+ INSERT INTO observations_with_full_replica_identity(tenant_id, payload) SELECT 'tenant_id', jsonb_build_object('name', 29.3);
+ INSERT INTO observations_with_full_replica_identity(tenant_id, payload) SELECT 'tenant_id', jsonb_build_object('name', 29.3);
+
+step s2-update-observations_with_full_replica_identity:
+ UPDATE observations_with_full_replica_identity set observation_time='03/11/2019 02:00:00'::TIMESTAMP where tenant_id = 'tenant_id' and measurement_id = 3;
+
+step s2-end:
+   COMMIT;
+
+step s2-print-cluster-2:
+ -- row count per shard
+ SELECT
+  nodeport, shardid, success, result
+ FROM
+  run_command_on_placements('observations_with_full_replica_identity', 'select count(*) from %s')
+ ORDER BY
+  nodeport, shardid;
+ SELECT *
+ FROM
+  observations_with_full_replica_identity
+ ORDER BY
+ measurement_id;
+
+nodeport|shardid|success|result
+---------------------------------------------------------------------
+   57636|1500019|t      |     3
+(1 row)
+
+tenant_id|measurement_id|payload       |observation_time
+---------------------------------------------------------------------
+tenant_id|             1|{"name": 29.3}|Sun Mar 11 03:00:00 2018 PDT
+tenant_id|             2|{"name": 29.3}|Sun Mar 11 03:00:00 2018 PDT
+tenant_id|             3|{"name": 29.3}|Mon Mar 11 02:00:00 2019 PDT
+(3 rows)
+
+step s3-release-advisory-lock:
+    SELECT pg_advisory_unlock(44000, 55152);
+
+pg_advisory_unlock
+---------------------------------------------------------------------
+t
+(1 row)
+
+step s1-create-distributed-table-observations-2-concurrently: <... completed>
+create_distributed_table_concurrently
+---------------------------------------------------------------------
+
+(1 row)
+
+step s2-print-cluster-2:
+ -- row count per shard
+ SELECT
+  nodeport, shardid, success, result
+ FROM
+  run_command_on_placements('observations_with_full_replica_identity', 'select count(*) from %s')
+ ORDER BY
+  nodeport, shardid;
+ SELECT *
+ FROM
+  observations_with_full_replica_identity
+ ORDER BY
+ measurement_id;
+
+nodeport|shardid|success|result
+---------------------------------------------------------------------
+   57637|1500021|t      |     3
+   57637|1500023|t      |     0
+   57638|1500020|t      |     0
+   57638|1500022|t      |     0
+(4 rows)
+
+tenant_id|measurement_id|payload       |observation_time
+---------------------------------------------------------------------
+tenant_id|             1|{"name": 29.3}|Sun Mar 11 03:00:00 2018 PDT
+tenant_id|             2|{"name": 29.3}|Sun Mar 11 03:00:00 2018 PDT
+tenant_id|             3|{"name": 29.3}|Mon Mar 11 02:00:00 2019 PDT
+(3 rows)
+
+
+starting permutation: s2-print-cluster-2 s3-acquire-advisory-lock s2-begin s1-alter-table s1-set-factor-1 s1-create-distributed-table-observations-2-concurrently s2-insert-observations_with_full_replica_identity s2-update-observations_with_full_replica_identity s2-delete-observations_with_full_replica_identity s2-end s2-print-cluster-2 s3-release-advisory-lock s2-print-cluster-2
+step s2-print-cluster-2:
+ -- row count per shard
+ SELECT
+  nodeport, shardid, success, result
+ FROM
+  run_command_on_placements('observations_with_full_replica_identity', 'select count(*) from %s')
+ ORDER BY
+  nodeport, shardid;
+ SELECT *
+ FROM
+  observations_with_full_replica_identity
+ ORDER BY
+ measurement_id;
+
+nodeport|shardid|success|result
+---------------------------------------------------------------------
+(0 rows)
+
+tenant_id|dummy|measurement_id|payload|observation_time
+---------------------------------------------------------------------
+(0 rows)
+
+step s3-acquire-advisory-lock:
+    SELECT pg_advisory_lock(44000, 55152);
+
+pg_advisory_lock
+---------------------------------------------------------------------
+
+(1 row)
+
+step s2-begin:
+    BEGIN;
+
+step s1-alter-table:
+ ALTER TABLE observations_with_pk DROP COLUMN dummy;
+ ALTER TABLE observations_with_full_replica_identity DROP COLUMN dummy;
+
+step s1-set-factor-1:
+ SET citus.shard_replication_factor TO 1;
+ SELECT citus_set_coordinator_host('localhost');
+
+citus_set_coordinator_host
+---------------------------------------------------------------------
+
+(1 row)
+
+step s1-create-distributed-table-observations-2-concurrently:
+ SELECT create_distributed_table_concurrently('observations_with_full_replica_identity','tenant_id');
+ <waiting ...>
+step s2-insert-observations_with_full_replica_identity: 
+ INSERT INTO observations_with_full_replica_identity(tenant_id, payload) SELECT 'tenant_id', jsonb_build_object('name', 29.3);
+ INSERT INTO observations_with_full_replica_identity(tenant_id, payload) SELECT 'tenant_id', jsonb_build_object('name', 29.3);
+ INSERT INTO observations_with_full_replica_identity(tenant_id, payload) SELECT 'tenant_id', jsonb_build_object('name', 29.3);
+
+step s2-update-observations_with_full_replica_identity:
+ UPDATE observations_with_full_replica_identity set observation_time='03/11/2019 02:00:00'::TIMESTAMP where tenant_id = 'tenant_id' and measurement_id = 3;
+
+step s2-delete-observations_with_full_replica_identity:
+ DELETE FROM observations_with_full_replica_identity where tenant_id = 'tenant_id' and measurement_id = 3 ;
+
+step s2-end:
+   COMMIT;
+
+step s2-print-cluster-2:
+ -- row count per shard
+ SELECT
+  nodeport, shardid, success, result
+ FROM
+  run_command_on_placements('observations_with_full_replica_identity', 'select count(*) from %s')
+ ORDER BY
+  nodeport, shardid;
+ SELECT *
+ FROM
+  observations_with_full_replica_identity
+ ORDER BY
+ measurement_id;
+
+nodeport|shardid|success|result
+---------------------------------------------------------------------
+   57636|1500024|t      |     2
+(1 row)
+
+tenant_id|measurement_id|payload       |observation_time
+---------------------------------------------------------------------
+tenant_id|             1|{"name": 29.3}|Sun Mar 11 03:00:00 2018 PDT
+tenant_id|             2|{"name": 29.3}|Sun Mar 11 03:00:00 2018 PDT
+(2 rows)
+
+step s3-release-advisory-lock:
+    SELECT pg_advisory_unlock(44000, 55152);
+
+pg_advisory_unlock
+---------------------------------------------------------------------
+t
+(1 row)
+
+step s1-create-distributed-table-observations-2-concurrently: <... completed>
+create_distributed_table_concurrently
+---------------------------------------------------------------------
+
+(1 row)
+
+step s2-print-cluster-2:
+ -- row count per shard
+ SELECT
+  nodeport, shardid, success, result
+ FROM
+  run_command_on_placements('observations_with_full_replica_identity', 'select count(*) from %s')
+ ORDER BY
+  nodeport, shardid;
+ SELECT *
+ FROM
+  observations_with_full_replica_identity
+ ORDER BY
+ measurement_id;
+
+nodeport|shardid|success|result
+---------------------------------------------------------------------
+   57637|1500026|t      |     2
+   57637|1500028|t      |     0
+   57638|1500025|t      |     0
+   57638|1500027|t      |     0
+(4 rows)
+
+tenant_id|measurement_id|payload       |observation_time
+---------------------------------------------------------------------
+tenant_id|             1|{"name": 29.3}|Sun Mar 11 03:00:00 2018 PDT
+tenant_id|             2|{"name": 29.3}|Sun Mar 11 03:00:00 2018 PDT
+(2 rows)
+

src/test/regress/expected/isolation_logical_replication_nonsu_nonbypassrls.out

@@ -0,0 +1,170 @@
+Parsed test spec with 3 sessions
+
+starting permutation: s1-table-owner-new_user s1-table-enable-rls s1-get-shard-distribution s1-user-spec s3-acquire-advisory-lock s1-begin s1-set-role s1-move-placement s2-insert s3-release-advisory-lock s1-reset-role s1-end s1-select s1-get-shard-distribution
+step s1-table-owner-new_user:
+    ALTER TABLE dist OWNER TO new_user;
+
+step s1-table-enable-rls:
+    ALTER TABLE dist ENABLE ROW LEVEL SECURITY;
+
+step s1-get-shard-distribution:
+    SELECT shardid, nodeport FROM pg_dist_placement INNER JOIN pg_dist_node ON (pg_dist_placement.groupid = pg_dist_node.groupid) WHERE shardstate != 4 AND shardid IN (SELECT * FROM selected_shard) ORDER BY nodeport;
+
+shardid|nodeport
+---------------------------------------------------------------------
+1234003|   57638
+(1 row)
+
+step s1-user-spec:
+    SELECT rolname, rolsuper, rolbypassrls FROM pg_authid WHERE rolname = 'new_user';
+
+rolname |rolsuper|rolbypassrls
+---------------------------------------------------------------------
+new_user|f       |f
+(1 row)
+
+step s3-acquire-advisory-lock:
+    SELECT pg_advisory_lock(44000, 55152);
+
+pg_advisory_lock
+---------------------------------------------------------------------
+
+(1 row)
+
+step s1-begin:
+ BEGIN;
+
+step s1-set-role:
+    SET ROLE new_user;
+
+step s1-move-placement:
+    SELECT citus_move_shard_placement((SELECT * FROM selected_shard), 'localhost', 57638, 'localhost', 57637);
+ <waiting ...>
+step s2-insert: 
+    INSERT INTO dist VALUES (23, 23);
+
+step s3-release-advisory-lock:
+    SELECT pg_advisory_unlock(44000, 55152);
+
+pg_advisory_unlock
+---------------------------------------------------------------------
+t
+(1 row)
+
+step s1-move-placement: <... completed>
+citus_move_shard_placement
+---------------------------------------------------------------------
+
+(1 row)
+
+step s1-reset-role:
+ RESET ROLE;
+
+step s1-end:
+ COMMIT;
+
+step s1-select:
+    SELECT * FROM dist ORDER BY column1;
+
+column1|column2
+---------------------------------------------------------------------
+     23|     23
+(1 row)
+
+step s1-get-shard-distribution:
+    SELECT shardid, nodeport FROM pg_dist_placement INNER JOIN pg_dist_node ON (pg_dist_placement.groupid = pg_dist_node.groupid) WHERE shardstate != 4 AND shardid IN (SELECT * FROM selected_shard) ORDER BY nodeport;
+
+shardid|nodeport
+---------------------------------------------------------------------
+1234003|   57637
+(1 row)
+
+
+starting permutation: s1-no-connection-cache s2-no-connection-cache s3-no-connection-cache s1-table-owner-new_user s1-table-force-rls s1-get-shard-distribution s1-user-spec s3-acquire-advisory-lock s1-begin s1-set-role s1-move-placement s2-insert s3-release-advisory-lock s1-reset-role s1-end s1-select s1-get-shard-distribution
+step s1-no-connection-cache:
+ SET citus.max_cached_conns_per_worker to 0;
+
+step s2-no-connection-cache:
+ SET citus.max_cached_conns_per_worker to 0;
+
+step s3-no-connection-cache:
+ SET citus.max_cached_conns_per_worker to 0;
+
+step s1-table-owner-new_user:
+    ALTER TABLE dist OWNER TO new_user;
+
+step s1-table-force-rls:
+    ALTER TABLE dist FORCE ROW LEVEL SECURITY;
+
+step s1-get-shard-distribution:
+    SELECT shardid, nodeport FROM pg_dist_placement INNER JOIN pg_dist_node ON (pg_dist_placement.groupid = pg_dist_node.groupid) WHERE shardstate != 4 AND shardid IN (SELECT * FROM selected_shard) ORDER BY nodeport;
+
+shardid|nodeport
+---------------------------------------------------------------------
+1234003|   57638
+(1 row)
+
+step s1-user-spec:
+    SELECT rolname, rolsuper, rolbypassrls FROM pg_authid WHERE rolname = 'new_user';
+
+rolname |rolsuper|rolbypassrls
+---------------------------------------------------------------------
+new_user|f       |f
+(1 row)
+
+step s3-acquire-advisory-lock:
+    SELECT pg_advisory_lock(44000, 55152);
+
+pg_advisory_lock
+---------------------------------------------------------------------
+
+(1 row)
+
+step s1-begin:
+ BEGIN;
+
+step s1-set-role:
+    SET ROLE new_user;
+
+step s1-move-placement:
+    SELECT citus_move_shard_placement((SELECT * FROM selected_shard), 'localhost', 57638, 'localhost', 57637);
+ <waiting ...>
+step s2-insert: 
+    INSERT INTO dist VALUES (23, 23);
+
+step s3-release-advisory-lock:
+    SELECT pg_advisory_unlock(44000, 55152);
+
+pg_advisory_unlock
+---------------------------------------------------------------------
+t
+(1 row)
+
+step s1-move-placement: <... completed>
+citus_move_shard_placement
+---------------------------------------------------------------------
+
+(1 row)
+
+step s1-reset-role:
+ RESET ROLE;
+
+step s1-end:
+ COMMIT;
+
+step s1-select:
+    SELECT * FROM dist ORDER BY column1;
+
+column1|column2
+---------------------------------------------------------------------
+     23|     23
+(1 row)
+
+step s1-get-shard-distribution:
+    SELECT shardid, nodeport FROM pg_dist_placement INNER JOIN pg_dist_node ON (pg_dist_placement.groupid = pg_dist_node.groupid) WHERE shardstate != 4 AND shardid IN (SELECT * FROM selected_shard) ORDER BY nodeport;
+
+shardid|nodeport
+---------------------------------------------------------------------
+1234003|   57637
+(1 row)
+

src/test/regress/expected/isolation_logical_replication_skip_fk_validation.out

@@ -0,0 +1,52 @@
+Parsed test spec with 3 sessions
+
+starting permutation: s1-start-session-level-connection s3-acquire-advisory-lock s2-move-placement s1-start-session-level-connection s1-insert-violation-into-shard s3-release-advisory-lock
+step s1-start-session-level-connection:
+ SELECT start_session_level_connection_to_node('localhost', 57638);
+
+start_session_level_connection_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
+step s3-acquire-advisory-lock:
+ SELECT pg_advisory_lock(44000, 55152);
+
+pg_advisory_lock
+---------------------------------------------------------------------
+
+(1 row)
+
+step s2-move-placement:
+ SELECT master_move_shard_placement((SELECT * FROM selected_shard_for_test_table), 'localhost', 57637, 'localhost', 57638);
+ <waiting ...>
+step s1-start-session-level-connection: 
+ SELECT start_session_level_connection_to_node('localhost', 57638);
+
+start_session_level_connection_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
+step s1-insert-violation-into-shard:
+ SELECT run_commands_on_session_level_connection_to_node(format('INSERT INTO t1_%s VALUES (-1, -1)', (SELECT * FROM selected_shard_for_test_table)));
+
+run_commands_on_session_level_connection_to_node
+---------------------------------------------------------------------
+
+(1 row)
+
+step s3-release-advisory-lock:
+ SELECT pg_advisory_unlock(44000, 55152);
+
+pg_advisory_unlock
+---------------------------------------------------------------------
+t
+(1 row)
+
+step s2-move-placement: <... completed>
+master_move_shard_placement
+---------------------------------------------------------------------
+
+(1 row)
+

src/test/regress/expected/isolation_rebalancer_deferred_drop.out

@@ -91,7 +91,7 @@ step s1-drop-marked-shards:
  <waiting ...>
 s1: WARNING:  canceling statement due to lock timeout
 step s1-drop-marked-shards: <... completed>
-s1: WARNING:  Failed to drop 1 orphaned shards out of 1
+s1: WARNING:  failed to clean up 1 orphaned shards out of 1
 step s1-commit:
     COMMIT;
 

src/test/regress/expected/issue_6543.out

@@ -0,0 +1,30 @@
+CREATE SCHEMA issue_6543;
+SET search_path TO issue_6543;
+SET citus.shard_count TO 4;
+SET citus.shard_replication_factor TO 1;
+SET citus.next_shard_id TO 67322500;
+CREATE TABLE event (
+  tenant_id varchar,
+  id bigint,
+  primary key (tenant_id, id)
+);
+CREATE TABLE page (
+  tenant_id varchar,
+  id int,
+  primary key (tenant_id, id)
+);
+SELECT create_distributed_table('event', 'tenant_id');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+SELECT create_distributed_table('page', 'tenant_id', colocate_with => 'event');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+alter table page add constraint fk21 foreign key (tenant_id, id) references event;
+SET client_min_messages TO WARNING;
+DROP SCHEMA issue_6543 CASCADE;

src/test/regress/expected/issue_6592.out

@@ -0,0 +1,37 @@
+-- https://github.com/citusdata/citus/issues/6592
+SET citus.next_shard_id TO 180000;
+CREATE TABLE ref_table_to_be_dropped_6592 (key int);
+SELECT create_reference_table('ref_table_to_be_dropped_6592');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE TABLE ref_table_oid AS SELECT oid FROM pg_class WHERE relname = 'ref_table_to_be_dropped_6592';
+SET citus.enable_ddl_propagation TO OFF;
+DROP TABLE ref_table_to_be_dropped_6592 CASCADE; -- citus_drop_all_shards doesn't drop shards and metadata
+RESET citus.enable_ddl_propagation;
+-- error out for the dropped reference table
+SET client_min_messages to ERROR;
+SELECT 1 FROM master_add_node('localhost', :master_port, groupId => 0);
+ERROR:  relation with OID XXXX does not exist
+RESET client_min_messages;
+\c - - - :worker_1_port
+SET citus.enable_ddl_propagation TO OFF;
+DELETE FROM pg_dist_partition WHERE logicalrelid = 'ref_table_to_be_dropped_6592'::regclass;
+DELETE FROM pg_dist_placement WHERE shardid = 180000;
+DELETE FROM pg_dist_shard WHERE shardid = 180000;
+DROP TABLE IF EXISTS ref_table_to_be_dropped_6592;
+DROP TABLE IF EXISTS ref_table_to_be_dropped_6592_180000;
+\c - - - :worker_2_port
+SET citus.enable_ddl_propagation TO OFF;
+DELETE FROM pg_dist_partition WHERE logicalrelid = 'ref_table_to_be_dropped_6592'::regclass;
+DELETE FROM pg_dist_placement WHERE shardid = 180000;
+DELETE FROM pg_dist_shard WHERE shardid = 180000;
+DROP TABLE IF EXISTS ref_table_to_be_dropped_6592;
+DROP TABLE IF EXISTS ref_table_to_be_dropped_6592_180000;
+\c - - - :master_port
+DELETE FROM pg_dist_placement WHERE shardid = 180000;
+DELETE FROM pg_dist_shard WHERE shardid = 180000;
+DELETE FROM pg_dist_partition WHERE logicalrelid IN (SELECT oid FROM ref_table_oid);
+DROP TABLE ref_table_oid;

src/test/regress/expected/multi_colocated_shard_rebalance.out

@@ -740,7 +740,7 @@ DETAIL:  from localhost:xxxxx
 (1 row)
 
 CALL citus_cleanup_orphaned_shards();
-LOG:  cleaning up public.test_with_pkey_13000042 on localhost:xxxxx which was left after a move
+LOG:  deferred drop of orphaned shard public.test_with_pkey_13000042 on localhost:xxxxx after a move completed
 NOTICE:  cleaned up 1 orphaned shards
 SET client_min_messages TO DEFAULT;
 -- we don't support multiple shard moves in a single transaction

src/test/regress/expected/multi_explain.out

@@ -636,21 +636,6 @@ Aggregate
                                             ->  Seq Scan on events_1400285 events
                                                   Filter: ((event_type)::text = ANY ('{click,submit,pay}'::text[]))
 -- Union and left join subquery pushdown
--- enable_group_by_reordering is a new GUC introduced in PG15
--- it does some optimization of the order of group by keys which results
--- in a different explain output plan between PG13/14 and PG15
--- Hence we set that GUC to off.
-SHOW server_version \gset
-SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15
-\gset
-\if :server_version_ge_15
-SET enable_group_by_reordering TO off;
-\endif
-SELECT DISTINCT 1 FROM run_command_on_workers($$ALTER SYSTEM SET enable_group_by_reordering TO off;$$);
-1
-SELECT run_command_on_workers($$SELECT pg_reload_conf()$$);
-(localhost,57637,t,t)
-(localhost,57638,t,t)
 EXPLAIN (COSTS OFF)
 SELECT
 	avg(array_length(events, 1)) AS event_average,
@@ -873,14 +858,6 @@ Sort
                                                                                 Sort Key: events_2.composite_id
                                                                                 ->  Seq Scan on events_1400285 events_2
                                                                                       Filter: ((composite_id >= '(1,-9223372036854775808)'::user_composite_type) AND (composite_id <= '(1,9223372036854775807)'::user_composite_type) AND ((event_type)::text = 'pay'::text))
-\if :server_version_ge_15
-RESET enable_group_by_reordering;
-\endif
-SELECT DISTINCT 1 FROM run_command_on_workers($$ALTER SYSTEM RESET enable_group_by_reordering;$$);
-1
-SELECT run_command_on_workers($$SELECT pg_reload_conf()$$);
-(localhost,57637,t,t)
-(localhost,57638,t,t)
 -- Lateral join subquery pushdown
 -- set subquery_pushdown due to limit in the query
 SET citus.subquery_pushdown to ON;

src/test/regress/expected/multi_extension.out

@@ -909,6 +909,24 @@ SELECT * FROM multi_extension.print_extension_changes();
                  | function worker_fix_partition_shard_index_names(regclass,text,text) void
 (4 rows)
 
+-- There was a bug when downgrading to 10.2-2 from 10.2-4
+-- Test that we do not have any issues with this particular downgrade
+ALTER EXTENSION citus UPDATE TO '10.2-2';
+ALTER EXTENSION citus UPDATE TO '10.2-4';
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
+-- Test downgrade to 10.2-4 from 10.2-5
+ALTER EXTENSION citus UPDATE TO '10.2-5';
+ALTER EXTENSION citus UPDATE TO '10.2-4';
+-- Should be empty result since upgrade+downgrade should be a no-op
+SELECT * FROM multi_extension.print_extension_changes();
+ previous_object | current_object
+---------------------------------------------------------------------
+(0 rows)
+
 -- Snapshot of state at 10.2-5
 ALTER EXTENSION citus UPDATE TO '10.2-5';
 SELECT * FROM multi_extension.print_extension_changes();
@@ -916,9 +934,6 @@ SELECT * FROM multi_extension.print_extension_changes();
 ---------------------------------------------------------------------
 (0 rows)
 
--- Test downgrade to 10.2-4 from 10.2-5
-ALTER EXTENSION citus UPDATE TO '10.2-4';
-ALTER EXTENSION citus UPDATE TO '10.2-5';
 -- Make sure that we defined dependencies from all rel objects (tables,
 -- indexes, sequences ..) to columnar table access method ...
 SELECT pg_class.oid INTO columnar_schema_members
@@ -1177,7 +1192,7 @@ DROP TABLE multi_extension.prev_objects, multi_extension.extension_diff;
 SHOW citus.version;
  citus.version
 ---------------------------------------------------------------------
- 11.1devel
+ 11.1.6
 (1 row)
 
 -- ensure no unexpected objects were created outside pg_catalog
@@ -1521,6 +1536,66 @@ SELECT count(*) FROM pg_stat_activity WHERE application_name = 'Citus Maintenanc
      1
 (1 row)
 
+-- confirm that we can create a distributed table concurrently on an empty node
+DROP EXTENSION citus;
+CREATE EXTENSION citus;
+CREATE TABLE test (x int, y int);
+INSERT INTO test VALUES (1,2);
+SET citus.shard_replication_factor TO 1;
+SET citus.defer_drop_after_shard_split TO off;
+SELECT create_distributed_table_concurrently('test','x');
+NOTICE:  relation test does not have a REPLICA IDENTITY or PRIMARY KEY
+DETAIL:  UPDATE and DELETE commands on the relation will error out during create_distributed_table_concurrently unless there is a REPLICA IDENTITY or PRIMARY KEY. INSERT commands will still work.
+ create_distributed_table_concurrently
+---------------------------------------------------------------------
+
+(1 row)
+
+DROP TABLE test;
+TRUNCATE pg_dist_node;
+-- confirm that we can create a distributed table on an empty node
+CREATE TABLE test (x int, y int);
+INSERT INTO test VALUES (1,2);
+SET citus.shard_replication_factor TO 1;
+SELECT create_distributed_table('test','x');
+NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is no longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.test$$)
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+DROP TABLE test;
+TRUNCATE pg_dist_node;
+-- confirm that we can create a reference table on an empty node
+CREATE TABLE test (x int, y int);
+INSERT INTO test VALUES (1,2);
+SELECT create_reference_table('test');
+NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is no longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$public.test$$)
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+DROP TABLE test;
+TRUNCATE pg_dist_node;
+-- confirm that we can create a local table on an empty node
+CREATE TABLE test (x int, y int);
+INSERT INTO test VALUES (1,2);
+SELECT citus_add_local_table_to_metadata('test');
+ citus_add_local_table_to_metadata
+---------------------------------------------------------------------
+
+(1 row)
+
+DROP TABLE test;
+DROP EXTENSION citus;
+CREATE EXTENSION citus;
 DROP TABLE version_mismatch_table;
 DROP SCHEMA multi_extension;
 ERROR:  cannot drop schema multi_extension because other objects depend on it

src/test/regress/expected/multi_fix_partition_shard_index_names.out

@@ -521,9 +521,9 @@ SELECT tablename, indexname FROM pg_indexes WHERE schemaname = 'fix_idx_names' O
                  tablename                 |                        indexname
 ---------------------------------------------------------------------
  date_partitioned_citus_local_table        | date_partitioned_citus_local_table_measureid_idx
- date_partitioned_citus_local_table_361369 | date_partitioned_citus_local_table_measureid_idx_361369
+ date_partitioned_citus_local_table_361377 | date_partitioned_citus_local_table_measureid_idx_361377
  partition_local_table                     | partition_local_table_measureid_idx
- partition_local_table_361370              | partition_local_table_measureid_idx_361370
+ partition_local_table_361378              | partition_local_table_measureid_idx_361378
 (4 rows)
 
 -- creating a single object should only need to trigger fixing the single object
@@ -753,7 +753,7 @@ DETAIL:  drop cascades to table not_partitioned
 drop cascades to table not_distributed
 drop cascades to table fk_table
 drop cascades to table p
-drop cascades to table date_partitioned_citus_local_table_361369
+drop cascades to table date_partitioned_citus_local_table_361377
 drop cascades to table date_partitioned_citus_local_table
 drop cascades to table parent_table
 SELECT citus_remove_node('localhost', :master_port);

src/test/regress/expected/multi_foreign_key.out

@@ -1172,5 +1172,108 @@ SELECT create_distributed_table ('dropfkeytest2', 'x', colocate_with:='none');
 
 (1 row)
 
+CREATE TABLE set_on_default_test_referenced(
+    col_1 int, col_2 int, col_3 int, col_4 int,
+    unique (col_1, col_3)
+);
+SELECT create_reference_table('set_on_default_test_referenced');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE TABLE set_on_default_test_referencing(
+    col_1 int, col_2 int, col_3 serial, col_4 int,
+    FOREIGN KEY(col_1, col_3)
+    REFERENCES set_on_default_test_referenced(col_1, col_3)
+    ON UPDATE SET DEFAULT
+);
+-- from distributed / reference to reference, fkey exists before calling the UDFs
+SELECT create_distributed_table('set_on_default_test_referencing', 'col_1');
+ERROR:  cannot create foreign key constraint since Citus does not support ON DELETE / UPDATE SET DEFAULT actions on the columns that default to sequences
+SELECT create_reference_table('set_on_default_test_referencing');
+ERROR:  cannot create foreign key constraint since Citus does not support ON DELETE / UPDATE SET DEFAULT actions on the columns that default to sequences
+DROP TABLE set_on_default_test_referencing;
+CREATE TABLE set_on_default_test_referencing(
+    col_1 serial, col_2 int, col_3 int, col_4 int
+);
+SELECT create_reference_table('set_on_default_test_referencing');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+-- from reference to reference, fkey doesn't exist before calling the UDFs
+ALTER TABLE set_on_default_test_referencing ADD CONSTRAINT fkey
+FOREIGN KEY(col_1, col_3) REFERENCES set_on_default_test_referenced(col_1, col_3)
+ON DELETE SET DEFAULT;
+ERROR:  cannot create foreign key constraint since Citus does not support ON DELETE / UPDATE SET DEFAULT actions on the columns that default to sequences
+DROP TABLE set_on_default_test_referencing;
+CREATE TABLE set_on_default_test_referencing(
+    col_1 int, col_2 serial, col_3 int, col_4 bigserial
+);
+SELECT create_reference_table('set_on_default_test_referencing');
+ create_reference_table
+---------------------------------------------------------------------
+
+(1 row)
+
+-- ok since referencing columns are not based on sequences
+ALTER TABLE set_on_default_test_referencing ADD CONSTRAINT fkey
+FOREIGN KEY(col_1, col_3) REFERENCES set_on_default_test_referenced(col_1, col_3)
+ON DELETE SET DEFAULT;
+DROP TABLE set_on_default_test_referencing;
+CREATE SEQUENCE test_sequence;
+CREATE TABLE set_on_default_test_referencing(
+    col_1 int, col_2 int, col_3 int DEFAULT nextval('test_sequence'), col_4 int
+);
+SELECT create_distributed_table('set_on_default_test_referencing', 'col_1');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+-- from distributed to reference, fkey doesn't exist before calling the UDFs
+ALTER TABLE set_on_default_test_referencing ADD CONSTRAINT fkey
+FOREIGN KEY(col_1, col_3) REFERENCES set_on_default_test_referenced(col_1, col_3)
+ON DELETE SET DEFAULT ON UPDATE SET DEFAULT;
+ERROR:  cannot create foreign key constraint since Citus does not support ON DELETE / UPDATE SET DEFAULT actions on the columns that default to sequences
+DROP TABLE set_on_default_test_referenced;
+CREATE TABLE set_on_default_test_referenced(
+    col_1 int, col_2 int, col_3 int, col_4 int,
+    unique (col_1, col_3)
+);
+SELECT create_distributed_table('set_on_default_test_referenced', 'col_1');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+DROP TABLE set_on_default_test_referencing;
+CREATE TABLE set_on_default_test_referencing(
+    col_1 bigserial, col_2 int, col_3 int DEFAULT nextval('test_sequence'), col_4 int,
+    FOREIGN KEY(col_1, col_3)
+    REFERENCES set_on_default_test_referenced(col_1, col_3)
+    ON DELETE SET DEFAULT
+);
+-- from distributed to distributed, fkey exists before calling the UDFs
+SELECT create_distributed_table('set_on_default_test_referencing', 'col_1');
+ERROR:  cannot create foreign key constraint since Citus does not support ON DELETE / UPDATE SET DEFAULT actions on the columns that default to sequences
+DROP TABLE set_on_default_test_referencing;
+CREATE TABLE set_on_default_test_referencing(
+    col_1 int DEFAULT nextval('test_sequence'), col_2 int, col_3 int, col_4 int
+);
+SELECT create_distributed_table('set_on_default_test_referencing', 'col_1');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+-- from distributed to distributed, fkey doesn't exist before calling the UDFs
+ALTER TABLE set_on_default_test_referencing ADD CONSTRAINT fkey
+FOREIGN KEY(col_1, col_3) REFERENCES set_on_default_test_referenced(col_1, col_3)
+ON DELETE SET DEFAULT;
+ERROR:  cannot create foreign key constraint since Citus does not support ON DELETE / UPDATE SET DEFAULT actions on the columns that default to sequences
 -- we no longer need those tables
-DROP TABLE referenced_by_reference_table, references_to_reference_table, reference_table, reference_table_second, referenced_local_table, self_referencing_reference_table, dropfkeytest2;
+DROP TABLE referenced_by_reference_table, references_to_reference_table, reference_table, reference_table_second, referenced_local_table, self_referencing_reference_table, dropfkeytest2,
+           set_on_default_test_referenced, set_on_default_test_referencing;

src/test/regress/expected/multi_move_mx.out

@@ -238,8 +238,40 @@ ORDER BY
 LIMIT 1 OFFSET 1;
 ERROR:  operation is not allowed on this node
 HINT:  Connect to the coordinator and run it again.
+-- Check that shards of a table with GENERATED columns can be moved.
+\c - - - :master_port
+SET citus.shard_count TO 4;
+SET citus.shard_replication_factor TO 1;
+CREATE TABLE mx_table_with_generated_column (a int, b int GENERATED ALWAYS AS ( a + 3 ) STORED, c int);
+SELECT create_distributed_table('mx_table_with_generated_column', 'a');
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+-- Check that dropped columns are handled properly in a move.
+ALTER TABLE mx_table_with_generated_column DROP COLUMN c;
+-- Move a shard from worker 1 to worker 2
+SELECT
+        citus_move_shard_placement(shardid, 'localhost', :worker_1_port, 'localhost', :worker_2_port, 'force_logical')
+FROM
+        pg_dist_shard NATURAL JOIN pg_dist_shard_placement
+WHERE
+        logicalrelid = 'mx_table_with_generated_column'::regclass
+	AND nodeport = :worker_1_port
+ORDER BY
+        shardid
+LIMIT 1;
+ citus_move_shard_placement
+---------------------------------------------------------------------
+
+(1 row)
+
 -- Cleanup
 \c - - - :master_port
+SET client_min_messages TO WARNING;
+CALL citus_cleanup_orphaned_resources();
+DROP TABLE mx_table_with_generated_column;
 DROP TABLE mx_table_1;
 DROP TABLE mx_table_2;
 DROP TABLE mx_table_3;

src/test/regress/expected/multi_mx_create_table.out

@@ -497,22 +497,22 @@ ORDER BY table_name::text;
 SELECT shard_name, table_name, citus_table_type, shard_size FROM citus_shards ORDER BY shard_name::text;
                            shard_name                           |                       table_name                       | citus_table_type | shard_size
 ---------------------------------------------------------------------
- app_analytics_events_mx_1220096                                | app_analytics_events_mx                                | distributed      |          0
- app_analytics_events_mx_1220096                                | app_analytics_events_mx                                | distributed      |          0
- app_analytics_events_mx_1220096                                | app_analytics_events_mx                                | distributed      |          0
- app_analytics_events_mx_1220096                                | app_analytics_events_mx                                | distributed      |          0
- app_analytics_events_mx_1220096                                | app_analytics_events_mx                                | distributed      |          0
- app_analytics_events_mx_1220096                                | app_analytics_events_mx                                | distributed      |          0
- app_analytics_events_mx_1220096                                | app_analytics_events_mx                                | distributed      |          0
- app_analytics_events_mx_1220097                                | app_analytics_events_mx                                | distributed      |          0
- app_analytics_events_mx_1220098                                | app_analytics_events_mx                                | distributed      |          0
- app_analytics_events_mx_1220098                                | app_analytics_events_mx                                | distributed      |          0
- app_analytics_events_mx_1220098                                | app_analytics_events_mx                                | distributed      |          0
- app_analytics_events_mx_1220098                                | app_analytics_events_mx                                | distributed      |          0
- app_analytics_events_mx_1220098                                | app_analytics_events_mx                                | distributed      |          0
- app_analytics_events_mx_1220098                                | app_analytics_events_mx                                | distributed      |          0
- app_analytics_events_mx_1220098                                | app_analytics_events_mx                                | distributed      |          0
- app_analytics_events_mx_1220099                                | app_analytics_events_mx                                | distributed      |          0
+ app_analytics_events_mx_1220096                                | app_analytics_events_mx                                | distributed      |       8192
+ app_analytics_events_mx_1220096                                | app_analytics_events_mx                                | distributed      |       8192
+ app_analytics_events_mx_1220096                                | app_analytics_events_mx                                | distributed      |       8192
+ app_analytics_events_mx_1220096                                | app_analytics_events_mx                                | distributed      |       8192
+ app_analytics_events_mx_1220096                                | app_analytics_events_mx                                | distributed      |       8192
+ app_analytics_events_mx_1220096                                | app_analytics_events_mx                                | distributed      |       8192
+ app_analytics_events_mx_1220096                                | app_analytics_events_mx                                | distributed      |       8192
+ app_analytics_events_mx_1220097                                | app_analytics_events_mx                                | distributed      |       8192
+ app_analytics_events_mx_1220098                                | app_analytics_events_mx                                | distributed      |       8192
+ app_analytics_events_mx_1220098                                | app_analytics_events_mx                                | distributed      |       8192
+ app_analytics_events_mx_1220098                                | app_analytics_events_mx                                | distributed      |       8192
+ app_analytics_events_mx_1220098                                | app_analytics_events_mx                                | distributed      |       8192
+ app_analytics_events_mx_1220098                                | app_analytics_events_mx                                | distributed      |       8192
+ app_analytics_events_mx_1220098                                | app_analytics_events_mx                                | distributed      |       8192
+ app_analytics_events_mx_1220098                                | app_analytics_events_mx                                | distributed      |       8192
+ app_analytics_events_mx_1220099                                | app_analytics_events_mx                                | distributed      |       8192
  articles_hash_mx_1220104                                       | articles_hash_mx                                       | distributed      |          0
  articles_hash_mx_1220104                                       | articles_hash_mx                                       | distributed      |          0
  articles_hash_mx_1220104                                       | articles_hash_mx                                       | distributed      |          0
@@ -608,22 +608,22 @@ SELECT shard_name, table_name, citus_table_type, shard_size FROM citus_shards OR
  citus_mx_test_schema.nation_hash_collation_search_path_1220046 | citus_mx_test_schema.nation_hash_collation_search_path | distributed      |          0
  citus_mx_test_schema.nation_hash_collation_search_path_1220046 | citus_mx_test_schema.nation_hash_collation_search_path | distributed      |          0
  citus_mx_test_schema.nation_hash_collation_search_path_1220047 | citus_mx_test_schema.nation_hash_collation_search_path | distributed      |       8192
- citus_mx_test_schema.nation_hash_composite_types_1220048       | citus_mx_test_schema.nation_hash_composite_types       | distributed      |       8192
- citus_mx_test_schema.nation_hash_composite_types_1220048       | citus_mx_test_schema.nation_hash_composite_types       | distributed      |       8192
- citus_mx_test_schema.nation_hash_composite_types_1220048       | citus_mx_test_schema.nation_hash_composite_types       | distributed      |       8192
- citus_mx_test_schema.nation_hash_composite_types_1220048       | citus_mx_test_schema.nation_hash_composite_types       | distributed      |       8192
- citus_mx_test_schema.nation_hash_composite_types_1220048       | citus_mx_test_schema.nation_hash_composite_types       | distributed      |       8192
- citus_mx_test_schema.nation_hash_composite_types_1220048       | citus_mx_test_schema.nation_hash_composite_types       | distributed      |       8192
- citus_mx_test_schema.nation_hash_composite_types_1220048       | citus_mx_test_schema.nation_hash_composite_types       | distributed      |       8192
- citus_mx_test_schema.nation_hash_composite_types_1220049       | citus_mx_test_schema.nation_hash_composite_types       | distributed      |       8192
- citus_mx_test_schema.nation_hash_composite_types_1220050       | citus_mx_test_schema.nation_hash_composite_types       | distributed      |          0
- citus_mx_test_schema.nation_hash_composite_types_1220050       | citus_mx_test_schema.nation_hash_composite_types       | distributed      |          0
- citus_mx_test_schema.nation_hash_composite_types_1220050       | citus_mx_test_schema.nation_hash_composite_types       | distributed      |          0
- citus_mx_test_schema.nation_hash_composite_types_1220050       | citus_mx_test_schema.nation_hash_composite_types       | distributed      |          0
- citus_mx_test_schema.nation_hash_composite_types_1220050       | citus_mx_test_schema.nation_hash_composite_types       | distributed      |          0
- citus_mx_test_schema.nation_hash_composite_types_1220050       | citus_mx_test_schema.nation_hash_composite_types       | distributed      |          0
- citus_mx_test_schema.nation_hash_composite_types_1220050       | citus_mx_test_schema.nation_hash_composite_types       | distributed      |          0
- citus_mx_test_schema.nation_hash_composite_types_1220051       | citus_mx_test_schema.nation_hash_composite_types       | distributed      |       8192
+ citus_mx_test_schema.nation_hash_composite_types_1220048       | citus_mx_test_schema.nation_hash_composite_types       | distributed      |      16384
+ citus_mx_test_schema.nation_hash_composite_types_1220048       | citus_mx_test_schema.nation_hash_composite_types       | distributed      |      16384
+ citus_mx_test_schema.nation_hash_composite_types_1220048       | citus_mx_test_schema.nation_hash_composite_types       | distributed      |      16384
+ citus_mx_test_schema.nation_hash_composite_types_1220048       | citus_mx_test_schema.nation_hash_composite_types       | distributed      |      16384
+ citus_mx_test_schema.nation_hash_composite_types_1220048       | citus_mx_test_schema.nation_hash_composite_types       | distributed      |      16384
+ citus_mx_test_schema.nation_hash_composite_types_1220048       | citus_mx_test_schema.nation_hash_composite_types       | distributed      |      16384
+ citus_mx_test_schema.nation_hash_composite_types_1220048       | citus_mx_test_schema.nation_hash_composite_types       | distributed      |      16384
+ citus_mx_test_schema.nation_hash_composite_types_1220049       | citus_mx_test_schema.nation_hash_composite_types       | distributed      |      16384
+ citus_mx_test_schema.nation_hash_composite_types_1220050       | citus_mx_test_schema.nation_hash_composite_types       | distributed      |       8192
+ citus_mx_test_schema.nation_hash_composite_types_1220050       | citus_mx_test_schema.nation_hash_composite_types       | distributed      |       8192
+ citus_mx_test_schema.nation_hash_composite_types_1220050       | citus_mx_test_schema.nation_hash_composite_types       | distributed      |       8192
+ citus_mx_test_schema.nation_hash_composite_types_1220050       | citus_mx_test_schema.nation_hash_composite_types       | distributed      |       8192
+ citus_mx_test_schema.nation_hash_composite_types_1220050       | citus_mx_test_schema.nation_hash_composite_types       | distributed      |       8192
+ citus_mx_test_schema.nation_hash_composite_types_1220050       | citus_mx_test_schema.nation_hash_composite_types       | distributed      |       8192
+ citus_mx_test_schema.nation_hash_composite_types_1220050       | citus_mx_test_schema.nation_hash_composite_types       | distributed      |       8192
+ citus_mx_test_schema.nation_hash_composite_types_1220051       | citus_mx_test_schema.nation_hash_composite_types       | distributed      |      16384
  citus_mx_test_schema_join_1.nation_hash_1220032                | citus_mx_test_schema_join_1.nation_hash                | distributed      |          0
  citus_mx_test_schema_join_1.nation_hash_1220032                | citus_mx_test_schema_join_1.nation_hash                | distributed      |          0
  citus_mx_test_schema_join_1.nation_hash_1220032                | citus_mx_test_schema_join_1.nation_hash                | distributed      |          0
@@ -696,109 +696,109 @@ SELECT shard_name, table_name, citus_table_type, shard_size FROM citus_shards OR
  customer_mx_1220084                                            | customer_mx                                            | reference        |          0
  customer_mx_1220084                                            | customer_mx                                            | reference        |          0
  customer_mx_1220084                                            | customer_mx                                            | reference        |          0
- labs_mx_1220102                                                | labs_mx                                                | distributed      |          0
- labs_mx_1220102                                                | labs_mx                                                | distributed      |          0
- labs_mx_1220102                                                | labs_mx                                                | distributed      |          0
- labs_mx_1220102                                                | labs_mx                                                | distributed      |          0
- labs_mx_1220102                                                | labs_mx                                                | distributed      |          0
- labs_mx_1220102                                                | labs_mx                                                | distributed      |          0
- labs_mx_1220102                                                | labs_mx                                                | distributed      |          0
- limit_orders_mx_1220092                                        | limit_orders_mx                                        | distributed      |          0
- limit_orders_mx_1220092                                        | limit_orders_mx                                        | distributed      |          0
- limit_orders_mx_1220092                                        | limit_orders_mx                                        | distributed      |          0
- limit_orders_mx_1220092                                        | limit_orders_mx                                        | distributed      |          0
- limit_orders_mx_1220092                                        | limit_orders_mx                                        | distributed      |          0
- limit_orders_mx_1220092                                        | limit_orders_mx                                        | distributed      |          0
- limit_orders_mx_1220092                                        | limit_orders_mx                                        | distributed      |          0
- limit_orders_mx_1220093                                        | limit_orders_mx                                        | distributed      |          0
- lineitem_mx_1220052                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220052                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220052                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220052                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220052                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220052                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220052                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220053                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220054                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220054                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220054                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220054                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220054                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220054                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220054                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220055                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220056                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220056                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220056                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220056                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220056                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220056                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220056                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220057                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220058                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220058                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220058                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220058                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220058                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220058                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220058                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220059                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220060                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220060                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220060                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220060                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220060                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220060                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220060                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220061                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220062                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220062                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220062                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220062                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220062                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220062                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220062                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220063                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220064                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220064                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220064                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220064                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220064                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220064                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220064                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220065                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220066                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220066                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220066                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220066                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220066                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220066                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220066                                            | lineitem_mx                                            | distributed      |          0
- lineitem_mx_1220067                                            | lineitem_mx                                            | distributed      |          0
- multiple_hash_mx_1220094                                       | multiple_hash_mx                                       | distributed      |          0
- multiple_hash_mx_1220094                                       | multiple_hash_mx                                       | distributed      |          0
- multiple_hash_mx_1220094                                       | multiple_hash_mx                                       | distributed      |          0
- multiple_hash_mx_1220094                                       | multiple_hash_mx                                       | distributed      |          0
- multiple_hash_mx_1220094                                       | multiple_hash_mx                                       | distributed      |          0
- multiple_hash_mx_1220094                                       | multiple_hash_mx                                       | distributed      |          0
- multiple_hash_mx_1220094                                       | multiple_hash_mx                                       | distributed      |          0
- multiple_hash_mx_1220095                                       | multiple_hash_mx                                       | distributed      |          0
- mx_ddl_table_1220088                                           | mx_ddl_table                                           | distributed      |       8192
- mx_ddl_table_1220088                                           | mx_ddl_table                                           | distributed      |       8192
- mx_ddl_table_1220088                                           | mx_ddl_table                                           | distributed      |       8192
- mx_ddl_table_1220088                                           | mx_ddl_table                                           | distributed      |       8192
- mx_ddl_table_1220088                                           | mx_ddl_table                                           | distributed      |       8192
- mx_ddl_table_1220088                                           | mx_ddl_table                                           | distributed      |       8192
- mx_ddl_table_1220088                                           | mx_ddl_table                                           | distributed      |       8192
- mx_ddl_table_1220089                                           | mx_ddl_table                                           | distributed      |       8192
- mx_ddl_table_1220090                                           | mx_ddl_table                                           | distributed      |       8192
- mx_ddl_table_1220090                                           | mx_ddl_table                                           | distributed      |       8192
- mx_ddl_table_1220090                                           | mx_ddl_table                                           | distributed      |       8192
- mx_ddl_table_1220090                                           | mx_ddl_table                                           | distributed      |       8192
- mx_ddl_table_1220090                                           | mx_ddl_table                                           | distributed      |       8192
- mx_ddl_table_1220090                                           | mx_ddl_table                                           | distributed      |       8192
- mx_ddl_table_1220090                                           | mx_ddl_table                                           | distributed      |       8192
- mx_ddl_table_1220091                                           | mx_ddl_table                                           | distributed      |       8192
+ labs_mx_1220102                                                | labs_mx                                                | distributed      |       8192
+ labs_mx_1220102                                                | labs_mx                                                | distributed      |       8192
+ labs_mx_1220102                                                | labs_mx                                                | distributed      |       8192
+ labs_mx_1220102                                                | labs_mx                                                | distributed      |       8192
+ labs_mx_1220102                                                | labs_mx                                                | distributed      |       8192
+ labs_mx_1220102                                                | labs_mx                                                | distributed      |       8192
+ labs_mx_1220102                                                | labs_mx                                                | distributed      |       8192
+ limit_orders_mx_1220092                                        | limit_orders_mx                                        | distributed      |      16384
+ limit_orders_mx_1220092                                        | limit_orders_mx                                        | distributed      |      16384
+ limit_orders_mx_1220092                                        | limit_orders_mx                                        | distributed      |      16384
+ limit_orders_mx_1220092                                        | limit_orders_mx                                        | distributed      |      16384
+ limit_orders_mx_1220092                                        | limit_orders_mx                                        | distributed      |      16384
+ limit_orders_mx_1220092                                        | limit_orders_mx                                        | distributed      |      16384
+ limit_orders_mx_1220092                                        | limit_orders_mx                                        | distributed      |      16384
+ limit_orders_mx_1220093                                        | limit_orders_mx                                        | distributed      |      16384
+ lineitem_mx_1220052                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220052                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220052                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220052                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220052                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220052                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220052                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220053                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220054                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220054                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220054                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220054                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220054                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220054                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220054                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220055                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220056                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220056                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220056                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220056                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220056                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220056                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220056                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220057                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220058                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220058                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220058                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220058                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220058                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220058                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220058                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220059                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220060                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220060                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220060                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220060                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220060                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220060                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220060                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220061                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220062                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220062                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220062                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220062                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220062                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220062                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220062                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220063                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220064                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220064                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220064                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220064                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220064                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220064                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220064                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220065                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220066                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220066                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220066                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220066                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220066                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220066                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220066                                            | lineitem_mx                                            | distributed      |      16384
+ lineitem_mx_1220067                                            | lineitem_mx                                            | distributed      |      16384
+ multiple_hash_mx_1220094                                       | multiple_hash_mx                                       | distributed      |       8192
+ multiple_hash_mx_1220094                                       | multiple_hash_mx                                       | distributed      |       8192
+ multiple_hash_mx_1220094                                       | multiple_hash_mx                                       | distributed      |       8192
+ multiple_hash_mx_1220094                                       | multiple_hash_mx                                       | distributed      |       8192
+ multiple_hash_mx_1220094                                       | multiple_hash_mx                                       | distributed      |       8192
+ multiple_hash_mx_1220094                                       | multiple_hash_mx                                       | distributed      |       8192
+ multiple_hash_mx_1220094                                       | multiple_hash_mx                                       | distributed      |       8192
+ multiple_hash_mx_1220095                                       | multiple_hash_mx                                       | distributed      |       8192
+ mx_ddl_table_1220088                                           | mx_ddl_table                                           | distributed      |      24576
+ mx_ddl_table_1220088                                           | mx_ddl_table                                           | distributed      |      24576
+ mx_ddl_table_1220088                                           | mx_ddl_table                                           | distributed      |      24576
+ mx_ddl_table_1220088                                           | mx_ddl_table                                           | distributed      |      24576
+ mx_ddl_table_1220088                                           | mx_ddl_table                                           | distributed      |      24576
+ mx_ddl_table_1220088                                           | mx_ddl_table                                           | distributed      |      24576
+ mx_ddl_table_1220088                                           | mx_ddl_table                                           | distributed      |      24576
+ mx_ddl_table_1220089                                           | mx_ddl_table                                           | distributed      |      24576
+ mx_ddl_table_1220090                                           | mx_ddl_table                                           | distributed      |      24576
+ mx_ddl_table_1220090                                           | mx_ddl_table                                           | distributed      |      24576
+ mx_ddl_table_1220090                                           | mx_ddl_table                                           | distributed      |      24576
+ mx_ddl_table_1220090                                           | mx_ddl_table                                           | distributed      |      24576
+ mx_ddl_table_1220090                                           | mx_ddl_table                                           | distributed      |      24576
+ mx_ddl_table_1220090                                           | mx_ddl_table                                           | distributed      |      24576
+ mx_ddl_table_1220090                                           | mx_ddl_table                                           | distributed      |      24576
+ mx_ddl_table_1220091                                           | mx_ddl_table                                           | distributed      |      24576
  nation_hash_1220000                                            | nation_hash                                            | distributed      |          0
  nation_hash_1220000                                            | nation_hash                                            | distributed      |          0
  nation_hash_1220000                                            | nation_hash                                            | distributed      |          0
@@ -871,77 +871,77 @@ SELECT shard_name, table_name, citus_table_type, shard_size FROM citus_shards OR
  nation_mx_1220085                                              | nation_mx                                              | reference        |          0
  nation_mx_1220085                                              | nation_mx                                              | reference        |          0
  nation_mx_1220085                                              | nation_mx                                              | reference        |          0
- objects_mx_1220103                                             | objects_mx                                             | distributed      |          0
- objects_mx_1220103                                             | objects_mx                                             | distributed      |          0
- objects_mx_1220103                                             | objects_mx                                             | distributed      |          0
- objects_mx_1220103                                             | objects_mx                                             | distributed      |          0
- objects_mx_1220103                                             | objects_mx                                             | distributed      |          0
- objects_mx_1220103                                             | objects_mx                                             | distributed      |          0
- objects_mx_1220103                                             | objects_mx                                             | distributed      |          0
- orders_mx_1220068                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220068                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220068                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220068                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220068                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220068                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220068                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220069                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220070                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220070                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220070                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220070                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220070                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220070                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220070                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220071                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220072                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220072                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220072                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220072                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220072                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220072                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220072                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220073                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220074                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220074                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220074                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220074                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220074                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220074                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220074                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220075                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220076                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220076                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220076                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220076                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220076                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220076                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220076                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220077                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220078                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220078                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220078                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220078                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220078                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220078                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220078                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220079                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220080                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220080                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220080                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220080                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220080                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220080                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220080                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220081                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220082                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220082                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220082                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220082                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220082                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220082                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220082                                              | orders_mx                                              | distributed      |          0
- orders_mx_1220083                                              | orders_mx                                              | distributed      |          0
+ objects_mx_1220103                                             | objects_mx                                             | distributed      |      16384
+ objects_mx_1220103                                             | objects_mx                                             | distributed      |      16384
+ objects_mx_1220103                                             | objects_mx                                             | distributed      |      16384
+ objects_mx_1220103                                             | objects_mx                                             | distributed      |      16384
+ objects_mx_1220103                                             | objects_mx                                             | distributed      |      16384
+ objects_mx_1220103                                             | objects_mx                                             | distributed      |      16384
+ objects_mx_1220103                                             | objects_mx                                             | distributed      |      16384
+ orders_mx_1220068                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220068                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220068                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220068                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220068                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220068                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220068                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220069                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220070                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220070                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220070                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220070                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220070                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220070                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220070                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220071                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220072                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220072                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220072                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220072                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220072                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220072                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220072                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220073                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220074                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220074                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220074                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220074                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220074                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220074                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220074                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220075                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220076                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220076                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220076                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220076                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220076                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220076                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220076                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220077                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220078                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220078                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220078                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220078                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220078                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220078                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220078                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220079                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220080                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220080                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220080                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220080                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220080                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220080                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220080                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220081                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220082                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220082                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220082                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220082                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220082                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220082                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220082                                              | orders_mx                                              | distributed      |       8192
+ orders_mx_1220083                                              | orders_mx                                              | distributed      |       8192
  part_mx_1220086                                                | part_mx                                                | reference        |          0
  part_mx_1220086                                                | part_mx                                                | reference        |          0
  part_mx_1220086                                                | part_mx                                                | reference        |          0
@@ -950,14 +950,14 @@ SELECT shard_name, table_name, citus_table_type, shard_size FROM citus_shards OR
  part_mx_1220086                                                | part_mx                                                | reference        |          0
  part_mx_1220086                                                | part_mx                                                | reference        |          0
  part_mx_1220086                                                | part_mx                                                | reference        |          0
- researchers_mx_1220100                                         | researchers_mx                                         | distributed      |          0
- researchers_mx_1220100                                         | researchers_mx                                         | distributed      |          0
- researchers_mx_1220100                                         | researchers_mx                                         | distributed      |          0
- researchers_mx_1220100                                         | researchers_mx                                         | distributed      |          0
- researchers_mx_1220100                                         | researchers_mx                                         | distributed      |          0
- researchers_mx_1220100                                         | researchers_mx                                         | distributed      |          0
- researchers_mx_1220100                                         | researchers_mx                                         | distributed      |          0
- researchers_mx_1220101                                         | researchers_mx                                         | distributed      |          0
+ researchers_mx_1220100                                         | researchers_mx                                         | distributed      |       8192
+ researchers_mx_1220100                                         | researchers_mx                                         | distributed      |       8192
+ researchers_mx_1220100                                         | researchers_mx                                         | distributed      |       8192
+ researchers_mx_1220100                                         | researchers_mx                                         | distributed      |       8192
+ researchers_mx_1220100                                         | researchers_mx                                         | distributed      |       8192
+ researchers_mx_1220100                                         | researchers_mx                                         | distributed      |       8192
+ researchers_mx_1220100                                         | researchers_mx                                         | distributed      |       8192
+ researchers_mx_1220101                                         | researchers_mx                                         | distributed      |       8192
  supplier_mx_1220087                                            | supplier_mx                                            | reference        |          0
  supplier_mx_1220087                                            | supplier_mx                                            | reference        |          0
  supplier_mx_1220087                                            | supplier_mx                                            | reference        |          0

src/test/regress/expected/multi_partitioning.out

@@ -4324,12 +4324,66 @@ WHERE schemaname = 'partitioning_schema' AND tablename ilike '%part_table_with_%
 (6 rows)
 
 \c - - - :master_port
+SET search_path TO partitioning_schema;
+-- create parent table
+CREATE TABLE stxdinp(i int, a int, b int) PARTITION BY RANGE (i);
+-- create partition
+CREATE TABLE stxdinp1 PARTITION OF stxdinp FOR VALUES FROM (1) TO (100);
+-- populate table
+INSERT INTO stxdinp SELECT 1, a/100, a/100 FROM generate_series(1, 999) a;
+-- create extended statistics
+CREATE STATISTICS stxdinp ON a, b FROM stxdinp;
+-- distribute parent table
+SELECT create_distributed_table('stxdinp', 'i');
+NOTICE:  Copying data from local table...
+NOTICE:  copying the data has completed
+DETAIL:  The local data in the table is no longer visible, but is still on disk.
+HINT:  To remove the local data, run: SELECT truncate_local_data_after_distributing_table($$partitioning_schema.stxdinp1$$)
+ create_distributed_table
+---------------------------------------------------------------------
+
+(1 row)
+
+-- run select query, works fine
+SELECT a, b FROM stxdinp GROUP BY 1, 2;
+ a | b
+---------------------------------------------------------------------
+ 1 | 1
+ 3 | 3
+ 7 | 7
+ 2 | 2
+ 8 | 8
+ 0 | 0
+ 5 | 5
+ 6 | 6
+ 9 | 9
+ 4 | 4
+(10 rows)
+
+-- partitions are processed recursively for PG15+
+VACUUM ANALYZE stxdinp;
+SELECT a, b FROM stxdinp GROUP BY 1, 2;
+ a | b
+---------------------------------------------------------------------
+ 1 | 1
+ 3 | 3
+ 7 | 7
+ 2 | 2
+ 8 | 8
+ 0 | 0
+ 5 | 5
+ 6 | 6
+ 9 | 9
+ 4 | 4
+(10 rows)
+
 DROP SCHEMA partitioning_schema CASCADE;
-NOTICE:  drop cascades to 4 other objects
-DETAIL:  drop cascades to table partitioning_schema."schema-test"
-drop cascades to table partitioning_schema.another_distributed_table
-drop cascades to table partitioning_schema.distributed_parent_table
-drop cascades to table partitioning_schema.part_table_with_very_long_name
+NOTICE:  drop cascades to 5 other objects
+DETAIL:  drop cascades to table "schema-test"
+drop cascades to table another_distributed_table
+drop cascades to table distributed_parent_table
+drop cascades to table part_table_with_very_long_name
+drop cascades to table stxdinp
 RESET search_path;
 DROP TABLE IF EXISTS
 	partitioning_hash_test,

src/test/regress/expected/multi_select_distinct.out

@@ -3,13 +3,6 @@
 --
 -- Tests select distinct, and select distinct on features.
 --
-SHOW server_version \gset
-SELECT substring(:'server_version', '\d+')::int >= 15 AS server_version_ge_15;
- server_version_ge_15
----------------------------------------------------------------------
- t
-(1 row)
-
 ANALYZE lineitem_hash_part;
 -- function calls are supported
 SELECT DISTINCT l_orderkey, now() FROM lineitem_hash_part LIMIT 0;
@@ -446,10 +439,9 @@ EXPLAIN (COSTS FALSE)
                                                        QUERY PLAN
 ---------------------------------------------------------------------
  Limit
+   ->  Unique
          ->  Sort
                Sort Key: remote_scan.l_suppkey, ((pg_catalog.sum(remote_scan.avg) / pg_catalog.sum(remote_scan.avg_1)))
-         ->  HashAggregate
-               Group Key: remote_scan.l_suppkey, (pg_catalog.sum(remote_scan.avg) / pg_catalog.sum(remote_scan.avg_1))
                ->  HashAggregate
                      Group Key: remote_scan.l_suppkey, remote_scan.worker_column_4
                      ->  Custom Scan (Citus Adaptive)
@@ -460,7 +452,7 @@ EXPLAIN (COSTS FALSE)
                                  ->  HashAggregate
                                        Group Key: l_suppkey, l_linenumber
                                        ->  Seq Scan on lineitem_hash_part_360041 lineitem_hash_part
-(15 rows)
+(14 rows)
 
 -- check the plan if the hash aggreate is disabled. This explain errors out due
 -- to a bug right now, expectation must be corrected after fixing it.
@@ -598,10 +590,9 @@ EXPLAIN (COSTS FALSE)
                                                 QUERY PLAN
 ---------------------------------------------------------------------
  Limit
+   ->  Unique
          ->  Sort
                Sort Key: ((sum(remote_scan.avg) / (pg_catalog.sum(remote_scan.avg_1))::double precision))
-         ->  HashAggregate
-               Group Key: (sum(remote_scan.avg) / (pg_catalog.sum(remote_scan.avg_1))::double precision)
                ->  HashAggregate
                      Group Key: remote_scan.worker_column_3, remote_scan.worker_column_4
                      ->  Custom Scan (Citus Adaptive)
@@ -612,7 +603,7 @@ EXPLAIN (COSTS FALSE)
                                  ->  HashAggregate
                                        Group Key: l_suppkey, l_linenumber
                                        ->  Seq Scan on lineitem_hash_part_360041 lineitem_hash_part
-(15 rows)
+(14 rows)
 
 -- check the plan if the hash aggreate is disabled. This explain errors out due
 -- to a bug right now, expectation must be corrected after fixing it.
@@ -674,10 +665,9 @@ EXPLAIN (COSTS FALSE)
                                                                  QUERY PLAN
 ---------------------------------------------------------------------
  Limit
+   ->  Unique
          ->  Sort
                Sort Key: (((pg_catalog.sum(remote_scan.dis))::bigint + COALESCE((pg_catalog.sum(remote_scan.dis_1))::bigint, '0'::bigint)))
-         ->  HashAggregate
-               Group Key: ((pg_catalog.sum(remote_scan.dis))::bigint + COALESCE((pg_catalog.sum(remote_scan.dis_1))::bigint, '0'::bigint))
                ->  HashAggregate
                      Group Key: remote_scan.worker_column_3, remote_scan.worker_column_4
                      ->  Custom Scan (Citus Adaptive)
@@ -688,7 +678,7 @@ EXPLAIN (COSTS FALSE)
                                  ->  HashAggregate
                                        Group Key: l_suppkey, l_linenumber
                                        ->  Seq Scan on lineitem_hash_part_360041 lineitem_hash_part
-(15 rows)
+(14 rows)
 
 -- check the plan if the hash aggreate is disabled. This explain errors out due
 -- to a bug right now, expectation must be corrected after fixing it.